forked from cloudera/cm_ext
-
Notifications
You must be signed in to change notification settings - Fork 0
The parcel repository format
philipl edited this page Mar 21, 2014
·
12 revisions
To make parcels easily accessible to Cloudera Manager, we have a concept of a parcel repository. Like a yum or apt package repository, this is a remote location that CM can connect to do discover and download parcels.
A parcel repository is very simple: It consists of a directory that contains a set of parcel files and a manifest.json file that describes them. This simple structure allows a repository to be constructed simply by publishing the directory through an http server.
- manifest.json
- CDH-5.0.0-0.cdh5b2.p0.27-el5.parcel
- CDH-5.0.0-0.cdh5b2.p0.27-el6.parcel
- CDH-5.0.0-0.cdh5b2.p0.27-precise.parcel
- CDH-5.0.0-0.cdh5b2.p0.27-sles11.parcel
- CDH-5.0.0-0.cdh5b2.p0.27-wheezy.parcel
Note that as we have one parcel for each supported distro, there is a separate entry in the manifest for each of them. CM takes care of only showing a single entry in its UI and auto-selecting the right parcel file based on the installed distro on each host in a cluster.
Since Cloudera Manager 5.0, it is possible to password protect a repository using http basic authentication. When specifying the repository url in CM settings, the credentials can be specified in the standard way.
For example:
http://username:[email protected]/repository
SSL/TLS encryption can be used for a parcel repository, through the use of https when specifying the parcel repository in CM, but it is important to remember that no certificate checking will be done.
We recommend that you use the make_manifest.py tool, provided in this git repo, to build your manifest.json files. This automatically handles generating hashes and copying parcel metadata where appropriate.
$ ./make_manifest.py /var/www/parcel-repo{
"lastUpdated": 13920649160000,
"parcels": [
{
"parcelName": "CDH-5.0.0-0.cdh5b2.p0.27-wheezy.parcel",
"components": [
{
"pkg_version": "0.7.0+cdh5.0.0+0",
"name": "bigtop-tomcat",
"version": "6.0.37-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.9.0+cdh5.0.0+19",
"name": "crunch",
"version": "0.9.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.4.0+cdh5.0.0+90",
"name": "flume-ng",
"version": "1.4.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-0.20-mapreduce",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-hdfs",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-httpfs",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-mapreduce",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-yarn",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.96.1.1+cdh5.0.0+23",
"name": "hbase",
"version": "0.96.1.1-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.3+cdh5.0.0+39",
"name": "hbase-solr",
"version": "1.3-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.12.0+cdh5.0.0+265",
"name": "hive",
"version": "0.12.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.12.0+cdh5.0.0+265",
"name": "hive-hcatalog",
"version": "0.12.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "3.5.0+cdh5.0.0+186",
"name": "hue",
"version": "3.5.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.2.3+cdh5.0.0+0",
"name": "impala",
"version": "1.2.3-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.10.0+cdh5.0.0+69",
"name": "kite",
"version": "0.10.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.0.0+cdh5.0.0+0",
"name": "llama",
"version": "1.0.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.8+cdh5.0.0+28",
"name": "mahout",
"version": "0.8-cdh5.0.0-beta-2"
},
{
"pkg_version": "4.0.0+cdh5.0.0+144",
"name": "oozie",
"version": "4.0.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.2.5+cdh5.0.0+29",
"name": "parquet",
"version": "1.2.5-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.12.0+cdh5.0.0+20",
"name": "pig",
"version": "0.12.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "4.4.0+cdh5.0.0+163",
"name": "solr",
"version": "4.4.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.9.0",
"name": "spark",
"version": "0.9.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.99.3+cdh5.0.0+19",
"name": "sqoop2",
"version": "1.99.3-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.4.4+cdh5.0.0+40",
"name": "sqoop",
"version": "1.4.4-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.8.2+cdh5.0.0+20",
"name": "whirr",
"version": "0.8.2-cdh5.0.0-beta-2"
},
{
"pkg_version": "3.4.5+cdh5.0.0+27",
"name": "zookeeper",
"version": "3.4.5-cdh5.0.0-beta-2"
}
],
"replaces": "IMPALA, SOLR, SPARK",
"hash": "b56fc785e6930ab86e81acfa5bbe29dba86a4b8a"
},
{
"parcelName": "CDH-5.0.0-0.cdh5b2.p0.27-el6.parcel",
"components": [
{
"pkg_version": "0.7.0+cdh5.0.0+0",
"name": "bigtop-tomcat",
"version": "6.0.37-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.9.0+cdh5.0.0+19",
"name": "crunch",
"version": "0.9.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.4.0+cdh5.0.0+90",
"name": "flume-ng",
"version": "1.4.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-0.20-mapreduce",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-hdfs",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-httpfs",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-mapreduce",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-yarn",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.3+cdh5.0.0+39",
"name": "hbase-solr",
"version": "1.3-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.96.1.1+cdh5.0.0+23",
"name": "hbase",
"version": "0.96.1.1-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.12.0+cdh5.0.0+265",
"name": "hive-hcatalog",
"version": "0.12.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.12.0+cdh5.0.0+265",
"name": "hive",
"version": "0.12.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "3.5.0+cdh5.0.0+186",
"name": "hue",
"version": "3.5.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.2.3+cdh5.0.0+0",
"name": "impala",
"version": "1.2.3-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.10.0+cdh5.0.0+69",
"name": "kite",
"version": "0.10.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.0.0+cdh5.0.0+0",
"name": "llama",
"version": "1.0.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.8+cdh5.0.0+28",
"name": "mahout",
"version": "0.8-cdh5.0.0-beta-2"
},
{
"pkg_version": "4.0.0+cdh5.0.0+144",
"name": "oozie",
"version": "4.0.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.2.5+cdh5.0.0+29",
"name": "parquet",
"version": "1.2.5-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.12.0+cdh5.0.0+20",
"name": "pig",
"version": "0.12.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "4.4.0+cdh5.0.0+163",
"name": "solr",
"version": "4.4.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.9.0",
"name": "spark",
"version": "0.9.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.4.4+cdh5.0.0+40",
"name": "sqoop",
"version": "1.4.4-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.99.3+cdh5.0.0+19",
"name": "sqoop2",
"version": "1.99.3-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.8.2+cdh5.0.0+20",
"name": "whirr",
"version": "0.8.2-cdh5.0.0-beta-2"
},
{
"pkg_version": "3.4.5+cdh5.0.0+27",
"name": "zookeeper",
"version": "3.4.5-cdh5.0.0-beta-2"
}
],
"replaces": "IMPALA, SOLR, SPARK",
"hash": "20542176d9f52877235583d6b013c0e219f781aa"
},
{
"parcelName": "CDH-5.0.0-0.cdh5b2.p0.27-precise.parcel",
"components": [
{
"pkg_version": "0.7.0+cdh5.0.0+0",
"name": "bigtop-tomcat",
"version": "6.0.37-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.9.0+cdh5.0.0+19",
"name": "crunch",
"version": "0.9.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.4.0+cdh5.0.0+90",
"name": "flume-ng",
"version": "1.4.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-0.20-mapreduce",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-hdfs",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-httpfs",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-mapreduce",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-yarn",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.96.1.1+cdh5.0.0+23",
"name": "hbase",
"version": "0.96.1.1-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.3+cdh5.0.0+39",
"name": "hbase-solr",
"version": "1.3-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.12.0+cdh5.0.0+265",
"name": "hive",
"version": "0.12.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.12.0+cdh5.0.0+265",
"name": "hive-hcatalog",
"version": "0.12.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "3.5.0+cdh5.0.0+186",
"name": "hue",
"version": "3.5.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.2.3+cdh5.0.0+0",
"name": "impala",
"version": "1.2.3-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.10.0+cdh5.0.0+69",
"name": "kite",
"version": "0.10.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.0.0+cdh5.0.0+0",
"name": "llama",
"version": "1.0.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.8+cdh5.0.0+28",
"name": "mahout",
"version": "0.8-cdh5.0.0-beta-2"
},
{
"pkg_version": "4.0.0+cdh5.0.0+144",
"name": "oozie",
"version": "4.0.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.2.5+cdh5.0.0+29",
"name": "parquet",
"version": "1.2.5-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.12.0+cdh5.0.0+20",
"name": "pig",
"version": "0.12.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "4.4.0+cdh5.0.0+163",
"name": "solr",
"version": "4.4.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.9.0",
"name": "spark",
"version": "0.9.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.99.3+cdh5.0.0+19",
"name": "sqoop2",
"version": "1.99.3-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.4.4+cdh5.0.0+40",
"name": "sqoop",
"version": "1.4.4-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.8.2+cdh5.0.0+20",
"name": "whirr",
"version": "0.8.2-cdh5.0.0-beta-2"
},
{
"pkg_version": "3.4.5+cdh5.0.0+27",
"name": "zookeeper",
"version": "3.4.5-cdh5.0.0-beta-2"
}
],
"replaces": "IMPALA, SOLR, SPARK",
"hash": "a944f2e6f9665516d9c5059850e026ec4acf0e2d"
},
{
"parcelName": "CDH-5.0.0-0.cdh5b2.p0.27-sles11.parcel",
"components": [
{
"pkg_version": "0.7.0+cdh5.0.0+0",
"name": "bigtop-tomcat",
"version": "6.0.37-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.9.0+cdh5.0.0+19",
"name": "crunch",
"version": "0.9.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.4.0+cdh5.0.0+90",
"name": "flume-ng",
"version": "1.4.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-0.20-mapreduce",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-hdfs",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-httpfs",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-mapreduce",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-yarn",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.96.1.1+cdh5.0.0+23",
"name": "hbase",
"version": "0.96.1.1-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.3+cdh5.0.0+39",
"name": "hbase-solr",
"version": "1.3-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.12.0+cdh5.0.0+265",
"name": "hive",
"version": "0.12.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.12.0+cdh5.0.0+265",
"name": "hive-hcatalog",
"version": "0.12.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "3.5.0+cdh5.0.0+186",
"name": "hue",
"version": "3.5.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.2.3+cdh5.0.0+0",
"name": "impala",
"version": "1.2.3-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.10.0+cdh5.0.0+69",
"name": "kite",
"version": "0.10.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.0.0+cdh5.0.0+0",
"name": "llama",
"version": "1.0.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.8+cdh5.0.0+28",
"name": "mahout",
"version": "0.8-cdh5.0.0-beta-2"
},
{
"pkg_version": "4.0.0+cdh5.0.0+144",
"name": "oozie",
"version": "4.0.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.2.5+cdh5.0.0+29",
"name": "parquet",
"version": "1.2.5-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.12.0+cdh5.0.0+20",
"name": "pig",
"version": "0.12.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "4.4.0+cdh5.0.0+163",
"name": "solr",
"version": "4.4.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.9.0",
"name": "spark",
"version": "0.9.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.99.3+cdh5.0.0+19",
"name": "sqoop2",
"version": "1.99.3-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.4.4+cdh5.0.0+40",
"name": "sqoop",
"version": "1.4.4-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.8.2+cdh5.0.0+20",
"name": "whirr",
"version": "0.8.2-cdh5.0.0-beta-2"
},
{
"pkg_version": "3.4.5+cdh5.0.0+27",
"name": "zookeeper",
"version": "3.4.5-cdh5.0.0-beta-2"
}
],
"replaces": "IMPALA, SOLR, SPARK",
"hash": "0858a7f8fc0207c21a244c4445cd3a59a1711fac"
},
{
"parcelName": "CDH-5.0.0-0.cdh5b2.p0.27-el5.parcel",
"components": [
{
"pkg_version": "0.7.0+cdh5.0.0+0",
"name": "bigtop-tomcat",
"version": "6.0.37-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.9.0+cdh5.0.0+19",
"name": "crunch",
"version": "0.9.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.4.0+cdh5.0.0+90",
"name": "flume-ng",
"version": "1.4.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-0.20-mapreduce",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-hdfs",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-httpfs",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-mapreduce",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "2.2.0+cdh5.0.0+1610",
"name": "hadoop-yarn",
"version": "2.2.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.96.1.1+cdh5.0.0+23",
"name": "hbase",
"version": "0.96.1.1-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.3+cdh5.0.0+39",
"name": "hbase-solr",
"version": "1.3-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.12.0+cdh5.0.0+265",
"name": "hive",
"version": "0.12.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.12.0+cdh5.0.0+265",
"name": "hive-hcatalog",
"version": "0.12.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "3.5.0+cdh5.0.0+186",
"name": "hue",
"version": "3.5.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.2.3+cdh5.0.0+0",
"name": "impala",
"version": "1.2.3-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.10.0+cdh5.0.0+69",
"name": "kite",
"version": "0.10.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.0.0+cdh5.0.0+0",
"name": "llama",
"version": "1.0.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.8+cdh5.0.0+28",
"name": "mahout",
"version": "0.8-cdh5.0.0-beta-2"
},
{
"pkg_version": "4.0.0+cdh5.0.0+144",
"name": "oozie",
"version": "4.0.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.2.5+cdh5.0.0+29",
"name": "parquet",
"version": "1.2.5-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.12.0+cdh5.0.0+20",
"name": "pig",
"version": "0.12.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "4.4.0+cdh5.0.0+163",
"name": "solr",
"version": "4.4.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.9.0",
"name": "spark",
"version": "0.9.0-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.99.3+cdh5.0.0+19",
"name": "sqoop2",
"version": "1.99.3-cdh5.0.0-beta-2"
},
{
"pkg_version": "1.4.4+cdh5.0.0+40",
"name": "sqoop",
"version": "1.4.4-cdh5.0.0-beta-2"
},
{
"pkg_version": "0.8.2+cdh5.0.0+20",
"name": "whirr",
"version": "0.8.2-cdh5.0.0-beta-2"
},
{
"pkg_version": "3.4.5+cdh5.0.0+27",
"name": "zookeeper",
"version": "3.4.5-cdh5.0.0-beta-2"
}
],
"replaces": "IMPALA, SOLR, SPARK",
"hash": "b97bc1f436962d4ee6919c606400b6cd21c0bd02"
}
]
}