Merge remote-tracking branch 'upstream/master' into refactor-dir-creation

Author: macmaster

archive_bins.sh

@@ -15,7 +15,7 @@ tarball_name=${1-release_master_bins.tar.gz}
 set -x
 
 vagrant ssh -c "tar cvpzf ~/${tarball_name} -C ~/chef-bcpc \
-    --exclude=bins/apt_key.*  \
+    --exclude=bins/apt_key.* --exclude=bins/Release.gpg \
     bins vendor/bootstrap vendor/cache gemfiles" | log_pretty
 vagrant ssh -c "cp -fv  ~/${tarball_name} \
     /chef-bcpc-host/${tarball_name}" | log_pretty

cookbooks/bach_deployment/attributes/default.rb

@@ -1,12 +1,12 @@
 force_default['bach']['deploy']['appfile']['data'] = {
-  appfoo: {
-    repo_url: 'http://bcpc.example.com',
-    copy_to: '',
-    copy_type: 'file',
-    runas: 'root',
-    filename: 'example.jar',
-    filemode: '0644',
-    fileowner: 'root',
-    checksum: '4bd'
-  }
+#  appfoo: {
+#    repo_url: 'http://bcpc.example.com',
+#    copy_to: '',
+#    copy_type: 'file',
+#    runas: 'root',
+#    filename: 'example.jar',
+#    filemode: '0644',
+#    fileowner: 'root',
+#    checksum: '4bd'
+#  }
 }

cookbooks/bcpc-hadoop/attributes/dir_creation.rb

@@ -47,13 +47,15 @@
 
 # HDFS Projects Directory Configuration
 default['bcpc']['hadoop']['dir_creation']['projects'].tap do |projects|
+  groups = node['bcpc']['hadoop']['dir_creation']['groups']
+
   # default entry values
   projects['defaults'] = {
     owner: 'hdfs',
     group: 'hdfs',
     perms: '1771',
-    space_quota: node['bcpc']['hadoop']['hdfs']['groups']['space_quota'],
-    ns_quota: node['bcpc']['hadoop']['hdfs']['groups']['ns_quota']
+    space_quota: groups['space_quota'],
+    ns_quota: groups['ns_quota']
   }
 
   projects['dirinfo'] = {

cookbooks/bcpc-hadoop/recipes/hdfs_group_directories.rb

@@ -5,11 +5,12 @@
   block do
     # Default configuration for group directory creation
     groups_dir_creation =
-      node.default['bcpc']['hadoop']['dir_creation']['groups']
+      node['bcpc']['hadoop']['dir_creation']['groups']
 
     # Create directories for existing LDAP groups.
-    # Add dirinfo entries for each user returned by LDAP queries.
-    dirinfo = groups_dir_creation['dirinfo']
+    # Add dirinfo entries for each group returned by LDAP queries.
+    dirinfo =
+      node.default['bcpc']['hadoop']['dir_creation']['groups']['dirinfo']
     (cluster_groups).each do |group|
       space_quota = groups_dir_creation["#{group}_space_quota"]
       ns_quota = groups_dir_creation["#{group}_ns_quota"]
@@ -20,6 +21,10 @@
     end
 
     # Create group directories in /groups.
-    dir_creation('/groups', groups_dir_creation['defaults'], dirinfo)
+    dir_creation(
+      '/groups',
+      node['bcpc']['hadoop']['dir_creation']['groups']['defaults'],
+      node['bcpc']['hadoop']['dir_creation']['groups']['dirinfo']
+    )
   end
 end

cookbooks/bcpc-hadoop/recipes/hdfs_project_directories.rb

@@ -2,13 +2,12 @@
 
 ruby_block 'hdfs_projects_directories' do
   block do 
-    # Default configuration for projects directory creation
-    projects_dir_creation = node['bcpc']['hadoop']['dir_creation']['projects']
-
-    # Use the dirinfo structure specified in the default attribute.
-    dirinfo = project_dir_creation['dirinfo']
-
     # Create projects directories in /projects.
-    dir_creation('/projects', projects_dir_creation['defaults'], dirinfo)
+    # Use the defaults and dirinfo specified in the attributes.
+    dir_creation(
+      '/projects',
+      node['bcpc']['hadoop']['dir_creation']['projects']['defaults'],
+      node['bcpc']['hadoop']['dir_creation']['projects']['dirinfo'],
+    )
   end
 end

cookbooks/bcpc-hadoop/recipes/hdfs_user_directories.rb

@@ -5,11 +5,12 @@
   block do
     # Default configuration for user directory creation
     user_dir_creation =
-      node.default['bcpc']['hadoop']['dir_creation']['user']
+      node['bcpc']['hadoop']['dir_creation']['user']
 
     # Create directories for existing LDAP users and role accounts.
     # Add dirinfo entries for each user returned by LDAP queries.
-    dirinfo = user_dir_creation['dirinfo']
+    dirinfo =
+      node.default['bcpc']['hadoop']['dir_creation']['user']['dirinfo']
     (cluster_users + cluster_roles).each do |user|
       space_quota = user_dir_creation["#{user}_space_quota"]
       ns_quota = user_dir_creation["#{user}_ns_quota"]
@@ -20,6 +21,10 @@
     end
 
     # Create user directories in /user.
-    dir_creation('/user', user_dir_creation['defaults'], dirinfo)
+    dir_creation(
+      '/user',
+      node['bcpc']['hadoop']['dir_creation']['user']['defaults'],
+      node['bcpc']['hadoop']['dir_creation']['user']['dirinfo']
+    )
   end
 end

stub-environment/environments/Test-Laptop.json

@@ -11,44 +11,44 @@
       },
       "domain_name": "bcpc.example.com",
       "management": {
-        "vip" : "10.0.101.5"
+        "vip" : "10.0.100.5"
       },
       "floating": {
-        "vip" : "192.168.101.5"
+        "vip" : "192.168.100.5"
       },
       "networks": {
         "subnet1": {
           "management": {
-            "vip" : "10.0.101.5",
+            "vip" : "10.0.100.5",
             "interface" : "eth0",
             "netmask" : "255.255.255.0",
-            "cidr" : "10.0.101.0/24",
-            "gateway" : "10.0.101.3"
+            "cidr" : "10.0.100.0/24",
+            "gateway" : "10.0.100.3"
           },
           "storage": {
             "interface" : "eth1",
             "netmask" : "255.255.255.0",
-            "cidr" : "172.16.101.0/24",
-            "gateway" : "172.16.101.3"
+            "cidr" : "172.16.100.0/24",
+            "gateway" : "172.16.100.3"
           },
           "floating": {
-            "vip" : "192.168.101.5",
+            "vip" : "192.168.100.5",
             "interface" : "eth2",
             "netmask" : "255.255.255.0",
-            "cidr" : "192.168.101.0/24",
-            "gateway" : "192.168.101.3",
-            "available_subnet" : "192.168.101.128/25"
+            "cidr" : "192.168.100.0/24",
+            "gateway" : "192.168.100.3",
+            "available_subnet" : "192.168.100.128/25"
           }
         }
       },
       "bootstrap": {
         "interface" : "eth0",
         "pxe_interface" : "eth1",
         "hostname" : "bcpc-bootstrap",
-        "server" : "10.0.101.3",
+        "server" : "10.0.100.3",
         "cluster_def_path": "/cluster-def/",
-        "dhcp_subnet" : "10.0.101.0",
-        "dhcp_range" : "10.0.101.14 10.0.101.250",
+        "dhcp_subnet" : "10.0.100.0",
+        "dhcp_range" : "10.0.100.14 10.0.100.250",
         "preseed": {
           "additional_packages": [ "openssh-server",
                                    "lldpd",
@@ -96,7 +96,7 @@
                      "3.pool.ntp.org" ]
     },
     "chef_client": {
-      "server_url": "http://10.0.101.3:4000",
+      "server_url": "http://10.0.100.3:4000",
       "cache_path": "/var/chef/cache",
       "config": {
          "verify_api_cert": false,

stub-environment/hadoop_cluster.txt

@@ -1,4 +1,4 @@
-1 bcpc-vm1 08:00:27:C0:FF:EE 10.0.101.11 - cobbler_profile bcpc.example.com role[BCPC-Hadoop-Head],role[BCPC-Hadoop-Head-Namenode],role[BCPC-Hadoop-Head-HBase],recipe[bcpc-hadoop::oozie],role[BCPC-Hadoop-Head-OpenTSDB-Server],recipe[bcpc-hadoop::copylog]
-2 bcpc-vm2 08:00:27:C0:FF:EE 10.0.101.12 - cobbler_profile bcpc.example.com role[BCPC-Hadoop-Head],role[BCPC-Hadoop-Head-ResourceManager],role[BCPC-Hadoop-Head-Namenode-Standby],role[BCPC-Hadoop-Head-HBase],role[BCPC-Hadoop-Head-MapReduce],role[BCPC-Hadoop-Head-Hive],role[BCPC-Hadoop-Head-OpenTSDB-Server],recipe[bcpc-hadoop::copylog],role[BCPC-Hadoop-Head-YarnTimeLineServer]
-3 bcpc-vm3 08:00:27:C0:FF:EE 10.0.101.13 - cobbler_profile bcpc.example.com role[BCPC-Hadoop-Head],role[BACH-Ambari-Server]
-- bcpc-vm4 08:00:27:C0:FF:EE 10.0.101.14 - cobbler_profile bcpc.example.com role[BCPC-Hadoop-Worker],recipe[bcpc-hadoop::copylog]
+1 bcpc-vm1 08:00:27:C0:FF:EE 10.0.100.11 - cobbler_profile bcpc.example.com role[BCPC-Hadoop-Head],role[BCPC-Hadoop-Head-Namenode],role[BCPC-Hadoop-Head-HBase],recipe[bcpc-hadoop::oozie],role[BCPC-Hadoop-Head-OpenTSDB-Server],recipe[bcpc-hadoop::copylog]
+2 bcpc-vm2 08:00:27:C0:FF:EE 10.0.100.12 - cobbler_profile bcpc.example.com role[BCPC-Hadoop-Head],role[BCPC-Hadoop-Head-ResourceManager],role[BCPC-Hadoop-Head-Namenode-Standby],role[BCPC-Hadoop-Head-HBase],role[BCPC-Hadoop-Head-MapReduce],role[BCPC-Hadoop-Head-Hive],role[BCPC-Hadoop-Head-OpenTSDB-Server],recipe[bcpc-hadoop::copylog],role[BCPC-Hadoop-Head-YarnTimeLineServer]
+3 bcpc-vm3 08:00:27:C0:FF:EE 10.0.100.13 - cobbler_profile bcpc.example.com role[BCPC-Hadoop-Head],role[BACH-Ambari-Server]
+- bcpc-vm4 08:00:27:C0:FF:EE 10.0.100.14 - cobbler_profile bcpc.example.com role[BCPC-Hadoop-Worker],recipe[bcpc-hadoop::copylog]

tests/automated_install.sh

@@ -19,7 +19,7 @@ if [[ "$(pwd)" != "$(git rev-parse --show-toplevel)" ]]; then
   printf '#### WARNING: This should be run in the git top level directory! ####\n' > /dev/stderr
 fi
 
-export BACH_ENVIRONMENT='Test-Laptop'
+export BACH_ENVIRONMENT=${BACH_ENVIRONMENT:-'Test-Laptop'}
 export BACH_CLUSTER_PREFIX=${BACH_CLUSTER_PREFIX:-''}
 export BOOTSTRAP_VM_MEM=${BOOTSTRAP_VM_MEM:-5096}
 export BOOTSTRAP_VM_CPUs=${BOOTSTRAP_VM_CPUS:-2}