BIGTOP-2708. puppet code always installs hdfs by default

11 files changed, 48 insertions, 21 deletions

diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index d9f6e019..1313082b 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -13,6 +13,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# The following is a bit of a tricky map. The idea here is that the keys
+# correspond to anything that could be specified in
+#   hadoop_cluster_node::cluster_components:
+# The values are maps from each role that a node can have in a cluster:
+#   client, gateway_server, library, master, worker, standby
+# to a role recognized by each puppet module's deploy class.
+#
+# Note that the code here will pass all these roles to all the deploy
+# classes defined in every single Bigtop's puppet module. This is similar
+# to how a visitor pattern works in OOP. One subtle ramification of this
+# approach is that you should make sure that deploy classes from different
+# modules do NOT accept same strings for role types.
+#
+# And if that wasn't enough of a head scratcher -- you also need to keep
+# in mind that there's no hdfs key in the following map, even though it
+# is a perfectly legal value for hadoop_cluster_node::cluster_components:
+# The reason for this is that hdfs is treated as an alias for either
+# hdfs-non-ha or hdfs-ha depending on whether HA for HDFS is either enabled
+# or disabled.
+
 $roles_map = {
   apex => {
     client => ["apex-client"],
@@ -217,14 +237,17 @@ class node_with_components inherits hadoop_cluster_node {
   }
 
   $given_components = $components_array[0] ? {
-    "all"   => delete(keys($roles_map), ["hdfs-non-ha", "hdfs-ha"]),
+    "all"   => delete(keys($roles_map), ["hdfs-non-ha", "hdfs-ha"]) << "hdfs",
     default => $components_array,
   }
   $ha_dependent_components = $ha_enabled ? {
-    true    => ["hdfs-ha"],
-    default => ["hdfs-non-ha"],
+    true    => "hdfs-ha",
+    default => "hdfs-non-ha",
+  }
+  $components = member($given_components, "hdfs") ? {
+    true    => delete($given_components, "hdfs") << $ha_dependent_components,
+    default => $given_components
   }
-  $components = concat($given_components, $ha_dependent_components)
 
   $master_role_types = ["master", "worker", "library"]
   $standby_role_types = ["standby", "library"]
diff --git a/bigtop-deploy/puppet/manifests/site.pp b/bigtop-deploy/puppet/manifests/site.pp
index ce5857f4..f35a4e0f 100644
--- a/bigtop-deploy/puppet/manifests/site.pp
+++ b/bigtop-deploy/puppet/manifests/site.pp
@@ -69,7 +69,10 @@ if ($provision_repo) {
 case $::operatingsystem {
     /Debian/: {
       require apt
-      require apt::backports
+      class { 'apt::backports':
+        pin => 500,
+      }
+      Class['apt::backports'] -> Package <||>
 
       package { "jdk":
         name => $jdk_package_name,
diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
index 30a68e2b..7d8e48ff 100644
--- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
@@ -45,7 +45,7 @@ class hadoop ($hadoop_security_authentication = "simple",
       }
     }
 
-    if ($hadoop::common_hdfs::ha != "disabled" and "standby-namenode" in $roles) {
+    if ("standby-namenode" in $roles and $hadoop::common_hdfs::ha != "disabled") {
       include hadoop::namenode
     }
 
@@ -67,7 +67,7 @@ class hadoop ($hadoop_security_authentication = "simple",
       }
     }
 
-    if ($hadoop::common_hdfs::ha == "disabled" and "secondarynamenode" in $roles) {
+    if ("secondarynamenode" in $roles and $hadoop::common_hdfs::ha == "disabled") {
       include hadoop::secondarynamenode
     }
 
diff --git a/docker/pseudo-cluster/config/hieradata/site.yaml b/docker/pseudo-cluster/config/hieradata/site.yaml
index a8ab29d6..1d298598 100644
--- a/docker/pseudo-cluster/config/hieradata/site.yaml
+++ b/docker/pseudo-cluster/config/hieradata/site.yaml
@@ -3,8 +3,9 @@ hadoop::hadoop_storage_dirs:
   - /data/1
   - /data/2
 hadoop_cluster_node::cluster_components:
-  - hadoop
+  - hdfs 
   - yarn
+  - mapreduce
 bigtop::bigtop_repo_uri: http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/ubuntu/trusty/x86_64
 bigtop::jdk_package_name: openjdk-8-jdk
 
diff --git a/docker/sandbox/site.yaml.template.centos-6_hadoop b/docker/sandbox/site.yaml.template.centos-6_hadoop
index f38c4fba..554431ee 100644
--- a/docker/sandbox/site.yaml.template.centos-6_hadoop
+++ b/docker/sandbox/site.yaml.template.centos-6_hadoop
@@ -16,5 +16,5 @@
 bigtop::hadoop_head_node: "head.node.fqdn"
 hadoop::hadoop_storage_dirs: [/data/1, /data/2]
 bigtop::bigtop_repo_uri: http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/6/x86_64
-hadoop_cluster_node::cluster_components: [hadoop, yarn]
+hadoop_cluster_node::cluster_components: [hdfs, yarn, mapreduce]
 bigtop::jdk_package_name: java-1.8.0-openjdk-devel.x86_64
diff --git a/docker/sandbox/site.yaml.template.debian-8_hadoop b/docker/sandbox/site.yaml.template.debian-8_hadoop
index 9a00c43b..d9c7c7f5 100644
--- a/docker/sandbox/site.yaml.template.debian-8_hadoop
+++ b/docker/sandbox/site.yaml.template.debian-8_hadoop
@@ -16,5 +16,5 @@
 bigtop::hadoop_head_node: "head.node.fqdn"
 hadoop::hadoop_storage_dirs: [/data/1, /data/2]
 bigtop::bigtop_repo_uri: http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/debian/8/x86_64
-hadoop_cluster_node::cluster_components: [hadoop, yarn]
+hadoop_cluster_node::cluster_components: [hdfs, yarn, mapreduce]
 bigtop::jdk_package_name: openjdk-8-jdk
diff --git a/provisioner/docker/config_centos6.yaml b/provisioner/docker/config_centos6.yaml
index 1f067314..9b7b6ee4 100644
--- a/provisioner/docker/config_centos6.yaml
+++ b/provisioner/docker/config_centos6.yaml
@@ -19,7 +19,7 @@ docker:
 
 repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/6/x86_64"
 distro: centos
-components: [hadoop, yarn]
+components: [hdfs, yarn, mapreduce]
 enable_local_repo: false
-smoke_test_components: [mapreduce, pig]
+smoke_test_components: [hdfs, yarn, mapreduce]
 jdk: "java-1.8.0-openjdk-devel.x86_64"
diff --git a/provisioner/docker/config_centos7.yaml b/provisioner/docker/config_centos7.yaml
index 95724ff7..3fe66e50 100644
--- a/provisioner/docker/config_centos7.yaml
+++ b/provisioner/docker/config_centos7.yaml
@@ -19,7 +19,7 @@ docker:
 
 repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/7/x86_64"
 distro: centos
-components: [hadoop, yarn]
+components: [hdfs, yarn, mapreduce]
 enable_local_repo: false
-smoke_test_components: [mapreduce, pig]
+smoke_test_components: [hdfs, yarn, mapreduce]
 jdk: "java-1.8.0-openjdk-devel.x86_64"
diff --git a/provisioner/docker/config_debian8.yaml b/provisioner/docker/config_debian8.yaml
index ea8d1501..d3e23999 100644
--- a/provisioner/docker/config_debian8.yaml
+++ b/provisioner/docker/config_debian8.yaml
@@ -19,7 +19,7 @@ docker:
 
 repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/debian/8/x86_64"
 distro: debian
-components: [hadoop, yarn]
+components: [hdfs, yarn, mapreduce]
 enable_local_repo: false
-smoke_test_components: [mapreduce, pig]
+smoke_test_components: [hdfs, yarn, mapreduce]
 jdk: "openjdk-8-jdk"
diff --git a/provisioner/docker/config_ubuntu_trusty.yaml b/provisioner/docker/config_ubuntu_xenial.yaml
index a56e97a3..3b454fa4 100644
--- a/provisioner/docker/config_ubuntu_trusty.yaml
+++ b/provisioner/docker/config_ubuntu_xenial.yaml
@@ -15,11 +15,11 @@
 
 docker:
         memory_limit: "4g"
-        image:  "bigtop/puppet:ubuntu-14.04"
+        image:  "bigtop/puppet:ubuntu-16.04"
 
 repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/ubuntu/trusty/x86_64"
 distro: debian
-components: [hadoop, yarn]
+components: [hdfs, yarn, mapreduce]
 enable_local_repo: false
-smoke_test_components: [mapreduce, pig]
+smoke_test_components: [hdfs, yarn, mapreduce]
 jdk: "openjdk-8-jdk"
diff --git a/provisioner/vagrant/vagrantconfig.yaml b/provisioner/vagrant/vagrantconfig.yaml
index aabc7bd9..49741044 100644
--- a/provisioner/vagrant/vagrantconfig.yaml
+++ b/provisioner/vagrant/vagrantconfig.yaml
@@ -19,8 +19,8 @@ box: "puppetlabs/centos-7.2-64-nocm"
 repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/7/x86_64"
 num_instances: 1
 distro: centos
-components: [hadoop, yarn, hbase]
+components: [hdfs, yarn, mapreduce]
 enable_local_repo: false
 run_smoke_tests: false
-smoke_test_components: [mapreduce, pig]
+smoke_test_components: [hdfs, yarn, mapreduce]
 jdk: "java-1.8.0-openjdk-devel.x86_64"