BIGTOP-2775: Make open jdk 8 available on bigtop/puppet:debian-8

12 files changed, 136 insertions, 89 deletions

diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index d116763d..6ff39cc9 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -55,9 +55,8 @@
 #bigtop::bigtop_repo_uri: "http://mirror.example.com/path/to/mirror/"
 
 # Use a pre-installed java environment. The default value of 'false' will cause
-# the configured 'bigtop::jdk_package_name' package to be installed. Setting
-# this to 'true' will ignore the configured 'bigtop::jdk_package_name' but
-# requires a compatible java environment be avaialble prior to Bigtop
+# the default jdk by bigtop to be installed.
+# this to 'true' requires a compatible java environment be avaialble prior to Bigtop
 # installation.
 #bigtop::jdk_preinstalled: false
 
diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml
index 02f8e45c..b0036c39 100644
--- a/bigtop-deploy/puppet/hieradata/site.yaml
+++ b/bigtop-deploy/puppet/hieradata/site.yaml
@@ -43,5 +43,4 @@ hadoop::hadoop_storage_dirs:
 
 #bigtop::roles_enabled: false
 
-#bigtop::jdk_package_name: "openjdk-7-jre-headless"
 #bigtop::bigtop_repo_uri: "http://mirror.example.com/path/to/mirror/"
diff --git a/bigtop-deploy/puppet/manifests/bigtop_repo.pp b/bigtop-deploy/puppet/manifests/bigtop_repo.pp
new file mode 100644
index 00000000..da782d50
--- /dev/null
+++ b/bigtop-deploy/puppet/manifests/bigtop_repo.pp
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class bigtop_repo {
+  case $::operatingsystem {
+    /(OracleLinux|Amazon|CentOS|Fedora|RedHat)/: {
+       $default_repo = "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/centos/7/x86_64"
+       yumrepo { "Bigtop":
+          baseurl => hiera("bigtop::bigtop_repo_uri", $default_repo),
+          descr => "Bigtop packages",
+          enabled => 1,
+          gpgcheck => 0,
+       }
+       Yumrepo<||> -> Package<||>
+    }
+    /(Ubuntu|Debian)/: {
+       include stdlib
+       include apt
+
+       $lower_os = downcase($operatingsystem)
+       # We use code name such as trusty for Ubuntu instead of release version in bigtop's binary convenience repos
+       if ($operatingsystem == "Ubuntu") { $release = $lsbdistcodename } else { $release = $operatingsystemmajrelease }
+       $default_repo = "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/${lower_os}/${release}/x86_64"
+
+       apt::conf { "disable_keys":
+          content => "APT::Get::AllowUnauthenticated 1;",
+          ensure => present
+       } ->
+       apt::source { "Bigtop":
+          location => hiera("bigtop::bigtop_repo_uri", $default_repo),
+          release => "bigtop",
+          repos => "contrib",
+          ensure => present,
+       } ->
+       exec {'bigtop-apt-update':
+          command => '/usr/bin/apt-get update'
+       }
+       Apt::Source<||> -> Exec['bigtop-apt-update'] -> Package<||>
+    }
+    default: {
+      notify{"WARNING: running on a neither yum nor apt platform -- make sure Bigtop repo is setup": }
+    }
+  }
+}
diff --git a/bigtop-deploy/puppet/manifests/jdk.pp b/bigtop-deploy/puppet/manifests/jdk.pp
new file mode 100644
index 00000000..152e8a41
--- /dev/null
+++ b/bigtop-deploy/puppet/manifests/jdk.pp
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class jdk {
+  case $::operatingsystem {
+    /Debian/: {
+      require apt
+      unless $operatingsystemmajrelease > "8" {
+         # we pin openjdk-8-* and ca-certificates-java to backports
+         require apt::backports
+
+         Exec['bigtop-apt-update'] ->
+         apt::pin { 'backports_jdk':
+            packages => 'openjdk-8-*',
+            priority => 500,
+            release  => 'jessie-backports',
+         } ->
+         apt::pin { 'backports_ca':
+            packages => 'ca-certificates-java',
+            priority => 500,
+            release  => 'jessie-backports',
+         } ->
+         exec {'own_update':
+            command => '/usr/bin/apt-get update'
+         } -> Package['jdk']
+      }
+      package { 'jdk':
+        name => 'openjdk-8-jdk',
+        ensure => present,
+      }
+
+     }
+    /Ubuntu/: {
+       include apt
+
+      package { 'jdk':
+        name => 'openjdk-8-jdk',
+        ensure  => present,
+      }
+    }
+    /(CentOS|Amazon|Fedora)/: {
+      package { 'jdk':
+        name => 'java-1.8.0-openjdk-devel',
+        ensure => present
+      }
+      if ($::operatingsystem == "Fedora") {
+        file { '/usr/lib/jvm/java-1.8.0-openjdk/jre/lib/security/cacerts':
+          ensure => 'link',
+          target => '/etc/pki/java/cacerts'
+        }
+      }
+    }
+    /OpenSuSE/: {
+      package { 'jdk':
+        name => 'java-1_8_0-openjdk-devel',
+        ensure => present
+      }
+    }
+  }
+}
diff --git a/bigtop-deploy/puppet/manifests/site.pp b/bigtop-deploy/puppet/manifests/site.pp
index bf0212cd..cd6e8d5a 100644
--- a/bigtop-deploy/puppet/manifests/site.pp
+++ b/bigtop-deploy/puppet/manifests/site.pp
@@ -13,86 +13,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Prepare default repo by detecting the environment automatically
-case $operatingsystem {
-    # Use CentOS 7 repo for other CentOS compatible OSs
-    /(OracleLinux|Amazon|RedHat)/: {
-      $default_repo = "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/centos/7/x86_64"
-    }
-    # Detect env to pick up default repo for other Bigtop supported OSs
-    default: {
-      include stdlib
-
-      $lower_os = downcase($operatingsystem)
-      # We use code name such as trusty for Ubuntu instead of release version in bigtop's binary convenience repos
-      if ($operatingsystem == "Ubuntu") { $release = $lsbdistcodename } else { $release = $operatingsystemmajrelease }
-      $default_repo = "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/${lower_os}/${release}/x86_64"
-    }
-}
-
 $jdk_preinstalled = hiera("bigtop::jdk_preinstalled", false)
-$jdk_package_name = hiera("bigtop::jdk_package_name", "jdk")
+if ( ! $jdk_preinstalled ) {
+   require jdk
+   Class['jdk'] -> Service<||>
+}
 
 $provision_repo = hiera("bigtop::provision_repo", true)
-
-stage {"pre": before => Stage["main"]}
-
 if ($provision_repo) {
-  case $::operatingsystem {
-    /(OracleLinux|Amazon|CentOS|Fedora|RedHat)/: {
-       yumrepo { "Bigtop":
-          baseurl => hiera("bigtop::bigtop_repo_uri", $default_repo),
-          descr => "Bigtop packages",
-          enabled => 1,
-          gpgcheck => 0,
-       }
-       Yumrepo<||> -> Package<||>
-    }
-    /(Ubuntu|Debian)/: {
-       include apt
-       apt::conf { "disable_keys":
-          content => "APT::Get::AllowUnauthenticated 1;",
-	  ensure => present
-       }
-       apt::source { "Bigtop":
-          location => hiera("bigtop::bigtop_repo_uri", $default_repo),
-          release => "bigtop",
-          repos => "contrib",
-          ensure => present,
-        }
-       Apt::Source<||> -> Exec['apt_update'] -> Package<||>
-    }
-    default: {
-      notify{"WARNING: running on a neither yum nor apt platform -- make sure Bigtop repo is setup": }
-    }
-  }
-}
-
-case $::operatingsystem {
-    /Debian/: {
-      require apt
-      class { 'apt::backports':
-        pin => 500,
-      }
-      Class['apt::backports'] -> Package <||>
-
-      package { "jdk":
-        name => $jdk_package_name,
-        ensure => present,
-      }
-    }
-    default: {
-      package { "jdk":
-        name => $jdk_package_name,
-        ensure => "installed",
-        alias => "jdk",
-        noop => $jdk_preinstalled,
-     }
-   }
+   require bigtop_repo
 }
 
 node default {
-
   $roles_enabled = hiera("bigtop::roles_enabled", false)
 
   if (!is_bool($roles_enabled)) {
diff --git a/provisioner/docker/config_centos6.yaml b/provisioner/docker/config_centos6.yaml
index 459f8a2a..7912aca6 100644
--- a/provisioner/docker/config_centos6.yaml
+++ b/provisioner/docker/config_centos6.yaml
@@ -22,4 +22,3 @@ distro: centos
 components: [hdfs, yarn, mapreduce]
 enable_local_repo: false
 smoke_test_components: [hdfs, yarn, mapreduce]
-jdk: "java-1.8.0-openjdk-devel.x86_64"
diff --git a/provisioner/docker/config_centos7.yaml b/provisioner/docker/config_centos7.yaml
index 6e3c28b3..6cdd7cfc 100644
--- a/provisioner/docker/config_centos7.yaml
+++ b/provisioner/docker/config_centos7.yaml
@@ -22,4 +22,3 @@ distro: centos
 components: [hdfs, yarn, mapreduce]
 enable_local_repo: false
 smoke_test_components: [hdfs, yarn, mapreduce]
-jdk: "java-1.8.0-openjdk-devel.x86_64"
diff --git a/provisioner/docker/config_debian8.yaml b/provisioner/docker/config_debian8.yaml
index 10915e50..c1a468a4 100644
--- a/provisioner/docker/config_debian8.yaml
+++ b/provisioner/docker/config_debian8.yaml
@@ -22,4 +22,3 @@ distro: debian
 components: [hdfs, yarn, mapreduce]
 enable_local_repo: false
 smoke_test_components: [hdfs, yarn, mapreduce]
-jdk: "openjdk-8-jdk"
diff --git a/provisioner/docker/config_ubuntu_xenial.yaml b/provisioner/docker/config_ubuntu_xenial.yaml
index 8f8cb878..54c91f77 100644
--- a/provisioner/docker/config_ubuntu_xenial.yaml
+++ b/provisioner/docker/config_ubuntu_xenial.yaml
@@ -22,4 +22,3 @@ distro: debian
 components: [hdfs, yarn, mapreduce]
 enable_local_repo: false
 smoke_test_components: [hdfs, yarn, mapreduce]
-jdk: "openjdk-8-jdk"
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index 0a84b076..2783a1fb 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -60,10 +60,9 @@ create() {
     # Fetch configurations form specificed yaml config file
     repo=$(get-yaml-config repo)
     components="[`echo $(get-yaml-config components) | sed 's/ /, /g'`]"
-    jdk=$(get-yaml-config jdk)
     distro=$(get-yaml-config distro)
     enable_local_repo=$(get-yaml-config enable_local_repo)
-    generate-config "$hadoop_head_node" "$repo" "$components" "$jdk"
+    generate-config "$hadoop_head_node" "$repo" "$components"
 
     # Start provisioning
     generate-hosts
@@ -90,7 +89,6 @@ bigtop::hadoop_head_node: $1
 hadoop::hadoop_storage_dirs: [/data/1, /data/2]
 bigtop::bigtop_repo_uri: $2
 hadoop_cluster_node::cluster_components: $3
-bigtop::jdk_package_name: $4
 EOF
 }
 
diff --git a/provisioner/vagrant/Vagrantfile b/provisioner/vagrant/Vagrantfile
index fc6bfc19..ae099631 100755
--- a/provisioner/vagrant/Vagrantfile
+++ b/provisioner/vagrant/Vagrantfile
@@ -59,9 +59,6 @@ smoke_test_components = CONF['smoke_test_components'].join(',')
 enable_local_repo = CONF['enable_local_repo']
 puts "vagrant conf local repo enabled:  #{enable_local_repo}"
 
-# JDK package name
-jdk = CONF['jdk']
-
 # master node hostname
 bigtop_master = "bigtop1.vagrant"
 
@@ -86,7 +83,6 @@ bigtop::hadoop_head_node: #{bigtop_master}
 hadoop::hadoop_storage_dirs: [/data/1, /data/2]
 bigtop::bigtop_repo_uri: #{repo}
 hadoop_cluster_node::cluster_components: #{components}
-bigtop::jdk_package_name: #{jdk}
 hadoop::common_hdfs::testonly_hdfs_sshkeys: "yes"
 EOF
 SCRIPT
diff --git a/provisioner/vagrant/vagrantconfig.yaml b/provisioner/vagrant/vagrantconfig.yaml
index 4fbb4e44..54604b01 100644
--- a/provisioner/vagrant/vagrantconfig.yaml
+++ b/provisioner/vagrant/vagrantconfig.yaml
@@ -23,4 +23,3 @@ components: [hdfs, yarn, mapreduce]
 enable_local_repo: false
 run_smoke_tests: false
 smoke_test_components: [hdfs, yarn, mapreduce]
-jdk: "java-1.8.0-openjdk-devel.x86_64"