diff options
74 files changed, 4445 insertions, 1094 deletions
diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/assets/data/version/version.json b/ambari-admin/src/main/resources/ui/admin-web/app/assets/data/version/version.json index 7c604f1882..95c063acb8 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/app/assets/data/version/version.json +++ b/ambari-admin/src/main/resources/ui/admin-web/app/assets/data/version/version.json @@ -1,100 +1,98 @@ { - "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions?fields=repository_versions/operatingSystems/repositories/*&repository_versions/RepositoryVersions/repository_version=2.2.0.1-901", + "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions?fields=repository_versions/operatingSystems/repositories/*&repository_versions/RepositoryVersions/repository_version=2.3.6.0-3509", "items" : [ { - "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.2", + "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.3", "Versions" : { "stack_name" : "HDP", - "stack_version" : "2.2" + "stack_version" : "2.3" }, "repository_versions" : [ { - "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.2/repository_versions/15", + "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.3/repository_versions/15", "RepositoryVersions" : { "id" : 15, - "repository_version" : "2.2.0.1-901", + "repository_version" : "2.3.6.0-3509", "stack_name" : "HDP", - "stack_version" : "2.2" - }, - "operating_systems" : [ - { - "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.2/repository_versions/15/operating_systems/redhat5", - "OperatingSystems" : { - "os_type" : "redhat5", - "repository_version_id" : 15, - "stack_name" : "HDP", - "stack_version" : "2.2" + "stack_version" : "2.3", + "type": "PATCH", + "release": { + "stack_id": "HDP-2.3", + "version": "2.3.6.0", + "build": "3509", + "compatible_with": "2.3.6.0-[1-9]", + "release_notes": "http://someurl" + }, + "services": [ + { + "name": "HDFS", + "display_name": "HDFS", + "versions": [ + { + "version": "2.1.1", + "version_id": "10", + "components": [ "NAMENODE"] + } + ] }, - "repositories" : [ - { - "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.2/repository_versions/15/operating_systems/redhat5/repositories/HDP-2.2.0.1-901", - "Repositories" : { - "base_url" : "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0", - "default_base_url" : "", - "latest_base_url" : "", - "mirrors_list" : "", - "os_type" : "redhat5", - "repo_id" : "HDP-2.2.0.1-901", - "repo_name" : "HDP", - "repository_version_id" : 15, - "stack_name" : "HDP", - "stack_version" : "2.2" + { + "name": "HIVE", + "display_name": "Hive", + "versions": [ + { + "version": "1.2.1" } - }, - { - "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.2/repository_versions/15/operating_systems/redhat5/repositories/HDP-UTILS-2.2.0.1-901", - "Repositories" : { - "base_url" : "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0", - "default_base_url" : "", - "latest_base_url" : "", - "mirrors_list" : "", - "os_type" : "redhat5", - "repo_id" : "HDP-UTILS-2.2.0.1-901", - "repo_name" : "HDP-UTILS", - "repository_version_id" : 15, - "stack_name" : "HDP", - "stack_version" : "2.2" + ] + }, + { + "name": "ZOOKEEPER", + "display_name": "ZooKeeper", + "versions": [ + { + "version": "3.4.5" } - } - ] - }, + ] + } + ] + }, + "operating_systems" : [ { - "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.2/repository_versions/15/operating_systems/redhat6", + "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.3/repository_versions/15/operating_systems/redhat6", "OperatingSystems" : { "os_type" : "redhat6", "repository_version_id" : 15, "stack_name" : "HDP", - "stack_version" : "2.2" + "stack_version" : "2.3" }, "repositories" : [ { - "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.2/repository_versions/15/operating_systems/redhat6/repositories/HDP-2.2.0.1-901", + "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.3/repository_versions/15/operating_systems/redhat6/repositories/HDP-2.3.6.0-3509", "Repositories" : { - "base_url" : "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0", + "base_url" : "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.3.6.0-3509", "default_base_url" : "", "latest_base_url" : "", "mirrors_list" : "", "os_type" : "redhat6", - "repo_id" : "HDP-2.2.0.1-901", + "repo_id" : "HDP-2.3.6.0-3509", "repo_name" : "HDP", "repository_version_id" : 15, "stack_name" : "HDP", - "stack_version" : "2.2" + "stack_version" : "2.3" } }, { - "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.2/repository_versions/15/operating_systems/redhat6/repositories/HDP-UTILS-2.2.0.1-901", + "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.3/repository_versions/15/operating_systems/redhat6/repositories/HDP-UTILS-2.3.6.0-3509", "Repositories" : { - "base_url" : "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0", + "base_url" : "http://s3.amazonaws.com/dev.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6", "default_base_url" : "", "latest_base_url" : "", "mirrors_list" : "", "os_type" : "redhat6", - "repo_id" : "HDP-UTILS-2.2.0.1-901", + "repo_id" : "HDP-UTILS-2.3.6.0-3509", "repo_name" : "HDP-UTILS", "repository_version_id" : 15, "stack_name" : "HDP", - "stack_version" : "2.2" + "stack_version" : "2.3" } } ] diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsCreateCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsCreateCtrl.js index 532e5f4899..931b7ecd15 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsCreateCtrl.js +++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsCreateCtrl.js @@ -23,40 +23,166 @@ angular.module('ambariAdminConsole') $scope.createController = true; $scope.osList = []; $scope.skipValidation = false; - $scope.selectedOS = 0; - $scope.repoSubversion = ""; $scope.clusterName = $routeParams.clusterName; $scope.subversionPattern = /^\d+\.\d+(-\d+)?$/; $scope.upgradeStack = { - selected: null, - options: [] - }; - $scope.fetchStackVersionFilterList = function () { - return Stack.allStackVersions() - .then(function (allStackVersions) { - var versions = []; - angular.forEach(allStackVersions, function (version) { - if (version.upgrade_packs.length > 0 && version.active) { - versions.push(version); - } + stack_name: '', + stack_version: '', + display_name: '' + }; + + $scope.option1 = { + index: 1, + displayName: 'Upload Version Definition File', + url: 'files://', + hasError: false + }; + $scope.option2 = { + index: 2, + displayName: 'Version Definition File URL', + url: 'https://', + hasError: false + }; + $scope.selectedOption = 1; + + /** + * User can select ONLY one option to upload version definition file + */ + $scope.toggleOptionSelect = function () { + $scope.option1.hasError = false; + $scope.option2.hasError = false; + }; + $scope.clearOptionsError = function () { + $scope.option1.hasError = false; + $scope.option2.hasError = false; + }; + $scope.readInfoButtonDisabled = function () { + return $scope.option1.selected ? !$scope.option1.url : !$scope.option2.url; + }; + + $scope.onFileSelect = function(){ + return { + link: function($scope,el){ + el.bind("change", function(e){ + $scope.file = (e.srcElement || e.target).files[0]; + $scope.getFile(); + }) + } + } + }; + +// $scope.uploadFile = function(){ +// var file = $scope.myFile; +// console.log('file is ' ); +// console.dir(file); +// var uploadUrl = "/fileUpload"; +// fileUpload.uploadFileToUrl(file, uploadUrl); +// }; + + /** + * Load selected file to current page content + */ + $scope.readVersionInfo = function(){ + if ($scope.option2.selected) { + var url = $scope.option2.url; + } + /// POST url first then get the version definition info + return Stack.getLatestRepo('HDP').then(function (response) { + $scope.id = response.id; + $scope.isPatch = response.type == 'PATCH'; + $scope.stackNameVersion = response.stackNameVersion || 'n/a'; + $scope.displayName = response.displayName || 'n/a'; + $scope.version = response.version || 'n/a'; + $scope.actualVersion = response.actualVersion || 'n/a'; + $scope.upgradeStack = { + stack_name: response.stackName, + stack_version: response.stackVersion, + display_name: response.displayName + }; + $scope.services = response.services || []; + //save default values of repos to check if they were changed + $scope.defaulfOSRepos = {}; + response.updateObj.operating_systems.forEach(function(os) { + $scope.defaulfOSRepos[os.OperatingSystems.os_type] = { + defaultBaseUrl: os.repositories[0].Repositories.base_url, + defaultUtilsUrl: os.repositories[1].Repositories.base_url + }; }); - $scope.upgradeStack.options = versions; - $scope.upgradeStack.selected = versions[versions.length - 1]; - $scope.afterStackVersionChange().then(function(){ - $scope.disableUnusedOS(); + $scope.repoVersionFullName = response.repoVersionFullName; + angular.forEach(response.osList, function (os) { + os.selected = true; }); - }) - .catch(function (data) { - Alert.error($t('versions.alerts.filterListError'), data.message); + $scope.osList = response.osList; + // load supported os type base on stack version + $scope.afterStackVersionRead(); }); }; - $scope.fetchStackVersionFilterList(); + + /** + * Load supported OS list + */ + $scope.afterStackVersionRead = function () { + Stack.getSupportedOSList($scope.upgradeStack.stack_name, $scope.upgradeStack.stack_version) + .then(function (data) { + var operatingSystems = data.operating_systems; + operatingSystems.map(function (os) { + var existingOSHash = {}; + angular.forEach($scope.osList, function (os) { + existingOSHash[os.OperatingSystems.os_type] = os; + }); + // if os not in the list, mark as un-selected, add this to the osList + if (!existingOSHash[os.OperatingSystems.os_type]) { + os.selected = false; + os.repositories.forEach(function(repo) { + repo.Repositories.base_url = ''; + }); + $scope.osList.push(os); + } + }); + }) + .catch(function (data) { + Alert.error($t('versions.alerts.osListError'), data.message); + }); + }; + + /** + * On click handler for removing OS + */ + $scope.removeOS = function() { + this.os.selected = false; + if (this.os.repositories) { + this.os.repositories.forEach(function(repo) { + repo.hasError = false; + }); + } + }; + /** + * On click handler for adding new OS + */ + $scope.addOS = function() { + this.os.selected = true; + if (this.os.repositories) { + this.os.repositories.forEach(function(repo) { + repo.hasError = false; + }); + } + }; + + $scope.isSaveButtonDisabled = function() { + var enabled = false; + $scope.osList.forEach(function(os) { + if (os.selected) { + enabled = true + } + }); + return !enabled; + } $scope.save = function () { - return Stack.validateBaseUrls($scope.skipValidation, $scope.osList, $scope.upgradeStack.selected).then(function (invalidUrls) { + return Stack.validateBaseUrls($scope.skipValidation, $scope.osList, $scope.upgradeStack).then(function (invalidUrls) { if (invalidUrls.length === 0) { - Stack.addRepo($scope.upgradeStack.selected, $scope.repoSubversion, $scope.osList) + Stack.addRepo($scope.upgradeStack, $scope.actualVersion, $scope.osList) .success(function () { var versionName = $scope.upgradeStack.selected.stack_version + '.' + $scope.repoSubversion; var stackName = $scope.upgradeStack.selected.stack_name; @@ -72,41 +198,11 @@ angular.module('ambariAdminConsole') }); }; - $scope.afterStackVersionChange = function () { - return Stack.getSupportedOSList($scope.upgradeStack.selected.stack_name, $scope.upgradeStack.selected.stack_version) - .then(function (data) { - var operatingSystems = data.operating_systems; - $scope.osList = operatingSystems.map(function (os) { - os.selected = false; - os.repositories.forEach(function(repo) { - repo.Repositories.base_url = ''; - }); - return os; - }); - }) - .catch(function (data) { - Alert.error($t('versions.alerts.osListError'), data.message); - }); - }; - - $scope.disableUnusedOS = function() { - Cluster.getClusterOS().then(function(usedOS){ - angular.forEach($scope.osList, function (os) { - if (os.OperatingSystems.os_type !== usedOS) { - os.disabled = true; - } - }); - }); - }; - - $scope.updateCurrentVersionInput = function () { - $scope.currentVersionInput = $scope.upgradeStack.selected.displayName + '.' + angular.element('[name="version"]')[0].value; + $scope.cancel = function () { + $scope.editVersionDisabled = true; + $location.path('/stackVersions'); }; - /** - * TODO create parent controller for StackVersionsEditCtrl and StackVersionsCreateCtrl and - * move this method to it - */ $scope.clearErrors = function() { if ($scope.osList) { $scope.osList.forEach(function(os) { @@ -118,27 +214,11 @@ angular.module('ambariAdminConsole') }); } }; - /** - * TODO create parent controller for StackVersionsEditCtrl and StackVersionsCreateCtrl and - * move this method to it - */ + $scope.clearError = function() { this.repository.hasError = false; }; - /** - * TODO create parent controller for StackVersionsEditCtrl and StackVersionsCreateCtrl and - * move this method to it - */ - $scope.toggleOSSelect = function () { - this.os.repositories.forEach(function(repo) { - repo.hasError = false; - }); - this.os.selected ? $scope.selectedOS++ : $scope.selectedOS--; - }; - /** - * TODO create parent controller for StackVersionsEditCtrl and StackVersionsCreateCtrl and - * move this method to it - */ + $scope.hasValidationErrors = function() { var hasErrors = false; if ($scope.osList) { @@ -154,13 +234,4 @@ angular.module('ambariAdminConsole') } return hasErrors; }; - /** - * TODO create parent controller for StackVersionsEditCtrl and StackVersionsCreateCtrl and - * move this method to it - */ - $scope.cancel = function () { - $scope.editVersionDisabled = true; - $location.path('/stackVersions'); - }; - }]); diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsEditCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsEditCtrl.js index 39a6700eda..cd9cf402fa 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsEditCtrl.js +++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsEditCtrl.js @@ -24,17 +24,27 @@ angular.module('ambariAdminConsole') $scope.osList = []; $scope.skipValidation = false; $scope.selectedOS = 0; + $scope.upgradeStack = { + stack_name: '', + stack_version: '', + display_name: '' + }; $scope.loadStackVersionInfo = function () { return Stack.getRepo($routeParams.versionId, $routeParams.stackName).then(function (response) { $scope.id = response.id; - $scope.stack = response.stack; - $scope.stackName = response.stackName; - $scope.versionName = response.versionName; - $scope.displayName = response.displayName; - $scope.stackVersion = response.stackVersion; + $scope.isPatch = response.type == 'PATCH'; + $scope.stackNameVersion = response.stackNameVersion || 'n/a'; + $scope.displayName = response.displayName || 'n/a'; + $scope.version = response.version || 'n/a'; + $scope.actualVersion = response.actualVersion || 'n/a'; $scope.updateObj = response.updateObj; - $scope.subversion = response.versionName.substring(4); // cut off stack version + $scope.upgradeStack = { + stack_name: response.stackName, + stack_version: response.stackVersion, + display_name: response.displayName + }; + $scope.services = response.services || []; //save default values of repos to check if they were changed $scope.defaulfOSRepos = {}; response.updateObj.operating_systems.forEach(function(os) { @@ -47,8 +57,10 @@ angular.module('ambariAdminConsole') angular.forEach(response.osList, function (os) { os.selected = true; }); - $scope.selectedOS = response.osList.length; $scope.osList = response.osList; + // load supported os type base on stack version + $scope.afterStackVersionRead(); + // if user reach here from UI click, repo status should be cached // otherwise re-fetch repo status from cluster end point. $scope.repoStatus = Cluster.repoStatusCache[$scope.id]; @@ -63,52 +75,40 @@ angular.module('ambariAdminConsole') } else { $scope.deleteEnabled = $scope.isDeletable(); } - $scope.addMissingOSList().then(function(){ - $scope.disableUnusedOS(); - }); + // fetch all repos to display the left menu + $scope.fetchRepos(); }); }; - $scope.isDeletable = function() { - return !($scope.repoStatus == 'current' || $scope.repoStatus == 'installed'); + /** + * Load supported OS list + */ + $scope.afterStackVersionRead = function () { + Stack.getSupportedOSList($scope.upgradeStack.stack_name, $scope.upgradeStack.stack_version) + .then(function (data) { + var operatingSystems = data.operating_systems; + operatingSystems.map(function (os) { + var existingOSHash = {}; + angular.forEach($scope.osList, function (os) { + existingOSHash[os.OperatingSystems.os_type] = os; + }); + // if os not in the list, mark as un-selected, add this to the osList + if (!existingOSHash[os.OperatingSystems.os_type]) { + os.selected = false; + os.repositories.forEach(function(repo) { + repo.Repositories.base_url = ''; + }); + $scope.osList.push(os); + } + }); + }) + .catch(function (data) { + Alert.error($t('versions.alerts.osListError'), data.message); + }); }; - $scope.addMissingOSList = function() { - return Stack.getSupportedOSList($scope.stackName, $scope.stackVersion) - .then(function (data) { - var existingOSHash = {}; - angular.forEach($scope.osList, function (os) { - existingOSHash[os.OperatingSystems.os_type] = os; - }); - var osList = data.operating_systems.map(function (os) { - return existingOSHash[os.OperatingSystems.os_type] || { - OperatingSystems: { - os_type : os.OperatingSystems.os_type - }, - repositories: [ - { - Repositories: { - base_url: '', - repo_id: 'HDP-' + $routeParams.versionId, - repo_name: 'HDP' - } - }, - { - Repositories: { - base_url: '', - repo_id: 'HDP-UTILS-' + $routeParams.versionId, - repo_name: 'HDP-UTILS' - } - } - ], - selected: false - }; - }); - $scope.osList = osList; - }) - .catch(function (data) { - Alert.error($t('versions.alerts.osListError'), data.message); - }); + $scope.isDeletable = function() { + return !($scope.repoStatus == 'current' || $scope.repoStatus == 'installed'); }; $scope.disableUnusedOS = function() { @@ -155,17 +155,13 @@ angular.module('ambariAdminConsole') }; $scope.updateRepoVersions = function () { - var upgradeStack = { - stack_name: $scope.stackName, - stack_version: $scope.stackVersion - }; - return Stack.validateBaseUrls($scope.skipValidation, $scope.osList, upgradeStack).then(function (invalidUrls) { + return Stack.validateBaseUrls($scope.skipValidation, $scope.osList, $scope.upgradeStack).then(function (invalidUrls) { if (invalidUrls.length === 0) { - Stack.updateRepo($scope.stackName, $scope.stackVersion, $scope.id, $scope.updateObj).then(function () { + Stack.updateRepo($scope.upgradeStack.stack_name, $scope.upgradeStack.stack_version, $scope.id, $scope.updateObj).then(function () { Alert.success($t('versions.alerts.versionEdited', { - stackName: $scope.stackName, - versionName: $scope.versionName, - displayName: $scope.displayName + stackName: $scope.upgradeStack.stack_name, + versionName: $scope.actualVersion, + displayName: $scope.repoVersionFullName })); $location.path('/stackVersions'); }).catch(function (data) { @@ -199,22 +195,56 @@ angular.module('ambariAdminConsole') $t('versions.deregister'), { "url": 'views/modals/BodyForDeregisterVersion.html', - "scope": {"displayName": $scope.displayName } + "scope": {"displayName": $scope.repoVersionFullName } } ).then(function() { - Stack.deleteRepo($scope.stackName, $scope.stackVersion, $scope.id).then( function () { - $location.path('/stackVersions'); - }).catch(function (data) { - Alert.error($t('versions.alerts.versionDeleteError'), data.message); + Stack.deleteRepo($scope.upgradeStack.stack_name, $scope.upgradeStack.stack_version, $scope.id).then( function () { + $location.path('/stackVersions'); + }).catch(function (data) { + Alert.error($t('versions.alerts.versionDeleteError'), data.message); + }); }); - }); }; $scope.loadStackVersionInfo(); - + + /** + * On click handler for removing OS + */ + $scope.removeOS = function() { + this.os.selected = false; + if (this.os.repositories) { + this.os.repositories.forEach(function(repo) { + repo.hasError = false; + }); + } + }; /** - * TODO create parent controller for StackVersionsEditCtrl and StackVersionsCreateCtrl and - * move this method to it + * On click handler for adding new OS */ + $scope.addOS = function() { + this.os.selected = true; + if (this.os.repositories) { + this.os.repositories.forEach(function(repo) { + repo.hasError = false; + }); + } + }; + + $scope.isSaveButtonDisabled = function() { + var enabled = false; + $scope.osList.forEach(function(os) { + if (os.selected) { + enabled = true + } + }); + return !enabled; + } + + $scope.cancel = function () { + $scope.editVersionDisabled = true; + $location.path('/stackVersions'); + }; + $scope.clearErrors = function() { if ($scope.osList) { $scope.osList.forEach(function(os) { @@ -226,29 +256,11 @@ angular.module('ambariAdminConsole') }); } }; - /** - * TODO create parent controller for StackVersionsEditCtrl and StackVersionsCreateCtrl and - * move this method to it - */ + $scope.clearError = function () { this.repository.hasError = false; }; - /** - * TODO create parent controller for StackVersionsEditCtrl and StackVersionsCreateCtrl and - * move this method to it - */ - $scope.toggleOSSelect = function () { - this.os.repositories.forEach(function (repo) { - repo.hasError = false; - }); - this.os.selected ? $scope.selectedOS++ : $scope.selectedOS--; - }; - - /** - * TODO create parent controller for StackVersionsEditCtrl and StackVersionsCreateCtrl and - * move this method to it - */ $scope.hasValidationErrors = function () { var hasErrors = false; if ($scope.osList) { @@ -265,13 +277,62 @@ angular.module('ambariAdminConsole') return hasErrors; }; + + // add all repos list + $scope.filter = { + version: '', + cluster: { + options: [], + current: null + } + }; + + $scope.pagination = { + totalRepos: 100, + maxVisiblePages: 1, + itemsPerPage: 100, + currentPage: 1 + }; + $scope.allRepos = []; + $scope.stackVersions = []; + + + /** - * TODO create parent controller for StackVersionsEditCtrl and StackVersionsCreateCtrl and - * move this method to it + * Formatted object to display all repos: + * + * [{ 'name': 'HDP-2.3', + * 'repos': ['2.3.6.0-2343', '2.3.4.1', '2.3.4.0-56'] + * }, + * { 'name': 'HDP-2.2', + * 'repos': ['2.2.6.0', '2.2.4.5', '2.2.4.0'] + * } + * ] + * */ - $scope.cancel = function () { - $scope.editVersionDisabled = true; - $location.path('/stackVersions'); + $scope.fetchRepos = function () { + return Stack.allRepos($scope.filter, $scope.pagination).then(function (repos) { + $scope.allRepos = repos.items.sort(function(a, b){return a.repository_version < b.repository_version}); + var existingStackHash = {}; + var stackVersions = []; + angular.forEach($scope.allRepos, function (repo) { + var stackVersionName = repo.stack_name + '-' + repo.stack_version; + var currentStackVersion = $scope.upgradeStack.stack_name + '-' + $scope.upgradeStack.stack_version; + repo.isActive = $scope.actualVersion == repo.repository_version; + if (!existingStackHash[stackVersionName]) { + existingStackHash[stackVersionName] = true; + stackVersions.push({ + 'name': stackVersionName, + 'isOpened': stackVersionName == currentStackVersion, + 'repos': [repo] + }); + } else { + if (stackVersions[stackVersions.length -1].repos) { + stackVersions[stackVersions.length -1].repos.push(repo); + } + } + }); + $scope.stackVersions = stackVersions; + }); }; - }]); diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsListCtrl.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsListCtrl.js index 010ec1b497..3a8233a83e 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsListCtrl.js +++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/controllers/stackVersions/StackVersionsListCtrl.js @@ -24,6 +24,7 @@ angular.module('ambariAdminConsole') return $t('common.' + key).toLowerCase(); } $scope.clusterName = $routeParams.clusterName; + $scope.filter = { version: '', cluster: { @@ -31,106 +32,52 @@ angular.module('ambariAdminConsole') current: null } }; - $scope.isNotEmptyFilter = true; $scope.pagination = { - totalRepos: 10, - maxVisiblePages: 20, - itemsPerPage: 10, + totalRepos: 100, + maxVisiblePages: 1, + itemsPerPage: 100, currentPage: 1 }; + $scope.allRepos = []; + $scope.stackVersions = []; - $scope.tableInfo = { - total: 0, - showed: 0, - filtered: 0 - }; - - $scope.stacks = []; - $scope.dropDownClusters = []; - $scope.selectedCluster = $scope.dropDownClusters[0]; - - $scope.resetPagination = function () { - $scope.pagination.currentPage = 1; - $scope.loadAllData(); - }; - - $scope.pageChanged = function () { - $scope.loadAllData(); - }; - - $scope.goToCluster = function() { - window.location.replace('/#/main/admin/stack/versions'); - }; - - $scope.clearFilters = function () { - $scope.filter.version = ''; - $scope.filter.cluster.current = $scope.filter.cluster.options[0]; - $scope.resetPagination(); - }; - - $scope.fetchRepoClusterStatus = function () { - var clusterName = ($scope.clusters && $scope.clusters.length > 0) ? $scope.clusters[0].Clusters.cluster_name : null; // only support one cluster at the moment - if (clusterName) { - angular.forEach($scope.repos, function (repo) { - Cluster.getRepoVersionStatus(clusterName, repo.id).then(function (response) { - repo.status = response.status; - repo.totalHosts = response.totalHosts; - repo.currentHosts = response.currentHosts; - repo.installedHosts = response.installedHosts; - repo.stackVersionId = response.stackVersionId; - repo.cluster = (repo.status == 'current' || repo.status == 'installed')? clusterName : ''; - }); - }); - } - }; - + /** + * Formatted object to display all repos: + * + * [{ 'name': 'HDP-2.3', + * 'repos': ['2.3.6.0-2343', '2.3.4.1', '2.3.4.0-56'] + * }, + * { 'name': 'HDP-2.2', + * 'repos': ['2.2.6.0', '2.2.4.5', '2.2.4.0'] + * } + * ] + * + */ $scope.fetchRepos = function () { return Stack.allRepos($scope.filter, $scope.pagination).then(function (repos) { - $scope.pagination.totalRepos = repos.itemTotal; - $scope.repos = repos.items; - $scope.tableInfo.total = repos.itemTotal; - $scope.tableInfo.showed = repos.showed; - }); - }; - - $scope.fillClusters = function (clusters) { - $scope.dropDownClusters = [].concat(clusters); - var options = [{label: $t('common.all'), value: ''}]; - angular.forEach(clusters, function (cluster) { - options.push({ - label: cluster.Clusters.cluster_name, - value: cluster.Clusters.cluster_name - }); + $scope.allRepos = repos.items.sort(function(a, b){return a.repository_version < b.repository_version}); + var existingStackHash = {}; + var stackVersions = []; + angular.forEach($scope.allRepos, function (repo) { + var stackVersionName = repo.stack_name + '-' + repo.stack_version; + if (!existingStackHash[stackVersionName]) { + existingStackHash[stackVersionName] = true; + stackVersions.push({ + 'name': stackVersionName, + 'isOpened': true, + 'repos': [repo] + }); + } else { + if (stackVersions[stackVersions.length -1].repos) { + stackVersions[stackVersions.length -1].repos.push(repo); + } + } }); - $scope.filter.cluster.options = options; - if (!$scope.filter.cluster.current) { - $scope.filter.cluster.current = options[0]; - } - }; - - $scope.fetchClusters = function () { - return Cluster.getAllClusters().then(function (clusters) { - if (clusters && clusters.length > 0) { - $scope.clusters = clusters; - $scope.fillClusters(clusters); - } - }); - }; - - $scope.loadAllData = function () { - $scope.fetchClusters() - .then(function () { - return $scope.fetchRepos(); - }) - .then(function () { - $scope.fetchRepoClusterStatus(); + $scope.stackVersions = stackVersions; }); }; - $scope.loadAllData(); + $scope.fetchRepos(); - $scope.$watch('filter', function (filter) { - $scope.isNotEmptyFilter = Boolean(filter.version || (filter.cluster.current && filter.cluster.current.value)); - }, true); }]); diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/i18n.config.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/i18n.config.js index 91a1645e23..aa0b830705 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/i18n.config.js +++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/i18n.config.js @@ -295,6 +295,10 @@ angular.module('ambariAdminConsole') 'current': 'Current', 'inUse': 'In Use', 'installed': 'Installed', + 'uploadFile': 'Upload Version Definition File', + 'enterURL': 'Version Definition File URL', + 'readInfo': 'Read Version Info', + 'browse': 'Browse', 'installOn': 'Install on...', 'register': 'Register Version', 'deregister': 'Deregister Version', @@ -304,14 +308,25 @@ angular.module('ambariAdminConsole') 'os': 'OS', 'baseURL': 'Base URL', 'skipValidation': 'Skip Repository Base URL validation (Advanced)', - + 'noVersions': 'Select version to display details.', + 'contents': { + 'title': 'Contents', + 'empty': 'No contents to display' + }, + 'details': { + 'stackName': 'Stack Name', + 'displayName': 'Display Name', + 'version': 'Version', + 'actualVersion': 'Actual Version', + 'releaseNotes': 'Release Notes' + }, 'changeBaseURLConfirmation': { 'title': 'Confirm Base URL Change', 'message': 'You are about to change repository Base URLs that are already in use. Please confirm that you intend to make this change and that the new Base URLs point to the same exact Stack version and build' }, 'alerts': { - 'baseURLs': 'Provide Base URLs for the Operating Systems you are configuring. Uncheck all other Operating Systems.', + 'baseURLs': 'Provide Base URLs for the Operating Systems you are configuring.', 'validationFailed': 'Some of the repositories failed validation. Make changes to the base url or skip validation if you are sure that urls are correct', 'skipValidationWarning': '<b>Warning:</b> This is for advanced users only. Use this option if you want to skip validation for Repository Base URLs.', 'filterListError': 'Fetch stack version filter list error', diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Stack.js b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Stack.js index 4c55967f72..ae67c7ecde 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Stack.js +++ b/ambari-admin/src/main/resources/ui/admin-web/app/scripts/services/Stack.js @@ -116,11 +116,11 @@ angular.module('ambariAdminConsole') return deferred.promise; }, - addRepo: function (stack, repoSubversion, osList) { + addRepo: function (stack, actualVersion, osList) { var url = '/stacks/' + stack.stack_name + '/versions/' + stack.stack_version + '/repository_versions/'; var payload = {}; var payloadWrap = { RepositoryVersions : payload }; - payload.repository_version = stack.stack_version + '.' + repoSubversion; + payload.repository_version = actualVersion; payload.display_name = stack.stack_name + '-' + payload.repository_version; payloadWrap.operating_systems = []; osList.forEach(function (osItem) { @@ -157,14 +157,27 @@ angular.module('ambariAdminConsole') var response = { id : data.repository_versions[0].RepositoryVersions.id, stackVersion : data.Versions.stack_version, - stack: data.Versions.stack_name + '-' + data.Versions.stack_version, stackName: data.Versions.stack_name, - versionName: data.repository_versions[0].RepositoryVersions.repository_version, - displayName : data.repository_versions[0].RepositoryVersions.display_name, + type: data.repository_versions[0].RepositoryVersions.release? data.repository_versions[0].RepositoryVersions.release.type: null, + stackNameVersion: data.Versions.stack_name + '-' + data.Versions.stack_version, /// HDP-2.3 + actualVersion: data.repository_versions[0].RepositoryVersions.repository_version, /// 2.3.4.0-3846 + version: data.repository_versions[0].RepositoryVersions.release ? data.repository_versions[0].RepositoryVersions.release.version: null, /// 2.3.4.0 + releaseNotes: data.repository_versions[0].RepositoryVersions.release ? data.repository_versions[0].RepositoryVersions.release.release_notes: null, + displayName: data.repository_versions[0].RepositoryVersions.release ? data.Versions.stack_name + '-' + data.repository_versions[0].RepositoryVersions.release.version : + data.Versions.stack_name + '-' + data.repository_versions[0].RepositoryVersions.repository_version.split('-')[0], //HDP-2.3.4.0 repoVersionFullName : data.Versions.stack_name + '-' + data.repository_versions[0].RepositoryVersions.repository_version, osList: data.repository_versions[0].operating_systems, updateObj: data.repository_versions[0] }; + var services = []; + angular.forEach(data.repository_versions[0].RepositoryVersions.services, function (service) { + services.push({ + name: service.name, + version: service.versions[0].version, + components: service.versions[0].components + }); + }); + response.services = services; deferred.resolve(response); }) .error(function (data) { @@ -173,6 +186,147 @@ angular.module('ambariAdminConsole') return deferred.promise; }, + getLatestRepo: function (stack_name) { + var url = Settings.baseUrl + '/stacks/' + stack_name + '/versions?' + + 'fields=repository_versions/operating_systems/repositories/*' + + ',repository_versions/RepositoryVersions/*'; // tbd + var deferred = $q.defer(); + $http.get(url, {mock: 'version/version.json'}) + .success(function (data) { + //data = data.items[0]; + data = { + "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.3", + "Versions" : { + "stack_name" : "HDP", + "stack_version" : "2.3" + }, + "repository_versions" : [ + { + "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.3/repository_versions/15", + "RepositoryVersions" : { + "id" : 15, + "repository_version" : "2.3.6.0-3509", + "stack_name" : "HDP", + "stack_version" : "2.3", + "type": "PATCH", + "release": { + "stack_id": "HDP-2.3", + "version": "2.3.6.0", + "build": "3509", + "compatible_with": "2.3.6.0-[1-9]", + "release_notes": "http://someurl" + }, + "services": [ + { + "name": "HDFS", + "display_name": "HDFS", + "versions": [ + { + "version": "2.1.1", + "version_id": "10", + "components": [ "NAMENODE"] + } + ] + }, + { + "name": "HIVE", + "display_name": "Hive", + "versions": [ + { + "version": "1.2.1" + } + ] + }, + { + "name": "ZOOKEEPER", + "display_name": "ZooKeeper", + "versions": [ + { + "version": "3.4.5" + } + ] + } + ] + }, + "operating_systems" : [ + { + "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.3/repository_versions/15/operating_systems/redhat6", + "OperatingSystems" : { + "os_type" : "redhat6", + "repository_version_id" : 15, + "stack_name" : "HDP", + "stack_version" : "2.3" + }, + "repositories" : [ + { + "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.3/repository_versions/15/operating_systems/redhat6/repositories/HDP-2.3.6.0-3509", + "Repositories" : { + "base_url" : "http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/BUILDS/2.3.6.0-3509", + "default_base_url" : "", + "latest_base_url" : "", + "mirrors_list" : "", + "os_type" : "redhat6", + "repo_id" : "HDP-2.3.6.0-3509", + "repo_name" : "HDP", + "repository_version_id" : 15, + "stack_name" : "HDP", + "stack_version" : "2.3" + } + }, + { + "href" : "http://c6401.ambari.apache.org:8080/api/v1/stacks/HDP/versions/2.3/repository_versions/15/operating_systems/redhat6/repositories/HDP-UTILS-2.3.6.0-3509", + "Repositories" : { + "base_url" : "http://s3.amazonaws.com/dev.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6", + "default_base_url" : "", + "latest_base_url" : "", + "mirrors_list" : "", + "os_type" : "redhat6", + "repo_id" : "HDP-UTILS-2.3.6.0-3509", + "repo_name" : "HDP-UTILS", + "repository_version_id" : 15, + "stack_name" : "HDP", + "stack_version" : "2.3" + } + } + ] + } + ] + } + ] + }; + + var response = { + id : data.repository_versions[0].RepositoryVersions.id, + stackVersion : data.Versions.stack_version, + stackName: data.Versions.stack_name, + type: data.repository_versions[0].RepositoryVersions.type, + stackNameVersion: data.Versions.stack_name + '-' + data.Versions.stack_version, /// HDP-2.3 + actualVersion: data.repository_versions[0].RepositoryVersions.repository_version, /// 2.3.4.0-3846 + version: data.repository_versions[0].RepositoryVersions.release ? data.repository_versions[0].RepositoryVersions.release.version: null, /// 2.3.4.0 + releaseNotes: data.repository_versions[0].RepositoryVersions.release ? data.repository_versions[0].RepositoryVersions.release.release_notes: null, + displayName: data.repository_versions[0].RepositoryVersions.release ? data.Versions.stack_name + '-' + data.repository_versions[0].RepositoryVersions.release.version : + data.Versions.stack_name + '-' + data.repository_versions[0].RepositoryVersions.repository_version.split('-')[0], //HDP-2.3.4.0 + repoVersionFullName : data.Versions.stack_name + '-' + data.repository_versions[0].RepositoryVersions.repository_version, + osList: data.repository_versions[0].operating_systems, + updateObj: data.repository_versions[0] + }; + var services = []; + angular.forEach(data.repository_versions[0].RepositoryVersions.services, function (service) { + services.push({ + name: service.display_name, + version: service.versions[0].version, + components: service.versions[0].components + }); + }); + response.services = services; + deferred.resolve(response); + }) + .error(function (data) { + deferred.reject(data); + }); + return deferred.promise; + }, + updateRepo: function (stackName, stackVersion, id, payload) { var url = Settings.baseUrl + '/stacks/' + stackName + '/versions/' + stackVersion + '/repository_versions/' + id; var deferred = $q.defer(); diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/styles/main.css b/ambari-admin/src/main/resources/ui/admin-web/app/styles/main.css index cc57fa396d..93f22711c2 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/app/styles/main.css +++ b/ambari-admin/src/main/resources/ui/admin-web/app/styles/main.css @@ -1378,9 +1378,104 @@ thead.view-permission-header > tr > th { } .enable-ldap input[type="checkbox"] { - margin-top: 10px; + margin-top: 10px; } .test-ldap-icon.ng-hide-add-active, .test-ldap-icon.ng-hide-remove { - display: inline-block!important; -}
\ No newline at end of file + display: inline-block!important; +} + +.left-menu-all-repos { + padding-left: 5px; + padding-right: 5px; +} + +.left-menu-all-repos .glyphicon.glyphicon-chevron-right{ + -webkit-transition: all 0.3s; + -o-transition: all 0.3s; + transition: all 0.3s; +} +.left-menu-all-repos .glyphicon.glyphicon-chevron-right.opened{ + -webkit-transform: rotateZ(90deg); + -ms-transform: rotateZ(90deg); + -o-transform: rotateZ(90deg); + transform: rotateZ(90deg); +} + +.left-menu-all-repos .stack-version-title { + font-size: 14px; + cursor: pointer; + text-decoration: none; + padding-left: 10px; +} + +.left-menu-all-repos .repos-table { + margin-bottom: 0px; +} + +.left-menu-all-repos .panel-body { + padding: 15px 0px; +} + +.left-menu-all-repos .repos-table .repos-td{ + border-top: none; + padding: 5px 10px; +} +.left-menu-all-repos .repos-table .repos-td > a { + text-decoration: none; +} +.left-menu-all-repos .repos-table .repos-td.active{ + background-color: #666;; +} +.left-menu-all-repos .repos-table .repos-td.active > a { + color: white; +} + +.register-version-options .read-info-button { + margin-bottom: 10px; +} + +.register-version-options .option-radio-button { + padding-top: 5px; +} + +.register-version-options .choose-file-input { + padding-top: 6px; + padding-bottom: 20px; +} + +.register-version-form .details-panel .patch-icon { + color: #ff4500; +} +.register-version-form .deregister-button { + margin-top: -23px; +} +.register-version-form .version-info { + padding-top: 7px; + margin-top: 0; + margin-bottom: 0; +} + +.register-version-form .contents-panel .version-contents-body { + max-height: 150px; + overflow: scroll; +} + +.register-version-form .repos-panel .remove-icon { + color: red; + margin: 20px 0px; + padding: 0px; + text-align: center; + cursor: pointer; +} + +.register-version-form .repos-panel .os-type-label { + margin-top: 27px;; +} + +#stack-versions .no-version-alert { + text-align: center; +} + + + diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/list.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/list.html index 5f5421bf28..3de92c1492 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/list.html +++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/list.html @@ -29,85 +29,34 @@ </div> </div> <hr/> - <table class="table table-striped table-hover"> - <thead> - <tr> - <th class="col-medium"> - <label>{{'common.name' | translate}}</label> - <input type="text" class="form-control" ng-change="resetPagination()" ng-model="filter.version" placeholder="{{'common.any' | translate}}"> - </th> - <th class="col-small"> - <label>{{'common.cluster' | translate}}</label> - <select class="form-control" - ng-change="resetPagination()" - ng-model="filter.cluster.current" - ng-options="item.label for item in filter.cluster.options track by item.value" - ></select> - </th> - <th></th> - </tr> - </thead> - <tbody> - <tr ng-repeat="repo in repos"> - <td class="col-medium"> - <a href="#/stackVersions/{{repo.stack_name}}/{{repo.repository_version}}/edit">{{repo.display_name}}</a> - </td> - <td class="col-small"> - <a href="/#/main/admin/stack/versions" ng-show="repo.cluster"> - {{repo.cluster}} - </a> - <span ng-show="!repo.cluster"> - {{'common.none' | translate}} - </span> - </td> - <td class="verison-label-row"> - <div ng-show="repo.status == 'current'"> - <span class="label {{'status-' + repo.status}}">{{'versions.current' | translate}}: {{repo.currentHosts}}/{{repo.totalHosts}}</span> - </div> - <div ng-show="repo.status == 'installed'"> - <span class="label {{'status-' + repo.status}}">{{'versions.installed' | translate}}: {{repo.installedHosts}}/{{repo.totalHosts}}</span> - </div> - <div ng-show="!repo.cluster"> - <div class="btn-group display-inline-block" dropdown is-open="viewsdropdown.isopen" ng-mouseover="viewsdropdown.isopen=true" ng-mouseout="viewsdropdown.isopen=false" ng-init="viewsdropdown.isopen=false"> - <a class="btn dropdown-toggle"> - <span>{{'versions.installOn' | translate}}</span> - </a> - <ul class="dropdown-menu" ng-show="viewsdropdown.isopen"> - <li ng-repeat="cluster in dropDownClusters"> - <a href="javascript:void(null)" ng-click="goToCluster()"> - <span>{{cluster.Clusters.cluster_name}}</span> - </a> - </li> - </ul> - </div> + + <accordion close-others="false" class="col-sm-2 left-menu-all-repos"> + <accordion-group ng-repeat="stackVersion in stackVersions" is-open="stackVersion.isOpened"> + <accordion-heading> + <div class="row stack-version-title"> + <i class="glyphicon glyphicon-chevron-right" ng-class="{'opened': stackVersion.isOpened}"></i> + {{stackVersion.name}} </div> - </td> - </tr> - </tbody> - </table> - <div class="alert alert-info col-sm-12" ng-show="!repos.length"> - {{'common.alerts.nothingToDisplay' | translate: '{term: getConstant("version")}'}} - </div> - <div class="col-sm-12 table-bar"> - <div class="pull-left filtered-info"> - <span>{{'common.filterInfo' | translate: '{showed: tableInfo.showed, total: tableInfo.total, term: getConstant("versions")}'}}</span> - <span ng-show="isNotEmptyFilter">- <a href ng-click="clearFilters()">{{'common.controls.clearFilters' | translate}}</a></span> + </accordion-heading> + <table class="table repos-table"> + <tbody> + <tr ng-repeat="repo in stackVersion.repos"> + <td class="repos-td"> + <a href="#/stackVersions/{{repo.stack_name}}/{{repo.repository_version}}/edit">{{repo.repository_version}}</a> + </td> + </tr> + </tbody> + </table> + </accordion-group> + <div class="alert alert-info" ng-show="stackVersions && !stackVersions.length"> + {{'versions.contents.empty' | translate}} </div> - <div class="pull-right left-margin"> - <pagination class="paginator" - total-items="pagination.totalRepos" - max-size="pagination.maxVisiblePages" - items-per-page="pagination.itemsPerPage" - ng-model="pagination.currentPage" - ng-change="pageChanged()" - ></pagination> - </div> - <div class="pull-right"> - <select class="form-control" - ng-model="pagination.itemsPerPage" - ng-options="item for item in [10, 25, 50, 100]" - ng-change="resetPagination()" - ></select> + </accordion> + + <form class="col-sm-10 form-horizontal" role="form"novalidate> + <div class="alert alert-info no-version-alert"> + {{'versions.noVersions' | translate}} </div> - </div> + </form> + </div> diff --git a/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/stackVersionPage.html b/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/stackVersionPage.html index 449d743590..1fad9a3ba5 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/stackVersionPage.html +++ b/ambari-admin/src/main/resources/ui/admin-web/app/views/stackVersions/stackVersionPage.html @@ -23,39 +23,109 @@ <li class="active" ng-if="createController">{{'versions.register' | translate}}</li> </ol> - <div class="pull-right top-margin-4" ng-switch="deleteEnabled" ng-if="editController"> - <button ng-switch-when="false" class="btn disabled btn-default" tooltip="Cannot delete version already installed.">{{'versions.deregister' | translate}}</button> - <button ng-switch-when="true" class="btn btn-danger" ng-click="delete()">{{'versions.deregister' | translate}}</button> + <div class="pull-right top-margin-4"> + <a href="#/stackVersions/create" class="btn btn-primary"> + <span class="glyphicon glyphicon-plus"></span> + {{'versions.register' | translate}} + </a> </div> </div> <hr> -<form class="form-horizontal register-version-form" role="form" name="repoRegForm" novalidate> - <div class="panel panel-default" ng-if="createController"> + +<div id="upload-definition-file-panel" ng-if="createController"> + <div class="clearfix register-version-options"> + <div class="col-sm-5 option-radio-button"> + <label class="option-label"> + <input type="radio" ng-model="selectedOption" value="{{option1.index}}" ng-change="toggleOptionSelect()"> {{'versions.uploadFile' | translate}} + </label> + </div> + <div class="col-sm-7"> + <input type="file" class="choose-file-input" ng-file-select="onFileSelect($files)"/> + </div> + </div> + <div class="clearfix register-version-options border-bottom bottom-margin"> + <div class="col-sm-5 option-radio-button"> + <label class="option-label"> + <input type="radio" ng-model="selectedOption" value="{{option2.index}}" ng-change="toggleOptionSelect()"> {{'versions.enterURL' | translate}} + </label> + </div> + <div class="col-sm-7"> + <div class="form-group {{option2.name}}" ng-class="{'has-error': option2.url.hasError }"> + <div class=""><input type="text" class="form-control" ng-model="option2.url" ng-change="clearOptionsError()" ng-disabled="!(selectedOption==2)"></div> + </div> + </div> + <div class="col-sm-12 read-info-button"> + <button class="btn btn-primary pull-right" ng-click="readVersionInfo()" + ng-disabled="readInfoButtonDisabled()">{{'versions.readInfo' | translate}}</button> + </div> + </div> +</div> + +<accordion close-others="false" class="col-sm-2 left-menu-all-repos" ng-if="editController"> + <accordion-group ng-repeat="stackVersion in stackVersions" is-open="stackVersion.isOpened"> + <accordion-heading> + <div class="row stack-version-title"> + <i class="glyphicon glyphicon-chevron-right" ng-class="{'opened': stackVersion.isOpened}"></i> + {{stackVersion.name}} + </div> + </accordion-heading> + <table class="table repos-table"> + <tbody> + <tr ng-repeat="repo in stackVersion.repos"> + <td class="repos-td" ng-class="{'active': repo.isActive}"> + <a href="#/stackVersions/{{repo.stack_name}}/{{repo.repository_version}}/edit">{{repo.repository_version}}</a> + </td> + </tr> + </tbody> + </table> + </accordion-group> + <div class="alert alert-info" ng-show="stackVersions && !stackVersions.length"> + {{'versions.contents.empty' | translate}} + </div> +</accordion> + +<form ng-class="{'col-sm-10': editController}" class="form-horizontal register-version-form" role="form" name="repoRegForm" novalidate> + <div class="panel panel-default details-panel"> <div class="panel-heading"> <h3 class="panel-title">{{'common.details' | translate}}</h3> + <div class="pull-right deregister-button" ng-switch="deleteEnabled" ng-if="editController"> + <button ng-switch-when="false" class="btn disabled btn-default" tooltip="Cannot delete version already installed.">{{'versions.deregister' | translate}}</button> + <button ng-switch-when="true" class="btn btn-danger" ng-click="delete()">{{'versions.deregister' | translate}}</button> + </div> </div> <div class="panel-body"> - <div class="form-inline repo-version-inline"> - <label class="control-label col-sm-2 repo-version-label">{{'common.name' | translate}}</label> - <div class="col-sm-10"> - <select class="form-control repo-version-select" - ng-model="$parent.upgradeStack.selected" - ng-options="o as o.displayName for o in upgradeStack.options" - ng-change="afterStackVersionChange()"> - </select> - <span class="bold-dot">.</span> - <div class="form-group" ng-class="{'has-error' : repoRegForm.version.$error.pattern}"> - <input class="form-control" name="version" type="text" ng-model="$parent.repoSubversion" ng-pattern="subversionPattern" - placeholder="{{'versions.placeholder' | translate}}" ng-change="updateCurrentVersionInput()" required/> - <span class="text-danger" ng-show="repoRegForm.version.$error.pattern"> -  {{'common.alerts.isInvalid' | translate: '{term: currentVersionInput}'}} - </span> - </div> - </div> + <div class="clearfix"> + <label class="control-label col-sm-3">{{'versions.details.stackName' | translate}}</label> + <div class="version-info col-sm-7">{{stackNameVersion}}</div> + <div class="col-sm-2 patch-icon" ng-if="isPatch"><span class="glyphicon glyphicon-tree-deciduous"></span>Patch</div> + </div> + <div class="clearfix"> + <label class="control-label col-sm-3">{{'versions.details.displayName' | translate}}</label> + <div class="version-info col-sm-9">{{displayName}}</div> + </div> + <div class="clearfix"> + <label class="control-label col-sm-3">{{'versions.details.version' | translate}}</label> + <div class="version-info col-sm-9">{{version}}</div> + </div> + <div class="clearfix"> + <label class="control-label col-sm-3">{{'versions.details.actualVersion' | translate}}</label> + <div class="version-info col-sm-9">{{actualVersion}}</div> </div> </div> </div> - <div class="panel panel-default"> + <div class="panel panel-default contents-panel"> + <div class="panel-heading"> + <h3 class="panel-title">{{'versions.contents.title' | translate}}</h3> + </div> + <div class="panel-body version-contents-body"> + <div class="alert alert-info hide-soft" ng-class="{'visible' : !services || !services.length}" role="alert">{{'versions.contents.empty' | translate}}</div> + <div class="clearfix" ng-repeat="service in services"> + <label class="control-label col-sm-3">{{service.name}}</label> + <div class="version-info col-sm-9">{{service.version}}</div> + </div> + </div> + </div> + <div class="panel panel-default repos-panel"> <div class="panel-heading"> <h3 class="panel-title">{{'versions.repos' | translate}}</h3> </div> @@ -67,22 +137,32 @@ <div class="name-label-adjust col-sm-2"><h5><label>{{'common.name' | translate}}</label></h5></div> <div class="col-sm-7"><h5><label >{{'versions.baseURL' | translate}}</label></h5></div> </div> - <div class="clearfix border-bottom bottom-margin" ng-repeat="os in osList"> - <div class="col-sm-2 os-checkbox"> - <div class="checkbox"> - <label> - <input type="checkbox" ng-model="os.selected" ng-change="toggleOSSelect()" ng-disabled="os.disabled"> {{os.OperatingSystems.os_type}} - </label> - </div> - </div> - <div class="col-sm-10"> - <div class="form-group {{repository.Repositories.repo_name}}" ng-class="{'has-error': repository.hasError }" ng-repeat="repository in os.repositories"> - <div class="col-sm-3"><label class="control-label">{{repository.Repositories.repo_name}}</label></div> - <div class="col-sm-9"><input type="text" class="form-control" ng-model="repository.Repositories.base_url" - ng-change="clearError()" ng-disabled="os.disabled"></div> + <div class="alert alert-info hide-soft" ng-class="{'visible' : !osList || !osList.length}" role="alert">{{'versions.contents.empty' | translate}}</div> + <div class="" ng-repeat="os in osList"> + <div ng-if="os.selected==true"> + <div class="clearfix border-bottom bottom-margin"> + <!-- show selected os in list table--> + <div class="col-sm-2 os-type-label"> + <label>{{os.OperatingSystems.os_type}}</label> + </div> + <div class="col-sm-9"> + <div class="form-group {{repository.Repositories.repo_name}}" ng-class="{'has-error': repository.hasError }" ng-repeat="repository in os.repositories"> + <div class="col-sm-3"><label class="control-label">{{repository.Repositories.repo_name}}</label></div> + <div class="col-sm-9"><input type="text" class="form-control" ng-model="repository.Repositories.base_url" + ng-change="clearError()" ng-disabled="!os.selected"></div> + </div> + </div> + <div class="col-sm-1 remove-icon" ng-click="removeOS()"><span class="glyphicon glyphicon-minus"></span>Remove</div> </div> </div> </div> + <div class="btn-group pull-right" dropdown> + <button class="btn dropdown-toggle"> + <span class="glyphicon glyphicon-plus"></span> Add OS <span class="caret"></span></button> + <ul class="dropdown-menu"> + <li ng-repeat="os in osList"><a ng-if="os.selected==false" ng-click="addOS()">{{os.OperatingSystems.os_type}}</a></li> + </ul> + </div> <div class="clearfix"> <div class="col-sm-12" id="skip-validation"> <div class="checkbox"> @@ -97,7 +177,7 @@ </div> <div class="col-sm-12"> <button class="btn btn-primary pull-right left-margin" ng-click="save()" - ng-disabled="(createController && repoRegForm.version.$invalid) || selectedOS === 0">{{'common.controls.save' | translate}}</button> + ng-disabled="isSaveButtonDisabled()">{{'common.controls.save' | translate}}</button> <button class="btn btn-default pull-right" ng-click="cancel()">{{'common.controls.cancel' | translate}}</button> </div> </form> diff --git a/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/stackVersions/StackversionsListCtrl_test.js b/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/stackVersions/StackversionsListCtrl_test.js index 1e47c200e7..6f168dbb2b 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/stackVersions/StackversionsListCtrl_test.js +++ b/ambari-admin/src/main/resources/ui/admin-web/test/unit/controllers/stackVersions/StackversionsListCtrl_test.js @@ -40,157 +40,5 @@ describe('#Cluster', function () { }); - describe('#fillClusters()', function () { - - var clusters = [ - { - Clusters: { - cluster_name: 'c0' - } - } - ], - cases = [ - { - prev: null, - current: { - label: 'All', - value: '' - }, - title: 'no cluster selected before' - }, - { - prev: { - label: 'c0', - value: 'c0' - }, - current: { - label: 'c0', - value: 'c0' - }, - title: 'cluster was selected before' - } - ]; - - angular.forEach(cases, function (item) { - it(item.title, function() { - scope.filter.cluster.current = item.prev; - scope.fillClusters(clusters); - expect(scope.dropDownClusters).toEqual(clusters); - expect(scope.filter.cluster.current).toEqual(item.current); - }); - }); - - }); - - describe('#isNotEmptyFilter', function () { - - var cases = [ - { - filter: { - version: '', - cluster: { - current: null - } - }, - isNotEmptyFilter: false, - title: 'no filters' - }, - { - filter: { - version: '', - cluster: { - current: { - value: '' - } - } - }, - isNotEmptyFilter: false, - title: 'empty filters' - }, - { - filter: { - version: 'a', - cluster: { - current: { - value: '' - } - } - }, - isNotEmptyFilter: true, - title: 'version filter' - }, - { - filter: { - version: '0', - cluster: { - current: { - value: '' - } - } - }, - isNotEmptyFilter: true, - title: 'version filter with "0" as string' - }, - { - filter: { - version: '', - cluster: { - current: { - value: 'a' - } - } - }, - isNotEmptyFilter: true, - title: 'cluster filter' - }, - { - filter: { - version: '', - cluster: { - current: { - value: '0' - } - } - }, - isNotEmptyFilter: true, - title: 'cluster filter with "0" as string' - }, - { - filter: { - version: 'a', - cluster: { - current: { - value: 'a' - } - } - }, - isNotEmptyFilter: true, - title: 'both filters' - }, - { - filter: { - version: '0', - cluster: { - current: { - value: '0' - } - } - }, - isNotEmptyFilter: true, - title: 'both filters with "0" as string' - } - ]; - - cases.forEach(function (item) { - it(item.title, function () { - $httpBackend.expectGET(/\/api\/v1\/clusters\?_=\d+/).respond(200); - scope.filter = item.filter; - scope.$digest(); - expect(scope.isNotEmptyFilter).toEqual(item.isNotEmptyFilter); - }); - }); - - }); - }); }); diff --git a/ambari-funtest/src/test/java/org/apache/ambari/funtest/server/tests/DeleteServiceTest.java b/ambari-funtest/src/test/java/org/apache/ambari/funtest/server/tests/DeleteServiceTest.java index 472bdc431f..747ae95a87 100644 --- a/ambari-funtest/src/test/java/org/apache/ambari/funtest/server/tests/DeleteServiceTest.java +++ b/ambari-funtest/src/test/java/org/apache/ambari/funtest/server/tests/DeleteServiceTest.java @@ -39,7 +39,6 @@ import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity; import org.apache.ambari.server.orm.entities.HostComponentStateEntity; import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity; import org.apache.ambari.server.orm.entities.ServiceDesiredStateEntityPK; -import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntityPK; import org.apache.ambari.server.state.State; import org.apache.commons.httpclient.HttpStatus; @@ -54,6 +53,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; + /** * Simple test that starts the local ambari server, * tests it's status and shuts down the server. @@ -181,8 +181,7 @@ public class DeleteServiceTest extends ServerTestBase { /** * ServiceComponentDesiredStateDAO */ - ServiceComponentDesiredStateEntityPK serviceComponentDesiredStateEntityPK = injector.getInstance(ServiceComponentDesiredStateEntityPK.class); - ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity = serviceComponentDesiredStateDAO.findByPK(serviceComponentDesiredStateEntityPK); + ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity = serviceComponentDesiredStateDAO.findById(0L); assertTrue(serviceComponentDesiredStateEntity == null); /** diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java index 4c12094020..da00d553e4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java @@ -412,6 +412,10 @@ public class ResourceInstanceFactoryImpl implements ResourceInstanceFactory { resourceDefinition = new SimpleResourceDefinition(Resource.Type.Setting, "setting", "settings"); break; + case VersionDefinition: + resourceDefinition = new VersionDefinitionResourceDefinition(); + break; + default: throw new IllegalArgumentException("Unsupported resource type: " + type); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/VersionDefinitionResourceDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/VersionDefinitionResourceDefinition.java new file mode 100644 index 0000000000..67d94394af --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/VersionDefinitionResourceDefinition.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.api.resources; + +import java.util.List; + +import org.apache.ambari.server.api.services.Request; +import org.apache.ambari.server.api.util.TreeNode; +import org.apache.ambari.server.controller.internal.RepositoryVersionResourceProvider; +import org.apache.ambari.server.controller.internal.VersionDefinitionResourceProvider; +import org.apache.ambari.server.controller.spi.Resource; + +import com.google.common.collect.Lists; + +/** + * The Resource Definition used for Version Definition files. + */ +public class VersionDefinitionResourceDefinition extends BaseResourceDefinition { + private static final String STACKS_NAME = new StackResourceDefinition().getPluralName(); + private static final String STACK_VERSIONS_NAME = new StackVersionResourceDefinition().getPluralName(); + private static final String REPO_VERSIONS_NAME = new RepositoryVersionResourceDefinition().getPluralName(); + + private static final String HREF_TEMPLATE = + STACKS_NAME + "/%s/" + STACK_VERSIONS_NAME + "/%s/" + REPO_VERSIONS_NAME; + + + public VersionDefinitionResourceDefinition() { + super(Resource.Type.VersionDefinition); + } + + @Override + public String getPluralName() { + return "version_definitions"; + } + + @Override + public String getSingularName() { + return "version_definition"; + } + + @Override + public List<PostProcessor> getPostProcessors() { + List<PostProcessor> list = Lists.newArrayList(); + + list.add(new HrefPostProcessor()); + + return list; + } + + + class HrefPostProcessor extends BaseHrefPostProcessor { + @Override + public void process(Request request, TreeNode<Resource> resultNode, String href) { + super.process(request, resultNode, href); + + Object stackNameObj = resultNode.getObject().getPropertyValue( + VersionDefinitionResourceProvider.VERSION_DEF_STACK_NAME); + Object stackVersionObj = resultNode.getObject().getPropertyValue( + VersionDefinitionResourceProvider.VERSION_DEF_STACK_VERSION); + + if (resultNode.getObject().getType() == Resource.Type.VersionDefinition && + null != stackNameObj && null != stackVersionObj && + null != resultNode.getProperty("href")) { + + String oldHref = resultNode.getProperty("href").toString(); + + String newPath = String.format(HREF_TEMPLATE, stackNameObj, stackVersionObj); + + String newHref = oldHref.replace(getPluralName(), newPath); + newHref = newHref.replace(VersionDefinitionResourceProvider.VERSION_DEF, + RepositoryVersionResourceProvider.REPOSITORY_VERSION); + + resultNode.setProperty("href", newHref); + } + } + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/VersionDefinitionService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/VersionDefinitionService.java new file mode 100644 index 0000000000..e6378504fa --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/VersionDefinitionService.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.services; + +import java.util.Collections; + +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; + +import org.apache.ambari.server.api.resources.ResourceInstance; +import org.apache.ambari.server.controller.internal.VersionDefinitionResourceProvider; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.commons.codec.binary.Base64; + +import com.google.gson.JsonObject; + +@Path("/version_definitions/") +public class VersionDefinitionService extends BaseService { + + @GET + @Produces(MediaType.TEXT_PLAIN) + public Response getServices(@Context HttpHeaders headers, @Context UriInfo ui) { + + return handleRequest(headers, null, ui, Request.Type.GET, + createResource(null)); + } + + @GET + @Path("{versionId}") + @Produces(MediaType.TEXT_PLAIN) + public Response getService(@Context HttpHeaders headers, @Context UriInfo ui, + @PathParam("versionId") Long versionId) { + + return handleRequest(headers, null, ui, Request.Type.GET, + createResource(versionId)); + } + + @POST + @Produces(MediaType.TEXT_PLAIN) + public Response createVersion(String body, @Context HttpHeaders headers, @Context UriInfo ui) { + return handleRequest(headers, body, ui, Request.Type.POST, + createResource(null)); + } + + /** + * Creates a version by directly POSTing the XML body. Unfortunately the request processor + * uses JSON, so an appropriate JSON structure must be made to get to the ResourceProvider. + * @param body the XML + * @param headers the headers + * @param ui the URI info + */ + @POST + @Consumes({MediaType.TEXT_XML}) + @Produces(MediaType.TEXT_PLAIN) + public Response createVersionByXml(String body, @Context HttpHeaders headers, + @Context UriInfo ui) throws Exception { + + String encoded = Base64.encodeBase64String(body.getBytes("UTF-8")); + + JsonObject obj = new JsonObject(); + obj.addProperty(VersionDefinitionResourceProvider.VERSION_DEF_BASE64_PROPERTY, encoded); + + JsonObject payload = new JsonObject(); + payload.add(VersionDefinitionResourceProvider.VERSION_DEF, obj); + + return handleRequest(headers, payload.toString(), ui, Request.Type.POST, + createResource(null)); + } + + protected ResourceInstance createResource(Long versionId) { + return createResource(Resource.Type.VersionDefinition, + Collections.singletonMap(Resource.Type.VersionDefinition, + null == versionId ? null : versionId.toString())); + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java index 30250ef036..620c651370 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java @@ -53,6 +53,7 @@ public class HostsRepositoryVersionCheck extends AbstractCheckDescriptor { super(CheckDescription.HOSTS_REPOSITORY_VERSION); } + @Override public boolean isApplicable(PrereqCheckRequest request) throws AmbariException { return super.isApplicable(request) && request.getRepositoryVersion() != null; } @@ -77,7 +78,7 @@ public class HostsRepositoryVersionCheck extends AbstractCheckDescriptor { for (HostVersionEntity hve : hostVersionDaoProvider.get().findByHost(host.getHostName())) { if (hve.getRepositoryVersion().getVersion().equals(request.getRepositoryVersion()) - && hve.getState() == RepositoryVersionState.INSTALLED) { + && (hve.getState() == RepositoryVersionState.INSTALLED || hve.getState() == RepositoryVersionState.NOT_REQUIRED)) { found = true; break; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index 35e9b6f799..8700f06e77 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -17,11 +17,20 @@ */ package org.apache.ambari.server.configuration; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.inject.Inject; -import com.google.inject.Singleton; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.security.cert.CertificateException; +import java.security.interfaces.RSAPublicKey; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; import org.apache.ambari.annotations.Experimental; import org.apache.ambari.annotations.ExperimentalFeature; @@ -34,32 +43,23 @@ import org.apache.ambari.server.orm.entities.StageEntity; import org.apache.ambari.server.security.ClientSecurityType; import org.apache.ambari.server.security.authorization.LdapServerProperties; import org.apache.ambari.server.security.authorization.jwt.JwtAuthenticationProperties; +import org.apache.ambari.server.security.encryption.CertificateUtils; import org.apache.ambari.server.security.encryption.CredentialProvider; import org.apache.ambari.server.state.stack.OsFamily; -import org.apache.ambari.server.security.encryption.CertificateUtils; import org.apache.ambari.server.utils.Parallel; import org.apache.ambari.server.utils.ShellCommandUtil; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.math.NumberUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Properties; - -import java.security.cert.CertificateException; -import java.security.interfaces.RSAPublicKey; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.inject.Inject; +import com.google.inject.Singleton; /** @@ -551,6 +551,26 @@ public class Configuration { public static final String VIEWS_HTTP_X_XSS_PROTECTION_HEADER_VALUE_KEY = "views.http.x-xss-protection"; public static final String VIEWS_HTTP_X_XSS_PROTECTION_HEADER_VALUE_DEFAULT = "1; mode=block"; + /* + * Version Definition URL + */ + /** + * The connection timeout for reading version definitions. + */ + private static final String VERSION_DEFINITION_CONNECT_TIMEOUT = "server.version_definition.connect.timeout.millis"; + /** + * Default connect timeout for reading version definitions. + */ + private static final int VERSION_DEFINITION_CONNECT_TIMEOUT_DEFAULT = 5000; + /** + * The read timeout for reading version definitions. + */ + private static final String VERSION_DEFINITION_READ_TIMEOUT = "server.version_definition.read.timeout.millis"; + /** + * Default read timeout for reading version definitions. + */ + private static final int VERSION_DEFINITION_READ_TIMEOUT_DEFAULT = 5000; + private static final Logger LOG = LoggerFactory.getLogger( Configuration.class); @@ -1949,7 +1969,7 @@ public class Configuration { /** * Get property-providers' thread pool core size. - * + * * @return the property-providers' thread pool core size */ public int getPropertyProvidersThreadPoolCoreSize() { @@ -1959,7 +1979,7 @@ public class Configuration { /** * Get property-providers' thread pool max size. - * + * * @return the property-providers' thread pool max size */ public int getPropertyProvidersThreadPoolMaxSize() { @@ -2500,4 +2520,21 @@ public class Configuration { } return attempts; } + + /** + * @return the connect timeout used when loading a version definition URL. + */ + public int getVersionDefinitionConnectTimeout() { + return NumberUtils.toInt( + properties.getProperty(VERSION_DEFINITION_CONNECT_TIMEOUT), + VERSION_DEFINITION_CONNECT_TIMEOUT_DEFAULT); + } + /** + * @return the read timeout used when loading a version definition URL + */ + public int getVersionDefinitionReadTimeout() { + return NumberUtils.toInt( + properties.getProperty(VERSION_DEFINITION_READ_TIMEOUT), + VERSION_DEFINITION_READ_TIMEOUT_DEFAULT); + } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java index 61fbd8ad51..586924baf4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractControllerResourceProvider.java @@ -219,7 +219,8 @@ public abstract class AbstractControllerResourceProvider extends AbstractAuthori return new RoleAuthorizationResourceProvider(managementController); case UserAuthorization: return new UserAuthorizationResourceProvider(managementController); - + case VersionDefinition: + return new VersionDefinitionResourceProvider(); default: throw new IllegalArgumentException("Unknown type " + type); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java index 3a1ac623fc..ce5606e397 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java @@ -57,8 +57,6 @@ import org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException; import org.apache.ambari.server.controller.spi.SystemException; import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; import org.apache.ambari.server.controller.utilities.PropertyHelper; -import org.apache.ambari.server.events.ActionFinalReportReceivedEvent; -import org.apache.ambari.server.events.publishers.AmbariEventPublisher; import org.apache.ambari.server.orm.dao.ClusterVersionDAO; import org.apache.ambari.server.orm.dao.HostComponentStateDAO; import org.apache.ambari.server.orm.dao.HostVersionDAO; @@ -75,11 +73,14 @@ import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.MaintenanceState; +import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.RepositoryVersionState; import org.apache.ambari.server.state.ServiceComponentHost; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.ServiceOsSpecific; import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.repository.AvailableService; +import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.utils.StageUtils; import org.apache.commons.lang.StringUtils; @@ -187,9 +188,6 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou private static Configuration configuration; @Inject - private static AmbariEventPublisher ambariEventPublisher; - - @Inject private static Injector injector; @Inject @@ -252,8 +250,10 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou for (HostVersionEntity hostVersionEntity : hostVersionDAO.findByClusterStackAndVersion( entity.getClusterEntity().getClusterName(), repoVersionStackId, entity.getRepositoryVersion().getVersion())) { + hostStates.get(hostVersionEntity.getState().name()).add(hostVersionEntity.getHostName()); } + StackId stackId = new StackId(entity.getRepositoryVersion().getStack()); RepositoryVersionEntity repoVerEntity = repositoryVersionDAO.findByStackAndVersion( stackId, entity.getRepositoryVersion().getVersion()); @@ -387,6 +387,8 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou stageId = 1L; } + boolean hasStage = false; + ArrayList<Stage> stages = new ArrayList<Stage>(batchCount); for (int batchId = 1; batchId <= batchCount; batchId++) { // Create next stage @@ -414,18 +416,55 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou // add the stage that was just created stages.add(stage); + // determine services for the repo + Set<String> serviceNames = new HashSet<>(); + if (RepositoryType.STANDARD != repoVersionEnt.getType() && null != repoVersionEnt.getVersionXsd()) { + VersionDefinitionXml xml = null; + try { + xml = repoVersionEnt.getRepositoryXml(); + + Collection<AvailableService> available = xml.getAvailableServices( + ami.getStack(stackId.getStackName(), stackId.getStackVersion())); + + // check if the service is part of the cluster + for (AvailableService as : available) { + if (cluster.getServices().containsKey(as.getName())) { + serviceNames.add(as.getName()); + } + } + + } catch (Exception e) { + String msg = String.format("Could not load repo xml for %s", repoVersionEnt.getDisplayName()); + LOG.error(msg, e); + throw new SystemException (msg); + } + } + // Populate with commands for host for (int i = 0; i < maxTasks && hostIterator.hasNext(); i++) { Host host = hostIterator.next(); - if (hostHasVersionableComponents(cluster, ami, stackId, host)) { - addHostVersionInstallCommandsToStage(desiredRepoVersion, - cluster, managementController, ami, stackId, perOsRepos, stage, host); + if (hostHasVersionableComponents(cluster, serviceNames, ami, stackId, host)) { + ActionExecutionContext actionContext = getHostVersionInstallCommand(desiredRepoVersion, + cluster, managementController, ami, stackId, serviceNames, perOsRepos, stage, host); + if (null != actionContext) { + try { + actionExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage); + hasStage = true; + } catch (AmbariException e) { + throw new SystemException("Cannot modify stage", e); + } + } } else { directTransitions.add(host); } } } + if (!hasStage) { + throw new SystemException(String.format("There are no hosts that have components to install repository %s", + desiredRepoVersion)); + } + req.addStages(stages); try { @@ -448,7 +487,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou desiredRepoVersion, clName), e); } } else { - // Move CSV into INSTALLING state (retry installation) + // Move cluster version into INSTALLING state (retry installation) cluster.transitionClusterVersion(stackId, desiredRepoVersion, RepositoryVersionState.INSTALLING); } @@ -456,24 +495,23 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou // Will also initialize all Host Versions in an INSTALLING state. cluster.transitionHostsToInstalling(clusterVersionEntity); - // Directly transition host versions to INSTALLED for hosts that don't have + // Directly transition host versions to NOT_REQUIRED for hosts that don't have // versionable components for(Host host : directTransitions) { - transitionHostVersionToInstalled(host, cluster, - clusterVersionEntity.getRepositoryVersion().getVersion()); + transitionHostVersionToNotRequired(host, cluster, + clusterVersionEntity.getRepositoryVersion()); } req.persist(); - } catch (AmbariException e) { throw new SystemException("Can not persist request", e); } return getRequestStatus(req.getRequestStatusResponse()); } - private void addHostVersionInstallCommandsToStage(final String desiredRepoVersion, + private ActionExecutionContext getHostVersionInstallCommand(final String desiredRepoVersion, Cluster cluster, AmbariManagementController managementController, AmbariMetaInfo ami, - final StackId stackId, Map<String, List<RepositoryEntity>> perOsRepos, Stage stage, Host host) + final StackId stackId, Set<String> repoServices, Map<String, List<RepositoryEntity>> perOsRepos, Stage stage1, Host host) throws SystemException { // Determine repositories for host String osFamily = host.getOsFamily(); @@ -484,12 +522,19 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou osFamily, desiredRepoVersion, stackId)); } + // determine packages for all services that are installed on host List<ServiceOsSpecific.Package> packages = new ArrayList<ServiceOsSpecific.Package>(); Set<String> servicesOnHost = new HashSet<String>(); List<ServiceComponentHost> components = cluster.getServiceComponentHosts(host.getHostName()); for (ServiceComponentHost component : components) { - servicesOnHost.add(component.getServiceName()); + if (repoServices.isEmpty() || repoServices.contains(component.getServiceName())) { + servicesOnHost.add(component.getServiceName()); + } + } + + if (servicesOnHost.isEmpty()) { + return null; } List<String> blacklistedPackagePrefixes = configuration.getRollingUpgradeSkipPackagesPrefixes(); for (String serviceName : servicesOnHost) { @@ -538,11 +583,8 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou params); actionContext.setTimeout(Short.valueOf(configuration.getDefaultAgentTaskTimeout(true))); - try { - actionExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage); - } catch (AmbariException e) { - throw new SystemException("Can not modify stage", e); - } + return actionContext; + } @@ -550,10 +592,16 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou * Returns true if there is at least one versionable component on host for a given * stack. */ - private boolean hostHasVersionableComponents(Cluster cluster, AmbariMetaInfo ami, StackId stackId, + private boolean hostHasVersionableComponents(Cluster cluster, Set<String> serviceNames, AmbariMetaInfo ami, StackId stackId, Host host) throws SystemException { + List<ServiceComponentHost> components = cluster.getServiceComponentHosts(host.getHostName()); + for (ServiceComponentHost component : components) { + if (!serviceNames.isEmpty() && !serviceNames.contains(component.getServiceName())) { + continue; + } + ComponentInfo componentInfo; try { componentInfo = ami.getComponent(stackId.getStackName(), @@ -574,29 +622,24 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou /** - * Sends event for host regarding successful repo version installation - * without actually running any commands on host. - * Transitioning host version to INSTALLED state manually would not be the - * best idea since some additional logic may be bound to event listeners. + * Sets host versions states to not-required. + * + * Transitioning host version to NOT_REQUIRED state manually is ok since + * other completion handlers set success/fail states correctly during heartbeat. + * The number of NOT_REQUIRED components for a cluster will be low. */ - private void transitionHostVersionToInstalled(Host host, Cluster cluster, String version) { - LOG.info(String.format("Transitioning version %s on host %s directly to installed" + + private void transitionHostVersionToNotRequired(Host host, Cluster cluster, RepositoryVersionEntity repoVersion) { + LOG.info(String.format("Transitioning version %s on host %s directly to %s" + " without distributing bits to host since it has no versionable components.", - version, host.getHostName())); - CommandReport report = new CommandReport(); - report.setRole(INSTALL_PACKAGES_ACTION); - report.setStdOut("Skipped distributing bits to host since it has " + - "no versionable components installed"); - report.setStdErr(""); - // We don't set actual repo version in structured output in order - // to avoid confusing server with fake data - report.setStructuredOut("{}"); - report.setExitCode(0); - report.setStatus(HostRoleStatus.COMPLETED.toString()); - ActionFinalReportReceivedEvent event = new ActionFinalReportReceivedEvent( - cluster.getClusterId(), host.getHostName(), - report, true); - ambariEventPublisher.publish(event); + repoVersion.getVersion(), host.getHostName(), RepositoryVersionState.NOT_REQUIRED)); + + for (HostVersionEntity hve : host.getAllHostVersions()) { + if (hve.getRepositoryVersion().equals(repoVersion)) { + hve.setState(RepositoryVersionState.NOT_REQUIRED); + hostVersionDAO.merge(hve); + } + } + } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProvider.java index 568406bfc3..87731c42dd 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProvider.java @@ -61,12 +61,14 @@ import org.apache.ambari.server.state.OperatingSystemInfo; import org.apache.ambari.server.state.RepositoryVersionState; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.StackInfo; +import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.state.stack.UpgradePack; import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper; import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.StringUtils; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.inject.Inject; import com.google.inject.Provider; @@ -78,7 +80,7 @@ import com.google.inject.persist.Transactional; public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourceProvider { // ----- Property ID constants --------------------------------------------- - + public static final String REPOSITORY_VERSION = "RepositoryVersions"; public static final String REPOSITORY_VERSION_ID_PROPERTY_ID = PropertyHelper.getPropertyId("RepositoryVersions", "id"); public static final String REPOSITORY_VERSION_STACK_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("RepositoryVersions", "stack_name"); public static final String REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID = PropertyHelper.getPropertyId("RepositoryVersions", "stack_version"); @@ -87,6 +89,16 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc public static final String SUBRESOURCE_OPERATING_SYSTEMS_PROPERTY_ID = new OperatingSystemResourceDefinition().getPluralName(); public static final String SUBRESOURCE_REPOSITORIES_PROPERTY_ID = new RepositoryResourceDefinition().getPluralName(); + public static final String REPOSITORY_VERSION_TYPE_PROPERTY_ID = "RepositoryVersions/type"; + public static final String REPOSITORY_VERSION_RELEASE_VERSION = "RepositoryVersions/release/version"; + public static final String REPOSITORY_VERSION_RELEASE_BUILD = "RepositoryVersions/release/build"; + public static final String REPOSITORY_VERSION_RELEASE_NOTES = "RepositoryVersions/release/notes"; + public static final String REPOSITORY_VERSION_RELEASE_COMPATIBLE_WITH = "RepositoryVersions/release/compatible_with"; + public static final String REPOSITORY_VERSION_AVAILABLE_SERVICES = "RepositoryVersions/services"; + + public static final String REPOSITORY_VERSION_PARENT_ID = "RepositoryVersions/parent_id"; + public static final String REPOSITORY_VERSION_HAS_CHILDREN = "RepositoryVersions/has_children"; + @SuppressWarnings("serial") private static Set<String> pkPropertyIds = new HashSet<String>() { { @@ -95,16 +107,21 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc }; @SuppressWarnings("serial") - public static Set<String> propertyIds = new HashSet<String>() { - { - add(REPOSITORY_VERSION_ID_PROPERTY_ID); - add(REPOSITORY_VERSION_REPOSITORY_VERSION_PROPERTY_ID); - add(REPOSITORY_VERSION_DISPLAY_NAME_PROPERTY_ID); - add(REPOSITORY_VERSION_STACK_NAME_PROPERTY_ID); - add(REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID); - add(SUBRESOURCE_OPERATING_SYSTEMS_PROPERTY_ID); - } - }; + public static Set<String> propertyIds = Sets.newHashSet( + REPOSITORY_VERSION_ID_PROPERTY_ID, + REPOSITORY_VERSION_REPOSITORY_VERSION_PROPERTY_ID, + REPOSITORY_VERSION_DISPLAY_NAME_PROPERTY_ID, + REPOSITORY_VERSION_STACK_NAME_PROPERTY_ID, + REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID, + SUBRESOURCE_OPERATING_SYSTEMS_PROPERTY_ID, + REPOSITORY_VERSION_TYPE_PROPERTY_ID, + REPOSITORY_VERSION_RELEASE_BUILD, + REPOSITORY_VERSION_RELEASE_COMPATIBLE_WITH, + REPOSITORY_VERSION_RELEASE_NOTES, + REPOSITORY_VERSION_RELEASE_VERSION, + REPOSITORY_VERSION_PARENT_ID, + REPOSITORY_VERSION_HAS_CHILDREN, + REPOSITORY_VERSION_AVAILABLE_SERVICES); @SuppressWarnings("serial") public static Map<Type, String> keyPropertyIds = new HashMap<Type, String>() { @@ -145,6 +162,7 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc */ public RepositoryVersionResourceProvider() { super(propertyIds, keyPropertyIds); + setRequiredCreateAuthorizations(EnumSet.of(RoleAuthorization.AMBARI_MANAGE_STACK_VERSIONS)); setRequiredDeleteAuthorizations(EnumSet.of(RoleAuthorization.AMBARI_MANAGE_STACK_VERSIONS)); setRequiredUpdateAuthorizations(EnumSet.of(RoleAuthorization.AMBARI_MANAGE_STACK_VERSIONS, RoleAuthorization.AMBARI_EDIT_STACK_REPOS)); @@ -175,12 +193,14 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID, REPOSITORY_VERSION_REPOSITORY_VERSION_PROPERTY_ID }; + for (String propertyName : requiredProperties) { if (properties.get(propertyName) == null) { throw new AmbariException("Property " + propertyName + " should be provided"); } } - final RepositoryVersionEntity entity = toRepositoryVersionEntity(properties); + + RepositoryVersionEntity entity = toRepositoryVersionEntity(properties); if (repositoryVersionDAO.findByDisplayName(entity.getDisplayName()) != null) { throw new AmbariException("Repository version with name " + entity.getDisplayName() + " already exists"); @@ -188,7 +208,13 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc if (repositoryVersionDAO.findByStackAndVersion(entity.getStack(), entity.getVersion()) != null) { throw new AmbariException("Repository version for stack " + entity.getStack() + " and version " + entity.getVersion() + " already exists"); } - validateRepositoryVersion(entity); + + if (!upgradePackExists(entity.getVersion())) { + throw new AmbariException("Stack " + entity.getStackId() + " doesn't have upgrade packages"); + } + + validateRepositoryVersion(repositoryVersionDAO, ambariMetaInfo, entity); + repositoryVersionDAO.create(entity); notifyCreate(Resource.Type.RepositoryVersion, request); return null; @@ -206,6 +232,7 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc final Set<String> requestedIds = getRequestPropertyIds(request, predicate); final Set<Map<String, Object>> propertyMaps = getPropertyMaps(predicate); + List<RepositoryVersionEntity> requestedEntities = new ArrayList<RepositoryVersionEntity>(); for (Map<String, Object> propertyMap: propertyMaps) { final StackId stackId = getStackInformationFromUrl(propertyMap); @@ -220,6 +247,7 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc throw new SystemException("Repository version should have numerical id"); } final RepositoryVersionEntity entity = repositoryVersionDAO.findByPK(id); + if (entity == null) { throw new NoSuchResourceException("There is no repository version with id " + id); } else { @@ -236,6 +264,38 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc setResourceProperty(resource, REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID, entity.getStackVersion(), requestedIds); setResourceProperty(resource, REPOSITORY_VERSION_DISPLAY_NAME_PROPERTY_ID, entity.getDisplayName(), requestedIds); setResourceProperty(resource, REPOSITORY_VERSION_REPOSITORY_VERSION_PROPERTY_ID, entity.getVersion(), requestedIds); + setResourceProperty(resource, REPOSITORY_VERSION_TYPE_PROPERTY_ID, entity.getType(), requestedIds); + + setResourceProperty(resource, REPOSITORY_VERSION_PARENT_ID, entity.getParentId(), requestedIds); + + List<RepositoryVersionEntity> children = entity.getChildren(); + setResourceProperty(resource, REPOSITORY_VERSION_HAS_CHILDREN, + null != children && !children.isEmpty(), requestedIds); + + final VersionDefinitionXml xml; + + try { + xml = entity.getRepositoryXml(); + } catch (Exception e) { + throw new SystemException(String.format("Could not load xml for Repository %s", entity.getId()), e); + } + + if (null != xml) { + final StackInfo stack; + + try { + stack = ambariMetaInfo.getStack(entity.getStackName(), entity.getStackVersion()); + } catch (AmbariException e) { + throw new SystemException(String.format("Could not load stack %s for Repository %s", + entity.getStackId().toString(), entity.getId())); + } + + setResourceProperty(resource, REPOSITORY_VERSION_RELEASE_VERSION, xml.release.version, requestedIds); + setResourceProperty(resource, REPOSITORY_VERSION_RELEASE_BUILD, xml.release.build, requestedIds); + setResourceProperty(resource, REPOSITORY_VERSION_RELEASE_COMPATIBLE_WITH, xml.release.compatibleWith, requestedIds); + setResourceProperty(resource, REPOSITORY_VERSION_RELEASE_NOTES, xml.release.releaseNotes, requestedIds); + setResourceProperty(resource, REPOSITORY_VERSION_AVAILABLE_SERVICES, xml.getAvailableServices(stack), requestedIds); + } resources.add(resource); } @@ -285,7 +345,12 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc entity.setDisplayName(propertyMap.get(REPOSITORY_VERSION_DISPLAY_NAME_PROPERTY_ID).toString()); } - validateRepositoryVersion(entity); + if (!upgradePackExists(entity.getVersion())) { + throw new AmbariException("Stack " + entity.getStackId() + " doesn't have upgrade packages"); + } + + validateRepositoryVersion(repositoryVersionDAO, ambariMetaInfo, entity); + repositoryVersionDAO.merge(entity); // @@ -369,20 +434,17 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc * @param repositoryVersion repository version * @throws AmbariException exception with error message */ - protected void validateRepositoryVersion(RepositoryVersionEntity repositoryVersion) throws AmbariException { + protected static void validateRepositoryVersion(RepositoryVersionDAO dao, + AmbariMetaInfo metaInfo, RepositoryVersionEntity repositoryVersion) throws AmbariException { final StackId requiredStack = new StackId(repositoryVersion.getStack()); final String requiredStackName = requiredStack.getStackName(); final String requiredStackVersion = requiredStack.getStackVersion(); final String requiredStackId = requiredStack.getStackId(); - if (!upgradePackExists(repositoryVersion.getVersion())) { - throw new AmbariException("Stack " + requiredStackId + " doesn't have upgrade packages"); - } - // List of all repo urls that are already added at stack Set<String> existingRepoUrls = new HashSet<String>(); - List<RepositoryVersionEntity> existingRepoVersions = repositoryVersionDAO.findByStack(requiredStack); + List<RepositoryVersionEntity> existingRepoVersions = dao.findByStack(requiredStack); for (RepositoryVersionEntity existingRepoVersion : existingRepoVersions) { for (OperatingSystemEntity operatingSystemEntity : existingRepoVersion.getOperatingSystems()) { for (RepositoryEntity repositoryEntity : operatingSystemEntity.getRepositories()) { @@ -396,9 +458,10 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc // check that repositories contain only supported operating systems final Set<String> osSupported = new HashSet<String>(); - for (OperatingSystemInfo osInfo: ambariMetaInfo.getOperatingSystems(requiredStackName, requiredStackVersion)) { + for (OperatingSystemInfo osInfo: metaInfo.getOperatingSystems(requiredStackName, requiredStackVersion)) { osSupported.add(osInfo.getOsType()); } + final Set<String> osRepositoryVersion = new HashSet<String>(); for (OperatingSystemEntity os: repositoryVersion.getOperatingSystems()) { osRepositoryVersion.add(os.getOsType()); @@ -435,7 +498,7 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc Collection<StackInfo> stacks = new ArrayList<StackInfo>(); // Search results only in the installed stacks - for (Cluster cluster:clusters.get().getClusters().values()){ + for (Cluster cluster : clusters.get().getClusters().values()){ stacks.add(ambariMetaInfo.getStack(cluster.getCurrentStackVersion().getStackName(), cluster.getCurrentStackVersion().getStackVersion())); } @@ -464,6 +527,7 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc */ protected RepositoryVersionEntity toRepositoryVersionEntity(Map<String, Object> properties) throws AmbariException { final RepositoryVersionEntity entity = new RepositoryVersionEntity(); + final String stackName = properties.get(REPOSITORY_VERSION_STACK_NAME_PROPERTY_ID).toString(); final String stackVersion = properties.get(REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID).toString(); @@ -481,6 +545,7 @@ public class RepositoryVersionResourceProvider extends AbstractAuthorizedResourc throw new AmbariException("Json structure for operating systems is incorrect", ex); } entity.setOperatingSystems(operatingSystemsJson); + return entity; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index 70440fcdcb..db8c0795e3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -24,6 +24,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; @@ -36,8 +37,6 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.google.inject.assistedinject.Assisted; -import com.google.inject.persist.Transactional; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; @@ -86,12 +85,15 @@ import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.DesiredConfig; +import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.state.UpgradeContext; import org.apache.ambari.server.state.UpgradeHelper; import org.apache.ambari.server.state.UpgradeHelper.UpgradeGroupHolder; +import org.apache.ambari.server.state.repository.AvailableService; +import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.state.stack.ConfigUpgradePack; import org.apache.ambari.server.state.stack.PrereqCheckStatus; import org.apache.ambari.server.state.stack.UpgradePack; @@ -115,6 +117,8 @@ import com.google.common.collect.Lists; import com.google.gson.Gson; import com.google.inject.Inject; import com.google.inject.Provider; +import com.google.inject.assistedinject.Assisted; +import com.google.inject.persist.Transactional; /** * Manages the ability to start and get status of upgrades. @@ -701,12 +705,37 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider StackId sourceStackId = null; StackId targetStackId = null; + Set<String> supportedServices = new HashSet<>(); + switch (direction) { case UPGRADE: sourceStackId = cluster.getCurrentStackVersion(); RepositoryVersionEntity targetRepositoryVersion = s_repoVersionDAO.findByStackNameAndVersion( sourceStackId.getStackName(), version); + + EnumSet<RepositoryType> serviceAware = EnumSet.of(RepositoryType.PATCH, RepositoryType.SERVICE); + if (serviceAware.contains(targetRepositoryVersion.getType())) { + + VersionDefinitionXml xml = null; + StackInfo stackInfo = s_metaProvider.get().getStack(sourceStackId.getStackName(), + sourceStackId.getStackVersion()); + + try { + xml = targetRepositoryVersion.getRepositoryXml(); + } catch (Exception e) { + throw new AmbariException(String.format("Could not load repository definition for version %s", version)); + } + + if (null != xml) { + Collection<AvailableService> services = xml.getAvailableServices(stackInfo); + + for (AvailableService available : services) { + supportedServices.add(available.getName()); + } + } + } + targetStackId = targetRepositoryVersion.getStackId(); break; case DOWNGRADE: @@ -717,6 +746,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider UpgradeContext ctx = new UpgradeContext(resolver, sourceStackId, targetStackId, version, direction, pack.getType()); + ctx.setSupportedServices(supportedServices); + if (direction.isDowngrade()) { if (requestMap.containsKey(UPGRADE_FROM_VERSION)) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java new file mode 100644 index 0000000000..3ab5169e2d --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProvider.java @@ -0,0 +1,475 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.controller.internal; + +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.StaticallyInject; +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.configuration.ComponentSSLConfiguration; +import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; +import org.apache.ambari.server.controller.spi.NoSuchResourceException; +import org.apache.ambari.server.controller.spi.Predicate; +import org.apache.ambari.server.controller.spi.Request; +import org.apache.ambari.server.controller.spi.RequestStatus; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException; +import org.apache.ambari.server.controller.spi.SystemException; +import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; +import org.apache.ambari.server.controller.utilities.PropertyHelper; +import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; +import org.apache.ambari.server.orm.dao.StackDAO; +import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; +import org.apache.ambari.server.orm.entities.StackEntity; +import org.apache.ambari.server.security.authorization.ResourceType; +import org.apache.ambari.server.security.authorization.RoleAuthorization; +import org.apache.ambari.server.state.RepositoryType; +import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.StackInfo; +import org.apache.ambari.server.state.repository.VersionDefinitionXml; +import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; + +import com.google.common.collect.Sets; +import com.google.inject.Inject; +import com.google.inject.Provider; + +/** + * The {@link VersionDefinitionResourceProvider} class deals with managing Version Definition + * files. + */ +@StaticallyInject +public class VersionDefinitionResourceProvider extends AbstractAuthorizedResourceProvider { + + public static final String VERSION_DEF = "VersionDefinition"; + public static final String VERSION_DEF_BASE64_PROPERTY = "version_base64"; + public static final String VERSION_DEF_STACK_NAME = "VersionDefinition/stack_name"; + public static final String VERSION_DEF_STACK_VERSION = "VersionDefinition/stack_version"; + + protected static final String VERSION_DEF_ID = "VersionDefinition/id"; + protected static final String VERSION_DEF_TYPE_PROPERTY_ID = "VersionDefinition/type"; + protected static final String VERSION_DEF_DEFINITION_URL = "VersionDefinition/version_url"; + + protected static final String VERSION_DEF_DEFINITION_BASE64 = PropertyHelper.getPropertyId(VERSION_DEF, VERSION_DEF_BASE64_PROPERTY); + protected static final String VERSION_DEF_FULL_VERSION = "VersionDefinition/repository_version"; + protected static final String VERSION_DEF_RELEASE_VERSION = "VersionDefinition/release/version"; + protected static final String VERSION_DEF_RELEASE_BUILD = "VersionDefinition/release/build"; + protected static final String VERSION_DEF_RELEASE_NOTES = "VersionDefinition/release/notes"; + protected static final String VERSION_DEF_RELEASE_COMPATIBLE_WITH = "VersionDefinition/release/compatible_with"; + protected static final String VERSION_DEF_AVAILABLE_SERVICES = "VersionDefinition/services"; + + @Inject + private static RepositoryVersionDAO s_repoVersionDAO; + + @Inject + private static Provider<AmbariMetaInfo> s_metaInfo; + + @Inject + private static Provider<RepositoryVersionHelper> s_repoVersionHelper; + + @Inject + private static StackDAO s_stackDAO; + + @Inject + private static Configuration s_configuration; + + /** + * Key property ids + */ + private static final Set<String> PK_PROPERTY_IDS = Sets.newHashSet( + VERSION_DEF_ID, + VERSION_DEF_STACK_NAME, + VERSION_DEF_STACK_VERSION, + VERSION_DEF_FULL_VERSION); + + /** + * The property ids for an version definition resource. + */ + private static final Set<String> PROPERTY_IDS = Sets.newHashSet( + VERSION_DEF_ID, + VERSION_DEF_TYPE_PROPERTY_ID, + VERSION_DEF_DEFINITION_URL, + VERSION_DEF_DEFINITION_BASE64, + VERSION_DEF_FULL_VERSION, + VERSION_DEF_RELEASE_NOTES, + VERSION_DEF_RELEASE_COMPATIBLE_WITH, + VERSION_DEF_RELEASE_VERSION, + VERSION_DEF_RELEASE_BUILD, + VERSION_DEF_AVAILABLE_SERVICES); + + /** + * The key property ids for an version definition resource. + */ + private static final Map<Resource.Type, String> KEY_PROPERTY_IDS = new HashMap<Resource.Type, String>(); + + static { + KEY_PROPERTY_IDS.put(Resource.Type.VersionDefinition, VERSION_DEF_ID); + } + + /** + * Constructor. + */ + VersionDefinitionResourceProvider() { + super(PROPERTY_IDS, KEY_PROPERTY_IDS); + + setRequiredCreateAuthorizations(EnumSet.of(RoleAuthorization.AMBARI_MANAGE_STACK_VERSIONS)); + + setRequiredGetAuthorizations(EnumSet.of( + RoleAuthorization.AMBARI_MANAGE_STACK_VERSIONS)); + } + + @Override + protected RequestStatus createResourcesAuthorized(final Request request) + throws SystemException, + UnsupportedPropertyException, ResourceAlreadyExistsException, + NoSuchParentResourceException { + + Set<Map<String, Object>> requestProperties = request.getProperties(); + + if (requestProperties.size() > 1) { + throw new IllegalArgumentException("Cannot process more than one file per request"); + } + + final Map<String, Object> properties = requestProperties.iterator().next(); + + if (!properties.containsKey(VERSION_DEF_DEFINITION_URL) && !properties.containsKey(VERSION_DEF_DEFINITION_BASE64)) { + throw new IllegalArgumentException(String.format("%s is required or upload the file directly", VERSION_DEF_DEFINITION_URL)); + } + + if (properties.containsKey(VERSION_DEF_DEFINITION_URL) && properties.containsKey(VERSION_DEF_DEFINITION_BASE64)) { + throw new IllegalArgumentException(String.format("Specify ONLY the url with %s or upload the file directly", + VERSION_DEF_DEFINITION_URL)); + } + + RepositoryVersionEntity entity = createResources(new Command<RepositoryVersionEntity>() { + @Override + public RepositoryVersionEntity invoke() throws AmbariException { + + String definitionUrl = (String) properties.get(VERSION_DEF_DEFINITION_URL); + String definitionBase64 = (String) properties.get(VERSION_DEF_DEFINITION_BASE64); + + XmlHolder holder = null; + if (null != definitionUrl) { + holder = loadXml(definitionUrl); + } else { + holder = loadXml(Base64.decodeBase64(definitionBase64)); + } + + RepositoryVersionEntity entity = toRepositoryVersionEntity(holder); + + RepositoryVersionResourceProvider.validateRepositoryVersion(s_repoVersionDAO, + s_metaInfo.get(), entity); + + checkForParent(holder, entity); + + s_repoVersionDAO.create(entity); + + return entity; + } + }); + + notifyCreate(Resource.Type.VersionDefinition, request); + + RequestStatusImpl status = new RequestStatusImpl(null, + Collections.singleton(toResource(entity, Collections.<String>emptySet()))); + + return status; + } + + @Override + public Set<Resource> getResources(Request request, Predicate predicate) + throws SystemException, UnsupportedPropertyException, + NoSuchResourceException, NoSuchParentResourceException { + + Set<Resource> results = new HashSet<Resource>(); + Set<String> requestPropertyIds = getRequestPropertyIds(request, predicate); + + if (null == predicate){ + List<RepositoryVersionEntity> versions = s_repoVersionDAO.findAllDefinitions(); + + for (RepositoryVersionEntity entity : versions) { + results.add(toResource(entity, requestPropertyIds)); + } + + } else { + for (Map<String, Object> propertyMap : getPropertyMaps(predicate)) { + String id = (String) propertyMap.get(VERSION_DEF_ID); + if (null == id) { + continue; + } + + RepositoryVersionEntity entity = s_repoVersionDAO.findByPK(Long.parseLong(id)); + if (null != entity) { + results.add(toResource(entity, requestPropertyIds)); + } + } + } + return results; + } + + @Override + protected RequestStatus updateResourcesAuthorized(final Request request, Predicate predicate) + throws SystemException, UnsupportedPropertyException, + NoSuchResourceException, NoSuchParentResourceException { + + throw new SystemException("Cannot update Version Definitions"); + } + + @Override + protected RequestStatus deleteResourcesAuthorized(Predicate predicate) + throws SystemException, UnsupportedPropertyException, + NoSuchResourceException, NoSuchParentResourceException { + throw new SystemException("Cannot delete Version Definitions"); + } + + /** + * In the case of a patch, check if there is a parent repo. + * @param entity the entity to check + */ + private void checkForParent(XmlHolder holder, RepositoryVersionEntity entity) throws AmbariException { + if (entity.getType() != RepositoryType.PATCH) { + return; + } + + List<RepositoryVersionEntity> entities = s_repoVersionDAO.findByStack(entity.getStackId()); + if (entities.isEmpty()) { + throw new IllegalArgumentException(String.format("Patch %s was uploaded, but there are no repositories for %s", + entity.getVersion(), entity.getStackId().toString())); + } + + List<RepositoryVersionEntity> matching = new ArrayList<>(); + + boolean emptyCompatible = StringUtils.isBlank(holder.xml.release.compatibleWith); + + for (RepositoryVersionEntity candidate : entities) { + String baseVersion = candidate.getVersion(); + if (baseVersion.lastIndexOf('-') > -1) { + baseVersion = baseVersion.substring(0, baseVersion.lastIndexOf('-')); + } + + if (emptyCompatible) { + if (baseVersion.equals(holder.xml.release.version)) { + matching.add(candidate); + } + } else { + if (baseVersion.matches(holder.xml.release.compatibleWith)) { + matching.add(candidate); + } + } + } + + if (matching.isEmpty()) { + String format = "No versions matched pattern %s"; + + throw new IllegalArgumentException(String.format(format, + emptyCompatible ? holder.xml.release.version : holder.xml.release.compatibleWith)); + } else if (matching.size() > 1) { + Set<String> versions= new HashSet<>(); + for (RepositoryVersionEntity match : matching) { + versions.add(match.getVersion()); + } + + throw new IllegalArgumentException(String.format("More than one repository matches patch %s: %s", + entity.getVersion(), StringUtils.join(versions, ", "))); + } + + RepositoryVersionEntity parent = matching.get(0); + + entity.setParent(parent); + } + + @Override + protected Set<String> getPKPropertyIds() { + return PK_PROPERTY_IDS; + } + + @Override + protected ResourceType getResourceType(Request request, Predicate predicate) { + return ResourceType.AMBARI; + } + + /** + * Load the xml data from a posted Base64 stream + * @param decoded the decoded Base64 data + * @return the XmlHolder instance + * @throws AmbariException + */ + private XmlHolder loadXml(byte[] decoded) { + XmlHolder holder = new XmlHolder(); + + try { + holder.xmlString = new String(decoded, "UTF-8"); + } catch (UnsupportedEncodingException e) { + holder.xmlString = new String(decoded); + } + + try { + holder.xml = VersionDefinitionXml.load(holder.xmlString); + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + + return holder; + } + + /** + * Load the xml data from a url + * @param definitionUrl + * @return the XmlHolder instance + * @throws AmbariException + */ + private XmlHolder loadXml(String definitionUrl) throws AmbariException { + XmlHolder holder = new XmlHolder(); + holder.url = definitionUrl; + + int connectTimeout = s_configuration.getVersionDefinitionConnectTimeout(); + int readTimeout = s_configuration.getVersionDefinitionReadTimeout(); + + try { + URI uri = new URI(definitionUrl); + InputStream stream = null; + + if (uri.getScheme().equalsIgnoreCase("file")) { + stream = uri.toURL().openStream(); + } else { + URLStreamProvider provider = new URLStreamProvider(connectTimeout, readTimeout, + ComponentSSLConfiguration.instance()); + + stream = provider.readFrom(definitionUrl); + } + + holder.xmlString = IOUtils.toString(stream, "UTF-8"); + holder.xml = VersionDefinitionXml.load(holder.xmlString); + } catch (Exception e) { + String err = String.format("Could not load url from %s. %s", + definitionUrl, e.getMessage()); + throw new AmbariException(err, e); + } + + return holder; + } + + /** + * Transforms a XML version defintion to an entity + * + * @param definitionUrl the String URL for loading + * @return constructed entity + * @throws AmbariException if some properties are missing or json has incorrect structure + */ + protected RepositoryVersionEntity toRepositoryVersionEntity(XmlHolder holder) throws AmbariException { + + // !!! TODO validate parsed object graph + + RepositoryVersionEntity entity = new RepositoryVersionEntity(); + + StackId stackId = new StackId(holder.xml.release.stackId); + + StackEntity stackEntity = s_stackDAO.find(stackId.getStackName(), stackId.getStackVersion()); + + entity.setStack(stackEntity); + entity.setOperatingSystems(s_repoVersionHelper.get().serializeOperatingSystems( + holder.xml.repositoryInfo.getRepositories())); + entity.setVersion(holder.xml.release.getFullVersion()); + entity.setDisplayName(stackId, holder.xml.release); + entity.setType(holder.xml.release.repositoryType); + entity.setVersionUrl(holder.url); + entity.setVersionXml(holder.xmlString); + entity.setVersionXsd(holder.xml.xsdLocation); + + return entity; + } + + /** + * Convert the given {@link RepositoryVersionEntity} to a {@link Resource}. + * + * @param entity + * the entity to convert. + * @param requestedIds + * the properties that were requested or {@code null} for all. + * @return the resource representation of the entity (never {@code null}). + */ + private Resource toResource(RepositoryVersionEntity entity, Set<String> requestedIds) + throws SystemException { + + Resource resource = new ResourceImpl(Resource.Type.VersionDefinition); + + resource.setProperty(VERSION_DEF_ID, entity.getId()); + + VersionDefinitionXml xml = null; + try { + xml = entity.getRepositoryXml(); + } catch (Exception e) { + String msg = String.format("Could not load version definition %s", entity.getId()); + throw new SystemException(msg, e); + } + + StackId stackId = new StackId(xml.release.stackId); + + // !!! these are needed for href + resource.setProperty(VERSION_DEF_STACK_NAME, stackId.getStackName()); + resource.setProperty(VERSION_DEF_STACK_VERSION, stackId.getStackVersion()); + + setResourceProperty(resource, VERSION_DEF_TYPE_PROPERTY_ID, entity.getType(), requestedIds); + setResourceProperty(resource, VERSION_DEF_DEFINITION_URL, entity.getVersionUrl(), requestedIds); + setResourceProperty(resource, VERSION_DEF_FULL_VERSION, entity.getVersion(), requestedIds); + setResourceProperty(resource, VERSION_DEF_RELEASE_BUILD, xml.release.build, requestedIds); + setResourceProperty(resource, VERSION_DEF_RELEASE_COMPATIBLE_WITH, xml.release.compatibleWith, requestedIds); + setResourceProperty(resource, VERSION_DEF_RELEASE_NOTES, xml.release.releaseNotes, requestedIds); + setResourceProperty(resource, VERSION_DEF_RELEASE_VERSION, xml.release.version, requestedIds); + + // !!! future do something with the manifest + + if (isPropertyRequested(VERSION_DEF_AVAILABLE_SERVICES, requestedIds)) { + StackInfo stack = null; + try { + stack = s_metaInfo.get().getStack(stackId.getStackName(), stackId.getStackVersion()); + } catch (AmbariException e) { + throw new SystemException(String.format("Could not load stack %s", stackId)); + } + + setResourceProperty(resource, VERSION_DEF_AVAILABLE_SERVICES, xml.getAvailableServices(stack), requestedIds); + } + + return resource; + } + + /** + * Convenience class to hold the xml String representation, the url, and the parsed object. + */ + private static class XmlHolder { + String url = null; + String xmlString = null; + VersionDefinitionXml xml = null; + } + + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/Resource.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/Resource.java index f5fa5f183b..3eb8c2d3b6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/Resource.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/spi/Resource.java @@ -150,7 +150,8 @@ public interface Resource { Credential, KerberosDescriptor, RoleAuthorization, - UserAuthorization; + UserAuthorization, + VersionDefinition; /** * Get the {@link Type} that corresponds to this InternalType. @@ -262,6 +263,7 @@ public interface Resource { public static final Type KerberosDescriptor = InternalType.KerberosDescriptor.getType(); public static final Type RoleAuthorization = InternalType.RoleAuthorization.getType(); public static final Type UserAuthorization = InternalType.UserAuthorization.getType(); + public static final Type VersionDefinition = InternalType.VersionDefinition.getType(); /** * The type name. diff --git a/ambari-server/src/main/java/org/apache/ambari/server/events/ActionFinalReportReceivedEvent.java b/ambari-server/src/main/java/org/apache/ambari/server/events/ActionFinalReportReceivedEvent.java index de797f33e6..e96fb8902a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/events/ActionFinalReportReceivedEvent.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/events/ActionFinalReportReceivedEvent.java @@ -17,7 +17,6 @@ */ package org.apache.ambari.server.events; -import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.agent.CommandReport; /** diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/RepositoryVersionDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/RepositoryVersionDAO.java index e1f71104b3..26f96e8235 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/RepositoryVersionDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/RepositoryVersionDAO.java @@ -18,21 +18,19 @@ package org.apache.ambari.server.orm.dao; import java.text.MessageFormat; -import java.util.Collections; -import java.util.Comparator; import java.util.List; import javax.persistence.TypedQuery; -import com.google.inject.persist.Transactional; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.orm.RequiresSession; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.orm.entities.StackEntity; +import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.StackId; -import org.apache.ambari.server.utils.VersionUtils; import com.google.inject.Singleton; +import com.google.inject.persist.Transactional; /** * DAO for repository versions. @@ -145,9 +143,28 @@ public class RepositoryVersionDAO extends CrudDAO<RepositoryVersionEntity, Long> * @return Returns the object created if successful, and throws an exception otherwise. * @throws AmbariException */ + public RepositoryVersionEntity create(StackEntity stackEntity, + String version, String displayName, + String operatingSystems) throws AmbariException { + return create(stackEntity, version, displayName, operatingSystems, + RepositoryType.STANDARD); + } + + /** + * Validates and creates an object. + * The version must be unique within this stack name (e.g., HDP, HDPWIN, BIGTOP). + * @param stackEntity Stack entity. + * @param version Stack version, e.g., 2.2 or 2.2.0.1-885 + * @param displayName Unique display name + * @param operatingSystems JSON structure of repository URLs for each OS + * @param type the repository type + * @return Returns the object created if successful, and throws an exception otherwise. + * @throws AmbariException + */ @Transactional public RepositoryVersionEntity create(StackEntity stackEntity, - String version, String displayName, String operatingSystems) throws AmbariException { + String version, String displayName, + String operatingSystems, RepositoryType type) throws AmbariException { if (stackEntity == null || version == null || version.isEmpty() || displayName == null || displayName.isEmpty()) { @@ -175,7 +192,20 @@ public class RepositoryVersionDAO extends CrudDAO<RepositoryVersionEntity, Long> RepositoryVersionEntity newEntity = new RepositoryVersionEntity( stackEntity, version, displayName, operatingSystems); + newEntity.setType(type); this.create(newEntity); return newEntity; } + + /** + * Retrieves repository version when they are loaded by a version definition file + * + * @return a list of entities, or an empty list when there are none + */ + @RequiresSession + public List<RepositoryVersionEntity> findAllDefinitions() { + final TypedQuery<RepositoryVersionEntity> query = entityManagerProvider.get().createNamedQuery( + "repositoryVersionsFromDefinition", RepositoryVersionEntity.class); + return daoUtils.selectList(query); + } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceComponentDesiredStateDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceComponentDesiredStateDAO.java index 341d1fda85..b8c2fcc797 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceComponentDesiredStateDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ServiceComponentDesiredStateDAO.java @@ -26,7 +26,6 @@ import javax.persistence.TypedQuery; import org.apache.ambari.server.orm.RequiresSession; import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity; -import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntityPK; import com.google.inject.Inject; import com.google.inject.Provider; @@ -38,9 +37,22 @@ public class ServiceComponentDesiredStateDAO { @Inject Provider<EntityManager> entityManagerProvider; + /** + * DAO utilities for dealing mostly with {@link TypedQuery} results. + */ + @Inject + private DaoUtils daoUtils; + + /** + * Gets a {@link ServiceComponentDesiredStateEntity} by its PK ID. + * + * @param id + * the ID. + * @return the entity or {@code null} if it does not exist. + */ @RequiresSession - public ServiceComponentDesiredStateEntity findByPK(ServiceComponentDesiredStateEntityPK primaryKey) { - return entityManagerProvider.get().find(ServiceComponentDesiredStateEntity.class, primaryKey); + public ServiceComponentDesiredStateEntity findById(long id) { + return entityManagerProvider.get().find(ServiceComponentDesiredStateEntity.class, id); } @RequiresSession @@ -55,6 +67,37 @@ public class ServiceComponentDesiredStateDAO { return null; } + /** + * Finds a {@link ServiceComponentDesiredStateEntity} by a combination of + * cluster, service, and component. + * + * @param clusterId + * the cluster ID + * @param serviceName + * the service name (not {@code null}) + * @param componentName + * the component name (not {@code null}) + */ + @RequiresSession + public ServiceComponentDesiredStateEntity findByName(long clusterId, String serviceName, + String componentName) { + EntityManager entityManager = entityManagerProvider.get(); + TypedQuery<ServiceComponentDesiredStateEntity> query = entityManager.createNamedQuery( + "ServiceComponentDesiredStateEntity.findByName", ServiceComponentDesiredStateEntity.class); + + query.setParameter("clusterId", clusterId); + query.setParameter("serviceName", serviceName); + query.setParameter("componentName", componentName); + + ServiceComponentDesiredStateEntity entity = null; + List<ServiceComponentDesiredStateEntity> entities = daoUtils.selectList(query); + if (null != entities && !entities.isEmpty()) { + entity = entities.get(0); + } + + return entity; + } + @Transactional public void refresh(ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity) { entityManagerProvider.get().refresh(serviceComponentDesiredStateEntity); @@ -76,8 +119,10 @@ public class ServiceComponentDesiredStateDAO { } @Transactional - public void removeByPK(ServiceComponentDesiredStateEntityPK primaryKey) { - ServiceComponentDesiredStateEntity entity = findByPK(primaryKey); - entityManagerProvider.get().remove(entity); + public void removeByName(long clusterId, String serviceName, String componentName) { + ServiceComponentDesiredStateEntity entity = findByName(clusterId, serviceName, componentName); + if (null != entity) { + entityManagerProvider.get().remove(entity); + } } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java index b57a467089..fd15200aa8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentDesiredStateEntity.java @@ -20,7 +20,6 @@ package org.apache.ambari.server.orm.entities; import static org.apache.commons.lang.StringUtils.defaultString; import javax.persistence.Basic; -import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.EnumType; @@ -95,6 +94,7 @@ public class HostComponentDesiredStateEntity { @Column(name = "admin_state", nullable = true, insertable = true, updatable = true) private HostComponentAdminState adminState; + @ManyToOne @JoinColumns({ @JoinColumn(name = "cluster_id", referencedColumnName = "cluster_id", nullable = false), @JoinColumn(name = "service_name", referencedColumnName = "service_name", nullable = false), diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java index f1af9b04ab..f92f645a1c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostComponentStateEntity.java @@ -18,7 +18,6 @@ package org.apache.ambari.server.orm.entities; -import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.EnumType; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java index b1e54de80c..3398709b60 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java @@ -21,14 +21,19 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import javax.persistence.Basic; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.FetchType; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.JoinColumn; import javax.persistence.Lob; +import javax.persistence.ManyToOne; import javax.persistence.NamedQueries; import javax.persistence.NamedQuery; import javax.persistence.OneToMany; @@ -38,7 +43,10 @@ import javax.persistence.TableGenerator; import javax.persistence.UniqueConstraint; import org.apache.ambari.server.StaticallyInject; +import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.repository.Release; +import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -62,7 +70,8 @@ import com.google.inject.Provider; @NamedQueries({ @NamedQuery(name = "repositoryVersionByDisplayName", query = "SELECT repoversion FROM RepositoryVersionEntity repoversion WHERE repoversion.displayName=:displayname"), @NamedQuery(name = "repositoryVersionByStack", query = "SELECT repoversion FROM RepositoryVersionEntity repoversion WHERE repoversion.stack.stackName=:stackName AND repoversion.stack.stackVersion=:stackVersion"), - @NamedQuery(name = "repositoryVersionByStackNameAndVersion", query = "SELECT repoversion FROM RepositoryVersionEntity repoversion WHERE repoversion.stack.stackName=:stackName AND repoversion.version=:version") + @NamedQuery(name = "repositoryVersionByStackNameAndVersion", query = "SELECT repoversion FROM RepositoryVersionEntity repoversion WHERE repoversion.stack.stackName=:stackName AND repoversion.version=:version"), + @NamedQuery(name = "repositoryVersionsFromDefinition", query = "SELECT repoversion FROM RepositoryVersionEntity repoversion WHERE repoversion.versionXsd IS NOT NULL") }) @StaticallyInject public class RepositoryVersionEntity { @@ -94,12 +103,35 @@ public class RepositoryVersionEntity { @Column(name = "repositories") private String operatingSystems; + @OneToMany(cascade = CascadeType.REMOVE, mappedBy = "repositoryVersion") private Set<ClusterVersionEntity> clusterVersionEntities; @OneToMany(cascade = CascadeType.REMOVE, mappedBy = "repositoryVersion") private Set<HostVersionEntity> hostVersionEntities; + @Column(name = "repo_type", nullable = false, insertable = true, updatable = true) + @Enumerated(value = EnumType.STRING) + private RepositoryType type = RepositoryType.STANDARD; + + @Basic(fetch=FetchType.LAZY) + @Lob + @Column(name="version_xml", insertable = true, updatable = true) + private String versionXml; + + @Column(name="version_url", nullable=true, insertable=true, updatable=true) + private String versionUrl; + + @Column(name="version_xsd", insertable = true, updatable = true) + private String versionXsd; + + @ManyToOne + @JoinColumn(name = "parent_id") + private RepositoryVersionEntity parent; + + @OneToMany(mappedBy = "parent") + private List<RepositoryVersionEntity> children; + // ----- RepositoryVersionEntity ------------------------------------------------------- public RepositoryVersionEntity() { @@ -173,6 +205,19 @@ public class RepositoryVersionEntity { this.displayName = displayName; } + /** + * @param stackId the stack id for the version + * @param release the XML release instance + */ + public void setDisplayName(StackId stackId, Release release) { + if (StringUtils.isNotBlank(release.display)) { + displayName = release.display; + } else { + displayName = stackId.getStackName() + "-" + release.getFullVersion(); + } + } + + public String getOperatingSystemsJson() { return operatingSystems; } @@ -214,6 +259,20 @@ public class RepositoryVersionEntity { return new StackId(stack.getStackName(), stack.getStackVersion()); } + /** + * @return the type + */ + public RepositoryType getType() { + return type; + } + + /** + * @param type the repo type + */ + public void setType(RepositoryType type) { + this.type = type; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -237,6 +296,7 @@ public class RepositoryVersionEntity { if (displayName != null ? !displayName.equals(that.displayName) : that.displayName != null) { return false; } + if (operatingSystems != null ? !operatingSystems.equals(that.operatingSystems) : that.operatingSystems != null) { return false; } @@ -244,6 +304,62 @@ public class RepositoryVersionEntity { return true; } + /** + * @return the XML that is the basis for the version + */ + public String getVersionXml() { + return versionXml; + } + + /** + * @param xml the XML that is the basis for the version + */ + public void setVersionXml(String xml) { + versionXml = xml; + } + + /** + * @return The url used for the version. Optional in case the XML was loaded via blob. + */ + public String getVersionUrl() { + return versionUrl; + } + + /** + * @param url the url used to load the XML. + */ + public void setVersionUrl(String url) { + versionUrl = url; + } + + /** + * @return the XSD name extracted from the XML. + */ + public String getVersionXsd() { + return versionXml; + } + + /** + * @param xsdLocation the XSD name extracted from XML. + */ + public void setVersionXsd(String xsdLocation) { + versionXsd = xsdLocation; + } + + /** + * Parse the version XML into its object representation. This causes the XML to be lazy-loaded + * from storage. + * @return {@code null} if the XSD (from the XML) is not available. + * @throws Exception + */ + public VersionDefinitionXml getRepositoryXml() throws Exception { + if (null == versionXsd) { + return null; + } + + return VersionDefinitionXml.load(getVersionXml()); + } + @Override public int hashCode() { int result = id != null ? id.hashCode() : 0; @@ -280,4 +396,27 @@ public class RepositoryVersionEntity { } return false; } + + /** + * @param parent + */ + public void setParent(RepositoryVersionEntity entity) { + parent = entity; + parent.children.add(this); + } + + /** + * @return the repositories that are denoted children + */ + public List<RepositoryVersionEntity> getChildren() { + return children; + } + + /** + * @return the parentId, or {@code null} if the entity is already a parent + */ + public Long getParentId() { + return null == parent ? null : parent.getId(); + } + } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceComponentDesiredStateEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceComponentDesiredStateEntity.java index bda2543322..d2d1b42cb7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceComponentDesiredStateEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceComponentDesiredStateEntity.java @@ -20,35 +20,59 @@ package org.apache.ambari.server.orm.entities; import java.util.Collection; -import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.EnumType; import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.JoinColumn; import javax.persistence.JoinColumns; import javax.persistence.ManyToOne; +import javax.persistence.NamedQueries; +import javax.persistence.NamedQuery; import javax.persistence.OneToMany; import javax.persistence.OneToOne; +import javax.persistence.Table; +import javax.persistence.TableGenerator; +import javax.persistence.UniqueConstraint; import org.apache.ambari.server.state.State; -@javax.persistence.IdClass(ServiceComponentDesiredStateEntityPK.class) -@javax.persistence.Table(name = "servicecomponentdesiredstate") @Entity +@Table( + name = "servicecomponentdesiredstate", + uniqueConstraints = @UniqueConstraint( + name = "unq_scdesiredstate_name", + columnNames = { "component_name", "service_name", "cluster_id" }) ) +@TableGenerator( + name = "servicecomponentdesiredstate_id_generator", + table = "ambari_sequences", + pkColumnName = "sequence_name", + valueColumnName = "sequence_value", + pkColumnValue = "servicecomponentdesiredstate_id_seq", + initialValue = 0) +@NamedQueries({ + @NamedQuery( + name = "ServiceComponentDesiredStateEntity.findByName", + query = "SELECT scds FROM ServiceComponentDesiredStateEntity scds WHERE scds.clusterId = :clusterId AND scds.serviceName = :serviceName AND scds.componentName = :componentName") }) public class ServiceComponentDesiredStateEntity { - @Column(name = "cluster_id", nullable = false, insertable = false, updatable = false, length = 10) @Id + @Column(name = "id", nullable = false, insertable = true, updatable = false) + @GeneratedValue( + strategy = GenerationType.TABLE, + generator = "servicecomponentdesiredstate_id_generator") + private Long id; + + @Column(name = "cluster_id", nullable = false, insertable = false, updatable = false, length = 10) private Long clusterId; @Column(name = "service_name", nullable = false, insertable = false, updatable = false) - @Id private String serviceName; @Column(name = "component_name", nullable = false, insertable = true, updatable = true) - @Id private String componentName; @Column(name = "desired_state", nullable = false, insertable = true, updatable = true) @@ -72,6 +96,10 @@ public class ServiceComponentDesiredStateEntity { @OneToMany(mappedBy = "serviceComponentDesiredStateEntity") private Collection<HostComponentDesiredStateEntity> hostComponentDesiredStateEntities; + public Long getId() { + return id; + } + public Long getClusterId() { return clusterId; } @@ -117,12 +145,16 @@ public class ServiceComponentDesiredStateEntity { if (this == o) { return true; } + if (o == null || getClass() != o.getClass()) { return false; } ServiceComponentDesiredStateEntity that = (ServiceComponentDesiredStateEntity) o; + if (id != null ? !id.equals(that.id) : that.id != null) { + return false; + } if (clusterId != null ? !clusterId.equals(that.clusterId) : that.clusterId != null) { return false; } @@ -144,7 +176,8 @@ public class ServiceComponentDesiredStateEntity { @Override public int hashCode() { - int result = clusterId != null ? clusterId.intValue() : 0; + int result = id != null ? id.hashCode() : 0; + result = 31 * result + (clusterId != null ? clusterId.hashCode() : 0); result = 31 * result + (serviceName != null ? serviceName.hashCode() : 0); result = 31 * result + (componentName != null ? componentName.hashCode() : 0); result = 31 * result + (desiredState != null ? desiredState.hashCode() : 0); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceComponentDesiredStateEntityPK.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceComponentDesiredStateEntityPK.java deleted file mode 100644 index d56e555f8e..0000000000 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceComponentDesiredStateEntityPK.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.server.orm.entities; - -import javax.persistence.Column; -import javax.persistence.Id; -import java.io.Serializable; - -@SuppressWarnings("serial") -public class ServiceComponentDesiredStateEntityPK implements Serializable { - private Long clusterId; - - @Column(name = "cluster_id", nullable = false, insertable = true, updatable = true, length = 10) - @Id - public Long getClusterId() { - return clusterId; - } - - public void setClusterId(Long clusterId) { - this.clusterId = clusterId; - } - - private String serviceName; - - @javax.persistence.Column(name = "service_name", nullable = false, insertable = false, updatable = false) - @Id - public String getServiceName() { - return serviceName; - } - - public void setServiceName(String serviceName) { - this.serviceName = serviceName; - } - - private String componentName; - - @Id - @Column(name = "component_name", nullable = false, insertable = true, updatable = true) - public String getComponentName() { - return componentName; - } - - public void setComponentName(String componentName) { - this.componentName = componentName; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ServiceComponentDesiredStateEntityPK that = (ServiceComponentDesiredStateEntityPK) o; - - if (clusterId != null ? !clusterId.equals(that.clusterId) : that.clusterId != null) return false; - if (componentName != null ? !componentName.equals(that.componentName) : that.componentName != null) return false; - if (serviceName != null ? !serviceName.equals(that.serviceName) : that.serviceName != null) return false; - - return true; - } - - @Override - public int hashCode() { - int result = clusterId != null ? clusterId.intValue() : 0; - result = 31 * result + (serviceName != null ? serviceName.hashCode() : 0); - result = 31 * result + (componentName != null ? componentName.hashCode() : 0); - return result; - } -} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java index def33f0a68..7d934bbfe3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java @@ -110,8 +110,8 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V /** * validity flag */ - protected boolean valid = true; - + protected boolean valid = true; + /** * Logger */ @@ -342,7 +342,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V setValid(false); stackInfo.setValid(false); setErrors(baseService.getErrors()); - stackInfo.setErrors(baseService.getErrors()); + stackInfo.setErrors(baseService.getErrors()); } } } @@ -439,7 +439,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V setValid(false); setErrors(stackInfo.getErrors()); } - + //todo: shouldn't blindly catch Exception, re-evaluate this. } catch (Exception e) { String error = "Exception caught while populating services for stack: " + @@ -475,7 +475,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V stackInfo.setValid(metaInfoXml.isValid()); setValid(metaInfoXml.isValid()); stackInfo.setErrors(metaInfoXml.getErrors()); - setErrors(metaInfoXml.getErrors()); + setErrors(metaInfoXml.getErrors()); return; } List<ServiceInfo> serviceInfos = metaInfoXml.getServices(); @@ -487,7 +487,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V stackInfo.setValid(false); setValid(false); stackInfo.setErrors(serviceModule.getErrors()); - setErrors(serviceModule.getErrors()); + setErrors(serviceModule.getErrors()); } } addServices(serviceModules); @@ -648,6 +648,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V * @throws AmbariException if unable to fully process the stack repositories */ private void processRepositories() throws AmbariException { + RepositoryXml rxml = stackDirectory.getRepoFile(); if (rxml == null) { return; @@ -658,15 +659,10 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V ", stackVersion=" + stackInfo.getVersion() + ", repoFolder=" + stackDirectory.getRepoDir()); - List<RepositoryInfo> repos = new ArrayList<RepositoryInfo>(); + List<RepositoryInfo> repos = rxml.getRepositories(); - for (RepositoryXml.Os o : rxml.getOses()) { - String osFamily = o.getFamily(); - for (String os : osFamily.split(",")) { - for (RepositoryXml.Repo r : o.getRepos()) { - repos.add(processRepository(osFamily, os, r)); - } - } + for (RepositoryInfo ri : repos) { + processRepository(ri); } stackInfo.getRepositories().addAll(repos); @@ -683,19 +679,11 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V * @param osType OS type * @param r repo */ - private RepositoryInfo processRepository(String osFamily, String osType, RepositoryXml.Repo r) { - RepositoryInfo ri = new RepositoryInfo(); - ri.setBaseUrl(r.getBaseUrl()); - ri.setDefaultBaseUrl(r.getBaseUrl()); - ri.setMirrorsList(r.getMirrorsList()); - ri.setOsType(osType.trim()); - ri.setRepoId(r.getRepoId()); - ri.setRepoName(r.getRepoName()); - ri.setLatestBaseUrl(r.getBaseUrl()); + private RepositoryInfo processRepository(RepositoryInfo ri) { LOG.debug("Checking for override for base_url"); String updatedUrl = stackContext.getUpdatedRepoUrl(stackInfo.getName(), stackInfo.getVersion(), - osFamily, r.getRepoId()); + ri.getOsType(), ri.getRepoId()); if (null != updatedUrl) { ri.setBaseUrl(updatedUrl); @@ -709,6 +697,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V return ri; } + /** * Merge role command order with the parent stack * @@ -716,9 +705,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V */ private void mergeRoleCommandOrder(StackModule parentStack) { - stackInfo.getRoleCommandOrder().merge(parentStack.stackInfo.getRoleCommandOrder()); - } @Override @@ -732,7 +719,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V } private Set<String> errorSet = new HashSet<String>(); - + @Override public void setErrors(String error) { errorSet.add(error); @@ -741,11 +728,11 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V @Override public Collection getErrors() { return errorSet; - } + } @Override public void setErrors(Collection error) { this.errorSet.addAll(error); } - + } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryType.java b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryType.java new file mode 100644 index 0000000000..f79d828496 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryType.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state; + +/** + * Identifies the type of repository + */ +public enum RepositoryType { + + /** + * Repository should be considered to have all components for a cluster + * deployment + */ + STANDARD, + + /** + * Repository may have only minimum components and is used for patching + * purposes + */ + PATCH, + + /** + * Repository is used to update services + */ + SERVICE + + + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryVersionState.java b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryVersionState.java index b0f85b40f2..344f35821e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryVersionState.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryVersionState.java @@ -80,51 +80,41 @@ package org.apache.ambari.server.state; */ public enum RepositoryVersionState { /** + * Repository version is not required + */ + NOT_REQUIRED, + /** * Repository version that is in the process of being installed. */ - INSTALLING(2), + INSTALLING, /** * Repository version that is installed and supported but not the active version. */ - INSTALLED(3), + INSTALLED, /** * Repository version that during the install process failed to install some components. */ - INSTALL_FAILED(0), + INSTALL_FAILED, /** * Repository version that is installed for some components but not for all. */ - OUT_OF_SYNC(1), + OUT_OF_SYNC, /** * Repository version that is installed and supported and is the active version. */ - CURRENT(7), + CURRENT, /** * Repository version that is in the process of upgrading to become the CURRENT active version, * and the previous active version transitions to an INSTALLED state. */ - UPGRADING(5), + UPGRADING, /** * Repository version that during the upgrade process failed to become the active version and must be remedied. */ - UPGRADE_FAILED(4), + UPGRADE_FAILED, /** * Repository version that finished upgrading and should be finalized to become CURRENT. */ - UPGRADED(6); - - /** - * Is used to determine cluster version state. - * @see org.apache.ambari.server.state.Cluster#recalculateClusterVersionState(String) - */ - private int priority; - - RepositoryVersionState(int priority) { - this.priority = priority; - } - - public int getPriority() { - return priority; - } + UPGRADED } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java index 7e1dd1d1e5..4afc85724c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceComponentImpl.java @@ -39,7 +39,6 @@ import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity; import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntityPK; import org.apache.ambari.server.orm.entities.HostComponentStateEntity; import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity; -import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntityPK; import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.state.cluster.ClusterImpl; import org.slf4j.Logger; @@ -74,8 +73,8 @@ public class ServiceComponentImpl implements ServiceComponent { private ServiceComponentHostFactory serviceComponentHostFactory; @Inject private AmbariMetaInfo ambariMetaInfo; - private ServiceComponentDesiredStateEntity desiredStateEntity; - private ServiceComponentDesiredStateEntityPK desiredStateEntityPK; + + ServiceComponentDesiredStateEntity desiredStateEntity; private Map<String, ServiceComponentHost> hostComponents; /** @@ -90,12 +89,13 @@ public class ServiceComponentImpl implements ServiceComponent { injector.injectMembers(this); clusterGlobalLock = service.getClusterGlobalLock(); this.service = service; - desiredStateEntity = new ServiceComponentDesiredStateEntity( ); + + desiredStateEntity = new ServiceComponentDesiredStateEntity(); desiredStateEntity.setComponentName(componentName); desiredStateEntity.setDesiredState(State.INIT); desiredStateEntity.setServiceName(service.getName()); desiredStateEntity.setClusterId(service.getClusterId()); - desiredStateEntityPK = getDesiredStateEntityPK(desiredStateEntity); + setDesiredStackVersion(service.getDesiredStackVersion()); hostComponents = new HashMap<String, ServiceComponentHost>(); @@ -125,11 +125,12 @@ public class ServiceComponentImpl implements ServiceComponent { injector.injectMembers(this); clusterGlobalLock = service.getClusterGlobalLock(); this.service = service; + desiredStateEntity = serviceComponentDesiredStateEntity; - this.componentName = serviceComponentDesiredStateEntity.getComponentName(); + componentName = serviceComponentDesiredStateEntity.getComponentName(); hostComponents = new HashMap<String, ServiceComponentHost>(); - for (HostComponentStateEntity hostComponentStateEntity : desiredStateEntity.getHostComponentStateEntities()) { + for (HostComponentStateEntity hostComponentStateEntity : serviceComponentDesiredStateEntity.getHostComponentStateEntities()) { HostComponentDesiredStateEntityPK pk = new HostComponentDesiredStateEntityPK(); pk.setClusterId(hostComponentStateEntity.getClusterId()); pk.setServiceName(hostComponentStateEntity.getServiceName()); @@ -167,8 +168,6 @@ public class ServiceComponentImpl implements ServiceComponent { + ", stackInfo=" + stackId.getStackId()); } - desiredStateEntityPK = getDesiredStateEntityPK(desiredStateEntity); - persisted = true; } @@ -179,15 +178,7 @@ public class ServiceComponentImpl implements ServiceComponent { @Override public String getName() { - ServiceComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity(); - if (desiredStateEntity != null) { - return desiredStateEntity.getComponentName(); - } else { - LOG.warn("Trying to fetch a member from an entity object that may " + - "have been previously deleted, serviceName = " + getServiceName() + ", " + - "componentName = " + componentName); - } - return null; + return componentName; } @Override @@ -372,7 +363,7 @@ public class ServiceComponentImpl implements ServiceComponent { ServiceComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity(); if (desiredStateEntity != null) { desiredStateEntity.setDesiredState(state); - saveIfPersisted(); + saveIfPersisted(desiredStateEntity); } else { LOG.warn("Setting a member on an entity object that may have been " + "previously deleted, serviceName = " + (service != null ? service.getName() : "")); @@ -418,7 +409,7 @@ public class ServiceComponentImpl implements ServiceComponent { ServiceComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity(); if (desiredStateEntity != null) { desiredStateEntity.setDesiredStack(stackEntity); - saveIfPersisted(); + saveIfPersisted(desiredStateEntity); } else { LOG.warn("Setting a member on an entity object that may have been " + "previously deleted, serviceName = " + (service != null ? service.getName() : "")); @@ -517,7 +508,7 @@ public class ServiceComponentImpl implements ServiceComponent { // service.refresh(); persisted = true; } else { - saveIfPersisted(); + saveIfPersisted(desiredStateEntity); } } finally { readWriteLock.writeLock().unlock(); @@ -536,9 +527,11 @@ public class ServiceComponentImpl implements ServiceComponent { pk.setServiceName(service.getName()); ClusterServiceEntity serviceEntity = clusterServiceDAO.findByPK(pk); + ServiceComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity(); desiredStateEntity.setClusterServiceEntity(serviceEntity); + serviceComponentDesiredStateDAO.create(desiredStateEntity); - clusterServiceDAO.merge(serviceEntity); + serviceEntity = clusterServiceDAO.merge(serviceEntity); } @Override @@ -547,10 +540,6 @@ public class ServiceComponentImpl implements ServiceComponent { readWriteLock.writeLock().lock(); try { if (isPersisted()) { - ServiceComponentDesiredStateEntityPK pk = new ServiceComponentDesiredStateEntityPK(); - pk.setComponentName(getName()); - pk.setClusterId(getClusterId()); - pk.setServiceName(getServiceName()); serviceComponentDesiredStateDAO.refresh(getDesiredStateEntity()); } } finally { @@ -564,9 +553,9 @@ public class ServiceComponentImpl implements ServiceComponent { * has already been acquired from {@link #readWriteLock}. */ @Transactional - void saveIfPersisted() { + void saveIfPersisted(ServiceComponentDesiredStateEntity desiredStateEntity) { if (isPersisted()) { - serviceComponentDesiredStateDAO.merge(desiredStateEntity); + desiredStateEntity = serviceComponentDesiredStateDAO.merge(desiredStateEntity); } } @@ -732,18 +721,10 @@ public class ServiceComponentImpl implements ServiceComponent { // Refresh cached reference after ever setter private ServiceComponentDesiredStateEntity getDesiredStateEntity() { - if (isPersisted()) { - desiredStateEntity = serviceComponentDesiredStateDAO.findByPK(desiredStateEntityPK); + if (!isPersisted()) { + return desiredStateEntity; } - return desiredStateEntity; - } - - private ServiceComponentDesiredStateEntityPK getDesiredStateEntityPK(ServiceComponentDesiredStateEntity desiredStateEntity) { - ServiceComponentDesiredStateEntityPK pk = new ServiceComponentDesiredStateEntityPK(); - pk.setClusterId(desiredStateEntity.getClusterId()); - pk.setComponentName(desiredStateEntity.getComponentName()); - pk.setServiceName(desiredStateEntity.getServiceName()); - return pk; + return serviceComponentDesiredStateDAO.findById(desiredStateEntity.getId()); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java index bd87a55e31..05aecac0e3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java @@ -19,8 +19,10 @@ package org.apache.ambari.server.state; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.stack.MasterHostResolver; @@ -84,6 +86,8 @@ public class UpgradeContext { */ private boolean m_autoSkipManualVerification = false; + private Set<String> m_supported = new HashSet<>(); + /** * Constructor. * @@ -350,4 +354,28 @@ public class UpgradeContext { public void setAutoSkipManualVerification(boolean autoSkipManualVerification) { m_autoSkipManualVerification = autoSkipManualVerification; } + + /** + * Sets the service names that are supported by an upgrade. This is used for + * {@link RepositoryType#PATCH} and {@link RepositoryType#SERVICE}. + * + * @param services the set of specific services + */ + public void setSupportedServices(Set<String> services) { + m_supported = services; + } + + /** + * Gets if a service is supported. If there are no services marked for the context, + * then ALL services are supported + * @param serviceName the service name to check. + * @return {@code true} when the service is supported + */ + public boolean isServiceSupported(String serviceName) { + if (m_supported.isEmpty() || m_supported.contains(serviceName)) { + return true; + } + + return false; + } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java index 8213a787bc..0d9176d1c8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java @@ -317,6 +317,10 @@ public class UpgradeHelper { // !!! cluster and service checks are empty here for (UpgradePack.OrderService service : services) { + if (!context.isServiceSupported(service.serviceName)) { + continue; + } + if (upgradePack.getType() == UpgradeType.ROLLING && !allTasks.containsKey(service.serviceName)) { continue; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java index 50e02a59ef..9c86ea16d6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java @@ -1335,7 +1335,7 @@ public class ClusterImpl implements Cluster { * UPGRADING: at least one host is UPGRADED, and the rest in UPGRADING|INSTALLED * INSTALLED: all hosts in INSTALLED * INSTALL_FAILED: at least one host in INSTALL_FAILED - * INSTALLING: all hosts in INSTALLING. Notice that if one host is CURRENT and another is INSTALLING, then the + * INSTALLING: all hosts in INSTALLING -or- INSTALLING and NOT_REQUIRED. Notice that if one host is CURRENT and another is INSTALLING, then the * effective version will be OUT_OF_SYNC. * OUT_OF_SYNC: otherwise * @param stateToHosts Map from state to the collection of hosts with that state @@ -1385,13 +1385,25 @@ public class ClusterImpl implements Cluster { } } - final int totalINSTALLING = stateToHosts.containsKey(RepositoryVersionState.INSTALLING) ? stateToHosts.get(RepositoryVersionState.INSTALLING).size() : 0; - final int totalINSTALLED = stateToHosts.containsKey(RepositoryVersionState.INSTALLED) ? stateToHosts.get(RepositoryVersionState.INSTALLED).size() : 0; - final int totalINSTALL_FAILED = stateToHosts.containsKey(RepositoryVersionState.INSTALL_FAILED) ? stateToHosts.get(RepositoryVersionState.INSTALL_FAILED).size() : 0; - if (totalINSTALLING + totalINSTALLED + totalINSTALL_FAILED== totalHosts) { + int totalInstalling = stateToHosts.containsKey(RepositoryVersionState.INSTALLING) ? stateToHosts.get(RepositoryVersionState.INSTALLING).size() : 0; + int totalInstalled = stateToHosts.containsKey(RepositoryVersionState.INSTALLED) ? stateToHosts.get(RepositoryVersionState.INSTALLED).size() : 0; + int totalNotRequired = stateToHosts.containsKey(RepositoryVersionState.NOT_REQUIRED) ? stateToHosts.get(RepositoryVersionState.NOT_REQUIRED).size() : 0; + int totalInstallFailed = stateToHosts.containsKey(RepositoryVersionState.INSTALL_FAILED) ? stateToHosts.get(RepositoryVersionState.INSTALL_FAILED).size() : 0; + + if (totalInstalling + totalInstalled + totalInstallFailed == totalHosts) { return RepositoryVersionState.INSTALLING; } + if (totalNotRequired > 0) { + if (totalInstalling + totalInstalled + totalNotRequired == totalHosts) { + return RepositoryVersionState.INSTALLING; + } + + if (totalInstalled + totalNotRequired == totalHosts) { + return RepositoryVersionState.INSTALLED; + } + } + // Also returns when have a mix of CURRENT and INSTALLING|INSTALLED|UPGRADING|UPGRADED LOG.warn("have a mix of CURRENT and INSTALLING|INSTALLED|UPGRADING|UPGRADED host versions, " + "returning OUT_OF_SYNC as cluster version. Host version states: " + stateToHosts.toString()); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/AvailableService.java b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/AvailableService.java new file mode 100644 index 0000000000..70f3c40a21 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/AvailableService.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state.repository; + +import java.util.ArrayList; +import java.util.List; + +import org.codehaus.jackson.annotate.JsonProperty; +import org.codehaus.jackson.map.annotate.JsonSerialize; +import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; + +/** + * A representation of a {@link ManifestService} that is available for upgrading. This + * object will be serialized directly in API responses. + */ +public class AvailableService { + + @JsonProperty("name") + private String name; + + @JsonProperty("display_name") + @JsonSerialize(include=Inclusion.NON_NULL) + private String displayName; + + private List<AvailableVersion> versions = new ArrayList<>(); + + AvailableService(String service, String serviceDisplay) { + name = service; + displayName = serviceDisplay; + } + + /** + * @return the service name + */ + public String getName() { + return name; + } + + /** + * @return the list of versions to append additional versions. + */ + public List<AvailableVersion> getVersions() { + return versions; + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/AvailableServiceReference.java b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/AvailableServiceReference.java new file mode 100644 index 0000000000..35b6ceacab --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/AvailableServiceReference.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state.repository; + +import java.util.HashSet; +import java.util.Set; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; + +/** + * An available service can be upgraded. This is mostly a marker class identifying the + * {@link ManifestService} with a set of optional components to upgrade. + */ +public class AvailableServiceReference { + + /** + * The reference id back to the service in the repository. + */ + @XmlAttribute(name="idref") + public String serviceIdReference; + + /** + * Set of components to update. + */ + @XmlElement(name="component") + public Set<String> components = new HashSet<>(); + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/AvailableVersion.java b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/AvailableVersion.java new file mode 100644 index 0000000000..b10a13e9a1 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/AvailableVersion.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state.repository; + +import java.util.Set; + +import org.codehaus.jackson.annotate.JsonProperty; +import org.codehaus.jackson.map.annotate.JsonSerialize; +import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; + +/** + * Represents a version of a {@link ManifestService} that is available for upgrading. + * + * This class is serialized directly in API responses. + */ +public class AvailableVersion { + + @JsonProperty("version") + private String version; + + @JsonProperty("version_id") + @JsonSerialize(include=Inclusion.NON_NULL) + private String versionId; + + @JsonProperty + private Set<Component> components; + + AvailableVersion(String version, String versionId, Set<Component> components) { + this.version = version; + this.versionId = versionId; + this.components = components; + } + + static class Component { + @JsonProperty("name") + private String name; + + @JsonProperty("display_name") + private String display; + + Component(String name, String display) { + this.name = name; + this.display = display; + } + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/ManifestService.java b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/ManifestService.java new file mode 100644 index 0000000000..2bbacd584c --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/ManifestService.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state.repository; + +import javax.xml.bind.annotation.XmlAttribute; + +/** + * Represents a service definition in the manifest. The service manifest is a list + * of all services available in a repository. + */ +public class ManifestService { + + /** + * The XML unique id for the service and version. + */ + @XmlAttribute(name="id") + public String serviceId; + + /** + * Name of the service. + */ + @XmlAttribute(name="name") + public String serviceName; + + /** + * Version of the service. This is the publicly available version. + */ + @XmlAttribute(name="version") + public String version; + + /** + * Version id of the service. This may be a build number. + */ + @XmlAttribute(name="version-id") + public String versionId; +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/Release.java b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/Release.java new file mode 100644 index 0000000000..6bcedf536a --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/Release.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state.repository; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.ambari.server.state.RepositoryType; +import org.apache.commons.lang.StringUtils; + +/** + * Release information for a repository. + */ +@XmlRootElement(name="release") +@XmlAccessorType(XmlAccessType.FIELD) +public class Release { + + /** + * The type of repository dictates how the repo should be installed and its upgradability. + */ + @XmlElement(name="type") + public RepositoryType repositoryType; + + /** + * The stack id for the repository. + */ + @XmlElement(name="stack-id") + public String stackId; + + /** + * The stack version. + */ + @XmlElement(name="version") + public String version; + + /** + * The build identifier. + */ + @XmlElement(name="build") + public String build; + + /** + * The compatability regex. This is used to relate the release to another release. + */ + @XmlElement(name="compatible-with") + public String compatibleWith; + + /** + * The release notes or link. + */ + @XmlElement(name="release-notes") + public String releaseNotes; + + /** + * The optional display name + */ + @XmlElement(name="display") + public String display; + + /** + * @return the full version + */ + public String getFullVersion() { + StringBuilder sb = new StringBuilder(version); + + if (StringUtils.isNotBlank(build)) { + sb.append('-').append(StringUtils.trim(build)); + } + + return sb.toString(); + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java new file mode 100644 index 0000000000..a610bd04e1 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java @@ -0,0 +1,224 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state.repository; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.xml.XMLConstants; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Unmarshaller; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamReader; +import javax.xml.transform.stream.StreamSource; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; + +import org.apache.ambari.server.state.ComponentInfo; +import org.apache.ambari.server.state.ServiceInfo; +import org.apache.ambari.server.state.StackInfo; +import org.apache.ambari.server.state.repository.AvailableVersion.Component; +import org.apache.ambari.server.state.stack.RepositoryXml; +import org.apache.commons.io.IOUtils; + +/** + * Class that wraps a repository definition file. + */ +@XmlRootElement(name="repository-version") +@XmlAccessorType(XmlAccessType.FIELD) +public class VersionDefinitionXml { + + /** + * Release details. + */ + @XmlElement(name = "release") + public Release release; + + /** + * The manifest of ALL services available in this repository. + */ + @XmlElementWrapper(name="manifest") + @XmlElement(name="service") + List<ManifestService> manifestServices = new ArrayList<>(); + + /** + * For PATCH and SERVICE repositories, this dictates what is available for upgrade + * from the manifest. + */ + @XmlElementWrapper(name="available-services") + @XmlElement(name="service") + List<AvailableServiceReference> availableServices = new ArrayList<>(); + + /** + * Represents the repository details. This is reused from stack repo info. + */ + @XmlElement(name="repository-info") + public RepositoryXml repositoryInfo; + + /** + * The xsd location. Never {@code null}. + */ + @XmlTransient + public String xsdLocation; + + @XmlTransient + private Map<String, AvailableService> availableMap; + + + /** + * @param stack the stack info needed to lookup service and component display names + * @return a collection of AvailableServices used for web service consumption + */ + public Collection<AvailableService> getAvailableServices(StackInfo stack) { + if (availableServices.isEmpty()) { + return Collections.emptyList(); + } + + if (null == availableMap) { + Map<String, ManifestService> manifests = buildManifest(); + availableMap = new HashMap<>(); + + for (AvailableServiceReference ref : availableServices) { + ManifestService ms = manifests.get(ref.serviceIdReference); + ServiceInfo service = stack.getService(ms.serviceName); + + if (!availableMap.containsKey(ms.serviceName)) { + String display = (null == service) ? ms.serviceName: service.getDisplayName(); + + availableMap.put(ms.serviceName, new AvailableService(ms.serviceName, display)); + } + + AvailableService as = availableMap.get(ms.serviceName); + as.getVersions().add(new AvailableVersion(ms.version, ms.versionId, + buildComponents(service, ref.components))); + } + } + + return availableMap.values(); + } + + /** + * @return the list of manifest services to a map for easier access. + */ + private Map<String, ManifestService> buildManifest() { + Map<String, ManifestService> normalized = new HashMap<>(); + + for (ManifestService ms : manifestServices) { + normalized.put(ms.serviceId, ms); + } + return normalized; + } + + /** + * @param service the service containing components + * @param components the set of components in the service + * @return the set of components name/display name pairs + */ + private Set<Component> buildComponents(ServiceInfo service, Set<String> components) { + Set<Component> set = new HashSet<>(); + + for (String component : components) { + ComponentInfo ci = service.getComponentByName(component); + String display = (null == ci) ? component : ci.getDisplayName(); + set.add(new Component(component, display)); + } + + return set; + } + + + + /** + * Parses a URL for a definition XML file into the object graph. + * @param url the URL to load. Can be a file URL reference also. + * @return the definition + */ + public static VersionDefinitionXml load(URL url) throws Exception { + + InputStream stream = null; + try { + stream = url.openStream(); + return load(stream); + } finally { + IOUtils.closeQuietly(stream); + } + } + + /** + * Parses an xml string. Used when the xml is in the database. + * @param xml the xml string + * @return the definition + */ + public static VersionDefinitionXml load(String xml) throws Exception { + return load(new ByteArrayInputStream(xml.getBytes("UTF-8"))); + } + + /** + * Parses a stream into an object graph + * @param stream the stream + * @return the definition + * @throws Exception + */ + private static VersionDefinitionXml load(InputStream stream) throws Exception { + + XMLInputFactory xmlFactory = XMLInputFactory.newInstance(); + XMLStreamReader xmlReader = xmlFactory.createXMLStreamReader(stream); + + xmlReader.nextTag(); + String xsdName = xmlReader.getAttributeValue(XMLConstants.W3C_XML_SCHEMA_INSTANCE_NS_URI, "noNamespaceSchemaLocation"); + + JAXBContext ctx = JAXBContext.newInstance(VersionDefinitionXml.class); + Unmarshaller unmarshaller = ctx.createUnmarshaller(); + + if (null != xsdName) { + InputStream xsdStream = VersionDefinitionXml.class.getClassLoader().getResourceAsStream(xsdName); + + if (null == xsdStream) { + throw new Exception(String.format("Could not load XSD identified by '%s'", xsdName)); + } + + SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + Schema schema = factory.newSchema(new StreamSource(xsdStream)); + unmarshaller.setSchema(schema); + } + + VersionDefinitionXml xml = (VersionDefinitionXml) unmarshaller.unmarshal(xmlReader); + xml.xsdLocation = xsdName; + + return xml; + } + + + + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java index eff063b9d2..f57b9364b4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/RepositoryXml.java @@ -29,7 +29,9 @@ import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; + import org.apache.ambari.server.stack.Validable; +import org.apache.ambari.server.state.RepositoryInfo; /** * Represents the repository file <code>$STACK_VERSION/repos/repoinfo.xml</code>. @@ -47,7 +49,7 @@ public class RepositoryXml implements Validable{ private boolean valid = true; /** - * + * * @return valid xml flag */ @Override @@ -56,17 +58,17 @@ public class RepositoryXml implements Validable{ } /** - * + * * @param valid set validity flag */ @Override public void setValid(boolean valid) { this.valid = valid; - } - + } + @XmlTransient private Set<String> errorSet = new HashSet<String>(); - + @Override public void setErrors(String error) { errorSet.add(error); @@ -75,27 +77,27 @@ public class RepositoryXml implements Validable{ @Override public Collection getErrors() { return errorSet; - } - + } + @Override public void setErrors(Collection error) { this.errorSet.addAll(error); - } - + } + /** * @return the latest URI defined, if any. */ public String getLatestURI() { return latestUri; } - + /** * @return the list of <code>os</code> elements. */ public List<Os> getOses() { return oses; } - + /** * The <code>os</code> tag. */ @@ -103,20 +105,20 @@ public class RepositoryXml implements Validable{ public static class Os { @XmlAttribute(name="family") private String family; - + @XmlElement(name="repo") private List<Repo> repos; private Os() { } - + /** * @return the os family */ public String getFamily() { return family; } - + /** * @return the list of repo elements */ @@ -138,7 +140,7 @@ public class RepositoryXml implements Validable{ private Repo() { } - + /** * @return the base url */ @@ -152,27 +154,56 @@ public class RepositoryXml implements Validable{ public String getMirrorsList() { return (null == mirrorslist || mirrorslist.isEmpty()) ? null : mirrorslist; } - + /** * @return the repo id */ public String getRepoId() { return repoid; } - + /** * @return the repo name */ public String getRepoName() { return reponame; } - + public String getLatestUri() { return latest; } - - - } - + + + } + + /** + * @return the list of repositories consumable by the web service. + */ + public List<RepositoryInfo> getRepositories() { + List<RepositoryInfo> repos = new ArrayList<>(); + + for (RepositoryXml.Os o : getOses()) { + String osFamily = o.getFamily(); + for (String os : osFamily.split(",")) { + for (RepositoryXml.Repo r : o.getRepos()) { + + RepositoryInfo ri = new RepositoryInfo(); + ri.setBaseUrl(r.getBaseUrl()); + ri.setDefaultBaseUrl(r.getBaseUrl()); + ri.setMirrorsList(r.getMirrorsList()); + ri.setOsType(os.trim()); + ri.setRepoId(r.getRepoId()); + ri.setRepoName(r.getRepoName()); + ri.setLatestBaseUrl(r.getBaseUrl()); + + repos.add(ri); + } + } + } + + return repos; + } + + } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java index bfb6214362..92828af1b2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java @@ -51,7 +51,6 @@ import org.apache.ambari.server.orm.entities.HostComponentStateEntity; import org.apache.ambari.server.orm.entities.HostEntity; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity; -import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntityPK; import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; @@ -1453,26 +1452,26 @@ public class ServiceComponentHostImpl implements ServiceComponentHost { @Transactional protected void persistEntities() { + ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity = serviceComponentDesiredStateDAO.findByName( + serviceComponent.getClusterId(), serviceComponent.getServiceName(), + serviceComponent.getName()); + HostEntity hostEntity = hostDAO.findByName(getHostName()); hostEntity.addHostComponentStateEntity(stateEntity); hostEntity.addHostComponentDesiredStateEntity(desiredStateEntity); - ServiceComponentDesiredStateEntityPK dpk = new ServiceComponentDesiredStateEntityPK(); - dpk.setClusterId(serviceComponent.getClusterId()); - dpk.setServiceName(serviceComponent.getServiceName()); - dpk.setComponentName(serviceComponent.getName()); - - ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity = serviceComponentDesiredStateDAO.findByPK(dpk); - serviceComponentDesiredStateEntity.getHostComponentDesiredStateEntities().add(desiredStateEntity); - desiredStateEntity.setServiceComponentDesiredStateEntity(serviceComponentDesiredStateEntity); desiredStateEntity.setHostEntity(hostEntity); + stateEntity.setServiceComponentDesiredStateEntity(serviceComponentDesiredStateEntity); stateEntity.setHostEntity(hostEntity); hostComponentStateDAO.create(stateEntity); hostComponentDesiredStateDAO.create(desiredStateEntity); + serviceComponentDesiredStateEntity.getHostComponentDesiredStateEntities().add( + desiredStateEntity); + HostComponentStateEntity stateEntity = hostComponentStateDAO.findByIndex(serviceComponent.getClusterId(), serviceComponent.getServiceName(), serviceComponent.getName(), hostEntity.getHostId()); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java index b00b0e8a78..ff947956df 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java @@ -64,9 +64,7 @@ import org.apache.ambari.server.orm.entities.HostEntity; import org.apache.ambari.server.orm.entities.HostRoleCommandEntity; import org.apache.ambari.server.orm.entities.KeyValueEntity; import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity; -import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntityPK; import org.apache.ambari.server.orm.entities.StackEntity; -import org.apache.ambari.server.orm.entities.UserEntity; import org.apache.ambari.server.state.HostComponentAdminState; import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.ServiceInfo; @@ -607,24 +605,16 @@ public class UpgradeCatalog150 extends AbstractUpgradeCatalog { List<ClusterEntity> clusterEntities = clusterDAO.findAll(); for (final ClusterEntity clusterEntity : clusterEntities) { - ServiceComponentDesiredStateEntityPK pkHS = new ServiceComponentDesiredStateEntityPK(); - pkHS.setComponentName("HISTORYSERVER"); - pkHS.setClusterId(clusterEntity.getClusterId()); - pkHS.setServiceName("MAPREDUCE"); - - ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntityHS = serviceComponentDesiredStateDAO.findByPK(pkHS); + ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntityHS = serviceComponentDesiredStateDAO.findByName( + clusterEntity.getClusterId(), "MAPREDUCE", "HISTORYSERVER"); // already have historyserver if(serviceComponentDesiredStateEntityHS != null) { continue; } - ServiceComponentDesiredStateEntityPK pkJT = new ServiceComponentDesiredStateEntityPK(); - pkJT.setComponentName("JOBTRACKER"); - pkJT.setClusterId(clusterEntity.getClusterId()); - pkJT.setServiceName("MAPREDUCE"); - - ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntityJT = serviceComponentDesiredStateDAO.findByPK(pkJT); + ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntityJT = serviceComponentDesiredStateDAO.findByName( + clusterEntity.getClusterId(), "MAPREDUCE", "JOBTRACKER"); // no jobtracker present probably mapreduce is not installed if(serviceComponentDesiredStateEntityJT == null) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java index 91de82aadb..b3b1b70cf8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java @@ -86,7 +86,6 @@ import org.apache.ambari.server.orm.entities.PrivilegeEntity; import org.apache.ambari.server.orm.entities.ResourceEntity; import org.apache.ambari.server.orm.entities.ResourceTypeEntity; import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity; -import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntityPK; import org.apache.ambari.server.orm.entities.ServiceDesiredStateEntity; import org.apache.ambari.server.orm.entities.ServiceDesiredStateEntityPK; import org.apache.ambari.server.orm.entities.StackEntity; @@ -769,11 +768,8 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog { List<ClusterEntity> clusterEntities = clusterDAO.findAll(); for (final ClusterEntity clusterEntity : clusterEntities) { - ServiceComponentDesiredStateEntityPK pkHCATInHcatalog = new ServiceComponentDesiredStateEntityPK(); - pkHCATInHcatalog.setComponentName(componentName); - pkHCATInHcatalog.setClusterId(clusterEntity.getClusterId()); - pkHCATInHcatalog.setServiceName(serviceNameToBeDeleted); - ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntityToDelete = serviceComponentDesiredStateDAO.findByPK(pkHCATInHcatalog); + ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntityToDelete = serviceComponentDesiredStateDAO.findByName( + clusterEntity.getClusterId(), serviceNameToBeDeleted, componentName); if (serviceComponentDesiredStateEntityToDelete == null) { continue; @@ -803,6 +799,7 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog { serviceComponentDesiredStateEntity.setDesiredStack(serviceComponentDesiredStateEntityToDelete.getDesiredStack()); serviceComponentDesiredStateEntity.setDesiredState(serviceComponentDesiredStateEntityToDelete.getDesiredState()); serviceComponentDesiredStateEntity.setClusterServiceEntity(clusterServiceEntity); + serviceComponentDesiredStateDAO.create(serviceComponentDesiredStateEntity); Iterator<HostComponentDesiredStateEntity> hostComponentDesiredStateIterator = serviceComponentDesiredStateEntityToDelete.getHostComponentDesiredStateEntities().iterator(); Iterator<HostComponentStateEntity> hostComponentStateIterator = serviceComponentDesiredStateEntityToDelete.getHostComponentStateEntities().iterator(); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java index 70b8f9ffea..143f9afb0f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java @@ -45,7 +45,6 @@ import org.apache.ambari.server.orm.entities.ClusterServiceEntityPK; import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity; import org.apache.ambari.server.orm.entities.HostComponentStateEntity; import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity; -import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntityPK; import org.apache.ambari.server.orm.entities.ServiceDesiredStateEntity; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; @@ -516,11 +515,8 @@ public class UpgradeCatalog200 extends AbstractUpgradeCatalog { } // remove component state - ServiceComponentDesiredStateEntityPK primaryKey = new ServiceComponentDesiredStateEntityPK(); - primaryKey.setClusterId(nagios.getClusterId()); - primaryKey.setComponentName(componentDesiredState.getComponentName()); - primaryKey.setServiceName(componentDesiredState.getServiceName()); - componentDesiredStateDao.removeByPK(primaryKey); + componentDesiredStateDao.removeByName(nagios.getClusterId(), + componentDesiredState.getServiceName(), componentDesiredState.getComponentName()); } // remove service state diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java index faf4b9693b..d97c0c1d3c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java @@ -18,12 +18,28 @@ package org.apache.ambari.server.upgrade; -import com.google.gson.Gson; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.inject.Inject; -import com.google.inject.Injector; +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.regex.Matcher; + +import javax.persistence.EntityManager; +import javax.persistence.Query; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaDelete; +import javax.persistence.criteria.Root; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; @@ -39,7 +55,6 @@ import org.apache.ambari.server.orm.entities.ArtifactEntity; import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity; import org.apache.ambari.server.orm.entities.HostComponentStateEntity; import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity; -import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntityPK; import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; @@ -58,26 +73,12 @@ import org.eclipse.persistence.internal.databaseaccess.FieldTypeDefinition; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.persistence.EntityManager; -import javax.persistence.Query; -import javax.persistence.criteria.CriteriaBuilder; -import javax.persistence.criteria.CriteriaDelete; -import javax.persistence.criteria.Root; -import java.net.URI; -import java.net.URISyntaxException; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.regex.Matcher; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.inject.Inject; +import com.google.inject.Injector; /** @@ -1078,11 +1079,9 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog { @Override public void run() { ServiceComponentDesiredStateDAO dao = injector.getInstance(ServiceComponentDesiredStateDAO.class); - ServiceComponentDesiredStateEntityPK entityPK = new ServiceComponentDesiredStateEntityPK(); - entityPK.setClusterId(cluster.getClusterId()); - entityPK.setServiceName("STORM"); - entityPK.setComponentName("STORM_REST_API"); - ServiceComponentDesiredStateEntity entity = dao.findByPK(entityPK); + ServiceComponentDesiredStateEntity entity = dao.findByName(cluster.getClusterId(), + "STORM", "STORM_REST_API"); + if (entity != null) { EntityManager em = getEntityManagerProvider().get(); CriteriaBuilder cb = em.getCriteriaBuilder(); @@ -1227,7 +1226,7 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog { JsonObject reporting = rootJson.getAsJsonObject("reporting"); JsonObject ok = reporting.getAsJsonObject("ok"); JsonObject warning = reporting.getAsJsonObject("warning"); - JsonObject critical = reporting.getAsJsonObject("critical"); + JsonObject critical = reporting.getAsJsonObject("critical"); rootJson.remove("type"); rootJson.remove("default_port"); @@ -1533,8 +1532,8 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog { hiveSiteAddProps.put("hive.server2.authentication.kerberos.keytab", ""); hiveSiteAddProps.put("hive.server2.authentication.kerberos.principal", ""); } - - + + updateConfigurationPropertiesForCluster(cluster, "hive-site", hiveSiteAddProps, hiveSiteRemoveProps, false, true); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java index 0f9d0e0e36..365526bc7b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java @@ -96,7 +96,6 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog { private static final String TEZ_COUNTERS_MAX = "tez.counters.max"; private static final String TEZ_COUNTERS_MAX_GROUPS = "tez.counters.max.groups"; - // ----- Constructors ------------------------------------------------------ /** diff --git a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql index 0fdfd2b77e..6c94999f6c 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql @@ -171,14 +171,18 @@ CREATE TABLE host_version ( PRIMARY KEY (id)); CREATE TABLE servicecomponentdesiredstate ( + id BIGINT NOT NULL, component_name VARCHAR(255) NOT NULL, cluster_id BIGINT NOT NULL, desired_stack_id BIGINT NOT NULL, desired_state VARCHAR(255) NOT NULL, service_name VARCHAR(255) NOT NULL, - PRIMARY KEY (component_name, cluster_id, service_name) + CONSTRAINT pk_servicecomponentdesiredstate PRIMARY KEY (id), + CONSTRAINT unq_scdesiredstate_name UNIQUE(component_name,service_name,cluster_id) ); +CREATE INDEX idx_sc_desired_state ON servicecomponentdesiredstate(component_name, service_name, cluster_id); + CREATE TABLE servicedesiredstate ( cluster_id BIGINT NOT NULL, desired_host_role_mapping INTEGER NOT NULL, @@ -551,6 +555,10 @@ CREATE TABLE repo_version ( version VARCHAR(255) NOT NULL, display_name VARCHAR(128) NOT NULL, repositories VARCHAR(3000) NOT NULL, + version_url VARCHAR(1024), + version_xml CLOB, + version_xsd VARCHAR(512), + parent_id BIGINT, PRIMARY KEY(repo_version_id) ); @@ -1035,7 +1043,9 @@ INSERT INTO ambari_sequences (sequence_name, sequence_value) union all select 'setting_id_seq', 0 FROM SYSIBM.SYSDUMMY1 union all - select 'hostcomponentstate_id_seq', 0 FROM SYSIBM.SYSDUMMY1; + select 'hostcomponentstate_id_seq', 0 FROM SYSIBM.SYSDUMMY1 + union all + select 'servicecomponentdesiredstate_id_seq', 0 FROM SYSIBM.SYSDUMMY1; INSERT INTO adminresourcetype (resource_type_id, resource_type_name) SELECT 1, 'AMBARI' FROM SYSIBM.SYSDUMMY1 diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql index 11e43c2c05..7f5eddcce2 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql @@ -172,14 +172,18 @@ CREATE TABLE host_version ( PRIMARY KEY (id)); CREATE TABLE servicecomponentdesiredstate ( + id BIGINT NOT NULL, component_name VARCHAR(100) NOT NULL, cluster_id BIGINT NOT NULL, desired_stack_id BIGINT NOT NULL, desired_state VARCHAR(255) NOT NULL, service_name VARCHAR(100) NOT NULL, - PRIMARY KEY (component_name, cluster_id, service_name) + CONSTRAINT pk_servicecomponentdesiredstate PRIMARY KEY (id), + CONSTRAINT unq_scdesiredstate_name UNIQUE(component_name,service_name,cluster_id) ); +CREATE INDEX idx_sc_desired_state ON servicecomponentdesiredstate(component_name, service_name, cluster_id); + CREATE TABLE servicedesiredstate ( cluster_id BIGINT NOT NULL, desired_host_role_mapping INTEGER NOT NULL, @@ -557,7 +561,12 @@ CREATE TABLE repo_version ( stack_id BIGINT NOT NULL, version VARCHAR(255) NOT NULL, display_name VARCHAR(128) NOT NULL, - repositories LONGTEXT NOT NULL, + repositories MEDIUMTEXT NOT NULL, + repo_type VARCHAR(255) DEFAULT 'STANDARD' NOT NULL, + version_url VARCHAR(1024), + version_xml MEDIUMTEXT, + version_xsd VARCHAR(512), + parent_id BIGINT, PRIMARY KEY(repo_version_id) ); diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql index 8d4ba2861a..e3e4336971 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql @@ -162,14 +162,18 @@ CREATE TABLE host_version ( PRIMARY KEY (id)); CREATE TABLE servicecomponentdesiredstate ( + id NUMBER(19) NOT NULL, component_name VARCHAR2(255) NOT NULL, cluster_id NUMBER(19) NOT NULL, desired_stack_id NUMBER(19) NOT NULL, desired_state VARCHAR2(255) NOT NULL, service_name VARCHAR2(255) NOT NULL, - PRIMARY KEY (component_name, cluster_id, service_name) + CONSTRAINT pk_servicecomponentdesiredstate PRIMARY KEY (alert_id), + CONSTRAINT unq_scdesiredstate_name UNIQUE(component_name,service_name,cluster_id) ); +CREATE INDEX idx_sc_desired_state ON servicecomponentdesiredstate(component_name, service_name, cluster_id); + CREATE TABLE servicedesiredstate ( cluster_id NUMBER(19) NOT NULL, desired_host_role_mapping NUMBER(10) NOT NULL, @@ -547,6 +551,11 @@ CREATE TABLE repo_version ( version VARCHAR2(255) NOT NULL, display_name VARCHAR2(128) NOT NULL, repositories CLOB NOT NULL, + repo_type VARCHAR2(255) DEFAULT 'STANDARD' NOT NULL, + version_url VARCHAR(1024), + version_xml CLOB, + version_xsd VARCHAR(512), + parent_id NUMBER(19), PRIMARY KEY(repo_version_id) ); @@ -989,6 +998,7 @@ INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_re INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_host_group_id_seq', 0); INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('setting_id_seq', 0); INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('hostcomponentstate_id_seq', 0); +INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('servicecomponentdesiredstate_id_seq', 0); INSERT INTO metainfo("metainfo_key", "metainfo_value") values ('version', '${ambariVersion}'); diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql index c762ac42a6..f5983a53db 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql @@ -171,14 +171,19 @@ CREATE TABLE host_version ( PRIMARY KEY (id)); CREATE TABLE servicecomponentdesiredstate ( + id BIGINT NOT NULL, component_name VARCHAR(255) NOT NULL, cluster_id BIGINT NOT NULL, desired_stack_id BIGINT NOT NULL, desired_state VARCHAR(255) NOT NULL, service_name VARCHAR(255) NOT NULL, - PRIMARY KEY (component_name, cluster_id, service_name) + CONSTRAINT pk_servicecomponentdesiredstate PRIMARY KEY (id), + CONSTRAINT unq_scdesiredstate_name UNIQUE(component_name,service_name,cluster_id) ); +CREATE INDEX idx_sc_desired_state ON servicecomponentdesiredstate(component_name, service_name, cluster_id); + + CREATE TABLE servicedesiredstate ( cluster_id BIGINT NOT NULL, desired_host_role_mapping INTEGER NOT NULL, @@ -551,6 +556,11 @@ CREATE TABLE repo_version ( version VARCHAR(255) NOT NULL, display_name VARCHAR(128) NOT NULL, repositories TEXT NOT NULL, + repo_type VARCHAR(255) DEFAULT 'STANDARD' NOT NULL, + version_url VARCHAR(1024), + version_xml VARCHAR TEXT, + version_xsd VARCHAR(512), + parent_id BIGINT, PRIMARY KEY(repo_version_id) ); @@ -1035,7 +1045,9 @@ INSERT INTO ambari_sequences (sequence_name, sequence_value) union all select 'setting_id_seq', 0 union all - select 'hostcomponentstate_id_seq', 0; + select 'hostcomponentstate_id_seq', 0 + union all + select 'servicecomponentdesiredstate_id_seq', 0; INSERT INTO adminresourcetype (resource_type_id, resource_type_name) SELECT 1, 'AMBARI' diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql index 81b41fe547..9d9a986a7f 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql @@ -196,15 +196,19 @@ CREATE TABLE ambari.host_version ( GRANT ALL PRIVILEGES ON TABLE ambari.host_version TO :username; CREATE TABLE ambari.servicecomponentdesiredstate ( + id BIGINT NOT NULL, component_name VARCHAR(255) NOT NULL, cluster_id BIGINT NOT NULL, desired_stack_id BIGINT NOT NULL, desired_state VARCHAR(255) NOT NULL, service_name VARCHAR(255) NOT NULL, - PRIMARY KEY (component_name, cluster_id, service_name) + CONSTRAINT pk_servicecomponentdesiredstate PRIMARY KEY (id), + CONSTRAINT unq_scdesiredstate_name UNIQUE(component_name,service_name,cluster_id) ); GRANT ALL PRIVILEGES ON TABLE ambari.servicecomponentdesiredstate TO :username; +CREATE INDEX idx_sc_desired_state ON ambari.servicecomponentdesiredstate(component_name, service_name, cluster_id); + CREATE TABLE ambari.servicedesiredstate ( cluster_id BIGINT NOT NULL, desired_host_role_mapping INTEGER NOT NULL, @@ -620,6 +624,11 @@ CREATE TABLE ambari.repo_version ( version VARCHAR(255) NOT NULL, display_name VARCHAR(128) NOT NULL, repositories TEXT NOT NULL, + repo_type VARCHAR(255) DEFAULT 'STANDARD' NOT NULL, + version_url VARCHAR(1024), + version_xml TEXT, + version_xsd VARCHAR(512), + parent_id BIGINT, PRIMARY KEY(repo_version_id) ); GRANT ALL PRIVILEGES ON TABLE ambari.repo_version TO :username; @@ -1134,7 +1143,9 @@ INSERT INTO ambari.ambari_sequences (sequence_name, sequence_value) union all select 'setting_id_seq', 0 union all - select 'hostcomponentstate_id_seq', 0; + select 'hostcomponentstate_id_seq', 0 + union all + select 'servicecomponentdesiredstate_id_seq', 0; INSERT INTO ambari.adminresourcetype (resource_type_id, resource_type_name) SELECT 1, 'AMBARI' diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql index f8c9b8d848..8c82afcffb 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql @@ -161,14 +161,18 @@ CREATE TABLE host_version ( PRIMARY KEY (id)); CREATE TABLE servicecomponentdesiredstate ( + id NUMERIC(19) NOT NULL, component_name VARCHAR(255) NOT NULL, cluster_id NUMERIC(19) NOT NULL, desired_stack_id NUMERIC(19) NOT NULL, desired_state VARCHAR(255) NOT NULL, service_name VARCHAR(255) NOT NULL, - PRIMARY KEY (component_name, cluster_id, service_name) + CONSTRAINT pk_servicecomponentdesiredstate PRIMARY KEY (id), + CONSTRAINT unq_scdesiredstate_name UNIQUE(component_name,service_name,cluster_id) ); +CREATE INDEX idx_sc_desired_state ON servicecomponentdesiredstate(component_name, service_name, cluster_id); + CREATE TABLE servicedesiredstate ( cluster_id NUMERIC(19) NOT NULL, desired_host_role_mapping INTEGER NOT NULL, @@ -548,6 +552,11 @@ CREATE TABLE repo_version ( display_name VARCHAR(128) NOT NULL, upgrade_package VARCHAR(255) NOT NULL, repositories TEXT NOT NULL, + repo_type VARCHAR(255) DEFAULT 'STANDARD' NOT NULL, + version_url VARCHAR(1024), + version_xml TEXT, + version_xsd VARCHAR(512), + parent_id NUMERIC(19), PRIMARY KEY(repo_version_id) ); @@ -986,6 +995,7 @@ INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_re INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_host_group_id_seq', 0); INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('setting_id_seq', 0); INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('hostcomponentstate_id_seq', 0); +INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('servicecomponentdesiredstate_id_seq', 0); insert into adminresourcetype (resource_type_id, resource_type_name) select 1, 'AMBARI' diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql index 324c24dc12..e36519eb70 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql @@ -181,13 +181,18 @@ CREATE TABLE hoststate ( PRIMARY KEY CLUSTERED (host_id)); CREATE TABLE servicecomponentdesiredstate ( + id BIGINT NOT NULL, component_name VARCHAR(255) NOT NULL, cluster_id BIGINT NOT NULL, desired_stack_id BIGINT NOT NULL, desired_state VARCHAR(255) NOT NULL, service_name VARCHAR(255) NOT NULL, - PRIMARY KEY CLUSTERED (component_name, cluster_id, service_name) - ); + PRIMARY KEY CLUSTERED (id), + CONSTRAINT pk_servicecomponentdesiredstate PRIMARY KEY (id), + CONSTRAINT unq_scdesiredstate_name UNIQUE(component_name,service_name,cluster_id) +); + +CREATE NONCLUSTERED INDEX idx_sc_desired_state ON servicecomponentdesiredstate(component_name, service_name, cluster_id); CREATE TABLE servicedesiredstate ( cluster_id BIGINT NOT NULL, @@ -652,6 +657,11 @@ CREATE TABLE repo_version ( version VARCHAR(255) NOT NULL, display_name VARCHAR(128) NOT NULL, repositories VARCHAR(MAX) NOT NULL, + repo_type VARCHAR(255) DEFAULT 'STANDARD' NOT NULL, + version_url VARCHAR(1024), + version_xml VARCHAR(MAX), + version_xsd VARCHAR(512), + parent_id BIGINT, PRIMARY KEY CLUSTERED (repo_version_id) ); @@ -1101,7 +1111,8 @@ BEGIN TRANSACTION ('topology_request_id_seq', 0), ('topology_host_group_id_seq', 0), ('setting_id_seq', 0), - ('hostcomponentstate_id_seq', 0); + ('hostcomponentstate_id_seq', 0), + ('servicecomponentdesiredstate_id_seq', 0); insert into adminresourcetype (resource_type_id, resource_type_name) values diff --git a/ambari-server/src/main/resources/version_definition.xsd b/ambari-server/src/main/resources/version_definition.xsd new file mode 100644 index 0000000000..77b4203423 --- /dev/null +++ b/ambari-server/src/main/resources/version_definition.xsd @@ -0,0 +1,187 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> + <xs:annotation> + <xs:documentation> + This XSD is used to validate a version definition file. You can verify the XML is valid + by running (on Linux): + xmllint --noout --load-trace --schema [path-to-this-file] [path-to-xml] + </xs:documentation> + </xs:annotation> + + <xs:complexType name="release-type"> + <xs:all> + <xs:element name="type" type="repo-type" /> + <xs:element name="stack-id" type="xs:string" /> + <xs:element name="version" type="xs:string" /> + <xs:element name="build" type="xs:string" /> + <xs:element name="compatible-with" type="xs:string" minOccurs="0"/> + <xs:element name="release-notes" type="xs:string" maxOccurs="1" /> + <xs:element name="display" type="xs:string" minOccurs="0" /> + <xs:element name="package-version" type="xs:string" minOccurs="0" /> + </xs:all> + </xs:complexType> + + <xs:simpleType name="repo-type"> + <xs:restriction base="xs:string"> + <xs:enumeration value="STANDARD" /> + <xs:enumeration value="SERVICE" /> + <xs:enumeration value="PATCH" /> + </xs:restriction> + </xs:simpleType> + + <xs:simpleType name="family-type"> + <xs:restriction base="xs:string"> + <xs:enumeration value="redhat6" /> + <xs:enumeration value="redhat7" /> + <xs:enumeration value="debian7" /> + <xs:enumeration value="ubuntu12" /> + <xs:enumeration value="ubuntu14" /> + <xs:enumeration value="suse11" /> + </xs:restriction> + </xs:simpleType> + + <xs:complexType name="manifest-service-type"> + <xs:annotation> + <xs:documentation> + The manifest element describes what services are included in the repository. This + is an exhaustive list and is NOT representative of what a cluster is allowed to + use for upgrading. + </xs:documentation> + </xs:annotation> + <xs:sequence> + <xs:element name="service" minOccurs="1" maxOccurs="unbounded"> + <xs:annotation> + <xs:documentation> + A service includes both the name and version represented in the repository. + An ID is also required as a reference elsewhere in the document. + </xs:documentation> + </xs:annotation> + <xs:complexType> + <xs:attribute name="name" type="xs:string" use="required" /> + <xs:attribute name="version" type="xs:string" use="required" /> + <xs:attribute name="id" type="xs:string" use="required" /> + <xs:attribute name="version-id" type="xs:string" /> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + + <xs:complexType name="available-services-type"> + <xs:annotation> + <xs:documentation> + Provides a list of services that are available to upgrade out of this repository. + A service may include a list of components that can be upgraded. These are specified + (generally) for patch upgrades only. + + A service must have an 'idref' attribute to tie it back to a service and version from + the 'manifest' element. + </xs:documentation> + </xs:annotation> + <xs:sequence> + <xs:element name="service" maxOccurs="unbounded"> + <xs:complexType> + <xs:sequence> + <xs:element name="component" minOccurs="0" maxOccurs="unbounded" /> + </xs:sequence> + <xs:attribute name="idref" use="required" type="xs:string" /> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + + <xs:complexType name="repository-info-type"> + <xs:sequence> + <xs:element name="os" maxOccurs="unbounded"> + <xs:complexType> + <xs:sequence> + <xs:element name="repo" maxOccurs="unbounded"> + <xs:complexType> + <xs:sequence> + <xs:element name="baseurl" type="xs:string" /> + <xs:element name="repoid" type="xs:string" /> + <xs:element name="reponame" type="xs:string" /> + </xs:sequence> + </xs:complexType> + </xs:element> + </xs:sequence> + <xs:attribute name="family" type="family-type" use="required" /> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + + <xs:complexType name="upgrade-type"> + <xs:sequence> + <xs:element name="configuration" maxOccurs="unbounded"> + <xs:complexType> + <xs:sequence> + <xs:element name="set" maxOccurs="unbounded"> + <xs:complexType> + <xs:attribute name="key" use="required" /> + <xs:attribute name="value" use="required" /> + </xs:complexType> + </xs:element> + </xs:sequence> + <xs:attribute name="type" use="required" /> + </xs:complexType> + </xs:element> + </xs:sequence> + </xs:complexType> + + <xs:element name="repository-version"> + <xs:annotation> + <xs:documentation> + This is the root element for the xml file. + </xs:documentation> + </xs:annotation> + <xs:complexType> + <xs:sequence> + <xs:element name="release" type="release-type" /> + <xs:element name="manifest" type="manifest-service-type" /> + <xs:element name="available-services" type="available-services-type" /> + <xs:element name="repository-info" type="repository-info-type"/> + <xs:element name="upgrade" minOccurs="0" type="upgrade-type" /> + </xs:sequence> + </xs:complexType> + + <xs:key name="service-id-key"> + <xs:annotation> + <xs:documentation> + This key is used to identify services used in other elements to the services + from the 'manifest' element. + </xs:documentation> + </xs:annotation> + <xs:selector xpath="./manifest/service" /> + <xs:field xpath="@id" /> + </xs:key> + + <xs:keyref name="available-services-id-keyref" refer="service-id-key"> + <xs:annotation> + <xs:documentation> + The key reference for 'available-services' to the 'manifest' service. + </xs:documentation> + </xs:annotation> + <xs:selector xpath="./available-services/service" /> + <xs:field xpath="@idref" /> + </xs:keyref> + + </xs:element> + +</xs:schema> diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java index 4529554467..433eee1ad7 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheckTest.java @@ -207,4 +207,61 @@ public class HostsRepositoryVersionCheckTest { Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus()); } + @Test + public void testPerformWithVersionNotRequired() throws Exception { + final HostsRepositoryVersionCheck hostsRepositoryVersionCheck = new HostsRepositoryVersionCheck(); + hostsRepositoryVersionCheck.clustersProvider = new Provider<Clusters>() { + + @Override + public Clusters get() { + return clusters; + } + }; + hostsRepositoryVersionCheck.repositoryVersionDaoProvider = new Provider<RepositoryVersionDAO>() { + @Override + public RepositoryVersionDAO get() { + return repositoryVersionDAO; + } + }; + hostsRepositoryVersionCheck.hostVersionDaoProvider = new Provider<HostVersionDAO>() { + @Override + public HostVersionDAO get() { + return hostVersionDAO; + } + }; + + final Cluster cluster = Mockito.mock(Cluster.class); + Mockito.when(cluster.getClusterId()).thenReturn(1L); + Mockito.when(cluster.getDesiredStackVersion()).thenReturn(new StackId()); + Mockito.when(clusters.getCluster("cluster")).thenReturn(cluster); + final Map<String, Host> hosts = new HashMap<String, Host>(); + final Host host1 = Mockito.mock(Host.class); + final Host host2 = Mockito.mock(Host.class); + final Host host3 = Mockito.mock(Host.class); + Mockito.when(host1.getMaintenanceState(1L)).thenReturn(MaintenanceState.OFF); + Mockito.when(host2.getMaintenanceState(1L)).thenReturn(MaintenanceState.OFF); + Mockito.when(host3.getMaintenanceState(1L)).thenReturn(MaintenanceState.OFF); + hosts.put("host1", host1); + hosts.put("host2", host2); + hosts.put("host3", host3); + Mockito.when(clusters.getHostsForCluster("cluster")).thenReturn(hosts); + + RepositoryVersionEntity rve = new RepositoryVersionEntity(); + rve.setVersion("1.1.1"); + + HostVersionEntity hve = new HostVersionEntity(); + hve.setRepositoryVersion(rve); + hve.setState(RepositoryVersionState.NOT_REQUIRED); + + Mockito.when( + hostVersionDAO.findByHost(Mockito.anyString())).thenReturn( + Collections.singletonList(hve)); + + PrerequisiteCheck check = new PrerequisiteCheck(null, null); + PrereqCheckRequest request = new PrereqCheckRequest("cluster"); + request.setRepositoryVersion("1.1.1"); + hostsRepositoryVersionCheck.perform(check, request); + Assert.assertEquals(PrereqCheckStatus.PASS, check.getStatus()); + } + } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java index eec1075512..c6d0c57cfe 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java @@ -29,8 +29,11 @@ import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; +import java.io.File; +import java.io.FileInputStream; import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; @@ -50,8 +53,8 @@ import org.apache.ambari.server.agent.CommandReport; import org.apache.ambari.server.agent.ExecutionCommand; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; -import org.apache.ambari.server.controller.AmbariActionExecutionHelper; import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.ExecuteActionRequest; import org.apache.ambari.server.controller.RequestStatusResponse; import org.apache.ambari.server.controller.ResourceProviderFactory; import org.apache.ambari.server.controller.spi.Request; @@ -61,16 +64,15 @@ import org.apache.ambari.server.controller.spi.ResourceProvider; import org.apache.ambari.server.controller.utilities.PropertyHelper; import org.apache.ambari.server.orm.GuiceJpaInitializer; import org.apache.ambari.server.orm.InMemoryDefaultTestModule; -import org.apache.ambari.server.orm.dao.ClusterDAO; import org.apache.ambari.server.orm.dao.ClusterVersionDAO; import org.apache.ambari.server.orm.dao.HostComponentStateDAO; -import org.apache.ambari.server.orm.dao.HostDAO; import org.apache.ambari.server.orm.dao.HostVersionDAO; import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; import org.apache.ambari.server.orm.dao.ResourceTypeDAO; import org.apache.ambari.server.orm.dao.StackDAO; import org.apache.ambari.server.orm.entities.ClusterEntity; import org.apache.ambari.server.orm.entities.ClusterVersionEntity; +import org.apache.ambari.server.orm.entities.HostVersionEntity; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.orm.entities.ResourceEntity; import org.apache.ambari.server.orm.entities.ResourceTypeEntity; @@ -82,8 +84,10 @@ import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.MaintenanceState; +import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.RepositoryVersionState; import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.ServiceComponent; import org.apache.ambari.server.state.ServiceComponentHost; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.ServiceOsSpecific; @@ -91,6 +95,7 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.cluster.ClusterImpl; import org.apache.ambari.server.topology.TopologyManager; import org.apache.ambari.server.utils.StageUtils; +import org.apache.commons.io.IOUtils; import org.easymock.Capture; import org.easymock.EasyMock; import org.easymock.IAnswer; @@ -117,18 +122,15 @@ public class ClusterStackVersionResourceProviderTest { private RepositoryVersionDAO repositoryVersionDAOMock; private ResourceTypeDAO resourceTypeDAO; private StackDAO stackDAO; - private ClusterDAO clusterDAO; private ClusterVersionDAO clusterVersionDAO; - private HostDAO hostDAO; private ConfigHelper configHelper; private Configuration configuration; private StageFactory stageFactory; - private AmbariActionExecutionHelper actionExecutionHelper; private HostVersionDAO hostVersionDAO; private HostComponentStateDAO hostComponentStateDAO; - private String operatingSystemsJson = "[\n" + + private static final String OS_JSON = "[\n" + " {\n" + " \"repositories\":[\n" + " {\n" + @@ -146,7 +148,6 @@ public class ClusterStackVersionResourceProviderTest { " }\n" + "]"; - @Before public void setup() throws Exception { // Create instances of mocks @@ -169,9 +170,6 @@ public class ClusterStackVersionResourceProviderTest { ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class); resourceTypeDAO = injector.getInstance(ResourceTypeDAO.class); stackDAO = injector.getInstance(StackDAO.class); - clusterDAO = injector.getInstance(ClusterDAO.class); - hostDAO = injector.getInstance(HostDAO.class); - } @After @@ -188,6 +186,10 @@ public class ClusterStackVersionResourceProviderTest { Cluster cluster = createNiceMock(Cluster.class); StackId stackId = new StackId("HDP", "2.0.1"); + RepositoryVersionEntity repoVersion = new RepositoryVersionEntity(); + repoVersion.setId(1l); + repoVersion.setOperatingSystems(OS_JSON); + Map<String, Host> hostsForCluster = new HashMap<String, Host>(); int hostCount = 10; for (int i = 0; i < hostCount; i++) { @@ -197,6 +199,8 @@ public class ClusterStackVersionResourceProviderTest { expect(host.getOsFamily()).andReturn("redhat6").anyTimes(); expect(host.getMaintenanceState(EasyMock.anyLong())).andReturn( MaintenanceState.OFF).anyTimes(); + expect(host.getAllHostVersions()).andReturn( + Collections.<HostVersionEntity>emptyList()).anyTimes(); replay(host); hostsForCluster.put(hostname, host); @@ -222,9 +226,6 @@ public class ClusterStackVersionResourceProviderTest { add(schAMS); }}; - RepositoryVersionEntity repoVersion = new RepositoryVersionEntity(); - repoVersion.setId(1l); - repoVersion.setOperatingSystems(operatingSystemsJson); ServiceOsSpecific.Package hdfsPackage = new ServiceOsSpecific.Package(); hdfsPackage.setName("hdfs"); @@ -350,6 +351,228 @@ public class ClusterStackVersionResourceProviderTest { // check that the success factor was populated in the stage Float successFactor = successFactors.get(Role.INSTALL_PACKAGES); Assert.assertEquals(Float.valueOf(0.85f), successFactor); + + + } + + @Test + public void testCreateResourcesForPatch() throws Exception { + Resource.Type type = Resource.Type.ClusterStackVersion; + + AmbariManagementController managementController = createMock(AmbariManagementController.class); + Clusters clusters = createNiceMock(Clusters.class); + Cluster cluster = createNiceMock(Cluster.class); + StackId stackId = new StackId("HDP", "2.0.1"); + + File f = new File("src/test/resources/hbase_version_test.xml"); + + RepositoryVersionEntity repoVersion = new RepositoryVersionEntity(); + repoVersion.setId(1l); + repoVersion.setOperatingSystems(OS_JSON); + repoVersion.setVersionXml(IOUtils.toString(new FileInputStream(f))); + repoVersion.setVersionXsd("version_definition.xsd"); + repoVersion.setType(RepositoryType.PATCH); + + ambariMetaInfo.getComponent("HDP", "2.1.1", "HBASE", "HBASE_MASTER").setVersionAdvertised(true); + + + Map<String, Host> hostsForCluster = new HashMap<String, Host>(); + int hostCount = 10; + for (int i = 0; i < hostCount; i++) { + String hostname = "host" + i; + Host host = createNiceMock(hostname, Host.class); + expect(host.getHostName()).andReturn(hostname).anyTimes(); + expect(host.getOsFamily()).andReturn("redhat6").anyTimes(); + expect(host.getMaintenanceState(EasyMock.anyLong())).andReturn( + MaintenanceState.OFF).anyTimes(); + expect(host.getAllHostVersions()).andReturn( + Collections.<HostVersionEntity>emptyList()).anyTimes(); + + replay(host); + hostsForCluster.put(hostname, host); + } + + Service hdfsService = createNiceMock(Service.class); + Service hbaseService = createNiceMock(Service.class); + expect(hdfsService.getName()).andReturn("HDFS").anyTimes(); + expect(hbaseService.getName()).andReturn("HBASE").anyTimes(); +// Service metricsService = createNiceMock(Service.class); + + ServiceComponent scNameNode = createNiceMock(ServiceComponent.class); + ServiceComponent scDataNode = createNiceMock(ServiceComponent.class); + ServiceComponent scHBaseMaster = createNiceMock(ServiceComponent.class); + ServiceComponent scMetricCollector = createNiceMock(ServiceComponent.class); + + expect(hdfsService.getServiceComponents()).andReturn(new HashMap<String, ServiceComponent>()); + expect(hbaseService.getServiceComponents()).andReturn(new HashMap<String, ServiceComponent>()); +// expect(metricsService.getServiceComponents()).andReturn(new HashMap<String, ServiceComponent>()); + + + Map<String, Service> serviceMap = new HashMap<>(); + serviceMap.put("HDFS", hdfsService); + serviceMap.put("HBASE", hbaseService); + + + final ServiceComponentHost schDatanode = createMock(ServiceComponentHost.class); + expect(schDatanode.getServiceName()).andReturn("HDFS").anyTimes(); + expect(schDatanode.getServiceComponentName()).andReturn("DATANODE").anyTimes(); + + final ServiceComponentHost schNamenode = createMock(ServiceComponentHost.class); + expect(schNamenode.getServiceName()).andReturn("HDFS").anyTimes(); + expect(schNamenode.getServiceComponentName()).andReturn("NAMENODE").anyTimes(); + + final ServiceComponentHost schAMS = createMock(ServiceComponentHost.class); + expect(schAMS.getServiceName()).andReturn("AMBARI_METRICS").anyTimes(); + expect(schAMS.getServiceComponentName()).andReturn("METRICS_COLLECTOR").anyTimes(); + + final ServiceComponentHost schHBM = createMock(ServiceComponentHost.class); + expect(schHBM.getServiceName()).andReturn("HBASE").anyTimes(); + expect(schHBM.getServiceComponentName()).andReturn("HBASE_MASTER").anyTimes(); + + // First host contains versionable components + final List<ServiceComponentHost> schsH1 = Arrays.asList(schDatanode, schNamenode, schAMS); + + // Second host does not contain versionable components + final List<ServiceComponentHost> schsH2 = Arrays.asList(schAMS); + + // Third host only has hbase + final List<ServiceComponentHost> schsH3 = Arrays.asList(schHBM); + + ServiceOsSpecific.Package hdfsPackage = new ServiceOsSpecific.Package(); + hdfsPackage.setName("hdfs"); + +// ServiceOsSpecific.Package hbasePackage = new ServiceOsSpecific.Package(); +// hbasePackage.setName("hbase"); + + List<ServiceOsSpecific.Package> packages = Collections.singletonList(hdfsPackage); + + ActionManager actionManager = createNiceMock(ActionManager.class); + + RequestStatusResponse response = createNiceMock(RequestStatusResponse.class); + ResourceProviderFactory resourceProviderFactory = createNiceMock(ResourceProviderFactory.class); + ResourceProvider csvResourceProvider = createNiceMock(ClusterStackVersionResourceProvider.class); + + AbstractControllerResourceProvider.init(resourceProviderFactory); + + Map<String, Map<String, String>> hostConfigTags = new HashMap<String, Map<String, String>>(); + expect(configHelper.getEffectiveDesiredTags(anyObject(ClusterImpl.class), anyObject(String.class))).andReturn(hostConfigTags); + + expect(managementController.getClusters()).andReturn(clusters).anyTimes(); + expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); + expect(managementController.getAuthName()).andReturn("admin").anyTimes(); + expect(managementController.getActionManager()).andReturn(actionManager).anyTimes(); + expect(managementController.getJdkResourceUrl()).andReturn("/JdkResourceUrl").anyTimes(); + expect(managementController.getPackagesForServiceHost(anyObject(ServiceInfo.class), + (Map<String, String>) anyObject(List.class), anyObject(String.class))). + andReturn(packages).times(1); // only one host has the versionable component + + expect(resourceProviderFactory.getHostResourceProvider(anyObject(Set.class), anyObject(Map.class), + eq(managementController))).andReturn(csvResourceProvider).anyTimes(); + + expect(clusters.getCluster(anyObject(String.class))).andReturn(cluster); + expect(clusters.getHostsForCluster(anyObject(String.class))).andReturn( + hostsForCluster).anyTimes(); + + String clusterName = "Cluster100"; + expect(cluster.getClusterId()).andReturn(1L).anyTimes(); + expect(cluster.getHosts()).andReturn(hostsForCluster.values()).atLeastOnce(); + expect(cluster.getServices()).andReturn(serviceMap).anyTimes(); + expect(cluster.getCurrentStackVersion()).andReturn(stackId); + expect(cluster.getServiceComponentHosts(anyObject(String.class))).andAnswer(new IAnswer<List<ServiceComponentHost>>() { + @Override + public List<ServiceComponentHost> answer() throws Throwable { + String hostname = (String) EasyMock.getCurrentArguments()[0]; + if (hostname.equals("host2")) { + return schsH2; + } else if (hostname.equals("host3")) { + return schsH3; + } else { + return schsH1; + } + } + }).anyTimes(); + + ExecutionCommand executionCommand = createNiceMock(ExecutionCommand.class); + ExecutionCommandWrapper executionCommandWrapper = createNiceMock(ExecutionCommandWrapper.class); + + expect(executionCommandWrapper.getExecutionCommand()).andReturn(executionCommand).anyTimes(); + + Stage stage = createNiceMock(Stage.class); + expect(stage.getExecutionCommandWrapper(anyObject(String.class), anyObject(String.class))). + andReturn(executionCommandWrapper).anyTimes(); + + Map<Role, Float> successFactors = new HashMap<>(); + expect(stage.getSuccessFactors()).andReturn(successFactors).atLeastOnce(); + + // Check that we create proper stage count + expect(stageFactory.createNew(anyLong(), anyObject(String.class), + anyObject(String.class), anyLong(), + anyObject(String.class), anyObject(String.class), anyObject(String.class), + anyObject(String.class))).andReturn(stage). + times((int) Math.ceil(hostCount / MAX_TASKS_PER_STAGE)); + + expect( + repositoryVersionDAOMock.findByStackAndVersion( + anyObject(StackId.class), + anyObject(String.class))).andReturn(repoVersion); + + Capture<org.apache.ambari.server.actionmanager.Request> c = Capture.newInstance(); + Capture<ExecuteActionRequest> ear = Capture.newInstance(); + + actionManager.sendActions(capture(c), capture(ear)); + expectLastCall().atLeastOnce(); + expect(actionManager.getRequestTasks(anyLong())).andReturn(Collections.<HostRoleCommand>emptyList()).anyTimes(); + + ClusterEntity clusterEntity = new ClusterEntity(); + clusterEntity.setClusterId(1l); + clusterEntity.setClusterName(clusterName); + ClusterVersionEntity cve = new ClusterVersionEntity(clusterEntity, + repoVersion, RepositoryVersionState.INSTALL_FAILED, 0, ""); + expect(clusterVersionDAO.findByClusterAndStackAndVersion(anyObject(String.class), + anyObject(StackId.class), anyObject(String.class))).andReturn(cve); + + TopologyManager topologyManager = injector.getInstance(TopologyManager.class); + StageUtils.setTopologyManager(topologyManager); + + // replay + replay(managementController, response, clusters, hdfsService, hbaseService, resourceProviderFactory, csvResourceProvider, + cluster, repositoryVersionDAOMock, configHelper, schDatanode, schNamenode, schAMS, schHBM, actionManager, + executionCommand, executionCommandWrapper,stage, stageFactory, clusterVersionDAO); + + ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( + type, + PropertyHelper.getPropertyIds(type), + PropertyHelper.getKeyPropertyIds(type), + managementController); + + injector.injectMembers(provider); + + // add the property map to a set for the request. add more maps for multiple creates + Set<Map<String, Object>> propertySet = new LinkedHashSet<Map<String, Object>>(); + + Map<String, Object> properties = new LinkedHashMap<String, Object>(); + + // add properties to the request map + properties.put(ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID, "Cluster100"); + properties.put(ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_REPOSITORY_VERSION_PROPERTY_ID, "2.2.0.1-885"); + properties.put(ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_STACK_PROPERTY_ID, "HDP"); + properties.put(ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_VERSION_PROPERTY_ID, "2.1.1"); + + propertySet.add(properties); + + // create the request + Request request = PropertyHelper.getCreateRequest(propertySet, null); + + RequestStatus status = provider.createResources(request); + Assert.assertNotNull(status); + + // verify + verify(managementController, response, clusters, stageFactory, stage); + + // check that the success factor was populated in the stage + Float successFactor = successFactors.get(Role.INSTALL_PACKAGES); + Assert.assertEquals(Float.valueOf(0.85f), successFactor); + } @@ -400,7 +623,7 @@ public class ClusterStackVersionResourceProviderTest { expect(clusters.getCluster(anyObject(String.class))).andReturn(cluster); RepositoryVersionEntity repoVersion = new RepositoryVersionEntity(); - repoVersion.setOperatingSystems(operatingSystemsJson); + repoVersion.setOperatingSystems(OS_JSON); StackEntity newDesiredStack = stackDAO.find("HDP", "2.0.1"); repoVersion.setStack(newDesiredStack); @@ -546,7 +769,7 @@ public class ClusterStackVersionResourceProviderTest { expect(clusters.getCluster(anyObject(String.class))).andReturn(cluster); RepositoryVersionEntity repoVersion = new RepositoryVersionEntity(); - repoVersion.setOperatingSystems(operatingSystemsJson); + repoVersion.setOperatingSystems(OS_JSON); StackEntity newDesiredStack = stackDAO.find("HDP", "2.0.1"); repoVersion.setStack(newDesiredStack); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProviderTest.java index 22cb3c3fe0..e031fc8a0f 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RepositoryVersionResourceProviderTest.java @@ -52,6 +52,7 @@ import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.OperatingSystemInfo; import org.apache.ambari.server.state.RepositoryInfo; import org.apache.ambari.server.state.RepositoryVersionState; +import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.state.stack.UpgradePack; @@ -155,12 +156,18 @@ public class RepositoryVersionResourceProviderTest { map.put("pack2", pack2); return map; } + + @Override + public ServiceInfo getService(String name) { + return new ServiceInfo(); + } + }; stackInfo.setName("HDP"); stackInfo.setVersion("1.1"); stacks.add(stackInfo); - Mockito.when(ambariMetaInfo.getStack(Mockito.anyString(), Mockito.anyString())).thenAnswer(new Answer<StackInfo>() { + Mockito.when(ambariMetaInfo.getStack(Mockito.anyString(), Mockito.anyString())).thenAnswer(new Answer<StackInfo>() { @Override public StackInfo answer(InvocationOnMock invocation) throws Throwable { final String stack = invocation.getArguments()[0].toString(); @@ -173,6 +180,7 @@ public class RepositoryVersionResourceProviderTest { } }); + Mockito.when(ambariMetaInfo.getStacks()).thenReturn(stacks); Mockito.when(ambariMetaInfo.getUpgradePacks(Mockito.anyString(), Mockito.anyString())).thenAnswer(new Answer<Map<String, UpgradePack>>() { @@ -266,6 +274,9 @@ public class RepositoryVersionResourceProviderTest { Assert.assertEquals(1, provider.getResources(getRequest, new AndPredicate(predicateStackName, predicateStackVersion)).size()); } + + + @Test public void testGetResourcesAsAdministrator() throws Exception { testGetResources(TestAuthenticationFactory.createAdministrator()); @@ -309,36 +320,37 @@ public class RepositoryVersionResourceProviderTest { StackEntity stackEntity = stackDAO.find("HDP", "1.1"); Assert.assertNotNull(stackEntity); - final RepositoryVersionResourceProvider provider = (RepositoryVersionResourceProvider) injector.getInstance(ResourceProviderFactory.class).getRepositoryVersionResourceProvider(); - final RepositoryVersionEntity entity = new RepositoryVersionEntity(); entity.setDisplayName("name"); entity.setStack(stackEntity); entity.setVersion("1.1"); entity.setOperatingSystems("[{\"OperatingSystems/os_type\":\"redhat6\",\"repositories\":[{\"Repositories/repo_id\":\"1\",\"Repositories/repo_name\":\"1\",\"Repositories/base_url\":\"http://example.com/repo1\"}]}]"); + final RepositoryVersionDAO repositoryVersionDAO = injector.getInstance(RepositoryVersionDAO.class); + AmbariMetaInfo info = injector.getInstance(AmbariMetaInfo.class); + // test valid usecases - provider.validateRepositoryVersion(entity); + RepositoryVersionResourceProvider.validateRepositoryVersion(repositoryVersionDAO, info, entity); entity.setVersion("1.1-17"); - provider.validateRepositoryVersion(entity); + RepositoryVersionResourceProvider.validateRepositoryVersion(repositoryVersionDAO, info, entity); entity.setVersion("1.1.1.1"); - provider.validateRepositoryVersion(entity); + RepositoryVersionResourceProvider.validateRepositoryVersion(repositoryVersionDAO, info, entity); entity.setVersion("1.1.343432.2"); - provider.validateRepositoryVersion(entity); + RepositoryVersionResourceProvider.validateRepositoryVersion(repositoryVersionDAO, info, entity); entity.setVersion("1.1.343432.2-234234324"); - provider.validateRepositoryVersion(entity); + RepositoryVersionResourceProvider.validateRepositoryVersion(repositoryVersionDAO, info, entity); // test invalid usecases entity.setOperatingSystems(jsonStringRedhat7); try { - provider.validateRepositoryVersion(entity); + RepositoryVersionResourceProvider.validateRepositoryVersion(repositoryVersionDAO, info, entity); Assert.fail("Should throw exception"); } catch (Exception ex) { } entity.setOperatingSystems(""); try { - provider.validateRepositoryVersion(entity); + RepositoryVersionResourceProvider.validateRepositoryVersion(repositoryVersionDAO, info, entity); Assert.fail("Should throw exception"); } catch (Exception ex) { } @@ -347,12 +359,12 @@ public class RepositoryVersionResourceProviderTest { stackEntity.setStackName("BIGTOP"); entity.setStack(bigtop); try { - provider.validateRepositoryVersion(entity); + RepositoryVersionResourceProvider.validateRepositoryVersion(repositoryVersionDAO, info, entity); Assert.fail("Should throw exception"); } catch (Exception ex) { } - final RepositoryVersionDAO repositoryVersionDAO = injector.getInstance(RepositoryVersionDAO.class); + entity.setDisplayName("name"); entity.setStack(stackEntity); entity.setVersion("1.1"); @@ -367,7 +379,7 @@ public class RepositoryVersionResourceProviderTest { entity2.setOperatingSystems("[{\"OperatingSystems/os_type\":\"redhat6\",\"repositories\":[{\"Repositories/repo_id\":\"1\",\"Repositories/repo_name\":\"1\",\"Repositories/base_url\":\"http://example.com/repo1\"}]}]"); try { - provider.validateRepositoryVersion(entity2); + RepositoryVersionResourceProvider.validateRepositoryVersion(repositoryVersionDAO, info, entity2); Assert.fail("Should throw exception: Base url http://example.com/repo1 is already defined for another repository version"); } catch (Exception ex) { } @@ -522,6 +534,7 @@ public class RepositoryVersionResourceProviderTest { Assert.assertEquals(false, RepositoryVersionEntity.isVersionInStack(sid2, "2.1")); } + @After public void after() { injector.getInstance(PersistService.class).stop(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java new file mode 100644 index 0000000000..efdf84e018 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java @@ -0,0 +1,240 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.controller.internal; + +import java.io.File; +import java.io.FileInputStream; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.controller.ResourceProviderFactory; +import org.apache.ambari.server.controller.predicate.AndPredicate; +import org.apache.ambari.server.controller.spi.Predicate; +import org.apache.ambari.server.controller.spi.Request; +import org.apache.ambari.server.controller.spi.RequestStatus; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.spi.ResourceProvider; +import org.apache.ambari.server.controller.utilities.PredicateBuilder; +import org.apache.ambari.server.controller.utilities.PropertyHelper; +import org.apache.ambari.server.orm.GuiceJpaInitializer; +import org.apache.ambari.server.orm.InMemoryDefaultTestModule; +import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; +import org.apache.ambari.server.orm.dao.StackDAO; +import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; +import org.apache.ambari.server.orm.entities.StackEntity; +import org.apache.ambari.server.security.TestAuthenticationFactory; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.io.IOUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.persist.PersistService; + +/** + * Tests the VersionDefinitionResourceProvider class + */ +public class VersionDefinitionResourceProviderTest { + private Injector injector; + + @Before + public void before() throws Exception { + injector = Guice.createInjector(new InMemoryDefaultTestModule()); + injector.getInstance(GuiceJpaInitializer.class); + + AmbariMetaInfo ami = injector.getInstance(AmbariMetaInfo.class); + ami.init(); + + StackDAO stackDao = injector.getInstance(StackDAO.class); + StackEntity stack = stackDao.find("HDP", "2.2.0"); + + RepositoryVersionEntity entity = new RepositoryVersionEntity(); + entity.setStack(stack); + entity.setDisplayName("2.2.0.0"); + entity.setVersion("2.3.4.4-1234"); + + RepositoryVersionDAO dao = injector.getInstance(RepositoryVersionDAO.class); + dao.create(entity); + + } + + @After + public void after() throws Exception { + injector.getInstance(PersistService.class).stop(); + } + + @Test + public void testWithParentFromBase64() throws Exception { + Authentication authentication = TestAuthenticationFactory.createAdministrator(); + SecurityContextHolder.getContext().setAuthentication(authentication); + + File file = new File("src/test/resources/version_definition_resource_provider.xml"); + + byte[] bytes = IOUtils.toByteArray(new FileInputStream(file)); + String base64Str = Base64.encodeBase64String(bytes); + + final ResourceProvider versionProvider = new VersionDefinitionResourceProvider(); + final ResourceProvider provider = injector.getInstance(ResourceProviderFactory.class) + .getRepositoryVersionResourceProvider(); + + final Set<Map<String, Object>> propertySet = new LinkedHashSet<Map<String, Object>>(); + final Map<String, Object> properties = new LinkedHashMap<String, Object>(); + properties.put(VersionDefinitionResourceProvider.VERSION_DEF_DEFINITION_BASE64, base64Str); + propertySet.add(properties); + + + final Request createRequest = PropertyHelper.getCreateRequest(propertySet, null); + RequestStatus status = versionProvider.createResources(createRequest); + Assert.assertEquals(1, status.getAssociatedResources().size()); + + Request getRequest = PropertyHelper.getReadRequest("VersionDefinition"); + Set<Resource> results = versionProvider.getResources(getRequest, null); + Assert.assertEquals(1, results.size()); + + final Predicate predicateStackName = new PredicateBuilder().property(RepositoryVersionResourceProvider.REPOSITORY_VERSION_STACK_NAME_PROPERTY_ID).equals("HDP").toPredicate(); + final Predicate predicateStackVersion = new PredicateBuilder().property(RepositoryVersionResourceProvider.REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID).equals("2.2.0").toPredicate(); + + results = provider.getResources(getRequest, + new AndPredicate(predicateStackName, predicateStackVersion)); + Assert.assertEquals(1, results.size()); + + getRequest = PropertyHelper.getReadRequest( + RepositoryVersionResourceProvider.REPOSITORY_VERSION_DISPLAY_NAME_PROPERTY_ID, + RepositoryVersionResourceProvider.REPOSITORY_VERSION_ID_PROPERTY_ID, + RepositoryVersionResourceProvider.REPOSITORY_VERSION_REPOSITORY_VERSION_PROPERTY_ID, + RepositoryVersionResourceProvider.REPOSITORY_VERSION_STACK_NAME_PROPERTY_ID, + RepositoryVersionResourceProvider.REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID, + RepositoryVersionResourceProvider.SUBRESOURCE_OPERATING_SYSTEMS_PROPERTY_ID, + RepositoryVersionResourceProvider.SUBRESOURCE_REPOSITORIES_PROPERTY_ID, + "RepositoryVersions/release", "RepositoryVersions/services", + "RepositoryVersions/has_children", "RepositoryVersions/parent_id"); + + results = provider.getResources(getRequest, + new AndPredicate(predicateStackName, predicateStackVersion)); + Assert.assertEquals(2, results.size()); + + Resource r = null; + for (Resource result : results) { + if (result.getPropertyValue("RepositoryVersions/repository_version").equals("2.2.0.8-5678")) { + r = result; + break; + } + } + + Assert.assertNotNull(r); + Map<String, Map<String, Object>> map = r.getPropertiesMap(); + Assert.assertTrue(map.containsKey("RepositoryVersions")); + + Map<String, Object> vals = map.get("RepositoryVersions"); + + Assert.assertEquals("2.2.0.8-5678", vals.get("repository_version")); + Assert.assertNotNull(vals.get("parent_id")); + Assert.assertEquals(Boolean.FALSE, vals.get("has_children")); + + + Assert.assertTrue(map.containsKey("RepositoryVersions/release")); + vals = map.get("RepositoryVersions/release"); + Assert.assertEquals("5678", vals.get("build")); + Assert.assertEquals("2.3.4.[1-9]", vals.get("compatible_with")); + Assert.assertEquals("http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.4/", + vals.get("notes")); + } + + @Test + public void testWithParent() throws Exception { + Authentication authentication = TestAuthenticationFactory.createAdministrator(); + SecurityContextHolder.getContext().setAuthentication(authentication); + + File file = new File("src/test/resources/version_definition_resource_provider.xml"); + + final ResourceProvider versionProvider = new VersionDefinitionResourceProvider(); + final ResourceProvider provider = injector.getInstance(ResourceProviderFactory.class) + .getRepositoryVersionResourceProvider(); + + final Set<Map<String, Object>> propertySet = new LinkedHashSet<Map<String, Object>>(); + final Map<String, Object> properties = new LinkedHashMap<String, Object>(); + properties.put(VersionDefinitionResourceProvider.VERSION_DEF_DEFINITION_URL, + file.toURI().toURL().toString()); + propertySet.add(properties); + + + final Request createRequest = PropertyHelper.getCreateRequest(propertySet, null); + RequestStatus status = versionProvider.createResources(createRequest); + Assert.assertEquals(1, status.getAssociatedResources().size()); + + Request getRequest = PropertyHelper.getReadRequest("VersionDefinition"); + Set<Resource> results = versionProvider.getResources(getRequest, null); + Assert.assertEquals(1, results.size()); + + final Predicate predicateStackName = new PredicateBuilder().property(RepositoryVersionResourceProvider.REPOSITORY_VERSION_STACK_NAME_PROPERTY_ID).equals("HDP").toPredicate(); + final Predicate predicateStackVersion = new PredicateBuilder().property(RepositoryVersionResourceProvider.REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID).equals("2.2.0").toPredicate(); + + results = provider.getResources(getRequest, + new AndPredicate(predicateStackName, predicateStackVersion)); + Assert.assertEquals(1, results.size()); + + getRequest = PropertyHelper.getReadRequest( + RepositoryVersionResourceProvider.REPOSITORY_VERSION_DISPLAY_NAME_PROPERTY_ID, + RepositoryVersionResourceProvider.REPOSITORY_VERSION_ID_PROPERTY_ID, + RepositoryVersionResourceProvider.REPOSITORY_VERSION_REPOSITORY_VERSION_PROPERTY_ID, + RepositoryVersionResourceProvider.REPOSITORY_VERSION_STACK_NAME_PROPERTY_ID, + RepositoryVersionResourceProvider.REPOSITORY_VERSION_STACK_VERSION_PROPERTY_ID, + RepositoryVersionResourceProvider.SUBRESOURCE_OPERATING_SYSTEMS_PROPERTY_ID, + RepositoryVersionResourceProvider.SUBRESOURCE_REPOSITORIES_PROPERTY_ID, + "RepositoryVersions/release", "RepositoryVersions/services", + "RepositoryVersions/has_children", "RepositoryVersions/parent_id"); + + results = provider.getResources(getRequest, + new AndPredicate(predicateStackName, predicateStackVersion)); + Assert.assertEquals(2, results.size()); + + Resource r = null; + for (Resource result : results) { + if (result.getPropertyValue("RepositoryVersions/repository_version").equals("2.2.0.8-5678")) { + r = result; + break; + } + } + + Assert.assertNotNull(r); + Map<String, Map<String, Object>> map = r.getPropertiesMap(); + Assert.assertTrue(map.containsKey("RepositoryVersions")); + + Map<String, Object> vals = map.get("RepositoryVersions"); + + Assert.assertEquals("2.2.0.8-5678", vals.get("repository_version")); + Assert.assertNotNull(vals.get("parent_id")); + Assert.assertEquals(Boolean.FALSE, vals.get("has_children")); + + + Assert.assertTrue(map.containsKey("RepositoryVersions/release")); + vals = map.get("RepositoryVersions/release"); + Assert.assertEquals("5678", vals.get("build")); + Assert.assertEquals("2.3.4.[1-9]", vals.get("compatible_with")); + Assert.assertEquals("http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.4/", + vals.get("notes")); + } +} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java index ddab65df63..4e8713b08b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java @@ -39,7 +39,6 @@ import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntityPK; import org.apache.ambari.server.orm.entities.HostComponentStateEntity; import org.apache.ambari.server.orm.entities.HostEntity; import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity; -import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntityPK; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -148,14 +147,8 @@ public class ServiceComponentTest { ServiceComponentDesiredStateDAO serviceComponentDesiredStateDAO = injector.getInstance(ServiceComponentDesiredStateDAO.class); - ServiceComponentDesiredStateEntityPK primaryKey = - new ServiceComponentDesiredStateEntityPK(); - primaryKey.setClusterId(cluster.getClusterId()); - primaryKey.setComponentName(componentName); - primaryKey.setServiceName(serviceName); - - ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity = - serviceComponentDesiredStateDAO.findByPK(primaryKey); + ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity = serviceComponentDesiredStateDAO.findByName( + cluster.getClusterId(), serviceName, componentName); ServiceComponent sc1 = serviceComponentFactory.createExisting(service, serviceComponentDesiredStateEntity); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java index eb5bf62083..b15157eecd 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java @@ -61,10 +61,12 @@ import org.apache.ambari.server.state.stack.upgrade.StageWrapper; import org.apache.ambari.server.state.stack.upgrade.Task; import org.apache.ambari.server.state.stack.upgrade.TaskWrapper; import org.apache.ambari.server.state.stack.upgrade.UpgradeType; +import org.apache.ambari.server.utils.EventBusSynchronizer; import org.easymock.EasyMock; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.springframework.security.core.context.SecurityContextHolder; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; @@ -74,7 +76,6 @@ import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.persist.PersistService; import com.google.inject.util.Modules; -import org.springframework.security.core.context.SecurityContextHolder; /** * Tests the {@link UpgradeHelper} class @@ -123,6 +124,8 @@ public class UpgradeHelperTest { // create an injector which will inject the mocks injector = Guice.createInjector(Modules.override(injectorModule).with(mockModule)); injector.getInstance(GuiceJpaInitializer.class); + EventBusSynchronizer.synchronizeAmbariEventPublisher(injector); + EventBusSynchronizer.synchronizeAlertEventPublisher(injector); helper = injector.getInstance(OrmTestHelper.class); ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class); @@ -257,6 +260,58 @@ public class UpgradeHelperTest { } @Test + public void testSupportedServiceUpgradeOrchestration() throws Exception { + Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar"); + assertTrue(upgrades.isEmpty()); + + upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); + + ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER"); + si.setDisplayName("Zk"); + ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER"); + ci.setDisplayName("ZooKeeper1 Server2"); + + assertTrue(upgrades.containsKey("upgrade_test")); + UpgradePack upgrade = upgrades.get("upgrade_test"); + assertNotNull(upgrade); + + makeCluster(); + + UpgradeContext context = new UpgradeContext(m_masterHostResolver, HDP_21, + HDP_21, UPGRADE_VERSION, Direction.UPGRADE, UpgradeType.ROLLING); + context.setSupportedServices(Collections.singleton("ZOOKEEPER")); + + + List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context); + + assertEquals(3, groups.size()); + + assertEquals("PRE_CLUSTER", groups.get(0).name); + assertEquals("ZOOKEEPER", groups.get(1).name); + assertEquals("POST_CLUSTER", groups.get(2).name); + + + UpgradeGroupHolder group = groups.get(1); + // check that the display name is being used + assertTrue(group.items.get(1).getText().contains("ZooKeeper1 Server2")); + assertEquals(group.items.get(5).getText(), "Service Check Zk"); + + UpgradeGroupHolder postGroup = groups.get(2); + assertEquals("POST_CLUSTER", postGroup.name); + assertEquals("Finalize Upgrade", postGroup.title); + assertEquals(3, postGroup.items.size()); + assertEquals("Confirm Finalize", postGroup.items.get(0).getText()); + assertEquals("Execute HDFS Finalize", postGroup.items.get(1).getText()); + assertEquals("Save Cluster State", postGroup.items.get(2).getText()); + assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType()); + + assertEquals(4, groups.get(0).items.size()); + assertEquals(6, groups.get(1).items.size()); + assertEquals(3, groups.get(2).items.size()); + } + + + @Test public void testUpgradeServerActionOrchestration() throws Exception { Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java new file mode 100644 index 0000000000..4ba59671e6 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state.repository; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.io.File; + +import org.apache.ambari.server.state.RepositoryType; +import org.apache.commons.io.FileUtils; +import org.junit.Test; + +/** + * Tests for repository definitions. + */ +public class VersionDefinitionTest { + + private static File file = new File("src/test/resources/version_definition_test.xml"); + + @Test + public void testLoadingString() throws Exception { + String xmlString = FileUtils.readFileToString(file); + VersionDefinitionXml xml = VersionDefinitionXml.load(xmlString); + + validateXml(xml); + } + + @Test + public void testLoadingUrl() throws Exception { + VersionDefinitionXml xml = VersionDefinitionXml.load(file.toURI().toURL()); + + validateXml(xml); + } + + private void validateXml(VersionDefinitionXml xml) throws Exception { + assertNotNull(xml.release); + assertEquals(RepositoryType.PATCH, xml.release.repositoryType); + assertEquals("HDP-2.3", xml.release.stackId); + assertEquals("2.3.4.1", xml.release.version); + assertEquals("2.3.4.[1-9]", xml.release.compatibleWith); + assertEquals("http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.4/", xml.release.releaseNotes); + + assertEquals(4, xml.manifestServices.size()); + assertEquals("HDFS-271", xml.manifestServices.get(0).serviceId); + assertEquals("HDFS", xml.manifestServices.get(0).serviceName); + assertEquals("2.7.1", xml.manifestServices.get(0).version); + assertEquals("10", xml.manifestServices.get(0).versionId); + + assertEquals(3, xml.availableServices.size()); + assertEquals("HDFS-271", xml.availableServices.get(0).serviceIdReference); + assertEquals(0, xml.availableServices.get(0).components.size()); + + assertEquals("HIVE-110", xml.availableServices.get(2).serviceIdReference); + assertEquals(1, xml.availableServices.get(2).components.size()); + + assertNotNull(xml.repositoryInfo); + assertEquals(2, xml.repositoryInfo.getOses().size()); + + assertEquals("redhat6", xml.repositoryInfo.getOses().get(0).getFamily()); + assertEquals(2, xml.repositoryInfo.getOses().get(0).getRepos().size()); + assertEquals("http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.0.0", + xml.repositoryInfo.getOses().get(0).getRepos().get(0).getBaseUrl()); + assertEquals("HDP-2.3", xml.repositoryInfo.getOses().get(0).getRepos().get(0).getRepoId()); + assertEquals("HDP", xml.repositoryInfo.getOses().get(0).getRepos().get(0).getRepoName()); + } + +} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java index 6bbcab7273..10e4993f56 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog170Test.java @@ -108,7 +108,6 @@ import org.apache.ambari.server.orm.entities.PrincipalEntity; import org.apache.ambari.server.orm.entities.PrivilegeEntity; import org.apache.ambari.server.orm.entities.ResourceEntity; import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity; -import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntityPK; import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.orm.entities.UserEntity; import org.apache.ambari.server.orm.entities.ViewEntity; @@ -620,11 +619,9 @@ public class UpgradeCatalog170Test { upgradeCatalog170.moveHcatalogIntoHiveService(); - ServiceComponentDesiredStateEntityPK pkHCATInHive = new ServiceComponentDesiredStateEntityPK(); - pkHCATInHive.setComponentName("HCAT"); - pkHCATInHive.setClusterId(clusterEntity.getClusterId()); - pkHCATInHive.setServiceName("HIVE"); - ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity = serviceComponentDesiredStateDAO.findByPK(pkHCATInHive); + ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity = serviceComponentDesiredStateDAO.findByName( + clusterEntity.getClusterId(), "HIVE", "HCAT"); + assertNotNull(serviceComponentDesiredStateEntity); HostComponentDesiredStateEntityPK hcDesiredStateEntityPk = new HostComponentDesiredStateEntityPK(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java index 8ff23f822e..dcac986542 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java @@ -71,7 +71,6 @@ import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntityPK; import org.apache.ambari.server.orm.entities.HostComponentStateEntity; import org.apache.ambari.server.orm.entities.HostEntity; import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity; -import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntityPK; import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; @@ -639,11 +638,9 @@ public class UpgradeCatalog200Test { upgradeCatalogHelper.addComponent(injector, clusterEntity, clusterServiceEntityNagios, hostEntity, "NAGIOS_SERVER", stackEntity); - ServiceComponentDesiredStateEntityPK pkNagiosServer = new ServiceComponentDesiredStateEntityPK(); - pkNagiosServer.setComponentName("NAGIOS_SERVER"); - pkNagiosServer.setClusterId(clusterEntity.getClusterId()); - pkNagiosServer.setServiceName("NAGIOS"); - ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity = serviceComponentDesiredStateDAO.findByPK(pkNagiosServer); + ServiceComponentDesiredStateEntity serviceComponentDesiredStateEntity = serviceComponentDesiredStateDAO.findByName( + clusterEntity.getClusterId(), "NAGIOS", "NAGIOS_SERVER"); + assertNotNull(serviceComponentDesiredStateEntity); HostComponentDesiredStateEntityPK hcDesiredStateEntityPk = new HostComponentDesiredStateEntityPK(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java index 83018a2f64..c641bbf81d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java @@ -18,13 +18,37 @@ package org.apache.ambari.server.upgrade; -import com.google.inject.AbstractModule; -import com.google.inject.Binder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.Provider; -import com.google.inject.persist.PersistService; +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertNotNull; +import static junit.framework.Assert.assertNull; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.capture; +import static org.easymock.EasyMock.createMockBuilder; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.createStrictMock; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; +import static org.easymock.EasyMock.verify; + +import java.io.File; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.net.URL; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import javax.persistence.EntityManager; + import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; @@ -46,7 +70,6 @@ import org.apache.ambari.server.orm.entities.ClusterStateEntity; import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity; import org.apache.ambari.server.orm.entities.HostEntity; import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity; -import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntityPK; import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; @@ -66,35 +89,13 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import javax.persistence.EntityManager; -import java.io.File; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.net.URL; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; - -import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertNotNull; -import static junit.framework.Assert.assertNull; -import static org.easymock.EasyMock.anyObject; -import static org.easymock.EasyMock.capture; -import static org.easymock.EasyMock.createMockBuilder; -import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.createStrictMock; -import static org.easymock.EasyMock.eq; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.expectLastCall; -import static org.easymock.EasyMock.replay; -import static org.easymock.EasyMock.reset; -import static org.easymock.EasyMock.verify; +import com.google.inject.AbstractModule; +import com.google.inject.Binder; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Module; +import com.google.inject.Provider; +import com.google.inject.persist.PersistService; /** * {@link org.apache.ambari.server.upgrade.UpgradeCatalog210} unit tests. @@ -601,14 +602,15 @@ public class UpgradeCatalog210Test { Assert.assertEquals(HostComponentAdminState.INSERVICE.name(), entity.getAdminState().name()); + // ensure the desired state exists + Assert.assertNotNull(componentDesiredStateDAO.findByName(clusterEntity.getClusterId(), "STORM", + "STORM_REST_API")); + UpgradeCatalog210 upgradeCatalog210 = injector.getInstance(UpgradeCatalog210.class); upgradeCatalog210.removeStormRestApiServiceComponent(); - ServiceComponentDesiredStateEntityPK entityPK = new ServiceComponentDesiredStateEntityPK(); - entityPK.setClusterId(clusterEntity.getClusterId()); - entityPK.setServiceName("STORM"); - entityPK.setComponentName("STORM_REST_API"); - Assert.assertNull(componentDesiredStateDAO.findByPK(entityPK)); + Assert.assertNull(componentDesiredStateDAO.findByName(clusterEntity.getClusterId(), "STORM", + "STORM_REST_API")); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java index 4c48972d46..4c11d10084 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogHelper.java @@ -170,6 +170,9 @@ public class UpgradeCatalogHelper { protected void addComponent(Injector injector, ClusterEntity clusterEntity, ClusterServiceEntity clusterServiceEntity, HostEntity hostEntity, String componentName, StackEntity desiredStackEntity) { + ServiceComponentDesiredStateDAO serviceComponentDesiredStateDAO = injector.getInstance( + ServiceComponentDesiredStateDAO.class); + ServiceComponentDesiredStateEntity componentDesiredStateEntity = new ServiceComponentDesiredStateEntity(); componentDesiredStateEntity.setClusterServiceEntity(clusterServiceEntity); componentDesiredStateEntity.setComponentName(componentName); @@ -177,6 +180,7 @@ public class UpgradeCatalogHelper { componentDesiredStateEntity.setDesiredStack(desiredStackEntity); componentDesiredStateEntity.setClusterServiceEntity(clusterServiceEntity); componentDesiredStateEntity.setClusterId(clusterServiceEntity.getClusterId()); + serviceComponentDesiredStateDAO.create(componentDesiredStateEntity); HostComponentDesiredStateDAO hostComponentDesiredStateDAO = injector.getInstance(HostComponentDesiredStateDAO.class); HostComponentDesiredStateEntity hostComponentDesiredStateEntity = new HostComponentDesiredStateEntity(); @@ -208,7 +212,6 @@ public class UpgradeCatalogHelper { componentDesiredStateEntity); ClusterServiceDAO clusterServiceDAO = injector.getInstance(ClusterServiceDAO.class); - ServiceComponentDesiredStateDAO serviceComponentDesiredStateDAO = injector.getInstance(ServiceComponentDesiredStateDAO.class); HostDAO hostDAO = injector.getInstance(HostDAO.class); serviceComponentDesiredStateDAO.merge(componentDesiredStateEntity); hostDAO.merge(hostEntity); diff --git a/ambari-server/src/test/resources/hbase_version_test.xml b/ambari-server/src/test/resources/hbase_version_test.xml new file mode 100644 index 0000000000..9df07ed656 --- /dev/null +++ b/ambari-server/src/test/resources/hbase_version_test.xml @@ -0,0 +1,59 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<repository-version xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="version_definition.xsd"> + + <release> + <type>PATCH</type> + <stack-id>HDP-2.3</stack-id> + <version>2.3.4.0</version> + <build>3396</build> + <compatible-with>2.3.2.[0-9]</compatible-with> + <release-notes>http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.4/</release-notes> + </release> + + <manifest> + <service id="HBASE-112" name="HBASE" version="1.1.2" version-id="2_3_4_0-3396" /> + </manifest> + + <available-services> + <service idref="HBASE-112" /> + </available-services> + + <repository-info> + <os family="redhat6"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.4.0</baseurl> + <repoid>HDP-2.3</repoid> + <reponame>HDP</reponame> + </repo> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl> + <repoid>HDP-UTILS-1.1.0.20</repoid> + <reponame>HDP-UTILS</reponame> + </repo> + </os> + </repository-info> + + <upgrade> + <configuration type="hdfs-site"> + <set key="foo" value="bar" /> + </configuration> + </upgrade> +</repository-version> diff --git a/ambari-server/src/test/resources/version_definition_resource_provider.xml b/ambari-server/src/test/resources/version_definition_resource_provider.xml new file mode 100644 index 0000000000..55cd1c14fe --- /dev/null +++ b/ambari-server/src/test/resources/version_definition_resource_provider.xml @@ -0,0 +1,72 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<repository-version xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="version_definition.xsd"> + + <release> + <type>PATCH</type> + <stack-id>HDP-2.2.0</stack-id> + <version>2.2.0.8</version> + <build>5678</build> + <compatible-with>2.3.4.[1-9]</compatible-with> + <release-notes>http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.4/</release-notes> + </release> + + <manifest> + <service id="HDFS-271" name="HDFS" version="2.7.1" version-id="10" /> + <service id="HIVE-110" name="HIVE" version="1.1.0" /> + <service id="HIVE-200" name="HIVE" version="2.0.0" /> + </manifest> + + <available-services> + <service idref="HDFS-271" /> + <service idref="HIVE-110"> + <component>HIVE_METASTORE</component> + </service> + </available-services> + + <repository-info> + <os family="redhat6"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.0.0</baseurl> + <repoid>HDP-2.3</repoid> + <reponame>HDP</reponame> + </repo> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl> + <repoid>HDP-UTILS-1.1.0.20</repoid> + <reponame>HDP-UTILS</reponame> + </repo> + </os> + <os family="suse11"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/centos7/2.x/updates/2.3.0.0</baseurl> + <repoid>HDP-2.3</repoid> + <reponame>HDP</reponame> + </repo> + </os> + + </repository-info> + + <upgrade> + <configuration type="hdfs-site"> + <set key="foo" value="bar" /> + </configuration> + </upgrade> +</repository-version> diff --git a/ambari-server/src/test/resources/version_definition_test.xml b/ambari-server/src/test/resources/version_definition_test.xml new file mode 100644 index 0000000000..69ea58161f --- /dev/null +++ b/ambari-server/src/test/resources/version_definition_test.xml @@ -0,0 +1,79 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<repository-version xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:noNamespaceSchemaLocation="version_definition.xsd"> + + <release> + <type>PATCH</type> + <stack-id>HDP-2.3</stack-id> + <version>2.3.4.1</version> + <build>1234</build> + <compatible-with>2.3.4.[1-9]</compatible-with> + <release-notes>http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.4/</release-notes> + </release> + + <manifest> + <service id="HDFS-271" name="HDFS" version="2.7.1" version-id="10" /> + <service id="HIVE-110" name="HIVE" version="1.1.0" /> + <service id="HIVE-200" name="HIVE" version="2.0.0" /> + <service id="HBASE-899" name="HBASE" version="8.9.9" /> + </manifest> + + <available-services> + <service idref="HDFS-271" /> + <service idref="HIVE-200" /> + <service idref="HIVE-110"> + <component>HIVE_METASTORE</component> + </service> + </available-services> + + <repository-info> + <os family="redhat6"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.0.0</baseurl> + <repoid>HDP-2.3</repoid> + <reponame>HDP</reponame> + </repo> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl> + <repoid>HDP-UTILS-1.1.0.20</repoid> + <reponame>HDP-UTILS</reponame> + </repo> + </os> + <os family="redhat7"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP/centos7/2.x/updates/2.3.0.0</baseurl> + <repoid>HDP-2.3</repoid> + <reponame>HDP</reponame> + </repo> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos7</baseurl> + <repoid>HDP-UTILS-1.1.0.20</repoid> + <reponame>HDP-UTILS</reponame> + </repo> + </os> + + </repository-info> + + <upgrade> + <configuration type="hdfs-site"> + <set key="foo" value="bar" /> + </configuration> + </upgrade> +</repository-version> diff --git a/contrib/version-builder/example.sh b/contrib/version-builder/example.sh new file mode 100755 index 0000000000..a93ddb66ea --- /dev/null +++ b/contrib/version-builder/example.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +filename="version_241-12345.xml" + +python version_builder.py --file $filename --release-type PATCH +python version_builder.py --file $filename --release-stack HDP-2.3 +python version_builder.py --file $filename --release-version 2.4.1.1 +python version_builder.py --file $filename --release-build 12345 +python version_builder.py --file $filename --release-notes http://example.com +python version_builder.py --file $filename --release-display HDP-2.4.1.1-1234-patch +python version_builder.py --file $filename --release-compatible 2.4.[0-1].0 + +# call any number of times for each service in the repo +python version_builder.py --file $filename --manifest --manifest-id HDFS-271 --manifest-service HDFS --manifest-version 2.7.1.2.4 +python version_builder.py --file $filename --manifest --manifest-id HBASE-132 --manifest-service HBASE --manifest-version 1.3.2.4.3 + +#call any number of times for the target services to upgrade +python version_builder.py --file $filename --available --manifest-id HDFS-271 + +#call any number of times for repo per os +python version_builder.py --file $filename --repo --repo-os redhat6 --repo-id HDP-2.3 --repo-name HDP --repo-url http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.4.0 +python version_builder.py --file $filename --repo --repo-os redhat6 --repo-id HDP-UTILS-1.1.0.20 --repo-name HDP-UTILS --repo-url http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6 + +python version_builder.py --file $filename --finalize --xsd ../../ambari-server/src/main/resources/version_definition.xsd + +# to upload this to running Ambari instance on localhost: +# curl -u admin:admin -H 'Content-Type: text/xml' -X POST -d @$filename http://localhost:8080/api/v1/version_definitions diff --git a/contrib/version-builder/version_builder.py b/contrib/version-builder/version_builder.py new file mode 100644 index 0000000000..6c20a4739e --- /dev/null +++ b/contrib/version-builder/version_builder.py @@ -0,0 +1,354 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import optparse +import os +import subprocess +import sys +import xml.etree.ElementTree as ET + +def load_file(filename): + """ + Loads the specified XML file + """ + if os.path.exists(filename): + tree = ET.ElementTree() + tree.parse(filename) + root = tree.getroot() + else: + attribs = {} + attribs['xmlns:xsi'] = "http://www.w3.org/2001/XMLSchema-instance" + attribs['xsi:noNamespaceSchemaLocation'] = "version_definition.xsd" + root = ET.Element("repository-version", attribs) + + ET.SubElement(root, "release") + ET.SubElement(root, "manifest") + ET.SubElement(root, "available-services") + ET.SubElement(root, "repository-info") + + return root + +def save_file(xml, filename): + """ + Saves the XML file + """ + p = subprocess.Popen(['xmllint', '--format', '--output', filename, '-'], stdout=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate(input=ET.tostring(xml)) + +def check_xmllint(): + """ + Verifies utility xmllint is available + """ + try: + p = subprocess.Popen(['xmllint', '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) + (stdout, stderr) = p.communicate() + + if p.returncode != 0: + raise Exception("xmllint command does not appear to be available") + + except: + raise Exception("xmllint command does not appear to be available") + + +def validate_file(filename, xsdfile): + """ + Validates the XML file against the XSD + """ + args = ['xmllint', '--noout', '--load-trace', '--schema', xsdfile, filename] + + p = subprocess.Popen(args, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = p.communicate() + + if p.returncode != 0: + raise Exception(stderr) + + if len(stdout) > 0: + print stdout + + if len(stderr) > 0: + print stderr + + +def update_simple(parent, name, value): + """ + Helper method to either update or create the element + """ + element = parent.find('./' + name) + + if element is None: + element = ET.SubElement(parent, name) + element.text = value + else: + element.text = value + +def process_release(xmlroot, options): + """ + Create elements of the 'release' parent + """ + release_element = xmlroot.find("./release") + + if release_element is None: + raise Exception("Element 'release' is not found") + + if options.release_type: + update_simple(release_element, "type", options.release_type) + + if options.release_stack: + update_simple(release_element, "stack-id", options.release_stack) + + if options.release_version: + update_simple(release_element, "version", options.release_version) + + if options.release_build: + update_simple(release_element, "build", options.release_build) + + if options.release_compatible: + update_simple(release_element, "compatible-with", options.release_compatible) + + if options.release_notes: + update_simple(release_element, "release-notes", options.release_notes) + + if options.release_display: + update_simple(release_element, "display", options.release_display) + + if options.release_package_version: + update_simple(release_element, "package-version", options.release_package_version) + +def process_manifest(xmlroot, options): + """ + Creates the manifest element + """ + if not options.manifest: + return + + manifest_element = xmlroot.find("./manifest") + + if manifest_element is None: + raise Exception("Element 'manifest' is not found") + + service_element = manifest_element.find("./service[@id='{0}']".format(options.manifest_id)) + + if service_element is None: + service_element = ET.SubElement(manifest_element, "service") + service_element.set('id', options.manifest_id) + + service_element.set('name', options.manifest_service) + service_element.set('version', options.manifest_version) + if options.manifest_version_id: + service_element.set('version-id', options.manifest_version_id) + +def process_available(xmlroot, options): + """ + Processes available service elements + """ + if not options.available: + return + + manifest_element = xmlroot.find("./manifest") + if manifest_element is None: + raise Exception("'manifest' element is not found") + + service_element = manifest_element.find("./service[@id='{0}']".format(options.manifest_id)) + if service_element is None: + raise Exception("Cannot add an available service for {0}; it's not on the manifest".format(options.manifest_id)) + + available_element = xmlroot.find("./available-services") + if available_element is None: + raise Exception("'available-services' is not found") + + service_element = available_element.find("./service[@idref='{0}']".format(options.manifest_id)) + + if service_element is not None: + available_element.remove(service_element) + + service_element = ET.SubElement(available_element, "service") + service_element.set('idref', options.manifest_id) + + if options.available_components: + components = options.available_components.split(',') + for component in components: + e = ET.SubElement(service_element, 'component') + e.text = component + + +def process_repo(xmlroot, options): + """ + Processes repository options. This method doesn't update or create individual elements, it + creates the entire repo structure + """ + if not options.repo: + return + + repo_parent = xmlroot.find("./repository-info") + if repo_parent is None: + raise Exception("'repository-info' element is not found") + + os_element = repo_parent.find("./os[@family='{0}']".format(options.repo_os)) + if os_element is None: + os_element = ET.SubElement(repo_parent, 'os') + os_element.set('family', options.repo_os) + + repo_element = os_element.find("./repo/[reponame='{0}']".format(options.repo_name)) + + if repo_element is not None: + os_element.remove(repo_element) + + repo_element = ET.SubElement(os_element, 'repo') + e = ET.SubElement(repo_element, 'baseurl') + e.text = options.repo_url + + e = ET.SubElement(repo_element, 'repoid') + e.text = options.repo_id + + e = ET.SubElement(repo_element, 'reponame') + e.text = options.repo_name + +def validate_manifest(parser, options): + """ + Validates manifest options from the command line + """ + if not options.manifest: + return + + template = "When specifying --manifest, {0} is also required" + + if not options.manifest_id: + parser.error(template.format("--manifest-id")) + + if not options.manifest_service: + parser.error(template.format("--manifest-service")) + + if not options.manifest_version: + parser.error(template.format("--manifest-version")) + +def validate_available(parser, options): + """ + Validates available service options from the command line + """ + if not options.available: + return + + if not options.manifest_id: + parser.error("When specifying --available, --manifest-id is also required") + +def validate_repo(parser, options): + """ + Validates repo options from the command line + """ + if not options.repo: + return + + template = "When specifying --repo, {0} is also required" + + if not options.repo_os: + parser.error(template.format("--repo-os")) + + if not options.repo_url: + parser.error(template.format("--repo-url")) + + if not options.repo_id: + parser.error(template.format("--repo-id")) + + if not options.repo_name: + parser.error(template.format("--repo-name")) + + +def main(argv): + parser = optparse.OptionParser( + epilog="OS utility 'xmllint' is required for this tool to function. It handles pretty-printing and XSD validation.") + + parser.add_option('--file', dest='filename', + help="The output XML file") + + parser.add_option('--finalize', action='store_true', dest='finalize', + help="Finalize and validate the XML file") + parser.add_option('--xsd', dest='xsd_file', + help="The XSD location when finalizing") + + parser.add_option('--release-type', type='choice', choices=['STANDARD', 'PATCH'], dest='release_type' , + help="Indicate the release type: i.e. STANDARD or PATCH") + parser.add_option('--release-stack', dest='release_stack', + help="The stack id: e.g. HDP-2.4") + parser.add_option('--release-version', dest='release_version', + help="The release version without build number: e.g. 2.4.0.1") + parser.add_option('--release-build', dest='release_build', + help="The release build number: e.g. 1234") + parser.add_option('--release-compatible', dest='release_compatible', + help="Regular Expression string to identify version compatibility for patches: e.g. 2.4.1.[0-9]") + parser.add_option('--release-notes', dest='release_notes', + help="A http link to the documentation notes") + parser.add_option('--release-display', dest='release_display', + help="The display name for this release") + parser.add_option('--release-package-version', dest='release_package_version', + help="Identifier to use when installing packages, generally a part of the package name") + + parser.add_option('--manifest', action='store_true', dest='manifest', + help="Add a manifest service with other options: --manifest-id, --manifest-service, --manifest-version, --manifest-version-id") + parser.add_option('--manifest-id', dest='manifest_id', + help="Unique ID for a service in a manifest. Required when specifying --manifest and --available") + parser.add_option('--manifest-service', dest='manifest_service') + parser.add_option('--manifest-version', dest='manifest_version') + parser.add_option('--manifest-version-id', dest='manifest_version_id') + + parser.add_option('--available', action='store_true', dest='available', + help="Add an available service with other options: --manifest-id, --available-components") + parser.add_option('--available-components', dest='available_components', + help="A CSV of service components that are intended to be upgraded via patch. \ + Omitting this implies the entire service should be upgraded") + + parser.add_option('--repo', action='store_true', dest='repo', + help="Add repository data with options: --repo-os, --repo-url, --repo-id, --repo-name") + parser.add_option('--repo-os', dest='repo_os', + help="The operating system type: i.e. redhat6, redhat7, debian7, ubuntu12, ubuntu14, suse11") + parser.add_option('--repo-url', dest='repo_url', + help="The base url for the repository data") + parser.add_option('--repo-id', dest='repo_id', help="The ID of the repo") + parser.add_option('--repo-name', dest='repo_name', help="The name of the repo") + + (options, args) = parser.parse_args() + + check_xmllint() + + # validate_filename + if not options.filename: + parser.error("--file option is required") + + validate_manifest(parser, options) + validate_available(parser, options) + validate_repo(parser, options) + + # validate_finalize + if options.finalize and not options.xsd_file: + parser.error("Must supply XSD (--xsd) when finalizing") + + # load file + root = load_file(options.filename) + + process_release(root, options) + process_manifest(root, options) + process_available(root, options) + process_repo(root, options) + + # save file + save_file(root, options.filename) + + if options.finalize: + validate_file(options.filename, options.xsd_file) + +if __name__ == "__main__": + main(sys.argv) |