diff options
108 files changed, 1732 insertions, 665 deletions
diff --git a/ambari-agent/src/main/python/ambari_agent/RecoveryManager.py b/ambari-agent/src/main/python/ambari_agent/RecoveryManager.py index b5795b235c..ed537ca692 100644 --- a/ambari-agent/src/main/python/ambari_agent/RecoveryManager.py +++ b/ambari-agent/src/main/python/ambari_agent/RecoveryManager.py @@ -88,7 +88,6 @@ class RecoveryManager: self.allowed_desired_states = [self.STARTED, self.INSTALLED] self.allowed_current_states = [self.INIT, self.INSTALLED] self.enabled_components = [] - self.disabled_components = [] self.statuses = {} self.__status_lock = threading.RLock() self.__command_lock = threading.RLock() @@ -107,7 +106,7 @@ class RecoveryManager: self.actions = {} - self.update_config(6, 60, 5, 12, recovery_enabled, auto_start_only, "", "") + self.update_config(6, 60, 5, 12, recovery_enabled, auto_start_only, "") pass @@ -213,19 +212,12 @@ class RecoveryManager: pass """ - Whether specific components are enabled/disabled for recovery. Being enabled takes - precedence over being disabled. When specific components are enabled then only - those components are enabled. When specific components are disabled then all of - the other components are enabled. + Whether specific components are enabled for recovery. """ def configured_for_recovery(self, component): - if len(self.disabled_components) == 0 and len(self.enabled_components) == 0: - return True - if len(self.disabled_components) > 0 and component not in self.disabled_components \ - and len(self.enabled_components) == 0: - return True if len(self.enabled_components) > 0 and component in self.enabled_components: return True + return False def requires_recovery(self, component): @@ -547,8 +539,8 @@ class RecoveryManager: "maxCount" : 10, "windowInMinutes" : 60, "retryGap" : 0, - "disabledComponents" : "a,b", - "enabledComponents" : "c,d"} + "components" : "a,b" + } """ recovery_enabled = False @@ -558,7 +550,6 @@ class RecoveryManager: retry_gap = 5 max_lifetime_count = 12 enabled_components = "" - disabled_components = "" if reg_resp and "recoveryConfig" in reg_resp: @@ -578,17 +569,16 @@ class RecoveryManager: if 'maxLifetimeCount' in config: max_lifetime_count = self._read_int_(config['maxLifetimeCount'], max_lifetime_count) - if 'enabledComponents' in config: - enabled_components = config['enabledComponents'] - if 'disabledComponents' in config: - disabled_components = config['disabledComponents'] + if 'components' in config: + enabled_components = config['components'] + self.update_config(max_count, window_in_min, retry_gap, max_lifetime_count, recovery_enabled, auto_start_only, - enabled_components, disabled_components) + enabled_components) pass def update_config(self, max_count, window_in_min, retry_gap, max_lifetime_count, recovery_enabled, - auto_start_only, enabled_components, disabled_components): + auto_start_only, enabled_components): """ Update recovery configuration, recovery is disabled if configuration values are not correct @@ -619,7 +609,6 @@ class RecoveryManager: self.retry_gap_in_sec = retry_gap * 60 self.auto_start_only = auto_start_only self.max_lifetime_count = max_lifetime_count - self.disabled_components = [] self.enabled_components = [] self.allowed_desired_states = [self.STARTED, self.INSTALLED] @@ -635,19 +624,13 @@ class RecoveryManager: if len(component.strip()) > 0: self.enabled_components.append(component.strip()) - if disabled_components is not None and len(disabled_components) > 0: - components = disabled_components.split(",") - for component in components: - if len(component.strip()) > 0: - self.disabled_components.append(component.strip()) - self.recovery_enabled = recovery_enabled if self.recovery_enabled: logger.info( "==> Auto recovery is enabled with maximum %s in %s minutes with gap of %s minutes between and" - " lifetime max being %s. Enabled components - %s and Disabled components - %s", + " lifetime max being %s. Enabled components - %s", self.max_count, self.window_in_min, self.retry_gap, self.max_lifetime_count, - ', '.join(self.enabled_components), ', '.join(self.disabled_components)) + ', '.join(self.enabled_components)) pass diff --git a/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py b/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py index 715a60cb8c..f7e28949cc 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py +++ b/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py @@ -338,7 +338,7 @@ class TestActionQueue(TestCase): config.set('agent', 'tolerate_download_failures', "true") dummy_controller = MagicMock() dummy_controller.recovery_manager = RecoveryManager(tempfile.mktemp()) - dummy_controller.recovery_manager.update_config(5, 5, 1, 11, True, False, "", "") + dummy_controller.recovery_manager.update_config(5, 5, 1, 11, True, False, "") actionQueue = ActionQueue(config, dummy_controller) unfreeze_flag = threading.Event() diff --git a/ambari-agent/src/test/python/ambari_agent/TestRecoveryManager.py b/ambari-agent/src/test/python/ambari_agent/TestRecoveryManager.py index 0321a20964..ef81704f21 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestRecoveryManager.py +++ b/ambari-agent/src/test/python/ambari_agent/TestRecoveryManager.py @@ -141,6 +141,7 @@ class _TestRecoveryManager(TestCase): mock_uds.reset_mock() + rm.update_config(12, 5, 1, 15, True, False, "NODEMANAGER") rm.process_execution_commands([self.exec_command1, self.exec_command2, self.exec_command3]) mock_uds.assert_has_calls([call("NODEMANAGER", "INSTALLED")], [call("NODEMANAGER", "STARTED")]) @@ -173,32 +174,29 @@ class _TestRecoveryManager(TestCase): rm = RecoveryManager(tempfile.mktemp(), True, False) self.assertTrue(rm.enabled()) - rm.update_config(0, 60, 5, 12, True, False, "", "") + config = rm.update_config(0, 60, 5, 12, True, False, "") self.assertFalse(rm.enabled()) - rm.update_config(6, 60, 5, 12, True, False, "", "") + rm.update_config(6, 60, 5, 12, True, False, "") self.assertTrue(rm.enabled()) - rm.update_config(6, 0, 5, 12, True, False, "", "") + rm.update_config(6, 0, 5, 12, True, False, "") self.assertFalse(rm.enabled()) - rm.update_config(6, 60, 0, 12, True, False, "", "") + rm.update_config(6, 60, 0, 12, True, False, "") self.assertFalse(rm.enabled()) - rm.update_config(6, 60, 1, 12, True, False, None, None) + rm.update_config(6, 60, 1, 12, True, False, None) self.assertTrue(rm.enabled()) - rm.update_config(6, 60, 61, 12, True, False, "", None) + rm.update_config(6, 60, 61, 12, True, False, None) self.assertFalse(rm.enabled()) - rm.update_config(6, 60, 5, 0, True, False, None, "") - self.assertFalse(rm.enabled()) - - rm.update_config(6, 60, 5, 4, True, False, "", "") + rm.update_config(6, 60, 5, 4, True, False, "") self.assertFalse(rm.enabled()) # maximum 2 in 2 minutes and at least 1 minute wait - rm.update_config(2, 5, 1, 4, True, False, "", "") + rm.update_config(2, 5, 1, 4, True, False, "") self.assertTrue(rm.enabled()) # T = 1000-2 @@ -224,7 +222,7 @@ class _TestRecoveryManager(TestCase): self.assertFalse(rm.may_execute("NODEMANAGER")) # too soon # maximum 2 in 2 minutes and no min wait - rm.update_config(2, 5, 1, 5, True, True, "", "") + rm.update_config(2, 5, 1, 5, True, True, "") # T = 1500-3 self.assertTrue(rm.execute("NODEMANAGER2")) @@ -244,7 +242,7 @@ class _TestRecoveryManager(TestCase): def test_recovery_required(self): rm = RecoveryManager(tempfile.mktemp(), True, False) - + rm.update_config(12, 5, 1, 15, True, False, "NODEMANAGER") rm.update_current_status("NODEMANAGER", "INSTALLED") rm.update_desired_status("NODEMANAGER", "INSTALLED") self.assertFalse(rm.requires_recovery("NODEMANAGER")) @@ -292,13 +290,13 @@ class _TestRecoveryManager(TestCase): def test_recovery_required2(self): rm = RecoveryManager(tempfile.mktemp(), True, True) - rm.update_config(15, 5, 1, 16, True, False, "", "") + rm.update_config(15, 5, 1, 16, True, False, "NODEMANAGER") rm.update_current_status("NODEMANAGER", "INSTALLED") rm.update_desired_status("NODEMANAGER", "STARTED") self.assertTrue(rm.requires_recovery("NODEMANAGER")) rm = RecoveryManager(tempfile.mktemp(), True, True) - rm.update_config(15, 5, 1, 16, True, False, "NODEMANAGER", "") + rm.update_config(15, 5, 1, 16, True, False, "NODEMANAGER") rm.update_current_status("NODEMANAGER", "INSTALLED") rm.update_desired_status("NODEMANAGER", "STARTED") self.assertTrue(rm.requires_recovery("NODEMANAGER")) @@ -308,26 +306,16 @@ class _TestRecoveryManager(TestCase): self.assertFalse(rm.requires_recovery("DATANODE")) rm = RecoveryManager(tempfile.mktemp(), True, True) - rm.update_config(15, 5, 1, 16, True, False, "", "NODEMANAGER") + rm.update_config(15, 5, 1, 16, True, False, "") rm.update_current_status("NODEMANAGER", "INSTALLED") rm.update_desired_status("NODEMANAGER", "STARTED") self.assertFalse(rm.requires_recovery("NODEMANAGER")) rm.update_current_status("DATANODE", "INSTALLED") rm.update_desired_status("DATANODE", "STARTED") - self.assertTrue(rm.requires_recovery("DATANODE")) - - rm.update_config(15, 5, 1, 16, True, False, "", "NODEMANAGER") - rm.update_config(15, 5, 1, 16, True, False, "NODEMANAGER", "") - rm.update_current_status("NODEMANAGER", "INSTALLED") - rm.update_desired_status("NODEMANAGER", "STARTED") - self.assertTrue(rm.requires_recovery("NODEMANAGER")) - - rm.update_current_status("DATANODE", "INSTALLED") - rm.update_desired_status("DATANODE", "STARTED") self.assertFalse(rm.requires_recovery("DATANODE")) - rm.update_config(15, 5, 1, 16, True, False, "NODEMANAGER", "NODEMANAGER") + rm.update_config(15, 5, 1, 16, True, False, "NODEMANAGER") rm.update_current_status("NODEMANAGER", "INSTALLED") rm.update_desired_status("NODEMANAGER", "STARTED") self.assertTrue(rm.requires_recovery("NODEMANAGER")) @@ -392,12 +380,12 @@ class _TestRecoveryManager(TestCase): 4200, 4201, 4202, 4300, 4301, 4302] rm = RecoveryManager(tempfile.mktemp(), True) - rm.update_config(15, 5, 1, 16, True, False, "", "") + rm.update_config(15, 5, 1, 16, True, False, "") command1 = copy.deepcopy(self.command) rm.store_or_update_command(command1) - + rm.update_config(12, 5, 1, 15, True, False, "NODEMANAGER") rm.update_current_status("NODEMANAGER", "INSTALLED") rm.update_desired_status("NODEMANAGER", "STARTED") self.assertEqual("INSTALLED", rm.get_current_status("NODEMANAGER")) @@ -423,14 +411,14 @@ class _TestRecoveryManager(TestCase): self.assertEqual(1, len(commands)) self.assertEqual("INSTALL", commands[0]["roleCommand"]) - rm.update_config(2, 5, 1, 5, True, True, "", "") + rm.update_config(2, 5, 1, 5, True, True, "") rm.update_current_status("NODEMANAGER", "INIT") rm.update_desired_status("NODEMANAGER", "INSTALLED") commands = rm.get_recovery_commands() self.assertEqual(0, len(commands)) - rm.update_config(12, 5, 1, 15, True, False, "", "") + rm.update_config(12, 5, 1, 15, True, False, "NODEMANAGER") rm.update_current_status("NODEMANAGER", "INIT") rm.update_desired_status("NODEMANAGER", "INSTALLED") @@ -471,25 +459,25 @@ class _TestRecoveryManager(TestCase): def test_update_rm_config(self, mock_uc): rm = RecoveryManager(tempfile.mktemp()) rm.update_configuration_from_registration(None) - mock_uc.assert_has_calls([call(6, 60, 5, 12, False, True, "", "")]) + mock_uc.assert_has_calls([call(6, 60, 5, 12, False, True, "")]) mock_uc.reset_mock() rm.update_configuration_from_registration({}) - mock_uc.assert_has_calls([call(6, 60, 5, 12, False, True, "", "")]) + mock_uc.assert_has_calls([call(6, 60, 5, 12, False, True, "")]) mock_uc.reset_mock() rm.update_configuration_from_registration( {"recoveryConfig": { "type" : "DEFAULT"}} ) - mock_uc.assert_has_calls([call(6, 60, 5, 12, False, True, "", "")]) + mock_uc.assert_has_calls([call(6, 60, 5, 12, False, True, "")]) mock_uc.reset_mock() rm.update_configuration_from_registration( {"recoveryConfig": { "type" : "FULL"}} ) - mock_uc.assert_has_calls([call(6, 60, 5, 12, True, False, "", "")]) + mock_uc.assert_has_calls([call(6, 60, 5, 12, True, False, "")]) mock_uc.reset_mock() rm.update_configuration_from_registration( @@ -497,7 +485,7 @@ class _TestRecoveryManager(TestCase): "type" : "AUTO_START", "max_count" : "med"}} ) - mock_uc.assert_has_calls([call(6, 60, 5, 12, True, True, "", "")]) + mock_uc.assert_has_calls([call(6, 60, 5, 12, True, True, "")]) mock_uc.reset_mock() rm.update_configuration_from_registration( @@ -507,10 +495,9 @@ class _TestRecoveryManager(TestCase): "windowInMinutes" : 20, "retryGap" : 2, "maxLifetimeCount" : 5, - "enabledComponents" : " A,B", - "disabledComponents" : "C"}} + "components" : " A,B"}} ) - mock_uc.assert_has_calls([call(5, 20, 2, 5, True, True, " A,B", "C")]) + mock_uc.assert_has_calls([call(5, 20, 2, 5, True, True, " A,B")]) pass @patch.object(RecoveryManager, "_now_") @@ -522,7 +509,7 @@ class _TestRecoveryManager(TestCase): rec_st = rm.get_recovery_status() self.assertEquals(rec_st, {"summary": "DISABLED"}) - rm.update_config(2, 5, 1, 4, True, True, "", "") + rm.update_config(2, 5, 1, 4, True, True, "") rec_st = rm.get_recovery_status() self.assertEquals(rec_st, {"summary": "RECOVERABLE", "componentReports": []}) @@ -566,12 +553,12 @@ class _TestRecoveryManager(TestCase): [1000, 1001, 1002, 1003, 1104, 1105, 1106, 1807, 1808, 1809, 1810, 1811, 1812] rm = RecoveryManager(tempfile.mktemp(), True) - rm.update_config(5, 5, 1, 11, True, False, "", "") + rm.update_config(5, 5, 1, 11, True, False, "") command1 = copy.deepcopy(self.command) rm.store_or_update_command(command1) - + rm.update_config(12, 5, 1, 15, True, False, "NODEMANAGER") rm.update_current_status("NODEMANAGER", "INSTALLED") rm.update_desired_status("NODEMANAGER", "STARTED") @@ -607,23 +594,24 @@ class _TestRecoveryManager(TestCase): def test_configured_for_recovery(self): rm = RecoveryManager(tempfile.mktemp(), True) + rm.update_config(12, 5, 1, 15, True, False, "A,B") self.assertTrue(rm.configured_for_recovery("A")) self.assertTrue(rm.configured_for_recovery("B")) - rm.update_config(5, 5, 1, 11, True, False, "", "") - self.assertTrue(rm.configured_for_recovery("A")) - self.assertTrue(rm.configured_for_recovery("B")) + rm.update_config(5, 5, 1, 11, True, False, "") + self.assertFalse(rm.configured_for_recovery("A")) + self.assertFalse(rm.configured_for_recovery("B")) - rm.update_config(5, 5, 1, 11, True, False, "A", "") + rm.update_config(5, 5, 1, 11, True, False, "A") self.assertTrue(rm.configured_for_recovery("A")) self.assertFalse(rm.configured_for_recovery("B")) - rm.update_config(5, 5, 1, 11, True, False, "", "B,C") + rm.update_config(5, 5, 1, 11, True, False, "A") self.assertTrue(rm.configured_for_recovery("A")) self.assertFalse(rm.configured_for_recovery("B")) self.assertFalse(rm.configured_for_recovery("C")) - rm.update_config(5, 5, 1, 11, True, False, "A, D, F ", "B,C") + rm.update_config(5, 5, 1, 11, True, False, "A, D, F ") self.assertTrue(rm.configured_for_recovery("A")) self.assertFalse(rm.configured_for_recovery("B")) self.assertFalse(rm.configured_for_recovery("C")) @@ -637,7 +625,7 @@ class _TestRecoveryManager(TestCase): [1000, 1071, 1372] rm = RecoveryManager(tempfile.mktemp(), True) - rm.update_config(2, 5, 1, 4, True, True, "", "") + rm.update_config(2, 5, 1, 4, True, True, "") rm.execute("COMPONENT") actions = rm.get_actions_copy()["COMPONENT"] @@ -655,7 +643,7 @@ class _TestRecoveryManager(TestCase): def test_is_action_info_stale(self, time_mock): rm = RecoveryManager(tempfile.mktemp(), True) - rm.update_config(5, 60, 5, 16, True, False, "", "") + rm.update_config(5, 60, 5, 16, True, False, "") time_mock.return_value = 0 self.assertFalse(rm.is_action_info_stale("COMPONENT_NAME")) diff --git a/ambari-common/src/main/python/ambari_commons/os_check.py b/ambari-common/src/main/python/ambari_commons/os_check.py index c5457bb2a5..b430c869d0 100644 --- a/ambari-common/src/main/python/ambari_commons/os_check.py +++ b/ambari-common/src/main/python/ambari_commons/os_check.py @@ -56,6 +56,8 @@ RESOURCES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "resou # family JSON data OSFAMILY_JSON_RESOURCE = "os_family.json" +JSON_OS_MAPPING = "mapping" +JSON_OS_ALIASES = "aliases" JSON_OS_TYPE = "distro" JSON_OS_VERSION = "versions" JSON_EXTENDS = "extends" @@ -76,6 +78,8 @@ VER_NT_SERVER = 3 _IS_ORACLE_LINUX = os.path.exists('/etc/oracle-release') _IS_REDHAT_LINUX = os.path.exists('/etc/redhat-release') +SYSTEM_RELEASE_FILE = "/etc/system-release" + def _is_oracle_linux(): return _IS_ORACLE_LINUX @@ -84,16 +88,16 @@ def _is_redhat_linux(): def advanced_check(distribution): distribution = list(distribution) - if os.path.exists("/etc/issue"): - with open("/etc/issue", "rb") as fp: + if os.path.exists(SYSTEM_RELEASE_FILE): + with open(SYSTEM_RELEASE_FILE, "rb") as fp: issue_content = fp.read() if "Amazon" in issue_content: distribution[0] = "amazon" - search_groups = re.search('(\d+)\.(\d+)', issue_content) + search_groups = re.search('(\d+\.\d+)', issue_content) if search_groups: - distribution[1] = search_groups.group(1) # if version is 2015.09 only get 2015. + distribution[1] = search_groups.group(1) return tuple(distribution) @@ -114,16 +118,24 @@ class OS_CONST_TYPE(type): f = open(os.path.join(RESOURCES_DIR, OSFAMILY_JSON_RESOURCE)) json_data = eval(f.read()) f.close() - for family in json_data: + + if JSON_OS_MAPPING not in json_data: + raise Exception("Invalid {0}".format(OSFAMILY_JSON_RESOURCE)) + + json_mapping_data = json_data[JSON_OS_MAPPING] + + for family in json_mapping_data: cls.FAMILY_COLLECTION += [family] - cls.OS_COLLECTION += json_data[family][JSON_OS_TYPE] + cls.OS_COLLECTION += json_mapping_data[family][JSON_OS_TYPE] cls.OS_FAMILY_COLLECTION += [{ 'name': family, - 'os_list': json_data[family][JSON_OS_TYPE] + 'os_list': json_mapping_data[family][JSON_OS_TYPE] }] - if JSON_EXTENDS in json_data[family]: - cls.OS_FAMILY_COLLECTION[-1][JSON_EXTENDS] = json_data[family][JSON_EXTENDS] + if JSON_EXTENDS in json_mapping_data[family]: + cls.OS_FAMILY_COLLECTION[-1][JSON_EXTENDS] = json_mapping_data[family][JSON_EXTENDS] + + cls.OS_TYPE_ALIASES = json_data[JSON_OS_ALIASES] if JSON_OS_ALIASES in json_data else {} except: raise Exception("Couldn't load '%s' file" % OSFAMILY_JSON_RESOURCE) @@ -194,7 +206,24 @@ class OSCheck: distribution = ("Darwin", "TestOnly", "1.1.1", "1.1.1", "1.1") return distribution - + + @staticmethod + def get_alias(os_type, os_version): + version_parts = os_version.split('.') + full_os_and_major_version = os_type + version_parts[0] + + if full_os_and_major_version in OSConst.OS_TYPE_ALIASES: + alias = OSConst.OS_TYPE_ALIASES[full_os_and_major_version] + re_groups = re.search('(\D+)(\d+)$', alias).groups() + os_type = re_groups[0] + os_major_version = re_groups[1] + + version_parts[0] = os_major_version + os_version = '.'.join(version_parts) + + return os_type, os_version + + @staticmethod def get_os_type(): """ @@ -205,6 +234,10 @@ class OSCheck: In case cannot detect - exit. """ + return OSCheck.get_alias(OSCheck._get_os_type(), OSCheck._get_os_version())[0] + + @staticmethod + def _get_os_type(): # Read content from /etc/*-release file # Full release name dist = OSCheck.os_distribution() @@ -212,18 +245,18 @@ class OSCheck: # special cases if _is_oracle_linux(): - return 'oraclelinux' + operatingSystem = 'oraclelinux' elif operatingSystem.startswith('suse linux enterprise server'): - return 'sles' + operatingSystem = 'sles' elif operatingSystem.startswith('red hat enterprise linux'): - return 'redhat' + operatingSystem = 'redhat' elif operatingSystem.startswith('darwin'): - return 'mac' + operatingSystem = 'mac' - if operatingSystem != '': - return operatingSystem - else: + if operatingSystem == '': raise Exception("Cannot detect os type. Exiting...") + + return operatingSystem @staticmethod def get_os_family(): @@ -257,11 +290,15 @@ class OSCheck: In case cannot detect raises exception. """ + return OSCheck.get_alias(OSCheck._get_os_type(), OSCheck._get_os_version())[1] + + @staticmethod + def _get_os_version(): # Read content from /etc/*-release file # Full release name dist = OSCheck.os_distribution() dist = dist[1] - + if dist: return dist else: diff --git a/ambari-common/src/main/python/ambari_commons/resources/os_family.json b/ambari-common/src/main/python/ambari_commons/resources/os_family.json index 13014fc3a0..1558c1b101 100644 --- a/ambari-common/src/main/python/ambari_commons/resources/os_family.json +++ b/ambari-common/src/main/python/ambari_commons/resources/os_family.json @@ -1,72 +1,69 @@ { - "redhat": { - "distro": [ - "redhat", - "fedora", - "centos", - "oraclelinux", - "ascendos", - "xenserver", - "oel", - "ovs", - "cloudlinux", - "slc", - "scientific", - "psbm", - "centos linux" - ], - "versions": [ - 6, - 7 - ] - }, - "amazon": { - "extends" : "redhat", - "distro": [ - "amazon" - ], - "versions": [ - 2015 - ] - }, - "debian": { - "extends" : "ubuntu", - "distro": [ - "debian" - ], - "versions": [ - 7 - ] - }, - "ubuntu": { - "distro": [ - "ubuntu" - ], - "versions": [ - 12, - 14 - ] - }, - "suse": { - "distro": [ - "sles", - "sled", - "opensuse", - "suse" - ], - "versions": [ - 11 - ] - }, - "winsrv": { - "distro": [ - "win2008server", - "win2008serverr2", - "win2012server", - "win2012serverr2" - ], - "versions": [ - 6 - ] - } + "mapping": { + "redhat": { + "distro": [ + "redhat", + "fedora", + "centos", + "oraclelinux", + "amazon", + "ascendos", + "xenserver", + "oel", + "ovs", + "cloudlinux", + "slc", + "scientific", + "psbm", + "centos linux" + ], + "versions": [ + 6, + 7 + ] + }, + "debian": { + "extends" : "ubuntu", + "distro": [ + "debian" + ], + "versions": [ + 7 + ] + }, + "ubuntu": { + "distro": [ + "ubuntu" + ], + "versions": [ + 12, + 14 + ] + }, + "suse": { + "distro": [ + "sles", + "sled", + "opensuse", + "suse" + ], + "versions": [ + 11 + ] + }, + "winsrv": { + "distro": [ + "win2008server", + "win2008serverr2", + "win2012server", + "win2012serverr2" + ], + "versions": [ + 6 + ] + } + }, + "aliases": { + "amazon2015": "amazon6" + } } diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py index 0fb0fe4b5c..4ca3b7b5f6 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py @@ -19,7 +19,8 @@ limitations under the License. Ambari Agent """ -__all__ = ["is_lzo_enabled", "should_install_phoenix", "should_install_ams_collector", "should_install_mysql", "should_install_mysl_connector"] +__all__ = ["is_lzo_enabled", "should_install_phoenix", "should_install_ams_collector", "should_install_ams_grafana", + "should_install_mysql", "should_install_mysl_connector"] import os from resource_management.libraries.script import Script @@ -41,6 +42,10 @@ def should_install_ams_collector(): config = Script.get_config() return 'role' in config and config['role'] == "METRICS_COLLECTOR" +def should_install_ams_grafana(): + config = Script.get_config() + return 'role' in config and config['role'] == "METRICS_GRAFANA" + def should_install_mysql(): config = Script.get_config() hive_database = config['configurations']['hive-env']['hive_database'] diff --git a/ambari-funtest/pom.xml b/ambari-funtest/pom.xml index b9c591fc68..4fa342f07f 100644 --- a/ambari-funtest/pom.xml +++ b/ambari-funtest/pom.xml @@ -496,11 +496,6 @@ <version>4.2.5</version> </dependency> <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>14.0.1</version> - </dependency> - <dependency> <groupId>com.google.code.findbugs</groupId> <artifactId>jsr305</artifactId> <version>1.3.9</version> diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/services/AMBARI_METRICS/metainfo.xml b/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/services/AMBARI_METRICS/metainfo.xml index be99b830d3..fe3b860318 100644 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/services/AMBARI_METRICS/metainfo.xml +++ b/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/services/AMBARI_METRICS/metainfo.xml @@ -69,7 +69,7 @@ <packages> <package> <name>ambari-metrics-collector</name> - <condition>package_chooser.should_install_ams_collector()</condition> + <condition>should_install_ams_collector</condition> </package> <package> <name>ambari-metrics-monitor</name> diff --git a/ambari-metrics/ambari-metrics-grafana/README.md b/ambari-metrics/ambari-metrics-grafana/README.md index 0a138e8f8c..7eb84a7024 100644 --- a/ambari-metrics/ambari-metrics-grafana/README.md +++ b/ambari-metrics/ambari-metrics-grafana/README.md @@ -240,4 +240,24 @@ http://GRAFANA_HOST:3000 --- +<a name="templating"></a> +### Templating +**Templating allows you to dynamically change graphs by selecting your host from dropdown. To created a templated dashboard, you can follow these steps.** + +> 1. Click on the "cog" on the top, select "Templating" +> 2. Click on "+ New" +> 3. Enter a name for your variable. Ex: "hosts". +> 4. Type: query +> 5. Datasource: This should be set to the name you had chosen for your Datasource. Default is "AMBARI_METRICS" +> 6. Query: This needs to be "hosts". It is case sensitive as well. +> 7. Once you've entered the query and you focus your cursor anywhere else, you should see the "Preview of values" updated with the hostnames in your cluster. +> 8. You can close the Templating Variables section. +> 9. You should now have a dropdown on your dashboard with hosts from your cluster. +> 10. When you now add a graph, and select your component and metric, the plotted graph will show you metrics for the selected hostname from the dropdown. +> 11. The legend on the graph will also now update with the selected host. + +**Templalted dashboards do support multiple metrics in a single graph.** + + + diff --git a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js index 374501c200..e89ad0e73c 100644 --- a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js +++ b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js @@ -27,7 +27,7 @@ define([ var module = angular.module('grafana.services'); - module.factory('AmbariMetricsDatasource', function ($q, backendSrv) { + module.factory('AmbariMetricsDatasource', function ($q, backendSrv, templateSrv) { /** * AMS Datasource Constructor */ @@ -43,7 +43,9 @@ define([ .then(function (items) { allMetrics = []; appIds = []; - var fake = "timeline_metric_store_watcher"; delete items[fake]; + //We remove a couple of components from the list that do not contain any + //pertinent metrics. + delete items.timeline_metric_store_watcher; delete items.amssmoketestfake; for (var key in items) { if (items.hasOwnProperty(key)) { items[key].forEach(function (_item) { @@ -97,10 +99,11 @@ define([ } var series = []; var metricData = res.metrics[0].metrics; + var hostLegend = res.metrics[0].hostname ? ' on ' + res.metrics[0].hostname : ''; var timeSeries = {}; if (target.hosts === undefined || target.hosts.trim() === "") { timeSeries = { - target: target.metric, + target: target.metric + hostLegend, datapoints: [] }; } else { @@ -119,81 +122,42 @@ define([ }; }; - var precisionSetting = ''; + var getHostAppIdData = function(target) { - if (target.shouldAddPrecision) { - precisionSetting = '&precision=' + target.precision; - } else { - precisionSetting = ''; - } - if (target.shouldAddPrecision && target.shouldComputeRate) { - return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.metric + "._rate._" - + target.aggregator + "&hostname=" + target.hosts + '&appId=' + target.app + '&startTime=' + from - + '&endTime=' + to + precisionSetting).then( - getMetricsData(target) - ); - } else if (target.shouldComputeRate) { - return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.metric + "._rate._" - + target.aggregator + "&hostname=" + target.hosts + '&appId=' + target.app + '&startTime=' + from - + '&endTime=' + to).then( - getMetricsData(target) - ); - } else if (target.shouldAddPrecision){ - return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.metric + "._" - + target.aggregator + "&hostname=" + target.hosts + '&appId=' + target.app + '&startTime=' + from - + '&endTime=' + to + precisionSetting).then( - getMetricsData(target) - ); - } else { - return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.metric + "._" - + target.aggregator + "&hostname=" + target.hosts + '&appId=' + target.app + '&startTime=' + from - + '&endTime=' + to).then( + var precision = target.shouldAddPrecision ? '&precision=' + target.precision : ''; + var rate = target.shouldComputeRate ? '._rate._' : '._'; + return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.metric + rate + + target.aggregator + "&hostname=" + target.hosts + '&appId=' + target.app + '&startTime=' + from + + '&endTime=' + to + precision).then( getMetricsData(target) - ); - } + ); }; var getServiceAppIdData = function(target) { - if (target.shouldAddPrecision) { precisionSetting = '&precision=' + target.precision; - } else { precisionSetting = ''; } - if (target.shouldAddPrecision && target.shouldComputeRate) { - return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.metric + "._rate._" - + target.aggregator + '&appId=' + target.app + '&startTime=' + from + '&endTime=' + to + precisionSetting) - .then( - getMetricsData(target) - ); - } else if (target.shouldAddPrecision) { - return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.metric + "._" - + target.aggregator + '&appId=' + target.app + '&startTime=' + from + '&endTime=' + to + precisionSetting) - .then( - getMetricsData(target) - ); - } else if (target.shouldComputeRate) { - return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.metric + "._rate._" - + target.aggregator + '&appId=' + target.app + '&startTime=' + from + '&endTime=' + to).then( - getMetricsData(target) - ); - } else { - return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.metric + "._" - + target.aggregator + '&appId=' + target.app + '&startTime=' + from + '&endTime=' + to).then( - getMetricsData(target) - ); - } + var templatedHost = (_.isEmpty(templateSrv.variables)) ? "" : templateSrv.variables[0].options.filter(function(host) + { return host.selected; }).map(function(hostName) { return hostName.value; }); + var precision = target.shouldAddPrecision ? '&precision=' + target.precision : ''; + var rate = target.shouldComputeRate ? '._rate._' : '._'; + return backendSrv.get(self.url + '/ws/v1/timeline/metrics?metricNames=' + target.metric + rate + + target.aggregator + '&hostname=' + templatedHost + '&appId=' + target.app + '&startTime=' + from + + '&endTime=' + to + precision).then( + getMetricsData(target) + ); }; // Time Ranges var from = Math.floor(options.range.from.valueOf() / 1000); var to = Math.floor(options.range.to.valueOf() / 1000); - var metricsPromises = _.map(options.targets, function(target) { - console.debug('target app=' + target.app + ',' + - 'target metric=' + target.metric + ' on host=' + target.hosts); - if (!!target.hosts) { - return getHostAppIdData(target); - } else { - return getServiceAppIdData(target); - } - }); + console.debug('target app=' + target.app + ',' + + 'target metric=' + target.metric + ' on host=' + target.hosts); + if (!!target.hosts) { + return getHostAppIdData(target); + } else { + return getServiceAppIdData(target); + } + }); + return $q.all(metricsPromises).then(function(metricsDataArray) { var data = _.map(metricsDataArray, function(metricsData) { return metricsData.data; @@ -215,6 +179,32 @@ define([ }; /** + * AMS Datasource Templating Variables. + */ + AmbariMetricsDatasource.prototype.metricFindQuery = function (query) { + var interpolated; + try { + interpolated = templateSrv.replace(query); + } catch (err) { + return $q.reject(err); + } + return this.doAmbariRequest({ + method: 'GET', + url: '/ws/v1/timeline/metrics/' + interpolated + }) + .then(function (results) { + //Remove fakehostname from the list of hosts on the cluster. + var fake = "fakehostname"; delete results.data[fake]; + return _.map(_.keys(results.data), function (hostName) { + return { + text: hostName, + expandable: hostName.expandable ? true : false + }; + }); + }); + }; + + /** * AMS Datasource - Test Data Source Connection. * * Added Check to see if Datasource is working. Throws up an error in the @@ -277,6 +267,7 @@ define([ console.log(query); return this.doAmbariRequest({method: 'GET', url: '/ws/v1/timeline/metrics/hosts'}) .then(function (results) { + //Remove fakehostname from the list of hosts on the cluster. var fake = "fakehostname"; delete results.data[fake]; return _.map(Object.keys(results.data), function (hostName) { return {text: hostName}; @@ -301,4 +292,4 @@ define([ return AmbariMetricsDatasource; }); } -); +);
\ No newline at end of file diff --git a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/partials/query.editor.html b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/partials/query.editor.html index d4dffb4bdf..f51866e626 100644 --- a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/partials/query.editor.html +++ b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/partials/query.editor.html @@ -82,10 +82,10 @@ </li> - <li class="tight-form-item" style="width: 86px"> - Hosts <a bs-tooltip="'if host is selected, aggregator is ignored.'" data-placement="bottom"><i class="fa fa-info-circle"></i></a> + <li class="tight-form-item" style="width: 86px" ng-hide="dashboard.templating.list.length > 0"> + Hosts </li> - <li> + <li ng-hide="dashboard.templating.list.length > 0"> <input type="text" class="input-large tight-form-input" ng-model="target.hosts" spellcheck='false' bs-typeahead="suggestHosts" placeholder="host name" data-min-length=0 data-items=100 ng-blur="targetBlur()"> diff --git a/ambari-metrics/ambari-metrics-grafana/pom.xml b/ambari-metrics/ambari-metrics-grafana/pom.xml index 8304416ccc..c737b2b026 100644 --- a/ambari-metrics/ambari-metrics-grafana/pom.xml +++ b/ambari-metrics/ambari-metrics-grafana/pom.xml @@ -103,16 +103,12 @@ dest="${project.build.directory}/grafana" compression="gzip" /> - <get - src="${ambari-grafana.tar}" - dest="${project.build.directory}/grafana/ambari-grafana.tgz" - usetimestamp="true" - /> - <untar - src="${project.build.directory}/grafana/ambari-grafana.tgz" - dest="${project.build.directory}/grafana/${grafana.folder}/public/app/plugins/datasource" - compression="gzip" - /> + <copy + todir="${project.build.directory}/grafana/${grafana.folder}/public/app/plugins/datasource"> + <fileset dir="${project.build.directory}/../"> + <include name="ambari-metrics/"/> + </fileset> + </copy> </target> </configuration> </execution> diff --git a/ambari-metrics/ambari-metrics-grafana/screenshots/20-templating.png b/ambari-metrics/ambari-metrics-grafana/screenshots/20-templating.png Binary files differnew file mode 100644 index 0000000000..71b8b9f87d --- /dev/null +++ b/ambari-metrics/ambari-metrics-grafana/screenshots/20-templating.png diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java index 5ee8b44480..b5ec6e8d83 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java @@ -143,14 +143,16 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin aggregatorDailyThread.start(); } - int initDelay = configuration.getTimelineMetricsServiceWatcherInitDelay(); - int delay = configuration.getTimelineMetricsServiceWatcherDelay(); - // Start the watchdog - executorService.scheduleWithFixedDelay( - new TimelineMetricStoreWatcher(this, configuration), initDelay, delay, - TimeUnit.SECONDS); - LOG.info("Started watchdog for timeline metrics store with initial " + - "delay = " + initDelay + ", delay = " + delay); + if (!configuration.isTimelineMetricsServiceWatcherDisabled()) { + int initDelay = configuration.getTimelineMetricsServiceWatcherInitDelay(); + int delay = configuration.getTimelineMetricsServiceWatcherDelay(); + // Start the watchdog + executorService.scheduleWithFixedDelay( + new TimelineMetricStoreWatcher(this, configuration), initDelay, delay, + TimeUnit.SECONDS); + LOG.info("Started watchdog for timeline metrics store with initial " + + "delay = " + initDelay + ", delay = " + delay); + } isInitialized = true; } diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java index 7fa1d2dc16..1c86ebb53b 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java @@ -221,14 +221,14 @@ public class PhoenixHBaseAccessor { String encoding = metricsConf.get(HBASE_ENCODING_SCHEME, DEFAULT_ENCODING); String compression = metricsConf.get(HBASE_COMPRESSION_SCHEME, DEFAULT_TABLE_COMPRESSION); - String precisionTtl = metricsConf.get(PRECISION_TABLE_TTL, "86400"); //1 day - String hostMinTtl = metricsConf.get(HOST_MINUTE_TABLE_TTL, "604800"); //7 days - String hostHourTtl = metricsConf.get(HOST_HOUR_TABLE_TTL, "2592000"); //30 days - String hostDailyTtl = metricsConf.get(HOST_DAILY_TABLE_TTL, "31536000"); //1 year - String clusterSecTtl = metricsConf.get(CLUSTER_SECOND_TABLE_TTL, "2592000"); //7 days - String clusterMinTtl = metricsConf.get(CLUSTER_MINUTE_TABLE_TTL, "7776000"); //30 days - String clusterHourTtl = metricsConf.get(CLUSTER_HOUR_TABLE_TTL, "31536000"); //1 year - String clusterDailyTtl = metricsConf.get(CLUSTER_DAILY_TABLE_TTL, "63072000"); //2 years + String precisionTtl = getDaysInSeconds(metricsConf.get(PRECISION_TABLE_TTL, "1")); //1 day + String hostMinTtl = getDaysInSeconds(metricsConf.get(HOST_MINUTE_TABLE_TTL, "7")); //7 days + String hostHourTtl = getDaysInSeconds(metricsConf.get(HOST_HOUR_TABLE_TTL, "30")); //30 days + String hostDailyTtl = getDaysInSeconds(metricsConf.get(HOST_DAILY_TABLE_TTL, "365")); //1 year + String clusterSecTtl = getDaysInSeconds(metricsConf.get(CLUSTER_SECOND_TABLE_TTL, "7")); //7 days + String clusterMinTtl = getDaysInSeconds(metricsConf.get(CLUSTER_MINUTE_TABLE_TTL, "30")); //30 days + String clusterHourTtl = getDaysInSeconds(metricsConf.get(CLUSTER_HOUR_TABLE_TTL, "365")); //1 year + String clusterDailyTtl = getDaysInSeconds(metricsConf.get(CLUSTER_DAILY_TABLE_TTL, "730")); //2 years try { LOG.info("Initializing metrics schema..."); @@ -1255,4 +1255,9 @@ public class PhoenixHBaseAccessor { return metadataMap; } + + private String getDaysInSeconds(String daysString) { + double days = Double.valueOf(daysString.trim()); + return String.valueOf((int)(days*86400)); + } } diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java index 46f61fbc74..cc9a2f368c 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java @@ -181,6 +181,9 @@ public class TimelineMetricConfiguration { public static final String HANDLER_THREAD_COUNT = "timeline.metrics.service.handler.thread.count"; + public static final String WATCHER_DISABLED = + "timeline.metrics.service.watcher.disabled"; + public static final String WATCHER_INITIAL_DELAY = "timeline.metrics.service.watcher.initial.delay"; @@ -273,6 +276,13 @@ public class TimelineMetricConfiguration { return 20; } + public boolean isTimelineMetricsServiceWatcherDisabled() { + if (metricsConf != null) { + return Boolean.parseBoolean(metricsConf.get(WATCHER_DISABLED, "false")); + } + return false; + } + public int getTimelineMetricsServiceWatcherInitDelay() { if (metricsConf != null) { return Integer.parseInt(metricsConf.get(WATCHER_INITIAL_DELAY, "600")); diff --git a/ambari-metrics/pom.xml b/ambari-metrics/pom.xml index 9a39122dfc..1f8768328c 100644 --- a/ambari-metrics/pom.xml +++ b/ambari-metrics/pom.xml @@ -52,7 +52,6 @@ <hadoop.winpkg.folder>hadoop-2.6.0.2.2.4.2-0002</hadoop.winpkg.folder> <grafana.folder>grafana-2.6.0</grafana.folder> <grafana.tar>https://grafanarel.s3.amazonaws.com/builds/grafana-2.6.0.linux-x64.tar.gz</grafana.tar> - <ambari-grafana.tar>https://github.com/u39kun/ambari-grafana/raw/master/dist/ambari-grafana.tgz</ambari-grafana.tar> <resmonitor.install.dir> /usr/lib/python2.6/site-packages/resource_monitoring </resmonitor.install.dir> diff --git a/ambari-project/pom.xml b/ambari-project/pom.xml index f19ca14db0..ed940044df 100644 --- a/ambari-project/pom.xml +++ b/ambari-project/pom.xml @@ -220,7 +220,7 @@ <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> - <version>14.0.1</version> + <version>16.0</version> </dependency> <dependency> <groupId>com.google.code.findbugs</groupId> diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index b500fb2a80..51fd88b458 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -1179,11 +1179,6 @@ <version>4.2.5</version> </dependency> <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>14.0.1</version> - </dependency> - <dependency> <groupId>com.google.code.findbugs</groupId> <artifactId>jsr305</artifactId> <version>1.3.9</version> diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java index 210fe176c7..248ce4bb70 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java @@ -989,7 +989,44 @@ public class HeartBeatHandler { if(response.getAgentConfig() != null) { LOG.debug("Agent configuration map set to " + response.getAgentConfig()); } - response.setRecoveryConfig(RecoveryConfig.getRecoveryConfig(config)); + + // + // Filter the enabled components by maintenance mode + // + + // Build a map of component name => Service component host + // for easy look up of maintenance state by component name. + // As of now, a host can belong to only one cluster. + // Clusters::getClustersForHost(hostname) returns one item. + Map<String, ServiceComponentHost> schFromComponentName = new HashMap<>(); + + for (Cluster cl : clusterFsm.getClustersForHost(hostname)) { + List<ServiceComponentHost> scHosts = cl.getServiceComponentHosts(hostname); + for (ServiceComponentHost sch : scHosts) { + schFromComponentName.put(sch.getServiceComponentName(), sch); + } + } + + // Keep only the components that have maintenance state set to OFF + List<String> enabledComponents = new ArrayList<>(); + String[] confEnabledComponents = config.getEnabledComponents().split(","); + + for (String componentName : confEnabledComponents) { + ServiceComponentHost sch = schFromComponentName.get(componentName); + + // Append the component name only if it is + // in the host and not in maintenance mode. + if (sch != null && sch.getMaintenanceState() == MaintenanceState.OFF) { + enabledComponents.add(componentName); + } + } + + // Overwrite the pre-constructed RecoveryConfig's list of + // enabled components with the filtered list + RecoveryConfig rc = RecoveryConfig.getRecoveryConfig(config); + rc.setEnabledComponents(StringUtils.join(enabledComponents, ',')); + response.setRecoveryConfig(rc); + if(response.getRecoveryConfig() != null) { LOG.info("Recovery configuration set to " + response.getRecoveryConfig().toString()); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/RecoveryConfig.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/RecoveryConfig.java index 3f558ebb4a..3da86092ff 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/RecoveryConfig.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/RecoveryConfig.java @@ -48,21 +48,9 @@ public class RecoveryConfig { @SerializedName("maxLifetimeCount") private String maxLifetimeCount; - @SerializedName("enabledComponents") + @SerializedName("components") private String enabledComponents; - @SerializedName("disabledComponents") - private String disabledComponents; - - - public String getDisabledComponents() { - return disabledComponents; - } - - public void setDisabledComponents(String disabledComponents) { - this.disabledComponents = disabledComponents; - } - public String getEnabledComponents() { return enabledComponents; } @@ -118,7 +106,6 @@ public class RecoveryConfig { rc.setRetryGap(conf.getNodeRecoveryRetryGap()); rc.setType(conf.getNodeRecoveryType()); rc.setWindowInMinutes(conf.getNodeRecoveryWindowInMin()); - rc.setDisabledComponents(conf.getDisabledComponents()); rc.setEnabledComponents(conf.getEnabledComponents()); return rc; } @@ -131,8 +118,7 @@ public class RecoveryConfig { buffer.append(", windowInMinutes=").append(windowInMinutes); buffer.append(", retryGap=").append(retryGap); buffer.append(", maxLifetimeCount=").append(maxLifetimeCount); - buffer.append(", disabledComponents=").append(disabledComponents); - buffer.append(", enabledComponents=").append(enabledComponents); + buffer.append(", components=").append(enabledComponents); buffer.append('}'); return buffer.toString(); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/notifications/dispatchers/AlertScriptDispatcher.java b/ambari-server/src/main/java/org/apache/ambari/server/notifications/dispatchers/AlertScriptDispatcher.java index 092aaf4ba4..907588dc47 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/notifications/dispatchers/AlertScriptDispatcher.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/notifications/dispatchers/AlertScriptDispatcher.java @@ -40,6 +40,8 @@ import org.apache.commons.lang.SystemUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.escape.Escaper; +import com.google.common.escape.Escapers; import com.google.inject.Inject; /** @@ -82,6 +84,18 @@ public class AlertScriptDispatcher implements NotificationDispatcher { private static final long DEFAULT_SCRIPT_TIMEOUT = 5000L; /** + * Used to escape text being passed into the shell command. + */ + public static final Escaper SHELL_ESCAPE; + + static { + final Escapers.Builder builder = Escapers.builder(); + builder.addEscape('\"', "\\\""); + builder.addEscape('!', "\\!"); + SHELL_ESCAPE = builder.build(); + } + + /** * Configuration data from the ambari.properties file. */ @Inject @@ -242,8 +256,8 @@ public class AlertScriptDispatcher implements NotificationDispatcher { // these could have spaces in them, so quote them so they don't mess up the // command line - String alertLabel = "\"" + definition.getLabel() + "\""; - String alertText = "\"" + alertInfo.getAlertText() + "\""; + String alertLabel = "\"" + SHELL_ESCAPE.escape(definition.getLabel()) + "\""; + String alertText = "\"" + SHELL_ESCAPE.escape(alertInfo.getAlertText()) + "\""; Object[] params = new Object[] { script, definitionName, alertLabel, serviceName, alertState.name(), alertText }; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/services/AlertNoticeDispatchService.java b/ambari-server/src/main/java/org/apache/ambari/server/state/services/AlertNoticeDispatchService.java index 3a88f37ae4..a27bc1d33b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/services/AlertNoticeDispatchService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/services/AlertNoticeDispatchService.java @@ -712,7 +712,7 @@ public class AlertNoticeDispatchService extends AbstractScheduledService { * * @param history */ - protected AlertInfo(AlertHistoryEntity history) { + public AlertInfo(AlertHistoryEntity history) { m_history = history; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/services/CachedAlertFlushService.java b/ambari-server/src/main/java/org/apache/ambari/server/state/services/CachedAlertFlushService.java index 72bf68ab73..2e38c9bc16 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/services/CachedAlertFlushService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/services/CachedAlertFlushService.java @@ -76,7 +76,7 @@ public class CachedAlertFlushService extends AbstractScheduledService { protected void startUp() throws Exception { boolean enabled = m_configuration.isAlertCacheEnabled(); if (!enabled) { - stop(); + stopAsync(); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/JsonOsFamilyRoot.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/JsonOsFamilyRoot.java new file mode 100644 index 0000000000..3f9158f807 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/JsonOsFamilyRoot.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state.stack; + +import java.util.Map; + +public class JsonOsFamilyRoot { + private Map<String, JsonOsFamilyEntry> mapping; + private Map<String, String> aliases; + + public Map<String, JsonOsFamilyEntry> getMapping() { + return mapping; + } + public void setMapping(Map<String, JsonOsFamilyEntry> mapping) { + this.mapping = mapping; + } + public Map<String, String> getAliases() { + return aliases; + } + public void setAliases(Map<String, String> aliases) { + this.aliases = aliases; + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java index 37a6db3c36..e494c4421b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java @@ -48,11 +48,14 @@ public class OsFamily { private final String os_pattern = "([\\D]+|(?:[\\D]+[\\d]+[\\D]+))([\\d]*)"; private final String OS_DISTRO = "distro"; private final String OS_VERSION = "versions"; + private final String OS_MAPPING = "mapping"; + private final String OS_ALIASES = "aliases"; private final String LOAD_CONFIG_MSG = "Could not load OS family definition from %s file"; private final String FILE_NAME = "os_family.json"; private final Logger LOG = LoggerFactory.getLogger(OsFamily.class); private Map<String, JsonOsFamilyEntry> osMap = null; + private JsonOsFamilyRoot jsonOsFamily = null; /** * Initialize object @@ -77,9 +80,10 @@ public class OsFamily { if (!f.exists()) throw new Exception(); inputStream = new FileInputStream(f); - Type type = new TypeToken<Map<String, JsonOsFamilyEntry>>() {}.getType(); + Type type = new TypeToken<JsonOsFamilyRoot>() {}.getType(); Gson gson = new Gson(); - osMap = gson.fromJson(new InputStreamReader(inputStream), type); + jsonOsFamily = gson.fromJson(new InputStreamReader(inputStream), type); + osMap = jsonOsFamily.getMapping(); } catch (Exception e) { LOG.error(String.format(LOAD_CONFIG_MSG, new File(SharedResourcesPath, FILE_NAME).toString())); throw new RuntimeException(e); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java index 7e2683d9ee..88b3151fbc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java @@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory; import java.sql.SQLException; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; /** * Upgrade catalog for version 2.2.2. @@ -52,8 +53,16 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog { "timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier"; private static final String CLUSTER_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER = "timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier"; - - + private static final String TIMELINE_METRICS_SERVICE_WATCHER_DISBALED = "timeline.metrics.service.watcher.disabled"; + private static final String AMS_MODE = "timeline.metrics.service.operation.mode"; + public static final String PRECISION_TABLE_TTL = "timeline.metrics.host.aggregator.ttl"; + public static final String HOST_MINUTE_TABLE_TTL = "timeline.metrics.host.aggregator.minute.ttl"; + public static final String HOST_HOUR_TABLE_TTL = "timeline.metrics.host.aggregator.hourly.ttl"; + public static final String HOST_DAILY_TABLE_TTL = "timeline.metrics.host.aggregator.daily.ttl"; + public static final String CLUSTER_SECOND_TABLE_TTL = "timeline.metrics.cluster.aggregator.second.ttl"; + public static final String CLUSTER_MINUTE_TABLE_TTL = "timeline.metrics.cluster.aggregator.minute.ttl"; + public static final String CLUSTER_HOUR_TABLE_TTL = "timeline.metrics.cluster.aggregator.hourly.ttl"; + public static final String CLUSTER_DAILY_TABLE_TTL = "timeline.metrics.cluster.aggregator.daily.ttl"; // ----- Constructors ------------------------------------------------------ @@ -175,6 +184,94 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog { } + if (!amsSiteProperties.containsKey(TIMELINE_METRICS_SERVICE_WATCHER_DISBALED)) { + LOG.info("Add config " + TIMELINE_METRICS_SERVICE_WATCHER_DISBALED + " = false"); + newProperties.put(TIMELINE_METRICS_SERVICE_WATCHER_DISBALED, String.valueOf(false)); + } + + boolean isDistributed = false; + if ("distributed".equals(amsSite.getProperties().get(AMS_MODE))) { + isDistributed = true; + } + + if (amsSiteProperties.containsKey(PRECISION_TABLE_TTL)) { + String oldTtl = amsSiteProperties.get(PRECISION_TABLE_TTL); + String newTtl = convertToDaysIfInSeconds(oldTtl); + if (isDistributed) { + if ("86400".equals(oldTtl)) { + newTtl = "7.0"; // 7 days + } + } + newProperties.put(PRECISION_TABLE_TTL, newTtl); + LOG.info("Setting value of " + PRECISION_TABLE_TTL + " : " + newTtl); + } + + if (amsSiteProperties.containsKey(HOST_MINUTE_TABLE_TTL)) { + String oldTtl = amsSiteProperties.get(HOST_MINUTE_TABLE_TTL); + String newTtl = convertToDaysIfInSeconds(oldTtl); + newProperties.put(HOST_MINUTE_TABLE_TTL, newTtl); + LOG.info("Setting value of " + HOST_MINUTE_TABLE_TTL + " : " + newTtl); + } + + if (amsSiteProperties.containsKey(HOST_MINUTE_TABLE_TTL)) { + String oldTtl = amsSiteProperties.get(HOST_MINUTE_TABLE_TTL); + String newTtl = convertToDaysIfInSeconds(oldTtl); + newProperties.put(HOST_MINUTE_TABLE_TTL, newTtl); + LOG.info("Setting value of " + HOST_MINUTE_TABLE_TTL + " : " + newTtl); + } + + if (amsSiteProperties.containsKey(HOST_HOUR_TABLE_TTL)) { + String oldTtl = amsSiteProperties.get(HOST_HOUR_TABLE_TTL); + String newTtl = convertToDaysIfInSeconds(oldTtl); + newProperties.put(HOST_HOUR_TABLE_TTL, newTtl); + LOG.info("Setting value of " + HOST_HOUR_TABLE_TTL + " : " + newTtl); + } + + if (amsSiteProperties.containsKey(HOST_DAILY_TABLE_TTL)) { + String oldTtl = amsSiteProperties.get(HOST_DAILY_TABLE_TTL); + String newTtl = convertToDaysIfInSeconds(oldTtl); + newProperties.put(HOST_DAILY_TABLE_TTL, newTtl); + LOG.info("Setting value of " + HOST_DAILY_TABLE_TTL + " : " + newTtl); + } + + if (amsSiteProperties.containsKey(CLUSTER_SECOND_TABLE_TTL)) { + String oldTtl = amsSiteProperties.get(CLUSTER_SECOND_TABLE_TTL); + String newTtl = convertToDaysIfInSeconds(oldTtl); + + if ("2592000".equals(oldTtl)) { + newTtl = "7.0"; // 7 days + } + + newProperties.put(CLUSTER_SECOND_TABLE_TTL, newTtl); + LOG.info("Setting value of " + CLUSTER_SECOND_TABLE_TTL + " : " + newTtl); + } + + if (amsSiteProperties.containsKey(CLUSTER_MINUTE_TABLE_TTL)) { + String oldTtl = amsSiteProperties.get(CLUSTER_MINUTE_TABLE_TTL); + String newTtl = convertToDaysIfInSeconds(oldTtl); + + if ("7776000".equals(oldTtl)) { + newTtl = "30.0"; // 30 days + } + + newProperties.put(CLUSTER_MINUTE_TABLE_TTL, newTtl); + LOG.info("Setting value of " + CLUSTER_MINUTE_TABLE_TTL + " : " + newTtl); + } + + if (amsSiteProperties.containsKey(CLUSTER_HOUR_TABLE_TTL)) { + String oldTtl = amsSiteProperties.get(CLUSTER_HOUR_TABLE_TTL); + String newTtl = convertToDaysIfInSeconds(oldTtl); + newProperties.put(CLUSTER_HOUR_TABLE_TTL, newTtl); + LOG.info("Setting value of " + CLUSTER_HOUR_TABLE_TTL + " : " + newTtl); + } + + if (amsSiteProperties.containsKey(CLUSTER_DAILY_TABLE_TTL)) { + String oldTtl = amsSiteProperties.get(CLUSTER_DAILY_TABLE_TTL); + String newTtl = convertToDaysIfInSeconds(oldTtl); + newProperties.put(CLUSTER_DAILY_TABLE_TTL, newTtl); + LOG.info("Setting value of " + CLUSTER_DAILY_TABLE_TTL + " : " + newTtl); + } + updateConfigurationPropertiesForCluster(cluster, AMS_SITE, newProperties, true, true); } @@ -183,4 +280,19 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog { } } + private String convertToDaysIfInSeconds(String secondsString) { + + int seconds = Integer.valueOf(secondsString); + double days = 0.0; + + if (seconds >= 86400) { + days += TimeUnit.SECONDS.toDays(seconds); + } + + days += ((float)seconds % 86400.0) / 86400.0; + days = Math.round(days * 100.0)/100.0; + + return String.valueOf(days); + } + } diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-site.xml b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-site.xml index 2e7576823e..c2e0200924 100644 --- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-site.xml +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-site.xml @@ -24,6 +24,7 @@ <property> <name>instance.volumes</name> <value>hdfs://localhost:8020/apps/accumulo/data</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>Location of Accumulo data files in HDFS.</description> <value-attributes> <overridable>false</overridable> @@ -33,6 +34,7 @@ <property> <name>instance.zookeeper.host</name> <value>localhost:2181</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>Comma-separated list of ZooKeeper server:port pairs.</description> <value-attributes> diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml index a3f0028000..180b43bb71 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml @@ -283,59 +283,65 @@ </property> <property> <name>timeline.metrics.host.aggregator.daily.ttl</name> - <value>31536000</value> + <value>365</value> <description> Host based daily resolution data purge interval. Default is 1 year. </description> </property> <property> <name>timeline.metrics.host.aggregator.hourly.ttl</name> - <value>2592000</value> + <value>30</value> <description> Host based hourly resolution data purge interval. Default is 30 days. </description> </property> <property> <name>timeline.metrics.host.aggregator.minute.ttl</name> - <value>604800</value> + <value>7</value> <description> Host based minute resolution data purge interval. Default is 7 days. </description> </property> <property> <name>timeline.metrics.cluster.aggregator.second.ttl</name> - <value>2592000</value> + <value>7</value> <description> Cluster wide second resolution data purge interval. Default is 7 days. </description> </property> <property> <name>timeline.metrics.cluster.aggregator.minute.ttl</name> - <value>7776000</value> + <value>30</value> <description> Cluster wide minute resolution data purge interval. Default is 30 days. </description> </property> <property> <name>timeline.metrics.cluster.aggregator.hourly.ttl</name> - <value>31536000</value> + <value>365</value> <description> Cluster wide hourly resolution data purge interval. Default is 1 year. </description> </property> <property> <name>timeline.metrics.cluster.aggregator.daily.ttl</name> - <value>63072000</value> + <value>730</value> <description> Cluster wide daily resolution data purge interval. Default is 2 years. </description> </property> <property> <name>timeline.metrics.host.aggregator.ttl</name> - <value>86400</value> + <value>1</value> <description> - 1 minute resolution data purge interval. Default is 1 day. + 1 minute resolution data purge interval. Default is 1 day for embedded mode and 7 days for Distributed mode. </description> + <depends-on> + <property> + <type>ams-site</type> + <name>timeline.metrics.service.operation.mode</name> + </property> + </depends-on> </property> <property> <name>timeline.metrics.hbase.data.block.encoding</name> @@ -495,6 +501,19 @@ </description> </property> <property> + <name>timeline.metrics.service.watcher.disabled</name> + <value>false</value> + <description> + Disable Timeline Metric Store watcher thread. Disabled by default in AMS distributed mode. + </description> + <depends-on> + <property> + <type>ams-site</type> + <name>timeline.metrics.service.operation.mode</name> + </property> + </depends-on> + </property> + <property> <name>timeline.metrics.service.watcher.initial.delay</name> <value>600</value> <description> diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml index c3dbc6b00b..3a832eb99c 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml @@ -83,6 +83,7 @@ <package> <name>ambari-metrics-collector</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_ams_collector</condition> </package> <package> <name>ambari-metrics-monitor</name> @@ -95,6 +96,7 @@ <package> <name>ambari-metrics-grafana</name> <skipUpgrade>true</skipUpgrade> + <condition>should_install_ams_grafana</condition> </package> <package> <name>gcc</name> diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py index ddefed7fad..533d3ec952 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_collector.py @@ -31,7 +31,9 @@ from ambari_commons.os_family_impl import OsFamilyImpl class AmsCollector(Script): def install(self, env): - self.install_packages(env, exclude_packages = ['ambari-metrics-grafana']) + import params + env.set_params(params) + self.install_packages(env) def configure(self, env, action = None): import params diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py index d96309c4dd..6dd60f56da 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana.py @@ -26,7 +26,10 @@ from metrics_grafana_util import create_ams_datasource class AmsGrafana(Script): def install(self, env): - self.install_packages(env, exclude_packages = ['ambari-metrics-collector']) + import params + env.set_params(params) + self.install_packages(env) + self.configure(env) # for security def configure(self, env, action = None): import params diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py index c4a91e121a..5ab40b02d2 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_grafana_util.py @@ -58,8 +58,9 @@ def create_ams_datasource(): Logger.info("Ambari Metrics Grafana datasource already present. Checking Metrics Collector URL") datasource_url = datasources_json[i]["url"] - if datasource_url == (params.metric_collector_host + ":" + params.metric_collector_port - + "/ws/v1/timeline/metrics") : + if datasource_url == (params.ams_grafana_protocol + "://" + + params.metric_collector_host + ":" + + params.metric_collector_port): Logger.info("Metrics Collector URL validation succeeded. Skipping datasource creation") GRAFANA_CONNECT_TRIES = 0 # No need to create datasource again @@ -84,6 +85,7 @@ def create_ams_datasource(): try: ams_datasource_json = Template('metrics_grafana_datasource.json.j2', ams_datasource_name=METRICS_GRAFANA_DATASOURCE_NAME, + ams_grafana_protocol=params.ams_grafana_protocol, ams_collector_host=params.metric_collector_host, ams_collector_port=params.metric_collector_port).get_content() diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_monitor.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_monitor.py index 921bed7ac4..d98384ba1d 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_monitor.py +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/metrics_monitor.py @@ -27,7 +27,7 @@ class AmsMonitor(Script): def install(self, env): import params env.set_params(params) - self.install_packages(env, exclude_packages = ['ambari-metrics-collector', 'ambari-metrics-grafana']) + self.install_packages(env) self.configure(env) # for security def configure(self, env): diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2 index 1b03a2d060..a803da5ba0 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2 +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/metrics_grafana_datasource.json.j2 @@ -20,7 +20,7 @@ "name": "{{ams_datasource_name}}", "type": "ambarimetrics", "access": "proxy", - "url": "{{ams_collector_host}}:{{ams_collector_port}}/ws/v1/timeline/metrics", + "url": "{{ams_grafana_protocol}}://{{ams_collector_host}}:{{ams_collector_port}}", "password": "", "user": "", "database": "", diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml index 74b1537305..ed9f1bc7f9 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml @@ -102,6 +102,7 @@ <property> <name>atlas.server.bind.address</name> <value>localhost</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description></description> </property> <property> diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml index 66a297c8c5..431d9632e3 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml @@ -120,6 +120,7 @@ <property> <name>*.broker.url</name> <value>tcp://localhost:61616</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>Default Active MQ url</description> </property> <property> diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml index 21ab4459b8..ce625eb33d 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml @@ -76,6 +76,16 @@ <scriptType>PYTHON</scriptType> <timeout>600</timeout> </commandScript> + <customCommands> + <customCommand> + <name>ACTIVATE_STANDBY</name> + <commandScript> + <script>scripts/hawqstandby.py</script> + <scriptType>PYTHON</scriptType> + <timeout>1200</timeout> + </commandScript> + </customCommand> + </customCommands> <dependencies> <dependency> <name>HDFS/HDFS_CLIENT</name> diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py index 01de99a773..6a2d9ba43f 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py @@ -27,6 +27,7 @@ YARN = "yarn" CLUSTER = "cluster" IMMEDIATE = "immediate" FAST = "fast" +ACTIVATE = "activate" # Users root_user = "root" diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstandby.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstandby.py index 7f5bab4e93..7da7f6df8a 100644 --- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstandby.py +++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstandby.py @@ -51,8 +51,10 @@ class HawqStandby(Script): from hawqstatus import get_pid_file check_process_status(get_pid_file()) - def activatestandby(self, env): - pass + def activate_standby(self, env): + import utils + utils.exec_hawq_operation(hawq_constants.ACTIVATE, "{0} -a -M {1} -v".format(hawq_constants.STANDBY, hawq_constants.FAST)) + if __name__ == "__main__": HawqStandby().execute() diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-site.xml b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-site.xml index 4fefb1e511..3575bf2f8e 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-site.xml +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-site.xml @@ -23,6 +23,7 @@ <property> <name>hbase.rootdir</name> <value>hdfs://localhost:8020/apps/hbase/data</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>The directory shared by region servers and into which HBase persists. The URL should be 'fully-qualified' to include the filesystem scheme. For example, to specify the diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/core-site.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/core-site.xml index 4d73250e8a..d216605224 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/core-site.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/core-site.xml @@ -59,6 +59,7 @@ <name>fs.defaultFS</name> <!-- cluster variant --> <value>hdfs://localhost:8020</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>The name of the default file system. Either the literal string "local" or a host:port for HDFS.</description> <final>true</final> diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml index ea5514da55..260fe65130 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/configuration/hdfs-site.xml @@ -255,6 +255,7 @@ <property> <name>dfs.namenode.http-address</name> <value>localhost:50070</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>The name of the default file system. Either the literal string "local" or a host:port for HDFS.</description> <final>true</final> @@ -366,6 +367,7 @@ <!-- cluster variant --> <name>dfs.namenode.secondary.http-address</name> <value>localhost:50090</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>Address of secondary namenode web server</description> </property> @@ -373,6 +375,7 @@ <property> <name>dfs.namenode.https-address</name> <value>localhost:50470</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>The https address where namenode binds</description> </property> diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml index 80da3a1955..4087c72cbe 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml @@ -46,6 +46,7 @@ limitations under the License. <property> <name>javax.jdo.option.ConnectionURL</name> <value>jdbc:mysql://localhost/hive?createDatabaseIfNotExist=true</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <display-name>Database URL</display-name> <description>JDBC connect string for a JDBC metastore</description> <value-attributes> @@ -124,6 +125,7 @@ limitations under the License. <property> <name>hive.metastore.uris</name> <value>thrift://localhost:9083</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>URI for client to contact metastore server</description> </property> @@ -444,6 +446,7 @@ limitations under the License. <property> <name>atlas.rest.address</name> <value>http://localhost:21000</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <depends-on> <property> <type>application-properties</type> diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml index 72a57105b2..be2f22133b 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/webhcat-site.xml @@ -117,6 +117,7 @@ limitations under the License. <property> <name>templeton.zookeeper.hosts</name> <value>localhost:2181</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>ZooKeeper servers, as comma separated host:port pairs</description> <value-attributes> <type>multiLine</type> diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-broker.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-broker.xml index b23075d39b..6d0c8c662a 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-broker.xml +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/configuration/kafka-broker.xml @@ -45,6 +45,7 @@ <property> <name>zookeeper.connect</name> <value>localhost:2181</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description> Zookeeper also allows you to add a "chroot" path which will make all kafka data for this cluster appear under a particular path. This is a way to setup multiple Kafka clusters or other applications on the same zookeeper cluster. To do this give a connection @@ -295,6 +296,7 @@ <property> <name>kafka.ganglia.metrics.host</name> <value>localhost</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description> Ganglia host </description> </property> <property> diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/ranger-knox-plugin-properties.xml b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/ranger-knox-plugin-properties.xml index 0ab649cd61..eef1daf9af 100644 --- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/ranger-knox-plugin-properties.xml +++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/configuration/ranger-knox-plugin-properties.xml @@ -113,6 +113,7 @@ <property> <name>XAAUDIT.HDFS.DESTINATION_DIRECTORY</name> <value>hdfs://localhost:8020/ranger/audit/%app-type%/%time:yyyyMMdd%</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description></description> <depends-on> <property> diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-site.xml b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-site.xml index 5a188758d5..4a8e60c673 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-site.xml +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-site.xml @@ -26,6 +26,7 @@ <property> <name>oozie.base.url</name> <value>http://localhost:11000/oozie</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>Base Oozie URL.</description> </property> diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/configuration/oozie-site.xml b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/configuration/oozie-site.xml index 2ecc0ca4c1..107816dc0e 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/configuration/oozie-site.xml +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.2.3/configuration/oozie-site.xml @@ -35,6 +35,7 @@ <property> <name>oozie.base.url</name> <value>http://localhost:11000/oozie</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>Base Oozie URL.</description> </property> diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-env.xml b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-env.xml index 0cffae0365..d73272d3b3 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-env.xml +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-env.xml @@ -268,6 +268,7 @@ <property> <name>xasecure.audit.destination.hdfs.dir</name> <value>hdfs://localhost:8020/ranger/audit/%app-type%/%time:yyyyMMdd%</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <display-name>Destination HDFS Directory</display-name> <description>HDFS folder to write audit to, make sure all service user has required permissions. This property is overridable at service level</description> <depends-on> diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-site.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-site.xml index 889e3107c2..e559d92900 100644 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-site.xml +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-site.xml @@ -23,6 +23,7 @@ <property> <name>hadoop.kms.key.provider.uri</name> <value>dbks://http@localhost:9292/kms</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>URI of the backing KeyProvider for the KMS.</description> </property> diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml index 4719dbb4af..aada3632fc 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml @@ -42,6 +42,7 @@ <property> <name>storm.zookeeper.servers</name> <value>['localhost']</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>A list of hosts of ZooKeeper servers used to manage the cluster.</description> <value-attributes> <type>multiLine</type> diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml index 5737c4d47d..f30b807a17 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml @@ -257,12 +257,14 @@ <property> <name>mapreduce.jobhistory.address</name> <value>localhost:10020</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>Enter your JobHistoryServer hostname.</description> </property> <property> <name>mapreduce.jobhistory.webapp.address</name> <value>localhost:19888</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>Enter your JobHistoryServer hostname.</description> </property> diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml index 802a1f5d0b..59d4964615 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml @@ -26,24 +26,28 @@ <property> <name>yarn.resourcemanager.hostname</name> <value>localhost</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>The hostname of the RM.</description> </property> <property> <name>yarn.resourcemanager.resource-tracker.address</name> <value>localhost:8025</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description> The address of ResourceManager. </description> </property> <property> <name>yarn.resourcemanager.scheduler.address</name> <value>localhost:8030</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>The address of the scheduler interface.</description> </property> <property> <name>yarn.resourcemanager.address</name> <value>localhost:8050</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description> The address of the applications manager interface in the RM. @@ -53,6 +57,7 @@ <property> <name>yarn.resourcemanager.admin.address</name> <value>localhost:8141</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>The address of the RM admin interface.</description> </property> @@ -372,6 +377,7 @@ <property> <name>yarn.resourcemanager.webapp.address</name> <value>localhost:8088</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description> The address of the RM web application. </description> @@ -380,6 +386,7 @@ <property> <name>yarn.resourcemanager.webapp.https.address</name> <value>localhost:8090</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description> The https address of the RM web application. </description> @@ -396,6 +403,7 @@ <property> <name>yarn.log.server.url</name> <value>http://localhost:19888/jobhistory/logs</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description> URI for the HistoryServer's log resource </description> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py index a53dd8b80b..af21008794 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py @@ -507,8 +507,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor): operatingMode = services["configurations"]["ams-site"]["properties"]["timeline.metrics.service.operation.mode"] if operatingMode == "distributed": + putAmsSiteProperty("timeline.metrics.service.watcher.disabled", 'true') + putAmsSiteProperty("timeline.metrics.host.aggregator.ttl", 7) putAmsHbaseSiteProperty("hbase.cluster.distributed", 'true') else: + putAmsSiteProperty("timeline.metrics.service.watcher.disabled", 'false') + putAmsSiteProperty("timeline.metrics.host.aggregator.ttl", 1) putAmsHbaseSiteProperty("hbase.cluster.distributed", 'false') rootDir = "file:///var/lib/ambari-metrics-collector/hbase" @@ -564,7 +568,6 @@ class HDP206StackAdvisor(DefaultStackAdvisor): putAmsHbaseSiteProperty("hbase.hregion.memstore.flush.size", 134217728) putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.upperLimit", 0.35) putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.lowerLimit", 0.3) - putAmsSiteProperty("timeline.metrics.host.aggregator.ttl", 86400) if len(amsCollectorHosts) > 1: pass diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml index aad9ba5ccd..56dc017ff1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml @@ -44,6 +44,7 @@ limitations under the License. <property> <name>javax.jdo.option.ConnectionURL</name> <value>jdbc:mysql://localhost/hive?createDatabaseIfNotExist=true</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <display-name>Database URL</display-name> <description>JDBC connect string for a JDBC metastore</description> <value-attributes> @@ -120,6 +121,7 @@ limitations under the License. <property> <name>hive.metastore.uris</name> <value>thrift://localhost:9083</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>URI for client to contact metastore server</description> </property> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/OOZIE/configuration/oozie-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/OOZIE/configuration/oozie-site.xml index af1a0a8373..b4d29658df 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/OOZIE/configuration/oozie-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/OOZIE/configuration/oozie-site.xml @@ -26,6 +26,7 @@ <property> <name>oozie.base.url</name> <value>http://localhost:11000/oozie</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>Base Oozie URL.</description> </property> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-site.xml index 667c37439c..732cef18a2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-site.xml @@ -61,6 +61,7 @@ <property> <name>yarn.timeline-service.webapp.address</name> <value>localhost:8188</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description> The http address of the timeline service web application. </description> @@ -69,6 +70,7 @@ <property> <name>yarn.timeline-service.webapp.https.address</name> <value>localhost:8190</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description> The http address of the timeline service web application. </description> @@ -77,6 +79,7 @@ <property> <name>yarn.timeline-service.address</name> <value>localhost:10200</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description> This is default address for the timeline server to start the RPC server. diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/repos/repoinfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/repos/repoinfo.xml index 9decf51e71..dbf8506bff 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/repos/repoinfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/repos/repoinfo.xml @@ -29,18 +29,6 @@ <reponame>HDP-UTILS</reponame> </repo> </os> - <os family="amazon2015"> - <repo> - <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.6.0</baseurl> - <repoid>HDP-2.2</repoid> - <reponame>HDP</reponame> - </repo> - <repo> - <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl> - <repoid>HDP-UTILS-1.1.0.20</repoid> - <reponame>HDP-UTILS</reponame> - </repo> - </os> <os family="suse11"> <repo> <baseurl>http://public-repo-1.hortonworks.com/HDP/suse11sp3/2.x/updates/2.2.6.0</baseurl> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/configuration/ranger-hbase-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/configuration/ranger-hbase-plugin-properties.xml index 4e871517a6..fb6de71a58 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/configuration/ranger-hbase-plugin-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/configuration/ranger-hbase-plugin-properties.xml @@ -106,6 +106,7 @@ <property> <name>XAAUDIT.HDFS.DESTINATION_DIRECTORY</name> <value>hdfs://localhost:8020/ranger/audit/%app-type%/%time:yyyyMMdd%</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description></description> <depends-on> <property> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml index 8bb6369cdd..d8ab47aff6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/configuration/ranger-hdfs-plugin-properties.xml @@ -110,8 +110,8 @@ <property> <name>XAAUDIT.HDFS.DESTINATION_DIRECTORY</name> - <value>hdfs://localhost:8020/ranger/audit/%app-type%/%time:yyyyMMdd% - </value> + <value>hdfs://localhost:8020/ranger/audit/%app-type%/%time:yyyyMMdd%</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description></description> <depends-on> <property> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml index 87adb4baa8..471af6e987 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml @@ -130,26 +130,6 @@ export METASTORE_PORT={{hive_metastore_port}} </property> <property> - <name>cost_based_optimizer</name> - <display-name>Cost Based Optimizer</display-name> - <value>Off</value> - <value-attributes> - <type>value-list</type> - <entries> - <entry> - <value>On</value> - <label>On</label> - </entry> - <entry> - <value>Off</value> - <label>Off</label> - </entry> - </entries> - <selection-cardinality>1</selection-cardinality> - </value-attributes> - </property> - - <property> <name>hive_security_authorization</name> <display-name>Choose Authorization</display-name> <value>None</value> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml index 35db348b03..25bb468f1d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml @@ -43,6 +43,7 @@ limitations under the License. <property> <name>hive.zookeeper.quorum</name> <value>localhost:2181</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>List of ZooKeeper servers to talk to. This is needed for: 1. Read/write locks - when hive.lock.manager is set to org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager, @@ -114,6 +115,7 @@ limitations under the License. <property> <name>hive.cluster.delegation.token.store.zookeeper.connectString</name> <value>localhost:2181</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>The ZooKeeper token store connect string.</description> </property> @@ -345,6 +347,7 @@ limitations under the License. <property> <name>hive.metastore.uris</name> <value>thrift://localhost:9083</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>Thrift URI for the remote metastore. Used by metastore client to connect to remote metastore.</description> </property> @@ -363,6 +366,7 @@ limitations under the License. <property> <name>javax.jdo.option.ConnectionURL</name> <value>jdbc:mysql://localhost/hive?createDatabaseIfNotExist=true</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <display-name>Database URL</display-name> <description>JDBC connect string for a JDBC metastore</description> <value-attributes> @@ -467,12 +471,6 @@ limitations under the License. <name>hive.cbo.enable</name> <value>true</value> <description>Flag to control enabling Cost Based Optimizations using Calcite framework.</description> - <depends-on> - <property> - <type>hive-env</type> - <name>cost_based_optimizer</name> - </property> - </depends-on> </property> <property> @@ -816,10 +814,6 @@ limitations under the License. </value-attributes> <depends-on> <property> - <type>hive-env</type> - <name>cost_based_optimizer</name> - </property> - <property> <type>hive-site</type> <name>hive.cbo.enable</name> </property> @@ -851,10 +845,6 @@ limitations under the License. </value-attributes> <depends-on> <property> - <type>hive-env</type> - <name>cost_based_optimizer</name> - </property> - <property> <type>hive-site</type> <name>hive.cbo.enable</name> </property> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/ranger-hive-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/ranger-hive-plugin-properties.xml index a9ceb29c86..756e95390b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/ranger-hive-plugin-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/ranger-hive-plugin-properties.xml @@ -95,6 +95,7 @@ <property> <name>XAAUDIT.HDFS.DESTINATION_DIRECTORY</name> <value>hdfs://localhost:8020/ranger/audit/%app-type%/%time:yyyyMMdd%</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description></description> <depends-on> <property> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/ranger-storm-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/ranger-storm-plugin-properties.xml index ef07f4f301..13acbffb63 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/ranger-storm-plugin-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/configuration/ranger-storm-plugin-properties.xml @@ -106,6 +106,7 @@ <property> <name>XAAUDIT.HDFS.DESTINATION_DIRECTORY</name> <value>hdfs://localhost:8020/ranger/audit/%app-type%/%time:yyyyMMdd%</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description></description> <depends-on> <property> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml index e75442f907..ee12ba3029 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml @@ -39,6 +39,7 @@ <property> <name>hadoop.registry.zk.quorum</name> <value>localhost:2181</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description> List of hostname:port pairs defining the zookeeper quorum binding for the registry </description> @@ -98,6 +99,7 @@ <property> <name>yarn.resourcemanager.zk-address</name> <value>localhost:2181</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description> List Host:Port of the ZooKeeper servers to be used by the RM. comma separated host:port pairs, each corresponding to a zk server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" If the optional chroot suffix is used the example would look like: "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002/app/a" where the client would be rooted at "/app/a" and all paths would be relative to this root - ie getting/setting/etc... "/foo/bar" would result in operations being run on "/app/a/foo/bar" (from the server perspective). </description> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py index 9e43865eff..dc968cc5d2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py @@ -353,14 +353,11 @@ class HDP22StackAdvisor(HDP21StackAdvisor): putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864") # CBO - putHiveEnvProperty("cost_based_optimizer", "On") - if str(configurations["hive-env"]["properties"]["cost_based_optimizer"]).lower() == "on": - putHiveSiteProperty("hive.cbo.enable", "true") - else: - putHiveSiteProperty("hive.cbo.enable", "false") - hive_cbo_enable = configurations["hive-site"]["properties"]["hive.cbo.enable"] - putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable) - putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable) + if "hive-site" in services["configurations"] and "hive.cbo.enable" in services["configurations"]["hive-site"]["properties"]: + hive_cbo_enable = services["configurations"]["hive-site"]["properties"]["hive.cbo.enable"] + putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable) + putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable) + putHiveSiteProperty("hive.compute.query.using.stats", "true") # Interactive Query diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/repos/repoinfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/repos/repoinfo.xml index 279134b988..142b87d858 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/repos/repoinfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/repos/repoinfo.xml @@ -41,18 +41,6 @@ <reponame>HDP-UTILS</reponame> </repo> </os> - <os family="amazon2015"> - <repo> - <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.0.0</baseurl> - <repoid>HDP-2.3</repoid> - <reponame>HDP</reponame> - </repo> - <repo> - <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl> - <repoid>HDP-UTILS-1.1.0.20</repoid> - <reponame>HDP-UTILS</reponame> - </repo> - </os> <os family="suse11"> <repo> <baseurl>http://public-repo-1.hortonworks.com/HDP/suse11sp3/2.x/updates/2.3.0.0</baseurl> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/KAFKA/configuration/kafka-broker.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/KAFKA/configuration/kafka-broker.xml index 8c2f34a938..6e924e2e8f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/KAFKA/configuration/kafka-broker.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/KAFKA/configuration/kafka-broker.xml @@ -22,6 +22,7 @@ <property> <name>listeners</name> <value>PLAINTEXT://localhost:6667</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>host and port where kafka broker will be accepting connnections. localhost will be subsituted with hostname.</description> </property> <property> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/KAFKA/configuration/ranger-kafka-plugin-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/KAFKA/configuration/ranger-kafka-plugin-properties.xml index 490e0592d9..18429a18c0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/KAFKA/configuration/ranger-kafka-plugin-properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/KAFKA/configuration/ranger-kafka-plugin-properties.xml @@ -48,6 +48,7 @@ <property> <name>zookeeper.connect</name> <value>localhost:2181</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>Used for repository creation on ranger admin</description> </property> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER/configuration/ranger-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER/configuration/ranger-env.xml index 1e86e7d00e..dce8a90796 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER/configuration/ranger-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER/configuration/ranger-env.xml @@ -58,6 +58,7 @@ <display-name>JDBC connect string for root user</display-name> <description>JDBC connect string - auto populated based on other values. This is to be used by root user</description> <value>jdbc:mysql://localhost</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <value-attributes> <overridable>false</overridable> </value-attributes> @@ -187,6 +188,7 @@ <property> <name>xasecure.audit.destination.hdfs.dir</name> <value>hdfs://localhost:8020</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <display-name>Destination HDFS Directory</display-name> <description>HDFS folder to write audit to, make sure all service user has required permissions. This property is overridable at service level</description> <depends-on> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-site.xml index ca6d2b6d10..6a5e6db70f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-site.xml @@ -22,6 +22,7 @@ <property> <name>nimbus.seeds</name> <value>localhost</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>Comma-delimited list of the hosts running nimbus server.</description> <value-attributes> <type>componentHosts</type> @@ -52,6 +53,7 @@ <property> <name>nimbus.host</name> <value>localhost</value> + <property-type>DONT_ADD_ON_UPGRADE</property-type> <description>Deprecated config in favor of nimbus.seeds used during non HA mode.</description> <deleted>true</deleted> </property> diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/repos/repoinfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/repos/repoinfo.xml index 6ac43f95dd..54bd3dad62 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/repos/repoinfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/repos/repoinfo.xml @@ -41,18 +41,6 @@ <reponame>HDP-UTILS</reponame> </repo> </os> - <os family="redhat2015"> - <repo> - <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/updates/2.4.0.0</baseurl> - <repoid>HDP-2.4</repoid> - <reponame>HDP</reponame> - </repo> - <repo> - <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl> - <repoid>HDP-UTILS-1.1.0.20</repoid> - <reponame>HDP-UTILS</reponame> - </repo> - </os> <os family="suse11"> <repo> <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/suse11sp3/2.x/updates/2.4.0.0</baseurl> diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml index 0efd749595..8dde3fd33f 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-env.xml @@ -77,26 +77,6 @@ </property> <property> - <name>cost_based_optimizer</name> - <display-name>Cost Based Optimizer</display-name> - <value>Off</value> - <value-attributes> - <type>value-list</type> - <entries> - <entry> - <value>On</value> - <label>On</label> - </entry> - <entry> - <value>Off</value> - <label>Off</label> - </entry> - </entries> - <selection-cardinality>1</selection-cardinality> - </value-attributes> - </property> - - <property> <name>hive_security_authorization</name> <display-name>Choose Authorization</display-name> <value>None</value> diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml index 6ef7f32981..be7c943fa5 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/HIVE/configuration/hive-site.xml @@ -789,10 +789,6 @@ limitations under the License. </value-attributes> <depends-on> <property> - <type>hive-env</type> - <name>cost_based_optimizer</name> - </property> - <property> <type>hive-site</type> <name>hive.cbo.enable</name> </property> @@ -824,10 +820,6 @@ limitations under the License. </value-attributes> <depends-on> <property> - <type>hive-env</type> - <name>cost_based_optimizer</name> - </property> - <property> <type>hive-site</type> <name>hive.cbo.enable</name> </property> diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py index afdb7cf744..1ed4ce06dc 100644 --- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/stack_advisor.py @@ -362,14 +362,11 @@ class HDPWIN22StackAdvisor(HDPWIN21StackAdvisor): putHiveSiteProperty("hive.exec.reducers.bytes.per.reducer", "67108864") # CBO - putHiveEnvProperty("cost_based_optimizer", "On") - if str(configurations["hive-env"]["properties"]["cost_based_optimizer"]).lower() == "on": - putHiveSiteProperty("hive.cbo.enable", "true") - else: - putHiveSiteProperty("hive.cbo.enable", "false") - hive_cbo_enable = configurations["hive-site"]["properties"]["hive.cbo.enable"] - putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable) - putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable) + if "hive-site" in services["configurations"] and "hive.cbo.enable" in services["configurations"]["hive-site"]["properties"]: + hive_cbo_enable = services["configurations"]["hive-site"]["properties"]["hive.cbo.enable"] + putHiveSiteProperty("hive.stats.fetch.partition.stats", hive_cbo_enable) + putHiveSiteProperty("hive.stats.fetch.column.stats", hive_cbo_enable) + putHiveSiteProperty("hive.compute.query.using.stats", "true") # Interactive Query diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java index d3945dc242..2a4cec8a2d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java @@ -183,9 +183,11 @@ public class TestHeartbeatHandler { public TemporaryFolder temporaryFolder = new TemporaryFolder(); + private InMemoryDefaultTestModule module; + @Before public void setup() throws Exception { - InMemoryDefaultTestModule module = new InMemoryDefaultTestModule(){ + module = new InMemoryDefaultTestModule(){ @Override protected void configure() { @@ -856,11 +858,23 @@ public class TestHeartbeatHandler { Clusters fsm = clusters; HeartBeatHandler handler = new HeartBeatHandler(fsm, new ActionQueue(), am, injector); - clusters.addHost(DummyHostname1); + Cluster cluster = getDummyCluster(); + Service hdfs = cluster.addService(HDFS); + hdfs.persist(); + hdfs.addServiceComponent(DATANODE).persist(); + hdfs.getServiceComponent(DATANODE).addServiceComponentHost(DummyHostname1).persist(); + hdfs.addServiceComponent(NAMENODE).persist(); + hdfs.getServiceComponent(NAMENODE).addServiceComponentHost(DummyHostname1).persist(); + hdfs.addServiceComponent(HDFS_CLIENT).persist(); + hdfs.getServiceComponent(HDFS_CLIENT).addServiceComponentHost(DummyHostname1).persist(); + Host hostObject = clusters.getHost(DummyHostname1); hostObject.setIPv4("ipv4"); hostObject.setIPv6("ipv6"); + // add recovery.enabled_components to ambari configuration + module.getProperties().put("recovery.enabled_components", "NAMENODE,DATANODE"); + Register reg = new Register(); HostInfo hi = new HostInfo(); hi.setHostName(DummyHostname1); @@ -877,6 +891,7 @@ public class TestHeartbeatHandler { assertEquals(rc.getMaxLifetimeCount(), "10"); assertEquals(rc.getRetryGap(), "2"); assertEquals(rc.getWindowInMinutes(), "23"); + assertEquals(rc.getEnabledComponents(), "NAMENODE,DATANODE"); rc = RecoveryConfig.getRecoveryConfig(new Configuration()); assertEquals(rc.getMaxCount(), "6"); @@ -884,6 +899,65 @@ public class TestHeartbeatHandler { assertEquals(rc.getMaxLifetimeCount(), "12"); assertEquals(rc.getRetryGap(), "5"); assertEquals(rc.getWindowInMinutes(), "60"); + assertEquals(rc.getEnabledComponents(), ""); + + // clean up + module.getProperties().remove("recovery.enabled_components"); + } + + // + // Same as testRegistrationRecoveryConfig but will test + // maintenance mode set to ON for a service component host + // + @Test + public void testRegistrationRecoveryConfigMaintenanceMode() + throws AmbariException, InvalidStateTransitionException { + ActionManager am = getMockActionManager(); + replay(am); + Clusters fsm = clusters; + HeartBeatHandler handler = new HeartBeatHandler(fsm, new ActionQueue(), am, + injector); + Cluster cluster = getDummyCluster(); + Service hdfs = cluster.addService(HDFS); + hdfs.persist(); + hdfs.addServiceComponent(DATANODE).persist(); + hdfs.getServiceComponent(DATANODE).addServiceComponentHost(DummyHostname1).persist(); + hdfs.addServiceComponent(NAMENODE).persist(); + hdfs.getServiceComponent(NAMENODE).addServiceComponentHost(DummyHostname1).persist(); + hdfs.addServiceComponent(HDFS_CLIENT).persist(); + hdfs.getServiceComponent(HDFS_CLIENT).addServiceComponentHost(DummyHostname1).persist(); + + Host hostObject = clusters.getHost(DummyHostname1); + hostObject.setIPv4("ipv4"); + hostObject.setIPv6("ipv6"); + + // add recovery.enabled_components to ambari configuration + module.getProperties().put("recovery.enabled_components", "NAMENODE,DATANODE,HDFS_CLIENT"); + + // set maintenance mode on HDFS_CLIENT on host1 to true + ServiceComponentHost schHdfsClient = hdfs.getServiceComponent(HDFS_CLIENT).getServiceComponentHost(DummyHostname1); + schHdfsClient.setMaintenanceState(MaintenanceState.ON); + + Register reg = new Register(); + HostInfo hi = new HostInfo(); + hi.setHostName(DummyHostname1); + hi.setOS(DummyOsType); + reg.setHostname(DummyHostname1); + reg.setCurrentPingPort(DummyCurrentPingPort); + reg.setHardwareProfile(hi); + reg.setAgentVersion(metaInfo.getServerVersion()); + reg.setPrefix(Configuration.PREFIX_DIR); + RegistrationResponse rr = handler.handleRegistration(reg); + RecoveryConfig rc = rr.getRecoveryConfig(); + assertEquals(rc.getMaxCount(), "4"); + assertEquals(rc.getType(), "FULL"); + assertEquals(rc.getMaxLifetimeCount(), "10"); + assertEquals(rc.getRetryGap(), "2"); + assertEquals(rc.getWindowInMinutes(), "23"); + assertEquals(rc.getEnabledComponents(), "NAMENODE,DATANODE"); // HDFS_CLIENT is in maintenance mode + + // clean up + module.getProperties().remove("recovery.enabled_components"); } @Test diff --git a/ambari-server/src/test/java/org/apache/ambari/server/notifications/dispatchers/AlertScriptDispatcherTest.java b/ambari-server/src/test/java/org/apache/ambari/server/notifications/dispatchers/AlertScriptDispatcherTest.java index 12141c9992..9e0e406925 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/notifications/dispatchers/AlertScriptDispatcherTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/notifications/dispatchers/AlertScriptDispatcherTest.java @@ -20,6 +20,7 @@ package org.apache.ambari.server.notifications.dispatchers; import java.lang.reflect.Field; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.UUID; import java.util.concurrent.Executor; @@ -28,8 +29,12 @@ import org.apache.ambari.server.notifications.DispatchCallback; import org.apache.ambari.server.notifications.DispatchFactory; import org.apache.ambari.server.notifications.Notification; import org.apache.ambari.server.notifications.NotificationDispatcher; +import org.apache.ambari.server.orm.entities.AlertDefinitionEntity; +import org.apache.ambari.server.orm.entities.AlertHistoryEntity; +import org.apache.ambari.server.state.AlertState; import org.apache.ambari.server.state.alert.AlertNotification; import org.apache.ambari.server.state.alert.TargetType; +import org.apache.ambari.server.state.services.AlertNoticeDispatchService.AlertInfo; import org.apache.ambari.server.state.stack.OsFamily; import org.easymock.EasyMock; import org.junit.Before; @@ -44,6 +49,8 @@ import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; +import junit.framework.Assert; + /** * Tests {@link AlertScriptDispatcher}. */ @@ -217,6 +224,60 @@ public class AlertScriptDispatcherTest { } /** + * Tests that arguments given to the {@link ProcessBuilder} are properly + * escaped. + * + * @throws Exception + */ + @Test + public void testArgumentEscaping() throws Exception { + final String ALERT_DEFINITION_NAME = "mock_alert_with_quotes"; + final String ALERT_DEFINITION_LABEL = "Mock alert with Quotes"; + final String ALERT_LABEL = "Alert Label"; + final String ALERT_SERVICE_NAME = "FOO_SERVICE"; + final String ALERT_TEXT = "Did you know, \"Quotes are hard!!!\""; + final String ALERT_TEXT_ESCAPED = "Did you know, \\\"Quotes are hard\\!\\!\\!\\\""; + + DispatchCallback callback = EasyMock.createNiceMock(DispatchCallback.class); + AlertNotification notification = new AlertNotification(); + notification.Callback = callback; + notification.CallbackIds = Collections.singletonList(UUID.randomUUID().toString()); + + AlertDefinitionEntity definition = new AlertDefinitionEntity(); + definition.setDefinitionName(ALERT_DEFINITION_NAME); + definition.setLabel(ALERT_DEFINITION_LABEL); + + AlertHistoryEntity history = new AlertHistoryEntity(); + history.setAlertDefinition(definition); + history.setAlertLabel(ALERT_LABEL); + history.setAlertText(ALERT_TEXT); + history.setAlertState(AlertState.OK); + history.setServiceName(ALERT_SERVICE_NAME); + + AlertInfo alertInfo = new AlertInfo(history); + notification.setAlertInfo(alertInfo); + + AlertScriptDispatcher dispatcher = new AlertScriptDispatcher(); + m_injector.injectMembers(dispatcher); + + ProcessBuilder processBuilder = dispatcher.getProcessBuilder(SCRIPT_CONFIG_VALUE, notification); + List<String> commands = processBuilder.command(); + Assert.assertEquals(3, commands.size()); + Assert.assertEquals("sh", commands.get(0)); + Assert.assertEquals("-c", commands.get(1)); + + StringBuilder buffer = new StringBuilder(); + buffer.append(SCRIPT_CONFIG_VALUE).append(" "); + buffer.append(ALERT_DEFINITION_NAME).append(" "); + buffer.append("\"").append(ALERT_DEFINITION_LABEL).append("\"").append(" "); + buffer.append(ALERT_SERVICE_NAME).append(" "); + buffer.append(AlertState.OK).append(" "); + buffer.append("\"").append(ALERT_TEXT_ESCAPED).append("\""); + + Assert.assertEquals(buffer.toString(), commands.get(2)); + } + + /** * */ private class MockModule extends AbstractModule { diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java index 67b9da51e5..6061e067bc 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java @@ -133,12 +133,31 @@ public class UpgradeCatalog222Test { { put("timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier", String.valueOf(1)); put("timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier", String.valueOf(1)); + put("timeline.metrics.service.operation.mode", "distributed"); + put("timeline.metrics.host.aggregator.ttl", String.valueOf(86400)); + put("timeline.metrics.host.aggregator.minute.ttl", String.valueOf(604800)); + put("timeline.metrics.host.aggregator.hourly.ttl", String.valueOf(2592000)); + put("timeline.metrics.host.aggregator.daily.ttl", String.valueOf(31536000)); + put("timeline.metrics.cluster.aggregator.second.ttl", String.valueOf(21600)); //Less than 1 day + put("timeline.metrics.cluster.aggregator.minute.ttl", String.valueOf(7776000)); + put("timeline.metrics.cluster.aggregator.hourly.ttl", String.valueOf(31536000)); + put("timeline.metrics.cluster.aggregator.daily.ttl", String.valueOf(63072000)); } }; Map<String, String> newPropertiesAmsSite = new HashMap<String, String>() { { put("timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier", String.valueOf(2)); put("timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier", String.valueOf(2)); + put("timeline.metrics.service.watcher.disabled", String.valueOf(false)); + put("timeline.metrics.host.aggregator.ttl", String.valueOf(7.0)); + put("timeline.metrics.host.aggregator.minute.ttl", String.valueOf(7.0)); + put("timeline.metrics.host.aggregator.hourly.ttl", String.valueOf(30.0)); + put("timeline.metrics.host.aggregator.daily.ttl", String.valueOf(365.0)); + put("timeline.metrics.cluster.aggregator.second.ttl", String.valueOf(0.25)); + put("timeline.metrics.cluster.aggregator.minute.ttl", String.valueOf(30.0)); + put("timeline.metrics.cluster.aggregator.hourly.ttl", String.valueOf(365.0)); + put("timeline.metrics.cluster.aggregator.daily.ttl", String.valueOf(730.0)); + put("timeline.metrics.service.operation.mode", "distributed"); } }; EasyMockSupport easyMockSupport = new EasyMockSupport(); @@ -151,7 +170,7 @@ public class UpgradeCatalog222Test { put("normal", cluster); }}).once(); expect(cluster.getDesiredConfigByType("ams-site")).andReturn(mockAmsSite).atLeastOnce(); - expect(mockAmsSite.getProperties()).andReturn(oldPropertiesAmsSite).times(2); + expect(mockAmsSite.getProperties()).andReturn(oldPropertiesAmsSite).anyTimes(); Injector injector = easyMockSupport.createNiceMock(Injector.class); expect(injector.getInstance(Gson.class)).andReturn(null).anyTimes(); diff --git a/ambari-server/src/test/python/TestOSCheck.py b/ambari-server/src/test/python/TestOSCheck.py index cf114a1a9a..d919fbc8be 100644 --- a/ambari-server/src/test/python/TestOSCheck.py +++ b/ambari-server/src/test/python/TestOSCheck.py @@ -30,7 +30,7 @@ from mock.mock import MagicMock from only_for_platform import os_distro_value, os_distro_value_linux -from ambari_commons import OSCheck +from ambari_commons import OSCheck, OSConst import os_check_type utils = __import__('ambari_server.utils').utils @@ -50,7 +50,7 @@ class TestOSCheck(TestCase): # 1 - Any system mock_is_oracle_linux.return_value = False - mock_linux_distribution.return_value = ('my_os', '', '') + mock_linux_distribution.return_value = ('my_os', '2015.09', '') result = OSCheck.get_os_type() self.assertEquals(result, 'my_os') @@ -66,13 +66,13 @@ class TestOSCheck(TestCase): # 3 - path exist: '/etc/oracle-release' mock_is_oracle_linux.return_value = True - mock_linux_distribution.return_value = ('some_os', '', '') + mock_linux_distribution.return_value = ('some_os', '1234', '') result = OSCheck.get_os_type() self.assertEquals(result, 'oraclelinux') # 4 - Common system mock_is_oracle_linux.return_value = False - mock_linux_distribution.return_value = ('CenToS', '', '') + mock_linux_distribution.return_value = ('CenToS', '4.56', '') result = OSCheck.get_os_type() self.assertEquals(result, 'centos') @@ -99,31 +99,31 @@ class TestOSCheck(TestCase): # 1 - Any system mock_exists.return_value = False - mock_linux_distribution.return_value = ('MY_os', '', '') + mock_linux_distribution.return_value = ('MY_os', '5.6.7', '') result = OSCheck.get_os_family() self.assertEquals(result, 'my_os') # 2 - Redhat mock_exists.return_value = False - mock_linux_distribution.return_value = ('Centos Linux', '', '') + mock_linux_distribution.return_value = ('Centos Linux', '2.4', '') result = OSCheck.get_os_family() self.assertEquals(result, 'redhat') # 3 - Ubuntu mock_exists.return_value = False - mock_linux_distribution.return_value = ('Ubuntu', '', '') + mock_linux_distribution.return_value = ('Ubuntu', '14.04', '') result = OSCheck.get_os_family() self.assertEquals(result, 'ubuntu') # 4 - Suse mock_exists.return_value = False mock_linux_distribution.return_value = ( - 'suse linux enterprise server', '', '') + 'suse linux enterprise server', '11.3', '') result = OSCheck.get_os_family() self.assertEquals(result, 'suse') mock_exists.return_value = False - mock_linux_distribution.return_value = ('SLED', '', '') + mock_linux_distribution.return_value = ('SLED', '1.2.3.4.5', '') result = OSCheck.get_os_family() self.assertEquals(result, 'suse') @@ -141,7 +141,7 @@ class TestOSCheck(TestCase): def test_get_os_version(self, mock_linux_distribution): # 1 - Any system - mock_linux_distribution.return_value = ('', '123.45', '') + mock_linux_distribution.return_value = ('some_os', '123.45', '') result = OSCheck.get_os_version() self.assertEquals(result, '123.45') @@ -159,7 +159,7 @@ class TestOSCheck(TestCase): def test_get_os_major_version(self, mock_linux_distribution): # 1 - mock_linux_distribution.return_value = ('', '123.45.67', '') + mock_linux_distribution.return_value = ('abcd_os', '123.45.67', '') result = OSCheck.get_os_major_version() self.assertEquals(result, '123') @@ -167,6 +167,21 @@ class TestOSCheck(TestCase): mock_linux_distribution.return_value = ('Suse', '11', '') result = OSCheck.get_os_major_version() self.assertEquals(result, '11') + + @patch.object(OSCheck, "os_distribution") + def test_aliases(self, mock_linux_distribution): + OSConst.OS_TYPE_ALIASES['qwerty_os123'] = 'aliased_os5' + OSConst.OS_FAMILY_COLLECTION.append({ + 'name': 'aliased_os_family', + 'os_list': ["aliased_os"] + }) + + mock_linux_distribution.return_value = ('qwerty_os', '123.45.67', '') + + self.assertEquals(OSCheck.get_os_type(), 'aliased_os') + self.assertEquals(OSCheck.get_os_major_version(), '5') + self.assertEquals(OSCheck.get_os_version(), '5.45.67') + self.assertEquals(OSCheck.get_os_family(), 'aliased_os_family') @patch.object(OSCheck, "os_distribution") def test_get_os_release_name(self, mock_linux_distribution): diff --git a/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json b/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json index 7829f05804..0fbc282444 100644 --- a/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json +++ b/ambari-server/src/test/python/stacks/2.1/configs/hive-metastore-upgrade.json @@ -77,8 +77,7 @@ "hive_user": "hive", "hive_database": "New MySQL Database", "hive.heapsize": "703", - "hcat_pid_dir": "/var/run/webhcat", - "cost_based_optimizer": "On", + "hcat_pid_dir": "/var/run/webhcat", "webhcat_user": "hcat", "hive_security_authorization": "None", "hive_exec_orc_storage_strategy": "SPEED", diff --git a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py index 0a8f05b992..d2497fd4f2 100644 --- a/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py @@ -1019,7 +1019,6 @@ class TestHDP22StackAdvisor(TestCase): }, 'hive-env': { 'properties': { - 'cost_based_optimizer': 'On', 'hive_exec_orc_storage_strategy': 'SPEED', 'hive_security_authorization': 'None', 'hive_timeline_logging_enabled': 'true', @@ -1033,7 +1032,6 @@ class TestHDP22StackAdvisor(TestCase): 'hive.server2.tez.initialize.default.sessions': 'false', 'hive.server2.tez.sessions.per.default.queue': '1', 'hive.auto.convert.join.noconditionaltask.size': '268435456', - 'hive.cbo.enable': 'true', 'hive.compactor.initiator.on': 'false', 'hive.compactor.worker.threads': '0', 'hive.compute.query.using.stats': 'true', @@ -1176,7 +1174,8 @@ class TestHDP22StackAdvisor(TestCase): "hive.server2.authentication.kerberos.keytab": "", "hive.server2.authentication.kerberos.principal": "", "hive.server2.authentication.pam.services": "", - "hive.server2.custom.authentication.class": "" + "hive.server2.custom.authentication.class": "", + "hive.cbo.enable": "true" } }, "hiveserver2-site": { @@ -1251,7 +1250,8 @@ class TestHDP22StackAdvisor(TestCase): "hive.server2.authentication.kerberos.keytab": "", "hive.server2.authentication.kerberos.principal": "", "hive.server2.authentication.pam.services": "", - "hive.server2.custom.authentication.class": "" + "hive.server2.custom.authentication.class": "", + "hive.cbo.enable": "true" } }, "hiveserver2-site": { @@ -1315,10 +1315,8 @@ class TestHDP22StackAdvisor(TestCase): #test recommendations services["configurations"]["hive-site"]["properties"]["hive.cbo.enable"] = "false" services["configurations"]["hive-env"]["properties"]["hive_security_authorization"] = "sqlstdauth" - services["changed-configurations"] = [{"type": "hive-site", "name": "hive.cbo.enable"}, - {"type": "hive-env", "name": "hive_security_authorization"}] + services["changed-configurations"] = [{"type": "hive-env", "name": "hive_security_authorization"}] expected["hive-env"]["properties"]["hive_security_authorization"] = "sqlstdauth" - expected["hive-site"]["properties"]["hive.cbo.enable"] = "false" expected["hive-site"]["properties"]["hive.stats.fetch.partition.stats"]="false" expected["hive-site"]["properties"]["hive.stats.fetch.column.stats"]="false" expected["hive-site"]["properties"]["hive.security.authorization.enabled"]="true" @@ -2052,7 +2050,8 @@ class TestHDP22StackAdvisor(TestCase): "properties": { "timeline.metrics.cluster.aggregate.splitpoints": " ", "timeline.metrics.host.aggregate.splitpoints": " ", - "timeline.metrics.host.aggregator.ttl": "86400" + "timeline.metrics.host.aggregator.ttl": "1", + 'timeline.metrics.service.watcher.disabled': 'false' } } } @@ -2230,6 +2229,8 @@ class TestHDP22StackAdvisor(TestCase): expected['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '512' expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '102' expected['ams-hbase-env']['properties']['regionserver_xmn_size'] = '384' + expected['ams-site']['properties']['timeline.metrics.host.aggregator.ttl'] = '7' + expected['ams-site']['properties']['timeline.metrics.service.watcher.disabled'] = 'true' self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts) self.assertEquals(configurations, expected) diff --git a/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json b/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json index 51ae36af7d..20a672e055 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json +++ b/ambari-server/src/test/python/stacks/2.3/common/services-sparkts-hive.json @@ -2402,47 +2402,6 @@ }, "dependencies" : [ ] }, { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer", - "StackConfigurations" : { - "property_depends_on" : [ ], - "property_name" : "cost_based_optimizer", - "service_name" : "HIVE", - "stack_name" : "HDP", - "stack_version" : "2.3", - "type" : "hive-env.xml" - }, - "dependencies" : [ { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer/dependencies/hive.cbo.enable", - "StackConfigurationDependency" : { - "dependency_name" : "hive.cbo.enable", - "dependency_type" : "hive-site", - "property_name" : "cost_based_optimizer", - "service_name" : "HIVE", - "stack_name" : "HDP", - "stack_version" : "2.3" - } - }, { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer/dependencies/hive.stats.fetch.column.stats", - "StackConfigurationDependency" : { - "dependency_name" : "hive.stats.fetch.column.stats", - "dependency_type" : "hive-site", - "property_name" : "cost_based_optimizer", - "service_name" : "HIVE", - "stack_name" : "HDP", - "stack_version" : "2.3" - } - }, { - "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/cost_based_optimizer/dependencies/hive.stats.fetch.partition.stats", - "StackConfigurationDependency" : { - "dependency_name" : "hive.stats.fetch.partition.stats", - "dependency_type" : "hive-site", - "property_name" : "cost_based_optimizer", - "service_name" : "HIVE", - "stack_name" : "HDP", - "stack_version" : "2.3" - } - } ] - }, { "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/datanucleus.autoCreateSchema", "StackConfigurations" : { "property_depends_on" : [ ], @@ -4199,9 +4158,6 @@ "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/hive.stats.fetch.column.stats", "StackConfigurations" : { "property_depends_on" : [ { - "type" : "hive-env", - "name" : "cost_based_optimizer" - }, { "type" : "hive-site", "name" : "hive.cbo.enable" } ], @@ -4216,9 +4172,6 @@ "href" : "/api/v1/stacks/HDP/versions/2.3/services/HIVE/configurations/hive.stats.fetch.partition.stats", "StackConfigurations" : { "property_depends_on" : [ { - "type" : "hive-env", - "name" : "cost_based_optimizer" - }, { "type" : "hive-site", "name" : "hive.cbo.enable" } ], diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 72e77184e3..545a2b5bcf 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -882,7 +882,6 @@ class TestHDP23StackAdvisor(TestCase): }, 'hive-env': { 'properties': { - 'cost_based_optimizer': 'On', 'hive_exec_orc_storage_strategy': 'SPEED', 'hive_security_authorization': 'None', 'hive_timeline_logging_enabled': 'true', @@ -896,7 +895,6 @@ class TestHDP23StackAdvisor(TestCase): 'hive.server2.tez.initialize.default.sessions': 'false', 'hive.server2.tez.sessions.per.default.queue': '1', 'hive.auto.convert.join.noconditionaltask.size': '268435456', - 'hive.cbo.enable': 'true', 'hive.compactor.initiator.on': 'false', 'hive.compactor.worker.threads': '0', 'hive.compute.query.using.stats': 'true', @@ -1040,7 +1038,8 @@ class TestHDP23StackAdvisor(TestCase): "hive.server2.authentication.kerberos.keytab": "", "hive.server2.authentication.kerberos.principal": "", "hive.server2.authentication.pam.services": "", - "hive.server2.custom.authentication.class": "" + "hive.server2.custom.authentication.class": "", + "hive.cbo.enable": "true" } }, "hiveserver2-site": { diff --git a/ambari-web/app/controllers/main/admin/service_auto_start.js b/ambari-web/app/controllers/main/admin/service_auto_start.js index fd288eda3b..40712922a1 100644 --- a/ambari-web/app/controllers/main/admin/service_auto_start.js +++ b/ambari-web/app/controllers/main/admin/service_auto_start.js @@ -21,6 +21,11 @@ var App = require('app'); App.MainAdminServiceAutoStartController = Em.Controller.extend({ name: 'mainAdminServiceAutoStartController', + clusterConfigs: null, + componentsConfigs: [], + isSaveDisabled: true, + valueChanged: false, + loadClusterConfig: function () { return App.ajax.send({ name: 'config.tags.site', @@ -31,6 +36,161 @@ App.MainAdminServiceAutoStartController = Em.Controller.extend({ }); }, + loadComponentsConfigs: function () { + return App.ajax.send({ + name: 'components.get_category', + sender: this, + success: 'loadComponentsConfigsSuccess' + }); + }, + + loadComponentsConfigsSuccess: function (data) { + this.set('componentsConfigs', data.items); + }, + + tabs: function() { + var services = {}; + var tabs = []; + this.get('componentsConfigs').forEach(function(component) { + var serviceComponentInfo = component.ServiceComponentInfo; + if (serviceComponentInfo.category === "MASTER" || serviceComponentInfo.category === "SLAVE") { + var componentRecovery = Ember.Object.create({ + display_name: App.format.role(serviceComponentInfo.component_name), + component_name: serviceComponentInfo.component_name, + //recovery_enabled: serviceComponentInfo.recovery_enabled === 'true', + recovery_enabled: false, + valueChanged: false, + service_name: serviceComponentInfo.service_name + }); + if (services[serviceComponentInfo.service_name]) { + services[serviceComponentInfo.service_name].get('componentRecovery').push(componentRecovery); + services[serviceComponentInfo.service_name].set('enabledComponents', services[serviceComponentInfo.service_name].get('enabledComponents') + (componentRecovery.get('recovery_enabled') ? 1 : 0)); + services[serviceComponentInfo.service_name].set('totalComponents', services[serviceComponentInfo.service_name].get('totalComponents') + 1); + } else { + services[serviceComponentInfo.service_name] = Ember.Object.create({ + service_name: serviceComponentInfo.service_name, + display_name: App.format.role(serviceComponentInfo.service_name), + headingClass: "." + serviceComponentInfo.service_name, + isActive: false, + componentRecovery: [componentRecovery], + enabledComponents: componentRecovery.recovery_enabled ? 1 : 0, + totalComponents: 1, + indicator: function() { + var percentage = this.get('enabledComponents') / this.get('totalComponents'); + var indicator = "icon-adjust"; + if (percentage === 1) { + indicator = "icon-circle"; + } else if (percentage === 0) { + indicator = "icon-circle-blank"; + } + return indicator; + }.property('enabledComponents', 'totalComponents') + }); + } + } + }); + for (var service in services ) { + tabs.push(services[service]); + } + if (tabs.length) { + tabs[0].set('isActive', true); + } + return tabs; + }.property('componentsConfigs'), + + checkValuesChange: function () { + var valuesChanged = this.get('valueChanged'); + this.get('tabs').forEach(function (service) { + service.get('componentRecovery').forEach(function (component) { + valuesChanged = valuesChanged || component.get('valueChanged'); + }); + }); + this.set('isSaveDisabled', !valuesChanged); + }.observes('valueChanged'), + + doReload: function () { + window.location.reload(); + }, + + /** + * If some configs are changed and user navigates away or select another config-group, show this popup with propose to save changes + * @param {object} transitionCallback - callback with action to change configs view + * @return {App.ModalPopup} + * @method showSavePopup + */ + showSavePopup: function (transitionCallback) { + var self = this; + return App.ModalPopup.show({ + header: Em.I18n.t('dashboard.configHistory.info-bar.save.popup.title'), + footerClass: Em.View.extend({ + templateName: require('templates/main/service/info/save_popup_footer'), + }), + primary: Em.I18n.t('common.save'), + secondary: Em.I18n.t('common.cancel'), + onSave: function () { + //save cluster setting + if (self.get('valueChanged')) { + self.saveClusterConfigs(self.get('clusterConfigs')); + } + //save component settings + var enabledComponents = []; + var disabledComponents = []; + self.get('tabs').forEach(function (service) { + service.get('componentRecovery').forEach(function (component) { + if (component.get('valueChanged')) { + if (component.get('recoveryEnabled')) { + enabledComponents.push(component.get('component_name')); + } else { + disabledComponents.push(component.get('component_name')); + } + } + }); + }); + if (enabledComponents.length){ + App.ajax.send({ + name: 'components.update', + sender: this, + data: { + ServiceComponentInfo: { + recovery_enabled: "true" + }, + query: 'ServiceComponentInfo/component_name.in(' + enabledComponents.join(',') + ')' + } + }); + } + if (disabledComponents.length){ + App.ajax.send({ + name: 'components.update', + sender: this, + data: { + ServiceComponentInfo: { + recovery_enabled: "false" + }, + query: 'ServiceComponentInfo/component_name.in(' + disabledComponents.join(',') + ')' + } + }); + } + if (typeof transitionCallback === 'function') { + transitionCallback(); + } else { + self.doReload(); + } + this.hide(); + }, + onDiscard: function () { + if (typeof transitionCallback === 'function') { + transitionCallback(); + } else { + self.doReload(); + } + this.hide(); + }, + onCancel: function () { + this.hide(); + } + }); + }, + saveClusterConfigs: function (clusterConfigs) { return App.ajax.send({ name: 'admin.save_configs', @@ -41,5 +201,4 @@ App.MainAdminServiceAutoStartController = Em.Controller.extend({ } }); } - }); diff --git a/ambari-web/app/controllers/main/alerts/manage_alert_notifications_controller.js b/ambari-web/app/controllers/main/alerts/manage_alert_notifications_controller.js index 244b8f07c9..79e524e006 100644 --- a/ambari-web/app/controllers/main/alerts/manage_alert_notifications_controller.js +++ b/ambari-web/app/controllers/main/alerts/manage_alert_notifications_controller.js @@ -61,7 +61,9 @@ App.ManageAlertNotificationsController = Em.Controller.extend({ value: '', defaultValue: 'custom', disabled: false, - isAll: Em.computed.equal('value', 'all') + isAll: function () { + return this.get('value') == 'all'; + }.property('value') }), method: { label: Em.I18n.t('alerts.actions.manage_alert_notifications_popup.method'), @@ -157,7 +159,7 @@ App.ManageAlertNotificationsController = Em.Controller.extend({ * used in Type combobox * @type {Array} */ - methods: ['EMAIL', 'SNMP', 'Custom SNMP'], + methods: ['EMAIL', 'SNMP'], /** * List of available value for Severity Filter @@ -218,8 +220,7 @@ App.ManageAlertNotificationsController = Em.Controller.extend({ 'mail.smtp.from', 'mail.smtp.host', 'mail.smtp.port', - 'mail.smtp.starttls.enable', - 'ambari.dispatch-property.script' + 'mail.smtp.starttls.enable' ], validationMap: { @@ -241,16 +242,6 @@ App.ManageAlertNotificationsController = Em.Controller.extend({ validator: 'retypePasswordValidation' } ], - 'Custom SNMP': [ - { - errorKey: 'portError', - validator: 'portValidation' - }, - { - errorKey: 'hostError', - validator: 'hostsValidation' - } - ], SNMP: [ { errorKey: 'portError', @@ -331,25 +322,24 @@ App.ManageAlertNotificationsController = Em.Controller.extend({ fillEditCreateInputs: function (addCopyToName) { var inputFields = this.get('inputFields'); var selectedAlertNotification = this.get('selectedAlertNotification'); - var props = selectedAlertNotification.get('properties'); inputFields.set('name.value', (addCopyToName ? 'Copy of ' : '') + selectedAlertNotification.get('name')); inputFields.set('groups.value', selectedAlertNotification.get('groups').toArray()); - inputFields.set('email.value', props['ambari.dispatch.recipients'] ? - props['ambari.dispatch.recipients'].join(', ') : ''); - inputFields.set('SMTPServer.value', props['mail.smtp.host']); - inputFields.set('SMTPPort.value', props['mail.smtp.port']); - inputFields.set('SMTPUseAuthentication.value', props['mail.smtp.auth'] !== "false"); - inputFields.set('SMTPUsername.value', props['ambari.dispatch.credential.username']); - inputFields.set('SMTPPassword.value', props['ambari.dispatch.credential.password']); - inputFields.set('retypeSMTPPassword.value', props['ambari.dispatch.credential.password']); - inputFields.set('SMTPSTARTTLS.value', props['mail.smtp.starttls.enable'] !== "false"); - inputFields.set('emailFrom.value', props['mail.smtp.from']); - inputFields.set('version.value', props['ambari.dispatch.snmp.version']); - inputFields.set('OIDs.value', props['ambari.dispatch.snmp.oids.trap']); - inputFields.set('community.value', props['ambari.dispatch.snmp.community']); - inputFields.set('host.value', props['ambari.dispatch.recipients'] ? - props['ambari.dispatch.recipients'].join(', ') : ''); - inputFields.set('port.value', props['ambari.dispatch.snmp.port']); + inputFields.set('email.value', selectedAlertNotification.get('properties')['ambari.dispatch.recipients'] ? + selectedAlertNotification.get('properties')['ambari.dispatch.recipients'].join(', ') : ''); + inputFields.set('SMTPServer.value', selectedAlertNotification.get('properties')['mail.smtp.host']); + inputFields.set('SMTPPort.value', selectedAlertNotification.get('properties')['mail.smtp.port']); + inputFields.set('SMTPUseAuthentication.value', selectedAlertNotification.get('properties')['mail.smtp.auth'] !== "false"); + inputFields.set('SMTPUsername.value', selectedAlertNotification.get('properties')['ambari.dispatch.credential.username']); + inputFields.set('SMTPPassword.value', selectedAlertNotification.get('properties')['ambari.dispatch.credential.password']); + inputFields.set('retypeSMTPPassword.value', selectedAlertNotification.get('properties')['ambari.dispatch.credential.password']); + inputFields.set('SMTPSTARTTLS.value', selectedAlertNotification.get('properties')['mail.smtp.starttls.enable'] !== "false"); + inputFields.set('emailFrom.value', selectedAlertNotification.get('properties')['mail.smtp.from']); + inputFields.set('version.value', selectedAlertNotification.get('properties')['ambari.dispatch.snmp.version']); + inputFields.set('OIDs.value', selectedAlertNotification.get('properties')['ambari.dispatch.snmp.oids.trap']); + inputFields.set('community.value', selectedAlertNotification.get('properties')['ambari.dispatch.snmp.community']); + inputFields.set('host.value', selectedAlertNotification.get('properties')['ambari.dispatch.recipients'] ? + selectedAlertNotification.get('properties')['ambari.dispatch.recipients'].join(', ') : ''); + inputFields.set('port.value', selectedAlertNotification.get('properties')['ambari.dispatch.snmp.port']); inputFields.set('severityFilter.value', selectedAlertNotification.get('alertStates')); inputFields.set('global.value', selectedAlertNotification.get('global')); inputFields.set('allGroups.value', selectedAlertNotification.get('global') ? 'all' : 'custom'); @@ -358,13 +348,14 @@ App.ManageAlertNotificationsController = Em.Controller.extend({ inputFields.set('description.value', selectedAlertNotification.get('description')); inputFields.set('method.value', selectedAlertNotification.get('type')); inputFields.get('customProperties').clear(); + var properties = selectedAlertNotification.get('properties'); var ignoredCustomProperties = this.get('ignoredCustomProperties'); - Em.keys(props).forEach(function (k) { + Em.keys(properties).forEach(function (k) { if (ignoredCustomProperties.contains(k)) return; inputFields.get('customProperties').pushObject({ name: k, - value: props[k], - defaultValue: props[k] + value: properties[k], + defaultValue: properties[k] }); }); }, @@ -403,7 +394,7 @@ App.ManageAlertNotificationsController = Em.Controller.extend({ validationMap = self.get('validationMap'); self.get('methods').forEach(function (method) { var validations = validationMap[method]; - if (method === currentMethod) { + if (method == currentMethod) { validations.mapProperty('validator').forEach(function (key) { this.get(key).call(this); }, this); @@ -427,7 +418,7 @@ App.ManageAlertNotificationsController = Em.Controller.extend({ if (!newName) { this.set('nameError', true); errorMessage = Em.I18n.t('alerts.actions.manage_alert_notifications_popup.error.name.empty'); - } else if (newName && newName !== this.get('currentName') && self.get('alertNotifications').mapProperty('name').contains(newName)) { + } else if (newName && newName != this.get('currentName') && self.get('alertNotifications').mapProperty('name').contains(newName)) { this.set('nameError', true); errorMessage = Em.I18n.t('alerts.actions.manage_alert_notifications_popup.error.name.existed'); } else { @@ -484,7 +475,7 @@ App.ManageAlertNotificationsController = Em.Controller.extend({ hostsValidation: function() { var inputValue = this.get('controller.inputFields.host.value').trim(), - hostError = false; + hostError = false;; if (!this.get('isEmailMethodSelected')) { var array = inputValue.split(','); hostError = array.some(function(hostname) { @@ -540,7 +531,7 @@ App.ManageAlertNotificationsController = Em.Controller.extend({ * @method selectAllGroups */ selectAllGroups: function () { - if (this.get('controller.inputFields.allGroups.value') === 'custom') { + if (this.get('controller.inputFields.allGroups.value') == 'custom') { this.set('groupSelect.selection', this.get('groupSelect.content').slice()); } }, @@ -550,7 +541,7 @@ App.ManageAlertNotificationsController = Em.Controller.extend({ * @method clearAllGroups */ clearAllGroups: function () { - if (this.get('controller.inputFields.allGroups.value') === 'custom') { + if (this.get('controller.inputFields.allGroups.value') == 'custom') { this.set('groupSelect.selection', []); } }, @@ -654,9 +645,6 @@ App.ManageAlertNotificationsController = Em.Controller.extend({ properties['ambari.dispatch.snmp.community'] = inputFields.get('community.value'); properties['ambari.dispatch.recipients'] = inputFields.get('host.value').replace(/\s/g, '').split(','); properties['ambari.dispatch.snmp.port'] = inputFields.get('port.value'); - if (inputFields.get('method.value') === 'SNMP') { - properties['ambari.dispatch-property.script'] = "org.apache.ambari.contrib.snmp.script"; - } } inputFields.get('customProperties').forEach(function (customProperty) { properties[customProperty.name] = customProperty.value; @@ -671,7 +659,7 @@ App.ManageAlertNotificationsController = Em.Controller.extend({ properties: properties } }; - if (inputFields.get('allGroups.value') === 'custom') { + if (inputFields.get('allGroups.value') == 'custom') { apiObject.AlertTarget.groups = inputFields.get('groups.value').mapProperty('id'); } return apiObject; diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js index 00fb4b9897..59877a5efa 100644 --- a/ambari-web/app/messages.js +++ b/ambari-web/app/messages.js @@ -2307,6 +2307,7 @@ Em.I18n.translations = { 'hosts.host.summary.hostname':'Hostname', 'hosts.host.summary.agentHeartbeat':'Heartbeat', 'hosts.host.summary.hostMetrics':'Host Metrics', + 'hosts.host.summary.hostLogMetrics':'Host Log Metrics', 'hosts.host.summary.addComponent':'Add Component', 'hosts.host.summary.currentVersion':'Current Version', diff --git a/ambari-web/app/mixins/common/widgets/widget_mixin.js b/ambari-web/app/mixins/common/widgets/widget_mixin.js index df7ff7622b..b6fe353003 100644 --- a/ambari-web/app/mixins/common/widgets/widget_mixin.js +++ b/ambari-web/app/mixins/common/widgets/widget_mixin.js @@ -119,6 +119,7 @@ App.WidgetMixin = Ember.Mixin.create({ context: this, startCallName: 'getHostComponentMetrics', successCallback: this.getHostComponentMetricsSuccessCallback, + errorCallback: this.getMetricsErrorCallback, completeCallback: function () { requestCounter--; if (requestCounter === 0) this.onMetricsLoaded(); @@ -130,6 +131,7 @@ App.WidgetMixin = Ember.Mixin.create({ context: this, startCallName: 'getServiceComponentMetrics', successCallback: this.getMetricsSuccessCallback, + errorCallback: this.getMetricsErrorCallback, completeCallback: function () { requestCounter--; if (requestCounter === 0) this.onMetricsLoaded(); @@ -271,12 +273,41 @@ App.WidgetMixin = Ember.Mixin.create({ if (!Em.isNone(metric_data)) { _metric.data = metric_data; this.get('metrics').pushObject(_metric); + } else if (this.get('graphView')) { + var graph = this.get('childViews') && this.get('childViews').findProperty('_showMessage'); + if (graph) { + graph.set('hasData', false); + this.set('isExportButtonHidden', true); + graph._showMessage('info', this.t('graphs.noData.title'), this.t('graphs.noDataAtTime.message')); + this.get('metrics').clear(); + } } }, this); } }, /** + * error callback on getting aggregated metrics and host component metrics + * @param {object} xhr + * @param {string} textStatus + * @param {string} errorThrown + */ + getMetricsErrorCallback: function (xhr, textStatus, errorThrown) { + if (this.get('graphView')) { + var graph = this.get('childViews') && this.get('childViews').findProperty('_showMessage'); + if (graph) { + if (xhr.readyState == 4 && xhr.status) { + textStatus = xhr.status + " " + textStatus; + } + graph.set('hasData', false); + this.set('isExportButtonHidden', true); + graph._showMessage('warn', this.t('graphs.error.title'), this.t('graphs.error.message').format(textStatus, errorThrown)); + this.get('metrics').clear(); + } + } + }, + + /** * make GET call to get metrics value for all host components * @param {object} request * @return {$.ajax} @@ -704,6 +735,7 @@ App.WidgetLoadAggregator = Em.Object.create({ bulks[id].subRequests = [{ context: request.context, successCallback: request.successCallback, + errorCallback: request.errorCallback, completeCallback: request.completeCallback }]; } else { @@ -711,6 +743,7 @@ App.WidgetLoadAggregator = Em.Object.create({ bulks[id].subRequests.push({ context: request.context, successCallback: request.successCallback, + errorCallback: request.errorCallback, completeCallback: request.completeCallback }); } @@ -732,11 +765,17 @@ App.WidgetLoadAggregator = Em.Object.create({ _request.subRequests.forEach(function (subRequest) { subRequest.successCallback.call(subRequest.context, response); }, this); - }).complete(function () { - _request.subRequests.forEach(function (subRequest) { - subRequest.completeCallback.call(subRequest.context); - }, this); - }); + }).fail(function (xhr, textStatus, errorThrown) { + _request.subRequests.forEach(function (subRequest) { + if (subRequest.errorCallback) { + subRequest.errorCallback.call(subRequest.context, xhr, textStatus, errorThrown); + } + }, this); + }).complete(function () { + _request.subRequests.forEach(function (subRequest) { + subRequest.completeCallback.call(subRequest.context); + }, this); + }); })(bulks[id]); } } diff --git a/ambari-web/app/routes/main.js b/ambari-web/app/routes/main.js index cbaf34d4e6..419f8457c0 100644 --- a/ambari-web/app/routes/main.js +++ b/ambari-web/app/routes/main.js @@ -265,13 +265,16 @@ module.exports = Em.Route.extend(App.RouterRedirections, { }), logs: Em.Route.extend({ - route: '/logs', + route: '/logs:query', connectOutlets: function (router, context) { if (App.get('supports.logSearch')) { router.get('mainHostDetailsController').connectOutlet('mainHostLogs') } else { router.transitionTo('summary'); } + }, + serialize: function(router, params) { + return this.serializeQueryParams(router, params, 'mainHostDetailsController'); } }), @@ -525,6 +528,14 @@ module.exports = Em.Route.extend(App.RouterRedirections, { connectOutlets: function (router) { router.set('mainAdminController.category', "serviceAutoStart"); router.get('mainAdminController').connectOutlet('mainAdminServiceAutoStart'); + }, + exitRoute: function (router, context, callback) { + var controller = router.get('mainAdminServiceAutoStartController'); + if (!controller.get('isSaveDisabled')) { + controller.showSavePopup(callback); + } else { + callback(); + } } }), diff --git a/ambari-web/app/styles/application.less b/ambari-web/app/styles/application.less index a1969db332..57b7e76b6c 100644 --- a/ambari-web/app/styles/application.less +++ b/ambari-web/app/styles/application.less @@ -2245,6 +2245,7 @@ a:focus { left: 60px; overflow: visible; position: relative; + text-align: center; } .chart-y-axis { position: absolute; @@ -2260,6 +2261,10 @@ a:focus { margin-top: 35px !important; } } + .alert { + display: inline-block; + padding-right: 14px; + } } position: relative; margin: 20px 15px 0 15px; @@ -3807,6 +3812,11 @@ table.graphs { .bootstrap-switch { margin-left: 20px; } + + .nav i { + margin-top: 0; + color: @green; + } } .admin-user-settings { diff --git a/ambari-web/app/templates/main/admin/service_auto_start.hbs b/ambari-web/app/templates/main/admin/service_auto_start.hbs index b5c10328ce..5d9b67c68f 100644 --- a/ambari-web/app/templates/main/admin/service_auto_start.hbs +++ b/ambari-web/app/templates/main/admin/service_auto_start.hbs @@ -24,14 +24,48 @@ {{t admin.serviceAutoStart.header.text}} </div> <br/> - <div class="cluster-switcher"> - <p>{{t admin.serviceAutoStart.body.text}} - {{view Ember.Checkbox checkedBinding="view.switcherValue"}} - </p> + <div class="cluster-switcher row-fluid"> + <div class="span4"> + {{t admin.serviceAutoStart.body.text}} + </div> + <div class="span4"> + {{view Ember.Checkbox checkedBinding="view.switcherValue"}} + </div> + <div class="pull-right operations-button pull-right"> + <button class="btn" {{action doReload target="controller"}} {{bindAttr disabled="controller.isSaveDisabled"}}>{{t common.discard}}</button> + <button class="btn btn-success" {{action showSavePopup target="controller"}} {{bindAttr disabled="controller.isSaveDisabled"}}>{{t common.save}}</button> + </div> </div> {{#if view.switcherValue}} - <div> + <div class="tabs-left"> <hr> + <ul class="nav nav-tabs"> + {{#each tab in controller.tabs}} + <li {{bindAttr class="tab.isActive:active"}}> + <a href="#" data-toggle="tab" {{bindAttr data-target="tab.headingClass"}}> + {{tab.display_name}} + <i {{bindAttr class=":pull-right tab.indicator"}}> + </i> + </a> + </li> + {{/each}} + </ul> + <div class="tab-content"> + {{#each tab in controller.tabs}} + <div {{bindAttr class=":tab-pane tab.isActive:active tab.service_name :row-fluid"}}> + {{#each component in tab.componentRecovery}} + <div class="row-fluid"> + <div class="span4"> + {{component.display_name}} + </div> + <div class="span8"> + {{view App.MainAdminServiceAutoStartComponentView recoveryEnabledBinding="component.recovery_enabled" componentBinding="component" tabBinding="tab"}} + </div> + </div> + {{/each}} + </div> + {{/each}} + </div> </div> {{/if}} -</div> +</div>
\ No newline at end of file diff --git a/ambari-web/app/templates/main/admin/service_auto_start/component_auto_start.hbs b/ambari-web/app/templates/main/admin/service_auto_start/component_auto_start.hbs new file mode 100644 index 0000000000..56af5adba2 --- /dev/null +++ b/ambari-web/app/templates/main/admin/service_auto_start/component_auto_start.hbs @@ -0,0 +1,19 @@ +{{! +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +}} + +{{view Ember.Checkbox checkedBinding="view.recoveryEnabled"}}
\ No newline at end of file diff --git a/ambari-web/app/templates/main/host/log_metrics.hbs b/ambari-web/app/templates/main/host/log_metrics.hbs new file mode 100644 index 0000000000..22a39bee16 --- /dev/null +++ b/ambari-web/app/templates/main/host/log_metrics.hbs @@ -0,0 +1,26 @@ +{{! +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +}} + +<div class="row-fluid log-metrics-charts mtl"> + {{#each item in view.logsData}} + <div class="span6 text-center mtl"> + {{view view.chartView contentBinding="item"}} + <a href="#" {{action transitionByService item target="view"}}>{{item.service.displayName}}</a> + </div> + {{/each}} +</div> diff --git a/ambari-web/app/templates/main/host/summary.hbs b/ambari-web/app/templates/main/host/summary.hbs index 6b4c7a571c..17a0b69837 100644 --- a/ambari-web/app/templates/main/host/summary.hbs +++ b/ambari-web/app/templates/main/host/summary.hbs @@ -168,19 +168,31 @@ </div> </div> </div> - {{!metrics}} - {{#unless view.isNoHostMetricsService}} - <div class="span6"> + <div class="span6"> + {{!metrics}} + {{#unless view.isNoHostMetricsService}} <div class="box"> <div class="box-header"> <h4>{{t hosts.host.summary.hostMetrics}}</h4> {{view view.timeRangeListView}} </div> <div> - {{view App.MainHostMetricsView contentBinding="view.content"}} + {{view App.MainHostMetricsView contentBinding="view.content"}} </div> </div> - </div> + {{/unless}} + + {{!logs metrics}} + {{#if App.supports.logSearch}} + <div class="box"> + <div class="box-header"> + <h4>{{t hosts.host.summary.hostLogMetrics}}</h4> + </div> + <div> + {{view App.MainHostLogMetrics contentBinding="view.content"}} + </div> + </div> + {{/if}} </div> - {{/unless}} + </div> </div> diff --git a/ambari-web/app/utils/ajax/ajax.js b/ambari-web/app/utils/ajax/ajax.js index 8b89365d16..ce6a196ff9 100644 --- a/ambari-web/app/utils/ajax/ajax.js +++ b/ambari-web/app/utils/ajax/ajax.js @@ -2340,6 +2340,27 @@ var urls = { 'real': '/clusters/{clusterName}/components?fields=host_components/HostRoles/host_name,ServiceComponentInfo/component_name,ServiceComponentInfo/started_count{urlParams}&minimal_response=true', 'mock': '' }, + 'components.get_category': { + 'real': '/clusters/{clusterName}/components?fields=ServiceComponentInfo/component_name,ServiceComponentInfo/service_name,ServiceComponentInfo/category,ServiceComponentInfo/recovery_enabled&minimal_response=true', + 'mock': '' + }, + 'components.update': { + 'real': '/clusters/{clusterName}/components?{urlParams}', + 'mock': '', + 'type': 'PUT', + 'format': function (data) { + return { + data: JSON.stringify({ + RequestInfo: { + query: data.query + }, + Body: { + ServiceComponentInfo: data.ServiceComponentInfo + } + }) + } + } + }, 'hosts.all.install': { 'real': '/hosts?minimal_response=true', 'mock': '' diff --git a/ambari-web/app/utils/ember_reopen.js b/ambari-web/app/utils/ember_reopen.js index 512b3da7de..bf091da934 100644 --- a/ambari-web/app/utils/ember_reopen.js +++ b/ambari-web/app/utils/ember_reopen.js @@ -242,6 +242,23 @@ Ember.TextArea.reopen({ attributeBindings: ['readonly'] }); +/** + * Simply converts query string to object. + * + * @param {string} queryString query string e.g. '?param1=value1¶m2=value2' + * @return {object} converted object + */ +function parseQueryParams(queryString) { + if (!queryString) { + return {}; + } + return queryString.replace(/^\?/, '').split('&').map(decodeURIComponent) + .reduce(function(p, c) { + var keyVal = c.split('='); + p[keyVal[0]] = keyVal[1]; + return p; + }, {}); +}; Ember.Route.reopen({ /** @@ -257,6 +274,35 @@ Ember.Route.reopen({ */ exitRoute: function (router, context, callback) { callback(); + }, + + /** + * Query Params serializer. This method should be used inside <code>serialize</code> method. + * You need to specify `:query` dynamic sygment in your route's <code>route</code> attribute + * e.g. Em.Route.extend({ route: '/login:query'}) and return result of this method. + * This method will set <code>serializedQuery</code> property to specified controller by name. + * For concrete example see `app/routes/main.js`. + * + * @example + * queryParams: Em.Route.extend({ + * route: '/queryDemo:query', + * serialize: function(route, params) { + * return this.serializeQueryParams(route, params, 'controllerNameToSetQueryObject'); + * } + * }); + * // now when navigated to http://example.com/#/queryDemo?param1=value1¶m2=value2 + * // App.router.get('controllerNameToSetQueryObject').get('serializedQuery') + * // will return { param1: 'value1', param2: 'value2' } + * + * @param {Em.Router} router router instance passed to <code>serialize</code> method + * @param {object} params dynamic segment passed to <code>seriazlie</code> + * @param {string} controllerName name of the controller to set `serializedQuery` as result + * @return {object} + */ + serializeQueryParams: function(router, params, controllerName) { + var controller = router.get(controllerName); + controller.set('serializedQuery', parseQueryParams(params ? params.query : '')); + return params || { query: ''}; } }); diff --git a/ambari-web/app/views.js b/ambari-web/app/views.js index 570f6cd078..2440086773 100644 --- a/ambari-web/app/views.js +++ b/ambari-web/app/views.js @@ -128,6 +128,7 @@ require('views/main/host/summary'); require('views/main/host/configs'); require('views/main/host/configs_service'); require('views/main/host/configs_service_menu'); +require('views/main/host/log_metrics'); require('views/main/host/metrics'); require('views/main/host/stack_versions_view'); require('views/main/host/add_view'); @@ -141,6 +142,7 @@ require('views/main/host/metrics/processes'); require('views/main/host/addHost/step4_view'); require('views/main/admin'); require('views/main/admin/service_auto_start'); +require('views/main/admin/service_auto_start/component_auto_start'); require('views/main/admin/highAvailability/nameNode/wizard_view'); require('views/main/admin/highAvailability/progress_view'); require('views/main/admin/highAvailability/nameNode/rollback_view'); diff --git a/ambari-web/app/views/common/chart/pie.js b/ambari-web/app/views/common/chart/pie.js index 0280f87452..ce9bda4b48 100644 --- a/ambari-web/app/views/common/chart/pie.js +++ b/ambari-web/app/views/common/chart/pie.js @@ -82,16 +82,15 @@ App.ChartPieView = Em.View.extend({ .append("svg:g") .attr("transform", "translate(" + thisChart.get('w') / 2 + "," + thisChart.get('h') / 2 + ")")); - this.set('arcs', thisChart.get('svg').selectAll("path") + this.set('arcs', thisChart.get('svg').selectAll(".arc") .data(thisChart.donut(thisChart.get('data'))) - .enter().append("svg:path") + .enter() + .append("svg:g").attr('class', 'arc') + .append('svg:path') .attr("fill", function (d, i) { return thisChart.palette.color(i); }) .attr("d", thisChart.get('arc')) - ); - } - -});
\ No newline at end of file +}); diff --git a/ambari-web/app/views/common/widget/graph_widget_view.js b/ambari-web/app/views/common/widget/graph_widget_view.js index da30dca302..6feaa28d5d 100644 --- a/ambari-web/app/views/common/widget/graph_widget_view.js +++ b/ambari-web/app/views/common/widget/graph_widget_view.js @@ -311,7 +311,9 @@ App.GraphWidgetView = Em.View.extend(App.WidgetMixin, App.ExportMetricsMixin, { self.set('parentView.isExportMenuHidden', true); }); this.setYAxisFormatter(); - this.loadData(); + if (!arguments.length || this.get('parentView.data.length')) { + this.loadData(); + } var self = this; Em.run.next(function () { if (self.get('isPreview')) { diff --git a/ambari-web/app/views/main/admin/service_auto_start.js b/ambari-web/app/views/main/admin/service_auto_start.js index 2d3ae16e6c..dcbbfc93e2 100644 --- a/ambari-web/app/views/main/admin/service_auto_start.js +++ b/ambari-web/app/views/main/admin/service_auto_start.js @@ -28,7 +28,7 @@ App.MainAdminServiceAutoStartView = Em.View.extend({ */ switcherValue: false, - clusterConfigs: null, + savedRecoveryEnabled: false, didInsertElement: function () { var self = this; @@ -41,12 +41,14 @@ App.MainAdminServiceAutoStartView = Em.View.extend({ } ]; App.router.get('configurationController').getConfigsByTags(tag).done(function (data) { - self.set('clusterConfigs', data[0].properties); - self.set('switcherValue', self.get('clusterConfigs.recovery_enabled') === 'true'); + self.set('controller.clusterConfigs', data[0].properties); + self.set('switcherValue', data[0].properties.recovery_enabled === 'true'); + self.set('savedRecoveryEnabled', self.get('switcherValue')); // plugin should be initiated after applying binding for switcherValue Em.run.later('sync', function() { self.initSwitcher(); }.bind(self), 10); + self.get('controller').loadComponentsConfigs(); }); }); }, @@ -58,8 +60,8 @@ App.MainAdminServiceAutoStartView = Em.View.extend({ */ updateClusterConfigs: function (state){ this.set('switcherValue', state); - this.set('clusterConfigs.recovery_enabled', '' + state); - this.get('controller').saveClusterConfigs(this.get('clusterConfigs')); + this.set('controller.clusterConfigs.recovery_enabled', '' + state); + this.set('controller.valueChanged', this.get('savedRecoveryEnabled') !== state); }, /** @@ -70,7 +72,7 @@ App.MainAdminServiceAutoStartView = Em.View.extend({ initSwitcher: function () { var self = this; if (this.$()) { - var switcher = this.$("input:eq(0)").bootstrapSwitch({ + this.$("input:eq(0)").bootstrapSwitch({ onText: Em.I18n.t('common.enabled'), offText: Em.I18n.t('common.disabled'), offColor: 'default', diff --git a/ambari-web/app/views/main/admin/service_auto_start/component_auto_start.js b/ambari-web/app/views/main/admin/service_auto_start/component_auto_start.js new file mode 100644 index 0000000000..cc910b8f57 --- /dev/null +++ b/ambari-web/app/views/main/admin/service_auto_start/component_auto_start.js @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var App = require('app'); + +App.MainAdminServiceAutoStartComponentView = Em.View.extend({ + templateName: require('templates/main/admin/service_auto_start/component_auto_start'), + + recoveryEnabled: false, + savedRecoveryEnabled: false, + tab: null, + component: null, + + didInsertElement: function () { + this.set('savedRecoveryEnabled', this.get('recoveryEnabled')); + this.initSwitcher(); + }, + + /** + * Init switcher plugin. + * + * @method initSwitcher + */ + initSwitcher: function () { + var self = this; + if (this.$()) { + this.$("input:eq(0)").bootstrapSwitch({ + onText: Em.I18n.t('common.enabled'), + offText: Em.I18n.t('common.disabled'), + offColor: 'default', + onColor: 'success', + handleWidth: Math.max(Em.I18n.t('common.enabled').length, Em.I18n.t('common.disabled').length) * 8, + onSwitchChange: function (event, state) { + self.set('tab.enabledComponents', self.get('tab.enabledComponents') + (state ? 1 : -1)); + self.set('recoveryEnabled', state); + self.set('component.valueChanged', self.get('savedRecoveryEnabled') !== state); + self.get('parentView.controller').checkValuesChange(); + } + }); + } + } +});
\ No newline at end of file diff --git a/ambari-web/app/views/main/dashboard/cluster_metrics/cpu.js b/ambari-web/app/views/main/dashboard/cluster_metrics/cpu.js index 407487d0bd..eea3040cb0 100644 --- a/ambari-web/app/views/main/dashboard/cluster_metrics/cpu.js +++ b/ambari-web/app/views/main/dashboard/cluster_metrics/cpu.js @@ -43,21 +43,21 @@ App.ChartClusterMetricsCPU = App.ChartLinearTimeView.extend({ idle = null, data = Em.get(jsonData, this.get('seriesTemplate.path')); if (data) { - for (var name in data) { + Object.keys(data).forEach(function (name) { var seriesData = data[name]; if (seriesData) { var s = { name: name, data: seriesData }; - if (name.indexOf('Idle') > -1) { + if (name.contains('Idle')) { //CPU idle metric should be the last in series array idle = s; - continue; + return; } dataArray.push(s); } - } + }); if (idle) { dataArray.push(idle); } @@ -66,7 +66,7 @@ App.ChartClusterMetricsCPU = App.ChartLinearTimeView.extend({ }, colorForSeries: function (series) { - if (Em.I18n.t('dashboard.clusterMetrics.cpu.displayNames.idle') == series.name){ + if (series.name && series.name.contains(Em.I18n.t('dashboard.clusterMetrics.cpu.displayNames.idle'))) { return '#CFECEC'; } return null; diff --git a/ambari-web/app/views/main/host/log_metrics.js b/ambari-web/app/views/main/host/log_metrics.js new file mode 100644 index 0000000000..20f5ec6d53 --- /dev/null +++ b/ambari-web/app/views/main/host/log_metrics.js @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var App = require('app'); + +/** + * @typedef {Ember.Object} LogLevelItemObject + * @property {string} level level name + * @property {number} counter + */ +/** + * @typedef {Object} ServiceLogMetricsObject + * @property {App.Service} service model instance + * @property {LogLevelItemObject[]} logs + */ +App.MainHostLogMetrics = Em.View.extend({ + templateName: require('templates/main/host/log_metrics'), + classNames: ['host-log-metrics'], + + /** + * @type {ServiceLogMetricsObject[]} + */ + logsData: function() { + var services = this.get('content').get('hostComponents').mapProperty('service').uniq(); + var logLevels = ['fatal', 'critical', 'error', 'warning', 'info', 'debug']; + return services.map(function(service) { + var levels = logLevels.map(function(level) { + return Em.Object.create({ + level: level, + counter: Math.ceil(Math.random()*10) + }); + }); + return Em.Object.create({ + service: service, + logs: levels + }); + }); + }.property('content'), + + /** + * @type {Ember.View} Pie Chart view + * @extends App.PieChartView + */ + chartView: App.ChartPieView.extend({ + classNames: ['log-metrics-chart'], + w: 150, + h: 150, + stroke: '#fff', + strokeWidth: 1, + levelColors: { + FATAL: '#B10202', + CRITICAL: '#E00505', + ERROR: App.healthStatusRed, + INFO: App.healthStatusGreen, + WARNING: App.healthStatusOrange, + DEBUG: '#1e61f7' + }, + innerR: 36, + donut: d3.layout.pie().sort(null).value(function(d) { return d.get('counter'); }), + + prepareChartData: function(content) { + this.set('data', content.get('logs')); + }, + + didInsertElement: function() { + this.prepareChartData(this.get('content')); + this._super(); + this.appendLabels(); + this.formatCenterText(); + this.attachArcEvents(); + this.colorizeArcs(); + }, + + attachArcEvents: function() { + var self = this; + this.get('svg').selectAll('.arc') + .on('mouseover', function(d) { + self.get('svg').select('g.center-text').select('text') + .text(d.data.get('level').capitalize() + ": " + d.data.get('counter')); + }) + .on('mouseout', function() { + self.get('svg').select('g.center-text').select('text').text(''); + }); + }, + + formatCenterText: function() { + this.get('svg') + .append('svg:g') + .attr('class', 'center-text') + .attr('render-order', 1) + .append('svg:text') + .attr('transform', "translate(0,0)") + .attr('text-anchor', 'middle') + .attr('stroke', '#000') + .attr('stroke-width', 0) + }, + + appendLabels: function() { + var labelArc = d3.svg.arc() + .outerRadius(this.get('outerR') - 15) + .innerRadius(this.get('outerR') - 15); + this.get('svg').selectAll('.arc') + .append('text') + .attr('transform', function(d) { return "translate(" + labelArc.centroid(d) + ")"; }) + .attr('stroke', '#000') + .attr('stroke-width', 0) + .attr('font-size', '12px') + .attr('dy', '.50em') + .text(function(d) { return d.data.get('counter'); }); + }, + + colorizeArcs: function() { + var self = this; + this.get('svg').selectAll('.arc path') + .attr('fill', function(d) { + return self.get('levelColors')[d.data.get('level').toUpperCase()]; + }); + } + }), + + + transitionByService: function(e) { + var service = e.context; + App.router.transitionTo('logs', {query: '?service_name=' + service.get('service.serviceName')}); + } +}); diff --git a/ambari-web/app/views/main/host/logs_view.js b/ambari-web/app/views/main/host/logs_view.js index dfe00e410b..b51f9556ab 100644 --- a/ambari-web/app/views/main/host/logs_view.js +++ b/ambari-web/app/views/main/host/logs_view.js @@ -67,6 +67,10 @@ App.MainHostLogsView = App.TableView.extend({ serviceNameFilterView: filters.createSelectView({ column: 1, fieldType: 'filter-input-width', + didInsertElement: function() { + this.setValue(Em.getWithDefault(this, 'controller.serializedQuery.service_name', '')); + this._super(); + }, content: function() { return [{ value: '', @@ -86,6 +90,10 @@ App.MainHostLogsView = App.TableView.extend({ componentNameFilterView: filters.createSelectView({ column: 2, fieldType: 'filter-input-width', + didInsertElement: function() { + this.setValue(Em.getWithDefault(this, 'controller.serializedQuery.component_name', '')); + this._super(); + }, content: function() { var hostName = this.get('parentView').get('host.hostName'), hostComponents = App.HostComponent.find().filterProperty('hostName', hostName), @@ -108,6 +116,10 @@ App.MainHostLogsView = App.TableView.extend({ fileExtensionsFilter: filters.createSelectView({ column: 3, fieldType: 'filter-input-width', + didInsertElement: function() { + this.setValue(Em.getWithDefault(this, 'controller.serializedQuery.file_extension', '')); + this._super(); + }, content: function() { return [{ value: '', diff --git a/ambari-web/test/controllers/main/alerts/manage_alert_notifications_controller_test.js b/ambari-web/test/controllers/main/alerts/manage_alert_notifications_controller_test.js index 283c0af0f7..19d0c1bf57 100644 --- a/ambari-web/test/controllers/main/alerts/manage_alert_notifications_controller_test.js +++ b/ambari-web/test/controllers/main/alerts/manage_alert_notifications_controller_test.js @@ -699,77 +699,56 @@ describe('App.ManageAlertNotificationsController', function () { describe("#formatNotificationAPIObject()", function () { - var inputFields; - - beforeEach(function () { - inputFields = Em.Object.create({ - name: { - value: 'test_name' - }, - groups: { - value: [{id: 1}, {id: 2}, {id: 3}] - }, - allGroups: { - value: 'custom' - }, - global: { - value: false - }, - method: { - value: 'EMAIL' - }, - email: { - value: 'test1@test.test, test2@test.test,test3@test.test , test4@test.test' - }, - severityFilter: { - value: ['OK', 'CRITICAL'] - }, - SMTPServer: { - value: 's1' - }, - SMTPPort: { - value: '25' - }, - SMTPUseAuthentication: { - value: "true" - }, - SMTPUsername: { - value: 'user' - }, - SMTPPassword: { - value: 'pass' - }, - SMTPSTARTTLS: { - value: "true" - }, - emailFrom: { - value: 'from' - }, - description: { - value: 'test_description' - }, - host: { - value: '' - }, - customProperties: [ - {name: 'n1', value: 'v1'}, - {name: 'n2', value: 'v2'} - ] - }); - }); - - it('should set property `ambari.dispatch-property.script` for SNMP type', function () { - Em.set(inputFields, 'method.value', 'SNMP'); - controller.set('inputFields', inputFields); - var result = controller.formatNotificationAPIObject(); - expect(result.AlertTarget.properties['ambari.dispatch-property.script']).to.be.equal('org.apache.ambari.contrib.snmp.script'); - }); - - it('should not set property `ambari.dispatch-property.script` for EMAIL type', function () { - Em.set(inputFields, 'method.value', 'EMAIL'); - controller.set('inputFields', inputFields); - var result = controller.formatNotificationAPIObject(); - expect(result.AlertTarget.properties).to.not.have.property('ambari.dispatch-property.script'); + var inputFields = Em.Object.create({ + name: { + value: 'test_name' + }, + groups: { + value: [{id: 1}, {id: 2}, {id: 3}] + }, + allGroups: { + value: 'custom' + }, + global: { + value: false + }, + method: { + value: 'EMAIL' + }, + email: { + value: 'test1@test.test, test2@test.test,test3@test.test , test4@test.test' + }, + severityFilter: { + value: ['OK', 'CRITICAL'] + }, + SMTPServer: { + value: 's1' + }, + SMTPPort: { + value: '25' + }, + SMTPUseAuthentication: { + value: "true" + }, + SMTPUsername: { + value: 'user' + }, + SMTPPassword: { + value: 'pass' + }, + SMTPSTARTTLS: { + value: "true" + }, + emailFrom: { + value: 'from' + }, + description: { + value: 'test_description' + }, + customProperties: [ + {name: 'n1', value: 'v1'}, + {name: 'n2', value: 'v2'} + ] }); it("should create object with properties from inputFields values", function () { diff --git a/ambari-web/test/mixins/common/widget_mixin_test.js b/ambari-web/test/mixins/common/widget_mixin_test.js index 00606f4512..91f628f626 100644 --- a/ambari-web/test/mixins/common/widget_mixin_test.js +++ b/ambari-web/test/mixins/common/widget_mixin_test.js @@ -384,6 +384,129 @@ describe('App.WidgetMixin', function () { }); }); }); + + describe('#getMetricsErrorCallback()', function () { + + var obj, + view = Em.Object.create({ + _showMessage: Em.K + }), + cases = [ + { + graphView: null, + metricsLength: 1, + showMessageCallCount: 0, + isExportButtonHidden: false, + title: 'no graph view' + }, + { + graphView: {}, + metricsLength: 1, + showMessageCallCount: 0, + isExportButtonHidden: false, + title: 'no childViews property' + }, + { + graphView: {}, + childViews: [], + metricsLength: 1, + showMessageCallCount: 0, + isExportButtonHidden: false, + title: 'no child views' + }, + { + graphView: {}, + childViews: [Em.Object.create({})], + metricsLength: 1, + showMessageCallCount: 0, + isExportButtonHidden: false, + title: 'no view with _showMessage method' + }, + { + graphView: {}, + childViews: [Em.Object.create({}), view], + metricsLength: 0, + showMessageCallCount: 1, + isExportButtonHidden: true, + title: 'graph view is available' + } + ], + messageCases = [ + { + readyState: 2, + status: 0, + textStatus: 'error', + title: 'incomplete request' + }, + { + readyState: 4, + status: 0, + textStatus: 'error', + title: 'no status code' + }, + { + readyState: 4, + status: 404, + textStatus: '404 error', + title: 'status code available' + } + ]; + + beforeEach(function () { + sinon.spy(view, '_showMessage'); + }); + + afterEach(function () { + view._showMessage.restore(); + }); + + cases.forEach(function (item) { + + describe(item.title, function () { + + beforeEach(function () { + obj = Em.Object.create(App.WidgetMixin, { + metrics: [{}], + isExportButtonHidden: false, + graphView: item.graphView, + childViews: item.childViews + }); + obj.getMetricsErrorCallback({}); + }); + + it('metrics array', function () { + expect(obj.get('metrics')).to.have.length(item.metricsLength); + }); + + it('error message', function () { + expect(view._showMessage.callCount).to.equal(item.showMessageCallCount); + }); + + it('export button display', function () { + expect(obj.get('isExportButtonHidden')).to.equal(item.isExportButtonHidden); + }); + + }); + + }); + + messageCases.forEach(function (item) { + + it(item.title, function () { + obj = Em.Object.create(App.WidgetMixin, { + graphView: Em.Object.create({}), + childViews: [view] + }); + obj.getMetricsErrorCallback({ + readyState: item.readyState, + status: item.status + }, 'error', 'Not Found'); + expect(view._showMessage.firstCall.args).to.eql(['warn', Em.I18n.t('graphs.error.title'), Em.I18n.t('graphs.error.message').format(item.textStatus, 'Not Found')]); + }); + + }); + + }); }); @@ -502,6 +625,7 @@ describe('App.WidgetLoadAggregator', function () { f1: function () { return { done: Em.K, + fail: Em.K, complete: Em.K } }, diff --git a/ambari-web/test/utils/ember_reopen_test.js b/ambari-web/test/utils/ember_reopen_test.js index eda5e81c06..aa50a50b03 100644 --- a/ambari-web/test/utils/ember_reopen_test.js +++ b/ambari-web/test/utils/ember_reopen_test.js @@ -78,4 +78,61 @@ describe('Ember functionality extension', function () { }); + describe('#Em.Route', function() { + describe('#serializeQueryParams', function() { + var route, + cases = [ + { + m: 'No query params', + params: undefined, + e: { + result: {query: ''}, + serializedQuery: {} + } + }, + { + m: 'Query params ?param1=value1¶m2=value2', + params: { query: '?param1=value1¶m2=value2'}, + e: { + result: {query: '?param1=value1¶m2=value2'}, + serializedQuery: {param1: 'value1', param2: 'value2'} + } + }, + { + m: 'Query params with encodedComponent ?param1=value1%30¶m2=value2', + params: { query: '?param1=value1%30¶m2=value2'}, + e: { + result: {query: '?param1=value1%30¶m2=value2'}, + serializedQuery: {param1: 'value10', param2: 'value2'} + } + } + ]; + + beforeEach(function() { + route = Ember.Route.create({ + route: 'demo:query', + serialize: function(router, params) { + return this.serializeQueryParams(router, params, 'testController'); + } + }); + }); + + afterEach(function() { + route.destroy(); + route = null; + }); + + cases.forEach(function(test) { + it(test.m, function() { + var ctrl = Em.Object.create({}); + var router = Em.Object.create({ + testController: ctrl + }); + var ret = route.serialize(router, test.params); + expect(ret).to.be.eql(test.e.result); + expect(ctrl.get('serializedQuery')).to.be.eql(test.e.serializedQuery); + }); + }); + }); + }); }); diff --git a/ambari-web/test/views/common/log_file_search_view_test.js b/ambari-web/test/views/common/log_file_search_view_test.js index a5f940e368..ca208b30c4 100644 --- a/ambari-web/test/views/common/log_file_search_view_test.js +++ b/ambari-web/test/views/common/log_file_search_view_test.js @@ -32,8 +32,7 @@ describe('App.LogFileSearchView', function() { isIncluded: !!isIncluded }); }; - - [ + var cases = [ { viewContent: { keywordsFilterValue: 'some_keyword' |
