diff options
53 files changed, 377 insertions, 384 deletions
diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 6f8653ba..1aa954bd 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -19,8 +19,8 @@ For more information on migrating from WA2 to WA3 please see the Not all of WA2 extensions have been ported for the initial 3.0.0 release. We have ported the ones we believe to be most widely used and useful. The porting -work will continue, and more of WA2's extensions will be in the future releases, -however we do not intend to port absolutely everything, as some things we +work will continue, and more of WA2's extensions will be in the future releases. +However, we do not intend to port absolutely everything, as some things we believe to be no longer useful. .. note:: If there a particular WA2 extension you would like to see in WA3 that @@ -31,6 +31,15 @@ believe to be no longer useful. New Features ~~~~~~~~~~~~ +- Python 3 support. WA now runs on both Python 2 and Python 3. + + .. warning:: Python 2 support should now be considered depricated. Python 2 + will still be fully supported up to the next major release + (v3.1). After that, Python 2 will be supported for existing + functionality, however there will be no guarantee that newly + added functionality would be compatible with Python 2. Support + for Python 2 will be dropped completely after release v3.2. + - There is a new Output API which can be used to aid in post processing a run's output. For more information please see :ref:`output_processing_api`. - All "augmentations" can now be enabled on a per workload basis (in WA2 this @@ -81,9 +81,10 @@ params = dict( 'pyYAML', # YAML-formatted agenda parsing 'requests', # Fetch assets over HTTP 'devlib>=0.0.4', # Interacting with devices - 'louie', # callbacks dispatch + 'louie-latest', # callbacks dispatch 'wrapt', # better decorators 'pandas>=0.13.1', # Data analysis and manipulation + 'future', # Python 2-3 compatiblity ], dependency_links=['https://github.com/ARM-software/devlib/tarball/master#egg=devlib-0.0.4'], @@ -104,7 +105,7 @@ params = dict( ], ) -all_extras = list(chain(params['extras_require'].itervalues())) +all_extras = list(chain(iter(params['extras_require'].values()))) params['extras_require']['everything'] = all_extras setup(**params) diff --git a/wa/commands/create.py b/wa/commands/create.py index b6024c3b..415bfa39 100644 --- a/wa/commands/create.py +++ b/wa/commands/create.py @@ -94,9 +94,9 @@ class CreateWorkloadSubcommand(SubCommand): self.parser.add_argument('-f', '--force', action='store_true', help='Create the new workload even if a workload with the specified ' + 'name already exists.') - self.parser.add_argument('-k', '--kind', metavar='KIND', default='basic', choices=create_funcs.keys(), + self.parser.add_argument('-k', '--kind', metavar='KIND', default='basic', choices=list(create_funcs.keys()), help='The type of workload to be created. The available options ' + - 'are: {}'.format(', '.join(create_funcs.keys()))) + 'are: {}'.format(', '.join(list(create_funcs.keys())))) def execute(self, state, args): # pylint: disable=R0201 where = args.path or 'local' @@ -179,7 +179,7 @@ def create_workload(name, kind='basic', where='local', check_name=True, **kwargs except KeyError: raise CommandError('Unknown workload type: {}'.format(kind)) - print 'Workload created in {}'.format(workload_dir) + print('Workload created in {}'.format(workload_dir)) def create_template_workload(path, name, kind, class_name): diff --git a/wa/commands/list.py b/wa/commands/list.py index 1b5637fe..2fa651f1 100644 --- a/wa/commands/list.py +++ b/wa/commands/list.py @@ -71,7 +71,7 @@ def list_targets(): output = DescriptionListFormatter() for target in targets: output.add_item(target.description or '', target.name) - print output.format_data() + print(output.format_data()) def list_plugins(args, filters): @@ -80,7 +80,7 @@ def list_plugins(args, filters): filtered_results = [] for result in results: passed = True - for k, v in filters.iteritems(): + for k, v in filters.items(): if getattr(result, k) != v: passed = False break @@ -95,7 +95,7 @@ def list_plugins(args, filters): output = DescriptionListFormatter() for result in sorted(filtered_results, key=lambda x: x.name): output.add_item(get_summary(result), result.name) - print output.format_data() + print(output.format_data()) def check_platform(plugin, platform): diff --git a/wa/commands/revent.py b/wa/commands/revent.py index 45d6e207..1da367fd 100644 --- a/wa/commands/revent.py +++ b/wa/commands/revent.py @@ -24,6 +24,10 @@ from wa.framework.target.manager import TargetManager from wa.utils.revent import ReventRecorder +if sys.version_info[0] == 3: + raw_input = input + + class RecordCommand(Command): name = 'record' @@ -146,7 +150,7 @@ class RecordCommand(Command): if os.path.exists(host_path): msg = 'Revent file \'{}\' already exists, overwrite? [y/n]' self.logger.info(msg.format(revent_file_name)) - if raw_input('') == 'y': + if input('') == 'y': os.remove(host_path) else: msg = 'Did not pull and overwrite \'{}\'' @@ -222,7 +226,7 @@ class RecordCommand(Command): if not file_name: file_name = '{}.revent'.format(self.target.model) if not output_path: - output_path = os.getcwdu() + output_path = os.getcwd() return output_path, file_name diff --git a/wa/commands/show.py b/wa/commands/show.py index d19bcb8a..e5fdd38c 100644 --- a/wa/commands/show.py +++ b/wa/commands/show.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import sys from subprocess import call, Popen, PIPE from wa import Command @@ -66,7 +67,11 @@ class ShowCommand(Command): if which('pandoc'): p = Popen(['pandoc', '-f', 'rst', '-t', 'man'], stdin=PIPE, stdout=PIPE, stderr=PIPE) - output, _ = p.communicate(rst_output) + if sys.version_info[0] == 3: + output, _ = p.communicate(rst_output.encode(sys.stdin.encoding)) + output = output.decode(sys.stdout.encoding) + else: + output, _ = p.communicate(rst_output) # Make sure to double escape back slashes output = output.replace('\\', '\\\\\\') @@ -78,7 +83,7 @@ class ShowCommand(Command): call('echo "{}" | man -l -'.format(escape_double_quotes(output)), shell=True) else: - print rst_output + print(rst_output) def get_target_description(name): diff --git a/wa/framework/configuration/core.py b/wa/framework/configuration/core.py index 9a571a22..f0a4f738 100644 --- a/wa/framework/configuration/core.py +++ b/wa/framework/configuration/core.py @@ -95,11 +95,11 @@ class RebootPolicy(object): __repr__ = __str__ - def __cmp__(self, other): + def __eq__(self, other): if isinstance(other, RebootPolicy): - return cmp(self.policy, other.policy) + return self.policy == other.policy else: - return cmp(self.policy, other) + return self.policy == other def to_pod(self): return self.policy @@ -127,7 +127,7 @@ class LoggingConfig(dict): def __init__(self, config=None): dict.__init__(self) if isinstance(config, dict): - config = {identifier(k.lower()): v for k, v in config.iteritems()} + config = {identifier(k.lower()): v for k, v in config.items()} self['regular_format'] = config.pop('regular_format', self.defaults['regular_format']) self['verbose_format'] = config.pop('verbose_format', self.defaults['verbose_format']) self['file_format'] = config.pop('file_format', self.defaults['file_format']) @@ -135,9 +135,9 @@ class LoggingConfig(dict): self['color'] = config.pop('color', self.defaults['color']) if config: message = 'Unexpected logging configuration parameters: {}' - raise ValueError(message.format(bad_vals=', '.join(config.keys()))) + raise ValueError(message.format(bad_vals=', '.join(list(config.keys())))) elif config is None: - for k, v in self.defaults.iteritems(): + for k, v in self.defaults.items(): self[k] = v else: raise ValueError(config) @@ -360,7 +360,7 @@ class Configuration(object): cfg_point.set_value(instance, value) if pod: msg = 'Invalid entry(ies) for "{}": "{}"' - raise ValueError(msg.format(cls.name, '", "'.join(pod.keys()))) + raise ValueError(msg.format(cls.name, '", "'.join(list(pod.keys())))) return instance def __init__(self): @@ -380,7 +380,7 @@ class Configuration(object): def update_config(self, values, check_mandatory=True): - for k, v in values.iteritems(): + for k, v in values.items(): self.set(k, v, check_mandatory=check_mandatory) def validate(self): @@ -824,7 +824,7 @@ class JobSpec(Configuration): def update_config(self, source, check_mandatory=True): self._sources.append(source) values = source.config - for k, v in values.iteritems(): + for k, v in values.items(): if k == "id": continue elif k.endswith('_parameters'): @@ -849,7 +849,7 @@ class JobSpec(Configuration): if not config: continue - for name, cfg_point in cfg_points.iteritems(): + for name, cfg_point in cfg_points.items(): if name in config: value = config.pop(name) cfg_point.set_value(workload_params, value, @@ -873,7 +873,7 @@ class JobSpec(Configuration): runtime_parameters[source] = global_runtime_params[source] # Add runtime parameters from JobSpec - for source, values in self.to_merge['runtime_parameters'].iteritems(): + for source, values in self.to_merge['runtime_parameters'].items(): runtime_parameters[source] = values # Merge @@ -884,9 +884,9 @@ class JobSpec(Configuration): for source in self._sources[1:]]) # ignore first id, "global" # ensure *_parameters are always obj_dict's - self.boot_parameters = obj_dict((self.boot_parameters or {}).items()) - self.runtime_parameters = obj_dict((self.runtime_parameters or {}).items()) - self.workload_parameters = obj_dict((self.workload_parameters or {}).items()) + self.boot_parameters = obj_dict(list((self.boot_parameters or {}).items())) + self.runtime_parameters = obj_dict(list((self.runtime_parameters or {}).items())) + self.workload_parameters = obj_dict(list((self.workload_parameters or {}).items())) if self.label is None: self.label = self.workload_name @@ -903,7 +903,7 @@ class JobGenerator(object): self._read_augmentations = True if self._enabled_instruments is None: self._enabled_instruments = [] - for entry in self._enabled_augmentations.merge_with(self.disabled_augmentations).values(): + for entry in list(self._enabled_augmentations.merge_with(self.disabled_augmentations).values()): entry_cls = self.plugin_cache.get_plugin_class(entry) if entry_cls.kind == 'instrument': self._enabled_instruments.append(entry) @@ -914,7 +914,7 @@ class JobGenerator(object): self._read_augmentations = True if self._enabled_processors is None: self._enabled_processors = [] - for entry in self._enabled_augmentations.merge_with(self.disabled_augmentations).values(): + for entry in list(self._enabled_augmentations.merge_with(self.disabled_augmentations).values()): entry_cls = self.plugin_cache.get_plugin_class(entry) if entry_cls.kind == 'output_processor': self._enabled_processors.append(entry) @@ -934,7 +934,7 @@ class JobGenerator(object): self.job_spec_template.name = "globally specified job spec configuration" self.job_spec_template.id = "global" # Load defaults - for cfg_point in JobSpec.configuration.itervalues(): + for cfg_point in JobSpec.configuration.values(): cfg_point.set_value(self.job_spec_template, check_mandatory=False) self.root_node = SectionNode(self.job_spec_template) @@ -996,7 +996,7 @@ class JobGenerator(object): break else: continue - self.update_augmentations(job_spec.augmentations.values()) + self.update_augmentations(list(job_spec.augmentations.values())) specs.append(job_spec) return specs diff --git a/wa/framework/configuration/execution.py b/wa/framework/configuration/execution.py index d9192da7..13ed81af 100644 --- a/wa/framework/configuration/execution.py +++ b/wa/framework/configuration/execution.py @@ -1,5 +1,7 @@ import random -from itertools import izip_longest, groupby, chain +from itertools import groupby, chain + +from future.moves.itertools import zip_longest from wa.framework.configuration.core import (MetaConfiguration, RunConfiguration, JobGenerator, Status, settings) @@ -157,8 +159,8 @@ def permute_by_iteration(specs): all_tuples = [] for spec in chain(*groups): all_tuples.append([(spec, i + 1) - for i in xrange(spec.iterations)]) - for t in chain(*map(list, izip_longest(*all_tuples))): + for i in range(spec.iterations)]) + for t in chain(*list(map(list, zip_longest(*all_tuples)))): if t is not None: yield t @@ -183,8 +185,8 @@ def permute_by_section(specs): all_tuples = [] for spec in chain(*groups): all_tuples.append([(spec, i + 1) - for i in xrange(spec.iterations)]) - for t in chain(*map(list, izip_longest(*all_tuples))): + for i in range(spec.iterations)]) + for t in chain(*list(map(list, zip_longest(*all_tuples)))): if t is not None: yield t @@ -196,7 +198,7 @@ def permute_randomly(specs): """ result = [] for spec in specs: - for i in xrange(1, spec.iterations + 1): + for i in range(1, spec.iterations + 1): result.append((spec, i)) random.shuffle(result) for t in result: @@ -214,5 +216,5 @@ permute_map = { def permute_iterations(specs, exec_order): if exec_order not in permute_map: msg = 'Unknown execution order "{}"; must be in: {}' - raise ValueError(msg.format(exec_order, permute_map.keys())) + raise ValueError(msg.format(exec_order, list(permute_map.keys()))) return permute_map[exec_order](specs) diff --git a/wa/framework/configuration/parsers.py b/wa/framework/configuration/parsers.py index 48359d60..7fc6fa41 100644 --- a/wa/framework/configuration/parsers.py +++ b/wa/framework/configuration/parsers.py @@ -21,6 +21,7 @@ from wa.framework.exception import ConfigError from wa.utils import log from wa.utils.serializer import json, read_pod, SerializerSyntaxError from wa.utils.types import toggle_set, counter +from functools import reduce logger = logging.getLogger('config') @@ -47,27 +48,27 @@ class ConfigParser(object): merge_augmentations(raw) # Get WA core configuration - for cfg_point in state.settings.configuration.itervalues(): + for cfg_point in state.settings.configuration.values(): value = pop_aliased_param(cfg_point, raw) if value is not None: logger.debug('Setting meta "{}" to "{}"'.format(cfg_point.name, value)) state.settings.set(cfg_point.name, value) # Get run specific configuration - for cfg_point in state.run_config.configuration.itervalues(): + for cfg_point in state.run_config.configuration.values(): value = pop_aliased_param(cfg_point, raw) if value is not None: logger.debug('Setting run "{}" to "{}"'.format(cfg_point.name, value)) state.run_config.set(cfg_point.name, value) # Get global job spec configuration - for cfg_point in JobSpec.configuration.itervalues(): + for cfg_point in JobSpec.configuration.values(): value = pop_aliased_param(cfg_point, raw) if value is not None: logger.debug('Setting global "{}" to "{}"'.format(cfg_point.name, value)) state.jobs_config.set_global_value(cfg_point.name, value) - for name, values in raw.iteritems(): + for name, values in raw.items(): # Assume that all leftover config is for a plug-in or a global # alias it is up to PluginCache to assert this assumption logger.debug('Caching "{}" with "{}"'.format(name, values)) @@ -106,7 +107,7 @@ class AgendaParser(object): if raw: msg = 'Invalid top level agenda entry(ies): "{}"' - raise ConfigError(msg.format('", "'.join(raw.keys()))) + raise ConfigError(msg.format('", "'.join(list(raw.keys())))) sect_ids, wkl_ids = self._collect_ids(sections, global_workloads) self._process_global_workloads(state, global_workloads, wkl_ids) @@ -301,7 +302,7 @@ def _construct_valid_entry(raw, seen_ids, prefix, jobs_config): merge_augmentations(raw) # Validate all workload_entry - for name, cfg_point in JobSpec.configuration.iteritems(): + for name, cfg_point in JobSpec.configuration.items(): value = pop_aliased_param(cfg_point, raw) if value is not None: value = cfg_point.kind(value) @@ -317,7 +318,7 @@ def _construct_valid_entry(raw, seen_ids, prefix, jobs_config): # error if there are unknown workload_entry if raw: msg = 'Invalid entry(ies) in "{}": "{}"' - raise ConfigError(msg.format(workload_entry['id'], ', '.join(raw.keys()))) + raise ConfigError(msg.format(workload_entry['id'], ', '.join(list(raw.keys())))) return workload_entry @@ -339,7 +340,7 @@ def _collect_valid_id(entry_id, seen_ids, entry_type): def _get_workload_entry(workload): - if isinstance(workload, basestring): + if isinstance(workload, str): workload = {'name': workload} elif not isinstance(workload, dict): raise ConfigError('Invalid workload entry: "{}"') diff --git a/wa/framework/configuration/plugin_cache.py b/wa/framework/configuration/plugin_cache.py index 2549416f..c0f766bf 100644 --- a/wa/framework/configuration/plugin_cache.py +++ b/wa/framework/configuration/plugin_cache.py @@ -90,11 +90,11 @@ class PluginCache(object): msg = 'configuration provided for unknown plugin "{}"' raise ConfigError(msg.format(plugin_name)) - if not hasattr(values, 'iteritems'): + if not hasattr(values, 'items'): msg = 'Plugin configuration for "{}" not a dictionary ({} is {})' raise ConfigError(msg.format(plugin_name, repr(values), type(values))) - for name, value in values.iteritems(): + for name, value in values.items(): if (plugin_name not in GENERIC_CONFIGS and name not in self.get_plugin_parameters(plugin_name)): msg = "'{}' is not a valid parameter for '{}'" @@ -124,7 +124,7 @@ class PluginCache(object): for source in self.sources: if source not in plugin_config: continue - for name, value in plugin_config[source].iteritems(): + for name, value in plugin_config[source].items(): cfg_points[name].set_value(config, value=value) else: # A more complicated merge that involves priority of sources and @@ -136,7 +136,7 @@ class PluginCache(object): def get_plugin(self, name, kind=None, *args, **kwargs): config = self.get_plugin_config(name) - kwargs = dict(config.items() + kwargs.items()) + kwargs = dict(list(config.items()) + list(kwargs.items())) return self.loader.get_plugin(name, kind=kind, *args, **kwargs) def get_plugin_class(self, name, kind=None): @@ -154,18 +154,18 @@ class PluginCache(object): def _set_plugin_defaults(self, plugin_name, config): cfg_points = self.get_plugin_parameters(plugin_name) - for cfg_point in cfg_points.itervalues(): + for cfg_point in cfg_points.values(): cfg_point.set_value(config, check_mandatory=False) try: _, alias_params = self.resolve_alias(plugin_name) - for name, value in alias_params.iteritems(): + for name, value in alias_params.items(): cfg_points[name].set_value(config, value) except NotFoundError: pass def _set_from_global_aliases(self, plugin_name, config): - for alias, param in self._global_alias_map[plugin_name].iteritems(): + for alias, param in self._global_alias_map[plugin_name].items(): if alias in self.global_alias_values: for source in self.sources: if source not in self.global_alias_values[alias]: @@ -230,7 +230,7 @@ class PluginCache(object): # Validate final configuration merged_config.name = specific_name - for cfg_point in ms.cfg_points.itervalues(): + for cfg_point in ms.cfg_points.values(): cfg_point.validate(merged_config, check_mandatory=is_final) def __getattr__(self, name): @@ -285,7 +285,7 @@ class MergeState(object): def update_config_from_source(final_config, source, state): if source in state.generic_config: final_config.name = state.generic_name - for name, cfg_point in state.cfg_points.iteritems(): + for name, cfg_point in state.cfg_points.items(): if name in state.generic_config[source]: if name in state.seen_specific_config: msg = ('"{generic_name}" configuration "{config_name}" has ' @@ -307,7 +307,7 @@ def update_config_from_source(final_config, source, state): if source in state.specific_config: final_config.name = state.specific_name - for name, cfg_point in state.cfg_points.iteritems(): + for name, cfg_point in state.cfg_points.items(): if name in state.specific_config[source]: state.seen_specific_config[name].append(str(source)) value = state.specific_config[source].pop(name) diff --git a/wa/framework/configuration/tree.py b/wa/framework/configuration/tree.py index e41b94fc..4cf32e32 100644 --- a/wa/framework/configuration/tree.py +++ b/wa/framework/configuration/tree.py @@ -39,7 +39,7 @@ class JobSpecSource(object): def _log_self(self): logger.debug('Creating {} node'.format(self.kind)) with log.indentcontext(): - for key, value in self.config.iteritems(): + for key, value in self.config.items(): logger.debug('"{}" to "{}"'.format(key, value)) diff --git a/wa/framework/exception.py b/wa/framework/exception.py index 167b3185..bb6e839e 100644 --- a/wa/framework/exception.py +++ b/wa/framework/exception.py @@ -20,7 +20,11 @@ from wa.utils.misc import get_traceback class WAError(Exception): """Base class for all Workload Automation exceptions.""" - pass + @property + def message(self): + if self.args: + return self.args[0] + return '' class NotFoundError(WAError): diff --git a/wa/framework/execution.py b/wa/framework/execution.py index a80ed215..db0e1e81 100644 --- a/wa/framework/execution.py +++ b/wa/framework/execution.py @@ -464,7 +464,7 @@ class Runner(object): self.logger.info('Skipping remaining jobs.') self.context.skip_remaining_jobs() except Exception as e: - message = e.message if e.message else str(e) + message = e.args[0] if e.args else str(e) log.log_error(e, self.logger) self.logger.error('Skipping remaining jobs due to "{}".'.format(e)) self.context.skip_remaining_jobs() diff --git a/wa/framework/getters.py b/wa/framework/getters.py index 3af23546..d82b78aa 100644 --- a/wa/framework/getters.py +++ b/wa/framework/getters.py @@ -18,7 +18,7 @@ This module contains the standard set of resource getters used by Workload Automation. """ -import httplib +import http.client import json import logging import os @@ -233,13 +233,17 @@ class Http(ResourceGetter): return {} index_url = urljoin(self.url, 'index.json') response = self.geturl(index_url) - if response.status_code != httplib.OK: + if response.status_code != http.client.OK: message = 'Could not fetch "{}"; recieved "{} {}"' self.logger.error(message.format(index_url, response.status_code, response.reason)) return {} - return json.loads(response.content) + if sys.version_info[0] == 3: + content = response.content.decode('utf-8') + else: + content = response.content + return json.loads(content) def download_asset(self, asset, owner_name): url = urljoin(self.url, owner_name, asset['path']) @@ -252,7 +256,7 @@ class Http(ResourceGetter): return local_path self.logger.debug('Downloading {}'.format(url)) response = self.geturl(url, stream=True) - if response.status_code != httplib.OK: + if response.status_code != http.client.OK: message = 'Could not download asset "{}"; recieved "{} {}"' self.logger.warning(message.format(url, response.status_code, @@ -275,7 +279,7 @@ class Http(ResourceGetter): if not assets: return None asset_map = {a['path']: a for a in assets} - paths = get_path_matches(resource, asset_map.keys()) + paths = get_path_matches(resource, list(asset_map.keys())) local_paths = [] for path in paths: local_paths.append(self.download_asset(asset_map[path], @@ -292,7 +296,7 @@ class Http(ResourceGetter): asset_map = {a['path']: a for a in assets} if resource.kind in ['jar', 'revent']: - path = get_generic_resource(resource, asset_map.keys()) + path = get_generic_resource(resource, list(asset_map.keys())) if path: return asset_map[path] elif resource.kind == 'executable': diff --git a/wa/framework/host.py b/wa/framework/host.py index e43761b0..32e34732 100644 --- a/wa/framework/host.py +++ b/wa/framework/host.py @@ -90,7 +90,7 @@ def convert_wa2_agenda(filepath, output_path): default=True), ]) - for param in orig_agenda.keys(): + for param in list(orig_agenda.keys()): for cfg_point in config_points: if param == cfg_point.name or param in cfg_point.aliases: if cfg_point.name == 'augmentations': @@ -105,7 +105,7 @@ def convert_wa2_agenda(filepath, output_path): # Convert plugin configuration output.write("# Plugin Configuration\n") - for param in orig_agenda.keys(): + for param in list(orig_agenda.keys()): if pluginloader.has_plugin(param): entry = {param: orig_agenda.pop(param)} yaml.dump(format_parameter(entry), output, default_flow_style=False) @@ -114,7 +114,7 @@ def convert_wa2_agenda(filepath, output_path): # Write any additional aliased parameters into new config plugin_cache = PluginCache() output.write("# Additional global aliases\n") - for param in orig_agenda.keys(): + for param in list(orig_agenda.keys()): if plugin_cache.is_global_alias(param): entry = {param: orig_agenda.pop(param)} yaml.dump(format_parameter(entry), output, default_flow_style=False) @@ -123,7 +123,7 @@ def convert_wa2_agenda(filepath, output_path): def format_parameter(param): if isinstance(param, dict): - return {identifier(k) : v for k, v in param.iteritems()} + return {identifier(k) : v for k, v in param.items()} else: return param diff --git a/wa/framework/instrument.py b/wa/framework/instrument.py index d7691fc4..47748847 100644 --- a/wa/framework/instrument.py +++ b/wa/framework/instrument.py @@ -165,7 +165,7 @@ def priority(priority): def decorate(func): def wrapper(*args, **kwargs): return func(*args, **kwargs) - wrapper.func_name = func.func_name + wrapper.__name__ = func.__name__ if priority in signal.CallbackPriority.levels: wrapper.priority = signal.CallbackPriority(priority) else: @@ -255,7 +255,7 @@ class ManagedCallback(object): global failures_detected # pylint: disable=W0603 failures_detected = True log_error(e, logger) - context.add_event(e.message) + context.add_event(e.args[0] if e.args else str(e)) if isinstance(e, WorkloadError): context.set_status('FAILED') elif isinstance(e, TargetError) or isinstance(e, TimeoutError): @@ -268,7 +268,7 @@ class ManagedCallback(object): def __repr__(self): text = 'ManagedCallback({}, {})' - return text.format(self.instrument.name, self.callback.im_func.func_name) + return text.format(self.instrument.name, self.callback.__func__.__name__) __str__ = __repr__ diff --git a/wa/framework/job.py b/wa/framework/job.py index 0280e059..2be52fc9 100644 --- a/wa/framework/job.py +++ b/wa/framework/job.py @@ -85,7 +85,7 @@ class Job(object): enabled_instruments = set(i.name for i in instrument.get_enabled()) enabled_output_processors = set(p.name for p in pm.get_enabled()) - for augmentation in self.spec.augmentations.values(): + for augmentation in list(self.spec.augmentations.values()): augmentation_cls = context.cm.plugin_cache.get_plugin_class(augmentation) if augmentation_cls.kind == 'instrument': instruments_to_enable.add(augmentation) diff --git a/wa/framework/output.py b/wa/framework/output.py index d15fbb09..31d227fa 100644 --- a/wa/framework/output.py +++ b/wa/framework/output.py @@ -10,7 +10,7 @@ from wa.framework.configuration.execution import CombinedConfig from wa.framework.exception import HostError from wa.framework.run import RunState, RunInfo from wa.framework.target.info import TargetInfo -from wa.utils.misc import touch, ensure_directory_exists +from wa.utils.misc import touch, ensure_directory_exists, isiterable from wa.utils.serializer import write_pod, read_pod, is_pod from wa.utils.types import enum, numeric @@ -229,7 +229,7 @@ class RunOutput(Output): if os.path.isfile(self.jobsfile): self.job_specs = self.read_job_specs() - for job_state in self.state.jobs.itervalues(): + for job_state in self.state.jobs.values(): job_path = os.path.join(self.basepath, job_state.output_name) job = JobOutput(job_path, job_state.id, job_state.label, job_state.iteration, @@ -387,14 +387,14 @@ class Result(object): if key not in self.metadata: return self.add_metadata(key, *args) - if hasattr(self.metadata[key], 'iteritems'): + if hasattr(self.metadata[key], 'items'): if len(args) == 2: self.metadata[key][args[0]] = args[1] elif len(args) > 2: # assume list of key-value pairs for k, v in args: self.metadata[key][k] = v - elif hasattr(args[0], 'iteritems'): - for k, v in args[0].iteritems(): + elif hasattr(args[0], 'items'): + for k, v in args[0].items(): self.metadata[key][k] = v else: raise ValueError('Invalid value for key "{}": {}'.format(key, args)) diff --git a/wa/framework/plugin.py b/wa/framework/plugin.py index 52a019b4..904de198 100644 --- a/wa/framework/plugin.py +++ b/wa/framework/plugin.py @@ -25,6 +25,8 @@ from collections import OrderedDict, defaultdict from itertools import chain from copy import copy +from future.utils import with_metaclass + from wa.framework.configuration.core import settings, ConfigurationPoint as Parameter from wa.framework.exception import (NotFoundError, PluginLoaderError, TargetError, ValidationError, ConfigError, HostError) @@ -34,7 +36,10 @@ from wa.utils.misc import (ensure_directory_exists as _d, walk_modules, load_cla from wa.utils.types import identifier -MODNAME_TRANS = string.maketrans(':/\\.', '____') +if sys.version_info[0] == 3: + MODNAME_TRANS = str.maketrans(':/\\.', '____') +else: + MODNAME_TRANS = string.maketrans(':/\\.', '____') class AttributeCollection(object): @@ -50,7 +55,7 @@ class AttributeCollection(object): @property def values(self): - return self._attrs.values() + return list(self._attrs.values()) def __init__(self, attrcls): self._attrcls = attrcls @@ -61,7 +66,7 @@ class AttributeCollection(object): if p.name in self._attrs: if p.override: newp = copy(self._attrs[p.name]) - for a, v in p.__dict__.iteritems(): + for a, v in p.__dict__.items(): if v is not None: setattr(newp, a, v) if not hasattr(newp, "_overridden"): @@ -77,7 +82,7 @@ class AttributeCollection(object): append = add def __str__(self): - return 'AC({})'.format(map(str, self._attrs.values())) + return 'AC({})'.format(list(map(str, list(self._attrs.values())))) __repr__ = __str__ @@ -212,14 +217,14 @@ class PluginMeta(type): if hasattr(cls, 'aliases'): aliases, cls.aliases = cls.aliases, AliasCollection() for alias in aliases: - if isinstance(alias, basestring): + if isinstance(alias, str): alias = Alias(alias) alias.validate(cls) alias.plugin_name = cls.name cls.aliases.add(alias) -class Plugin(object): +class Plugin(with_metaclass(PluginMeta, object)): """ Base class for all WA plugins. An plugin is basically a plug-in. It extends the functionality of WA in some way. Plugins are discovered and @@ -230,7 +235,6 @@ class Plugin(object): ``~/.workload_automation/``. """ - __metaclass__ = PluginMeta kind = None name = None @@ -334,7 +338,7 @@ class Plugin(object): can = has def _load_module(self, loader, module_spec): - if isinstance(module_spec, basestring): + if isinstance(module_spec, str): name = module_spec params = {} elif isinstance(module_spec, dict): @@ -342,7 +346,7 @@ class Plugin(object): msg = 'Invalid module spec: {}; dict must have exctly one key -- '\ 'the module name.' raise ValueError(msg.format(module_spec)) - name, params = module_spec.items()[0] + name, params = list(module_spec.items())[0] else: message = 'Invalid module spec: {}; must be a string or a one-key dict.' raise ValueError(message.format(module_spec)) @@ -491,7 +495,7 @@ class PluginLoader(object): """ name, base_kwargs = self.resolve_alias(name) - kwargs = OrderedDict(chain(base_kwargs.iteritems(), kwargs.iteritems())) + kwargs = OrderedDict(chain(iter(base_kwargs.items()), iter(kwargs.items()))) cls = self.get_plugin_class(name, kind) plugin = cls(*args, **kwargs) return plugin @@ -514,10 +518,10 @@ class PluginLoader(object): """ if kind is None: - return self.plugins.values() + return list(self.plugins.values()) if kind not in self.kind_map: raise ValueError('Unknown plugin type: {}'.format(kind)) - return self.kind_map[kind].values() + return list(self.kind_map[kind].values()) def has_plugin(self, name, kind=None): """ @@ -625,7 +629,7 @@ class PluginLoader(object): modname = os.path.splitext(filepath[1:])[0].translate(MODNAME_TRANS) module = imp.load_source(modname, filepath) self._discover_in_module(module) - except (SystemExit, ImportError), e: + except (SystemExit, ImportError) as e: if self.keep_going: self.logger.warning('Failed to load {}'.format(filepath)) self.logger.warning('Got: {}'.format(e)) @@ -639,7 +643,7 @@ class PluginLoader(object): def _discover_in_module(self, module): # NOQA pylint: disable=too-many-branches self.logger.debug('Checking module %s', module.__name__) with log.indentcontext(): - for obj in vars(module).itervalues(): + for obj in vars(module).values(): if inspect.isclass(obj): if not issubclass(obj, Plugin): continue diff --git a/wa/framework/pluginloader.py b/wa/framework/pluginloader.py index 548f42d9..c702a2d0 100644 --- a/wa/framework/pluginloader.py +++ b/wa/framework/pluginloader.py @@ -21,7 +21,7 @@ class __LoaderWrapper(object): def kinds(self): if not self._loader: self.reset() - return self._loader.kind_map.keys() + return list(self._loader.kind_map.keys()) @property def kind_map(self): diff --git a/wa/framework/resource.py b/wa/framework/resource.py index f677098a..98c1f285 100644 --- a/wa/framework/resource.py +++ b/wa/framework/resource.py @@ -287,7 +287,7 @@ def loose_version_matching(config_version, apk_version): if len(apk_version) < len(config_version): return False # More specific version requested than available - for i in xrange(len(config_version)): + for i in range(len(config_version)): if config_version[i] != apk_version[i]: return False return True diff --git a/wa/framework/run.py b/wa/framework/run.py index 39f8498b..579c9973 100644 --- a/wa/framework/run.py +++ b/wa/framework/run.py @@ -76,7 +76,7 @@ class RunState(object): @property def num_completed_jobs(self): - return sum(1 for js in self.jobs.itervalues() + return sum(1 for js in self.jobs.values() if js.status > Status.RUNNING) def __init__(self): @@ -95,7 +95,7 @@ class RunState(object): def get_status_counts(self): counter = Counter() - for job_state in self.jobs.itervalues(): + for job_state in self.jobs.values(): counter[job_state.status] += 1 return counter @@ -103,7 +103,7 @@ class RunState(object): return OrderedDict( status=str(self.status), timestamp=self.timestamp, - jobs=[j.to_pod() for j in self.jobs.itervalues()], + jobs=[j.to_pod() for j in self.jobs.values()], ) diff --git a/wa/framework/target/descriptor.py b/wa/framework/target/descriptor.py index d0c96ae4..418b17ab 100644 --- a/wa/framework/target/descriptor.py +++ b/wa/framework/target/descriptor.py @@ -28,7 +28,7 @@ def list_target_descriptions(loader=pluginloader): raise PluginLoaderError(msg.format(desc.name, prev_dtor.name, descriptor.name)) targets[desc.name] = desc - return targets.values() + return list(targets.values()) def get_target_description(name, loader=pluginloader): @@ -47,11 +47,11 @@ def instantiate_target(tdesc, params, connect=None, extra_platform_params=None): tp, pp, cp = {}, {}, {} for supported_params, new_params in (target_params, tp), (platform_params, pp), (conn_params, cp): - for name, value in supported_params.iteritems(): + for name, value in supported_params.items(): if value.default and name == value.name: new_params[name] = value.default - for name, value in params.iteritems(): + for name, value in params.items(): if name in target_params: tp[name] = value elif name in platform_params: @@ -64,7 +64,7 @@ def instantiate_target(tdesc, params, connect=None, extra_platform_params=None): msg = 'Unexpected parameter for {}: {}' raise ValueError(msg.format(tdesc.name, name)) - for pname, pval in (extra_platform_params or {}).iteritems(): + for pname, pval in (extra_platform_params or {}).items(): if pname in pp: raise RuntimeError('Platform parameter clash: {}'.format(pname)) pp[pname] = pval @@ -121,7 +121,7 @@ class TargetDescription(object): vals = [] elif isiterable(vals): if hasattr(vals, 'values'): - vals = v.values() + vals = list(v.values()) else: msg = '{} must be iterable; got "{}"' raise ValueError(msg.format(attr, vals)) @@ -453,11 +453,11 @@ class DefaultTargetDescriptor(TargetDescriptor): def get_descriptions(self): result = [] - for target_name, target_tuple in TARGETS.iteritems(): + for target_name, target_tuple in TARGETS.items(): (target, conn), target_params = self._get_item(target_tuple) assistant = ASSISTANTS[target_name] conn_params = CONNECTION_PARAMS[conn] - for platform_name, platform_tuple in PLATFORMS.iteritems(): + for platform_name, platform_tuple in PLATFORMS.items(): (platform, plat_conn), platform_params = self._get_item(platform_tuple) name = '{}_{}'.format(platform_name, target_name) td = TargetDescription(name, self) @@ -484,11 +484,11 @@ class DefaultTargetDescriptor(TargetDescriptor): return cls, params param_map = OrderedDict((p.name, copy(p)) for p in params) - for name, value in defaults.iteritems(): + for name, value in defaults.items(): if name not in param_map: raise ValueError('Unexpected default "{}"'.format(name)) param_map[name].default = value - return cls, param_map.values() + return cls, list(param_map.values()) @@ -522,7 +522,7 @@ def create_target_description(name, *args, **kwargs): def _get_target_defaults(target): specificity = 0 res = ('linux', TARGETS['linux']) # fallback to a generic linux target - for name, ttup in TARGETS.iteritems(): + for name, ttup in TARGETS.items(): if issubclass(target, ttup[0][0]): new_spec = len(inspect.getmro(ttup[0][0])) if new_spec > specificity: @@ -540,7 +540,7 @@ def add_description_for_target(target, description=None, **kwargs): if 'platform' not in kwargs: kwargs['platform'] = Platform if 'platform_params' not in kwargs: - for (plat, conn), params, _ in PLATFORMS.itervalues(): + for (plat, conn), params, _ in PLATFORMS.values(): if plat == kwargs['platform']: kwargs['platform_params'] = params if conn is not None and kwargs['conn'] is None: diff --git a/wa/framework/target/info.py b/wa/framework/target/info.py index 8b1bfb32..a96af240 100644 --- a/wa/framework/target/info.py +++ b/wa/framework/target/info.py @@ -10,7 +10,7 @@ def cpuinfo_from_pod(pod): cpuinfo.sections = pod['cpuinfo'] lines = [] for section in cpuinfo.sections: - for key, value in section.iteritems(): + for key, value in section.items(): line = '{}: {}'.format(key, value) lines.append(line) lines.append('') @@ -35,7 +35,7 @@ def kernel_config_from_pod(pod): config = KernelConfig('') config._config = pod['kernel_config'] lines = [] - for key, value in config._config.iteritems(): + for key, value in config._config.items(): if value == 'n': lines.append('# {} is not set'.format(key)) else: diff --git a/wa/framework/target/runtime_config.py b/wa/framework/target/runtime_config.py index 7d2b2f84..4353eb7b 100644 --- a/wa/framework/target/runtime_config.py +++ b/wa/framework/target/runtime_config.py @@ -33,7 +33,7 @@ class RuntimeConfig(Plugin): @property def supported_parameters(self): - return self._runtime_params.values() + return list(self._runtime_params.values()) @property def core_names(self): @@ -166,12 +166,12 @@ class HotplugRuntimeConfig(RuntimeConfig): def validate_parameters(self): if len(self.num_cores) == self.target.number_of_cpus: - if all(v is False for v in self.num_cores.values()): + if all(v is False for v in list(self.num_cores.values())): raise ValueError('Cannot set number of all cores to 0') def commit(self): '''Online all CPUs required in order before then off-lining''' - num_cores = sorted(self.num_cores.iteritems()) + num_cores = sorted(self.num_cores.items()) for cpu, online in num_cores: if online: self.target.hotplug.online(cpu) @@ -190,7 +190,7 @@ class SysfileValuesRuntimeConfig(RuntimeConfig): #pylint: disable=unused-argument @staticmethod def set_sysfile(obj, value, core): - for path, value in value.iteritems(): + for path, value in value.items(): verify = True if path.endswith('!'): verify = False @@ -222,7 +222,7 @@ class SysfileValuesRuntimeConfig(RuntimeConfig): return def commit(self): - for path, (value, verify) in self.sysfile_values.iteritems(): + for path, (value, verify) in self.sysfile_values.items(): self.target.write_value(path, value, verify=verify) def clear(self): @@ -255,7 +255,7 @@ class FreqValue(object): raise TargetError(msg.format(value)) elif isinstance(value, int) and value in self.values: return value - elif isinstance(value, basestring): + elif isinstance(value, str): value = caseless_string(value) if value in ['min', 'max']: return value @@ -675,7 +675,7 @@ class IdleStateValue(object): if self.values is None: return value - if isinstance(value, basestring): + if isinstance(value, str): value = caseless_string(value) if value == 'all': return [state[0] for state in self.values] diff --git a/wa/framework/target/runtime_parameter_manager.py b/wa/framework/target/runtime_parameter_manager.py index 534e17f8..c2340ace 100644 --- a/wa/framework/target/runtime_parameter_manager.py +++ b/wa/framework/target/runtime_parameter_manager.py @@ -39,7 +39,7 @@ class RuntimeParameterManager(object): def merge_runtime_parameters(self, parameters): merged_params = obj_dict() for source in parameters: - for name, value in parameters[source].iteritems(): + for name, value in parameters[source].items(): cp = self.get_cfg_point(name) cp.set_value(merged_params, value) return dict(merged_params) @@ -60,7 +60,7 @@ class RuntimeParameterManager(object): # Stores a set of parameters performing isolated validation when appropriate def set_runtime_parameters(self, parameters): - for name, value in parameters.iteritems(): + for name, value in parameters.items(): cfg = self.get_config_for_name(name) if cfg is None: msg = 'Unsupported runtime parameter: "{}"' @@ -74,14 +74,14 @@ class RuntimeParameterManager(object): def get_config_for_name(self, name): name = caseless_string(name) - for k, v in self.runtime_params.iteritems(): + for k, v in self.runtime_params.items(): if name == k: return v.rt_config return None def get_cfg_point(self, name): name = caseless_string(name) - for k, v in self.runtime_params.iteritems(): + for k, v in self.runtime_params.items(): if name == k: return v.cfg_point raise ConfigError('Unknown runtime parameter: {}'.format(name)) diff --git a/wa/framework/version.py b/wa/framework/version.py index 1ccbf3ba..d0963736 100644 --- a/wa/framework/version.py +++ b/wa/framework/version.py @@ -14,6 +14,7 @@ # import os +import sys from collections import namedtuple from subprocess import Popen, PIPE @@ -45,4 +46,7 @@ def get_commit(): p.wait() if p.returncode: return None - return std[:8] + if sys.version_info[0] == 3: + return std[:8].decode(sys.stdout.encoding) + else: + return std[:8] diff --git a/wa/instruments/dmesg.py b/wa/instruments/dmesg.py index 48036a83..cae48180 100644 --- a/wa/instruments/dmesg.py +++ b/wa/instruments/dmesg.py @@ -31,7 +31,7 @@ class DmesgInstrument(Instrument): name = 'dmesg' parameters = [ - Parameter('loglevel', kind=int, allowed_values=range(8), + Parameter('loglevel', kind=int, allowed_values=list(range(8)), description='Set loglevel for console output.') ] diff --git a/wa/instruments/energy_measurement.py b/wa/instruments/energy_measurement.py index 6d9b50d1..f46f62a3 100644 --- a/wa/instruments/energy_measurement.py +++ b/wa/instruments/energy_measurement.py @@ -15,7 +15,7 @@ # pylint: disable=W0613,E1101 -from __future__ import division + from collections import defaultdict import os import shutil @@ -413,9 +413,9 @@ class EnergyMeasurement(Instrument): self.params = obj_dict() instrument_parameters = {identifier(k): v - for k, v in self.instrument_parameters.iteritems()} + for k, v in self.instrument_parameters.items()} supported_params = self.backend.get_parameters() - for name, param in supported_params.iteritems(): + for name, param in supported_params.items(): value = instrument_parameters.pop(name, None) param.set_value(self.params, value) if instrument_parameters: @@ -426,7 +426,7 @@ class EnergyMeasurement(Instrument): def initialize(self, context): self.instruments = self.backend.get_instruments(self.target, context.run_output.metadir, **self.params) - for instrument in self.instruments.itervalues(): + for instrument in self.instruments.values(): if not (instrument.mode & CONTINUOUS): msg = '{} instrument does not support continuous measurement collection' raise ConfigError(msg.format(self.instrument)) @@ -436,26 +436,26 @@ class EnergyMeasurement(Instrument): # Check that the expeccted channels exist. # If there are multiple Instruments, they were all constructed with # the same channels param, so check them all. - for instrument in self.instruments.itervalues(): + for instrument in self.instruments.values(): if not instrument.get_channels(channel): raise ConfigError('No channels found for "{}"'.format(channel)) def setup(self, context): - for instrument in self.instruments.itervalues(): + for instrument in self.instruments.values(): instrument.reset(sites=self.sites, kinds=self.kinds, channels=self.channels) def start(self, context): - for instrument in self.instruments.itervalues(): + for instrument in self.instruments.values(): instrument.start() def stop(self, context): - for instrument in self.instruments.itervalues(): + for instrument in self.instruments.values(): instrument.stop() def update_output(self, context): - for device, instrument in self.instruments.iteritems(): + for device, instrument in self.instruments.items(): # Append the device key to the filename and artifact name, unless # it's None (as it will be for backends with only 1 # devce/instrument) @@ -501,7 +501,7 @@ class EnergyMeasurement(Instrument): # the devlib instrument, before we potentially appended a device key to # it) if len(self.instruments) > 1: - for name, metrics in metrics_by_name.iteritems(): + for name, metrics in metrics_by_name.items(): units = metrics[0].units value = sum(m.value for m in metrics) context.add_metric(name, value, units) diff --git a/wa/instruments/hwmon.py b/wa/instruments/hwmon.py index 8c6caebe..aa838543 100644 --- a/wa/instruments/hwmon.py +++ b/wa/instruments/hwmon.py @@ -58,11 +58,11 @@ class HwmonInstrument(Instrument): measurements_before = {m.channel.label: m for m in self.before} measurements_after = {m.channel.label: m for m in self.after} - if measurements_before.keys() != measurements_after.keys(): + if list(measurements_before.keys()) != list(measurements_after.keys()): self.logger.warning( 'hwmon before/after measurements returned different entries!') - for label, measurement_after in measurements_after.iteritems(): + for label, measurement_after in measurements_after.items(): if label not in measurements_before: continue # We've already warned about this measurement_before = measurements_before[label] diff --git a/wa/instruments/misc.py b/wa/instruments/misc.py index f9eae9c5..0610717e 100644 --- a/wa/instruments/misc.py +++ b/wa/instruments/misc.py @@ -30,16 +30,17 @@ import re import logging import time import tarfile -from itertools import izip, izip_longest from subprocess import CalledProcessError -from devlib.exception import TargetError +from future.moves.itertools import zip_longest +from devlib.exception import TargetError from devlib.utils.android import ApkInfo from wa import Instrument, Parameter, very_fast from wa.framework.exception import ConfigError from wa.framework.instrument import slow +from wa.utils.diff import diff_sysfs_dirs, diff_interrupt_files from wa.utils.misc import as_relative, diff_tokens, write_table from wa.utils.misc import ensure_file_directory_exists as _f from wa.utils.misc import ensure_directory_exists as _d @@ -112,7 +113,7 @@ class SysfsExtractor(Instrument): _d(os.path.join(context.output_directory, 'diff', self._local_dir(d))) for d in self.paths ] - self.device_and_host_paths = zip(self.paths, before_dirs, after_dirs, diff_dirs) + self.device_and_host_paths = list(zip(self.paths, before_dirs, after_dirs, diff_dirs)) if self.use_tmpfs: for d in self.paths: @@ -177,7 +178,7 @@ class SysfsExtractor(Instrument): self.logger.error('sysfs files were not pulled from the device.') self.device_and_host_paths.remove(paths) # Path is removed to skip diffing it for _, before_dir, after_dir, diff_dir in self.device_and_host_paths: - _diff_sysfs_dirs(before_dir, after_dir, diff_dir) + diff_sysfs_dirs(before_dir, after_dir, diff_dir) def teardown(self, context): self._one_time_setup_done = [] @@ -280,7 +281,7 @@ class InterruptStatsInstrument(Instrument): def update_output(self, context): # If workload execution failed, the after_file may not have been created. if os.path.isfile(self.after_file): - _diff_interrupt_files(self.before_file, self.after_file, _f(self.diff_file)) + diff_interrupt_files(self.before_file, self.after_file, _f(self.diff_file)) class DynamicFrequencyInstrument(SysfsExtractor): @@ -307,83 +308,3 @@ class DynamicFrequencyInstrument(SysfsExtractor): super(DynamicFrequencyInstrument, self).validate() if not self.tmpfs_mount_point.endswith('-cpufreq'): # pylint: disable=access-member-before-definition self.tmpfs_mount_point += '-cpufreq' - - -def _diff_interrupt_files(before, after, result): # pylint: disable=R0914 - output_lines = [] - with open(before) as bfh: - with open(after) as ofh: - for bline, aline in izip(bfh, ofh): - bchunks = bline.strip().split() - while True: - achunks = aline.strip().split() - if achunks[0] == bchunks[0]: - diffchunks = [''] - diffchunks.append(achunks[0]) - diffchunks.extend([diff_tokens(b, a) for b, a - in zip(bchunks[1:], achunks[1:])]) - output_lines.append(diffchunks) - break - else: # new category appeared in the after file - diffchunks = ['>'] + achunks - output_lines.append(diffchunks) - try: - aline = ofh.next() - except StopIteration: - break - - # Offset heading columns by one to allow for row labels on subsequent - # lines. - output_lines[0].insert(0, '') - - # Any "columns" that do not have headings in the first row are not actually - # columns -- they are a single column where space-spearated words got - # split. Merge them back together to prevent them from being - # column-aligned by write_table. - table_rows = [output_lines[0]] - num_cols = len(output_lines[0]) - for row in output_lines[1:]: - table_row = row[:num_cols] - table_row.append(' '.join(row[num_cols:])) - table_rows.append(table_row) - - with open(result, 'w') as wfh: - write_table(table_rows, wfh) - - -def _diff_sysfs_dirs(before, after, result): # pylint: disable=R0914 - before_files = [] - os.path.walk(before, - lambda arg, dirname, names: arg.extend([os.path.join(dirname, f) for f in names]), - before_files - ) - before_files = filter(os.path.isfile, before_files) - files = [os.path.relpath(f, before) for f in before_files] - after_files = [os.path.join(after, f) for f in files] - diff_files = [os.path.join(result, f) for f in files] - - for bfile, afile, dfile in zip(before_files, after_files, diff_files): - if not os.path.isfile(afile): - logger.debug('sysfs_diff: {} does not exist or is not a file'.format(afile)) - continue - - with open(bfile) as bfh, open(afile) as afh: # pylint: disable=C0321 - with open(_f(dfile), 'w') as dfh: - for i, (bline, aline) in enumerate(izip_longest(bfh, afh), 1): - if aline is None: - logger.debug('Lines missing from {}'.format(afile)) - break - bchunks = re.split(r'(\W+)', bline) - achunks = re.split(r'(\W+)', aline) - if len(bchunks) != len(achunks): - logger.debug('Token length mismatch in {} on line {}'.format(bfile, i)) - dfh.write('xxx ' + bline) - continue - if ((len([c for c in bchunks if c.strip()]) == len([c for c in achunks if c.strip()]) == 2) and - (bchunks[0] == achunks[0])): - # if there are only two columns and the first column is the - # same, assume it's a "header" column and do not diff it. - dchunks = [bchunks[0]] + [diff_tokens(b, a) for b, a in zip(bchunks[1:], achunks[1:])] - else: - dchunks = [diff_tokens(b, a) for b, a in zip(bchunks, achunks)] - dfh.write(''.join(dchunks)) diff --git a/wa/instruments/trace_cmd.py b/wa/instruments/trace_cmd.py index d008e216..6ecd1a37 100644 --- a/wa/instruments/trace_cmd.py +++ b/wa/instruments/trace_cmd.py @@ -15,7 +15,7 @@ # pylint: disable=W0613,E1101 -from __future__ import division + import os from devlib import FtraceCollector diff --git a/wa/output_processors/cpustates.py b/wa/output_processors/cpustates.py index 938c2993..ecab9b80 100755 --- a/wa/output_processors/cpustates.py +++ b/wa/output_processors/cpustates.py @@ -14,9 +14,10 @@ # import os -import csv from collections import OrderedDict +from devlib.utils.csvutil import csvwriter + from wa import OutputProcessor, Parameter from wa.utils.types import list_of_strings from wa.utils.cpustates import report_power_stats @@ -29,14 +30,14 @@ def _get_cpustates_description(): """ output_lines = [] lines = iter(report_power_stats.__doc__.split('\n')) - line = lines.next() + line = next(lines) while True: try: if line.strip().startswith(':param'): while line.strip(): - line = lines.next() + line = next(lines) output_lines.append(line) - line = lines.next() + line = next(lines) except StopIteration: break return '\n'.join(output_lines) @@ -105,7 +106,7 @@ class CpuStatesProcessor(OutputProcessor): split_wfi_states=self.split_wfi_states, ) - for report in reports.itervalues(): + for report in reports.values(): output.add_artifact(report.name, report.filepath, kind='data') iteration_id = (output.id, output.label, output.iteration) @@ -118,7 +119,7 @@ class CpuStatesProcessor(OutputProcessor): parallel_rows = [] powerstate_rows = [] - for iteration_id, reports in self.iteration_reports.iteritems(): + for iteration_id, reports in self.iteration_reports.items(): job_id, workload, iteration = iteration_id parallel_report = reports['parallel-stats'] powerstate_report = reports['power-state-stats'] @@ -132,8 +133,7 @@ class CpuStatesProcessor(OutputProcessor): for s in stats]) outpath = output.get_path('parallel-stats.csv') - with open(outpath, 'w') as wfh: - writer = csv.writer(wfh) + with csvwriter(outpath) as writer: writer.writerow(['id', 'workload', 'iteration', 'cluster', 'number_of_cores', 'total_time', '%time', '%running_time']) @@ -141,8 +141,7 @@ class CpuStatesProcessor(OutputProcessor): output.add_artifact('run-parallel-stats', outpath, kind='export') outpath = output.get_path('power-state-stats.csv') - with open(outpath, 'w') as wfh: - writer = csv.writer(wfh) + with csvwriter(outpath) as writer: headers = ['id', 'workload', 'iteration', 'state'] headers += ['{} CPU{}'.format(c, i) for i, c in enumerate(powerstate_report.core_names)] diff --git a/wa/output_processors/csvproc.py b/wa/output_processors/csvproc.py index 9a90006e..8d7f6887 100644 --- a/wa/output_processors/csvproc.py +++ b/wa/output_processors/csvproc.py @@ -1,4 +1,6 @@ -import csv +import sys + +from devlib.utils.csvutil import csvwriter from wa import OutputProcessor, Parameter from wa.framework.exception import ConfigError @@ -64,7 +66,7 @@ class CsvReportProcessor(OutputProcessor): classifiers = set([]) for out in outputs: for metric in out.metrics: - classifiers.update(metric.classifiers.keys()) + classifiers.update(list(metric.classifiers.keys())) extra_columns = list(classifiers) elif self.extra_columns: extra_columns = self.extra_columns @@ -72,8 +74,7 @@ class CsvReportProcessor(OutputProcessor): extra_columns = [] outfile = output.get_path('results.csv') - with open(outfile, 'wb') as wfh: - writer = csv.writer(wfh) + with csvwriter(outfile) as writer: writer.writerow(['id', 'workload', 'iteration', 'metric', ] + extra_columns + ['value', 'units']) diff --git a/wa/output_processors/status.py b/wa/output_processors/status.py index f284f86b..f844255a 100644 --- a/wa/output_processors/status.py +++ b/wa/output_processors/status.py @@ -49,8 +49,8 @@ class StatusTxtReporter(OutputProcessor): txt = '{}/{} iterations completed without error\n' wfh.write(txt.format(counter[Status.OK], len(output.jobs))) wfh.write('\n') - status_lines = [map(str, [o.id, o.label, o.iteration, o.status, - o.event_summary]) + status_lines = [list(map(str, [o.id, o.label, o.iteration, o.status, + o.event_summary])) for o in output.jobs] write_table(status_lines, wfh, align='<<>><') diff --git a/wa/utils/cpustates.py b/wa/utils/cpustates.py index 494b747e..c2557cd4 100755 --- a/wa/utils/cpustates.py +++ b/wa/utils/cpustates.py @@ -13,16 +13,17 @@ # limitations under the License. # -from __future__ import division + import os import sys -import csv import re import logging from ctypes import c_int32 from collections import defaultdict import argparse +from devlib.utils.csvutil import create_writer, csvwriter + from wa.utils.trace_cmd import TraceCmdParser, trace_has_marker, TRACE_MARKER_START, TRACE_MARKER_STOP @@ -114,7 +115,7 @@ class SystemPowerState(object): self.timestamp = None self.cpus = [] idle_state = -1 if no_idle else None - for _ in xrange(num_cores): + for _ in range(num_cores): self.cpus.append(CpuPowerState(idle_state=idle_state)) def copy(self): @@ -331,8 +332,7 @@ class PowerStateTransitions(object): def __init__(self, output_directory): self.filepath = os.path.join(output_directory, 'state-transitions-timeline.csv') - self._wfh = open(self.filepath, 'w') - self.writer = csv.writer(self._wfh) + self.writer, self._wfh = create_writer(self.filepath) headers = ['timestamp', 'cpu_id', 'frequency', 'idle_state'] self.writer.writerow(headers) @@ -360,8 +360,7 @@ class PowerStateTimeline(object): def __init__(self, output_directory, cpus): self.filepath = os.path.join(output_directory, 'power-state-timeline.csv') self.idle_state_names = {cpu.id: [s.name for s in cpu.cpuidle.states] for cpu in cpus} - self._wfh = open(self.filepath, 'w') - self.writer = csv.writer(self._wfh) + self.writer, self._wfh = create_writer(self.filepath) headers = ['ts'] + ['{} CPU{}'.format(cpu.name, cpu.id) for cpu in cpus] self.writer.writerow(headers) @@ -405,7 +404,7 @@ class ParallelStats(object): clusters.append(cpu.cpufreq.related_cpus) for i, clust in enumerate(clusters): - self.clusters[i] = set(clust) + self.clusters[str(i)] = set(clust) self.clusters['all'] = set([cpu.id for cpu in cpus]) self.first_timestamp = None @@ -419,7 +418,7 @@ class ParallelStats(object): delta = timestamp - self.last_timestamp active_cores = [i for i, c in enumerate(self.previous_states) if c and c[0] == -1] - for cluster, cluster_cores in self.clusters.iteritems(): + for cluster, cluster_cores in self.clusters.items(): clust_active_cores = len(cluster_cores.intersection(active_cores)) self.parallel_times[cluster][clust_active_cores] += delta if clust_active_cores: @@ -438,7 +437,7 @@ class ParallelStats(object): total_time = self.last_timestamp - self.first_timestamp for cluster in sorted(self.parallel_times): running_time = self.running_times[cluster] - for n in xrange(len(self.clusters[cluster]) + 1): + for n in range(len(self.clusters[cluster]) + 1): time = self.parallel_times[cluster][n] time_pc = time / total_time if not self.use_ratios: @@ -474,8 +473,7 @@ class ParallelReport(object): self.values.append(value) def write(self): - with open(self.filepath, 'w') as wfh: - writer = csv.writer(wfh) + with csvwriter(self.filepath) as writer: writer.writerow(['cluster', 'number_of_cores', 'total_time', '%time', '%running_time']) writer.writerows(self.values) @@ -520,7 +518,7 @@ class PowerStateStats(object): total_time = self.last_timestamp - self.first_timestamp state_stats = defaultdict(lambda: [None] * len(self.core_names)) - for cpu, states in self.cpu_states.iteritems(): + for cpu, states in self.cpu_states.items(): for state in states: time = states[state] time_pc = time / total_time @@ -543,8 +541,7 @@ class PowerStateStatsReport(object): self.precision = precision def write(self): - with open(self.filepath, 'w') as wfh: - writer = csv.writer(wfh) + with csvwriter(self.filepath) as writer: headers = ['state'] + ['{} CPU{}'.format(c, i) for i, c in enumerate(self.core_names)] writer.writerow(headers) @@ -561,8 +558,7 @@ class CpuUtilizationTimeline(object): def __init__(self, output_directory, cpus): self.filepath = os.path.join(output_directory, 'utilization-timeline.csv') - self._wfh = open(self.filepath, 'w') - self.writer = csv.writer(self._wfh) + self.writer, self._wfh = create_writer(self.filepath) headers = ['ts'] + ['{} CPU{}'.format(cpu.name, cpu.id) for cpu in cpus] self.writer.writerow(headers) diff --git a/wa/utils/diff.py b/wa/utils/diff.py index 9318f15c..c1bf3e67 100644 --- a/wa/utils/diff.py +++ b/wa/utils/diff.py @@ -1,11 +1,18 @@ -from wa.utils.misc import write_table +import os +import re + +from future.moves.itertools import zip_longest + +from wa.utils.misc import as_relative, diff_tokens, write_table +from wa.utils.misc import ensure_file_directory_exists as _f +from wa.utils.misc import ensure_directory_exists as _d def diff_interrupt_files(before, after, result): # pylint: disable=R0914 output_lines = [] with open(before) as bfh: with open(after) as ofh: - for bline, aline in izip(bfh, ofh): + for bline, aline in zip(bfh, ofh): bchunks = bline.strip().split() while True: achunks = aline.strip().split() @@ -20,7 +27,7 @@ def diff_interrupt_files(before, after, result): # pylint: disable=R0914 diffchunks = ['>'] + achunks output_lines.append(diffchunks) try: - aline = ofh.next() + aline = next(ofh) except StopIteration: break @@ -45,11 +52,9 @@ def diff_interrupt_files(before, after, result): # pylint: disable=R0914 def diff_sysfs_dirs(before, after, result): # pylint: disable=R0914 before_files = [] - os.path.walk(before, - lambda arg, dirname, names: arg.extend([os.path.join(dirname, f) for f in names]), - before_files - ) - before_files = filter(os.path.isfile, before_files) + for root, dirs, files in os.walk(before): + before_files.extend([os.path.join(root, f) for f in files]) + before_files = list(filter(os.path.isfile, before_files)) files = [os.path.relpath(f, before) for f in before_files] after_files = [os.path.join(after, f) for f in files] diff_files = [os.path.join(result, f) for f in files] @@ -61,7 +66,7 @@ def diff_sysfs_dirs(before, after, result): # pylint: disable=R0914 with open(bfile) as bfh, open(afile) as afh: # pylint: disable=C0321 with open(_f(dfile), 'w') as dfh: - for i, (bline, aline) in enumerate(izip_longest(bfh, afh), 1): + for i, (bline, aline) in enumerate(zip_longest(bfh, afh), 1): if aline is None: logger.debug('Lines missing from {}'.format(afile)) break diff --git a/wa/utils/doc.py b/wa/utils/doc.py index f20cf39d..c0589e66 100644 --- a/wa/utils/doc.py +++ b/wa/utils/doc.py @@ -164,16 +164,16 @@ def format_simple_table(rows, headers=None, align='>', show_borders=True, border """Formats a simple table.""" if not rows: return '' - rows = [map(str, r) for r in rows] + rows = [list(map(str, r)) for r in rows] num_cols = len(rows[0]) # cycle specified alignments until we have num_cols of them. This is # consitent with how such cases are handled in R, pandas, etc. it = cycle(align) - align = [it.next() for _ in xrange(num_cols)] + align = [next(it) for _ in range(num_cols)] - cols = zip(*rows) - col_widths = [max(map(len, c)) for c in cols] + cols = list(zip(*rows)) + col_widths = [max(list(map(len, c))) for c in cols] if headers: col_widths = [max(len(h), cw) for h, cw in zip(headers, col_widths)] row_format = ' '.join(['{:%s%s}' % (align[i], w) for i, w in enumerate(col_widths)]) @@ -259,12 +259,12 @@ def indent(text, spaces=4): def format_literal(lit): - if isinstance(lit, basestring): + if isinstance(lit, str): return '``\'{}\'``'.format(lit) elif hasattr(lit, 'pattern'): # regex return '``r\'{}\'``'.format(lit.pattern) elif isinstance(lit, dict): - content = indent(',\n'.join("{}: {}".format(key,val) for (key,val) in lit.iteritems())) + content = indent(',\n'.join("{}: {}".format(key,val) for (key,val) in lit.items())) return '::\n\n{}'.format(indent('{{\n{}\n}}'.format(content))) else: return '``{}``'.format(lit) @@ -287,7 +287,7 @@ def get_params_rst(parameters): text += indent('\nconstraint: ``{}``\n'.format(get_type_name(param.constraint))) if param.default: value = param.default - if isinstance(value, basestring) and value.startswith(USER_HOME): + if isinstance(value, str) and value.startswith(USER_HOME): value = value.replace(USER_HOME, '~') text += indent('\ndefault: {}\n'.format(format_literal(value))) text += '\n' @@ -298,7 +298,7 @@ def get_aliases_rst(aliases): text = '' for alias in aliases: param_str = ', '.join(['{}={}'.format(n, format_literal(v)) - for n, v in alias.params.iteritems()]) + for n, v in alias.params.items()]) text += '{}\n{}\n\n'.format(alias.name, indent(param_str)) return text diff --git a/wa/utils/exec_control.py b/wa/utils/exec_control.py index b98e8a06..a23336a4 100644 --- a/wa/utils/exec_control.py +++ b/wa/utils/exec_control.py @@ -12,7 +12,7 @@ def activate_environment(name): #pylint: disable=W0603 global __active_environment - if name not in __environments.keys(): + if name not in list(__environments.keys()): init_environment(name) __active_environment = name @@ -24,7 +24,7 @@ def init_environment(name): :raises: ``ValueError`` if an environment with name ``name`` already exists. """ - if name in __environments.keys(): + if name in list(__environments.keys()): msg = "Environment {} already exists".format(name) raise ValueError(msg) __environments[name] = [] @@ -39,7 +39,7 @@ def reset_environment(name=None): """ if name is not None: - if name not in __environments.keys(): + if name not in list(__environments.keys()): msg = "Environment {} does not exist".format(name) raise ValueError(msg) __environments[name] = [] @@ -75,7 +75,7 @@ def once_per_class(method): if __active_environment is None: activate_environment('default') - func_id = repr(method.func_name) + repr(args[0].__class__) + func_id = repr(method.__name__) + repr(args[0].__class__) if func_id in __environments[__active_environment]: return diff --git a/wa/utils/log.py b/wa/utils/log.py index a254374e..1012e7ab 100644 --- a/wa/utils/log.py +++ b/wa/utils/log.py @@ -128,13 +128,13 @@ def disable(logs): def __enable_logger(logger): - if isinstance(logger, basestring): + if isinstance(logger, str): logger = logging.getLogger(logger) logger.propagate = True def __disable_logger(logger): - if isinstance(logger, basestring): + if isinstance(logger, str): logger = logging.getLogger(logger) logger.propagate = False diff --git a/wa/utils/misc.py b/wa/utils/misc.py index 9caf7b3b..bb9ab546 100644 --- a/wa/utils/misc.py +++ b/wa/utils/misc.py @@ -18,7 +18,7 @@ Miscellaneous functions that don't fit anywhere else. """ -from __future__ import division + import os import sys import re @@ -30,9 +30,13 @@ import traceback import logging import random import hashlib +import sys from datetime import datetime, timedelta from operator import mul -from StringIO import StringIO +if sys.version_info[0] == 3: + from io import StringIO +else: + from io import BytesIO as StringIO from itertools import chain, cycle from distutils.spawn import find_executable @@ -83,11 +87,11 @@ def diff_tokens(before_token, after_token): def prepare_table_rows(rows): """Given a list of lists, make sure they are prepared to be formatted into a table by making sure each row has the same number of columns and stringifying all values.""" - rows = [map(str, r) for r in rows] - max_cols = max(map(len, rows)) + rows = [list(map(str, r)) for r in rows] + max_cols = max(list(map(len, rows))) for row in rows: pad = max_cols - len(row) - for _ in xrange(pad): + for _ in range(pad): row.append('') return rows @@ -102,10 +106,10 @@ def write_table(rows, wfh, align='>', headers=None): # pylint: disable=R0914 # cycle specified alignments until we have max_cols of them. This is # consitent with how such cases are handled in R, pandas, etc. it = cycle(align) - align = [it.next() for _ in xrange(num_cols)] + align = [next(it) for _ in range(num_cols)] - cols = zip(*rows) - col_widths = [max(map(len, c)) for c in cols] + cols = list(zip(*rows)) + col_widths = [max(list(map(len, c))) for c in cols] row_format = ' '.join(['{:%s%s}' % (align[i], w) for i, w in enumerate(col_widths)]) row_format += '\n' @@ -144,7 +148,7 @@ def _check_remove_item(the_list, item): """Helper function for merge_lists that implements checking wether an items should be removed from the list and doing so if needed. Returns ``True`` if the item has been removed and ``False`` otherwise.""" - if not isinstance(item, basestring): + if not isinstance(item, str): return False if not item.startswith('~'): return False @@ -275,7 +279,7 @@ def get_article(word): def get_random_string(length): """Returns a random ASCII string of the specified length).""" - return ''.join(random.choice(string.ascii_letters + string.digits) for _ in xrange(length)) + return ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(length)) class LoadSyntaxError(Exception): @@ -307,9 +311,9 @@ def load_struct_from_python(filepath=None, text=None): while modname in sys.modules: # highly unlikely, but... modname = get_random_string(RAND_MOD_NAME_LEN) mod = imp.new_module(modname) - exec text in mod.__dict__ # pylint: disable=exec-used + exec(text, mod.__dict__) # pylint: disable=exec-used return dict((k, v) - for k, v in mod.__dict__.iteritems() + for k, v in mod.__dict__.items() if not k.startswith('_')) except SyntaxError as e: raise LoadSyntaxError(e.message, filepath, e.lineno) @@ -404,7 +408,7 @@ def istextfile(fileobj, blocksize=512): def categorize(v): if hasattr(v, 'merge_with') and hasattr(v, 'merge_into'): return 'o' - elif hasattr(v, 'iteritems'): + elif hasattr(v, 'items'): return 'm' elif isiterable(v): return 's' @@ -515,13 +519,14 @@ def merge_sequencies(s1, s2): return type(s2)(unique(chain(s1, s2))) + def merge_maps(m1, m2): - return type(m2)(chain(m1.iteritems(), m2.iteritems())) + return type(m2)(chain(iter(m1.items()), iter(m2.items()))) def merge_dicts_simple(base, other): result = base.copy() - for key, value in (other or {}).iteritems(): + for key, value in (other or {}).items(): result[key] = merge_config_values(result.get(key), value) return result @@ -534,11 +539,11 @@ def touch(path): def get_object_name(obj): if hasattr(obj, 'name'): return obj.name - elif hasattr(obj, 'im_func'): - return '{}.{}'.format(get_object_name(obj.im_class), - obj.im_func.func_name) + elif hasattr(obj, '__func__') and hasattr(obj, '__self__'): + return '{}.{}'.format(get_object_name(obj.__self__.__class__), + obj.__func__.__name__) elif hasattr(obj, 'func_name'): - return obj.func_name + return obj.__name__ elif hasattr(obj, '__name__'): return obj.__name__ elif hasattr(obj, '__class__'): @@ -557,7 +562,7 @@ def resolve_cpus(name, target): - 'all' - returns all cpus - '' - Empty name will also return all cpus """ - cpu_list = range(target.number_of_cpus) + cpu_list = list(range(target.number_of_cpus)) # Support for passing cpu no directly if isinstance(name, int): diff --git a/wa/utils/revent.py b/wa/utils/revent.py index 22839b97..50552277 100644 --- a/wa/utils/revent.py +++ b/wa/utils/revent.py @@ -13,7 +13,7 @@ # limitations under the License. # -from __future__ import division + import os import struct import signal @@ -88,7 +88,7 @@ class UinputDeviceInfo(object): self.abs_bits = bytearray(parts[3]) self.num_absinfo = parts[4] self.absinfo = [absinfo(*read_struct(fh, absinfo_struct)) - for _ in xrange(self.num_absinfo)] + for _ in range(self.num_absinfo)] def __str__(self): return 'UInputInfo({})'.format(self.__dict__) @@ -145,7 +145,7 @@ class ReventRecording(object): if self.stream: events = self._iter_events() try: - first = last = events.next() + first = last = next(events) except StopIteration: self._duration = 0 for last in events: @@ -230,7 +230,7 @@ class ReventRecording(object): def _read_devices(self, fh): num_devices, = read_struct(fh, u32_struct) - for _ in xrange(num_devices): + for _ in range(num_devices): self.device_paths.append(read_string(fh)) def _read_gamepad_info(self, fh): @@ -243,7 +243,7 @@ class ReventRecording(object): raise RuntimeError(msg) self.fh.seek(self._events_start) if self.version >= 2: - for _ in xrange(self.num_events): + for _ in range(self.num_events): yield ReventEvent(self.fh) else: file_size = os.path.getsize(self.filepath) diff --git a/wa/utils/serializer.py b/wa/utils/serializer.py index 514d757e..1b7c1a74 100644 --- a/wa/utils/serializer.py +++ b/wa/utils/serializer.py @@ -71,7 +71,7 @@ POD_TYPES = [ dict, set, str, - unicode, + str, int, float, bool, @@ -104,7 +104,7 @@ class WAJSONDecoder(_json.JSONDecoder): d = _json.JSONDecoder.decode(self, s, **kwargs) def try_parse_object(v): - if isinstance(v, basestring): + if isinstance(v, str): if v.startswith('REGEX:'): _, flags, pattern = v.split(':', 2) return re.compile(pattern, int(flags or 0)) @@ -122,8 +122,8 @@ class WAJSONDecoder(_json.JSONDecoder): def load_objects(d): pairs = [] - for k, v in d.iteritems(): - if hasattr(v, 'iteritems'): + for k, v in d.items(): + if hasattr(v, 'items'): pairs.append((k, load_objects(v))) elif isiterable(v): pairs.append((k, [try_parse_object(i) for i in v])) @@ -160,13 +160,13 @@ class json(object): _mapping_tag = _yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG -_regex_tag = u'tag:wa:regex' -_level_tag = u'tag:wa:level' -_cpu_mask_tag = u'tag:wa:cpu_mask' +_regex_tag = 'tag:wa:regex' +_level_tag = 'tag:wa:level' +_cpu_mask_tag = 'tag:wa:cpu_mask' def _wa_dict_representer(dumper, data): - return dumper.represent_mapping(_mapping_tag, data.iteritems()) + return dumper.represent_mapping(_mapping_tag, iter(data.items())) def _wa_regex_representer(dumper, data): @@ -248,17 +248,17 @@ class python(object): def loads(s, *args, **kwargs): pod = {} try: - exec s in pod # pylint: disable=exec-used + exec(s, pod) # pylint: disable=exec-used except SyntaxError as e: raise SerializerSyntaxError(e.message, e.lineno) - for k in pod.keys(): + for k in list(pod.keys()): if k.startswith('__'): del pod[k] return pod def read_pod(source, fmt=None): - if isinstance(source, basestring): + if isinstance(source, str): with open(source) as fh: return _read_pod(fh, fmt) elif hasattr(source, 'read') and (hasattr(source, 'name') or fmt): @@ -269,7 +269,7 @@ def read_pod(source, fmt=None): def write_pod(pod, dest, fmt=None): - if isinstance(dest, basestring): + if isinstance(dest, str): with open(dest, 'w') as wfh: return _write_pod(pod, wfh, fmt) elif hasattr(dest, 'write') and (hasattr(dest, 'name') or fmt): @@ -323,8 +323,8 @@ def _write_pod(pod, wfh, fmt=None): def is_pod(obj): if type(obj) not in POD_TYPES: return False - if hasattr(obj, 'iteritems'): - for k, v in obj.iteritems(): + if hasattr(obj, 'items'): + for k, v in obj.items(): if not (is_pod(k) and is_pod(v)): return False elif isiterable(obj): diff --git a/wa/utils/terminalsize.py b/wa/utils/terminalsize.py index 828ca3e6..f211b795 100644 --- a/wa/utils/terminalsize.py +++ b/wa/utils/terminalsize.py @@ -89,5 +89,5 @@ def _get_terminal_size_linux(): if __name__ == "__main__": sizex, sizey = get_terminal_size() - print 'width =', sizex, 'height =', sizey + print('width =', sizex, 'height =', sizey) diff --git a/wa/utils/trace_cmd.py b/wa/utils/trace_cmd.py index 8eaf4d59..65900c79 100644 --- a/wa/utils/trace_cmd.py +++ b/wa/utils/trace_cmd.py @@ -114,7 +114,7 @@ class DroppedEventsEvent(object): def try_convert_to_numeric(v): try: if isiterable(v): - return map(numeric, v) + return list(map(numeric, v)) else: return numeric(v) except ValueError: @@ -153,13 +153,13 @@ def regex_body_parser(regex, flags=0): If regex is a pre-compiled object, flags will be ignored. """ - if isinstance(regex, basestring): + if isinstance(regex, str): regex = re.compile(regex, flags) def regex_parser_func(event, text): match = regex.search(text) if match: - for k, v in match.groupdict().iteritems(): + for k, v in match.groupdict().items(): try: event.fields[k] = int(v) except ValueError: @@ -321,7 +321,7 @@ class TraceCmdParser(object): continue body_parser = EVENT_PARSER_MAP.get(event_name, default_body_parser) - if isinstance(body_parser, basestring) or isinstance(body_parser, re._pattern_type): # pylint: disable=protected-access + if isinstance(body_parser, str) or isinstance(body_parser, re._pattern_type): # pylint: disable=protected-access body_parser = regex_body_parser(body_parser) yield TraceCmdEvent(parser=body_parser, **match.groupdict()) diff --git a/wa/utils/types.py b/wa/utils/types.py index 29d8cf49..6256b9c9 100644 --- a/wa/utils/types.py +++ b/wa/utils/types.py @@ -29,10 +29,19 @@ import os import re import numbers import shlex +import sys from bisect import insort -from urllib import quote, unquote +if sys.version_info[0] == 3: + from urllib.parse import quote, unquote + from past.builtins import basestring + long = int +else: + from urllib import quote, unquote from collections import defaultdict, MutableMapping from copy import copy +from functools import total_ordering + +from future.utils import with_metaclass from devlib.utils.types import identifier, boolean, integer, numeric, caseless_string @@ -47,7 +56,7 @@ def list_of_strs(value): """ if not isiterable(value): raise ValueError(value) - return map(str, value) + return list(map(str, value)) list_of_strings = list_of_strs @@ -59,7 +68,7 @@ def list_of_ints(value): """ if not isiterable(value): raise ValueError(value) - return map(int, value) + return list(map(int, value)) list_of_integers = list_of_ints @@ -72,7 +81,7 @@ def list_of_numbers(value): """ if not isiterable(value): raise ValueError(value) - return map(numeric, value) + return list(map(numeric, value)) def list_of_bools(value, interpret_strings=True): @@ -88,9 +97,9 @@ def list_of_bools(value, interpret_strings=True): if not isiterable(value): raise ValueError(value) if interpret_strings: - return map(boolean, value) + return list(map(boolean, value)) else: - return map(bool, value) + return list(map(bool, value)) def list_of(type_): @@ -98,16 +107,16 @@ def list_of(type_): attempts to convert all elements in the passed value to the specifed ``type_``, raising ``ValueError`` on error.""" def __init__(self, values): - list.__init__(self, map(type_, values)) + list.__init__(self, list(map(type_, values))) def append(self, value): list.append(self, type_(value)) def extend(self, other): - list.extend(self, map(type_, other)) + list.extend(self, list(map(type_, other))) def from_pod(cls, pod): - return cls(map(type_, pod)) + return cls(list(map(type_, pod))) def _to_pod(self): return self @@ -132,7 +141,7 @@ def list_or_string(value): a one-element list with stringified value will be returned. """ - if isinstance(value, basestring): + if isinstance(value, str): return [value] else: try: @@ -147,11 +156,11 @@ def list_or_caseless_string(value): not iterable a one-element list with stringified value will be returned. """ - if isinstance(value, basestring): + if isinstance(value, str): return [caseless_string(value)] else: try: - return map(caseless_string, value) + return list(map(caseless_string, value)) except ValueError: return [caseless_string(value)] @@ -229,8 +238,8 @@ class arguments(list): def __init__(self, value=None): if isiterable(value): - super(arguments, self).__init__(map(str, value)) - elif isinstance(value, basestring): + super(arguments, self).__init__(list(map(str, value))) + elif isinstance(value, str): posix = os.name != 'nt' super(arguments, self).__init__(shlex.split(value, posix=posix)) elif value is None: @@ -242,7 +251,7 @@ class arguments(list): return super(arguments, self).append(str(value)) def extend(self, values): - return super(arguments, self).extend(map(str, values)) + return super(arguments, self).extend(list(map(str, values))) def __str__(self): return ' '.join(self) @@ -288,7 +297,7 @@ class prioritylist(object): self.__delitem__(index) def _priority_index(self, element): - for priority, elements in self.elements.iteritems(): + for priority, elements in self.elements.items(): if element in elements: return (priority, elements.index(element)) raise IndexError(element) @@ -333,7 +342,7 @@ class prioritylist(object): else: index_range = [index] elif isinstance(index, slice): - index_range = range(index.start or 0, index.stop, index.step or 1) + index_range = list(range(index.start or 0, index.stop, index.step or 1)) else: raise ValueError('Invalid index {}'.format(index)) current_global_offset = 0 @@ -391,7 +400,7 @@ class toggle_set(set): def __init__(self, *args): if args: value = args[0] - if isinstance(value, basestring): + if isinstance(value, str): msg = 'invalid type for toggle_set: "{}"' raise TypeError(msg.format(type(value))) set.__init__(self, *args) @@ -507,12 +516,15 @@ class obj_dict(MutableMapping): raise AttributeError("No such attribute: " + name) def __getattr__(self, name): + if 'dict' not in self.__dict__: + raise AttributeError("No such attribute: " + name) if name in self.__dict__['dict']: return self.__dict__['dict'][name] else: raise AttributeError("No such attribute: " + name) +@total_ordering class level(object): """ A level has a name and behaves like a string when printed, however it also @@ -538,11 +550,8 @@ class level(object): def __repr__(self): return '{}({})'.format(self.name, self.value) - def __cmp__(self, other): - if isinstance(other, level): - return cmp(self.value, other.value) - else: - return cmp(self.value, other) + def __hash__(self): + return hash(self.name) def __eq__(self, other): if isinstance(other, level): @@ -552,13 +561,24 @@ class level(object): else: return self.value == other - def __ne__(self, other): + def __lt__(self, other): if isinstance(other, level): - return self.value != other.value + return self.value < other.value elif isinstance(other, basestring): - return self.name != other + return self.name < other else: - return self.value != other + return self.value < other + + +class _EnumMeta(type): + + def __str__(cls): + return str(cls.levels) + + def __getattr__(self, name): + name = name.lower() + if name in self.__dict__: + return self.__dict__[name] def enum(args, start=0, step=1): @@ -583,11 +603,7 @@ def enum(args, start=0, step=1): """ - class Enum(object): - - class __metaclass__(type): - def __str__(cls): - return str(cls.levels) + class Enum(with_metaclass(_EnumMeta, object)): @classmethod def from_pod(cls, pod): @@ -642,14 +658,14 @@ class ParameterDict(dict): # Function to determine the appropriate prefix based on the parameters type @staticmethod def _get_prefix(obj): - if isinstance(obj, basestring): + if isinstance(obj, str): prefix = 's' elif isinstance(obj, float): prefix = 'f' - elif isinstance(obj, long): - prefix = 'd' elif isinstance(obj, bool): prefix = 'b' + elif isinstance(obj, long): + prefix = 'i' elif isinstance(obj, int): prefix = 'i' elif obj is None: @@ -686,7 +702,7 @@ class ParameterDict(dict): elif value_type == 'b': return boolean(value) elif value_type == 'd': - return long(value) + return int(value) elif value_type == 'f': return float(value) elif value_type == 'i': @@ -700,7 +716,7 @@ class ParameterDict(dict): raise ValueError('Unknown {} {}'.format(type(string), string)) def __init__(self, *args, **kwargs): - for k, v in kwargs.iteritems(): + for k, v in kwargs.items(): self.__setitem__(k, v) dict.__init__(self, *args) @@ -714,7 +730,7 @@ class ParameterDict(dict): return dict.__contains__(self, self._encode(item)) def __iter__(self): - return iter((k, self._decode(v)) for (k, v) in self.items()) + return iter((k, self._decode(v)) for (k, v) in list(self.items())) def iteritems(self): return self.__iter__() @@ -730,7 +746,10 @@ class ParameterDict(dict): return (key, self._decode(value)) def iter_encoded_items(self): - return dict.iteritems(self) + if sys.version_info[0] == 3: + return dict.items(self) + else: + return dict.iteritems(self) def get_encoded_value(self, name): return dict.__getitem__(self, name) @@ -743,7 +762,7 @@ class ParameterDict(dict): if isinstance(d, ParameterDict): dict.update(self, d) else: - for k, v in d.iteritems(): + for k, v in d.items(): self[k] = v @@ -762,7 +781,7 @@ class cpu_mask(object): self._mask = 0 if isinstance(cpus, int): self._mask = cpus - elif isinstance(cpus, basestring): + elif isinstance(cpus, str): if cpus[:2] == '0x' or cpus[:2] == '0X': self._mask = int(cpus, 16) else: diff --git a/wa/workloads/exoplayer/__init__.py b/wa/workloads/exoplayer/__init__.py index dea5a7cf..7e51682f 100644 --- a/wa/workloads/exoplayer/__init__.py +++ b/wa/workloads/exoplayer/__init__.py @@ -18,7 +18,13 @@ import re import os import time -import urllib + +from future.standard_library import install_aliases +install_aliases() + +import urllib.request +import urllib.parse +import urllib.error from wa import ApkWorkload, Parameter, ConfigError, WorkloadError from wa.framework.configuration.core import settings @@ -81,7 +87,7 @@ class ExoPlayer(ApkWorkload): Playback duration of the video file. This becomes the duration of the workload. If provided must be shorter than the length of the media. """), - Parameter('format', allowed_values=DOWNLOAD_URLS.keys(), + Parameter('format', allowed_values=list(DOWNLOAD_URLS.keys()), description=""" Specifies which format video file to play. Default is {} """.format(default_format)), @@ -137,7 +143,7 @@ class ExoPlayer(ApkWorkload): filename = '{}_{}'.format(format_resolution, os.path.basename(url)) filepath = os.path.join(self.video_directory, filename) self.logger.info('Downloading {} to {}...'.format(url, filepath)) - urllib.urlretrieve(url, filepath) + urllib.request.urlretrieve(url, filepath) return filepath else: if len(files) > 1: @@ -172,7 +178,7 @@ class ExoPlayer(ApkWorkload): self.play_cmd = 'am start -a {} -d "file://{}"'.format(self.action, self.device_video_file) - self.monitor = self.target.get_logcat_monitor(REGEXPS.values()) + self.monitor = self.target.get_logcat_monitor(list(REGEXPS.values())) self.monitor.start() def run(self, context): diff --git a/wa/workloads/hackbench/__init__.py b/wa/workloads/hackbench/__init__.py index 70a2550f..8ba54a19 100644 --- a/wa/workloads/hackbench/__init__.py +++ b/wa/workloads/hackbench/__init__.py @@ -84,7 +84,7 @@ class Hackbench(Workload): results_file = context.get_artifact_path('hackbench-results') with open(results_file) as fh: for line in fh: - for label, (regex, units) in regex_map.iteritems(): + for label, (regex, units) in regex_map.items(): match = regex.search(line) if match: context.add_metric(label, float(match.group(1)), units) diff --git a/wa/workloads/jankbench/__init__.py b/wa/workloads/jankbench/__init__.py index d24f2e2d..0522a85f 100644 --- a/wa/workloads/jankbench/__init__.py +++ b/wa/workloads/jankbench/__init__.py @@ -15,7 +15,7 @@ # pylint: disable=E1101,W0201,E0203 -from __future__ import division + import os import re import select @@ -23,6 +23,7 @@ import json import threading import sqlite3 import subprocess +import sys from copy import copy import pandas as pd @@ -143,7 +144,7 @@ class Jankbench(ApkWorkload): for test_name, rep in results.index: test_results = results.ix[test_name, rep] - for metric, value in test_results.iteritems(): + for metric, value in test_results.items(): context.add_metric(metric, value, units=None, lower_is_better=True, classifiers={'test_name': test_name, 'rep': rep}) @@ -222,6 +223,8 @@ class JankbenchRunMonitor(threading.Thread): ready, _, _ = select.select([proc.stdout, proc.stderr], [], [], 2) if ready: line = ready[0].readline() + if sys.version_info[0] == 3: + line = line.decode(sys.stdout.encoding) if self.regex.search(line): self.run_ended.set() diff --git a/wa/workloads/meabo/__init__.py b/wa/workloads/meabo/__init__.py index c3b8244e..84761668 100644 --- a/wa/workloads/meabo/__init__.py +++ b/wa/workloads/meabo/__init__.py @@ -145,7 +145,7 @@ class Meabo(Workload): Controls which phases to run. ''', constraint=lambda x: all(0 < v <=10 for v in x), - default=range(1, 11), + default=list(range(1, 11)), ), Parameter( 'threads', diff --git a/wa/workloads/openssl/__init__.py b/wa/workloads/openssl/__init__.py index 9c991082..264ceeb6 100644 --- a/wa/workloads/openssl/__init__.py +++ b/wa/workloads/openssl/__init__.py @@ -102,7 +102,7 @@ class Openssl(Workload): parts = line.split(':') if parts[0] == '+F': # evp ciphers - for bs, value in zip(BLOCK_SIZES, map(float, parts[3:])): + for bs, value in zip(BLOCK_SIZES, list(map(float, parts[3:]))): value = value / 2**20 # to MB context.add_metric('score', value, 'MB/s', classifiers={'block_size': bs}) diff --git a/wa/workloads/sysbench/__init__.py b/wa/workloads/sysbench/__init__.py index 5ebb7b5b..4a26f261 100644 --- a/wa/workloads/sysbench/__init__.py +++ b/wa/workloads/sysbench/__init__.py @@ -135,16 +135,16 @@ class Sysbench(Workload): with open(self.host_results_file) as fh: find_line_with('General statistics:', fh) - extract_metric('total time', fh.next(), context.output) - extract_metric('total number of events', fh.next(), context.output, lower_is_better=False) + extract_metric('total time', next(fh), context.output) + extract_metric('total number of events', next(fh), context.output, lower_is_better=False) find_line_with('response time:', fh) - extract_metric('min', fh.next(), context.output, 'response time ') - extract_metric('avg', fh.next(), context.output, 'response time ') - extract_metric('max', fh.next(), context.output, 'response time ') - extract_metric('approx. 95 percentile', fh.next(), context.output) + extract_metric('min', next(fh), context.output, 'response time ') + extract_metric('avg', next(fh), context.output, 'response time ') + extract_metric('max', next(fh), context.output, 'response time ') + extract_metric('approx. 95 percentile', next(fh), context.output) find_line_with('Threads fairness:', fh) - extract_threads_fairness_metric('events', fh.next(), context.output) - extract_threads_fairness_metric('execution time', fh.next(), context.output) + extract_threads_fairness_metric('events', next(fh), context.output) + extract_threads_fairness_metric('execution time', next(fh), context.output) def teardown(self, context): self.target.remove(self.target_results_file) @@ -155,7 +155,7 @@ class Sysbench(Workload): def _build_command(self, **parameters): param_strings = ['--{}={}'.format(k.replace('_', '-'), v) - for k, v in parameters.iteritems()] + for k, v in parameters.items()] if self.file_test_mode: param_strings.append('--file-test-mode={}'.format(self.file_test_mode)) sysbench_command = '{} {} {} run'.format(self.target_binary, ' '.join(param_strings), self.cmd_params) diff --git a/wa/workloads/vellamo/__init__.py b/wa/workloads/vellamo/__init__.py index db227737..fdfb81d8 100644 --- a/wa/workloads/vellamo/__init__.py +++ b/wa/workloads/vellamo/__init__.py @@ -17,7 +17,7 @@ import os import json import re -from HTMLParser import HTMLParser +from html.parser import HTMLParser from wa import ApkUiautoWorkload, Parameter from wa.utils.types import list_of_strs @@ -48,7 +48,7 @@ class Vellamo(ApkUiautoWorkload): '3.0': ['Browser', 'Metal', 'Multi'], '3.2.4': ['Browser', 'Metal', 'Multi'], } - valid_versions = benchmark_types.keys() + valid_versions = list(benchmark_types.keys()) summary_metrics = None parameters = [ @@ -119,7 +119,7 @@ class Vellamo(ApkUiautoWorkload): benchmark.name = benchmark.name.replace(' ', '_') context.add_metric('{}_Total'.format(benchmark.name), benchmark.score) - for name, score in benchmark.metrics.items(): + for name, score in list(benchmark.metrics.items()): name = name.replace(' ', '_') context.add_metric('{}_{}'.format(benchmark.name, name), score) |