1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
|
"""
Database utility functions which use but are not actually models themselves
Used to allow models.py to be shortened and easier to follow.
"""
from __future__ import unicode_literals
# pylint: disable=wrong-import-order
import os
import yaml
import jinja2
import logging
from django.db.models import Q, Case, When, IntegerField, Sum
from lava_scheduler_app.models import (
Device,
TestJob,
validate_job,
validate_device,
Worker
)
from lava_results_app.dbutils import map_metadata
# pylint: disable=too-many-branches,too-many-statements,too-many-locals
def match_vlan_interface(device, job_def):
if not isinstance(job_def, dict):
raise RuntimeError("Invalid vlan interface data")
if 'protocols' not in job_def or 'lava-vland' not in job_def['protocols'] or not device:
return False
interfaces = []
logger = logging.getLogger('lava-master')
device_dict = device.load_configuration()
if not device_dict or device_dict.get('parameters', {}).get('interfaces', None) is None:
return False
for vlan_name in job_def['protocols']['lava-vland']:
tag_list = job_def['protocols']['lava-vland'][vlan_name]['tags']
for interface in device_dict['parameters']['interfaces']:
tags = device_dict['parameters']['interfaces'][interface]['tags']
if not tags:
continue
logger.info(
"Job requests %s for %s, device %s provides %s for %s",
tag_list, vlan_name, device.hostname, tags, interface)
if set(tags) & set(tag_list) == set(tag_list) and interface not in interfaces:
logger.info("Matched vlan %s to interface %s on %s", vlan_name, interface, device)
interfaces.append(interface)
# matched, do not check any further interfaces of this device for this vlan
break
logger.info("Matched: %s", (len(interfaces) == len(job_def['protocols']['lava-vland'].keys())))
return len(interfaces) == len(job_def['protocols']['lava-vland'].keys())
# TODO: check the list of exception that can be raised
def testjob_submission(job_definition, user, original_job=None):
"""
Single submission frontend for YAML
:param job_definition: string of the job submission
:param user: user attempting the submission
:return: a job or a list of jobs
:raises: SubmissionException, Device.DoesNotExist,
DeviceType.DoesNotExist, DevicesUnavailableException,
ValueError
"""
validate_job(job_definition)
# returns a single job or a list (not a QuerySet) of job objects.
job = TestJob.from_yaml_and_user(job_definition, user, original_job=original_job)
return job
def parse_job_description(job):
filename = os.path.join(job.output_dir, 'description.yaml')
logger = logging.getLogger('lava-master')
try:
with open(filename, 'r') as f_describe:
description = f_describe.read()
pipeline = yaml.load(description)
except (IOError, yaml.YAMLError):
logger.error("'Unable to open and parse '%s'", filename)
return
if not map_metadata(description, job):
logger.warning("[%d] unable to map metadata", job.id)
# add the compatibility result from the master to the definition for comparison on the slave.
try:
compat = int(pipeline['compatibility'])
except (TypeError, ValueError):
compat = pipeline['compatibility'] if pipeline is not None else None
logger.error("[%d] Unable to parse job compatibility: %s",
job.id, compat)
compat = 0
job.pipeline_compatibility = compat
job.save(update_fields=['pipeline_compatibility'])
def device_type_summary(visible=None):
devices = Device.objects.filter(
~Q(health=Device.HEALTH_RETIRED) & Q(device_type__in=visible)).only(
'state', 'health', 'is_public', 'device_type', 'hostname').values('device_type').annotate(
idle=Sum(
Case(
When(state=Device.STATE_IDLE, health__in=[Device.HEALTH_GOOD, Device.HEALTH_UNKNOWN], worker_host__state=Worker.STATE_ONLINE, then=1),
default=0, output_field=IntegerField()
)
),
busy=Sum(
Case(
When(state__in=[Device.STATE_RESERVED, Device.STATE_RUNNING], then=1),
default=0, output_field=IntegerField()
)
),
offline=Sum(
Case(
When(Q(state=Device.STATE_IDLE) & (Q(worker_host__state=Worker.STATE_OFFLINE) | ~Q(health__in=[Device.HEALTH_GOOD, Device.HEALTH_UNKNOWN])),
then=1),
default=0, output_field=IntegerField()
)
),
restricted=Sum(
Case(
When(is_public=False, then=1),
default=0, output_field=IntegerField()
)
),).order_by('device_type')
return devices
def load_devicetype_template(device_type_name, raw=False):
"""
Loads the bare device-type template as a python dictionary object for
representation within the device_type templates.
No device-specific details are parsed - default values only, so some
parts of the dictionary may be unexpectedly empty. Not to be used when
rendering device configuration for a testjob.
:param device_type_name: DeviceType.name (string)
:param raw: if True, return the raw yaml
:return: None or a dictionary of the device type template.
"""
path = os.path.dirname(Device.CONFIG_PATH)
type_loader = jinja2.FileSystemLoader([os.path.join(path, 'device-types')])
env = jinja2.Environment(
loader=jinja2.ChoiceLoader([type_loader]),
trim_blocks=True)
try:
template = env.get_template("%s.jinja2" % device_type_name)
data = template.render()
if not data:
return None
return data if raw else yaml.safe_load(data)
except (jinja2.TemplateError, yaml.error.YAMLError):
return None
def invalid_template(dt):
"""
Careful with the inverted logic here.
Return True if the template is invalid.
See unit tests in test_device.py
"""
d_template = bool(load_devicetype_template(dt.name)) # returns None on error ( == False)
if not d_template:
queryset = Device.objects.filter(Q(device_type=dt), ~Q(health=Device.HEALTH_RETIRED))
extends = set([device.get_extends() for device in queryset])
if not extends:
return True
for extend in extends:
if not extend:
return True
d_template = not bool(load_devicetype_template(extend.replace('.jinja2', '')))
# if d_template is False, template is valid, invalid_template returns False
if d_template:
return True
else:
d_template = False # template exists, invalid check is False
return d_template
|
