blob: 41a9f819dfcf835577aa26863cbef22566efa5f4 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05002# BitBake Toaster Implementation
3#
4# Copyright (C) 2013 Intel Corporation
5#
Brad Bishopc342db32019-05-15 21:57:59 -04006# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05009from __future__ import unicode_literals
10
Patrick Williamsc0f7c042017-02-23 20:41:17 -060011from django.db import models, IntegrityError, DataError
12from django.db.models import F, Q, Sum, Count
Patrick Williamsc124f4f2015-09-15 14:41:29 -050013from django.utils import timezone
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050014from django.utils.encoding import force_bytes
Patrick Williamsc124f4f2015-09-15 14:41:29 -050015
16from django.core.urlresolvers import reverse
17
18from django.core import validators
19from django.conf import settings
20import django.db.models.signals
21
Patrick Williamsc0f7c042017-02-23 20:41:17 -060022import sys
23import os
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050024import re
25import itertools
Patrick Williamsc0f7c042017-02-23 20:41:17 -060026from signal import SIGUSR1
Patrick Williamsc124f4f2015-09-15 14:41:29 -050027
Brad Bishop6e60e8b2018-02-01 10:27:11 -050028
Patrick Williamsc124f4f2015-09-15 14:41:29 -050029import logging
30logger = logging.getLogger("toaster")
31
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050032if 'sqlite' in settings.DATABASES['default']['ENGINE']:
33 from django.db import transaction, OperationalError
34 from time import sleep
35
36 _base_save = models.Model.save
37 def save(self, *args, **kwargs):
38 while True:
39 try:
40 with transaction.atomic():
41 return _base_save(self, *args, **kwargs)
42 except OperationalError as err:
43 if 'database is locked' in str(err):
44 logger.warning("%s, model: %s, args: %s, kwargs: %s",
45 err, self.__class__, args, kwargs)
46 sleep(0.5)
47 continue
48 raise
49
50 models.Model.save = save
51
52 # HACK: Monkey patch Django to fix 'database is locked' issue
53
54 from django.db.models.query import QuerySet
55 _base_insert = QuerySet._insert
56 def _insert(self, *args, **kwargs):
57 with transaction.atomic(using=self.db, savepoint=False):
58 return _base_insert(self, *args, **kwargs)
59 QuerySet._insert = _insert
60
61 from django.utils import six
62 def _create_object_from_params(self, lookup, params):
63 """
64 Tries to create an object using passed params.
65 Used by get_or_create and update_or_create
66 """
67 try:
68 obj = self.create(**params)
69 return obj, True
Patrick Williamsc0f7c042017-02-23 20:41:17 -060070 except (IntegrityError, DataError):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050071 exc_info = sys.exc_info()
72 try:
73 return self.get(**lookup), False
74 except self.model.DoesNotExist:
75 pass
76 six.reraise(*exc_info)
77
78 QuerySet._create_object_from_params = _create_object_from_params
79
80 # end of HACK
Patrick Williamsc124f4f2015-09-15 14:41:29 -050081
82class GitURLValidator(validators.URLValidator):
83 import re
84 regex = re.compile(
85 r'^(?:ssh|git|http|ftp)s?://' # http:// or https://
86 r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
87 r'localhost|' # localhost...
88 r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
89 r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
90 r'(?::\d+)?' # optional port
91 r'(?:/?|[/?]\S+)$', re.IGNORECASE)
92
93def GitURLField(**kwargs):
94 r = models.URLField(**kwargs)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060095 for i in range(len(r.validators)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050096 if isinstance(r.validators[i], validators.URLValidator):
97 r.validators[i] = GitURLValidator()
98 return r
99
100
101class ToasterSetting(models.Model):
102 name = models.CharField(max_length=63)
103 helptext = models.TextField()
104 value = models.CharField(max_length=255)
105
106 def __unicode__(self):
107 return "Setting %s = %s" % (self.name, self.value)
108
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600109
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500110class ProjectManager(models.Manager):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800111 def create_project(self, name, release, existing_project=None):
112 if existing_project and (release is not None):
113 prj = existing_project
114 prj.bitbake_version = release.bitbake_version
115 prj.release = release
116 # Delete the previous ProjectLayer mappings
117 for pl in ProjectLayer.objects.filter(project=prj):
118 pl.delete()
119 elif release is not None:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600120 prj = self.model(name=name,
121 bitbake_version=release.bitbake_version,
122 release=release)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500123 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600124 prj = self.model(name=name,
125 bitbake_version=None,
126 release=None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500127 prj.save()
128
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600129 for defaultconf in ToasterSetting.objects.filter(
130 name__startswith="DEFCONF_"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500131 name = defaultconf.name[8:]
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800132 pv,create = ProjectVariable.objects.get_or_create(project=prj,name=name)
133 pv.value = defaultconf.value
134 pv.save()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500135
136 if release is None:
137 return prj
138
139 for rdl in release.releasedefaultlayer_set.all():
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600140 lv = Layer_Version.objects.filter(
141 layer__name=rdl.layer_name,
142 release=release).first()
143
144 if lv:
145 ProjectLayer.objects.create(project=prj,
146 layercommit=lv,
147 optional=False)
148 else:
149 logger.warning("Default project layer %s not found" %
150 rdl.layer_name)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500151
152 return prj
153
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500154 # return single object with is_default = True
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500155 def get_or_create_default_project(self):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600156 projects = super(ProjectManager, self).filter(is_default=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500157
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500158 if len(projects) > 1:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500159 raise Exception('Inconsistent project data: multiple ' +
160 'default projects (i.e. with is_default=True)')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500161 elif len(projects) < 1:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500162 options = {
163 'name': 'Command line builds',
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600164 'short_description':
165 'Project for builds started outside Toaster',
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500166 'is_default': True
167 }
168 project = Project.objects.create(**options)
169 project.save()
170
171 return project
172 else:
173 return projects[0]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500174
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500175
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500176class Project(models.Model):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500177 search_allowed_fields = ['name', 'short_description', 'release__name',
178 'release__branch_name']
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500179 name = models.CharField(max_length=100)
180 short_description = models.CharField(max_length=50, blank=True)
181 bitbake_version = models.ForeignKey('BitbakeVersion', null=True)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500182 release = models.ForeignKey("Release", null=True)
183 created = models.DateTimeField(auto_now_add=True)
184 updated = models.DateTimeField(auto_now=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500185 # This is a horrible hack; since Toaster has no "User" model available when
186 # running in interactive mode, we can't reference the field here directly
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500187 # Instead, we keep a possible null reference to the User id,
188 # as not to force
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500189 # hard links to possibly missing models
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500190 user_id = models.IntegerField(null=True)
191 objects = ProjectManager()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500192
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800193 # build directory override (e.g. imported)
194 builddir = models.TextField()
195 # merge the Toaster configure attributes directly into the standard conf files
196 merged_attr = models.BooleanField(default=False)
197
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500198 # set to True for the project which is the default container
199 # for builds initiated by the command line etc.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500200 is_default= models.BooleanField(default=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500201
202 def __unicode__(self):
203 return "%s (Release %s, BBV %s)" % (self.name, self.release, self.bitbake_version)
204
205 def get_current_machine_name(self):
206 try:
207 return self.projectvariable_set.get(name="MACHINE").value
208 except (ProjectVariable.DoesNotExist,IndexError):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500209 return None;
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500210
211 def get_number_of_builds(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500212 """Return the number of builds which have ended"""
213
214 return self.build_set.exclude(
215 Q(outcome=Build.IN_PROGRESS) |
216 Q(outcome=Build.CANCELLED)
217 ).count()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500218
219 def get_last_build_id(self):
220 try:
221 return Build.objects.filter( project = self.id ).order_by('-completed_on')[0].id
222 except (Build.DoesNotExist,IndexError):
223 return( -1 )
224
225 def get_last_outcome(self):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500226 build_id = self.get_last_build_id()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500227 if (-1 == build_id):
228 return( "" )
229 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500230 return Build.objects.filter( id = build_id )[ 0 ].outcome
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500231 except (Build.DoesNotExist,IndexError):
232 return( "not_found" )
233
234 def get_last_target(self):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500235 build_id = self.get_last_build_id()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500236 if (-1 == build_id):
237 return( "" )
238 try:
239 return Target.objects.filter(build = build_id)[0].target
240 except (Target.DoesNotExist,IndexError):
241 return( "not_found" )
242
243 def get_last_errors(self):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500244 build_id = self.get_last_build_id()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500245 if (-1 == build_id):
246 return( 0 )
247 try:
248 return Build.objects.filter(id = build_id)[ 0 ].errors.count()
249 except (Build.DoesNotExist,IndexError):
250 return( "not_found" )
251
252 def get_last_warnings(self):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500253 build_id = self.get_last_build_id()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500254 if (-1 == build_id):
255 return( 0 )
256 try:
257 return Build.objects.filter(id = build_id)[ 0 ].warnings.count()
258 except (Build.DoesNotExist,IndexError):
259 return( "not_found" )
260
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500261 def get_last_build_extensions(self):
262 """
263 Get list of file name extensions for images produced by the most
264 recent build
265 """
266 last_build = Build.objects.get(pk = self.get_last_build_id())
267 return last_build.get_image_file_extensions()
268
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500269 def get_last_imgfiles(self):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500270 build_id = self.get_last_build_id()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500271 if (-1 == build_id):
272 return( "" )
273 try:
274 return Variable.objects.filter(build = build_id, variable_name = "IMAGE_FSTYPES")[ 0 ].variable_value
275 except (Variable.DoesNotExist,IndexError):
276 return( "not_found" )
277
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500278 def get_all_compatible_layer_versions(self):
279 """ Returns Queryset of all Layer_Versions which are compatible with
280 this project"""
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500281 queryset = None
282
283 # guard on release, as it can be null
284 if self.release:
285 queryset = Layer_Version.objects.filter(
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600286 (Q(release=self.release) &
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500287 Q(build=None) &
288 Q(project=None)) |
289 Q(project=self))
290 else:
291 queryset = Layer_Version.objects.none()
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500292
293 return queryset
294
295 def get_project_layer_versions(self, pk=False):
296 """ Returns the Layer_Versions currently added to this project """
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500297 layer_versions = self.projectlayer_set.all().values_list('layercommit',
298 flat=True)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500299
300 if pk is False:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500301 return Layer_Version.objects.filter(pk__in=layer_versions)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500302 else:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500303 return layer_versions
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500304
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500305
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800306 def get_default_image_recipe(self):
307 try:
308 return self.projectvariable_set.get(name="DEFAULT_IMAGE").value
309 except (ProjectVariable.DoesNotExist,IndexError):
310 return None;
311
312 def get_is_new(self):
313 return self.get_variable(Project.PROJECT_SPECIFIC_ISNEW)
314
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500315 def get_available_machines(self):
316 """ Returns QuerySet of all Machines which are provided by the
317 Layers currently added to the Project """
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500318 queryset = Machine.objects.filter(
319 layer_version__in=self.get_project_layer_versions())
320
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500321 return queryset
322
323 def get_all_compatible_machines(self):
324 """ Returns QuerySet of all the compatible machines available to the
325 project including ones from Layers not currently added """
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500326 queryset = Machine.objects.filter(
327 layer_version__in=self.get_all_compatible_layer_versions())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500328
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500329 return queryset
330
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500331 def get_available_distros(self):
332 """ Returns QuerySet of all Distros which are provided by the
333 Layers currently added to the Project """
334 queryset = Distro.objects.filter(
335 layer_version__in=self.get_project_layer_versions())
336
337 return queryset
338
339 def get_all_compatible_distros(self):
340 """ Returns QuerySet of all the compatible Wind River distros available to the
341 project including ones from Layers not currently added """
342 queryset = Distro.objects.filter(
343 layer_version__in=self.get_all_compatible_layer_versions())
344
345 return queryset
346
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500347 def get_available_recipes(self):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500348 """ Returns QuerySet of all the recipes that are provided by layers
349 added to this project """
350 queryset = Recipe.objects.filter(
351 layer_version__in=self.get_project_layer_versions())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500352
353 return queryset
354
355 def get_all_compatible_recipes(self):
356 """ Returns QuerySet of all the compatible Recipes available to the
357 project including ones from Layers not currently added """
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500358 queryset = Recipe.objects.filter(
359 layer_version__in=self.get_all_compatible_layer_versions()).exclude(name__exact='')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500361 return queryset
362
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800363 # Project Specific status management
364 PROJECT_SPECIFIC_STATUS = 'INTERNAL_PROJECT_SPECIFIC_STATUS'
365 PROJECT_SPECIFIC_CALLBACK = 'INTERNAL_PROJECT_SPECIFIC_CALLBACK'
366 PROJECT_SPECIFIC_ISNEW = 'INTERNAL_PROJECT_SPECIFIC_ISNEW'
367 PROJECT_SPECIFIC_DEFAULTIMAGE = 'PROJECT_SPECIFIC_DEFAULTIMAGE'
368 PROJECT_SPECIFIC_NONE = ''
369 PROJECT_SPECIFIC_NEW = '1'
370 PROJECT_SPECIFIC_EDIT = '2'
371 PROJECT_SPECIFIC_CLONING = '3'
372 PROJECT_SPECIFIC_CLONING_SUCCESS = '4'
373 PROJECT_SPECIFIC_CLONING_FAIL = '5'
374
375 def get_variable(self,variable,default_value = ''):
376 try:
377 return self.projectvariable_set.get(name=variable).value
378 except (ProjectVariable.DoesNotExist,IndexError):
379 return default_value
380
381 def set_variable(self,variable,value):
382 pv,create = ProjectVariable.objects.get_or_create(project = self, name = variable)
383 pv.value = value
384 pv.save()
385
386 def get_default_image(self):
387 return self.get_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE)
388
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500389 def schedule_build(self):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500390
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500391 from bldcontrol.models import BuildRequest, BRTarget, BRLayer
392 from bldcontrol.models import BRBitbake, BRVariable
393
394 try:
395 now = timezone.now()
396 build = Build.objects.create(project=self,
397 completed_on=now,
398 started_on=now)
399
400 br = BuildRequest.objects.create(project=self,
401 state=BuildRequest.REQ_QUEUED,
402 build=build)
403 BRBitbake.objects.create(req=br,
404 giturl=self.bitbake_version.giturl,
405 commit=self.bitbake_version.branch,
406 dirpath=self.bitbake_version.dirpath)
407
408 for t in self.projecttarget_set.all():
409 BRTarget.objects.create(req=br, target=t.target, task=t.task)
410 Target.objects.create(build=br.build, target=t.target,
411 task=t.task)
412 # If we're about to build a custom image recipe make sure
413 # that layer is currently in the project before we create the
414 # BRLayer objects
415 customrecipe = CustomImageRecipe.objects.filter(
416 name=t.target,
417 project=self).first()
418 if customrecipe:
419 ProjectLayer.objects.get_or_create(
420 project=self,
421 layercommit=customrecipe.layer_version,
422 optional=False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500423
424 for l in self.projectlayer_set.all().order_by("pk"):
425 commit = l.layercommit.get_vcs_reference()
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500426 logger.debug("Adding layer to build %s" %
427 l.layercommit.layer.name)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600428 BRLayer.objects.create(
429 req=br,
430 name=l.layercommit.layer.name,
431 giturl=l.layercommit.layer.vcs_url,
432 commit=commit,
433 dirpath=l.layercommit.dirpath,
434 layer_version=l.layercommit,
435 local_source_dir=l.layercommit.layer.local_source_dir
436 )
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500437
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500438 for v in self.projectvariable_set.all():
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500439 BRVariable.objects.create(req=br, name=v.name, value=v.value)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500440
441 try:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500442 br.build.machine = self.projectvariable_set.get(
443 name='MACHINE').value
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500444 br.build.save()
445 except ProjectVariable.DoesNotExist:
446 pass
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500447
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500448 br.save()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600449 signal_runbuilds()
450
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500451 except Exception:
452 # revert the build request creation since we're not done cleanly
453 br.delete()
454 raise
455 return br
456
457class Build(models.Model):
458 SUCCEEDED = 0
459 FAILED = 1
460 IN_PROGRESS = 2
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500461 CANCELLED = 3
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500462
463 BUILD_OUTCOME = (
464 (SUCCEEDED, 'Succeeded'),
465 (FAILED, 'Failed'),
466 (IN_PROGRESS, 'In Progress'),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500467 (CANCELLED, 'Cancelled'),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500468 )
469
470 search_allowed_fields = ['machine', 'cooker_log_path', "target__target", "target__target_image_file__file_name"]
471
472 project = models.ForeignKey(Project) # must have a project
473 machine = models.CharField(max_length=100)
474 distro = models.CharField(max_length=100)
475 distro_version = models.CharField(max_length=100)
476 started_on = models.DateTimeField()
477 completed_on = models.DateTimeField()
478 outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS)
479 cooker_log_path = models.CharField(max_length=500)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600480 build_name = models.CharField(max_length=100, default='')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500481 bitbake_version = models.CharField(max_length=50)
482
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600483 # number of recipes to parse for this build
484 recipes_to_parse = models.IntegerField(default=1)
485
486 # number of recipes parsed so far for this build
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500487 recipes_parsed = models.IntegerField(default=1)
488
489 # number of repos to clone for this build
490 repos_to_clone = models.IntegerField(default=1)
491
492 # number of repos cloned so far for this build (default off)
493 repos_cloned = models.IntegerField(default=1)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600494
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800495 # Hint on current progress item
496 progress_item = models.CharField(max_length=40)
497
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500498 @staticmethod
499 def get_recent(project=None):
500 """
501 Return recent builds as a list; if project is set, only return
502 builds for that project
503 """
504
505 builds = Build.objects.all()
506
507 if project:
508 builds = builds.filter(project=project)
509
510 finished_criteria = \
511 Q(outcome=Build.SUCCEEDED) | \
512 Q(outcome=Build.FAILED) | \
513 Q(outcome=Build.CANCELLED)
514
515 recent_builds = list(itertools.chain(
516 builds.filter(outcome=Build.IN_PROGRESS).order_by("-started_on"),
517 builds.filter(finished_criteria).order_by("-completed_on")[:3]
518 ))
519
520 # add percentage done property to each build; this is used
521 # to show build progress in mrb_section.html
522 for build in recent_builds:
523 build.percentDone = build.completeper()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600524 build.outcomeText = build.get_outcome_text()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500525
526 return recent_builds
527
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600528 def started(self):
529 """
530 As build variables are only added for a build when its BuildStarted event
531 is received, a build with no build variables is counted as
532 "in preparation" and not properly started yet. This method
533 will return False if a build has no build variables (it never properly
534 started), or True otherwise.
535
536 Note that this is a temporary workaround for the fact that we don't
537 have a fine-grained state variable on a build which would allow us
538 to record "in progress" (BuildStarted received) vs. "in preparation".
539 """
540 variables = Variable.objects.filter(build=self)
541 return len(variables) > 0
542
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500543 def completeper(self):
544 tf = Task.objects.filter(build = self)
545 tfc = tf.count()
546 if tfc > 0:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500547 completeper = tf.exclude(outcome=Task.OUTCOME_NA).count()*100 // tfc
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500548 else:
549 completeper = 0
550 return completeper
551
552 def eta(self):
553 eta = timezone.now()
554 completeper = self.completeper()
555 if self.completeper() > 0:
556 eta += ((eta - self.started_on)*(100-completeper))/completeper
557 return eta
558
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600559 def has_images(self):
560 """
561 Returns True if at least one of the targets for this build has an
562 image file associated with it, False otherwise
563 """
564 targets = Target.objects.filter(build_id=self.id)
565 has_images = False
566 for target in targets:
567 if target.has_images():
568 has_images = True
569 break
570 return has_images
571
572 def has_image_recipes(self):
573 """
574 Returns True if a build has any targets which were built from
575 image recipes.
576 """
577 image_recipes = self.get_image_recipes()
578 return len(image_recipes) > 0
579
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500580 def get_image_file_extensions(self):
581 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600582 Get string of file name extensions for images produced by this build;
583 note that this is the actual list of extensions stored on Target objects
584 for this build, and not the value of IMAGE_FSTYPES.
585
586 Returns comma-separated string, e.g. "vmdk, ext4"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500587 """
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500588 extensions = []
589
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600590 targets = Target.objects.filter(build_id = self.id)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500591 for target in targets:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600592 if not target.is_image:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500593 continue
594
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600595 target_image_files = Target_Image_File.objects.filter(
596 target_id=target.id)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500597
598 for target_image_file in target_image_files:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600599 extensions.append(target_image_file.suffix)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500600
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600601 extensions = list(set(extensions))
602 extensions.sort()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500603
604 return ', '.join(extensions)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500605
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600606 def get_image_fstypes(self):
607 """
608 Get the IMAGE_FSTYPES variable value for this build as a de-duplicated
609 list of image file suffixes.
610 """
611 image_fstypes = Variable.objects.get(
612 build=self, variable_name='IMAGE_FSTYPES').variable_value
613 return list(set(re.split(r' {1,}', image_fstypes)))
614
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500615 def get_sorted_target_list(self):
616 tgts = Target.objects.filter(build_id = self.id).order_by( 'target' );
617 return( tgts );
618
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500619 def get_recipes(self):
620 """
621 Get the recipes related to this build;
622 note that the related layer versions and layers are also prefetched
623 by this query, as this queryset can be sorted by these objects in the
624 build recipes view; prefetching them here removes the need
625 for another query in that view
626 """
627 layer_versions = Layer_Version.objects.filter(build=self)
628 criteria = Q(layer_version__id__in=layer_versions)
629 return Recipe.objects.filter(criteria) \
630 .select_related('layer_version', 'layer_version__layer')
631
632 def get_image_recipes(self):
633 """
634 Returns a list of image Recipes (custom and built-in) related to this
635 build, sorted by name; note that this has to be done in two steps, as
636 there's no way to get all the custom image recipes and image recipes
637 in one query
638 """
639 custom_image_recipes = self.get_custom_image_recipes()
640 custom_image_recipe_names = custom_image_recipes.values_list('name', flat=True)
641
642 not_custom_image_recipes = ~Q(name__in=custom_image_recipe_names) & \
643 Q(is_image=True)
644
645 built_image_recipes = self.get_recipes().filter(not_custom_image_recipes)
646
647 # append to the custom image recipes and sort
648 customisable_image_recipes = list(
649 itertools.chain(custom_image_recipes, built_image_recipes)
650 )
651
652 return sorted(customisable_image_recipes, key=lambda recipe: recipe.name)
653
654 def get_custom_image_recipes(self):
655 """
656 Returns a queryset of CustomImageRecipes related to this build,
657 sorted by name
658 """
659 built_recipe_names = self.get_recipes().values_list('name', flat=True)
660 criteria = Q(name__in=built_recipe_names) & Q(project=self.project)
661 queryset = CustomImageRecipe.objects.filter(criteria).order_by('name')
662 return queryset
663
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500664 def get_outcome_text(self):
665 return Build.BUILD_OUTCOME[int(self.outcome)][1]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500666
667 @property
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500668 def failed_tasks(self):
669 """ Get failed tasks for the build """
670 tasks = self.task_build.all()
671 return tasks.filter(order__gt=0, outcome=Task.OUTCOME_FAILED)
672
673 @property
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500674 def errors(self):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500675 return (self.logmessage_set.filter(level=LogMessage.ERROR) |
676 self.logmessage_set.filter(level=LogMessage.EXCEPTION) |
677 self.logmessage_set.filter(level=LogMessage.CRITICAL))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500678
679 @property
680 def warnings(self):
681 return self.logmessage_set.filter(level=LogMessage.WARNING)
682
683 @property
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500684 def timespent(self):
685 return self.completed_on - self.started_on
686
687 @property
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500688 def timespent_seconds(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500689 return self.timespent.total_seconds()
690
691 @property
692 def target_labels(self):
693 """
694 Sorted (a-z) "target1:task, target2, target3" etc. string for all
695 targets in this build
696 """
697 targets = self.target_set.all()
698 target_labels = [target.target +
699 (':' + target.task if target.task else '')
700 for target in targets]
701 target_labels.sort()
702
703 return target_labels
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500704
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600705 def get_buildrequest(self):
706 buildrequest = None
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500707 if hasattr(self, 'buildrequest'):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600708 buildrequest = self.buildrequest
709 return buildrequest
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500710
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600711 def is_queued(self):
712 from bldcontrol.models import BuildRequest
713 buildrequest = self.get_buildrequest()
714 if buildrequest:
715 return buildrequest.state == BuildRequest.REQ_QUEUED
716 else:
717 return False
718
719 def is_cancelling(self):
720 from bldcontrol.models import BuildRequest
721 buildrequest = self.get_buildrequest()
722 if buildrequest:
723 return self.outcome == Build.IN_PROGRESS and \
724 buildrequest.state == BuildRequest.REQ_CANCELLING
725 else:
726 return False
727
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500728 def is_cloning(self):
729 """
730 True if the build is still cloning repos
731 """
732 return self.outcome == Build.IN_PROGRESS and \
733 self.repos_cloned < self.repos_to_clone
734
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600735 def is_parsing(self):
736 """
737 True if the build is still parsing recipes
738 """
739 return self.outcome == Build.IN_PROGRESS and \
740 self.recipes_parsed < self.recipes_to_parse
741
742 def is_starting(self):
743 """
744 True if the build has no completed tasks yet and is still just starting
745 tasks.
746
747 Note that the mechanism for testing whether a Task is "done" is whether
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500748 its outcome field is set, as per the completeper() method.
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600749 """
750 return self.outcome == Build.IN_PROGRESS and \
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500751 self.task_build.exclude(outcome=Task.OUTCOME_NA).count() == 0
752
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600753
754 def get_state(self):
755 """
756 Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress',
757 'Cancelled' (Build outcomes); or 'Queued', 'Cancelling' (states
758 dependent on the BuildRequest state).
759
760 This works around the fact that we have BuildRequest states as well
761 as Build states, but really we just want to know the state of the build.
762 """
763 if self.is_cancelling():
764 return 'Cancelling';
765 elif self.is_queued():
766 return 'Queued'
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500767 elif self.is_cloning():
768 return 'Cloning'
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600769 elif self.is_parsing():
770 return 'Parsing'
771 elif self.is_starting():
772 return 'Starting'
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500773 else:
774 return self.get_outcome_text()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500775
776 def __str__(self):
777 return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()]))
778
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500779class ProjectTarget(models.Model):
780 project = models.ForeignKey(Project)
781 target = models.CharField(max_length=100)
782 task = models.CharField(max_length=100, null=True)
783
784class Target(models.Model):
785 search_allowed_fields = ['target', 'file_name']
786 build = models.ForeignKey(Build)
787 target = models.CharField(max_length=100)
788 task = models.CharField(max_length=100, null=True)
789 is_image = models.BooleanField(default = False)
790 image_size = models.IntegerField(default=0)
791 license_manifest_path = models.CharField(max_length=500, null=True)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600792 package_manifest_path = models.CharField(max_length=500, null=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500793
794 def package_count(self):
795 return Target_Installed_Package.objects.filter(target_id__exact=self.id).count()
796
797 def __unicode__(self):
798 return self.target
799
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600800 def get_similar_targets(self):
801 """
802 Get target sfor the same machine, task and target name
803 (e.g. 'core-image-minimal') from a successful build for this project
804 (but excluding this target).
805
806 Note that we only look for targets built by this project because
807 projects can have different configurations from each other, and put
808 their artifacts in different directories.
809
810 The possibility of error when retrieving candidate targets
811 is minimised by the fact that bitbake will rebuild artifacts if MACHINE
812 (or various other variables) change. In this case, there is no need to
813 clone artifacts from another target, as those artifacts will have
814 been re-generated for this target anyway.
815 """
816 query = ~Q(pk=self.pk) & \
817 Q(target=self.target) & \
818 Q(build__machine=self.build.machine) & \
819 Q(build__outcome=Build.SUCCEEDED) & \
820 Q(build__project=self.build.project)
821
822 return Target.objects.filter(query)
823
824 def get_similar_target_with_image_files(self):
825 """
826 Get the most recent similar target with Target_Image_Files associated
827 with it, for the purpose of cloning those files onto this target.
828 """
829 similar_target = None
830
831 candidates = self.get_similar_targets()
832 if candidates.count() == 0:
833 return similar_target
834
835 task_subquery = Q(task=self.task)
836
837 # we can look for a 'build' task if this task is a 'populate_sdk_ext'
838 # task, as the latter also creates images; and vice versa; note that
839 # 'build' targets can have their task set to '';
840 # also note that 'populate_sdk' does not produce image files
841 image_tasks = [
842 '', # aka 'build'
843 'build',
844 'image',
845 'populate_sdk_ext'
846 ]
847 if self.task in image_tasks:
848 task_subquery = Q(task__in=image_tasks)
849
850 # annotate with the count of files, to exclude any targets which
851 # don't have associated files
852 candidates = candidates.annotate(num_files=Count('target_image_file'))
853
854 query = task_subquery & Q(num_files__gt=0)
855
856 candidates = candidates.filter(query)
857
858 if candidates.count() > 0:
859 candidates.order_by('build__completed_on')
860 similar_target = candidates.last()
861
862 return similar_target
863
864 def get_similar_target_with_sdk_files(self):
865 """
866 Get the most recent similar target with TargetSDKFiles associated
867 with it, for the purpose of cloning those files onto this target.
868 """
869 similar_target = None
870
871 candidates = self.get_similar_targets()
872 if candidates.count() == 0:
873 return similar_target
874
875 # annotate with the count of files, to exclude any targets which
876 # don't have associated files
877 candidates = candidates.annotate(num_files=Count('targetsdkfile'))
878
879 query = Q(task=self.task) & Q(num_files__gt=0)
880
881 candidates = candidates.filter(query)
882
883 if candidates.count() > 0:
884 candidates.order_by('build__completed_on')
885 similar_target = candidates.last()
886
887 return similar_target
888
889 def clone_image_artifacts_from(self, target):
890 """
891 Make clones of the Target_Image_Files and TargetKernelFile objects
892 associated with Target target, then associate them with this target.
893
894 Note that for Target_Image_Files, we only want files from the previous
895 build whose suffix matches one of the suffixes defined in this
896 target's build's IMAGE_FSTYPES configuration variable. This prevents the
897 Target_Image_File object for an ext4 image being associated with a
898 target for a project which didn't produce an ext4 image (for example).
899
900 Also sets the license_manifest_path and package_manifest_path
901 of this target to the same path as that of target being cloned from, as
902 the manifests are also build artifacts but are treated differently.
903 """
904
905 image_fstypes = self.build.get_image_fstypes()
906
907 # filter out any image files whose suffixes aren't in the
908 # IMAGE_FSTYPES suffixes variable for this target's build
909 image_files = [target_image_file \
910 for target_image_file in target.target_image_file_set.all() \
911 if target_image_file.suffix in image_fstypes]
912
913 for image_file in image_files:
914 image_file.pk = None
915 image_file.target = self
916 image_file.save()
917
918 kernel_files = target.targetkernelfile_set.all()
919 for kernel_file in kernel_files:
920 kernel_file.pk = None
921 kernel_file.target = self
922 kernel_file.save()
923
924 self.license_manifest_path = target.license_manifest_path
925 self.package_manifest_path = target.package_manifest_path
926 self.save()
927
928 def clone_sdk_artifacts_from(self, target):
929 """
930 Clone TargetSDKFile objects from target and associate them with this
931 target.
932 """
933 sdk_files = target.targetsdkfile_set.all()
934 for sdk_file in sdk_files:
935 sdk_file.pk = None
936 sdk_file.target = self
937 sdk_file.save()
938
939 def has_images(self):
940 """
941 Returns True if this target has one or more image files attached to it.
942 """
943 return self.target_image_file_set.all().count() > 0
944
945# kernel artifacts for a target: bzImage and modules*
946class TargetKernelFile(models.Model):
947 target = models.ForeignKey(Target)
948 file_name = models.FilePathField()
949 file_size = models.IntegerField()
950
951 @property
952 def basename(self):
953 return os.path.basename(self.file_name)
954
955# SDK artifacts for a target: sh and manifest files
956class TargetSDKFile(models.Model):
957 target = models.ForeignKey(Target)
958 file_name = models.FilePathField()
959 file_size = models.IntegerField()
960
961 @property
962 def basename(self):
963 return os.path.basename(self.file_name)
964
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500965class Target_Image_File(models.Model):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500966 # valid suffixes for image files produced by a build
967 SUFFIXES = {
968 'btrfs', 'cpio', 'cpio.gz', 'cpio.lz4', 'cpio.lzma', 'cpio.xz',
969 'cramfs', 'elf', 'ext2', 'ext2.bz2', 'ext2.gz', 'ext2.lzma', 'ext4',
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600970 'ext4.gz', 'ext3', 'ext3.gz', 'hdddirect', 'hddimg', 'iso', 'jffs2',
971 'jffs2.sum', 'multiubi', 'qcow2', 'squashfs', 'squashfs-lzo',
972 'squashfs-xz', 'tar', 'tar.bz2', 'tar.gz', 'tar.lz4', 'tar.xz', 'ubi',
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500973 'ubifs', 'vdi', 'vmdk', 'wic', 'wic.bmap', 'wic.bz2', 'wic.gz', 'wic.lzma'
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500974 }
975
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500976 target = models.ForeignKey(Target)
977 file_name = models.FilePathField(max_length=254)
978 file_size = models.IntegerField()
979
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500980 @property
981 def suffix(self):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600982 """
983 Suffix for image file, minus leading "."
984 """
985 for suffix in Target_Image_File.SUFFIXES:
986 if self.file_name.endswith(suffix):
987 return suffix
988
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500989 filename, suffix = os.path.splitext(self.file_name)
990 suffix = suffix.lstrip('.')
991 return suffix
992
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500993class Target_File(models.Model):
994 ITYPE_REGULAR = 1
995 ITYPE_DIRECTORY = 2
996 ITYPE_SYMLINK = 3
997 ITYPE_SOCKET = 4
998 ITYPE_FIFO = 5
999 ITYPE_CHARACTER = 6
1000 ITYPE_BLOCK = 7
1001 ITYPES = ( (ITYPE_REGULAR ,'regular'),
1002 ( ITYPE_DIRECTORY ,'directory'),
1003 ( ITYPE_SYMLINK ,'symlink'),
1004 ( ITYPE_SOCKET ,'socket'),
1005 ( ITYPE_FIFO ,'fifo'),
1006 ( ITYPE_CHARACTER ,'character'),
1007 ( ITYPE_BLOCK ,'block'),
1008 )
1009
1010 target = models.ForeignKey(Target)
1011 path = models.FilePathField()
1012 size = models.IntegerField()
1013 inodetype = models.IntegerField(choices = ITYPES)
1014 permission = models.CharField(max_length=16)
1015 owner = models.CharField(max_length=128)
1016 group = models.CharField(max_length=128)
1017 directory = models.ForeignKey('Target_File', related_name="directory_set", null=True)
1018 sym_target = models.ForeignKey('Target_File', related_name="symlink_set", null=True)
1019
1020
1021class Task(models.Model):
1022
1023 SSTATE_NA = 0
1024 SSTATE_MISS = 1
1025 SSTATE_FAILED = 2
1026 SSTATE_RESTORED = 3
1027
1028 SSTATE_RESULT = (
1029 (SSTATE_NA, 'Not Applicable'), # For rest of tasks, but they still need checking.
1030 (SSTATE_MISS, 'File not in cache'), # the sstate object was not found
1031 (SSTATE_FAILED, 'Failed'), # there was a pkg, but the script failed
1032 (SSTATE_RESTORED, 'Succeeded'), # successfully restored
1033 )
1034
1035 CODING_NA = 0
1036 CODING_PYTHON = 2
1037 CODING_SHELL = 3
1038
1039 TASK_CODING = (
1040 (CODING_NA, 'N/A'),
1041 (CODING_PYTHON, 'Python'),
1042 (CODING_SHELL, 'Shell'),
1043 )
1044
1045 OUTCOME_NA = -1
1046 OUTCOME_SUCCESS = 0
1047 OUTCOME_COVERED = 1
1048 OUTCOME_CACHED = 2
1049 OUTCOME_PREBUILT = 3
1050 OUTCOME_FAILED = 4
1051 OUTCOME_EMPTY = 5
1052
1053 TASK_OUTCOME = (
1054 (OUTCOME_NA, 'Not Available'),
1055 (OUTCOME_SUCCESS, 'Succeeded'),
1056 (OUTCOME_COVERED, 'Covered'),
1057 (OUTCOME_CACHED, 'Cached'),
1058 (OUTCOME_PREBUILT, 'Prebuilt'),
1059 (OUTCOME_FAILED, 'Failed'),
1060 (OUTCOME_EMPTY, 'Empty'),
1061 )
1062
1063 TASK_OUTCOME_HELP = (
1064 (OUTCOME_SUCCESS, 'This task successfully completed'),
1065 (OUTCOME_COVERED, 'This task did not run because its output is provided by another task'),
1066 (OUTCOME_CACHED, 'This task restored output from the sstate-cache directory or mirrors'),
1067 (OUTCOME_PREBUILT, 'This task did not run because its outcome was reused from a previous build'),
1068 (OUTCOME_FAILED, 'This task did not complete'),
1069 (OUTCOME_EMPTY, 'This task has no executable content'),
1070 (OUTCOME_NA, ''),
1071 )
1072
1073 search_allowed_fields = [ "recipe__name", "recipe__version", "task_name", "logfile" ]
1074
1075 def __init__(self, *args, **kwargs):
1076 super(Task, self).__init__(*args, **kwargs)
1077 try:
1078 self._helptext = HelpText.objects.get(key=self.task_name, area=HelpText.VARIABLE, build=self.build).text
1079 except HelpText.DoesNotExist:
1080 self._helptext = None
1081
1082 def get_related_setscene(self):
1083 return Task.objects.filter(task_executed=True, build = self.build, recipe = self.recipe, task_name=self.task_name+"_setscene")
1084
1085 def get_outcome_text(self):
1086 return Task.TASK_OUTCOME[int(self.outcome) + 1][1]
1087
1088 def get_outcome_help(self):
1089 return Task.TASK_OUTCOME_HELP[int(self.outcome)][1]
1090
1091 def get_sstate_text(self):
1092 if self.sstate_result==Task.SSTATE_NA:
1093 return ''
1094 else:
1095 return Task.SSTATE_RESULT[int(self.sstate_result)][1]
1096
1097 def get_executed_display(self):
1098 if self.task_executed:
1099 return "Executed"
1100 return "Not Executed"
1101
1102 def get_description(self):
1103 return self._helptext
1104
1105 build = models.ForeignKey(Build, related_name='task_build')
1106 order = models.IntegerField(null=True)
1107 task_executed = models.BooleanField(default=False) # True means Executed, False means Not/Executed
1108 outcome = models.IntegerField(choices=TASK_OUTCOME, default=OUTCOME_NA)
1109 sstate_checksum = models.CharField(max_length=100, blank=True)
1110 path_to_sstate_obj = models.FilePathField(max_length=500, blank=True)
1111 recipe = models.ForeignKey('Recipe', related_name='tasks')
1112 task_name = models.CharField(max_length=100)
1113 source_url = models.FilePathField(max_length=255, blank=True)
1114 work_directory = models.FilePathField(max_length=255, blank=True)
1115 script_type = models.IntegerField(choices=TASK_CODING, default=CODING_NA)
1116 line_number = models.IntegerField(default=0)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001117
1118 # start/end times
1119 started = models.DateTimeField(null=True)
1120 ended = models.DateTimeField(null=True)
1121
1122 # in seconds; this is stored to enable sorting
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001123 elapsed_time = models.DecimalField(max_digits=8, decimal_places=2, null=True)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001124
1125 # in bytes; note that disk_io is stored to enable sorting
1126 disk_io = models.IntegerField(null=True)
1127 disk_io_read = models.IntegerField(null=True)
1128 disk_io_write = models.IntegerField(null=True)
1129
1130 # in seconds
1131 cpu_time_user = models.DecimalField(max_digits=8, decimal_places=2, null=True)
1132 cpu_time_system = models.DecimalField(max_digits=8, decimal_places=2, null=True)
1133
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001134 sstate_result = models.IntegerField(choices=SSTATE_RESULT, default=SSTATE_NA)
1135 message = models.CharField(max_length=240)
1136 logfile = models.FilePathField(max_length=255, blank=True)
1137
1138 outcome_text = property(get_outcome_text)
1139 sstate_text = property(get_sstate_text)
1140
1141 def __unicode__(self):
1142 return "%d(%d) %s:%s" % (self.pk, self.build.pk, self.recipe.name, self.task_name)
1143
1144 class Meta:
1145 ordering = ('order', 'recipe' ,)
1146 unique_together = ('build', 'recipe', 'task_name', )
1147
1148
1149class Task_Dependency(models.Model):
1150 task = models.ForeignKey(Task, related_name='task_dependencies_task')
1151 depends_on = models.ForeignKey(Task, related_name='task_dependencies_depends')
1152
1153class Package(models.Model):
1154 search_allowed_fields = ['name', 'version', 'revision', 'recipe__name', 'recipe__version', 'recipe__license', 'recipe__layer_version__layer__name', 'recipe__layer_version__branch', 'recipe__layer_version__commit', 'recipe__layer_version__local_path', 'installed_name']
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001155 build = models.ForeignKey('Build', null=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001156 recipe = models.ForeignKey('Recipe', null=True)
1157 name = models.CharField(max_length=100)
1158 installed_name = models.CharField(max_length=100, default='')
1159 version = models.CharField(max_length=100, blank=True)
1160 revision = models.CharField(max_length=32, blank=True)
1161 summary = models.TextField(blank=True)
1162 description = models.TextField(blank=True)
1163 size = models.IntegerField(default=0)
1164 installed_size = models.IntegerField(default=0)
1165 section = models.CharField(max_length=80, blank=True)
1166 license = models.CharField(max_length=80, blank=True)
1167
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001168 @property
1169 def is_locale_package(self):
1170 """ Returns True if this package is identifiable as a locale package """
1171 if self.name.find('locale') != -1:
1172 return True
1173 return False
1174
1175 @property
1176 def is_packagegroup(self):
1177 """ Returns True is this package is identifiable as a packagegroup """
1178 if self.name.find('packagegroup') != -1:
1179 return True
1180 return False
1181
1182class CustomImagePackage(Package):
1183 # CustomImageRecipe fields to track pacakges appended,
1184 # included and excluded from a CustomImageRecipe
1185 recipe_includes = models.ManyToManyField('CustomImageRecipe',
1186 related_name='includes_set')
1187 recipe_excludes = models.ManyToManyField('CustomImageRecipe',
1188 related_name='excludes_set')
1189 recipe_appends = models.ManyToManyField('CustomImageRecipe',
1190 related_name='appends_set')
1191
1192
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001193class Package_DependencyManager(models.Manager):
1194 use_for_related_fields = True
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001195 TARGET_LATEST = "use-latest-target-for-target"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001196
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001197 def get_queryset(self):
1198 return super(Package_DependencyManager, self).get_queryset().exclude(package_id = F('depends_on__id'))
1199
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001200 def for_target_or_none(self, target):
1201 """ filter the dependencies to be displayed by the supplied target
1202 if no dependences are found for the target then try None as the target
1203 which will return the dependences calculated without the context of a
1204 target e.g. non image recipes.
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001205
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001206 returns: { size, packages }
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001207 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001208 package_dependencies = self.all_depends().order_by('depends_on__name')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001209
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001210 if target is self.TARGET_LATEST:
1211 installed_deps =\
1212 package_dependencies.filter(~Q(target__target=None))
1213 else:
1214 installed_deps =\
1215 package_dependencies.filter(Q(target__target=target))
1216
1217 packages_list = None
1218 total_size = 0
1219
1220 # If we have installed depdencies for this package and target then use
1221 # these to display
1222 if installed_deps.count() > 0:
1223 packages_list = installed_deps
1224 total_size = installed_deps.aggregate(
1225 Sum('depends_on__size'))['depends_on__size__sum']
1226 else:
1227 new_list = []
1228 package_names = []
1229
1230 # Find dependencies for the package that we know about even if
1231 # it's not installed on a target e.g. from a non-image recipe
1232 for p in package_dependencies.filter(Q(target=None)):
1233 if p.depends_on.name in package_names:
1234 continue
1235 else:
1236 package_names.append(p.depends_on.name)
1237 new_list.append(p.pk)
1238 # while we're here we may as well total up the size to
1239 # avoid iterating again
1240 total_size += p.depends_on.size
1241
1242 # We want to return a queryset here for consistency so pick the
1243 # deps from the new_list
1244 packages_list = package_dependencies.filter(Q(pk__in=new_list))
1245
1246 return {'packages': packages_list,
1247 'size': total_size}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001248
1249 def all_depends(self):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001250 """ Returns just the depends packages and not any other dep_type
1251 Note that this is for any target
1252 """
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001253 return self.filter(Q(dep_type=Package_Dependency.TYPE_RDEPENDS) |
1254 Q(dep_type=Package_Dependency.TYPE_TRDEPENDS))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001255
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001256
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001257class Package_Dependency(models.Model):
1258 TYPE_RDEPENDS = 0
1259 TYPE_TRDEPENDS = 1
1260 TYPE_RRECOMMENDS = 2
1261 TYPE_TRECOMMENDS = 3
1262 TYPE_RSUGGESTS = 4
1263 TYPE_RPROVIDES = 5
1264 TYPE_RREPLACES = 6
1265 TYPE_RCONFLICTS = 7
1266 ' TODO: bpackage should be changed to remove the DEPENDS_TYPE access '
1267 DEPENDS_TYPE = (
1268 (TYPE_RDEPENDS, "depends"),
1269 (TYPE_TRDEPENDS, "depends"),
1270 (TYPE_TRECOMMENDS, "recommends"),
1271 (TYPE_RRECOMMENDS, "recommends"),
1272 (TYPE_RSUGGESTS, "suggests"),
1273 (TYPE_RPROVIDES, "provides"),
1274 (TYPE_RREPLACES, "replaces"),
1275 (TYPE_RCONFLICTS, "conflicts"),
1276 )
1277 """ Indexed by dep_type, in view order, key for short name and help
1278 description which when viewed will be printf'd with the
1279 package name.
1280 """
1281 DEPENDS_DICT = {
1282 TYPE_RDEPENDS : ("depends", "%s is required to run %s"),
1283 TYPE_TRDEPENDS : ("depends", "%s is required to run %s"),
1284 TYPE_TRECOMMENDS : ("recommends", "%s extends the usability of %s"),
1285 TYPE_RRECOMMENDS : ("recommends", "%s extends the usability of %s"),
1286 TYPE_RSUGGESTS : ("suggests", "%s is suggested for installation with %s"),
1287 TYPE_RPROVIDES : ("provides", "%s is provided by %s"),
1288 TYPE_RREPLACES : ("replaces", "%s is replaced by %s"),
1289 TYPE_RCONFLICTS : ("conflicts", "%s conflicts with %s, which will not be installed if this package is not first removed"),
1290 }
1291
1292 package = models.ForeignKey(Package, related_name='package_dependencies_source')
1293 depends_on = models.ForeignKey(Package, related_name='package_dependencies_target') # soft dependency
1294 dep_type = models.IntegerField(choices=DEPENDS_TYPE)
1295 target = models.ForeignKey(Target, null=True)
1296 objects = Package_DependencyManager()
1297
1298class Target_Installed_Package(models.Model):
1299 target = models.ForeignKey(Target)
1300 package = models.ForeignKey(Package, related_name='buildtargetlist_package')
1301
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001302
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001303class Package_File(models.Model):
1304 package = models.ForeignKey(Package, related_name='buildfilelist_package')
1305 path = models.FilePathField(max_length=255, blank=True)
1306 size = models.IntegerField()
1307
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001308
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001309class Recipe(models.Model):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001310 search_allowed_fields = ['name', 'version', 'file_path', 'section',
1311 'summary', 'description', 'license',
1312 'layer_version__layer__name',
1313 'layer_version__branch', 'layer_version__commit',
1314 'layer_version__local_path',
1315 'layer_version__layer_source']
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001316
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001317 up_date = models.DateTimeField(null=True, default=None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001318
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001319 name = models.CharField(max_length=100, blank=True)
1320 version = models.CharField(max_length=100, blank=True)
1321 layer_version = models.ForeignKey('Layer_Version',
1322 related_name='recipe_layer_version')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001323 summary = models.TextField(blank=True)
1324 description = models.TextField(blank=True)
1325 section = models.CharField(max_length=100, blank=True)
1326 license = models.CharField(max_length=200, blank=True)
1327 homepage = models.URLField(blank=True)
1328 bugtracker = models.URLField(blank=True)
1329 file_path = models.FilePathField(max_length=255)
1330 pathflags = models.CharField(max_length=200, blank=True)
1331 is_image = models.BooleanField(default=False)
1332
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001333 def __unicode__(self):
1334 return "Recipe " + self.name + ":" + self.version
1335
1336 def get_vcs_recipe_file_link_url(self):
1337 return self.layer_version.get_vcs_file_link_url(self.file_path)
1338
1339 def get_description_or_summary(self):
1340 if self.description:
1341 return self.description
1342 elif self.summary:
1343 return self.summary
1344 else:
1345 return ""
1346
1347 class Meta:
1348 unique_together = (("layer_version", "file_path", "pathflags"), )
1349
1350
1351class Recipe_DependencyManager(models.Manager):
1352 use_for_related_fields = True
1353
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001354 def get_queryset(self):
1355 return super(Recipe_DependencyManager, self).get_queryset().exclude(recipe_id = F('depends_on__id'))
1356
1357class Provides(models.Model):
1358 name = models.CharField(max_length=100)
1359 recipe = models.ForeignKey(Recipe)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001360
1361class Recipe_Dependency(models.Model):
1362 TYPE_DEPENDS = 0
1363 TYPE_RDEPENDS = 1
1364
1365 DEPENDS_TYPE = (
1366 (TYPE_DEPENDS, "depends"),
1367 (TYPE_RDEPENDS, "rdepends"),
1368 )
1369 recipe = models.ForeignKey(Recipe, related_name='r_dependencies_recipe')
1370 depends_on = models.ForeignKey(Recipe, related_name='r_dependencies_depends')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001371 via = models.ForeignKey(Provides, null=True, default=None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001372 dep_type = models.IntegerField(choices=DEPENDS_TYPE)
1373 objects = Recipe_DependencyManager()
1374
1375
1376class Machine(models.Model):
1377 search_allowed_fields = ["name", "description", "layer_version__layer__name"]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001378 up_date = models.DateTimeField(null = True, default = None)
1379
1380 layer_version = models.ForeignKey('Layer_Version')
1381 name = models.CharField(max_length=255)
1382 description = models.CharField(max_length=255)
1383
1384 def get_vcs_machine_file_link_url(self):
1385 path = 'conf/machine/'+self.name+'.conf'
1386
1387 return self.layer_version.get_vcs_file_link_url(path)
1388
1389 def __unicode__(self):
1390 return "Machine " + self.name + "(" + self.description + ")"
1391
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001392
1393
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001394
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001395
1396class BitbakeVersion(models.Model):
1397
1398 name = models.CharField(max_length=32, unique = True)
1399 giturl = GitURLField()
1400 branch = models.CharField(max_length=32)
1401 dirpath = models.CharField(max_length=255)
1402
1403 def __unicode__(self):
1404 return "%s (Branch: %s)" % (self.name, self.branch)
1405
1406
1407class Release(models.Model):
1408 """ A release is a project template, used to pre-populate Project settings with a configuration set """
1409 name = models.CharField(max_length=32, unique = True)
1410 description = models.CharField(max_length=255)
1411 bitbake_version = models.ForeignKey(BitbakeVersion)
1412 branch_name = models.CharField(max_length=50, default = "")
1413 helptext = models.TextField(null=True)
1414
1415 def __unicode__(self):
1416 return "%s (%s)" % (self.name, self.branch_name)
1417
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001418 def __str__(self):
1419 return self.name
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001420
1421class ReleaseDefaultLayer(models.Model):
1422 release = models.ForeignKey(Release)
1423 layer_name = models.CharField(max_length=100, default="")
1424
1425
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001426class LayerSource(object):
1427 """ Where the layer metadata came from """
1428 TYPE_LOCAL = 0
1429 TYPE_LAYERINDEX = 1
1430 TYPE_IMPORTED = 2
1431 TYPE_BUILD = 3
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001432
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001433 SOURCE_TYPE = (
1434 (TYPE_LOCAL, "local"),
1435 (TYPE_LAYERINDEX, "layerindex"),
1436 (TYPE_IMPORTED, "imported"),
1437 (TYPE_BUILD, "build"),
1438 )
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001439
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001440 def types_dict():
1441 """ Turn the TYPES enums into a simple dictionary """
1442 dictionary = {}
1443 for key in LayerSource.__dict__:
1444 if "TYPE" in key:
1445 dictionary[key] = getattr(LayerSource, key)
1446 return dictionary
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001447
1448
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001449class Layer(models.Model):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001450
1451 up_date = models.DateTimeField(null=True, default=timezone.now)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001452
1453 name = models.CharField(max_length=100)
1454 layer_index_url = models.URLField()
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001455 vcs_url = GitURLField(default=None, null=True)
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001456 local_source_dir = models.TextField(null=True, default=None)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001457 vcs_web_url = models.URLField(null=True, default=None)
1458 vcs_web_tree_base_url = models.URLField(null=True, default=None)
1459 vcs_web_file_base_url = models.URLField(null=True, default=None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001460
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001461 summary = models.TextField(help_text='One-line description of the layer',
1462 null=True, default=None)
1463 description = models.TextField(null=True, default=None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001464
1465 def __unicode__(self):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001466 return "%s / %s " % (self.name, self.summary)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001467
1468
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001469class Layer_Version(models.Model):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001470 """
1471 A Layer_Version either belongs to a single project or no project
1472 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001473 search_allowed_fields = ["layer__name", "layer__summary",
1474 "layer__description", "layer__vcs_url",
1475 "dirpath", "release__name", "commit", "branch"]
1476
1477 build = models.ForeignKey(Build, related_name='layer_version_build',
1478 default=None, null=True)
1479
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001480 layer = models.ForeignKey(Layer, related_name='layer_version_layer')
1481
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001482 layer_source = models.IntegerField(choices=LayerSource.SOURCE_TYPE,
1483 default=0)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001484
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001485 up_date = models.DateTimeField(null=True, default=timezone.now)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001486
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001487 # To which metadata release does this layer version belong to
1488 release = models.ForeignKey(Release, null=True, default=None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001489
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001490 branch = models.CharField(max_length=80)
1491 commit = models.CharField(max_length=100)
1492 # If the layer is in a subdir
1493 dirpath = models.CharField(max_length=255, null=True, default=None)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001494
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001495 # if -1, this is a default layer
1496 priority = models.IntegerField(default=0)
1497
1498 # where this layer exists on the filesystem
1499 local_path = models.FilePathField(max_length=1024, default="/")
1500
1501 # Set if this layer is restricted to a particular project
1502 project = models.ForeignKey('Project', null=True, default=None)
1503
1504 # code lifted, with adaptations, from the layerindex-web application
1505 # https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001506 def _handle_url_path(self, base_url, path):
1507 import re, posixpath
1508 if base_url:
1509 if self.dirpath:
1510 if path:
1511 extra_path = self.dirpath + '/' + path
1512 # Normalise out ../ in path for usage URL
1513 extra_path = posixpath.normpath(extra_path)
1514 # Minor workaround to handle case where subdirectory has been added between branches
1515 # (should probably support usage URL per branch to handle this... sigh...)
1516 if extra_path.startswith('../'):
1517 extra_path = extra_path[3:]
1518 else:
1519 extra_path = self.dirpath
1520 else:
1521 extra_path = path
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001522 branchname = self.release.name
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001523 url = base_url.replace('%branch%', branchname)
1524
1525 # If there's a % in the path (e.g. a wildcard bbappend) we need to encode it
1526 if extra_path:
1527 extra_path = extra_path.replace('%', '%25')
1528
1529 if '%path%' in base_url:
1530 if extra_path:
1531 url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '\\1', url)
1532 else:
1533 url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '', url)
1534 return url.replace('%path%', extra_path)
1535 else:
1536 return url + extra_path
1537 return None
1538
1539 def get_vcs_link_url(self):
1540 if self.layer.vcs_web_url is None:
1541 return None
1542 return self.layer.vcs_web_url
1543
1544 def get_vcs_file_link_url(self, file_path=""):
1545 if self.layer.vcs_web_file_base_url is None:
1546 return None
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001547 return self._handle_url_path(self.layer.vcs_web_file_base_url,
1548 file_path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001549
1550 def get_vcs_dirpath_link_url(self):
1551 if self.layer.vcs_web_tree_base_url is None:
1552 return None
1553 return self._handle_url_path(self.layer.vcs_web_tree_base_url, '')
1554
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001555 def get_vcs_reference(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001556 if self.commit is not None and len(self.commit) > 0:
1557 return self.commit
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001558 if self.branch is not None and len(self.branch) > 0:
1559 return self.branch
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001560 if self.release is not None:
1561 return self.release.name
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001562 return 'N/A'
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001563
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001564 def get_detailspage_url(self, project_id=None):
1565 """ returns the url to the layer details page uses own project
1566 field if project_id is not specified """
1567
1568 if project_id is None:
1569 project_id = self.project.pk
1570
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001571 return reverse('layerdetails', args=(project_id, self.pk))
1572
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001573 def get_alldeps(self, project_id):
1574 """Get full list of unique layer dependencies."""
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001575 def gen_layerdeps(lver, project, depth):
1576 if depth == 0:
1577 return
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001578 for ldep in lver.dependencies.all():
1579 yield ldep.depends_on
1580 # get next level of deps recursively calling gen_layerdeps
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001581 for subdep in gen_layerdeps(ldep.depends_on, project, depth-1):
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001582 yield subdep
1583
1584 project = Project.objects.get(pk=project_id)
1585 result = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001586 projectlvers = [player.layercommit for player in
1587 project.projectlayer_set.all()]
1588 # protect against infinite layer dependency loops
1589 maxdepth = 20
1590 for dep in gen_layerdeps(self, project, maxdepth):
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001591 # filter out duplicates and layers already belonging to the project
1592 if dep not in result + projectlvers:
1593 result.append(dep)
1594
1595 return sorted(result, key=lambda x: x.layer.name)
1596
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001597 def __unicode__(self):
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001598 return ("id %d belongs to layer: %s" % (self.pk, self.layer.name))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001599
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001600 def __str__(self):
1601 if self.release:
1602 release = self.release.name
1603 else:
1604 release = "No release set"
1605
1606 return "%d %s (%s)" % (self.pk, self.layer.name, release)
1607
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001608
1609class LayerVersionDependency(models.Model):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001610
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001611 layer_version = models.ForeignKey(Layer_Version,
1612 related_name="dependencies")
1613 depends_on = models.ForeignKey(Layer_Version,
1614 related_name="dependees")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001615
1616class ProjectLayer(models.Model):
1617 project = models.ForeignKey(Project)
1618 layercommit = models.ForeignKey(Layer_Version, null=True)
1619 optional = models.BooleanField(default = True)
1620
1621 def __unicode__(self):
1622 return "%s, %s" % (self.project.name, self.layercommit)
1623
1624 class Meta:
1625 unique_together = (("project", "layercommit"),)
1626
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001627class CustomImageRecipe(Recipe):
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001628
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001629 # CustomImageRecipe's belong to layers called:
1630 LAYER_NAME = "toaster-custom-images"
1631
1632 search_allowed_fields = ['name']
1633 base_recipe = models.ForeignKey(Recipe, related_name='based_on_recipe')
1634 project = models.ForeignKey(Project)
1635 last_updated = models.DateTimeField(null=True, default=None)
1636
1637 def get_last_successful_built_target(self):
1638 """ Return the last successful built target object if one exists
1639 otherwise return None """
1640 return Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
1641 Q(build__project=self.project) &
1642 Q(target=self.name)).last()
1643
1644 def update_package_list(self):
1645 """ Update the package list from the last good build of this
1646 CustomImageRecipe
1647 """
1648 # Check if we're aldready up-to-date or not
1649 target = self.get_last_successful_built_target()
1650 if target == None:
1651 # So we've never actually built this Custom recipe but what about
1652 # the recipe it's based on?
1653 target = \
1654 Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
1655 Q(build__project=self.project) &
1656 Q(target=self.base_recipe.name)).last()
1657 if target == None:
1658 return
1659
1660 if target.build.completed_on == self.last_updated:
1661 return
1662
1663 self.includes_set.clear()
1664
1665 excludes_list = self.excludes_set.values_list('name', flat=True)
1666 appends_list = self.appends_set.values_list('name', flat=True)
1667
1668 built_packages_list = \
1669 target.target_installed_package_set.values_list('package__name',
1670 flat=True)
1671 for built_package in built_packages_list:
1672 # Is the built package in the custom packages list?
1673 if built_package in excludes_list:
1674 continue
1675
1676 if built_package in appends_list:
1677 continue
1678
1679 cust_img_p = \
1680 CustomImagePackage.objects.get(name=built_package)
1681 self.includes_set.add(cust_img_p)
1682
1683
1684 self.last_updated = target.build.completed_on
1685 self.save()
1686
1687 def get_all_packages(self):
1688 """Get the included packages and any appended packages"""
1689 self.update_package_list()
1690
1691 return CustomImagePackage.objects.filter((Q(recipe_appends=self) |
1692 Q(recipe_includes=self)) &
1693 ~Q(recipe_excludes=self))
1694
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001695 def get_base_recipe_file(self):
1696 """Get the base recipe file path if it exists on the file system"""
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001697 path_schema_one = "%s/%s" % (self.base_recipe.layer_version.local_path,
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001698 self.base_recipe.file_path)
1699
1700 path_schema_two = self.base_recipe.file_path
1701
Brad Bishop5dd7cbb2018-09-05 22:26:40 -07001702 path_schema_three = "%s/%s" % (self.base_recipe.layer_version.layer.local_source_dir,
1703 self.base_recipe.file_path)
1704
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001705 if os.path.exists(path_schema_one):
1706 return path_schema_one
1707
1708 # The path may now be the full path if the recipe has been built
1709 if os.path.exists(path_schema_two):
1710 return path_schema_two
1711
Brad Bishop5dd7cbb2018-09-05 22:26:40 -07001712 # Or a local path if all layers are local
1713 if os.path.exists(path_schema_three):
1714 return path_schema_three
1715
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001716 return None
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001717
1718 def generate_recipe_file_contents(self):
1719 """Generate the contents for the recipe file."""
1720 # If we have no excluded packages we only need to _append
1721 if self.excludes_set.count() == 0:
1722 packages_conf = "IMAGE_INSTALL_append = \" "
1723
1724 for pkg in self.appends_set.all():
1725 packages_conf += pkg.name+' '
1726 else:
1727 packages_conf = "IMAGE_FEATURES =\"\"\nIMAGE_INSTALL = \""
1728 # We add all the known packages to be built by this recipe apart
1729 # from locale packages which are are controlled with IMAGE_LINGUAS.
1730 for pkg in self.get_all_packages().exclude(
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001731 name__icontains="locale"):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001732 packages_conf += pkg.name+' '
1733
1734 packages_conf += "\""
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001735
1736 base_recipe_path = self.get_base_recipe_file()
1737 if base_recipe_path:
1738 base_recipe = open(base_recipe_path, 'r').read()
1739 else:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001740 # Pass back None to trigger error message to user
1741 return None
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001742
1743 # Add a special case for when the recipe we have based a custom image
1744 # recipe on requires another recipe.
1745 # For example:
1746 # "require core-image-minimal.bb" is changed to:
1747 # "require recipes-core/images/core-image-minimal.bb"
1748
1749 req_search = re.search(r'(require\s+)(.+\.bb\s*$)',
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001750 base_recipe,
1751 re.MULTILINE)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001752 if req_search:
1753 require_filename = req_search.group(2).strip()
1754
1755 corrected_location = Recipe.objects.filter(
1756 Q(layer_version=self.base_recipe.layer_version) &
1757 Q(file_path__icontains=require_filename)).last().file_path
1758
1759 new_require_line = "require %s" % corrected_location
1760
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001761 base_recipe = base_recipe.replace(req_search.group(0),
1762 new_require_line)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001763
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001764 info = {
1765 "date": timezone.now().strftime("%Y-%m-%d %H:%M:%S"),
1766 "base_recipe": base_recipe,
1767 "recipe_name": self.name,
1768 "base_recipe_name": self.base_recipe.name,
1769 "license": self.license,
1770 "summary": self.summary,
1771 "description": self.description,
1772 "packages_conf": packages_conf.strip()
1773 }
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001774
1775 recipe_contents = ("# Original recipe %(base_recipe_name)s \n"
1776 "%(base_recipe)s\n\n"
1777 "# Recipe %(recipe_name)s \n"
1778 "# Customisation Generated by Toaster on %(date)s\n"
1779 "SUMMARY = \"%(summary)s\"\n"
1780 "DESCRIPTION = \"%(description)s\"\n"
1781 "LICENSE = \"%(license)s\"\n"
1782 "%(packages_conf)s") % info
1783
1784 return recipe_contents
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001785
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001786class ProjectVariable(models.Model):
1787 project = models.ForeignKey(Project)
1788 name = models.CharField(max_length=100)
1789 value = models.TextField(blank = True)
1790
1791class Variable(models.Model):
1792 search_allowed_fields = ['variable_name', 'variable_value',
1793 'vhistory__file_name', "description"]
1794 build = models.ForeignKey(Build, related_name='variable_build')
1795 variable_name = models.CharField(max_length=100)
1796 variable_value = models.TextField(blank=True)
1797 changed = models.BooleanField(default=False)
1798 human_readable_name = models.CharField(max_length=200)
1799 description = models.TextField(blank=True)
1800
1801class VariableHistory(models.Model):
1802 variable = models.ForeignKey(Variable, related_name='vhistory')
1803 value = models.TextField(blank=True)
1804 file_name = models.FilePathField(max_length=255)
1805 line_number = models.IntegerField(null=True)
1806 operation = models.CharField(max_length=64)
1807
1808class HelpText(models.Model):
1809 VARIABLE = 0
1810 HELPTEXT_AREA = ((VARIABLE, 'variable'), )
1811
1812 build = models.ForeignKey(Build, related_name='helptext_build')
1813 area = models.IntegerField(choices=HELPTEXT_AREA)
1814 key = models.CharField(max_length=100)
1815 text = models.TextField()
1816
1817class LogMessage(models.Model):
1818 EXCEPTION = -1 # used to signal self-toaster-exceptions
1819 INFO = 0
1820 WARNING = 1
1821 ERROR = 2
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001822 CRITICAL = 3
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001823
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001824 LOG_LEVEL = (
1825 (INFO, "info"),
1826 (WARNING, "warn"),
1827 (ERROR, "error"),
1828 (CRITICAL, "critical"),
1829 (EXCEPTION, "toaster exception")
1830 )
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001831
1832 build = models.ForeignKey(Build)
1833 task = models.ForeignKey(Task, blank = True, null=True)
1834 level = models.IntegerField(choices=LOG_LEVEL, default=INFO)
Patrick Williamsf1e5d692016-03-30 15:21:19 -05001835 message = models.TextField(blank=True, null=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001836 pathname = models.FilePathField(max_length=255, blank=True)
1837 lineno = models.IntegerField(null=True)
1838
1839 def __str__(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001840 return force_bytes('%s %s %s' % (self.get_level_display(), self.message, self.build))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001841
1842def invalidate_cache(**kwargs):
1843 from django.core.cache import cache
1844 try:
1845 cache.clear()
1846 except Exception as e:
1847 logger.warning("Problem with cache backend: Failed to clear cache: %s" % e)
1848
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001849def signal_runbuilds():
1850 """Send SIGUSR1 to runbuilds process"""
Brad Bishop6e60e8b2018-02-01 10:27:11 -05001851 try:
1852 with open(os.path.join(os.getenv('BUILDDIR', '.'),
1853 '.runbuilds.pid')) as pidf:
1854 os.kill(int(pidf.read()), SIGUSR1)
1855 except FileNotFoundError:
1856 logger.info("Stopping existing runbuilds: no current process found")
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001857
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001858class Distro(models.Model):
1859 search_allowed_fields = ["name", "description", "layer_version__layer__name"]
1860 up_date = models.DateTimeField(null = True, default = None)
1861
1862 layer_version = models.ForeignKey('Layer_Version')
1863 name = models.CharField(max_length=255)
1864 description = models.CharField(max_length=255)
1865
1866 def get_vcs_distro_file_link_url(self):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001867 path = 'conf/distro/%s.conf' % self.name
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001868 return self.layer_version.get_vcs_file_link_url(path)
1869
1870 def __unicode__(self):
1871 return "Distro " + self.name + "(" + self.description + ")"
1872
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001873django.db.models.signals.post_save.connect(invalidate_cache)
1874django.db.models.signals.post_delete.connect(invalidate_cache)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001875django.db.models.signals.m2m_changed.connect(invalidate_cache)