Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 1 | from django.core.management.base import NoArgsCommand, CommandError |
| 2 | from django.db import transaction |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 3 | from django.db.models import Q |
| 4 | |
| 5 | from bldcontrol.bbcontroller import getBuildEnvironmentController |
| 6 | from bldcontrol.bbcontroller import ShellCmdException, BuildSetupException |
| 7 | from bldcontrol.models import BuildRequest, BuildEnvironment |
| 8 | from bldcontrol.models import BRError, BRVariable |
| 9 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 10 | from orm.models import Build, ToasterSetting, LogMessage, Target |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 11 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 12 | import os |
| 13 | import logging |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 14 | import time |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 15 | import sys |
| 16 | import traceback |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 17 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 18 | logger = logging.getLogger("toaster") |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 19 | |
| 20 | class Command(NoArgsCommand): |
| 21 | args = "" |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 22 | help = "Schedules and executes build requests as possible." |
| 23 | "Does not return (interrupt with Ctrl-C)" |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 24 | |
| 25 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 26 | @transaction.atomic |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 27 | def _selectBuildEnvironment(self): |
| 28 | bec = getBuildEnvironmentController(lock = BuildEnvironment.LOCK_FREE) |
| 29 | bec.be.lock = BuildEnvironment.LOCK_LOCK |
| 30 | bec.be.save() |
| 31 | return bec |
| 32 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 33 | @transaction.atomic |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 34 | def _selectBuildRequest(self): |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 35 | br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 36 | return br |
| 37 | |
| 38 | def schedule(self): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 39 | try: |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 40 | # select the build environment and the request to build |
| 41 | br = self._selectBuildRequest() |
| 42 | if br: |
| 43 | br.state = BuildRequest.REQ_INPROGRESS |
| 44 | br.save() |
| 45 | else: |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 46 | return |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 47 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 48 | try: |
| 49 | bec = self._selectBuildEnvironment() |
| 50 | except IndexError as e: |
| 51 | # we could not find a BEC; postpone the BR |
| 52 | br.state = BuildRequest.REQ_QUEUED |
| 53 | br.save() |
| 54 | logger.debug("runbuilds: No build env") |
| 55 | return |
| 56 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 57 | logger.debug("runbuilds: starting build %s, environment %s" % \ |
| 58 | (str(br).decode('utf-8'), bec.be)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 59 | |
| 60 | # let the build request know where it is being executed |
| 61 | br.environment = bec.be |
| 62 | br.save() |
| 63 | |
| 64 | # this triggers an async build |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 65 | bec.triggerBuild(br.brbitbake, br.brlayer_set.all(), |
| 66 | br.brvariable_set.all(), br.brtarget_set.all(), |
| 67 | "%d:%d" % (br.pk, bec.be.pk)) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 68 | |
| 69 | except Exception as e: |
| 70 | logger.error("runbuilds: Error launching build %s" % e) |
| 71 | traceback.print_exc(e) |
| 72 | if "[Errno 111] Connection refused" in str(e): |
| 73 | # Connection refused, read toaster_server.out |
| 74 | errmsg = bec.readServerLogFile() |
| 75 | else: |
| 76 | errmsg = str(e) |
| 77 | |
| 78 | BRError.objects.create(req = br, |
| 79 | errtype = str(type(e)), |
| 80 | errmsg = errmsg, |
| 81 | traceback = traceback.format_exc(e)) |
| 82 | br.state = BuildRequest.REQ_FAILED |
| 83 | br.save() |
| 84 | bec.be.lock = BuildEnvironment.LOCK_FREE |
| 85 | bec.be.save() |
| 86 | |
| 87 | def archive(self): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 88 | for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 89 | if br.build == None: |
| 90 | br.state = BuildRequest.REQ_FAILED |
Patrick Williams | d7e9631 | 2015-09-22 08:09:05 -0500 | [diff] [blame] | 91 | else: |
| 92 | br.state = BuildRequest.REQ_COMPLETED |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 93 | br.save() |
| 94 | |
| 95 | def cleanup(self): |
| 96 | from django.utils import timezone |
| 97 | from datetime import timedelta |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 98 | # environments locked for more than 30 seconds |
| 99 | # they should be unlocked |
| 100 | BuildEnvironment.objects.filter( |
| 101 | Q(buildrequest__state__in=[BuildRequest.REQ_FAILED, |
| 102 | BuildRequest.REQ_COMPLETED, |
| 103 | BuildRequest.REQ_CANCELLING]) & |
| 104 | Q(lock=BuildEnvironment.LOCK_LOCK) & |
| 105 | Q(updated__lt=timezone.now() - timedelta(seconds = 30)) |
| 106 | ).update(lock=BuildEnvironment.LOCK_FREE) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 107 | |
| 108 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 109 | # update all Builds that were in progress and failed to start |
| 110 | for br in BuildRequest.objects.filter( |
| 111 | state=BuildRequest.REQ_FAILED, |
| 112 | build__outcome=Build.IN_PROGRESS): |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 113 | # transpose the launch errors in ToasterExceptions |
| 114 | br.build.outcome = Build.FAILED |
| 115 | for brerror in br.brerror_set.all(): |
| 116 | logger.debug("Saving error %s" % brerror) |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 117 | LogMessage.objects.create(build=br.build, |
| 118 | level=LogMessage.EXCEPTION, |
| 119 | message=brerror.errmsg) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 120 | br.build.save() |
| 121 | |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 122 | # we don't have a true build object here; hence, toasterui |
| 123 | # didn't have a change to release the BE lock |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 124 | br.environment.lock = BuildEnvironment.LOCK_FREE |
| 125 | br.environment.save() |
| 126 | |
| 127 | |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 128 | # update all BuildRequests without a build created |
| 129 | for br in BuildRequest.objects.filter(build = None): |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 130 | br.build = Build.objects.create(project=br.project, |
| 131 | completed_on=br.updated, |
| 132 | started_on=br.created) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 133 | br.build.outcome = Build.FAILED |
| 134 | try: |
| 135 | br.build.machine = br.brvariable_set.get(name='MACHINE').value |
| 136 | except BRVariable.DoesNotExist: |
| 137 | pass |
| 138 | br.save() |
| 139 | # transpose target information |
| 140 | for brtarget in br.brtarget_set.all(): |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 141 | Target.objects.create(build=br.build, |
| 142 | target=brtarget.target, |
| 143 | task=brtarget.task) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 144 | # transpose the launch errors in ToasterExceptions |
| 145 | for brerror in br.brerror_set.all(): |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 146 | LogMessage.objects.create(build=br.build, |
| 147 | level=LogMessage.EXCEPTION, |
| 148 | message=brerror.errmsg) |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 149 | |
| 150 | br.build.save() |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 151 | |
| 152 | # Make sure the LOCK is removed for builds which have been fully |
| 153 | # cancelled |
| 154 | for br in BuildRequest.objects.filter( |
| 155 | Q(build__outcome=Build.CANCELLED) & |
| 156 | Q(state=BuildRequest.REQ_CANCELLING) & |
| 157 | ~Q(environment=None)): |
| 158 | br.environment.lock = BuildEnvironment.LOCK_FREE |
| 159 | br.environment.save() |
Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame] | 160 | |
| 161 | |
| 162 | def handle_noargs(self, **options): |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 163 | while True: |
| 164 | try: |
| 165 | self.cleanup() |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 166 | except Exception as e: |
| 167 | logger.warn("runbuilds: cleanup exception %s" % str(e)) |
| 168 | |
| 169 | try: |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 170 | self.archive() |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 171 | except Exception as e: |
| 172 | logger.warn("runbuilds: archive exception %s" % str(e)) |
| 173 | |
| 174 | try: |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 175 | self.schedule() |
Patrick Williams | d8c66bc | 2016-06-20 12:57:21 -0500 | [diff] [blame] | 176 | except Exception as e: |
| 177 | logger.warn("runbuilds: schedule exception %s" % str(e)) |
Patrick Williams | f1e5d69 | 2016-03-30 15:21:19 -0500 | [diff] [blame] | 178 | |
| 179 | time.sleep(1) |