Patrick Williams | c124f4f | 2015-09-15 14:41:29 -0500 | [diff] [blame^] | 1 | from django.core.management.base import NoArgsCommand, CommandError |
| 2 | from django.db import transaction |
| 3 | from orm.models import Build, ToasterSetting, LogMessage, Target |
| 4 | from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException |
| 5 | from bldcontrol.models import BuildRequest, BuildEnvironment, BRError, BRVariable |
| 6 | import os |
| 7 | import logging |
| 8 | |
| 9 | logger = logging.getLogger("ToasterScheduler") |
| 10 | |
| 11 | class Command(NoArgsCommand): |
| 12 | args = "" |
| 13 | help = "Schedules and executes build requests as possible. Does not return (interrupt with Ctrl-C)" |
| 14 | |
| 15 | |
| 16 | @transaction.commit_on_success |
| 17 | def _selectBuildEnvironment(self): |
| 18 | bec = getBuildEnvironmentController(lock = BuildEnvironment.LOCK_FREE) |
| 19 | bec.be.lock = BuildEnvironment.LOCK_LOCK |
| 20 | bec.be.save() |
| 21 | return bec |
| 22 | |
| 23 | @transaction.commit_on_success |
| 24 | def _selectBuildRequest(self): |
| 25 | br = BuildRequest.objects.filter(state = BuildRequest.REQ_QUEUED).order_by('pk')[0] |
| 26 | br.state = BuildRequest.REQ_INPROGRESS |
| 27 | br.save() |
| 28 | return br |
| 29 | |
| 30 | def schedule(self): |
| 31 | import traceback |
| 32 | try: |
| 33 | br = None |
| 34 | try: |
| 35 | # select the build environment and the request to build |
| 36 | br = self._selectBuildRequest() |
| 37 | except IndexError as e: |
| 38 | #logger.debug("runbuilds: No build request") |
| 39 | return |
| 40 | try: |
| 41 | bec = self._selectBuildEnvironment() |
| 42 | except IndexError as e: |
| 43 | # we could not find a BEC; postpone the BR |
| 44 | br.state = BuildRequest.REQ_QUEUED |
| 45 | br.save() |
| 46 | logger.debug("runbuilds: No build env") |
| 47 | return |
| 48 | |
| 49 | logger.debug("runbuilds: starting build %s, environment %s" % (br, bec.be)) |
| 50 | |
| 51 | # write the build identification variable |
| 52 | BRVariable.objects.create(req = br, name="TOASTER_BRBE", value="%d:%d" % (br.pk, bec.be.pk)) |
| 53 | |
| 54 | # let the build request know where it is being executed |
| 55 | br.environment = bec.be |
| 56 | br.save() |
| 57 | |
| 58 | # this triggers an async build |
| 59 | bec.triggerBuild(br.brbitbake_set.all(), br.brlayer_set.all(), br.brvariable_set.all(), br.brtarget_set.all()) |
| 60 | |
| 61 | except Exception as e: |
| 62 | logger.error("runbuilds: Error launching build %s" % e) |
| 63 | traceback.print_exc(e) |
| 64 | if "[Errno 111] Connection refused" in str(e): |
| 65 | # Connection refused, read toaster_server.out |
| 66 | errmsg = bec.readServerLogFile() |
| 67 | else: |
| 68 | errmsg = str(e) |
| 69 | |
| 70 | BRError.objects.create(req = br, |
| 71 | errtype = str(type(e)), |
| 72 | errmsg = errmsg, |
| 73 | traceback = traceback.format_exc(e)) |
| 74 | br.state = BuildRequest.REQ_FAILED |
| 75 | br.save() |
| 76 | bec.be.lock = BuildEnvironment.LOCK_FREE |
| 77 | bec.be.save() |
| 78 | |
| 79 | def archive(self): |
| 80 | ''' archives data from the builds ''' |
| 81 | artifact_storage_dir = ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value |
| 82 | for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE): |
| 83 | # save cooker log |
| 84 | if br.build == None: |
| 85 | br.state = BuildRequest.REQ_FAILED |
| 86 | br.save() |
| 87 | continue |
| 88 | build_artifact_storage_dir = os.path.join(artifact_storage_dir, "%d" % br.build.pk) |
| 89 | try: |
| 90 | os.makedirs(build_artifact_storage_dir) |
| 91 | except OSError as ose: |
| 92 | if "File exists" in str(ose): |
| 93 | pass |
| 94 | else: |
| 95 | raise ose |
| 96 | |
| 97 | file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt") |
| 98 | try: |
| 99 | with open(file_name, "w") as f: |
| 100 | f.write(br.environment.get_artifact(br.build.cooker_log_path).read()) |
| 101 | except IOError: |
| 102 | os.unlink(file_name) |
| 103 | |
| 104 | br.state = BuildRequest.REQ_COMPLETED |
| 105 | br.save() |
| 106 | |
| 107 | def cleanup(self): |
| 108 | from django.utils import timezone |
| 109 | from datetime import timedelta |
| 110 | # environments locked for more than 30 seconds - they should be unlocked |
| 111 | BuildEnvironment.objects.filter(buildrequest__state__in=[BuildRequest.REQ_FAILED, BuildRequest.REQ_COMPLETED]).filter(lock=BuildEnvironment.LOCK_LOCK).filter(updated__lt = timezone.now() - timedelta(seconds = 30)).update(lock = BuildEnvironment.LOCK_FREE) |
| 112 | |
| 113 | |
| 114 | # update all Builds that failed to start |
| 115 | |
| 116 | for br in BuildRequest.objects.filter(state = BuildRequest.REQ_FAILED, build__outcome = Build.IN_PROGRESS): |
| 117 | # transpose the launch errors in ToasterExceptions |
| 118 | br.build.outcome = Build.FAILED |
| 119 | for brerror in br.brerror_set.all(): |
| 120 | logger.debug("Saving error %s" % brerror) |
| 121 | LogMessage.objects.create(build = br.build, level = LogMessage.EXCEPTION, message = brerror.errmsg) |
| 122 | br.build.save() |
| 123 | |
| 124 | # we don't have a true build object here; hence, toasterui didn't have a change to release the BE lock |
| 125 | br.environment.lock = BuildEnvironment.LOCK_FREE |
| 126 | br.environment.save() |
| 127 | |
| 128 | |
| 129 | |
| 130 | # update all BuildRequests without a build created |
| 131 | for br in BuildRequest.objects.filter(build = None): |
| 132 | br.build = Build.objects.create(project = br.project, completed_on = br.updated, started_on = br.created) |
| 133 | br.build.outcome = Build.FAILED |
| 134 | try: |
| 135 | br.build.machine = br.brvariable_set.get(name='MACHINE').value |
| 136 | except BRVariable.DoesNotExist: |
| 137 | pass |
| 138 | br.save() |
| 139 | # transpose target information |
| 140 | for brtarget in br.brtarget_set.all(): |
| 141 | Target.objects.create(build = br.build, target= brtarget.target) |
| 142 | # transpose the launch errors in ToasterExceptions |
| 143 | for brerror in br.brerror_set.all(): |
| 144 | LogMessage.objects.create(build = br.build, level = LogMessage.EXCEPTION, message = brerror.errmsg) |
| 145 | |
| 146 | br.build.save() |
| 147 | pass |
| 148 | |
| 149 | |
| 150 | def handle_noargs(self, **options): |
| 151 | self.cleanup() |
| 152 | self.archive() |
| 153 | self.schedule() |