poky/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py

#
# SPDX-License-Identifier: GPL-2.0-only
#

from django.core.management.base import BaseCommand
from django.db import transaction
from django.db.models import Q

from bldcontrol.bbcontroller import getBuildEnvironmentController
from bldcontrol.models import BuildRequest, BuildEnvironment
from bldcontrol.models import BRError, BRVariable

from orm.models import Build, LogMessage, Target

import logging
import traceback
import signal
import os

logger = logging.getLogger("toaster")


class Command(BaseCommand):
    args = ""
    help = "Schedules and executes build requests as possible. "\
           "Does not return (interrupt with Ctrl-C)"

    @transaction.atomic
    def _selectBuildEnvironment(self):
        bec = getBuildEnvironmentController(lock=BuildEnvironment.LOCK_FREE)
        bec.be.lock = BuildEnvironment.LOCK_LOCK
        bec.be.save()
        return bec

    @transaction.atomic
    def _selectBuildRequest(self):
        br = BuildRequest.objects.filter(state=BuildRequest.REQ_QUEUED).first()
        return br

    def schedule(self):
        try:
            # select the build environment and the request to build
            br = self._selectBuildRequest()
            if br:
                br.state = BuildRequest.REQ_INPROGRESS
                br.save()
            else:
                return

            try:
                bec = self._selectBuildEnvironment()
            except IndexError as e:
                # we could not find a BEC; postpone the BR
                br.state = BuildRequest.REQ_QUEUED
                br.save()
                logger.debug("runbuilds: No build env (%s)" % e)
                return

            logger.info("runbuilds: starting build %s, environment %s" %
                        (br, bec.be))

            # let the build request know where it is being executed
            br.environment = bec.be
            br.save()

            # this triggers an async build
            bec.triggerBuild(br.brbitbake, br.brlayer_set.all(),
                             br.brvariable_set.all(), br.brtarget_set.all(),
                             "%d:%d" % (br.pk, bec.be.pk))

        except Exception as e:
            logger.error("runbuilds: Error launching build %s" % e)
            traceback.print_exc()
            if "[Errno 111] Connection refused" in str(e):
                # Connection refused, read toaster_server.out
                errmsg = bec.readServerLogFile()
            else:
                errmsg = str(e)

            BRError.objects.create(req=br, errtype=str(type(e)), errmsg=errmsg,
                                   traceback=traceback.format_exc())
            br.state = BuildRequest.REQ_FAILED
            br.save()
            bec.be.lock = BuildEnvironment.LOCK_FREE
            bec.be.save()
            # Cancel the pending build and report the exception to the UI
            log_object = LogMessage.objects.create(
                            build = br.build,
                            level = LogMessage.EXCEPTION,
                            message = errmsg)
            log_object.save()
            br.build.outcome = Build.FAILED
            br.build.save()

    def archive(self):
        for br in BuildRequest.objects.filter(state=BuildRequest.REQ_ARCHIVE):
            if br.build is None:
                br.state = BuildRequest.REQ_FAILED
            else:
                br.state = BuildRequest.REQ_COMPLETED
            br.save()

    def cleanup(self):
        from django.utils import timezone
        from datetime import timedelta
        # environments locked for more than 30 seconds
        # they should be unlocked
        BuildEnvironment.objects.filter(
            Q(buildrequest__state__in=[BuildRequest.REQ_FAILED,
                                       BuildRequest.REQ_COMPLETED,
                                       BuildRequest.REQ_CANCELLING]) &
            Q(lock=BuildEnvironment.LOCK_LOCK) &
            Q(updated__lt=timezone.now() - timedelta(seconds=30))
        ).update(lock=BuildEnvironment.LOCK_FREE)

        # update all Builds that were in progress and failed to start
        for br in BuildRequest.objects.filter(
                state=BuildRequest.REQ_FAILED,
                build__outcome=Build.IN_PROGRESS):
            # transpose the launch errors in ToasterExceptions
            br.build.outcome = Build.FAILED
            for brerror in br.brerror_set.all():
                logger.debug("Saving error %s" % brerror)
                LogMessage.objects.create(build=br.build,
                                          level=LogMessage.EXCEPTION,
                                          message=brerror.errmsg)
            br.build.save()

            # we don't have a true build object here; hence, toasterui
            # didn't have a change to release the BE lock
            br.environment.lock = BuildEnvironment.LOCK_FREE
            br.environment.save()

        # update all BuildRequests without a build created
        for br in BuildRequest.objects.filter(build=None):
            br.build = Build.objects.create(project=br.project,
                                            completed_on=br.updated,
                                            started_on=br.created)
            br.build.outcome = Build.FAILED
            try:
                br.build.machine = br.brvariable_set.get(name='MACHINE').value
            except BRVariable.DoesNotExist:
                pass
            br.save()
            # transpose target information
            for brtarget in br.brtarget_set.all():
                Target.objects.create(build=br.build,
                                      target=brtarget.target,
                                      task=brtarget.task)
            # transpose the launch errors in ToasterExceptions
            for brerror in br.brerror_set.all():
                LogMessage.objects.create(build=br.build,
                                          level=LogMessage.EXCEPTION,
                                          message=brerror.errmsg)

            br.build.save()

        # Make sure the LOCK is removed for builds which have been fully
        # cancelled
        for br in BuildRequest.objects.filter(
                      Q(build__outcome=Build.CANCELLED) &
                      Q(state=BuildRequest.REQ_CANCELLING) &
                      ~Q(environment=None)):
            br.environment.lock = BuildEnvironment.LOCK_FREE
            br.environment.save()

    def runbuild(self):
        try:
            self.cleanup()
        except Exception as e:
            logger.warning("runbuilds: cleanup exception %s" % str(e))

        try:
            self.archive()
        except Exception as e:
            logger.warning("runbuilds: archive exception %s" % str(e))

        try:
            self.schedule()
        except Exception as e:
            logger.warning("runbuilds: schedule exception %s" % str(e))

    # Test to see if a build pre-maturely died due to a bitbake crash
    def check_dead_builds(self):
        do_cleanup = False
        try:
            for br in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS):
                # Get the build directory
                if br.project.builddir:
                    builddir =  br.project.builddir
                else:
                    builddir = '%s-toaster-%d' % (br.environment.builddir,br.project.id)
                # Check log to see if there is a recent traceback
                toaster_ui_log = os.path.join(builddir, 'toaster_ui.log')
                test_file = os.path.join(builddir, '._toaster_check.txt')
                os.system("tail -n 50 %s > %s" % (os.path.join(builddir, 'toaster_ui.log'),test_file))
                traceback_text = ''
                is_traceback = False
                with open(test_file,'r') as test_file_fd:
                    test_file_tail = test_file_fd.readlines()
                    for line in test_file_tail:
                        if line.startswith('Traceback (most recent call last):'):
                            traceback_text = line
                            is_traceback = True
                        elif line.startswith('NOTE: ToasterUI waiting for events'):
                            # Ignore any traceback before new build start
                            traceback_text = ''
                            is_traceback = False
                        elif line.startswith('Note: Toaster traceback auto-stop'):
                            # Ignore any traceback before this previous traceback catch
                            traceback_text = ''
                            is_traceback = False
                        elif is_traceback:
                            traceback_text += line
                # Test the results
                is_stop = False
                if is_traceback:
                    # Found a traceback
                    errtype = 'Bitbake crash'
                    errmsg = 'Bitbake crash\n' + traceback_text
                    state = BuildRequest.REQ_FAILED
                    # Clean up bitbake files
                    bitbake_lock = os.path.join(builddir, 'bitbake.lock')
                    if os.path.isfile(bitbake_lock):
                        os.remove(bitbake_lock)
                    bitbake_sock = os.path.join(builddir, 'bitbake.sock')
                    if os.path.isfile(bitbake_sock):
                        os.remove(bitbake_sock)
                    if os.path.isfile(test_file):
                        os.remove(test_file)
                    # Add note to ignore this traceback on next check
                    os.system('echo "Note: Toaster traceback auto-stop" >> %s' % toaster_ui_log)
                    is_stop = True
                # Add more tests here
                #elif ...
                # Stop the build request?
                if is_stop:
                    brerror = BRError(
                        req = br,
                        errtype = errtype,
                        errmsg = errmsg,
                        traceback = traceback_text,
                        )
                    brerror.save()
                    br.state = state
                    br.save()
                    do_cleanup = True
            # Do cleanup
            if do_cleanup:
                self.cleanup()
        except Exception as e:
            logger.error("runbuilds: Error in check_dead_builds %s" % e)

    def handle(self, **options):
        pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
                                    ".runbuilds.pid")

        with open(pidfile_path, 'w') as pidfile:
            pidfile.write("%s" % os.getpid())

        # Clean up any stale/failed builds from previous Toaster run
        self.runbuild()

        signal.signal(signal.SIGUSR1, lambda sig, frame: None)

        while True:
            sigset = signal.sigtimedwait([signal.SIGUSR1], 5)
            if sigset:
                for sig in sigset:
                    # Consume each captured pending event
                    self.runbuild()
            else:
                # Check for build exceptions
                self.check_dead_builds()