poky/bitbake/lib/toaster/toastermain/management/commands/buildimport.py

#
# BitBake Toaster Implementation
#
# Copyright (C) 2018        Wind River Systems
#
# SPDX-License-Identifier: GPL-2.0-only
#

# buildimport: import a project for project specific configuration
#
# Usage:
#  (a) Set up Toaster environent
#
#  (b) Call buildimport
#      $ /path/to/bitbake/lib/toaster/manage.py buildimport \
#        --name=$PROJECTNAME \
#        --path=$BUILD_DIRECTORY \
#        --callback="$CALLBACK_SCRIPT" \
#        --command="configure|reconfigure|import"
#
#  (c) Return is "|Default_image=%s|Project_id=%d"
#
#  (d) Open Toaster to this project using for example:
#      $ xdg-open http://localhost:$toaster_port/toastergui/project_specific/$project_id
#
#  (e) To delete a project:
#      $ /path/to/bitbake/lib/toaster/manage.py buildimport \
#        --name=$PROJECTNAME --delete-project
#


# ../bitbake/lib/toaster/manage.py buildimport --name=test --path=`pwd` --callback="" --command=import

from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ObjectDoesNotExist
from orm.models import ProjectManager, Project, Release, ProjectVariable
from orm.models import Layer, Layer_Version, LayerSource, ProjectLayer
from toastergui.api import scan_layer_content
from django.db import OperationalError

import os
import re
import os.path
import subprocess
import shutil

# Toaster variable section delimiters
TOASTER_PROLOG = '#=== TOASTER_CONFIG_PROLOG ==='
TOASTER_EPILOG = '#=== TOASTER_CONFIG_EPILOG ==='

# quick development/debugging support
verbose = 2
def _log(msg):
    if 1 == verbose:
        print(msg)
    elif 2 == verbose:
        f1=open('/tmp/toaster.log', 'a')
        f1.write("|" + msg + "|\n" )
        f1.close()


__config_regexp__  = re.compile( r"""
    ^
    (?P<exp>export\s+)?
    (?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
    (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?

    \s* (
        (?P<colon>:=) |
        (?P<lazyques>\?\?=) |
        (?P<ques>\?=) |
        (?P<append>\+=) |
        (?P<prepend>=\+) |
        (?P<predot>=\.) |
        (?P<postdot>\.=) |
        =
    ) \s*

    (?!'[^']*'[^']*'$)
    (?!\"[^\"]*\"[^\"]*\"$)
    (?P<apo>['\"])
    (?P<value>.*)
    (?P=apo)
    $
    """, re.X)

class Command(BaseCommand):
    args    = "<name> <path> <release>"
    help    = "Import a command line build directory"
    vars    = {}
    toaster_vars = {}

    def add_arguments(self, parser):
        parser.add_argument(
            '--name', dest='name', required=True,
            help='name of the project',
            )
        parser.add_argument(
            '--path', dest='path', required=True,
            help='path to the project',
            )
        parser.add_argument(
            '--release', dest='release', required=False,
            help='release for the project',
            )
        parser.add_argument(
            '--callback', dest='callback', required=False,
            help='callback for project config update',
            )
        parser.add_argument(
            '--delete-project', dest='delete_project', required=False,
            help='delete this project from the database',
            )
        parser.add_argument(
            '--command', dest='command', required=False,
            help='command (configure,reconfigure,import)',
            )

    # Extract the bb variables from a conf file
    def scan_conf(self,fn):
        vars = self.vars
        toaster_vars = self.toaster_vars

        #_log("scan_conf:%s" % fn)
        if not os.path.isfile(fn):
            return
        f = open(fn, 'r')

        #statements = ast.StatementGroup()
        lineno = 0
        is_toaster_section = False
        while True:
            lineno = lineno + 1
            s = f.readline()
            if not s:
                break
            w = s.strip()
            # skip empty lines
            if not w:
                continue
            # evaluate Toaster sections
            if w.startswith(TOASTER_PROLOG):
                is_toaster_section = True
                continue
            if w.startswith(TOASTER_EPILOG):
                is_toaster_section = False
                continue
            s = s.rstrip()
            while s[-1] == '\\':
                s2 = f.readline().strip()
                lineno = lineno + 1
                if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
                    echo("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
                s = s[:-1] + s2
            # skip comments
            if s[0] == '#':
                continue
            # process the line for just assignments
            m = __config_regexp__.match(s)
            if m:
                groupd = m.groupdict()
                var = groupd['var']
                value = groupd['value']

                if groupd['lazyques']:
                    if not var in vars:
                        vars[var] = value
                    continue
                if groupd['ques']:
                    if not var in vars:
                        vars[var] = value
                    continue
                # preset empty blank for remaining operators
                if not var in vars:
                    vars[var] = ''
                if groupd['append']:
                    vars[var] += value
                elif groupd['prepend']:
                    vars[var] = "%s%s" % (value,vars[var])
                elif groupd['predot']:
                    vars[var] = "%s %s" % (value,vars[var])
                elif groupd['postdot']:
                    vars[var] = "%s %s" % (vars[var],value)
                else:
                    vars[var] = "%s" % (value)
                # capture vars in a Toaster section
                if is_toaster_section:
                    toaster_vars[var] = vars[var]

        # DONE WITH PARSING
        f.close()
        self.vars = vars
        self.toaster_vars = toaster_vars

    # Update the scanned project variables
    def update_project_vars(self,project,name):
        pv, create = ProjectVariable.objects.get_or_create(project = project, name = name)
        if (not name in self.vars.keys()) or (not self.vars[name]):
            self.vars[name] = pv.value
        else:
            if pv.value != self.vars[name]:
                pv.value = self.vars[name]
        pv.save()

    # Find the git version of the installation
    def find_layer_dir_version(self,path):
        #  * rocko               ...

        install_version = ''
        cwd = os.getcwd()
        os.chdir(path)
        p = subprocess.Popen(['git', 'branch', '-av'], stdout=subprocess.PIPE,
                                                stderr=subprocess.PIPE)
        out, err = p.communicate()
        out = out.decode("utf-8")
        for branch in out.split('\n'):
            if ('*' == branch[0:1]) and ('no branch' not in branch):
                install_version = re.sub(' .*','',branch[2:])
                break
            if 'remotes/m/master' in branch:
                install_version = re.sub('.*base/','',branch)
                break
        os.chdir(cwd)
        return install_version

    # Compute table of the installation's registered layer versions (branch or commit)
    def find_layer_dir_versions(self,INSTALL_URL_PREFIX):
        lv_dict = {}
        layer_versions = Layer_Version.objects.all()
        for lv in layer_versions:
            layer = Layer.objects.filter(pk=lv.layer.pk)[0]
            if layer.vcs_url:
                url_short = layer.vcs_url.replace(INSTALL_URL_PREFIX,'')
            else:
                url_short = ''
            # register the core, branch, and the version variations
            lv_dict["%s,%s,%s" % (url_short,lv.dirpath,'')] = (lv.id,layer.name)
            lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.branch)] = (lv.id,layer.name)
            lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.commit)] = (lv.id,layer.name)
            #_log("  (%s,%s,%s|%s) = (%s,%s)" % (url_short,lv.dirpath,lv.branch,lv.commit,lv.id,layer.name))
        return lv_dict

    # Apply table of all layer versions
    def extract_bblayers(self):
        # set up the constants
        bblayer_str = self.vars['BBLAYERS']
        TOASTER_DIR = os.environ.get('TOASTER_DIR')
        INSTALL_CLONE_PREFIX = os.path.dirname(TOASTER_DIR) + "/"
        TOASTER_CLONE_PREFIX = TOASTER_DIR + "/_toaster_clones/"
        INSTALL_URL_PREFIX = ''
        layers = Layer.objects.filter(name='openembedded-core')
        for layer in layers:
            if layer.vcs_url:
                INSTALL_URL_PREFIX = layer.vcs_url
                break
        INSTALL_URL_PREFIX = INSTALL_URL_PREFIX.replace("/poky","/")
        INSTALL_VERSION_DIR = TOASTER_DIR
        INSTALL_URL_POSTFIX = INSTALL_URL_PREFIX.replace(':','_')
        INSTALL_URL_POSTFIX = INSTALL_URL_POSTFIX.replace('/','_')
        INSTALL_URL_POSTFIX = "%s_%s" % (TOASTER_CLONE_PREFIX,INSTALL_URL_POSTFIX)

        # get the set of available layer:layer_versions
        lv_dict = self.find_layer_dir_versions(INSTALL_URL_PREFIX)

        # compute the layer matches
        layers_list = []
        for line in bblayer_str.split(' '):
            if not line:
                continue
            if line.endswith('/local'):
                continue

            # isolate the repo
            layer_path = line
            line = line.replace(INSTALL_URL_POSTFIX,'').replace(INSTALL_CLONE_PREFIX,'').replace('/layers/','/').replace('/poky/','/')

            # isolate the sub-path
            path_index = line.rfind('/')
            if path_index > 0:
                sub_path = line[path_index+1:]
                line = line[0:path_index]
            else:
                sub_path = ''

            # isolate the version
            if TOASTER_CLONE_PREFIX in layer_path:
                is_toaster_clone = True
                # extract version from name syntax
                version_index = line.find('_')
                if version_index > 0:
                    version = line[version_index+1:]
                    line = line[0:version_index]
                else:
                    version = ''
                _log("TOASTER_CLONE(%s/%s), version=%s" % (line,sub_path,version))
            else:
                is_toaster_clone = False
                # version is from the installation
                version = self.find_layer_dir_version(layer_path)
                _log("LOCAL_CLONE(%s/%s), version=%s" % (line,sub_path,version))

            # capture the layer information into layers_list
            layers_list.append( (line,sub_path,version,layer_path,is_toaster_clone) )
        return layers_list,lv_dict

    #
    def find_import_release(self,layers_list,lv_dict,default_release):
        #   poky,meta,rocko => 4;openembedded-core
        release = default_release
        for line,path,version,layer_path,is_toaster_clone in layers_list:
            key = "%s,%s,%s" % (line,path,version)
            if key in lv_dict:
                lv_id = lv_dict[key]
                if 'openembedded-core' == lv_id[1]:
                    _log("Find_import_release(%s):version=%s,Toaster=%s" % (lv_id[1],version,is_toaster_clone))
                    # only versions in Toaster managed layers are accepted
                    if not is_toaster_clone:
                        break
                    try:
                        release = Release.objects.get(name=version)
                    except:
                        pass
                    break
        _log("Find_import_release:RELEASE=%s" % release.name)
        return release

    # Apply the found conf layers
    def apply_conf_bblayers(self,layers_list,lv_dict,project,release=None):
        for line,path,version,layer_path,is_toaster_clone in layers_list:
            # Assert release promote if present
            if release:
                version = release
            # try to match the key to a layer_version
            key = "%s,%s,%s" % (line,path,version)
            key_short = "%s,%s,%s" % (line,path,'')
            lv_id = ''
            if key in lv_dict:
                lv_id = lv_dict[key]
                lv = Layer_Version.objects.get(pk=int(lv_id[0]))
                pl,created = ProjectLayer.objects.get_or_create(project=project,
                                                   layercommit=lv)
                pl.optional=False
                pl.save()
                _log("  %s => %s;%s" % (key,lv_id[0],lv_id[1]))
            elif key_short in lv_dict:
                lv_id = lv_dict[key_short]
                lv = Layer_Version.objects.get(pk=int(lv_id[0]))
                pl,created = ProjectLayer.objects.get_or_create(project=project,
                                                   layercommit=lv)
                pl.optional=False
                pl.save()
                _log("  %s ?> %s" % (key,lv_dict[key_short]))
            else:
                _log("%s <= %s" % (key,layer_path))
                found = False
                # does local layer already exist in this project?
                try:
                    for pl in ProjectLayer.objects.filter(project=project):
                        if pl.layercommit.layer.local_source_dir == layer_path:
                            found = True
                            _log("  Project Local Layer found!")
                except Exception as e:
                    _log("ERROR: Local Layer '%s'" % e)
                    pass

                if not found:
                    # Does Layer name+path already exist?
                    try:
                        layer_name_base = os.path.basename(layer_path)
                        _log("Layer_lookup: try '%s','%s'" % (layer_name_base,layer_path))
                        layer = Layer.objects.get(name=layer_name_base,local_source_dir = layer_path)
                        # Found! Attach layer_version and ProjectLayer
                        layer_version = Layer_Version.objects.create(
                            layer=layer,
                            project=project,
                            layer_source=LayerSource.TYPE_IMPORTED)
                        layer_version.save()
                        pl,created = ProjectLayer.objects.get_or_create(project=project,
                                                           layercommit=layer_version)
                        pl.optional=False
                        pl.save()
                        found = True
                        # add layer contents to this layer version
                        scan_layer_content(layer,layer_version)
                        _log("  Parent Local Layer found in db!")
                    except Exception as e:
                        _log("Layer_exists_test_failed: Local Layer '%s'" % e)
                        pass

                if not found:
                    # Insure that layer path exists, in case of user typo
                    if not os.path.isdir(layer_path):
                        _log("ERROR:Layer path '%s' not found" % layer_path)
                        continue
                    # Add layer to db and attach project to it
                    layer_name_base = os.path.basename(layer_path)
                    # generate a unique layer name
                    layer_name_matches = {}
                    for layer in Layer.objects.filter(name__contains=layer_name_base):
                        layer_name_matches[layer.name] = '1'
                    layer_name_idx = 0
                    layer_name_test = layer_name_base
                    while layer_name_test in layer_name_matches.keys():
                        layer_name_idx += 1
                        layer_name_test = "%s_%d" % (layer_name_base,layer_name_idx)
                    # create the layer and layer_verion objects
                    layer = Layer.objects.create(name=layer_name_test)
                    layer.local_source_dir = layer_path
                    layer_version = Layer_Version.objects.create(
                        layer=layer,
                        project=project,
                        layer_source=LayerSource.TYPE_IMPORTED)
                    layer.save()
                    layer_version.save()
                    pl,created = ProjectLayer.objects.get_or_create(project=project,
                                                       layercommit=layer_version)
                    pl.optional=False
                    pl.save()
                    # register the layer's content
                    _log("  Local Layer Add content")
                    scan_layer_content(layer,layer_version)
                    _log("  Local Layer Added '%s'!" % layer_name_test)

    # Scan the project's conf files (if any)
    def scan_conf_variables(self,project_path):
        # scan the project's settings, add any new layers or variables
        if os.path.isfile("%s/conf/local.conf" % project_path):
            self.scan_conf("%s/conf/local.conf" % project_path)
            self.scan_conf("%s/conf/bblayers.conf" % project_path)
            # Import then disable old style Toaster conf files (before 'merged_attr')
            old_toaster_local = "%s/conf/toaster.conf" % project_path
            if os.path.isfile(old_toaster_local):
                self.scan_conf(old_toaster_local)
                shutil.move(old_toaster_local, old_toaster_local+"_old")
            old_toaster_layer = "%s/conf/toaster-bblayers.conf" % project_path
            if os.path.isfile(old_toaster_layer):
                self.scan_conf(old_toaster_layer)
                shutil.move(old_toaster_layer, old_toaster_layer+"_old")

    # Scan the found conf variables (if any)
    def apply_conf_variables(self,project,layers_list,lv_dict,release=None):
        if self.vars:
            # Catch vars relevant to Toaster (in case no Toaster section)
            self.update_project_vars(project,'DISTRO')
            self.update_project_vars(project,'MACHINE')
            self.update_project_vars(project,'IMAGE_INSTALL_append')
            self.update_project_vars(project,'IMAGE_FSTYPES')
            self.update_project_vars(project,'PACKAGE_CLASSES')
            # These vars are typically only assigned by Toaster
            #self.update_project_vars(project,'DL_DIR')
            #self.update_project_vars(project,'SSTATE_DIR')

            # Assert found Toaster vars
            for var in self.toaster_vars.keys():
                pv, create = ProjectVariable.objects.get_or_create(project = project, name = var)
                pv.value = self.toaster_vars[var]
                _log("* Add/update Toaster var '%s' = '%s'" % (pv.name,pv.value))
                pv.save()

            # Assert found BBLAYERS
            if 0 < verbose:
                for pl in ProjectLayer.objects.filter(project=project):
                    release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
                    print(" BEFORE:ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
            self.apply_conf_bblayers(layers_list,lv_dict,project,release)
            if 0 < verbose:
                for pl in ProjectLayer.objects.filter(project=project):
                    release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
                    print(" AFTER :ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))


    def handle(self, *args, **options):
        project_name = options['name']
        project_path = options['path']
        project_callback = options['callback'] if options['callback'] else ''
        release_name = options['release'] if options['release'] else ''

        #
        # Delete project
        #

        if options['delete_project']:
            try:
                print("Project '%s' delete from Toaster database" % (project_name))
                project = Project.objects.get(name=project_name)
                # TODO: deep project delete
                project.delete()
                print("Project '%s' Deleted" % (project_name))
                return
            except Exception as e:
                print("Project '%s' not found, not deleted (%s)" % (project_name,e))
                return

        #
        # Create/Update/Import project
        #

        # See if project (by name) exists
        project = None
        try:
            # Project already exists
            project = Project.objects.get(name=project_name)
        except Exception as e:
            pass

        # Find the installation's default release
        default_release = Release.objects.get(id=1)

        # SANITY: if 'reconfig' but project does not exist (deleted externally), switch to 'import'
        if ("reconfigure" == options['command']) and (None == project):
            options['command'] = 'import'

        # 'Configure':
        if "configure" == options['command']:
            # Note: ignore any existing conf files
            # create project, SANITY: reuse any project of same name
            project = Project.objects.create_project(project_name,default_release,project)

        # 'Re-configure':
        if "reconfigure" == options['command']:
            # Scan the directory's conf files
            self.scan_conf_variables(project_path)
            # Scan the layer list
            layers_list,lv_dict = self.extract_bblayers()
            # Apply any new layers or variables
            self.apply_conf_variables(project,layers_list,lv_dict)

        # 'Import':
        if "import" == options['command']:
            # Scan the directory's conf files
            self.scan_conf_variables(project_path)
            # Remove these Toaster controlled variables
            for var in ('DL_DIR','SSTATE_DIR'):
                self.vars.pop(var, None)
                self.toaster_vars.pop(var, None)
            # Scan the layer list
            layers_list,lv_dict = self.extract_bblayers()
            # Find the directory's release, and promote to default_release if local paths
            release = self.find_import_release(layers_list,lv_dict,default_release)
            # create project, SANITY: reuse any project of same name
            project = Project.objects.create_project(project_name,release,project)
            # Apply any new layers or variables
            self.apply_conf_variables(project,layers_list,lv_dict,release)
            # WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific'
            project.set_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE','1')

        # Set up the project's meta data
        project.builddir = project_path
        project.merged_attr = True
        project.set_variable(Project.PROJECT_SPECIFIC_CALLBACK,project_callback)
        project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_EDIT)
        if ("configure" == options['command']) or ("import" == options['command']):
            # preset the mode and default image recipe
            project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NEW)
            project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,"core-image-minimal")
            # Assert any extended/custom actions or variables for new non-Toaster projects
            if not len(self.toaster_vars):
                pass
        else:
            project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NONE)

        # Save the updated Project
        project.save()

        _log("Buildimport:project='%s' at '%d'" % (project_name,project.id))

        if ('DEFAULT_IMAGE' in self.vars) and (self.vars['DEFAULT_IMAGE']):
            print("|Default_image=%s|Project_id=%d" % (self.vars['DEFAULT_IMAGE'],project.id))
        else:
            print("|Project_id=%d" % (project.id))