poky/bitbake/lib/toaster/toastergui/buildtables.py

#
# BitBake Toaster Implementation
#
# Copyright (C) 2016 Intel Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
#

from orm.models import Build, Task, Target, Package
from django.db.models import Q, Sum

import toastergui.tables as tables
from toastergui.widgets import ToasterTable
from toastergui.tablefilter import TableFilter
from toastergui.tablefilter import TableFilterActionToggle


class BuildTablesMixin(ToasterTable):
    def get_context_data(self, **kwargs):
        # We need to be explicit about which superclass we're calling here
        # Otherwise the MRO gets in a right mess
        context = ToasterTable.get_context_data(self, **kwargs)
        context['build'] = Build.objects.get(pk=kwargs['build_id'])
        return context


class BuiltPackagesTableBase(tables.PackagesTable):
    """ Table to display all the packages built in a build """
    def __init__(self, *args, **kwargs):
        super(BuiltPackagesTableBase, self).__init__(*args, **kwargs)
        self.title = "Packages built"
        self.default_orderby = "name"

    def setup_queryset(self, *args, **kwargs):
        build = Build.objects.get(pk=kwargs['build_id'])
        self.static_context_extra['build'] = build
        self.static_context_extra['target_name'] = None
        self.queryset = build.package_set.all().exclude(recipe=None)
        self.queryset = self.queryset.order_by(self.default_orderby)

    def setup_columns(self, *args, **kwargs):
        super(BuiltPackagesTableBase, self).setup_columns(*args, **kwargs)

        def pkg_link_template(val):
            """ return the template used for the link with the val as the
            element value i.e. inside the <a></a>"""

            return ('''
                    <a href="
                    {%% url "package_built_detail" extra.build.pk data.pk %%}
                    ">%s</a>
                    ''' % val)

        def recipe_link_template(val):
            return ('''
                    {%% if data.recipe %%}
                    <a href="
                    {%% url "recipe" extra.build.pk data.recipe.pk %%}
                    ">%(value)s</a>
                    {%% else %%}
                    %(value)s
                    {%% endif %%}
                    ''' % {'value': val})

        add_pkg_link_to = 'name'
        add_recipe_link_to = 'recipe__name'

        # Add the recipe and pkg build links to the required columns
        for column in self.columns:
            # Convert to template field style accessors
            tmplv = column['field_name'].replace('__', '.')
            tmplv = "{{data.%s}}" % tmplv

            if column['field_name'] is add_pkg_link_to:
                # Don't overwrite an existing template
                if column['static_data_template']:
                    column['static_data_template'] =\
                        pkg_link_template(column['static_data_template'])
                else:
                    column['static_data_template'] = pkg_link_template(tmplv)

                column['static_data_name'] = column['field_name']

            elif column['field_name'] is add_recipe_link_to:
                # Don't overwrite an existing template
                if column['static_data_template']:
                    column['static_data_template'] =\
                        recipe_link_template(column['static_data_template'])
                else:
                    column['static_data_template'] =\
                        recipe_link_template(tmplv)
                column['static_data_name'] = column['field_name']

        self.add_column(title="Layer",
                        field_name="recipe__layer_version__layer__name",
                        hidden=True,
                        orderable=True)

        layer_branch_template = '''
        {%if not data.recipe.layer_version.layer.local_source_dir %}
        <span class="text-muted">{{data.recipe.layer_version.branch}}</span>
        {% else %}
        <span class="text-muted">Not applicable</span>
        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.recipe.layer_version.layer.name}} is not in a Git repository, so there is no branch associated with it"> </span>
        {% endif %}
        '''

        self.add_column(title="Layer branch",
                        field_name="recipe__layer_version__branch",
                        hidden=True,
                        static_data_name="recipe__layer_version__branch",
                        static_data_template=layer_branch_template,
                        orderable=True)

        git_rev_template = '''
        {% if not data.recipe.layer_version.layer.local_source_dir %}
        {% with vcs_ref=data.recipe.layer_version.commit %}
        {% include 'snippets/gitrev_popover.html' %}
        {% endwith %}
        {% else %}
        <span class="text-muted">Not applicable</span>
        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.recipe.layer_version.layer.name}} is not in a Git repository, so there is no revision associated with it"> </span>
        {% endif %}
        '''

        self.add_column(title="Layer commit",
                        static_data_name='vcs_ref',
                        static_data_template=git_rev_template,
                        hidden=True)


class BuiltPackagesTable(BuildTablesMixin, BuiltPackagesTableBase):
    """ Show all the packages built for the selected build """
    def __init__(self, *args, **kwargs):
        super(BuiltPackagesTable, self).__init__(*args, **kwargs)
        self.title = "Packages built"
        self.default_orderby = "name"

        self.empty_state =\
            ('<strong>No packages were built.</strong> How did this happen? '
             'Well, BitBake reuses as much stuff as possible. '
             'If all of the packages needed were already built and available '
             'in your build infrastructure, BitBake '
             'will not rebuild any of them. This might be slightly confusing, '
             'but it does make everything faster.')

    def setup_columns(self, *args, **kwargs):
        super(BuiltPackagesTable, self).setup_columns(*args, **kwargs)

        def remove_dep_cols(columns):
            for column in columns:
                # We don't need these fields
                if column['static_data_name'] in ['reverse_dependencies',
                                                  'dependencies']:
                    continue

                yield column

        self.columns = list(remove_dep_cols(self.columns))


class InstalledPackagesTable(BuildTablesMixin, BuiltPackagesTableBase):
    """ Show all packages installed in an image """
    def __init__(self, *args, **kwargs):
        super(InstalledPackagesTable, self).__init__(*args, **kwargs)
        self.title = "Packages Included"
        self.default_orderby = "name"

    def make_package_list(self, target):
        # The database design means that you get the intermediate objects and
        # not package objects like you'd really want so we get them here
        pkgs = target.target_installed_package_set.values_list('package',
                                                               flat=True)
        return Package.objects.filter(pk__in=pkgs)

    def get_context_data(self, **kwargs):
        context = super(InstalledPackagesTable,
                        self).get_context_data(**kwargs)

        target = Target.objects.get(pk=kwargs['target_id'])
        packages = self.make_package_list(target)

        context['packages_sum'] = packages.aggregate(
            Sum('installed_size'))['installed_size__sum']

        context['target'] = target
        return context

    def setup_queryset(self, *args, **kwargs):
        build = Build.objects.get(pk=kwargs['build_id'])
        self.static_context_extra['build'] = build

        target = Target.objects.get(pk=kwargs['target_id'])
        # We send these separately because in the case of image details table
        # we don't have a target just the recipe name as the target
        self.static_context_extra['target_name'] = target.target
        self.static_context_extra['target_id'] = target.pk

        self.static_context_extra['add_links'] = True

        self.queryset = self.make_package_list(target)
        self.queryset = self.queryset.order_by(self.default_orderby)

    def setup_columns(self, *args, **kwargs):
        super(InstalledPackagesTable, self).setup_columns(**kwargs)
        self.add_column(title="Installed size",
                        static_data_name="installed_size",
                        static_data_template="{% load projecttags %}"
                        "{{data.size|filtered_filesizeformat}}",
                        orderable=True,
                        hidden=True)

        # Add the template to show installed name for installed packages
        install_name_tmpl =\
            ('<a href="{% url "package_included_detail" extra.build.pk'
             ' extra.target_id data.pk %}">{{data.name}}</a>'
             '{% if data.installed_name and data.installed_name !='
             ' data.name %}'
             '<span class="text-muted"> as {{data.installed_name}}</span>'
             ' <span class="glyphicon glyphicon-question-sign get-help hover-help"'
             ' title="{{data.name}} was renamed at packaging time and'
             ' was installed in your image as {{data.installed_name}}'
             '"></span>{% endif %} ')

        for column in self.columns:
            if column['static_data_name'] == 'name':
                column['static_data_template'] = install_name_tmpl
                break


class BuiltRecipesTable(BuildTablesMixin):
    """ Table to show the recipes that have been built in this build """

    def __init__(self, *args, **kwargs):
        super(BuiltRecipesTable, self).__init__(*args, **kwargs)
        self.title = "Recipes built"
        self.default_orderby = "name"

    def setup_queryset(self, *args, **kwargs):
        build = Build.objects.get(pk=kwargs['build_id'])
        self.static_context_extra['build'] = build
        self.queryset = build.get_recipes()
        self.queryset = self.queryset.order_by(self.default_orderby)

    def setup_columns(self, *args, **kwargs):
        recipe_name_tmpl =\
            '<a href="{% url "recipe" extra.build.pk data.pk %}">'\
            '{{data.name}}'\
            '</a>'

        recipe_file_tmpl =\
            '{{data.file_path}}'\
            '{% if data.pathflags %}<i>({{data.pathflags}})</i>'\
            '{% endif %}'

        git_branch_template = '''
        {% if data.layer_version.layer.local_source_dir %}
        <span class="text-muted">Not applicable</span>
        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.layer_version.layer.name}} is not in a Git repository, so there is no branch associated with it"> </span>
        {% else %}
        <span>{{data.layer_version.branch}}</span>
        {% endif %}
        '''

        git_rev_template = '''
        {% if data.layer_version.layer.local_source_dir %}
        <span class="text-muted">Not applicable</span>
        <span class="glyphicon glyphicon-question-sign get-help" data-original-title="" title="The source code of {{data.layer_version.layer.name}} is not in a Git repository, so there is no commit associated with it"> </span>
        {% else %}
        {% with vcs_ref=data.layer_version.commit %}
        {% include 'snippets/gitrev_popover.html' %}
        {% endwith %}
        {% endif %}
        '''

        depends_on_tmpl = '''
        {% with deps=data.r_dependencies_recipe.all %}
        {% with count=deps|length %}
        {% if count %}
        <a class="btn btn-default" title="
        <a href='{% url "recipe" extra.build.pk data.pk %}#dependencies'>
        {{data.name}}</a> dependencies"
        data-content="<ul class='list-unstyled'>
        {% for dep in deps|dictsort:"depends_on.name"%}
        <li><a href='{% url "recipe" extra.build.pk dep.depends_on.pk %}'>
        {{dep.depends_on.name}}</a></li>
        {% endfor %}
        </ul>">
         {{count}}
        </a>
        {% endif %}{% endwith %}{% endwith %}
        '''

        rev_depends_tmpl = '''
        {% with revs=data.r_dependencies_depends.all %}
        {% with count=revs|length %}
        {% if count %}
        <a class="btn btn-default"
        title="
        <a href='{% url "recipe" extra.build.pk data.pk %}#brought-in-by'>
        {{data.name}}</a> reverse dependencies"
        data-content="<ul class='list-unstyled'>
        {% for dep in revs|dictsort:"recipe.name" %}
        <li>
        <a href='{% url "recipe" extra.build.pk dep.recipe.pk %}'>
        {{dep.recipe.name}}
        </a></li>
        {% endfor %}
        </ul>">
        {{count}}
        </a>
        {% endif %}{% endwith %}{% endwith %}
        '''

        self.add_column(title="Recipe",
                        field_name="name",
                        static_data_name='name',
                        orderable=True,
                        hideable=False,
                        static_data_template=recipe_name_tmpl)

        self.add_column(title="Version",
                        hideable=False,
                        field_name="version")

        self.add_column(title="Dependencies",
                        static_data_name="dependencies",
                        static_data_template=depends_on_tmpl)

        self.add_column(title="Reverse dependencies",
                        static_data_name="revdeps",
                        static_data_template=rev_depends_tmpl,
                        help_text='Recipe build-time reverse dependencies'
                        ' (i.e. the recipes that depend on this recipe)')

        self.add_column(title="Recipe file",
                        field_name="file_path",
                        static_data_name="file_path",
                        static_data_template=recipe_file_tmpl,
                        hidden=True)

        self.add_column(title="Section",
                        field_name="section",
                        orderable=True,
                        hidden=True)

        self.add_column(title="License",
                        field_name="license",
                        help_text='Multiple license names separated by the'
                        ' pipe character indicates a choice between licenses.'
                        ' Multiple license names separated by the ampersand'
                        ' character indicates multiple licenses exist that'
                        ' cover different parts of the source',
                        orderable=True)

        self.add_column(title="Layer",
                        field_name="layer_version__layer__name",
                        orderable=True)

        self.add_column(title="Layer branch",
                        field_name="layer_version__branch",
                        static_data_name="layer_version__branch",
                        static_data_template=git_branch_template,
                        orderable=True,
                        hidden=True)

        self.add_column(title="Layer commit",
                        static_data_name="commit",
                        static_data_template=git_rev_template,
                        hidden=True)


class BuildTasksTable(BuildTablesMixin):
    """ Table to show the tasks that run in this build """

    def __init__(self, *args, **kwargs):
        super(BuildTasksTable, self).__init__(*args, **kwargs)
        self.title = "Tasks"
        self.default_orderby = "order"

        # Toggle these columns on off for Time/CPU usage/Disk I/O tables
        self.toggle_columns = {}

    def setup_queryset(self, *args, **kwargs):
        build = Build.objects.get(pk=kwargs['build_id'])
        self.static_context_extra['build'] = build
        self.queryset = build.task_build.filter(~Q(order=None))
        self.queryset = self.queryset.order_by(self.default_orderby)

    def setup_filters(self, *args, **kwargs):
        # Execution outcome types filter
        executed_outcome = TableFilter(name="execution_outcome",
                                       title="Filter Tasks by 'Executed")

        exec_outcome_action_exec = TableFilterActionToggle(
            "executed",
            "Executed Tasks",
            Q(task_executed=True))

        exec_outcome_action_not_exec = TableFilterActionToggle(
            "not_executed",
            "Not Executed Tasks",
            Q(task_executed=False))

        executed_outcome.add_action(exec_outcome_action_exec)
        executed_outcome.add_action(exec_outcome_action_not_exec)

        # Task outcome types filter
        task_outcome = TableFilter(name="task_outcome",
                                   title="Filter Task by 'Outcome'")

        for outcome_enum, title in Task.TASK_OUTCOME:
            if outcome_enum is Task.OUTCOME_NA:
                continue
            action = TableFilterActionToggle(
                title.replace(" ", "_").lower(),
                "%s Tasks" % title,
                Q(outcome=outcome_enum))

            task_outcome.add_action(action)

        # SSTATE outcome types filter
        sstate_outcome = TableFilter(name="sstate_outcome",
                                     title="Filter Task by 'Cache attempt'")

        for sstate_result_enum, title in Task.SSTATE_RESULT:
            action = TableFilterActionToggle(
                title.replace(" ", "_").lower(),
                "Tasks with '%s' attempts" % title,
                Q(sstate_result=sstate_result_enum))

            sstate_outcome.add_action(action)

        self.add_filter(sstate_outcome)
        self.add_filter(executed_outcome)
        self.add_filter(task_outcome)

    def setup_columns(self, *args, **kwargs):
        self.toggle_columns['order'] = len(self.columns)

        recipe_name_tmpl =\
            '<a href="{% url "recipe" extra.build.pk data.recipe.pk %}">'\
            '{{data.recipe.name}}'\
            '</a>'

        def task_link_tmpl(val):
            return ('<a name="task-{{data.order}}"'
                    'href="{%% url "task" extra.build.pk data.pk %%}">'
                    '%s'
                    '</a>') % str(val)

        self.add_column(title="Order",
                        static_data_name="order",
                        static_data_template='{{data.order}}',
                        hideable=False,
                        orderable=True)

        self.add_column(title="Task",
                        static_data_name="task_name",
                        static_data_template=task_link_tmpl(
                            "{{data.task_name}}"),
                        hideable=False,
                        orderable=True)

        self.add_column(title="Recipe",
                        static_data_name='recipe__name',
                        static_data_template=recipe_name_tmpl,
                        hideable=False,
                        orderable=True)

        self.add_column(title="Recipe version",
                        field_name='recipe__version',
                        hidden=True)

        self.add_column(title="Executed",
                        static_data_name="task_executed",
                        static_data_template='{{data.get_executed_display}}',
                        filter_name='execution_outcome',
                        orderable=True)

        self.static_context_extra['OUTCOME_FAILED'] = Task.OUTCOME_FAILED
        outcome_tmpl = '{{data.outcome_text}}'
        outcome_tmpl = ('%s '
                        '{%% if data.outcome = extra.OUTCOME_FAILED %%}'
                        '<a href="{%% url "build_artifact" extra.build.pk '
                        '          "tasklogfile" data.pk %%}">'
                        ' <span class="glyphicon glyphicon-download-alt'
                        ' get-help" title="Download task log file"></span>'
                        '</a> {%% endif %%}'
                        '<span class="glyphicon glyphicon-question-sign'
                        ' get-help hover-help" style="visibility: hidden;" '
                        'title="{{data.get_outcome_help}}"></span>'
                        ) % outcome_tmpl

        self.add_column(title="Outcome",
                        static_data_name="outcome",
                        static_data_template=outcome_tmpl,
                        filter_name="task_outcome",
                        orderable=True)

        self.toggle_columns['sstate_result'] = len(self.columns)

        self.add_column(title="Cache attempt",
                        static_data_name="sstate_result",
                        static_data_template='{{data.sstate_text}}',
                        filter_name="sstate_outcome",
                        orderable=True)

        self.toggle_columns['elapsed_time'] = len(self.columns)

        self.add_column(
            title="Time (secs)",
            static_data_name="elapsed_time",
            static_data_template='{% load projecttags %}{% load humanize %}'
            '{{data.elapsed_time|format_none_and_zero|floatformat:2}}',
            orderable=True,
            hidden=True)

        self.toggle_columns['cpu_time_sys'] = len(self.columns)

        self.add_column(
            title="System CPU time (secs)",
            static_data_name="cpu_time_system",
            static_data_template='{% load projecttags %}{% load humanize %}'
            '{{data.cpu_time_system|format_none_and_zero|floatformat:2}}',
            hidden=True,
            orderable=True)

        self.toggle_columns['cpu_time_user'] = len(self.columns)

        self.add_column(
            title="User CPU time (secs)",
            static_data_name="cpu_time_user",
            static_data_template='{% load projecttags %}{% load humanize %}'
            '{{data.cpu_time_user|format_none_and_zero|floatformat:2}}',
            hidden=True,
            orderable=True)

        self.toggle_columns['disk_io'] = len(self.columns)

        self.add_column(
            title="Disk I/O (ms)",
            static_data_name="disk_io",
            static_data_template='{% load projecttags %}{% load humanize %}'
            '{{data.disk_io|format_none_and_zero|filtered_filesizeformat}}',
            hidden=True,
            orderable=True)


class BuildTimeTable(BuildTasksTable):
    """ Same as tasks table but the Time column is default displayed"""

    def __init__(self, *args, **kwargs):
        super(BuildTimeTable, self).__init__(*args, **kwargs)
        self.default_orderby = "-elapsed_time"

    def setup_columns(self, *args, **kwargs):
        super(BuildTimeTable, self).setup_columns(**kwargs)

        self.columns[self.toggle_columns['order']]['hidden'] = True
        self.columns[self.toggle_columns['order']]['hideable'] = True
        self.columns[self.toggle_columns['sstate_result']]['hidden'] = True
        self.columns[self.toggle_columns['elapsed_time']]['hidden'] = False


class BuildCPUTimeTable(BuildTasksTable):
    """ Same as tasks table but the CPU usage columns are default displayed"""

    def __init__(self, *args, **kwargs):
        super(BuildCPUTimeTable, self).__init__(*args, **kwargs)
        self.default_orderby = "-cpu_time_system"

    def setup_columns(self, *args, **kwargs):
        super(BuildCPUTimeTable, self).setup_columns(**kwargs)

        self.columns[self.toggle_columns['order']]['hidden'] = True
        self.columns[self.toggle_columns['order']]['hideable'] = True
        self.columns[self.toggle_columns['sstate_result']]['hidden'] = True
        self.columns[self.toggle_columns['cpu_time_sys']]['hidden'] = False
        self.columns[self.toggle_columns['cpu_time_user']]['hidden'] = False


class BuildIOTable(BuildTasksTable):
    """ Same as tasks table but the Disk IO column is default displayed"""

    def __init__(self, *args, **kwargs):
        super(BuildIOTable, self).__init__(*args, **kwargs)
        self.default_orderby = "-disk_io"

    def setup_columns(self, *args, **kwargs):
        super(BuildIOTable, self).setup_columns(**kwargs)

        self.columns[self.toggle_columns['order']]['hidden'] = True
        self.columns[self.toggle_columns['order']]['hideable'] = True
        self.columns[self.toggle_columns['sstate_result']]['hidden'] = True
        self.columns[self.toggle_columns['disk_io']]['hidden'] = False