tools/ct_metrics/gen_csv_results.py

#!/usr/bin/env python

r"""
Use robot framework API to extract test result data from output.xml generated
by robot tests. For more information on the Robot Framework API, see
http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html
"""

import sys
import os
import getopt
import csv
import robot.errors
import re
import stat
from datetime import datetime
from robot.api import ExecutionResult
from robot.result.visitor import ResultVisitor
from xml.etree import ElementTree

# Remove the python library path to restore with local project path later.
save_path_0 = sys.path[0]
del sys.path[0]
sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))

from gen_arg import *
from gen_print import *
from gen_valid import *

# Restore sys.path[0].
sys.path.insert(0, save_path_0)


this_program = sys.argv[0]
info = " For more information:  " + this_program + '  -h'
if len(sys.argv) == 1:
    print (info)
    sys.exit(1)


parser = argparse.ArgumentParser(
    usage=info,
    description="%(prog)s uses a robot framework API to extract test result\
    data from output.xml generated by robot tests. For more information on the\
    Robot Framework API, see\
    http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html",
    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    prefix_chars='-+')

parser.add_argument(
    '--source',
    '-s',
    help='The output.xml robot test result file path.  This parameter is \
          required.')

parser.add_argument(
    '--dest',
    '-d',
    help='The directory path where the generated .csv files will go.  This \
          parameter is required.')

parser.add_argument(
    '--version_id',
    help='Driver version of openbmc firmware which was used during test,\
          e.g. "v2.1-215-g6e7eacb".  This parameter is required.')

parser.add_argument(
    '--platform',
    help='OpenBMC platform which was used during test,\
          e.g. "Witherspoon".  This parameter is required.')

parser.add_argument(
    '--level',
    help='OpenBMC release level which was used during test,\
          e.g. "Master", "OBMC920".  This parameter is required.')

parser.add_argument(
    '--test_phase',
    help='Name of testing phase, e.g. "DVT", "SVT", etc.\
          This parameter is optional.',
    default="FVT")

parser.add_argument(
    '--processor',
    help='Name of processor, e.g. "P9". This parameter is optional.',
    default="OPENPOWER")


# Populate stock_list with options we want.
stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)]


def exit_function(signal_number=0,
                  frame=None):
    r"""
    Execute whenever the program ends normally or with the signals that we
    catch (i.e. TERM, INT).
    """

    dprint_executing()

    dprint_var(signal_number)

    qprint_pgm_footer()


def signal_handler(signal_number,
                   frame):
    r"""
    Handle signals.  Without a function to catch a SIGTERM or SIGINT, the
    program would terminate immediately with return code 143 and without
    calling the exit_function.
    """

    # Our convention is to set up exit_function with atexit.register() so
    # there is no need to explicitly call exit_function from here.

    dprint_executing()

    # Calling exit prevents us from returning to the code that was running
    # when the signal was received.
    exit(0)


def validate_parms():
    r"""
    Validate program parameters, etc.  Return True or False (i.e. pass/fail)
    accordingly.
    """

    if not valid_file_path(source):
        return False

    if not valid_dir_path(dest):
        return False

    gen_post_validation(exit_function, signal_handler)

    return True


def parse_output_xml(xml_file_path, csv_dir_path, version_id, platform, level,
                     test_phase, processor):
    r"""
    Parse the robot-generated output.xml file and extract various test
    output data. Put the extracted information into a csv file in the "dest"
    folder.

    Description of argument(s):
    xml_file_path                   The path to a Robot-generated output.xml
                                    file.
    csv_dir_path                    The path to the directory that is to
                                    contain the .csv files generated by
                                    this function.
    version_id                      Version of the openbmc firmware
                                    (e.g. "v2.1-215-g6e7eacb").
    platform                        Platform of the openbmc system.
    level                           Release level of the OpenBMC system
                                    (e.g. "Master").
    """

    result = ExecutionResult(xml_file_path)
    result.configure(stat_config={'suite_stat_level': 2,
                                  'tag_stat_combine': 'tagANDanother'})

    stats = result.statistics
    print("--------------------------------------")
    total_tc = stats.total.critical.passed + stats.total.critical.failed
    print("Total Test Count:\t %d" % total_tc)
    print("Total Test Failed:\t %d" % stats.total.critical.failed)
    print("Total Test Passed:\t %d" % stats.total.critical.passed)
    print("Test Start Time:\t %s" % result.suite.starttime)
    print("Test End Time:\t\t %s" % result.suite.endtime)
    print("--------------------------------------")

    # Use ResultVisitor object and save off the test data info
    class TestResult(ResultVisitor):
        def __init__(self):
            self.testData = []

        def visit_test(self, test):
            self.testData += [test]

    collectDataObj = TestResult()
    result.visit(collectDataObj)

    # Write the result statistics attributes to CSV file
    l_csvlist = []

    # Default Test data
    l_subsys = 'OPENBMC'
    l_test_type = test_phase

    l_pse_rel = 'Master'
    if level:
        l_pse_rel = level

    l_env = 'HW'
    l_proc = processor
    l_platform_type = ""
    l_func_area = ""

    # System data from XML meta data
    # l_system_info = get_system_details(xml_file_path)

    # First let us try to collect information from keyboard input
    # If keyboard input cannot give both information, then find from xml file.
    if version_id and platform:
        l_driver = version_id
        l_platform_type = platform
        print("BMC Version_id:%s" % version_id)
        print("BMC Platform:%s" % platform)
    else:
        # System data from XML meta data
        l_system_info = get_system_details(xml_file_path)
        l_driver = l_system_info[0]
        l_platform_type = l_system_info[1]

    # Driver version id and platform are mandatorily required for CSV file
    # generation. If any one is not avaulable, exit CSV file generation
    # process.
    if l_driver and l_platform_type:
        print("Driver and system info set.")
    else:
        print("Both driver and system info need to be set.\
                CSV file is not generated.")
        sys.exit()

    # Default header
    l_header = ['test_start', 'test_end', 'subsys', 'test_type',
                'test_result', 'test_name', 'pse_rel', 'driver',
                'env', 'proc', 'platform_type', 'test_func_area']

    l_csvlist.append(l_header)

    # Generate CSV file onto the path with current time stamp
    l_base_dir = csv_dir_path
    l_timestamp = datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
    # Example: 2017-02-20-08-47-22_Witherspoon.csv
    l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv"

    print("Writing data into csv file:%s" % l_csvfile)

    for testcase in collectDataObj.testData:
        # Functional Area: Suite Name
        # Test Name: Test Case Name
        l_func_area = str(testcase.parent).split(' ', 1)[1]
        l_test_name = str(testcase)

        # Test Result pass=0 fail=1
        if testcase.status == 'PASS':
            l_test_result = 0
        else:
            l_test_result = 1

        # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S"
        l_stime = xml_to_csv_time(testcase.starttime)
        l_etime = xml_to_csv_time(testcase.endtime)
        # Data Sequence: test_start,test_end,subsys,test_type,
        #                test_result,test_name,pse_rel,driver,
        #                env,proc,platform_type,test_func_area,
        l_data = [l_stime, l_etime, l_subsys, l_test_type, l_test_result,
                  l_test_name, l_pse_rel, l_driver, l_env, l_proc,
                  l_platform_type, l_func_area]
        l_csvlist.append(l_data)

    # Open the file and write to the CSV file
    l_file = open(l_csvfile, "w")
    l_writer = csv.writer(l_file, lineterminator='\n')
    l_writer.writerows(l_csvlist)
    l_file.close()
    # Set file permissions 666.
    perm = stat.S_IRUSR + stat.S_IWUSR + stat.S_IRGRP + stat.S_IWGRP + stat.S_IROTH + stat.S_IWOTH
    os.chmod(l_csvfile, perm)


def xml_to_csv_time(xml_datetime):
    r"""
    Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format
    and return it.

    Description of argument(s):
    datetime                        The date in the following format: %Y%m%d
                                    %H:%M:%S.%f (This is the format
                                    typically found in an XML file.)

    The date returned will be in the following format: %Y-%m-%d-%H-%M-%S
    """

    # 20170206 05:05:19.342
    l_str = datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f")
    # 2017-02-06-05-05-19
    l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S")
    return str(l_str)


def get_system_details(xml_file_path):
    r"""
    Get the system data from output.xml generated by robot and return it.
    The list returned will be in the following order: [driver,platform]

    Description of argument(s):
    xml_file_path                   The relative or absolute path to the
                                    output.xml file.
    """

    bmc_version_id = ""
    bmc_platform = ""
    with open(xml_file_path, 'rt') as output:
        tree = ElementTree.parse(output)

    for node in tree.iter('msg'):
        # /etc/os-release output is logged in the XML as msg
        # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79"
        if '${output} = VERSION_ID=' in node.text:
            # Get BMC version (e.g. v1.99.1-96-g2a46570)
            bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1]

        # Platform is logged in the XML as msg.
        # Example: ${bmc_model} = Witherspoon BMC
        if '${bmc_model} = ' in node.text:
            bmc_platform = node.text.split(" = ")[1]

    print_vars(bmc_version_id, bmc_platform)
    return [str(bmc_version_id), str(bmc_platform)]


def main():

    if not gen_get_options(parser, stock_list):
        return False

    if not validate_parms():
        return False

    qprint_pgm_header()

    parse_output_xml(source, dest, version_id, platform, level,
                     test_phase, processor)

    return True


# Main

if not main():
    exit(1)