| #!/usr/bin/env python |
| |
| r""" |
| Use robot framework API to extract test result data from output.xml generated |
| by robot tests. For more information on the Robot Framework API, see |
| http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html |
| """ |
| |
| import sys |
| import os |
| import getopt |
| import csv |
| import robot.errors |
| import re |
| from datetime import datetime |
| from robot.api import ExecutionResult |
| from robot.result.visitor import ResultVisitor |
| from xml.etree import ElementTree |
| |
| # Remove the python library path to restore with local project path later. |
| save_path_0 = sys.path[0] |
| del sys.path[0] |
| sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib")) |
| |
| from gen_arg import * |
| from gen_print import * |
| from gen_valid import * |
| |
| # Restore sys.path[0]. |
| sys.path.insert(0, save_path_0) |
| |
| |
| this_program = sys.argv[0] |
| info = " For more information: " + this_program + ' -h' |
| if len(sys.argv) == 1: |
| print (info) |
| sys.exit(1) |
| |
| |
| parser = argparse.ArgumentParser( |
| usage=info, |
| description="%(prog)s uses a robot framework API to extract test result\ |
| data from output.xml generated by robot tests. For more information on the\ |
| Robot Framework API, see\ |
| http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html", |
| formatter_class=argparse.ArgumentDefaultsHelpFormatter, |
| prefix_chars='-+') |
| |
| parser.add_argument( |
| '--source', |
| '-s', |
| help='The output.xml robot test result file path. This parameter is \ |
| required.') |
| |
| parser.add_argument( |
| '--dest', |
| '-d', |
| help='The directory path where the generated .csv files will go. This \ |
| parameter is required.') |
| |
| parser.add_argument( |
| '--version_id', |
| help='Driver version of openbmc firmware which was used during test,\ |
| e.g. "v2.1-215-g6e7eacb". This parameter is required.') |
| |
| parser.add_argument( |
| '--platform', |
| help='OpenBMC platform which was used during test,\ |
| e.g. "Witherspoon". This parameter is required.') |
| |
| parser.add_argument( |
| '--level', |
| help='OpenBMC release level which was used during test,\ |
| e.g. "Master", "OBMC920". This parameter is required.') |
| |
| parser.add_argument( |
| '--test_phase', |
| help='Name of testing phase, e.g. "DVT", "SVT", etc.\ |
| This parameter is optional.', |
| default="FVT") |
| |
| parser.add_argument( |
| '--processor', |
| help='Name of processor, e.g. "P9". This parameter is optional.', |
| default="OPENPOWER") |
| |
| |
| # Populate stock_list with options we want. |
| stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)] |
| |
| |
| def exit_function(signal_number=0, |
| frame=None): |
| r""" |
| Execute whenever the program ends normally or with the signals that we |
| catch (i.e. TERM, INT). |
| """ |
| |
| dprint_executing() |
| |
| dprint_var(signal_number) |
| |
| qprint_pgm_footer() |
| |
| |
| def signal_handler(signal_number, |
| frame): |
| r""" |
| Handle signals. Without a function to catch a SIGTERM or SIGINT, the |
| program would terminate immediately with return code 143 and without |
| calling the exit_function. |
| """ |
| |
| # Our convention is to set up exit_function with atexit.register() so |
| # there is no need to explicitly call exit_function from here. |
| |
| dprint_executing() |
| |
| # Calling exit prevents us from returning to the code that was running |
| # when the signal was received. |
| exit(0) |
| |
| |
| def validate_parms(): |
| r""" |
| Validate program parameters, etc. Return True or False (i.e. pass/fail) |
| accordingly. |
| """ |
| |
| if not valid_file_path(source): |
| return False |
| |
| if not valid_dir_path(dest): |
| return False |
| |
| gen_post_validation(exit_function, signal_handler) |
| |
| return True |
| |
| |
| def parse_output_xml(xml_file_path, csv_dir_path, version_id, platform, level, |
| test_phase, processor): |
| r""" |
| Parse the robot-generated output.xml file and extract various test |
| output data. Put the extracted information into a csv file in the "dest" |
| folder. |
| |
| Description of argument(s): |
| xml_file_path The path to a Robot-generated output.xml |
| file. |
| csv_dir_path The path to the directory that is to |
| contain the .csv files generated by |
| this function. |
| version_id Version of the openbmc firmware |
| (e.g. "v2.1-215-g6e7eacb"). |
| platform Platform of the openbmc system. |
| level Release level of the OpenBMC system |
| (e.g. "Master"). |
| """ |
| |
| result = ExecutionResult(xml_file_path) |
| result.configure(stat_config={'suite_stat_level': 2, |
| 'tag_stat_combine': 'tagANDanother'}) |
| |
| stats = result.statistics |
| print("--------------------------------------") |
| total_tc = stats.total.critical.passed + stats.total.critical.failed |
| print("Total Test Count:\t %d" % total_tc) |
| print("Total Test Failed:\t %d" % stats.total.critical.failed) |
| print("Total Test Passed:\t %d" % stats.total.critical.passed) |
| print("Test Start Time:\t %s" % result.suite.starttime) |
| print("Test End Time:\t\t %s" % result.suite.endtime) |
| print("--------------------------------------") |
| |
| # Use ResultVisitor object and save off the test data info |
| class TestResult(ResultVisitor): |
| def __init__(self): |
| self.testData = [] |
| |
| def visit_test(self, test): |
| self.testData += [test] |
| |
| collectDataObj = TestResult() |
| result.visit(collectDataObj) |
| |
| # Write the result statistics attributes to CSV file |
| l_csvlist = [] |
| |
| # Default Test data |
| l_subsys = 'OPENBMC' |
| l_test_type = test_phase |
| |
| l_pse_rel = 'Master' |
| if level: |
| l_pse_rel = level |
| |
| l_env = 'HW' |
| l_proc = processor |
| l_platform_type = "" |
| l_func_area = "" |
| |
| # System data from XML meta data |
| # l_system_info = get_system_details(xml_file_path) |
| |
| # First let us try to collect information from keyboard input |
| # If keyboard input cannot give both information, then find from xml file. |
| if version_id and platform: |
| l_driver = version_id |
| l_platform_type = platform |
| print("BMC Version_id:%s" % version_id) |
| print("BMC Platform:%s" % platform) |
| else: |
| # System data from XML meta data |
| l_system_info = get_system_details(xml_file_path) |
| l_driver = l_system_info[0] |
| l_platform_type = l_system_info[1] |
| |
| # Driver version id and platform are mandatorily required for CSV file |
| # generation. If any one is not avaulable, exit CSV file generation |
| # process. |
| if l_driver and l_platform_type: |
| print("Driver and system info set.") |
| else: |
| print("Both driver and system info need to be set.\ |
| CSV file is not generated.") |
| sys.exit() |
| |
| # Default header |
| l_header = ['test_start', 'test_end', 'subsys', 'test_type', |
| 'test_result', 'test_name', 'pse_rel', 'driver', |
| 'env', 'proc', 'platform_type', 'test_func_area'] |
| |
| l_csvlist.append(l_header) |
| |
| # Generate CSV file onto the path with current time stamp |
| l_base_dir = csv_dir_path |
| l_timestamp = datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S") |
| # Example: 2017-02-20-08-47-22_Witherspoon.csv |
| l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv" |
| |
| print("Writing data into csv file:%s" % l_csvfile) |
| |
| for testcase in collectDataObj.testData: |
| # Functional Area: Suite Name |
| # Test Name: Test Case Name |
| l_func_area = str(testcase.parent).split(' ', 1)[1] |
| l_test_name = str(testcase) |
| |
| # Test Result pass=0 fail=1 |
| if testcase.status == 'PASS': |
| l_test_result = 0 |
| else: |
| l_test_result = 1 |
| |
| # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S" |
| l_stime = xml_to_csv_time(testcase.starttime) |
| l_etime = xml_to_csv_time(testcase.endtime) |
| # Data Sequence: test_start,test_end,subsys,test_type, |
| # test_result,test_name,pse_rel,driver, |
| # env,proc,platform_type,test_func_area, |
| l_data = [l_stime, l_etime, l_subsys, l_test_type, l_test_result, |
| l_test_name, l_pse_rel, l_driver, l_env, l_proc, |
| l_platform_type, l_func_area] |
| l_csvlist.append(l_data) |
| |
| # Open the file and write to the CSV file |
| l_file = open(l_csvfile, "w") |
| l_writer = csv.writer(l_file, lineterminator='\n') |
| l_writer.writerows(l_csvlist) |
| l_file.close() |
| |
| |
| def xml_to_csv_time(xml_datetime): |
| r""" |
| Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format |
| and return it. |
| |
| Description of argument(s): |
| datetime The date in the following format: %Y%m%d |
| %H:%M:%S.%f (This is the format |
| typically found in an XML file.) |
| |
| The date returned will be in the following format: %Y-%m-%d-%H-%M-%S |
| """ |
| |
| # 20170206 05:05:19.342 |
| l_str = datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f") |
| # 2017-02-06-05-05-19 |
| l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S") |
| return str(l_str) |
| |
| |
| def get_system_details(xml_file_path): |
| r""" |
| Get the system data from output.xml generated by robot and return it. |
| The list returned will be in the following order: [driver,platform] |
| |
| Description of argument(s): |
| xml_file_path The relative or absolute path to the |
| output.xml file. |
| """ |
| |
| bmc_version_id = "" |
| bmc_platform = "" |
| with open(xml_file_path, 'rt') as output: |
| tree = ElementTree.parse(output) |
| |
| for node in tree.iter('msg'): |
| # /etc/os-release output is logged in the XML as msg |
| # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79" |
| if '${output} = VERSION_ID=' in node.text: |
| # Get BMC version (e.g. v1.99.1-96-g2a46570) |
| bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1] |
| |
| # Platform is logged in the XML as msg. |
| # Example: ${bmc_model} = Witherspoon BMC |
| if '${bmc_model} = ' in node.text: |
| bmc_platform = node.text.split(" = ")[1] |
| |
| print_vars(bmc_version_id, bmc_platform) |
| return [str(bmc_version_id), str(bmc_platform)] |
| |
| |
| def main(): |
| |
| if not gen_get_options(parser, stock_list): |
| return False |
| |
| if not validate_parms(): |
| return False |
| |
| qprint_pgm_header() |
| |
| parse_output_xml(source, dest, version_id, platform, level, |
| test_phase, processor) |
| |
| return True |
| |
| |
| # Main |
| |
| if not main(): |
| exit(1) |