blob: a2dbc77d948f3501b6c019d1751a5fc4104dc016 [file] [log] [blame]
#!/usr/bin/env python
r"""
Use robot framework API to extract test result data from output.xml generated
by robot tests. For more information on the Robot Framework API, see
http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html
"""
import sys
import os
import getopt
import csv
import robot.errors
import re
import stat
from datetime import datetime
from robot.api import ExecutionResult
from robot.result.visitor import ResultVisitor
from xml.etree import ElementTree
# Remove the python library path to restore with local project path later.
save_path_0 = sys.path[0]
del sys.path[0]
sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
from gen_arg import *
from gen_print import *
from gen_valid import *
# Restore sys.path[0].
sys.path.insert(0, save_path_0)
this_program = sys.argv[0]
info = " For more information: " + this_program + ' -h'
if len(sys.argv) == 1:
print (info)
sys.exit(1)
parser = argparse.ArgumentParser(
usage=info,
description="%(prog)s uses a robot framework API to extract test result\
data from output.xml generated by robot tests. For more information on the\
Robot Framework API, see\
http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
prefix_chars='-+')
parser.add_argument(
'--source',
'-s',
help='The output.xml robot test result file path. This parameter is \
required.')
parser.add_argument(
'--dest',
'-d',
help='The directory path where the generated .csv files will go. This \
parameter is required.')
parser.add_argument(
'--version_id',
help='Driver version of openbmc firmware which was used during test,\
e.g. "v2.1-215-g6e7eacb". This parameter is required.')
parser.add_argument(
'--platform',
help='OpenBMC platform which was used during test,\
e.g. "Witherspoon". This parameter is required.')
parser.add_argument(
'--level',
help='OpenBMC release level which was used during test,\
e.g. "Master", "OBMC920". This parameter is required.')
parser.add_argument(
'--test_phase',
help='Name of testing phase, e.g. "DVT", "SVT", etc.\
This parameter is optional.',
default="FVT")
parser.add_argument(
'--processor',
help='Name of processor, e.g. "P9". This parameter is optional.',
default="OPENPOWER")
# Populate stock_list with options we want.
stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)]
def exit_function(signal_number=0,
frame=None):
r"""
Execute whenever the program ends normally or with the signals that we
catch (i.e. TERM, INT).
"""
dprint_executing()
dprint_var(signal_number)
qprint_pgm_footer()
def signal_handler(signal_number,
frame):
r"""
Handle signals. Without a function to catch a SIGTERM or SIGINT, the
program would terminate immediately with return code 143 and without
calling the exit_function.
"""
# Our convention is to set up exit_function with atexit.register() so
# there is no need to explicitly call exit_function from here.
dprint_executing()
# Calling exit prevents us from returning to the code that was running
# when the signal was received.
exit(0)
def validate_parms():
r"""
Validate program parameters, etc. Return True or False (i.e. pass/fail)
accordingly.
"""
if not valid_file_path(source):
return False
if not valid_dir_path(dest):
return False
gen_post_validation(exit_function, signal_handler)
return True
def parse_output_xml(xml_file_path, csv_dir_path, version_id, platform, level,
test_phase, processor):
r"""
Parse the robot-generated output.xml file and extract various test
output data. Put the extracted information into a csv file in the "dest"
folder.
Description of argument(s):
xml_file_path The path to a Robot-generated output.xml
file.
csv_dir_path The path to the directory that is to
contain the .csv files generated by
this function.
version_id Version of the openbmc firmware
(e.g. "v2.1-215-g6e7eacb").
platform Platform of the openbmc system.
level Release level of the OpenBMC system
(e.g. "Master").
"""
result = ExecutionResult(xml_file_path)
result.configure(stat_config={'suite_stat_level': 2,
'tag_stat_combine': 'tagANDanother'})
stats = result.statistics
print("--------------------------------------")
total_tc = stats.total.critical.passed + stats.total.critical.failed
print("Total Test Count:\t %d" % total_tc)
print("Total Test Failed:\t %d" % stats.total.critical.failed)
print("Total Test Passed:\t %d" % stats.total.critical.passed)
print("Test Start Time:\t %s" % result.suite.starttime)
print("Test End Time:\t\t %s" % result.suite.endtime)
print("--------------------------------------")
# Use ResultVisitor object and save off the test data info
class TestResult(ResultVisitor):
def __init__(self):
self.testData = []
def visit_test(self, test):
self.testData += [test]
collectDataObj = TestResult()
result.visit(collectDataObj)
# Write the result statistics attributes to CSV file
l_csvlist = []
# Default Test data
l_subsys = 'OPENBMC'
l_test_type = test_phase
l_pse_rel = 'Master'
if level:
l_pse_rel = level
l_env = 'HW'
l_proc = processor
l_platform_type = ""
l_func_area = ""
# System data from XML meta data
# l_system_info = get_system_details(xml_file_path)
# First let us try to collect information from keyboard input
# If keyboard input cannot give both information, then find from xml file.
if version_id and platform:
l_driver = version_id
l_platform_type = platform
print("BMC Version_id:%s" % version_id)
print("BMC Platform:%s" % platform)
else:
# System data from XML meta data
l_system_info = get_system_details(xml_file_path)
l_driver = l_system_info[0]
l_platform_type = l_system_info[1]
# Driver version id and platform are mandatorily required for CSV file
# generation. If any one is not avaulable, exit CSV file generation
# process.
if l_driver and l_platform_type:
print("Driver and system info set.")
else:
print("Both driver and system info need to be set.\
CSV file is not generated.")
sys.exit()
# Default header
l_header = ['test_start', 'test_end', 'subsys', 'test_type',
'test_result', 'test_name', 'pse_rel', 'driver',
'env', 'proc', 'platform_type', 'test_func_area']
l_csvlist.append(l_header)
# Generate CSV file onto the path with current time stamp
l_base_dir = csv_dir_path
l_timestamp = datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
# Example: 2017-02-20-08-47-22_Witherspoon.csv
l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv"
print("Writing data into csv file:%s" % l_csvfile)
for testcase in collectDataObj.testData:
# Functional Area: Suite Name
# Test Name: Test Case Name
l_func_area = str(testcase.parent).split(' ', 1)[1]
l_test_name = str(testcase)
# Test Result pass=0 fail=1
if testcase.status == 'PASS':
l_test_result = 0
else:
l_test_result = 1
# Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S"
l_stime = xml_to_csv_time(testcase.starttime)
l_etime = xml_to_csv_time(testcase.endtime)
# Data Sequence: test_start,test_end,subsys,test_type,
# test_result,test_name,pse_rel,driver,
# env,proc,platform_type,test_func_area,
l_data = [l_stime, l_etime, l_subsys, l_test_type, l_test_result,
l_test_name, l_pse_rel, l_driver, l_env, l_proc,
l_platform_type, l_func_area]
l_csvlist.append(l_data)
# Open the file and write to the CSV file
l_file = open(l_csvfile, "w")
l_writer = csv.writer(l_file, lineterminator='\n')
l_writer.writerows(l_csvlist)
l_file.close()
# Set file permissions 666.
perm = stat.S_IRUSR + stat.S_IWUSR + stat.S_IRGRP + stat.S_IWGRP + stat.S_IROTH + stat.S_IWOTH
os.chmod(l_csvfile, perm)
def xml_to_csv_time(xml_datetime):
r"""
Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format
and return it.
Description of argument(s):
datetime The date in the following format: %Y%m%d
%H:%M:%S.%f (This is the format
typically found in an XML file.)
The date returned will be in the following format: %Y-%m-%d-%H-%M-%S
"""
# 20170206 05:05:19.342
l_str = datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f")
# 2017-02-06-05-05-19
l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S")
return str(l_str)
def get_system_details(xml_file_path):
r"""
Get the system data from output.xml generated by robot and return it.
The list returned will be in the following order: [driver,platform]
Description of argument(s):
xml_file_path The relative or absolute path to the
output.xml file.
"""
bmc_version_id = ""
bmc_platform = ""
with open(xml_file_path, 'rt') as output:
tree = ElementTree.parse(output)
for node in tree.iter('msg'):
# /etc/os-release output is logged in the XML as msg
# Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79"
if '${output} = VERSION_ID=' in node.text:
# Get BMC version (e.g. v1.99.1-96-g2a46570)
bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1]
# Platform is logged in the XML as msg.
# Example: ${bmc_model} = Witherspoon BMC
if '${bmc_model} = ' in node.text:
bmc_platform = node.text.split(" = ")[1]
print_vars(bmc_version_id, bmc_platform)
return [str(bmc_version_id), str(bmc_platform)]
def main():
if not gen_get_options(parser, stock_list):
return False
if not validate_parms():
return False
qprint_pgm_header()
parse_output_xml(source, dest, version_id, platform, level,
test_phase, processor)
return True
# Main
if not main():
exit(1)