blob: bbc66d9ca3a6b63d9f8c21f1feca73951277b409 [file] [log] [blame]
#!/usr/bin/env python3
r"""
Use robot framework API to extract test result data from output.xml generated
by robot tests. For more information on the Robot Framework API, see
http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html
"""
import csv
import datetime
import getopt
import os
import re
import stat
import sys
from xml.etree import ElementTree
import robot.errors
from robot.api import ExecutionResult
from robot.result.visitor import ResultVisitor
# Remove the python library path to restore with local project path later.
save_path_0 = sys.path[0]
del sys.path[0]
sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
from gen_arg import * # NOQA
from gen_print import * # NOQA
from gen_valid import * # NOQA
# Restore sys.path[0].
sys.path.insert(0, save_path_0)
this_program = sys.argv[0]
info = " For more information: " + this_program + " -h"
if len(sys.argv) == 1:
print(info)
sys.exit(1)
parser = argparse.ArgumentParser(
usage=info,
description=(
"%(prog)s uses a robot framework API to extract test result data"
" from output.xml generated by robot tests. For more information on"
" the Robot Framework API, see "
" http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html"
),
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
prefix_chars="-+",
)
parser.add_argument(
"--source",
"-s",
help=(
"The output.xml robot test result file path. This parameter is "
" required."
),
)
parser.add_argument(
"--dest",
"-d",
help=(
"The directory path where the generated .csv files will go. This "
" parameter is required."
),
)
parser.add_argument(
"--version_id",
help=(
"Driver version of openbmc firmware which was used during test, "
' e.g. "v2.1-215-g6e7eacb". This parameter is required.'
),
)
parser.add_argument(
"--platform",
help=(
"OpenBMC platform which was used during test, e.g."
' "Witherspoon". This parameter is required.'
),
)
parser.add_argument(
"--level",
help=(
"OpenBMC release level which was used during test, e.g."
' "Master", "OBMC920". This parameter is required.'
),
)
parser.add_argument(
"--test_phase",
help=(
'Name of testing phase, e.g. "DVT", "SVT", etc. This'
" parameter is optional."
),
default="FVT",
)
parser.add_argument(
"--subsystem",
help=(
'Name of the subsystem, e.g. "OPENBMC" etc. This parameter is'
" optional."
),
default="OPENBMC",
)
parser.add_argument(
"--processor",
help='Name of processor, e.g. "P9". This parameter is optional.',
default="OPENPOWER",
)
# Populate stock_list with options we want.
stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)]
def exit_function(signal_number=0, frame=None):
r"""
Execute whenever the program ends normally or with the signals that we
catch (i.e. TERM, INT).
"""
dprint_executing()
dprint_var(signal_number)
qprint_pgm_footer()
def signal_handler(signal_number, frame):
r"""
Handle signals. Without a function to catch a SIGTERM or SIGINT, the
program would terminate immediately with return code 143 and without
calling the exit_function.
"""
# Our convention is to set up exit_function with atexit.register() so
# there is no need to explicitly call exit_function from here.
dprint_executing()
# Calling exit prevents us from returning to the code that was running
# when the signal was received.
exit(0)
def validate_parms():
r"""
Validate program parameters, etc. Return True or False (i.e. pass/fail)
accordingly.
"""
if not valid_file_path(source):
return False
if not valid_dir_path(dest):
return False
gen_post_validation(exit_function, signal_handler)
return True
def parse_output_xml(
xml_file_path,
csv_dir_path,
version_id,
platform,
level,
test_phase,
processor,
):
r"""
Parse the robot-generated output.xml file and extract various test
output data. Put the extracted information into a csv file in the "dest"
folder.
Description of argument(s):
xml_file_path The path to a Robot-generated output.xml
file.
csv_dir_path The path to the directory that is to
contain the .csv files generated by
this function.
version_id Version of the openbmc firmware
(e.g. "v2.1-215-g6e7eacb").
platform Platform of the openbmc system.
level Release level of the OpenBMC system
(e.g. "Master").
"""
# Initialize tallies
total_critical_tc = 0
total_critical_passed = 0
total_critical_failed = 0
total_non_critical_tc = 0
total_non_critical_passed = 0
total_non_critical_failed = 0
result = ExecutionResult(xml_file_path)
result.configure(
stat_config={
"suite_stat_level": 2,
"tag_stat_combine": "tagANDanother",
}
)
stats = result.statistics
print("--------------------------------------")
try:
total_critical_tc = (
stats.total.critical.passed + stats.total.critical.failed
)
total_critical_passed = stats.total.critical.passed
total_critical_failed = stats.total.critical.failed
except AttributeError:
pass
try:
total_non_critical_tc = stats.total.passed + stats.total.failed
total_non_critical_passed = stats.total.passed
total_non_critical_failed = stats.total.failed
except AttributeError:
pass
print(
"Total Test Count:\t %d" % (total_non_critical_tc + total_critical_tc)
)
print("Total Critical Test Failed:\t %d" % total_critical_failed)
print("Total Critical Test Passed:\t %d" % total_critical_passed)
print("Total Non-Critical Test Failed:\t %d" % total_non_critical_failed)
print("Total Non-Critical Test Passed:\t %d" % total_non_critical_passed)
print("Test Start Time:\t %s" % result.suite.starttime)
print("Test End Time:\t\t %s" % result.suite.endtime)
print("--------------------------------------")
# Use ResultVisitor object and save off the test data info
class TestResult(ResultVisitor):
def __init__(self):
self.testData = []
def visit_test(self, test):
self.testData += [test]
collectDataObj = TestResult()
result.visit(collectDataObj)
# Write the result statistics attributes to CSV file
l_csvlist = []
# Default Test data
l_test_type = test_phase
l_pse_rel = "Master"
if level:
l_pse_rel = level
l_env = "HW"
l_proc = processor
l_platform_type = ""
l_func_area = ""
# System data from XML meta data
# l_system_info = get_system_details(xml_file_path)
# First let us try to collect information from keyboard input
# If keyboard input cannot give both information, then find from xml file.
if version_id and platform:
l_driver = version_id
l_platform_type = platform
print("BMC Version_id:%s" % version_id)
print("BMC Platform:%s" % platform)
else:
# System data from XML meta data
l_system_info = get_system_details(xml_file_path)
l_driver = l_system_info[0]
l_platform_type = l_system_info[1]
# Driver version id and platform are mandatorily required for CSV file
# generation. If any one is not avaulable, exit CSV file generation
# process.
if l_driver and l_platform_type:
print("Driver and system info set.")
else:
print(
"Both driver and system info need to be set. CSV"
" file is not generated."
)
sys.exit()
# Default header
l_header = [
"test_start",
"test_end",
"subsys",
"test_type",
"test_result",
"test_name",
"pse_rel",
"driver",
"env",
"proc",
"platform_type",
"test_func_area",
]
l_csvlist.append(l_header)
# Generate CSV file onto the path with current time stamp
l_base_dir = csv_dir_path
l_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
# Example: 2017-02-20-08-47-22_Witherspoon.csv
l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv"
print("Writing data into csv file:%s" % l_csvfile)
for testcase in collectDataObj.testData:
# Functional Area: Suite Name
# Test Name: Test Case Name
l_func_area = str(testcase.parent).split(" ", 1)[1]
l_test_name = str(testcase)
# Test Result pass=0 fail=1
if testcase.status == "PASS":
l_test_result = 0
else:
l_test_result = 1
# Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S"
l_stime = xml_to_csv_time(testcase.starttime)
l_etime = xml_to_csv_time(testcase.endtime)
# Data Sequence: test_start,test_end,subsys,test_type,
# test_result,test_name,pse_rel,driver,
# env,proc,platform_type,test_func_area,
l_data = [
l_stime,
l_etime,
subsystem,
l_test_type,
l_test_result,
l_test_name,
l_pse_rel,
l_driver,
l_env,
l_proc,
l_platform_type,
l_func_area,
]
l_csvlist.append(l_data)
# Open the file and write to the CSV file
l_file = open(l_csvfile, "w")
l_writer = csv.writer(l_file, lineterminator="\n")
l_writer.writerows(l_csvlist)
l_file.close()
# Set file permissions 666.
perm = (
stat.S_IRUSR
+ stat.S_IWUSR
+ stat.S_IRGRP
+ stat.S_IWGRP
+ stat.S_IROTH
+ stat.S_IWOTH
)
os.chmod(l_csvfile, perm)
def xml_to_csv_time(xml_datetime):
r"""
Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format
and return it.
Description of argument(s):
datetime The date in the following format: %Y%m%d
%H:%M:%S.%f (This is the format
typically found in an XML file.)
The date returned will be in the following format: %Y-%m-%d-%H-%M-%S
"""
# 20170206 05:05:19.342
l_str = datetime.datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f")
# 2017-02-06-05-05-19
l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S")
return str(l_str)
def get_system_details(xml_file_path):
r"""
Get the system data from output.xml generated by robot and return it.
The list returned will be in the following order: [driver,platform]
Description of argument(s):
xml_file_path The relative or absolute path to the
output.xml file.
"""
bmc_version_id = ""
bmc_platform = ""
with open(xml_file_path, "rt") as output:
tree = ElementTree.parse(output)
for node in tree.iter("msg"):
# /etc/os-release output is logged in the XML as msg
# Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79"
if "${output} = VERSION_ID=" in node.text:
# Get BMC version (e.g. v1.99.1-96-g2a46570)
bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1]
# Platform is logged in the XML as msg.
# Example: ${bmc_model} = Witherspoon BMC
if "${bmc_model} = " in node.text:
bmc_platform = node.text.split(" = ")[1]
print_vars(bmc_version_id, bmc_platform)
return [str(bmc_version_id), str(bmc_platform)]
def main():
if not gen_get_options(parser, stock_list):
return False
if not validate_parms():
return False
qprint_pgm_header()
parse_output_xml(
source, dest, version_id, platform, level, test_phase, processor
)
return True
# Main
if not main():
exit(1)