tools/ct_metrics/gen_csv_results.py - openbmc/openbmc-test-automation - Gitiles

 #!/usr/bin/env python3

 r"""
 Use robot framework API to extract test result data from output.xml generated
 by robot tests. For more information on the Robot Framework API, see
 http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html
 """

 import sys
 import os
 import getopt
 import csv
 import robot.errors
 import re
 import stat
 import datetime

 from robot.api import ExecutionResult
 from robot.result.visitor import ResultVisitor
 from xml.etree import ElementTree

 from gen_arg import *
 from gen_print import *
 from gen_valid import *

 # Remove the python library path to restore with local project path later.
 save_path_0 = sys.path[0]
 del sys.path[0]
 sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))

 # Restore sys.path[0].
 sys.path.insert(0, save_path_0)


 this_program = sys.argv[0]
 info = " For more information:  " + this_program + '  -h'
 if len(sys.argv) == 1:
     print(info)
     sys.exit(1)


 parser = argparse.ArgumentParser(
     usage=info,
     description="%(prog)s uses a robot framework API to extract test result\
     data from output.xml generated by robot tests. For more information on the\
     Robot Framework API, see\
     http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html",
     formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     prefix_chars='-+')

 parser.add_argument(
     '--source',
     '-s',
     help='The output.xml robot test result file path.  This parameter is \
           required.')

 parser.add_argument(
     '--dest',
     '-d',
     help='The directory path where the generated .csv files will go.  This \
           parameter is required.')

 parser.add_argument(
     '--version_id',
     help='Driver version of openbmc firmware which was used during test,\
           e.g. "v2.1-215-g6e7eacb".  This parameter is required.')

 parser.add_argument(
     '--platform',
     help='OpenBMC platform which was used during test,\
           e.g. "Witherspoon".  This parameter is required.')

 parser.add_argument(
     '--level',
     help='OpenBMC release level which was used during test,\
           e.g. "Master", "OBMC920".  This parameter is required.')

 parser.add_argument(
     '--test_phase',
     help='Name of testing phase, e.g. "DVT", "SVT", etc.\
           This parameter is optional.',
     default="FVT")

 parser.add_argument(
     '--subsystem',
     help='Name of the subsystem, e.g. "OPENBMC" etc.\
           This parameter is optional.',
     default="OPENBMC")

 parser.add_argument(
     '--processor',
     help='Name of processor, e.g. "P9". This parameter is optional.',
     default="OPENPOWER")


 # Populate stock_list with options we want.
 stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)]


 def exit_function(signal_number=0,
                   frame=None):
     r"""
     Execute whenever the program ends normally or with the signals that we
     catch (i.e. TERM, INT).
     """

     dprint_executing()

     dprint_var(signal_number)

     qprint_pgm_footer()


 def signal_handler(signal_number,
                    frame):
     r"""
     Handle signals.  Without a function to catch a SIGTERM or SIGINT, the
     program would terminate immediately with return code 143 and without
     calling the exit_function.
     """

     # Our convention is to set up exit_function with atexit.register() so
     # there is no need to explicitly call exit_function from here.

     dprint_executing()

     # Calling exit prevents us from returning to the code that was running
     # when the signal was received.
     exit(0)


 def validate_parms():
     r"""
     Validate program parameters, etc.  Return True or False (i.e. pass/fail)
     accordingly.
     """

     if not valid_file_path(source):
         return False

     if not valid_dir_path(dest):
         return False

     gen_post_validation(exit_function, signal_handler)

     return True


 def parse_output_xml(xml_file_path, csv_dir_path, version_id, platform, level,
                      test_phase, processor):
     r"""
     Parse the robot-generated output.xml file and extract various test
     output data. Put the extracted information into a csv file in the "dest"
     folder.

     Description of argument(s):
     xml_file_path                   The path to a Robot-generated output.xml
                                     file.
     csv_dir_path                    The path to the directory that is to
                                     contain the .csv files generated by
                                     this function.
     version_id                      Version of the openbmc firmware
                                     (e.g. "v2.1-215-g6e7eacb").
     platform                        Platform of the openbmc system.
     level                           Release level of the OpenBMC system
                                     (e.g. "Master").
     """

     # Initialize tallies
     total_critical_tc = 0
     total_critical_passed = 0
     total_critical_failed = 0
     total_non_critical_tc = 0
     total_non_critical_passed = 0
     total_non_critical_failed = 0

     result = ExecutionResult(xml_file_path)
     result.configure(stat_config={'suite_stat_level': 2,
                                   'tag_stat_combine': 'tagANDanother'})

     stats = result.statistics
     print("--------------------------------------")
     try:
         total_critical_tc = stats.total.critical.passed + stats.total.critical.failed
         total_critical_passed = stats.total.critical.passed
         total_critical_failed = stats.total.critical.failed
     except AttributeError:
         pass

     try:
         total_non_critical_tc = stats.total.passed + stats.total.failed
         total_non_critical_passed = stats.total.passed
         total_non_critical_failed = stats.total.failed
     except AttributeError:
         pass

     print("Total Test Count:\t %d" % (total_non_critical_tc + total_critical_tc))

     print("Total Critical Test Failed:\t %d" % total_critical_failed)
     print("Total Critical Test Passed:\t %d" % total_critical_passed)
     print("Total Non-Critical Test Failed:\t %d" % total_non_critical_failed)
     print("Total Non-Critical Test Passed:\t %d" % total_non_critical_passed)
     print("Test Start Time:\t %s" % result.suite.starttime)
     print("Test End Time:\t\t %s" % result.suite.endtime)
     print("--------------------------------------")

     # Use ResultVisitor object and save off the test data info
     class TestResult(ResultVisitor):
         def __init__(self):
             self.testData = []

         def visit_test(self, test):
             self.testData += [test]

     collectDataObj = TestResult()
     result.visit(collectDataObj)

     # Write the result statistics attributes to CSV file
     l_csvlist = []

     # Default Test data
     l_test_type = test_phase

     l_pse_rel = 'Master'
     if level:
         l_pse_rel = level

     l_env = 'HW'
     l_proc = processor
     l_platform_type = ""
     l_func_area = ""

     # System data from XML meta data
     # l_system_info = get_system_details(xml_file_path)

     # First let us try to collect information from keyboard input
     # If keyboard input cannot give both information, then find from xml file.
     if version_id and platform:
         l_driver = version_id
         l_platform_type = platform
         print("BMC Version_id:%s" % version_id)
         print("BMC Platform:%s" % platform)
     else:
         # System data from XML meta data
         l_system_info = get_system_details(xml_file_path)
         l_driver = l_system_info[0]
         l_platform_type = l_system_info[1]

     # Driver version id and platform are mandatorily required for CSV file
     # generation. If any one is not avaulable, exit CSV file generation
     # process.
     if l_driver and l_platform_type:
         print("Driver and system info set.")
     else:
         print("Both driver and system info need to be set.\
                 CSV file is not generated.")
         sys.exit()

     # Default header
     l_header = ['test_start', 'test_end', 'subsys', 'test_type',
                 'test_result', 'test_name', 'pse_rel', 'driver',
                 'env', 'proc', 'platform_type', 'test_func_area']

     l_csvlist.append(l_header)

     # Generate CSV file onto the path with current time stamp
     l_base_dir = csv_dir_path
     l_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
     # Example: 2017-02-20-08-47-22_Witherspoon.csv
     l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv"

     print("Writing data into csv file:%s" % l_csvfile)

     for testcase in collectDataObj.testData:
         # Functional Area: Suite Name
         # Test Name: Test Case Name
         l_func_area = str(testcase.parent).split(' ', 1)[1]
         l_test_name = str(testcase)

         # Test Result pass=0 fail=1
         if testcase.status == 'PASS':
             l_test_result = 0
         else:
             l_test_result = 1

         # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S"
         l_stime = xml_to_csv_time(testcase.starttime)
         l_etime = xml_to_csv_time(testcase.endtime)
         # Data Sequence: test_start,test_end,subsys,test_type,
         #                test_result,test_name,pse_rel,driver,
         #                env,proc,platform_type,test_func_area,
         l_data = [l_stime, l_etime, subsystem, l_test_type, l_test_result,
                   l_test_name, l_pse_rel, l_driver, l_env, l_proc,
                   l_platform_type, l_func_area]
         l_csvlist.append(l_data)

     # Open the file and write to the CSV file
     l_file = open(l_csvfile, "w")
     l_writer = csv.writer(l_file, lineterminator='\n')
     l_writer.writerows(l_csvlist)
     l_file.close()
     # Set file permissions 666.
     perm = stat.S_IRUSR + stat.S_IWUSR + stat.S_IRGRP + stat.S_IWGRP + stat.S_IROTH + stat.S_IWOTH
     os.chmod(l_csvfile, perm)


 def xml_to_csv_time(xml_datetime):
     r"""
     Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format
     and return it.

     Description of argument(s):
     datetime                        The date in the following format: %Y%m%d
                                     %H:%M:%S.%f (This is the format
                                     typically found in an XML file.)

     The date returned will be in the following format: %Y-%m-%d-%H-%M-%S
     """

     # 20170206 05:05:19.342
     l_str = datetime.datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f")
     # 2017-02-06-05-05-19
     l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S")
     return str(l_str)


 def get_system_details(xml_file_path):
     r"""
     Get the system data from output.xml generated by robot and return it.
     The list returned will be in the following order: [driver,platform]

     Description of argument(s):
     xml_file_path                   The relative or absolute path to the
                                     output.xml file.
     """

     bmc_version_id = ""
     bmc_platform = ""
     with open(xml_file_path, 'rt') as output:
         tree = ElementTree.parse(output)

     for node in tree.iter('msg'):
         # /etc/os-release output is logged in the XML as msg
         # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79"
         if '${output} = VERSION_ID=' in node.text:
             # Get BMC version (e.g. v1.99.1-96-g2a46570)
             bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1]

         # Platform is logged in the XML as msg.
         # Example: ${bmc_model} = Witherspoon BMC
         if '${bmc_model} = ' in node.text:
             bmc_platform = node.text.split(" = ")[1]

     print_vars(bmc_version_id, bmc_platform)
     return [str(bmc_version_id), str(bmc_platform)]


 def main():

     if not gen_get_options(parser, stock_list):
         return False

     if not validate_parms():
         return False

     qprint_pgm_header()

     parse_output_xml(source, dest, version_id, platform, level,
                      test_phase, processor)

     return True


 # Main

 if not main():
     exit(1)
	#!/usr/bin/env python3

	r"""
	Use robot framework API to extract test result data from output.xml generated
	by robot tests. For more information on the Robot Framework API, see
	http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html
	"""

	import sys
	import os
	import getopt
	import csv
	import robot.errors
	import re
	import stat
	import datetime

	from robot.api import ExecutionResult
	from robot.result.visitor import ResultVisitor
	from xml.etree import ElementTree

	from gen_arg import *
	from gen_print import *
	from gen_valid import *

	# Remove the python library path to restore with local project path later.
	save_path_0 = sys.path[0]
	del sys.path[0]
	sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))

	# Restore sys.path[0].
	sys.path.insert(0, save_path_0)


	this_program = sys.argv[0]
	info = " For more information: " + this_program + ' -h'
	if len(sys.argv) == 1:
	print(info)
	sys.exit(1)


	parser = argparse.ArgumentParser(
	usage=info,
	description="%(prog)s uses a robot framework API to extract test result\
	data from output.xml generated by robot tests. For more information on the\
	Robot Framework API, see\
	http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html",
	formatter_class=argparse.ArgumentDefaultsHelpFormatter,
	prefix_chars='-+')

	parser.add_argument(
	'--source',
	'-s',
	help='The output.xml robot test result file path. This parameter is \
	required.')

	parser.add_argument(
	'--dest',
	'-d',
	help='The directory path where the generated .csv files will go. This \
	parameter is required.')

	parser.add_argument(
	'--version_id',
	help='Driver version of openbmc firmware which was used during test,\
	e.g. "v2.1-215-g6e7eacb". This parameter is required.')

	parser.add_argument(
	'--platform',
	help='OpenBMC platform which was used during test,\
	e.g. "Witherspoon". This parameter is required.')

	parser.add_argument(
	'--level',
	help='OpenBMC release level which was used during test,\
	e.g. "Master", "OBMC920". This parameter is required.')

	parser.add_argument(
	'--test_phase',
	help='Name of testing phase, e.g. "DVT", "SVT", etc.\
	This parameter is optional.',
	default="FVT")

	parser.add_argument(
	'--subsystem',
	help='Name of the subsystem, e.g. "OPENBMC" etc.\
	This parameter is optional.',
	default="OPENBMC")

	parser.add_argument(
	'--processor',
	help='Name of processor, e.g. "P9". This parameter is optional.',
	default="OPENPOWER")


	# Populate stock_list with options we want.
	stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)]


	def exit_function(signal_number=0,
	frame=None):
	r"""
	Execute whenever the program ends normally or with the signals that we
	catch (i.e. TERM, INT).
	"""

	dprint_executing()

	dprint_var(signal_number)

	qprint_pgm_footer()


	def signal_handler(signal_number,
	frame):
	r"""
	Handle signals. Without a function to catch a SIGTERM or SIGINT, the
	program would terminate immediately with return code 143 and without
	calling the exit_function.
	"""

	# Our convention is to set up exit_function with atexit.register() so
	# there is no need to explicitly call exit_function from here.

	dprint_executing()

	# Calling exit prevents us from returning to the code that was running
	# when the signal was received.
	exit(0)


	def validate_parms():
	r"""
	Validate program parameters, etc. Return True or False (i.e. pass/fail)
	accordingly.
	"""

	if not valid_file_path(source):
	return False

	if not valid_dir_path(dest):
	return False

	gen_post_validation(exit_function, signal_handler)

	return True


	def parse_output_xml(xml_file_path, csv_dir_path, version_id, platform, level,
	test_phase, processor):
	r"""
	Parse the robot-generated output.xml file and extract various test
	output data. Put the extracted information into a csv file in the "dest"
	folder.

	Description of argument(s):
	xml_file_path The path to a Robot-generated output.xml
	file.
	csv_dir_path The path to the directory that is to
	contain the .csv files generated by
	this function.
	version_id Version of the openbmc firmware
	(e.g. "v2.1-215-g6e7eacb").
	platform Platform of the openbmc system.
	level Release level of the OpenBMC system
	(e.g. "Master").
	"""

	# Initialize tallies
	total_critical_tc = 0
	total_critical_passed = 0
	total_critical_failed = 0
	total_non_critical_tc = 0
	total_non_critical_passed = 0
	total_non_critical_failed = 0

	result = ExecutionResult(xml_file_path)
	result.configure(stat_config={'suite_stat_level': 2,
	'tag_stat_combine': 'tagANDanother'})

	stats = result.statistics
	print("--------------------------------------")
	try:
	total_critical_tc = stats.total.critical.passed + stats.total.critical.failed
	total_critical_passed = stats.total.critical.passed
	total_critical_failed = stats.total.critical.failed
	except AttributeError:
	pass

	try:
	total_non_critical_tc = stats.total.passed + stats.total.failed
	total_non_critical_passed = stats.total.passed
	total_non_critical_failed = stats.total.failed
	except AttributeError:
	pass

	print("Total Test Count:\t %d" % (total_non_critical_tc + total_critical_tc))

	print("Total Critical Test Failed:\t %d" % total_critical_failed)
	print("Total Critical Test Passed:\t %d" % total_critical_passed)
	print("Total Non-Critical Test Failed:\t %d" % total_non_critical_failed)
	print("Total Non-Critical Test Passed:\t %d" % total_non_critical_passed)
	print("Test Start Time:\t %s" % result.suite.starttime)
	print("Test End Time:\t\t %s" % result.suite.endtime)
	print("--------------------------------------")

	# Use ResultVisitor object and save off the test data info
	class TestResult(ResultVisitor):
	def __init__(self):
	self.testData = []

	def visit_test(self, test):
	self.testData += [test]

	collectDataObj = TestResult()
	result.visit(collectDataObj)

	# Write the result statistics attributes to CSV file
	l_csvlist = []

	# Default Test data
	l_test_type = test_phase

	l_pse_rel = 'Master'
	if level:
	l_pse_rel = level

	l_env = 'HW'
	l_proc = processor
	l_platform_type = ""
	l_func_area = ""

	# System data from XML meta data
	# l_system_info = get_system_details(xml_file_path)

	# First let us try to collect information from keyboard input
	# If keyboard input cannot give both information, then find from xml file.
	if version_id and platform:
	l_driver = version_id
	l_platform_type = platform
	print("BMC Version_id:%s" % version_id)
	print("BMC Platform:%s" % platform)
	else:
	# System data from XML meta data
	l_system_info = get_system_details(xml_file_path)
	l_driver = l_system_info[0]
	l_platform_type = l_system_info[1]

	# Driver version id and platform are mandatorily required for CSV file
	# generation. If any one is not avaulable, exit CSV file generation
	# process.
	if l_driver and l_platform_type:
	print("Driver and system info set.")
	else:
	print("Both driver and system info need to be set.\
	CSV file is not generated.")
	sys.exit()

	# Default header
	l_header = ['test_start', 'test_end', 'subsys', 'test_type',
	'test_result', 'test_name', 'pse_rel', 'driver',
	'env', 'proc', 'platform_type', 'test_func_area']

	l_csvlist.append(l_header)

	# Generate CSV file onto the path with current time stamp
	l_base_dir = csv_dir_path
	l_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
	# Example: 2017-02-20-08-47-22_Witherspoon.csv
	l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv"

	print("Writing data into csv file:%s" % l_csvfile)

	for testcase in collectDataObj.testData:
	# Functional Area: Suite Name
	# Test Name: Test Case Name
	l_func_area = str(testcase.parent).split(' ', 1)[1]
	l_test_name = str(testcase)

	# Test Result pass=0 fail=1
	if testcase.status == 'PASS':
	l_test_result = 0
	else:
	l_test_result = 1

	# Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S"
	l_stime = xml_to_csv_time(testcase.starttime)
	l_etime = xml_to_csv_time(testcase.endtime)
	# Data Sequence: test_start,test_end,subsys,test_type,
	# test_result,test_name,pse_rel,driver,
	# env,proc,platform_type,test_func_area,
	l_data = [l_stime, l_etime, subsystem, l_test_type, l_test_result,
	l_test_name, l_pse_rel, l_driver, l_env, l_proc,
	l_platform_type, l_func_area]
	l_csvlist.append(l_data)

	# Open the file and write to the CSV file
	l_file = open(l_csvfile, "w")
	l_writer = csv.writer(l_file, lineterminator='\n')
	l_writer.writerows(l_csvlist)
	l_file.close()
	# Set file permissions 666.
	perm = stat.S_IRUSR + stat.S_IWUSR + stat.S_IRGRP + stat.S_IWGRP + stat.S_IROTH + stat.S_IWOTH
	os.chmod(l_csvfile, perm)


	def xml_to_csv_time(xml_datetime):
	r"""
	Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format
	and return it.

	Description of argument(s):
	datetime The date in the following format: %Y%m%d
	%H:%M:%S.%f (This is the format
	typically found in an XML file.)

	The date returned will be in the following format: %Y-%m-%d-%H-%M-%S
	"""

	# 20170206 05:05:19.342
	l_str = datetime.datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f")
	# 2017-02-06-05-05-19
	l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S")
	return str(l_str)


	def get_system_details(xml_file_path):
	r"""
	Get the system data from output.xml generated by robot and return it.
	The list returned will be in the following order: [driver,platform]

	Description of argument(s):
	xml_file_path The relative or absolute path to the
	output.xml file.
	"""

	bmc_version_id = ""
	bmc_platform = ""
	with open(xml_file_path, 'rt') as output:
	tree = ElementTree.parse(output)

	for node in tree.iter('msg'):
	# /etc/os-release output is logged in the XML as msg
	# Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79"
	if '${output} = VERSION_ID=' in node.text:
	# Get BMC version (e.g. v1.99.1-96-g2a46570)
	bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1]

	# Platform is logged in the XML as msg.
	# Example: ${bmc_model} = Witherspoon BMC
	if '${bmc_model} = ' in node.text:
	bmc_platform = node.text.split(" = ")[1]

	print_vars(bmc_version_id, bmc_platform)
	return [str(bmc_version_id), str(bmc_platform)]


	def main():

	if not gen_get_options(parser, stock_list):
	return False

	if not validate_parms():
	return False

	qprint_pgm_header()

	parse_output_xml(source, dest, version_id, platform, level,
	test_phase, processor)

	return True


	# Main

	if not main():
	exit(1)