blob: 62206b43239980bb8587a425955bae37d42fc89c [file] [log] [blame]
manasarm5dedfaf2018-02-07 14:54:54 +05301#!/usr/bin/env python
2
George Keishing9fbdf792016-10-18 06:16:09 -05003r"""
manasarm5dedfaf2018-02-07 14:54:54 +05304Use robot framework API to extract test result data from output.xml generated
5by robot tests. For more information on the Robot Framework API, see
6http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html
George Keishing9fbdf792016-10-18 06:16:09 -05007"""
manasarm5dedfaf2018-02-07 14:54:54 +05308
Gunnar Mills096cd562018-03-26 10:19:12 -05009import sys
10import os
George Keishing9fbdf792016-10-18 06:16:09 -050011import getopt
12import csv
George Keishing9fbdf792016-10-18 06:16:09 -050013import robot.errors
manasarm5dedfaf2018-02-07 14:54:54 +053014import re
Steven Sombar3e82e3b2019-03-21 10:33:52 -050015import stat
George Keishing937fe3c2019-10-29 11:20:34 -050016import datetime
George Keishing9fbdf792016-10-18 06:16:09 -050017from robot.api import ExecutionResult
18from robot.result.visitor import ResultVisitor
George Keishing74777bd2017-02-07 01:43:38 -060019from xml.etree import ElementTree
manasarm5dedfaf2018-02-07 14:54:54 +053020
21# Remove the python library path to restore with local project path later.
22save_path_0 = sys.path[0]
23del sys.path[0]
George Keishingc2a6f092019-02-20 12:26:54 -060024sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
manasarm5dedfaf2018-02-07 14:54:54 +053025
26from gen_arg import *
27from gen_print import *
28from gen_valid import *
29
30# Restore sys.path[0].
31sys.path.insert(0, save_path_0)
32
Steven Sombara5e32f22019-03-01 13:33:15 -060033
34this_program = sys.argv[0]
35info = " For more information: " + this_program + ' -h'
36if len(sys.argv) == 1:
37 print (info)
38 sys.exit(1)
39
40
manasarm5dedfaf2018-02-07 14:54:54 +053041parser = argparse.ArgumentParser(
Steven Sombara5e32f22019-03-01 13:33:15 -060042 usage=info,
manasarm5dedfaf2018-02-07 14:54:54 +053043 description="%(prog)s uses a robot framework API to extract test result\
44 data from output.xml generated by robot tests. For more information on the\
45 Robot Framework API, see\
George Keishing11b7af92018-06-11 10:39:01 -050046 http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html",
manasarm5dedfaf2018-02-07 14:54:54 +053047 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
48 prefix_chars='-+')
49
50parser.add_argument(
51 '--source',
52 '-s',
Steven Sombara5e32f22019-03-01 13:33:15 -060053 help='The output.xml robot test result file path. This parameter is \
54 required.')
manasarm5dedfaf2018-02-07 14:54:54 +053055
56parser.add_argument(
57 '--dest',
58 '-d',
Steven Sombara5e32f22019-03-01 13:33:15 -060059 help='The directory path where the generated .csv files will go. This \
60 parameter is required.')
manasarm5dedfaf2018-02-07 14:54:54 +053061
manasarme4f79c92018-02-22 13:02:46 +053062parser.add_argument(
63 '--version_id',
64 help='Driver version of openbmc firmware which was used during test,\
Steven Sombara5e32f22019-03-01 13:33:15 -060065 e.g. "v2.1-215-g6e7eacb". This parameter is required.')
manasarme4f79c92018-02-22 13:02:46 +053066
67parser.add_argument(
68 '--platform',
George Keishing19533d52018-04-09 03:08:03 -050069 help='OpenBMC platform which was used during test,\
Steven Sombara5e32f22019-03-01 13:33:15 -060070 e.g. "Witherspoon". This parameter is required.')
manasarme4f79c92018-02-22 13:02:46 +053071
George Keishing19533d52018-04-09 03:08:03 -050072parser.add_argument(
73 '--level',
74 help='OpenBMC release level which was used during test,\
Steven Sombara5e32f22019-03-01 13:33:15 -060075 e.g. "Master", "OBMC920". This parameter is required.')
76
77parser.add_argument(
78 '--test_phase',
79 help='Name of testing phase, e.g. "DVT", "SVT", etc.\
80 This parameter is optional.',
81 default="FVT")
82
83parser.add_argument(
84 '--processor',
85 help='Name of processor, e.g. "P9". This parameter is optional.',
86 default="OPENPOWER")
87
George Keishing19533d52018-04-09 03:08:03 -050088
manasarm5dedfaf2018-02-07 14:54:54 +053089# Populate stock_list with options we want.
90stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)]
91
92
93def exit_function(signal_number=0,
94 frame=None):
manasarm5dedfaf2018-02-07 14:54:54 +053095 r"""
96 Execute whenever the program ends normally or with the signals that we
97 catch (i.e. TERM, INT).
98 """
99
100 dprint_executing()
101
102 dprint_var(signal_number)
103
104 qprint_pgm_footer()
105
106
107def signal_handler(signal_number,
108 frame):
manasarm5dedfaf2018-02-07 14:54:54 +0530109 r"""
110 Handle signals. Without a function to catch a SIGTERM or SIGINT, the
111 program would terminate immediately with return code 143 and without
112 calling the exit_function.
113 """
114
115 # Our convention is to set up exit_function with atexit.register() so
116 # there is no need to explicitly call exit_function from here.
117
118 dprint_executing()
119
120 # Calling exit prevents us from returning to the code that was running
121 # when the signal was received.
122 exit(0)
123
124
125def validate_parms():
manasarm5dedfaf2018-02-07 14:54:54 +0530126 r"""
127 Validate program parameters, etc. Return True or False (i.e. pass/fail)
128 accordingly.
129 """
130
131 if not valid_file_path(source):
132 return False
133
134 if not valid_dir_path(dest):
135 return False
136
137 gen_post_validation(exit_function, signal_handler)
138
139 return True
George Keishing9fbdf792016-10-18 06:16:09 -0500140
141
Steven Sombara5e32f22019-03-01 13:33:15 -0600142def parse_output_xml(xml_file_path, csv_dir_path, version_id, platform, level,
143 test_phase, processor):
manasarm5dedfaf2018-02-07 14:54:54 +0530144 r"""
145 Parse the robot-generated output.xml file and extract various test
146 output data. Put the extracted information into a csv file in the "dest"
147 folder.
148
149 Description of argument(s):
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500150 xml_file_path The path to a Robot-generated output.xml
151 file.
152 csv_dir_path The path to the directory that is to
153 contain the .csv files generated by
154 this function.
155 version_id Version of the openbmc firmware
156 (e.g. "v2.1-215-g6e7eacb").
157 platform Platform of the openbmc system.
158 level Release level of the OpenBMC system
George Keishing360db632018-09-06 12:01:28 -0500159 (e.g. "Master").
manasarm5dedfaf2018-02-07 14:54:54 +0530160 """
161
George Keishing74777bd2017-02-07 01:43:38 -0600162 result = ExecutionResult(xml_file_path)
George Keishing9fbdf792016-10-18 06:16:09 -0500163 result.configure(stat_config={'suite_stat_level': 2,
164 'tag_stat_combine': 'tagANDanother'})
165
166 stats = result.statistics
George Keishingf8a9ebe2018-08-06 12:49:11 -0500167 print("--------------------------------------")
168 total_tc = stats.total.critical.passed + stats.total.critical.failed
169 print("Total Test Count:\t %d" % total_tc)
170 print("Total Test Failed:\t %d" % stats.total.critical.failed)
171 print("Total Test Passed:\t %d" % stats.total.critical.passed)
172 print("Test Start Time:\t %s" % result.suite.starttime)
173 print("Test End Time:\t\t %s" % result.suite.endtime)
174 print("--------------------------------------")
George Keishing9fbdf792016-10-18 06:16:09 -0500175
176 # Use ResultVisitor object and save off the test data info
177 class TestResult(ResultVisitor):
178 def __init__(self):
179 self.testData = []
180
181 def visit_test(self, test):
182 self.testData += [test]
183
184 collectDataObj = TestResult()
185 result.visit(collectDataObj)
186
187 # Write the result statistics attributes to CSV file
188 l_csvlist = []
George Keishing74777bd2017-02-07 01:43:38 -0600189
190 # Default Test data
George Keishing9fbdf792016-10-18 06:16:09 -0500191 l_subsys = 'OPENBMC'
Steven Sombara5e32f22019-03-01 13:33:15 -0600192 l_test_type = test_phase
George Keishing19533d52018-04-09 03:08:03 -0500193
George Keishing360db632018-09-06 12:01:28 -0500194 l_pse_rel = 'Master'
George Keishing19533d52018-04-09 03:08:03 -0500195 if level:
196 l_pse_rel = level
197
George Keishing74777bd2017-02-07 01:43:38 -0600198 l_env = 'HW'
Steven Sombara5e32f22019-03-01 13:33:15 -0600199 l_proc = processor
George Keishing74777bd2017-02-07 01:43:38 -0600200 l_platform_type = ""
201 l_func_area = ""
202
Gunnar Mills096cd562018-03-26 10:19:12 -0500203 # System data from XML meta data
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500204 # l_system_info = get_system_details(xml_file_path)
manasarme4f79c92018-02-22 13:02:46 +0530205
206 # First let us try to collect information from keyboard input
207 # If keyboard input cannot give both information, then find from xml file.
208 if version_id and platform:
209 l_driver = version_id
210 l_platform_type = platform
George Keishingf8a9ebe2018-08-06 12:49:11 -0500211 print("BMC Version_id:%s" % version_id)
212 print("BMC Platform:%s" % platform)
George Keishing74777bd2017-02-07 01:43:38 -0600213 else:
manasarme4f79c92018-02-22 13:02:46 +0530214 # System data from XML meta data
215 l_system_info = get_system_details(xml_file_path)
216 l_driver = l_system_info[0]
217 l_platform_type = l_system_info[1]
218
219 # Driver version id and platform are mandatorily required for CSV file
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500220 # generation. If any one is not avaulable, exit CSV file generation
221 # process.
manasarme4f79c92018-02-22 13:02:46 +0530222 if l_driver and l_platform_type:
George Keishingf8a9ebe2018-08-06 12:49:11 -0500223 print("Driver and system info set.")
manasarme4f79c92018-02-22 13:02:46 +0530224 else:
George Keishingf8a9ebe2018-08-06 12:49:11 -0500225 print("Both driver and system info need to be set.\
226 CSV file is not generated.")
George Keishing74777bd2017-02-07 01:43:38 -0600227 sys.exit()
George Keishing9fbdf792016-10-18 06:16:09 -0500228
229 # Default header
230 l_header = ['test_start', 'test_end', 'subsys', 'test_type',
George Keishing74777bd2017-02-07 01:43:38 -0600231 'test_result', 'test_name', 'pse_rel', 'driver',
232 'env', 'proc', 'platform_type', 'test_func_area']
George Keishing9fbdf792016-10-18 06:16:09 -0500233
234 l_csvlist.append(l_header)
235
236 # Generate CSV file onto the path with current time stamp
George Keishing74777bd2017-02-07 01:43:38 -0600237 l_base_dir = csv_dir_path
George Keishing937fe3c2019-10-29 11:20:34 -0500238 l_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
George Keishing74777bd2017-02-07 01:43:38 -0600239 # Example: 2017-02-20-08-47-22_Witherspoon.csv
240 l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv"
George Keishing9fbdf792016-10-18 06:16:09 -0500241
George Keishingf8a9ebe2018-08-06 12:49:11 -0500242 print("Writing data into csv file:%s" % l_csvfile)
George Keishing9fbdf792016-10-18 06:16:09 -0500243
244 for testcase in collectDataObj.testData:
George Keishing74777bd2017-02-07 01:43:38 -0600245 # Functional Area: Suite Name
246 # Test Name: Test Case Name
247 l_func_area = str(testcase.parent).split(' ', 1)[1]
248 l_test_name = str(testcase)
George Keishing9fbdf792016-10-18 06:16:09 -0500249
250 # Test Result pass=0 fail=1
251 if testcase.status == 'PASS':
252 l_test_result = 0
253 else:
254 l_test_result = 1
255
George Keishing74777bd2017-02-07 01:43:38 -0600256 # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S"
257 l_stime = xml_to_csv_time(testcase.starttime)
258 l_etime = xml_to_csv_time(testcase.endtime)
George Keishing9fbdf792016-10-18 06:16:09 -0500259 # Data Sequence: test_start,test_end,subsys,test_type,
George Keishing74777bd2017-02-07 01:43:38 -0600260 # test_result,test_name,pse_rel,driver,
George Keishing360db632018-09-06 12:01:28 -0500261 # env,proc,platform_type,test_func_area,
George Keishing74777bd2017-02-07 01:43:38 -0600262 l_data = [l_stime, l_etime, l_subsys, l_test_type, l_test_result,
263 l_test_name, l_pse_rel, l_driver, l_env, l_proc,
264 l_platform_type, l_func_area]
George Keishing9fbdf792016-10-18 06:16:09 -0500265 l_csvlist.append(l_data)
266
George Keishinga96e27c2016-12-04 23:05:04 -0600267 # Open the file and write to the CSV file
268 l_file = open(l_csvfile, "w")
269 l_writer = csv.writer(l_file, lineterminator='\n')
270 l_writer.writerows(l_csvlist)
271 l_file.close()
Steven Sombar3e82e3b2019-03-21 10:33:52 -0500272 # Set file permissions 666.
273 perm = stat.S_IRUSR + stat.S_IWUSR + stat.S_IRGRP + stat.S_IWGRP + stat.S_IROTH + stat.S_IWOTH
274 os.chmod(l_csvfile, perm)
George Keishing9fbdf792016-10-18 06:16:09 -0500275
276
George Keishing74777bd2017-02-07 01:43:38 -0600277def xml_to_csv_time(xml_datetime):
278 r"""
279 Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format
280 and return it.
281
manasarm5dedfaf2018-02-07 14:54:54 +0530282 Description of argument(s):
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500283 datetime The date in the following format: %Y%m%d
284 %H:%M:%S.%f (This is the format
285 typically found in an XML file.)
George Keishing74777bd2017-02-07 01:43:38 -0600286
287 The date returned will be in the following format: %Y-%m-%d-%H-%M-%S
288 """
manasarm5dedfaf2018-02-07 14:54:54 +0530289
George Keishing74777bd2017-02-07 01:43:38 -0600290 # 20170206 05:05:19.342
George Keishing937fe3c2019-10-29 11:20:34 -0500291 l_str = datetime.datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f")
George Keishing74777bd2017-02-07 01:43:38 -0600292 # 2017-02-06-05-05-19
293 l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S")
294 return str(l_str)
295
296
297def get_system_details(xml_file_path):
298 r"""
299 Get the system data from output.xml generated by robot and return it.
Gunnar Mills28e403b2017-10-25 16:16:38 -0500300 The list returned will be in the following order: [driver,platform]
George Keishing74777bd2017-02-07 01:43:38 -0600301
manasarm5dedfaf2018-02-07 14:54:54 +0530302 Description of argument(s):
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500303 xml_file_path The relative or absolute path to the
304 output.xml file.
George Keishing74777bd2017-02-07 01:43:38 -0600305 """
manasarm5dedfaf2018-02-07 14:54:54 +0530306
manasarme4f79c92018-02-22 13:02:46 +0530307 bmc_version_id = ""
George Keishing74777bd2017-02-07 01:43:38 -0600308 bmc_platform = ""
309 with open(xml_file_path, 'rt') as output:
310 tree = ElementTree.parse(output)
311
312 for node in tree.iter('msg'):
313 # /etc/os-release output is logged in the XML as msg
George Keishing360db632018-09-06 12:01:28 -0500314 # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79"
George Keishing74777bd2017-02-07 01:43:38 -0600315 if '${output} = VERSION_ID=' in node.text:
George Keishing360db632018-09-06 12:01:28 -0500316 # Get BMC version (e.g. v1.99.1-96-g2a46570)
manasarme4f79c92018-02-22 13:02:46 +0530317 bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1]
George Keishing74777bd2017-02-07 01:43:38 -0600318
319 # Platform is logged in the XML as msg.
320 # Example: ${bmc_model} = Witherspoon BMC
321 if '${bmc_model} = ' in node.text:
322 bmc_platform = node.text.split(" = ")[1]
323
manasarme4f79c92018-02-22 13:02:46 +0530324 print_vars(bmc_version_id, bmc_platform)
325 return [str(bmc_version_id), str(bmc_platform)]
George Keishing74777bd2017-02-07 01:43:38 -0600326
327
manasarm5dedfaf2018-02-07 14:54:54 +0530328def main():
George Keishing9fbdf792016-10-18 06:16:09 -0500329
manasarm5dedfaf2018-02-07 14:54:54 +0530330 if not gen_get_options(parser, stock_list):
331 return False
George Keishing9fbdf792016-10-18 06:16:09 -0500332
manasarm5dedfaf2018-02-07 14:54:54 +0530333 if not validate_parms():
334 return False
George Keishing9fbdf792016-10-18 06:16:09 -0500335
manasarm5dedfaf2018-02-07 14:54:54 +0530336 qprint_pgm_header()
George Keishing9fbdf792016-10-18 06:16:09 -0500337
Steven Sombara5e32f22019-03-01 13:33:15 -0600338 parse_output_xml(source, dest, version_id, platform, level,
339 test_phase, processor)
George Keishing9fbdf792016-10-18 06:16:09 -0500340
manasarm5dedfaf2018-02-07 14:54:54 +0530341 return True
342
343
344# Main
345
346if not main():
347 exit(1)