blob: a2a6735712ca2bcbcc75143af05eeb8ce9c23a70 [file] [log] [blame]
manasarm5dedfaf2018-02-07 14:54:54 +05301#!/usr/bin/env python
2
George Keishing9fbdf792016-10-18 06:16:09 -05003r"""
manasarm5dedfaf2018-02-07 14:54:54 +05304Use robot framework API to extract test result data from output.xml generated
5by robot tests. For more information on the Robot Framework API, see
6http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html
George Keishing9fbdf792016-10-18 06:16:09 -05007"""
manasarm5dedfaf2018-02-07 14:54:54 +05308
Gunnar Mills096cd562018-03-26 10:19:12 -05009import sys
10import os
George Keishing9fbdf792016-10-18 06:16:09 -050011import getopt
12import csv
George Keishing9fbdf792016-10-18 06:16:09 -050013import robot.errors
manasarm5dedfaf2018-02-07 14:54:54 +053014import re
15from datetime import datetime
George Keishing9fbdf792016-10-18 06:16:09 -050016from robot.api import ExecutionResult
17from robot.result.visitor import ResultVisitor
George Keishing74777bd2017-02-07 01:43:38 -060018from xml.etree import ElementTree
manasarm5dedfaf2018-02-07 14:54:54 +053019
20# Remove the python library path to restore with local project path later.
21save_path_0 = sys.path[0]
22del sys.path[0]
23sys.path.append(os.path.join(os.path.dirname(__file__), "../../../lib"))
24
25from gen_arg import *
26from gen_print import *
27from gen_valid import *
28
29# Restore sys.path[0].
30sys.path.insert(0, save_path_0)
31
32parser = argparse.ArgumentParser(
33 usage='%(prog)s [OPTIONS]',
34 description="%(prog)s uses a robot framework API to extract test result\
35 data from output.xml generated by robot tests. For more information on the\
36 Robot Framework API, see\
George Keishing11b7af92018-06-11 10:39:01 -050037 http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html",
manasarm5dedfaf2018-02-07 14:54:54 +053038 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
39 prefix_chars='-+')
40
41parser.add_argument(
42 '--source',
43 '-s',
44 help='The output.xml robot test result file path.')
45
46parser.add_argument(
47 '--dest',
48 '-d',
49 help='The directory path where the generated .csv files will go.')
50
manasarme4f79c92018-02-22 13:02:46 +053051parser.add_argument(
52 '--version_id',
53 help='Driver version of openbmc firmware which was used during test,\
54 e.g. "v2.1-215-g6e7eacb".')
55
56parser.add_argument(
57 '--platform',
George Keishing19533d52018-04-09 03:08:03 -050058 help='OpenBMC platform which was used during test,\
manasarme4f79c92018-02-22 13:02:46 +053059 e.g. "Witherspoon".')
60
George Keishing19533d52018-04-09 03:08:03 -050061parser.add_argument(
62 '--level',
63 help='OpenBMC release level which was used during test,\
64 e.g. "OBMC920".')
65
manasarm5dedfaf2018-02-07 14:54:54 +053066# Populate stock_list with options we want.
67stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)]
68
69
70def exit_function(signal_number=0,
71 frame=None):
manasarm5dedfaf2018-02-07 14:54:54 +053072 r"""
73 Execute whenever the program ends normally or with the signals that we
74 catch (i.e. TERM, INT).
75 """
76
77 dprint_executing()
78
79 dprint_var(signal_number)
80
81 qprint_pgm_footer()
82
83
84def signal_handler(signal_number,
85 frame):
manasarm5dedfaf2018-02-07 14:54:54 +053086 r"""
87 Handle signals. Without a function to catch a SIGTERM or SIGINT, the
88 program would terminate immediately with return code 143 and without
89 calling the exit_function.
90 """
91
92 # Our convention is to set up exit_function with atexit.register() so
93 # there is no need to explicitly call exit_function from here.
94
95 dprint_executing()
96
97 # Calling exit prevents us from returning to the code that was running
98 # when the signal was received.
99 exit(0)
100
101
102def validate_parms():
manasarm5dedfaf2018-02-07 14:54:54 +0530103 r"""
104 Validate program parameters, etc. Return True or False (i.e. pass/fail)
105 accordingly.
106 """
107
108 if not valid_file_path(source):
109 return False
110
111 if not valid_dir_path(dest):
112 return False
113
114 gen_post_validation(exit_function, signal_handler)
115
116 return True
George Keishing9fbdf792016-10-18 06:16:09 -0500117
118
George Keishing19533d52018-04-09 03:08:03 -0500119def parse_output_xml(xml_file_path, csv_dir_path, version_id, platform, level):
manasarm5dedfaf2018-02-07 14:54:54 +0530120 r"""
121 Parse the robot-generated output.xml file and extract various test
122 output data. Put the extracted information into a csv file in the "dest"
123 folder.
124
125 Description of argument(s):
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500126 xml_file_path The path to a Robot-generated output.xml
127 file.
128 csv_dir_path The path to the directory that is to
129 contain the .csv files generated by
130 this function.
131 version_id Version of the openbmc firmware
132 (e.g. "v2.1-215-g6e7eacb").
133 platform Platform of the openbmc system.
134 level Release level of the OpenBMC system
135 (e.g. "OBMC920").
manasarm5dedfaf2018-02-07 14:54:54 +0530136 """
137
George Keishing74777bd2017-02-07 01:43:38 -0600138 result = ExecutionResult(xml_file_path)
George Keishing9fbdf792016-10-18 06:16:09 -0500139 result.configure(stat_config={'suite_stat_level': 2,
140 'tag_stat_combine': 'tagANDanother'})
141
142 stats = result.statistics
143 print "--------------------------------------"
144 print "Total Test Count:\t",\
145 stats.total.critical.passed + stats.total.critical.failed
146 print "Total Test Failed:\t", stats.total.critical.failed
147 print "Total Test Passed:\t", stats.total.critical.passed
148 print "Test Start Time:\t", result.suite.starttime
149 print "Test End Time:\t\t", result.suite.endtime
150 print "--------------------------------------"
151
152 # Use ResultVisitor object and save off the test data info
153 class TestResult(ResultVisitor):
154 def __init__(self):
155 self.testData = []
156
157 def visit_test(self, test):
158 self.testData += [test]
159
160 collectDataObj = TestResult()
161 result.visit(collectDataObj)
162
163 # Write the result statistics attributes to CSV file
164 l_csvlist = []
George Keishing74777bd2017-02-07 01:43:38 -0600165
166 # Default Test data
George Keishing9fbdf792016-10-18 06:16:09 -0500167 l_subsys = 'OPENBMC'
168 l_test_type = 'FTC'
George Keishing19533d52018-04-09 03:08:03 -0500169
George Keishing9fbdf792016-10-18 06:16:09 -0500170 l_pse_rel = 'OBMC910'
George Keishing19533d52018-04-09 03:08:03 -0500171 if level:
172 l_pse_rel = level
173
George Keishing74777bd2017-02-07 01:43:38 -0600174 l_env = 'HW'
175 l_proc = 'P9'
176 l_platform_type = ""
177 l_func_area = ""
178
Gunnar Mills096cd562018-03-26 10:19:12 -0500179 # System data from XML meta data
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500180 # l_system_info = get_system_details(xml_file_path)
manasarme4f79c92018-02-22 13:02:46 +0530181
182 # First let us try to collect information from keyboard input
183 # If keyboard input cannot give both information, then find from xml file.
184 if version_id and platform:
185 l_driver = version_id
186 l_platform_type = platform
187 print "BMC Version_id:", version_id
188 print "BMC Platform:", platform
George Keishing74777bd2017-02-07 01:43:38 -0600189 else:
manasarme4f79c92018-02-22 13:02:46 +0530190 # System data from XML meta data
191 l_system_info = get_system_details(xml_file_path)
192 l_driver = l_system_info[0]
193 l_platform_type = l_system_info[1]
194
195 # Driver version id and platform are mandatorily required for CSV file
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500196 # generation. If any one is not avaulable, exit CSV file generation
197 # process.
manasarme4f79c92018-02-22 13:02:46 +0530198 if l_driver and l_platform_type:
199 print "Driver and system info set."
200 else:
201 print "\
202 Both driver and system info need to be set. CSV file is not generated."
George Keishing74777bd2017-02-07 01:43:38 -0600203 sys.exit()
George Keishing9fbdf792016-10-18 06:16:09 -0500204
205 # Default header
206 l_header = ['test_start', 'test_end', 'subsys', 'test_type',
George Keishing74777bd2017-02-07 01:43:38 -0600207 'test_result', 'test_name', 'pse_rel', 'driver',
208 'env', 'proc', 'platform_type', 'test_func_area']
George Keishing9fbdf792016-10-18 06:16:09 -0500209
210 l_csvlist.append(l_header)
211
212 # Generate CSV file onto the path with current time stamp
George Keishing74777bd2017-02-07 01:43:38 -0600213 l_base_dir = csv_dir_path
George Keishing9fbdf792016-10-18 06:16:09 -0500214 l_timestamp = datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
George Keishing74777bd2017-02-07 01:43:38 -0600215 # Example: 2017-02-20-08-47-22_Witherspoon.csv
216 l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv"
George Keishing9fbdf792016-10-18 06:16:09 -0500217
218 print "Writing data into csv file:", l_csvfile
219
220 for testcase in collectDataObj.testData:
George Keishing74777bd2017-02-07 01:43:38 -0600221 # Functional Area: Suite Name
222 # Test Name: Test Case Name
223 l_func_area = str(testcase.parent).split(' ', 1)[1]
224 l_test_name = str(testcase)
George Keishing9fbdf792016-10-18 06:16:09 -0500225
226 # Test Result pass=0 fail=1
227 if testcase.status == 'PASS':
228 l_test_result = 0
229 else:
230 l_test_result = 1
231
George Keishing74777bd2017-02-07 01:43:38 -0600232 # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S"
233 l_stime = xml_to_csv_time(testcase.starttime)
234 l_etime = xml_to_csv_time(testcase.endtime)
George Keishing9fbdf792016-10-18 06:16:09 -0500235 # Data Sequence: test_start,test_end,subsys,test_type,
George Keishing74777bd2017-02-07 01:43:38 -0600236 # test_result,test_name,pse_rel,driver,
237 # env,proc,platform_tyep,test_func_area,
238 l_data = [l_stime, l_etime, l_subsys, l_test_type, l_test_result,
239 l_test_name, l_pse_rel, l_driver, l_env, l_proc,
240 l_platform_type, l_func_area]
George Keishing9fbdf792016-10-18 06:16:09 -0500241 l_csvlist.append(l_data)
242
George Keishinga96e27c2016-12-04 23:05:04 -0600243 # Open the file and write to the CSV file
244 l_file = open(l_csvfile, "w")
245 l_writer = csv.writer(l_file, lineterminator='\n')
246 l_writer.writerows(l_csvlist)
247 l_file.close()
George Keishing9fbdf792016-10-18 06:16:09 -0500248
249
George Keishing74777bd2017-02-07 01:43:38 -0600250def xml_to_csv_time(xml_datetime):
251 r"""
252 Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format
253 and return it.
254
manasarm5dedfaf2018-02-07 14:54:54 +0530255 Description of argument(s):
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500256 datetime The date in the following format: %Y%m%d
257 %H:%M:%S.%f (This is the format
258 typically found in an XML file.)
George Keishing74777bd2017-02-07 01:43:38 -0600259
260 The date returned will be in the following format: %Y-%m-%d-%H-%M-%S
261 """
manasarm5dedfaf2018-02-07 14:54:54 +0530262
George Keishing74777bd2017-02-07 01:43:38 -0600263 # 20170206 05:05:19.342
264 l_str = datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f")
265 # 2017-02-06-05-05-19
266 l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S")
267 return str(l_str)
268
269
270def get_system_details(xml_file_path):
271 r"""
272 Get the system data from output.xml generated by robot and return it.
Gunnar Mills28e403b2017-10-25 16:16:38 -0500273 The list returned will be in the following order: [driver,platform]
George Keishing74777bd2017-02-07 01:43:38 -0600274
manasarm5dedfaf2018-02-07 14:54:54 +0530275 Description of argument(s):
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500276 xml_file_path The relative or absolute path to the
277 output.xml file.
George Keishing74777bd2017-02-07 01:43:38 -0600278 """
manasarm5dedfaf2018-02-07 14:54:54 +0530279
manasarme4f79c92018-02-22 13:02:46 +0530280 bmc_version_id = ""
George Keishing74777bd2017-02-07 01:43:38 -0600281 bmc_platform = ""
282 with open(xml_file_path, 'rt') as output:
283 tree = ElementTree.parse(output)
284
285 for node in tree.iter('msg'):
286 # /etc/os-release output is logged in the XML as msg
287 # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79-dirty"
288 if '${output} = VERSION_ID=' in node.text:
289 # Get BMC version (e.g. v1.99.1-96-g2a46570-dirty)
manasarme4f79c92018-02-22 13:02:46 +0530290 bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1]
George Keishing74777bd2017-02-07 01:43:38 -0600291
292 # Platform is logged in the XML as msg.
293 # Example: ${bmc_model} = Witherspoon BMC
294 if '${bmc_model} = ' in node.text:
295 bmc_platform = node.text.split(" = ")[1]
296
manasarme4f79c92018-02-22 13:02:46 +0530297 print_vars(bmc_version_id, bmc_platform)
298 return [str(bmc_version_id), str(bmc_platform)]
George Keishing74777bd2017-02-07 01:43:38 -0600299
300
manasarm5dedfaf2018-02-07 14:54:54 +0530301def main():
George Keishing9fbdf792016-10-18 06:16:09 -0500302
manasarm5dedfaf2018-02-07 14:54:54 +0530303 if not gen_get_options(parser, stock_list):
304 return False
George Keishing9fbdf792016-10-18 06:16:09 -0500305
manasarm5dedfaf2018-02-07 14:54:54 +0530306 if not validate_parms():
307 return False
George Keishing9fbdf792016-10-18 06:16:09 -0500308
manasarm5dedfaf2018-02-07 14:54:54 +0530309 qprint_pgm_header()
George Keishing9fbdf792016-10-18 06:16:09 -0500310
George Keishing19533d52018-04-09 03:08:03 -0500311 parse_output_xml(source, dest, version_id, platform, level)
George Keishing9fbdf792016-10-18 06:16:09 -0500312
manasarm5dedfaf2018-02-07 14:54:54 +0530313 return True
314
315
316# Main
317
318if not main():
319 exit(1)