blob: 4e8b4192e4e3111b9f48c7b0507d70c252ef943a [file] [log] [blame]
George Keishing04d93452017-05-03 09:14:15 -05001#!/usr/bin/env python
2
3r"""
4This module contains keyword functions to supplement robot's built in
5functions and use in test where generic robot keywords don't support.
George Keishing04d93452017-05-03 09:14:15 -05006"""
Steven Sombar130a04f2017-07-16 10:02:37 -05007
8try:
9 from robot.libraries.BuiltIn import BuiltIn
10 from robot.libraries import DateTime
11except ImportError:
12 pass
George Keishing04d93452017-05-03 09:14:15 -050013import time
Steven Sombar130a04f2017-07-16 10:02:37 -050014import os
15import difflib
16
17
18##########################################################################
19def json_inv_file_diff_check(file1_path,
20 file2_path,
21 diff_file_path,
22 skip_dictionary):
23 r"""
24 Compare the contents of two files which contain inventory data in
25 JSON format. The comparison is similar to the unix 'diff' command but
26 the output lists the hardware subsystem (category) where differences
27 are found, and some differences are selectively ignored. The items
28 ignored are defined in the skip_dictionary dictionary.
29
30 Description of arguments:
31 file1_path File containing JSON formatted data.
32 file2_path File to compare to file1 to.
33 diff_file_path File which will contain the resulting difference report.
34 skip_dictionary Dictionary which defines what inventory items
35 to ignore if there are differences in inventory
36 files -- some differences are expected or
37 immaterial. Specify items in this dictionary
38 if there are inventory items that should be
39 ignored whenever a comparison between inventory
40 files is done. For example, assigned processor
41 speed routinely varies depending upon the
42 needs of OCC/tmgt/ondemand governor.
43 Back-to-back inventory runs may show
44 processor speed differences even if nothing
45 else was run between them.
46 Each dictionary entry is of the form
47 category:leafname where category is a JSON
48 hardware category such as "processor", "memory",
49 "disk", "display", "network", etc.,
50 and leafname is a leaf node (atribute name)
51 within that category.
52 For example: {'processor':'size'}, or
53 {'processor':'size','memory':'claimed','display':'id'}.
54
55 Sample difference report:
56 Difference at line 102 (in section "memory":)
57 102 - "slot": "UOPWR.BAR.1315ACA-DIMM0",
58 102 + "slot": "0"
59 Difference at line 126 (in section "processor":)
60 126 - "size": 2151000000, +++ NOTE! This is an ignore item
61 126 + "size": 2201000000, +++ NOTE! This is an ignore item
62
63 Returns:
64 0 if both files contain the same information or they differ only in
65 items specified as those to ignore.
66 2 if FILES_DO_NOT_MATCH.
67 3 if INPUT_FILE_DOES_NOT_EXIST.
68 4 if IO_EXCEPTION_READING_FILE.
69 5 if IO_EXCEPTION_WRITING_FILE.
70 """
71
72 FILES_MATCH = 0
73 FILES_DO_NOT_MATCH = 2
74 INPUT_FILE_DOES_NOT_EXIST = 3
75 IO_EXCEPTION_READING_FILE = 4
76 IO_EXCEPTION_WRITING_FILE = 5
77
78 # Hardware categories which are reported in the JSON inventory files.
79 hardware_categories = ['\"processor\":', '\"memory\":', '\"disk\":',
80 '\"I/O\":', '\"display\":', '\"generic\":',
81 '\"network\":', '\"communication\":',
82 '\"printer\":', '\"input\":', '\"multimedia\":',
83 '\"tape\":']
84
85 # The minimum size in bytes a JSON file must be.
86 min_json_byte_size = 16
87
88 now = time.strftime("At %Y-%m-%d %H:%M:%S")
89
90 if (not os.path.exists(file1_path) or (not os.path.exists(file2_path))):
91 return INPUT_FILE_DOES_NOT_EXIST
92 try:
93 with open(file1_path, 'r') as file:
94 initial = file.readlines()
95 with open(file2_path, 'r') as file:
96 final = file.readlines()
97 except IOError:
98 file.close()
99 return IO_EXCEPTION_READING_FILE
100 except ValueError:
101 file.close()
102 return INPUT_FILE_MALFORMED
103 else:
104 file.close()
105
106 # Must have more than a trivial number of bytes.
107 if len(initial) <= min_json_byte_size:
108 return INPUT_FILE_MALFORMED
109
110 if (initial == final):
111 try:
112 file = open(diff_file_path, 'w')
113 except IOError:
114 file.close()
115 line_to_print = now + " found no difference between file " + \
116 file1_path + " and " + \
117 file2_path + "\n"
118 file.write(line_to_print)
119 file.close()
120 return FILES_MATCH
121
122 # Find the differences and write difference report to diff_file_path file.
123 try:
124 file = open(diff_file_path, 'w')
125 except IOError:
126 file.close()
127 return IO_EXCEPTION_WRITING_FILE
128
129 line_to_print = now + " compared files " + \
130 file1_path + " and " + \
131 file2_path + "\n"
132 file.write(line_to_print)
133
134 diff = difflib.ndiff(initial, final)
135 # The diff array contains all lines that match in the
136 # initial and final arrays, and also all lines that differ.
137 # The first two characters of each line
138 # are prefixed with two letters, defined as:
139 # '- ' This line is unique to initial
140 # '+ ' This line is line unique to final
141 # ' ' This line is common to both, and
142 # '? ' This line indicates approximate differences.
143 # For example, comparing two three-line files:
144 # This line is in both initial and final.
145 # This line is too but the next line is different in each file.
146 # - "size": 2101000000,
147 # ? - ^
148 # + "size": 2002000000,
149 # ? ^^
150
151 print_header_flag = False
152 category = ""
153 row_num = 1
154 item_we_cannot_ignore = False
155
156 for my_line in diff:
157 diff_item = my_line.strip('\n')
158 # If it's a Category, such as processor or memory,
159 # save it. We will print it out later.
160 # Most lines do not explicitly contain a category. As such the
161 # category for that line is found in a previous line, which
162 # will be the category we last found.
163 for hdw_cat in hardware_categories:
164 if (hdw_cat in diff_item):
165 # If we don't have a match we will reuse
166 # the prvious category.
167 category = hdw_cat
168 # Lines beginning with minus or plus or q-mark are
169 # true difference items.
170 # We want to look at those in more detail.
171 if diff_item.startswith('? '):
172 # we can ignore these
173 continue
174 if (diff_item.startswith('- ') or diff_item.startswith('+ ')):
175 # If we have not printed the header line for this
176 # difference, print it now.
177 if print_header_flag is False:
178 line_to_print = "Difference at line " + \
179 str(row_num) + " (in section " + \
180 category + ")\n"
181 file.write(line_to_print)
182 # If this is in the ignore dictionary, we'll print
183 # it but also add text that it is an ignore item.
184 skipitem = False
185 for key, value in skip_dictionary.iteritems():
186 if ((key in category.lower().strip()) and
187 (value in diff_item.lower().strip())):
188 line_to_print = " " + \
189 str(row_num) + " " + diff_item + \
Steven Sombara2216d22017-08-09 12:18:00 -0500190 " +++ NOTE! This line matches" + \
Steven Sombar130a04f2017-07-16 10:02:37 -0500191 " the inventory ignore list and" + \
192 " can be ignored. +++\n"
193 # set flag indicating this item is a skip item
194 skipitem = True
195 break
196 if skipitem is False:
197 # Its not a skip item, that is,
198 # this is not on the ignore list.
199 # Print the item and set the item_we_canot_ignore flag
200 # indicating we have an item not on the ignore list. The
201 # flag will determine the return code we
202 # pass back to the user at the end.
203 item_we_cannot_ignore = True
204 line_to_print = " " + \
205 str(row_num) + " " + diff_item + "\n"
206 file.write(line_to_print)
207 print_header_flag = True
208
209 else:
210 # Adjust row numbering as a difference is only one line
211 # but it takes several lines in the diff file.
212 if print_header_flag is True:
213 row_num = row_num + 1
214 print_header_flag = False
215 row_num = row_num + 1
216
217 # Make sure we end the file.
218 file.write("\n")
219 file.close()
220
221 if item_we_cannot_ignore:
222 # We have at least one diff_item not on the ignore list.
223 return FILES_DO_NOT_MATCH
224 else:
225 # Any differences were on the ignore list.
226 return FILES_MATCH
227###############################################################################
228
George Keishing04d93452017-05-03 09:14:15 -0500229
230###############################################################################
George Keishing04d93452017-05-03 09:14:15 -0500231
George Keishing4bbf5202017-05-18 06:55:53 -0500232
Steven Sombar130a04f2017-07-16 10:02:37 -0500233def run_until_keyword_fails(retry,
234 retry_interval,
235 name,
236 *args):
George Keishing04d93452017-05-03 09:14:15 -0500237 r"""
238 Execute a robot keyword repeatedly until it either fails or the timeout
239 value is exceeded.
240 Note: Opposite of robot keyword "Wait Until Keyword Succeeds".
241
242 Description of argument(s):
243 retry Max timeout time in hour(s).
244 retry_interval Time interval in minute(s) for looping.
245 name Robot keyword to execute.
246 args Robot keyword arguments.
247 """
248
249 # Convert the retry time in seconds
George Keishing4bbf5202017-05-18 06:55:53 -0500250 retry_seconds = DateTime.convert_time(retry)
George Keishing04d93452017-05-03 09:14:15 -0500251 timeout = time.time() + int(retry_seconds)
252
253 # Convert the interval time in seconds
George Keishing4bbf5202017-05-18 06:55:53 -0500254 interval_seconds = DateTime.convert_time(retry_interval)
George Keishing04d93452017-05-03 09:14:15 -0500255 interval = int(interval_seconds)
256
257 BuiltIn().log(timeout)
258 BuiltIn().log(interval)
259
260 while True:
George Keishing4bbf5202017-05-18 06:55:53 -0500261 status = BuiltIn().run_keyword_and_return_status(name, *args)
George Keishing04d93452017-05-03 09:14:15 -0500262
263 # Return if keywords returns as failure.
George Keishing4bbf5202017-05-18 06:55:53 -0500264 if status is False:
George Keishing04d93452017-05-03 09:14:15 -0500265 BuiltIn().log("Failed as expected")
George Keishing4bbf5202017-05-18 06:55:53 -0500266 return False
George Keishing04d93452017-05-03 09:14:15 -0500267 # Return if retry timeout as success.
268 elif time.time() > timeout > 0:
269 BuiltIn().log("Max retry timeout")
George Keishing4bbf5202017-05-18 06:55:53 -0500270 return True
George Keishing04d93452017-05-03 09:14:15 -0500271 time.sleep(interval)
272 BuiltIn().log(time.time())
273
George Keishing4bbf5202017-05-18 06:55:53 -0500274 return True
275###############################################################################
276
277
278###############################################################################
279def htx_error_log_to_list(htx_error_log_output):
280
281 r"""
282 Parse htx error log output string and return list of strings in the form
283 "<field name>:<field value>".
284 The output of this function may be passed to the build_error_dict function.
285
286 Description of argument(s):
287 htx_error_log_output Error entry string containing the stdout
288 generated by "htxcmdline -geterrlog".
289
290 Example of htx_error_log_output contents:
291
292 ######################## Result Starts Here ###############################
293 Currently running ECG/MDT : /usr/lpp/htx/mdt/mdt.whit
294 ===========================
295 ---------------------------------------------------------------------
296 Device id:/dev/nvidia0
297 Timestamp:Mar 29 19:41:54 2017
298 err=00000027
299 sev=1
300 Exerciser Name:hxenvidia
301 Serial No:Not Available
302 Part No:Not Available
303 Location:Not Available
304 FRU Number:Not Available
305 Device:Not Available
306 Error Text:cudaEventSynchronize for stopEvent returned err = 0039 from file
307 , line 430.
308 ---------------------------------------------------------------------
309 ---------------------------------------------------------------------
310 Device id:/dev/nvidia0
311 Timestamp:Mar 29 19:41:54 2017
312 err=00000027
313 sev=1
314 Exerciser Name:hxenvidia
315 Serial No:Not Available
316 Part No:Not Available
317 Location:Not Available
318 FRU Number:Not Available
319 Device:Not Available
320 Error Text:Hardware Exerciser stopped on error
321 ---------------------------------------------------------------------
322 ######################### Result Ends Here ################################
323
324 Example output:
325 Returns the lists of error string per entry
326 ['Device id:/dev/nvidia0',
327 'Timestamp:Mar 29 19:41:54 2017',
328 'err=00000027',
329 'sev=1',
330 'Exerciser Name:hxenvidia',
331 'Serial No:Not Available',
332 'Part No:Not Available',
333 'Location:Not Available',
334 'FRU Number:Not Available',
335 'Device:Not Available',
336 'Error Text:cudaEventSynchronize for stopEvent returned err = 0039
337 from file , line 430.']
338 """
339
340 # List which will hold all the list of entries.
341 error_list = []
342
343 temp_error_list = []
344 parse_walk = False
345
346 for line in htx_error_log_output.splitlines():
347 # Skip lines starting with "#"
348 if line.startswith("#"):
349 continue
350
351 # Mark line starting with "-" and set parse flag.
352 if line.startswith("-") and parse_walk is False:
353 parse_walk = True
354 continue
355 # Mark line starting with "-" and reset parse flag.
356 # Set temp error list to EMPTY.
357 elif line.startswith("-"):
358 error_list.append(temp_error_list)
359 parse_walk = False
360 temp_error_list = []
361 # Add entry to list if line is not emtpy
362 elif parse_walk:
363 temp_error_list.append(str(line))
364
365 return error_list
366###############################################################################
367
368
369###############################################################################
370def build_error_dict(htx_error_log_output):
371
372 r"""
373 Builds error list into a list of dictionary entries.
374
375 Description of argument(s):
376 error_list Error list entries.
377
378 Example output dictionary:
379 {
380 0:
381 {
382 'sev': '1',
383 'err': '00000027',
384 'Timestamp': 'Mar 29 19:41:54 2017',
385 'Part No': 'Not Available',
386 'Serial No': 'Not Available',
387 'Device': 'Not Available',
388 'FRU Number': 'Not Available',
389 'Location': 'Not Available',
390 'Device id': '/dev/nvidia0',
391 'Error Text': 'cudaEventSynchronize for stopEvent returned err = 0039
392 from file , line 430.',
393 'Exerciser Name': 'hxenvidia'
394 },
395 1:
396 {
397 'sev': '1',
398 'err': '00000027',
399 'Timestamp': 'Mar 29 19:41:54 2017',
400 'Part No': 'Not Available',
401 'Serial No': 'Not Available',
402 'Device': 'Not Available',
403 'FRU Number': 'Not Available',
404 'Location': 'Not Available',
405 'Device id': '/dev/nvidia0',
406 'Error Text': 'Hardware Exerciser stopped on error',
407 'Exerciser Name': 'hxenvidia'
408 }
409 },
410
411 """
412
413 # List which will hold all the list of entries.
414 error_list = []
415 error_list = htx_error_log_to_list(htx_error_log_output)
416
417 # dictionary which holds the error dictionry entry.
418 error_dict = {}
419
420 temp_error_dict = {}
421 error_index = 0
422
423 # Loop through the error list.
424 for entry_list in error_list:
425 # Loop through the first error list entry.
426 for entry in entry_list:
427 # Split string into list for key value update.
428 # Example: 'Device id:/dev/nvidia0'
429 # Example: 'err=00000027'
430 parm_split = re.split("[:=]", entry)
431 # Populate temp dictionary with key value pair data.
432 temp_error_dict[str(parm_split[0])] = parm_split[1]
433
434 # Update the master dictionary per entry index.
435 error_dict[error_index] = temp_error_dict
436 # Reset temp dict to EMPTY and increment index count.
437 temp_error_dict = {}
438 error_index += 1
439
440 return error_dict
George Keishing04d93452017-05-03 09:14:15 -0500441
442###############################################################################