blob: ceed9f8d34f713abb4acd587ba8a9168411c8145 [file] [log] [blame]
Michael Walshced4eb02017-09-19 16:49:13 -05001#!/usr/bin/env python
2
3r"""
4Define variable manipulation functions.
5"""
6
7import os
Michael Walsh05c68d92017-09-20 16:36:37 -05008import re
Michael Walshced4eb02017-09-19 16:49:13 -05009
10try:
11 from robot.utils import DotDict
12except ImportError:
13 pass
14
15import collections
16
17import gen_print as gp
18import gen_misc as gm
19
20
21def create_var_dict(*args):
Michael Walshced4eb02017-09-19 16:49:13 -050022 r"""
23 Create a dictionary whose keys/values are the arg names/arg values passed
24 to it and return it to the caller.
25
26 Note: The resulting dictionary will be ordered.
27
28 Description of argument(s):
29 *args An unlimited number of arguments to be processed.
30
31 Example use:
32
33 first_name = 'Steve'
34 last_name = 'Smith'
35 var_dict = create_var_dict(first_name, last_name)
36
37 gp.print_var(var_dict)
38
39 The print-out of the resulting var dictionary is:
40 var_dict:
41 var_dict[first_name]: Steve
42 var_dict[last_name]: Smith
43 """
44
45 try:
46 result_dict = collections.OrderedDict()
47 except AttributeError:
48 result_dict = DotDict()
49
50 arg_num = 1
51 for arg in args:
52 arg_name = gp.get_arg_name(None, arg_num, stack_frame_ix=2)
53 result_dict[arg_name] = arg
54 arg_num += 1
55
56 return result_dict
57
58
59default_record_delim = ':'
60default_key_val_delim = '.'
61
62
63def join_dict(dict,
64 record_delim=default_record_delim,
65 key_val_delim=default_key_val_delim):
Michael Walshced4eb02017-09-19 16:49:13 -050066 r"""
67 Join a dictionary's keys and values into a string and return the string.
68
69 Description of argument(s):
70 dict The dictionary whose keys and values are
71 to be joined.
72 record_delim The delimiter to be used to separate
73 dictionary pairs in the resulting string.
74 key_val_delim The delimiter to be used to separate keys
75 from values in the resulting string.
76
77 Example use:
78
79 gp.print_var(var_dict)
80 str1 = join_dict(var_dict)
81 gp.pvar(str1)
82
83 Program output.
84 var_dict:
85 var_dict[first_name]: Steve
86 var_dict[last_name]: Smith
87 str1:
88 first_name.Steve:last_name.Smith
89 """
90
91 format_str = '%s' + key_val_delim + '%s'
92 return record_delim.join([format_str % (key, value) for (key, value) in
Gunnar Mills096cd562018-03-26 10:19:12 -050093 dict.items()])
Michael Walshced4eb02017-09-19 16:49:13 -050094
95
96def split_to_dict(string,
97 record_delim=default_record_delim,
98 key_val_delim=default_key_val_delim):
Michael Walshced4eb02017-09-19 16:49:13 -050099 r"""
100 Split a string into a dictionary and return it.
101
102 This function is the complement to join_dict.
103
104 Description of argument(s):
105 string The string to be split into a dictionary.
106 The string must have the proper delimiters
107 in it. A string created by join_dict
108 would qualify.
109 record_delim The delimiter to be used to separate
110 dictionary pairs in the input string.
111 key_val_delim The delimiter to be used to separate
112 keys/values in the input string.
113
114 Example use:
115
116 gp.print_var(str1)
117 new_dict = split_to_dict(str1)
118 gp.print_var(new_dict)
119
120
121 Program output.
122 str1:
123 first_name.Steve:last_name.Smith
124 new_dict:
125 new_dict[first_name]: Steve
126 new_dict[last_name]: Smith
127 """
128
129 try:
130 result_dict = collections.OrderedDict()
131 except AttributeError:
132 result_dict = DotDict()
133
134 raw_keys_values = string.split(record_delim)
135 for key_value in raw_keys_values:
136 key_value_list = key_value.split(key_val_delim)
137 try:
138 result_dict[key_value_list[0]] = key_value_list[1]
139 except IndexError:
140 result_dict[key_value_list[0]] = ""
141
142 return result_dict
143
144
145def create_file_path(file_name_dict,
146 dir_path="/tmp/",
147 file_suffix=""):
Michael Walshced4eb02017-09-19 16:49:13 -0500148 r"""
149 Create a file path using the given parameters and return it.
150
151 Description of argument(s):
152 file_name_dict A dictionary with keys/values which are to
153 appear as part of the file name.
154 dir_path The dir_path that is to appear as part of
155 the file name.
156 file_suffix A suffix to be included as part of the
157 file name.
158 """
159
160 dir_path = gm.add_trailing_slash(dir_path)
161 return dir_path + join_dict(file_name_dict) + file_suffix
162
163
164def parse_file_path(file_path):
Michael Walshced4eb02017-09-19 16:49:13 -0500165 r"""
166 Parse a file path created by create_file_path and return the result as a
167 dictionary.
168
169 This function is the complement to create_file_path.
170
171 Description of argument(s):
172 file_path The file_path.
173
174 Example use:
175 gp.pvar(boot_results_file_path)
176 file_path_data = parse_file_path(boot_results_file_path)
177 gp.pvar(file_path_data)
178
179 Program output.
180
181 boot_results_file_path:
182 /tmp/pgm_name.obmc_boot_test:openbmc_nickname.beye6:master_pid.2039:boot_re
183 sults
184 file_path_data:
185 file_path_data[dir_path]: /tmp/
186 file_path_data[pgm_name]: obmc_boot_test
187 file_path_data[openbmc_nickname]: beye6
188 file_path_data[master_pid]: 2039
189 file_path_data[boot_results]:
190 """
191
192 try:
193 result_dict = collections.OrderedDict()
194 except AttributeError:
195 result_dict = DotDict()
196
197 dir_path = os.path.dirname(file_path) + os.sep
198 file_path = os.path.basename(file_path)
199
200 result_dict['dir_path'] = dir_path
201
202 result_dict.update(split_to_dict(file_path))
203
204 return result_dict
Michael Walsh05c68d92017-09-20 16:36:37 -0500205
206
207def parse_key_value(string,
208 delim=":",
209 strip=" ",
210 to_lower=1,
211 underscores=1):
Michael Walsh05c68d92017-09-20 16:36:37 -0500212 r"""
213 Parse a key/value string and return as a key/value tuple.
214
215 This function is useful for parsing a line of program output or data that
216 is in the following form:
217 <key or variable name><delimiter><value>
218
219 An example of a key/value string would be as follows:
220
221 Current Limit State: No Active Power Limit
222
223 In the example shown, the delimiter is ":". The resulting key would be as
224 follows:
225 Current Limit State
226
227 Note: If one were to take the default values of to_lower=1 and
228 underscores=1, the resulting key would be as follows:
229 current_limit_state
230
231 The to_lower and underscores arguments are provided for those who wish to
232 have their key names have the look and feel of python variable names.
233
234 The resulting value for the example above would be as follows:
235 No Active Power Limit
236
237 Another example:
238 name=Mike
239
240 In this case, the delim would be "=", the key is "name" and the value is
241 "Mike".
242
243 Description of argument(s):
244 string The string to be parsed.
245 delim The delimiter which separates the key from
246 the value.
247 strip The characters (if any) to strip from the
248 beginning and end of both the key and the
249 value.
250 to_lower Change the key name to lower case.
251 underscores Change any blanks found in the key name to
252 underscores.
253 """
254
255 pair = string.split(delim)
256
257 key = pair[0].strip(strip)
258 if len(pair) == 0:
259 value = ""
260 else:
MICHAEL J. WALSH9509a0f2018-02-08 11:08:52 -0600261 value = delim.join(pair[1:]).strip(strip)
Michael Walsh05c68d92017-09-20 16:36:37 -0500262
263 if to_lower:
264 key = key.lower()
265 if underscores:
266 key = re.sub(r" ", "_", key)
267
268 return key, value
269
270
271def key_value_list_to_dict(list,
Michael Walshcad07132018-02-19 17:28:01 -0600272 process_indent=0,
Michael Walsh05c68d92017-09-20 16:36:37 -0500273 **args):
Michael Walsh05c68d92017-09-20 16:36:37 -0500274 r"""
275 Convert a list containing key/value strings to a dictionary and return it.
276
277 See docstring of parse_key_value function for details on key/value strings.
278
279 Example usage:
280
281 For the following value of list:
282
283 list:
284 list[0]: Current Limit State: No Active Power Limit
285 list[1]: Exception actions: Hard Power Off & Log Event to SEL
286 list[2]: Power Limit: 0 Watts
287 list[3]: Correction time: 0 milliseconds
288 list[4]: Sampling period: 0 seconds
289
290 And the following call in python:
291
292 power_limit = key_value_outbuf_to_dict(list)
293
294 The resulting power_limit directory would look like this:
295
296 power_limit:
297 [current_limit_state]: No Active Power Limit
298 [exception_actions]: Hard Power Off & Log Event to SEL
299 [power_limit]: 0 Watts
300 [correction_time]: 0 milliseconds
301 [sampling_period]: 0 seconds
302
Michael Walshcad07132018-02-19 17:28:01 -0600303 Another example containing a sub-list (see process_indent description
304 below):
305
306 Provides Device SDRs : yes
307 Additional Device Support :
308 Sensor Device
309 SEL Device
310 FRU Inventory Device
311 Chassis Device
312
313 Note that the 2 qualifications for containing a sub-list are met: 1)
314 'Additional Device Support' has no value and 2) The entries below it are
315 indented. In this case those entries contain no delimiters (":") so they
316 will be processed as a list rather than as a dictionary. The result would
317 be as follows:
318
319 mc_info:
320 mc_info[provides_device_sdrs]: yes
321 mc_info[additional_device_support]:
322 mc_info[additional_device_support][0]: Sensor Device
323 mc_info[additional_device_support][1]: SEL Device
324 mc_info[additional_device_support][2]: FRU Inventory Device
325 mc_info[additional_device_support][3]: Chassis Device
326
Michael Walsh05c68d92017-09-20 16:36:37 -0500327 Description of argument(s):
328 list A list of key/value strings. (See
329 docstring of parse_key_value function for
330 details).
Michael Walshcad07132018-02-19 17:28:01 -0600331 process_indent This indicates that indented
332 sub-dictionaries and sub-lists are to be
333 processed as such. An entry may have a
334 sub-dict or sub-list if 1) It has no value
335 other than blank 2) There are entries
336 below it that are indented.
Michael Walsh05c68d92017-09-20 16:36:37 -0500337 **args Arguments to be interpreted by
338 parse_key_value. (See docstring of
339 parse_key_value function for details).
340 """
341
342 try:
343 result_dict = collections.OrderedDict()
344 except AttributeError:
345 result_dict = DotDict()
346
Michael Walshcad07132018-02-19 17:28:01 -0600347 if not process_indent:
348 for entry in list:
349 key, value = parse_key_value(entry, **args)
350 result_dict[key] = value
351 return result_dict
352
353 # Process list while paying heed to indentation.
354 delim = args.get("delim", ":")
355 # Initialize "parent_" indentation level variables.
356 parent_indent = len(list[0]) - len(list[0].lstrip())
357 sub_list = []
Michael Walsh05c68d92017-09-20 16:36:37 -0500358 for entry in list:
Michael Walshc1dfc782017-09-26 16:08:51 -0500359 key, value = parse_key_value(entry, **args)
Michael Walshcad07132018-02-19 17:28:01 -0600360
361 indent = len(entry) - len(entry.lstrip())
362
363 if indent > parent_indent and parent_value == "":
364 # This line is indented compared to the parent entry and the
365 # parent entry has no value.
366 # Append the entry to sub_list for later processing.
367 sub_list.append(str(entry))
368 continue
369
370 # Process any outstanding sub_list and add it to
371 # result_dict[parent_key].
372 if len(sub_list) > 0:
373 if any(delim in word for word in sub_list):
374 # If delim is found anywhere in the sub_list, we'll process
375 # as a sub-dictionary.
376 result_dict[parent_key] = key_value_list_to_dict(sub_list,
377 **args)
378 else:
379 result_dict[parent_key] = map(str.strip, sub_list)
380 del sub_list[:]
381
Michael Walsh05c68d92017-09-20 16:36:37 -0500382 result_dict[key] = value
383
Michael Walshcad07132018-02-19 17:28:01 -0600384 parent_key = key
385 parent_value = value
386 parent_indent = indent
387
388 # Any outstanding sub_list to be processed?
389 if len(sub_list) > 0:
390 if any(delim in word for word in sub_list):
391 # If delim is found anywhere in the sub_list, we'll process as a
392 # sub-dictionary.
393 result_dict[parent_key] = key_value_list_to_dict(sub_list, **args)
394 else:
395 result_dict[parent_key] = map(str.strip, sub_list)
396
Michael Walsh05c68d92017-09-20 16:36:37 -0500397 return result_dict
398
399
400def key_value_outbuf_to_dict(out_buf,
401 **args):
Michael Walsh05c68d92017-09-20 16:36:37 -0500402 r"""
403 Convert a buffer with a key/value string on each line to a dictionary and
404 return it.
405
406 Each line in the out_buf should end with a \n.
407
408 See docstring of parse_key_value function for details on key/value strings.
409
410 Example usage:
411
412 For the following value of out_buf:
413
414 Current Limit State: No Active Power Limit
415 Exception actions: Hard Power Off & Log Event to SEL
416 Power Limit: 0 Watts
417 Correction time: 0 milliseconds
418 Sampling period: 0 seconds
419
420 And the following call in python:
421
422 power_limit = key_value_outbuf_to_dict(out_buf)
423
424 The resulting power_limit directory would look like this:
425
426 power_limit:
427 [current_limit_state]: No Active Power Limit
428 [exception_actions]: Hard Power Off & Log Event to SEL
429 [power_limit]: 0 Watts
430 [correction_time]: 0 milliseconds
431 [sampling_period]: 0 seconds
432
433 Description of argument(s):
434 out_buf A buffer with a key/value string on each
435 line. (See docstring of parse_key_value
436 function for details).
437 **args Arguments to be interpreted by
438 parse_key_value. (See docstring of
439 parse_key_value function for details).
440 """
441
442 # Create key_var_list and remove null entries.
443 key_var_list = list(filter(None, out_buf.split("\n")))
Michael Walshc1dfc782017-09-26 16:08:51 -0500444 return key_value_list_to_dict(key_var_list, **args)
Michael Walshdb560d42017-11-20 16:42:49 -0600445
446
Michael Walshdc978822018-07-12 15:34:13 -0500447def create_field_desc_regex(line):
448
449 r"""
450 Create a field descriptor regular expression based on the input line and
451 return it.
452
453 This function is designed for use by the list_to_report function (defined
454 below).
455
456 Example:
457
458 Given the following input line:
459
460 -------- ------------ ------------------ ------------------------
461
462 This function will return this regular expression:
463
464 (.{8}) (.{12}) (.{18}) (.{24})
465
466 This means that other report lines interpreted using the regular
467 expression are expected to have:
468 - An 8 character field
469 - 3 spaces
470 - A 12 character field
471 - One space
472 - An 18 character field
473 - One space
474 - A 24 character field
475
476 Description of argument(s):
477 line A line consisting of dashes to represent
478 fields and spaces to delimit fields.
479 """
480
481 # Split the line into a descriptors list. Example:
482 # descriptors:
483 # descriptors[0]: --------
484 # descriptors[1]:
485 # descriptors[2]:
486 # descriptors[3]: ------------
487 # descriptors[4]: ------------------
488 # descriptors[5]: ------------------------
489 descriptors = line.split(" ")
490
491 # Create regexes list. Example:
492 # regexes:
493 # regexes[0]: (.{8})
494 # regexes[1]:
495 # regexes[2]:
496 # regexes[3]: (.{12})
497 # regexes[4]: (.{18})
498 # regexes[5]: (.{24})
499 regexes = []
500 for descriptor in descriptors:
501 if descriptor == "":
502 regexes.append("")
503 else:
504 regexes.append("(.{" + str(len(descriptor)) + "})")
505
506 # Join the regexes list into a regex string.
507 field_desc_regex = ' '.join(regexes)
508
509 return field_desc_regex
510
511
Michael Walshdb560d42017-11-20 16:42:49 -0600512def list_to_report(report_list,
Michael Walsh64043d52018-09-21 16:40:44 -0500513 to_lower=1,
514 field_delim=None):
Michael Walshdb560d42017-11-20 16:42:49 -0600515 r"""
516 Convert a list containing report text lines to a report "object" and
517 return it.
518
519 The first entry in report_list must be a header line consisting of column
520 names delimited by white space. No column name may contain white space.
521 The remaining report_list entries should contain tabular data which
522 corresponds to the column names.
523
524 A report object is a list where each entry is a dictionary whose keys are
525 the field names from the first entry in report_list.
526
527 Example:
528 Given the following report_list as input:
529
530 rl:
531 rl[0]: Filesystem 1K-blocks Used Available Use% Mounted on
532 rl[1]: dev 247120 0 247120 0% /dev
533 rl[2]: tmpfs 248408 79792 168616 32% /run
534
535 This function will return a list of dictionaries as shown below:
536
537 df_report:
538 df_report[0]:
539 [filesystem]: dev
540 [1k-blocks]: 247120
541 [used]: 0
542 [available]: 247120
543 [use%]: 0%
544 [mounted]: /dev
545 df_report[1]:
546 [filesystem]: dev
547 [1k-blocks]: 247120
548 [used]: 0
549 [available]: 247120
550 [use%]: 0%
551 [mounted]: /dev
552
553 Notice that because "Mounted on" contains a space, "on" would be
554 considered the 7th field. In this case, there is never any data in field
555 7 so things work out nicely. A caller could do some pre-processing if
556 desired (e.g. change "Mounted on" to "Mounted_on").
557
Michael Walshdc978822018-07-12 15:34:13 -0500558 Example 2:
559
560 If the 2nd line of report data is a series of dashes and spaces as in the
561 following example, that line will serve to delineate columns.
562
563 The 2nd line of data is like this:
564 ID status size
565 tool,clientid,userid
566 -------- ------------ ------------------ ------------------------
567 20000001 in progress 0x7D0 ,,
568
Michael Walshdb560d42017-11-20 16:42:49 -0600569 Description of argument(s):
570 report_list A list where each entry is one line of
571 output from a report. The first entry
572 must be a header line which contains
573 column names. Column names may not
574 contain spaces.
575 to_lower Change the resulting key names to lower
576 case.
Michael Walsh64043d52018-09-21 16:40:44 -0500577 field_delim Indicates that there are field delimiters
578 in report_list entries (which should be
579 removed).
Michael Walshdb560d42017-11-20 16:42:49 -0600580 """
581
Michael Walshdc978822018-07-12 15:34:13 -0500582 if len(report_list) <= 1:
583 # If we don't have at least a descriptor line and one line of data,
584 # return an empty array.
585 return []
586
Michael Walsh64043d52018-09-21 16:40:44 -0500587 if field_delim is not None:
588 report_list = [re.sub("\\|", "", line) for line in report_list]
589
Michael Walshdb560d42017-11-20 16:42:49 -0600590 header_line = report_list[0]
591 if to_lower:
592 header_line = header_line.lower()
Michael Walshdc978822018-07-12 15:34:13 -0500593
594 field_desc_regex = ""
595 if re.match(r"^-[ -]*$", report_list[1]):
596 # We have a field descriptor line (as shown in example 2 above).
597 field_desc_regex = create_field_desc_regex(report_list[1])
598 field_desc_len = len(report_list[1])
599 pad_format_string = "%-" + str(field_desc_len) + "s"
600 # The field descriptor line has served its purpose. Deleting it.
601 del report_list[1]
602
603 # Process the header line by creating a list of column names.
604 if field_desc_regex == "":
605 columns = header_line.split()
606 else:
607 # Pad the line with spaces on the right to facilitate processing with
608 # field_desc_regex.
609 header_line = pad_format_string % header_line
610 columns = map(str.strip, re.findall(field_desc_regex, header_line)[0])
Michael Walshdb560d42017-11-20 16:42:49 -0600611
612 report_obj = []
613 for report_line in report_list[1:]:
Michael Walshdc978822018-07-12 15:34:13 -0500614 if field_desc_regex == "":
615 line = report_line.split()
616 else:
617 # Pad the line with spaces on the right to facilitate processing
618 # with field_desc_regex.
619 report_line = pad_format_string % report_line
620 line = map(str.strip, re.findall(field_desc_regex, report_line)[0])
Michael Walshdb560d42017-11-20 16:42:49 -0600621 try:
622 line_dict = collections.OrderedDict(zip(columns, line))
623 except AttributeError:
624 line_dict = DotDict(zip(columns, line))
625 report_obj.append(line_dict)
626
627 return report_obj
628
629
630def outbuf_to_report(out_buf,
631 **args):
Michael Walshdb560d42017-11-20 16:42:49 -0600632 r"""
633 Convert a text buffer containing report lines to a report "object" and
634 return it.
635
636 Refer to list_to_report (above) for more details.
637
638 Example:
639
640 Given the following out_buf:
641
Michael Walshdc978822018-07-12 15:34:13 -0500642 Filesystem 1K-blocks Used Available Use% Mounted
643 on
644 dev 247120 0 247120 0% /dev
645 tmpfs 248408 79792 168616 32% /run
Michael Walshdb560d42017-11-20 16:42:49 -0600646
647 This function will return a list of dictionaries as shown below:
648
649 df_report:
650 df_report[0]:
651 [filesystem]: dev
652 [1k-blocks]: 247120
653 [used]: 0
654 [available]: 247120
655 [use%]: 0%
656 [mounted]: /dev
657 df_report[1]:
658 [filesystem]: dev
659 [1k-blocks]: 247120
660 [used]: 0
661 [available]: 247120
662 [use%]: 0%
663 [mounted]: /dev
664
665 Other possible uses:
666 - Process the output of a ps command.
667 - Process the output of an ls command (the caller would need to supply
Michael Walshdc978822018-07-12 15:34:13 -0500668 column names)
Michael Walshdb560d42017-11-20 16:42:49 -0600669
670 Description of argument(s):
Michael Walshdc978822018-07-12 15:34:13 -0500671 out_buf A text report. The first line must be a
Michael Walshdb560d42017-11-20 16:42:49 -0600672 header line which contains column names.
673 Column names may not contain spaces.
674 **args Arguments to be interpreted by
675 list_to_report. (See docstring of
676 list_to_report function for details).
677 """
678
Michael Walsh255181c2018-08-07 15:06:23 -0500679 report_list = list(filter(None, out_buf.split("\n")))
Michael Walshdb560d42017-11-20 16:42:49 -0600680 return list_to_report(report_list, **args)
Michael Walsh7822b9e2019-03-12 16:34:38 -0500681
682
683def nested_get(key, dictionary):
684 r"""
685 Return a list of all values from the nested dictionary with the given key.
686
687 Example:
688
689 Given a dictionary named personnel with the following contents:
690
691 personnel:
692 [manager]:
693 [last_name]: Doe
694 [first_name]: John
695 [accountant]:
696 [last_name]: Smith
697 [first_name]: Will
698
699 The following code...
700
701 last_names = nested_get('last_name', personnel)
702 print_var(last_names)
703
704 Would result in the following data:
705
706 last_names:
707 last_names[0]: Doe
708 last_names[1]: Smith
709
710 Description of argument(s):
711 key The key value.
712 dictionary The nested dictionary.
713 """
714
715 result = []
716 for k, v in dictionary.items():
717 if isinstance(v, dict):
718 result += nested_get(key, v)
719 if k == key:
720 result.append(v)
721
722 return result