blob: 72a0f62837d512b970ff40e3771c73bab040117a [file] [log] [blame]
#!/usr/bin/env python
r"""
Define variable manipulation functions.
"""
import os
import re
try:
from robot.utils import DotDict
except ImportError:
pass
import collections
import gen_print as gp
import gen_misc as gm
import func_args as fa
def create_var_dict(*args):
r"""
Create a dictionary whose keys/values are the arg names/arg values passed to it and return it to the
caller.
Note: The resulting dictionary will be ordered.
Description of argument(s):
*args An unlimited number of arguments to be processed.
Example use:
first_name = 'Steve'
last_name = 'Smith'
var_dict = create_var_dict(first_name, last_name)
gp.print_var(var_dict)
The print-out of the resulting var dictionary is:
var_dict:
var_dict[first_name]: Steve
var_dict[last_name]: Smith
"""
try:
result_dict = collections.OrderedDict()
except AttributeError:
result_dict = DotDict()
arg_num = 1
for arg in args:
arg_name = gp.get_arg_name(None, arg_num, stack_frame_ix=2)
result_dict[arg_name] = arg
arg_num += 1
return result_dict
default_record_delim = ':'
default_key_val_delim = '.'
def join_dict(dict,
record_delim=default_record_delim,
key_val_delim=default_key_val_delim):
r"""
Join a dictionary's keys and values into a string and return the string.
Description of argument(s):
dict The dictionary whose keys and values are to be joined.
record_delim The delimiter to be used to separate dictionary pairs in the resulting
string.
key_val_delim The delimiter to be used to separate keys from values in the resulting
string.
Example use:
gp.print_var(var_dict)
str1 = join_dict(var_dict)
gp.print_var(str1)
Program output.
var_dict:
var_dict[first_name]: Steve
var_dict[last_name]: Smith
str1: first_name.Steve:last_name.Smith
"""
format_str = '%s' + key_val_delim + '%s'
return record_delim.join([format_str % (key, value) for (key, value) in
dict.items()])
def split_to_dict(string,
record_delim=default_record_delim,
key_val_delim=default_key_val_delim):
r"""
Split a string into a dictionary and return it.
This function is the complement to join_dict.
Description of argument(s):
string The string to be split into a dictionary. The string must have the
proper delimiters in it. A string created by join_dict would qualify.
record_delim The delimiter to be used to separate dictionary pairs in the input string.
key_val_delim The delimiter to be used to separate keys/values in the input string.
Example use:
gp.print_var(str1)
new_dict = split_to_dict(str1)
gp.print_var(new_dict)
Program output.
str1: first_name.Steve:last_name.Smith
new_dict:
new_dict[first_name]: Steve
new_dict[last_name]: Smith
"""
try:
result_dict = collections.OrderedDict()
except AttributeError:
result_dict = DotDict()
raw_keys_values = string.split(record_delim)
for key_value in raw_keys_values:
key_value_list = key_value.split(key_val_delim)
try:
result_dict[key_value_list[0]] = key_value_list[1]
except IndexError:
result_dict[key_value_list[0]] = ""
return result_dict
def create_file_path(file_name_dict,
dir_path="/tmp/",
file_suffix=""):
r"""
Create a file path using the given parameters and return it.
Description of argument(s):
file_name_dict A dictionary with keys/values which are to appear as part of the file
name.
dir_path The dir_path that is to appear as part of the file name.
file_suffix A suffix to be included as part of the file name.
"""
dir_path = gm.add_trailing_slash(dir_path)
return dir_path + join_dict(file_name_dict) + file_suffix
def parse_file_path(file_path):
r"""
Parse a file path created by create_file_path and return the result as a dictionary.
This function is the complement to create_file_path.
Description of argument(s):
file_path The file_path.
Example use:
gp.print_var(boot_results_file_path)
file_path_data = parse_file_path(boot_results_file_path)
gp.print_var(file_path_data)
Program output.
boot_results_file_path:
/tmp/pgm_name.obmc_boot_test:openbmc_nickname.beye6:master_pid.2039:boot_results
file_path_data:
file_path_data[dir_path]: /tmp/
file_path_data[pgm_name]: obmc_boot_test
file_path_data[openbmc_nickname]: beye6
file_path_data[master_pid]: 2039
file_path_data[boot_results]:
"""
try:
result_dict = collections.OrderedDict()
except AttributeError:
result_dict = DotDict()
dir_path = os.path.dirname(file_path) + os.sep
file_path = os.path.basename(file_path)
result_dict['dir_path'] = dir_path
result_dict.update(split_to_dict(file_path))
return result_dict
def parse_key_value(string,
delim=":",
strip=" ",
to_lower=1,
underscores=1):
r"""
Parse a key/value string and return as a key/value tuple.
This function is useful for parsing a line of program output or data that is in the following form:
<key or variable name><delimiter><value>
An example of a key/value string would be as follows:
Current Limit State: No Active Power Limit
In the example shown, the delimiter is ":". The resulting key would be as follows:
Current Limit State
Note: If one were to take the default values of to_lower=1 and underscores=1, the resulting key would be
as follows:
current_limit_state
The to_lower and underscores arguments are provided for those who wish to have their key names have the
look and feel of python variable names.
The resulting value for the example above would be as follows:
No Active Power Limit
Another example:
name=Mike
In this case, the delim would be "=", the key is "name" and the value is "Mike".
Description of argument(s):
string The string to be parsed.
delim The delimiter which separates the key from the value.
strip The characters (if any) to strip from the beginning and end of both the
key and the value.
to_lower Change the key name to lower case.
underscores Change any blanks found in the key name to underscores.
"""
pair = string.split(delim)
key = pair[0].strip(strip)
if len(pair) == 0:
value = ""
else:
value = delim.join(pair[1:]).strip(strip)
if to_lower:
key = key.lower()
if underscores:
key = re.sub(r" ", "_", key)
return key, value
def key_value_list_to_dict(key_value_list,
process_indent=0,
**args):
r"""
Convert a list containing key/value strings or tuples to a dictionary and return it.
See docstring of parse_key_value function for details on key/value strings.
Example usage:
For the following value of key_value_list:
key_value_list:
[0]: Current Limit State: No Active Power Limit
[1]: Exception actions: Hard Power Off & Log Event to SEL
[2]: Power Limit: 0 Watts
[3]: Correction time: 0 milliseconds
[4]: Sampling period: 0 seconds
And the following call in python:
power_limit = key_value_outbuf_to_dict(key_value_list)
The resulting power_limit directory would look like this:
power_limit:
[current_limit_state]: No Active Power Limit
[exception_actions]: Hard Power Off & Log Event to SEL
[power_limit]: 0 Watts
[correction_time]: 0 milliseconds
[sampling_period]: 0 seconds
For the following list:
headers:
headers[0]:
headers[0][0]: content-length
headers[0][1]: 559
headers[1]:
headers[1][0]: x-xss-protection
headers[1][1]: 1; mode=block
And the following call in python:
headers_dict = key_value_list_to_dict(headers)
The resulting headers_dict would look like this:
headers_dict:
[content-length]: 559
[x-xss-protection]: 1; mode=block
Another example containing a sub-list (see process_indent description below):
Provides Device SDRs : yes
Additional Device Support :
Sensor Device
SEL Device
FRU Inventory Device
Chassis Device
Note that the 2 qualifications for containing a sub-list are met: 1) 'Additional Device Support' has no
value and 2) The entries below it are indented. In this case those entries contain no delimiters (":")
so they will be processed as a list rather than as a dictionary. The result would be as follows:
mc_info:
mc_info[provides_device_sdrs]: yes
mc_info[additional_device_support]:
mc_info[additional_device_support][0]: Sensor Device
mc_info[additional_device_support][1]: SEL Device
mc_info[additional_device_support][2]: FRU Inventory Device
mc_info[additional_device_support][3]: Chassis Device
Description of argument(s):
key_value_list A list of key/value strings. (See docstring of parse_key_value function
for details).
process_indent This indicates that indented sub-dictionaries and sub-lists are to be
processed as such. An entry may have a sub-dict or sub-list if 1) It has
no value other than blank 2) There are entries below it that are
indented. Note that process_indent is not allowed for a list of tuples
(vs. a list of key/value strings).
**args Arguments to be interpreted by parse_key_value. (See docstring of
parse_key_value function for details).
"""
try:
result_dict = collections.OrderedDict()
except AttributeError:
result_dict = DotDict()
if not process_indent:
for entry in key_value_list:
if type(entry) is tuple:
key, value = entry
else:
key, value = parse_key_value(entry, **args)
result_dict[key] = value
return result_dict
# Process list while paying heed to indentation.
delim = args.get("delim", ":")
# Initialize "parent_" indentation level variables.
parent_indent = len(key_value_list[0]) - len(key_value_list[0].lstrip())
sub_list = []
for entry in key_value_list:
key, value = parse_key_value(entry, **args)
indent = len(entry) - len(entry.lstrip())
if indent > parent_indent and parent_value == "":
# This line is indented compared to the parent entry and the parent entry has no value.
# Append the entry to sub_list for later processing.
sub_list.append(str(entry))
continue
# Process any outstanding sub_list and add it to result_dict[parent_key].
if len(sub_list) > 0:
if any(delim in word for word in sub_list):
# If delim is found anywhere in the sub_list, we'll process as a sub-dictionary.
result_dict[parent_key] = key_value_list_to_dict(sub_list,
**args)
else:
result_dict[parent_key] = list(map(str.strip, sub_list))
del sub_list[:]
result_dict[key] = value
parent_key = key
parent_value = value
parent_indent = indent
# Any outstanding sub_list to be processed?
if len(sub_list) > 0:
if any(delim in word for word in sub_list):
# If delim is found anywhere in the sub_list, we'll process as a sub-dictionary.
result_dict[parent_key] = key_value_list_to_dict(sub_list, **args)
else:
result_dict[parent_key] = list(map(str.strip, sub_list))
return result_dict
def key_value_outbuf_to_dict(out_buf,
**args):
r"""
Convert a buffer with a key/value string on each line to a dictionary and return it.
Each line in the out_buf should end with a \n.
See docstring of parse_key_value function for details on key/value strings.
Example usage:
For the following value of out_buf:
Current Limit State: No Active Power Limit
Exception actions: Hard Power Off & Log Event to SEL
Power Limit: 0 Watts
Correction time: 0 milliseconds
Sampling period: 0 seconds
And the following call in python:
power_limit = key_value_outbuf_to_dict(out_buf)
The resulting power_limit directory would look like this:
power_limit:
[current_limit_state]: No Active Power Limit
[exception_actions]: Hard Power Off & Log Event to SEL
[power_limit]: 0 Watts
[correction_time]: 0 milliseconds
[sampling_period]: 0 seconds
Description of argument(s):
out_buf A buffer with a key/value string on each line. (See docstring of
parse_key_value function for details).
**args Arguments to be interpreted by parse_key_value. (See docstring of
parse_key_value function for details).
"""
# Create key_var_list and remove null entries.
key_var_list = list(filter(None, out_buf.split("\n")))
return key_value_list_to_dict(key_var_list, **args)
def key_value_outbuf_to_dicts(out_buf,
**args):
r"""
Convert a buffer containing multiple sections with key/value strings on each line to a list of
dictionaries and return it.
Sections in the output are delimited by blank lines.
Example usage:
For the following value of out_buf:
Maximum User IDs : 15
Enabled User IDs : 1
User ID : 1
User Name : root
Fixed Name : No
Access Available : callback
Link Authentication : enabled
IPMI Messaging : enabled
Privilege Level : ADMINISTRATOR
Enable Status : enabled
User ID : 2
User Name :
Fixed Name : No
Access Available : call-in / callback
Link Authentication : disabled
IPMI Messaging : disabled
Privilege Level : NO ACCESS
Enable Status : disabled
And the following call in python:
user_info = key_value_outbuf_to_dicts(out_buf)
The resulting user_info list would look like this:
user_info:
[0]:
[maximum_user_ids]: 15
[enabled_user_ids]: 1
[1]:
[user_id]: 1
[user_name]: root
[fixed_name]: No
[access_available]: callback
[link_authentication]: enabled
[ipmi_messaging]: enabled
[privilege_level]: ADMINISTRATOR
[enable_status]: enabled
[2]:
[user_id]: 2
[user_name]:
[fixed_name]: No
[access_available]: call-in / callback
[link_authentication]: disabled
[ipmi_messaging]: disabled
[privilege_level]: NO ACCESS
[enable_status]: disabled
Description of argument(s):
out_buf A buffer with multiple secionts of key/value strings on each line.
Sections are delimited by one or more blank lines (i.e. line feeds). (See
docstring of parse_key_value function for details).
**args Arguments to be interpreted by parse_key_value. (See docstring of
parse_key_value function for details).
"""
return [key_value_outbuf_to_dict(x, **args) for x in re.split('\n[\n]+', out_buf)]
def create_field_desc_regex(line):
r"""
Create a field descriptor regular expression based on the input line and return it.
This function is designed for use by the list_to_report function (defined below).
Example:
Given the following input line:
-------- ------------ ------------------ ------------------------
This function will return this regular expression:
(.{8}) (.{12}) (.{18}) (.{24})
This means that other report lines interpreted using the regular expression are expected to have:
- An 8 character field
- 3 spaces
- A 12 character field
- One space
- An 18 character field
- One space
- A 24 character field
Description of argument(s):
line A line consisting of dashes to represent fields and spaces to delimit
fields.
"""
# Split the line into a descriptors list. Example:
# descriptors:
# descriptors[0]: --------
# descriptors[1]:
# descriptors[2]:
# descriptors[3]: ------------
# descriptors[4]: ------------------
# descriptors[5]: ------------------------
descriptors = line.split(" ")
# Create regexes list. Example:
# regexes:
# regexes[0]: (.{8})
# regexes[1]:
# regexes[2]:
# regexes[3]: (.{12})
# regexes[4]: (.{18})
# regexes[5]: (.{24})
regexes = []
for descriptor in descriptors:
if descriptor == "":
regexes.append("")
else:
regexes.append("(.{" + str(len(descriptor)) + "})")
# Join the regexes list into a regex string.
field_desc_regex = ' '.join(regexes)
return field_desc_regex
def list_to_report(report_list,
to_lower=1,
field_delim=None):
r"""
Convert a list containing report text lines to a report "object" and return it.
The first entry in report_list must be a header line consisting of column names delimited by white space.
No column name may contain white space. The remaining report_list entries should contain tabular data
which corresponds to the column names.
A report object is a list where each entry is a dictionary whose keys are the field names from the first
entry in report_list.
Example:
Given the following report_list as input:
rl:
rl[0]: Filesystem 1K-blocks Used Available Use% Mounted on
rl[1]: dev 247120 0 247120 0% /dev
rl[2]: tmpfs 248408 79792 168616 32% /run
This function will return a list of dictionaries as shown below:
df_report:
df_report[0]:
[filesystem]: dev
[1k-blocks]: 247120
[used]: 0
[available]: 247120
[use%]: 0%
[mounted]: /dev
df_report[1]:
[filesystem]: dev
[1k-blocks]: 247120
[used]: 0
[available]: 247120
[use%]: 0%
[mounted]: /dev
Notice that because "Mounted on" contains a space, "on" would be considered the 7th field. In this case,
there is never any data in field 7 so things work out nicely. A caller could do some pre-processing if
desired (e.g. change "Mounted on" to "Mounted_on").
Example 2:
If the 2nd line of report data is a series of dashes and spaces as in the following example, that line
will serve to delineate columns.
The 2nd line of data is like this:
ID status size tool,clientid,userid
-------- ------------ ------------------ ------------------------
20000001 in progress 0x7D0 ,,
Description of argument(s):
report_list A list where each entry is one line of output from a report. The first
entry must be a header line which contains column names. Column names
may not contain spaces.
to_lower Change the resulting key names to lower case.
field_delim Indicates that there are field delimiters in report_list entries (which
should be removed).
"""
if len(report_list) <= 1:
# If we don't have at least a descriptor line and one line of data, return an empty array.
return []
if field_delim is not None:
report_list = [re.sub("\\|", "", line) for line in report_list]
header_line = report_list[0]
if to_lower:
header_line = header_line.lower()
field_desc_regex = ""
if re.match(r"^-[ -]*$", report_list[1]):
# We have a field descriptor line (as shown in example 2 above).
field_desc_regex = create_field_desc_regex(report_list[1])
field_desc_len = len(report_list[1])
pad_format_string = "%-" + str(field_desc_len) + "s"
# The field descriptor line has served its purpose. Deleting it.
del report_list[1]
# Process the header line by creating a list of column names.
if field_desc_regex == "":
columns = header_line.split()
else:
# Pad the line with spaces on the right to facilitate processing with field_desc_regex.
header_line = pad_format_string % header_line
columns = list(map(str.strip,
re.findall(field_desc_regex, header_line)[0]))
report_obj = []
for report_line in report_list[1:]:
if field_desc_regex == "":
line = report_line.split()
else:
# Pad the line with spaces on the right to facilitate processing with field_desc_regex.
report_line = pad_format_string % report_line
line = list(map(str.strip,
re.findall(field_desc_regex, report_line)[0]))
try:
line_dict = collections.OrderedDict(zip(columns, line))
except AttributeError:
line_dict = DotDict(zip(columns, line))
report_obj.append(line_dict)
return report_obj
def outbuf_to_report(out_buf,
**args):
r"""
Convert a text buffer containing report lines to a report "object" and return it.
Refer to list_to_report (above) for more details.
Example:
Given the following out_buf:
Filesystem 1K-blocks Used Available Use% Mounted on
dev 247120 0 247120 0% /dev
tmpfs 248408 79792 168616 32% /run
This function will return a list of dictionaries as shown below:
df_report:
df_report[0]:
[filesystem]: dev
[1k-blocks]: 247120
[used]: 0
[available]: 247120
[use%]: 0%
[mounted]: /dev
df_report[1]:
[filesystem]: dev
[1k-blocks]: 247120
[used]: 0
[available]: 247120
[use%]: 0%
[mounted]: /dev
Other possible uses:
- Process the output of a ps command.
- Process the output of an ls command (the caller would need to supply column names)
Description of argument(s):
out_buf A text report. The first line must be a header line which contains
column names. Column names may not contain spaces.
**args Arguments to be interpreted by list_to_report. (See docstring of
list_to_report function for details).
"""
report_list = list(filter(None, out_buf.split("\n")))
return list_to_report(report_list, **args)
def nested_get(key_name, structure):
r"""
Return a list of all values from the nested structure that have the given key name.
Example:
Given a dictionary structure named "personnel" with the following contents:
personnel:
[manager]:
[last_name]: Doe
[first_name]: John
[accountant]:
[last_name]: Smith
[first_name]: Will
The following code...
last_names = nested_get('last_name', personnel)
print_var(last_names)
Would result in the following data returned:
last_names:
last_names[0]: Doe
last_names[1]: Smith
Description of argument(s):
key_name The key name (e.g. 'last_name').
structure Any nested combination of lists or dictionaries (e.g. a dictionary, a
dictionary of dictionaries, a list of dictionaries, etc.). This function
will locate the given key at any level within the structure and include
its value in the returned list.
"""
result = []
if type(structure) is list:
for entry in structure:
result += nested_get(key_name, entry)
return result
elif gp.is_dict(structure):
for key, value in structure.items():
result += nested_get(key_name, value)
if key == key_name:
result.append(value)
return result
def match_struct(structure, match_dict, regex=False):
r"""
Return True or False to indicate whether the structure matches the match dictionary.
Example:
Given a dictionary structure named "personnel" with the following contents:
personnel:
[manager]:
[last_name]: Doe
[first_name]: John
[accountant]:
[last_name]: Smith
[first_name]: Will
The following call would return True.
match_struct(personnel, {'last_name': '^Doe$'}, regex=True)
Whereas the following call would return False.
match_struct(personnel, {'last_name': 'Johnson'}, regex=True)
Description of argument(s):
structure Any nested combination of lists or dictionaries. See the prolog of
get_nested() for details.
match_dict Each key/value pair in match_dict must exist somewhere in the structure
for the structure to be considered a match. A match value of None is
considered a special case where the structure would be considered a match
only if the key in question is found nowhere in the structure.
regex Indicates whether the values in the match_dict should be interpreted as
regular expressions.
"""
# The structure must match for each match_dict entry to be considered a match. Therefore, any failure
# to match is grounds for returning False.
for match_key, match_value in match_dict.items():
struct_key_values = nested_get(match_key, structure)
if match_value is None:
# Handle this as special case.
if len(struct_key_values) != 0:
return False
else:
if len(struct_key_values) == 0:
return False
if regex:
matches = [x for x in struct_key_values
if re.search(match_value, str(x))]
if not matches:
return False
elif match_value not in struct_key_values:
return False
return True
def filter_struct(structure, filter_dict, regex=False, invert=False):
r"""
Filter the structure by removing any entries that do NOT contain the keys/values specified in filter_dict
and return the result.
The selection process is directed only at the first-level entries of the structure.
Example:
Given a dictionary named "properties" that has the following structure:
properties:
[/redfish/v1/Systems/system/Processors]:
[Members]:
[0]:
[@odata.id]: /redfish/v1/Systems/system/Processors/cpu0
[1]:
[@odata.id]: /redfish/v1/Systems/system/Processors/cpu1
[/redfish/v1/Systems/system/Processors/cpu0]:
[Status]:
[State]: Enabled
[Health]: OK
[/redfish/v1/Systems/system/Processors/cpu1]:
[Status]:
[State]: Enabled
[Health]: Bad
The following call:
properties = filter_struct(properties, "[('Health', 'OK')]")
Would return a new properties dictionary that looks like this:
properties:
[/redfish/v1/Systems/system/Processors/cpu0]:
[Status]:
[State]: Enabled
[Health]: OK
Note that the first item in the original properties directory had no key anywhere in the structure named
"Health". Therefore, that item failed to make the cut. The next item did have a key named "Health"
whose value was "OK" so it was included in the new structure. The third item had a key named "Health"
but its value was not "OK" so it also failed to make the cut.
Description of argument(s):
structure Any nested combination of lists or dictionaries. See the prolog of
get_nested() for details.
filter_dict For each key/value pair in filter_dict, each entry in structure must
contain the same key/value pair at some level. A filter_dict value of
None is treated as a special case. Taking the example shown above,
[('State', None)] would mean that the result should only contain records
that have no State key at all.
regex Indicates whether the values in the filter_dict should be interpreted as
regular expressions.
invert Invert the results. Instead of including only matching entries in the
results, include only NON-matching entries in the results.
"""
# Convert filter_dict from a string containing a python object definition to an actual python object (if
# warranted).
filter_dict = fa.source_to_object(filter_dict)
# Determine whether structure is a list or a dictionary and process accordingly. The result returned
# will be of the same type as the structure.
if type(structure) is list:
result = []
for element in structure:
if match_struct(element, filter_dict, regex) != invert:
result.append(element)
else:
try:
result = collections.OrderedDict()
except AttributeError:
result = DotDict()
for struct_key, struct_value in structure.items():
if match_struct(struct_value, filter_dict, regex) != invert:
result[struct_key] = struct_value
return result
def split_dict_on_key(split_key, dictionary):
r"""
Split a dictionary into two dictionaries based on the first occurrence of the split key and return the
resulting sub-dictionaries.
Example:
dictionary = {'one': 1, 'two': 2, 'three':3, 'four':4}
dict1, dict2 = split_dict_on_key('three', dictionary)
pvars(dictionary, dict1, dict2)
Output:
dictionary:
[one]: 1
[two]: 2
[three]: 3
[four]: 4
dict1:
[one]: 1
[two]: 2
dict2:
[three]: 3
[four]: 4
Description of argument(s):
split_key The key value to be used to determine where the dictionary should be
split.
dictionary The dictionary to be split.
"""
dict1 = {}
dict2 = {}
found_split_key = False
for key in list(dictionary.keys()):
if key == split_key:
found_split_key = True
if found_split_key:
dict2[key] = dictionary[key]
else:
dict1[key] = dictionary[key]
return dict1, dict2