blob: 35fb594e7e54554be1ab96d296950b83fd8f0313 [file] [log] [blame]
#!/usr/bin/env python
r"""
Define variable manipulation functions.
"""
import os
import re
try:
from robot.utils import DotDict
except ImportError:
pass
import collections
import gen_print as gp
import gen_misc as gm
import func_args as fa
def create_var_dict(*args):
r"""
Create a dictionary whose keys/values are the arg names/arg values passed
to it and return it to the caller.
Note: The resulting dictionary will be ordered.
Description of argument(s):
*args An unlimited number of arguments to be processed.
Example use:
first_name = 'Steve'
last_name = 'Smith'
var_dict = create_var_dict(first_name, last_name)
gp.print_var(var_dict)
The print-out of the resulting var dictionary is:
var_dict:
var_dict[first_name]: Steve
var_dict[last_name]: Smith
"""
try:
result_dict = collections.OrderedDict()
except AttributeError:
result_dict = DotDict()
arg_num = 1
for arg in args:
arg_name = gp.get_arg_name(None, arg_num, stack_frame_ix=2)
result_dict[arg_name] = arg
arg_num += 1
return result_dict
default_record_delim = ':'
default_key_val_delim = '.'
def join_dict(dict,
record_delim=default_record_delim,
key_val_delim=default_key_val_delim):
r"""
Join a dictionary's keys and values into a string and return the string.
Description of argument(s):
dict The dictionary whose keys and values are
to be joined.
record_delim The delimiter to be used to separate
dictionary pairs in the resulting string.
key_val_delim The delimiter to be used to separate keys
from values in the resulting string.
Example use:
gp.print_var(var_dict)
str1 = join_dict(var_dict)
gp.print_var(str1)
Program output.
var_dict:
var_dict[first_name]: Steve
var_dict[last_name]: Smith
str1:
first_name.Steve:last_name.Smith
"""
format_str = '%s' + key_val_delim + '%s'
return record_delim.join([format_str % (key, value) for (key, value) in
dict.items()])
def split_to_dict(string,
record_delim=default_record_delim,
key_val_delim=default_key_val_delim):
r"""
Split a string into a dictionary and return it.
This function is the complement to join_dict.
Description of argument(s):
string The string to be split into a dictionary.
The string must have the proper delimiters
in it. A string created by join_dict
would qualify.
record_delim The delimiter to be used to separate
dictionary pairs in the input string.
key_val_delim The delimiter to be used to separate
keys/values in the input string.
Example use:
gp.print_var(str1)
new_dict = split_to_dict(str1)
gp.print_var(new_dict)
Program output.
str1:
first_name.Steve:last_name.Smith
new_dict:
new_dict[first_name]: Steve
new_dict[last_name]: Smith
"""
try:
result_dict = collections.OrderedDict()
except AttributeError:
result_dict = DotDict()
raw_keys_values = string.split(record_delim)
for key_value in raw_keys_values:
key_value_list = key_value.split(key_val_delim)
try:
result_dict[key_value_list[0]] = key_value_list[1]
except IndexError:
result_dict[key_value_list[0]] = ""
return result_dict
def create_file_path(file_name_dict,
dir_path="/tmp/",
file_suffix=""):
r"""
Create a file path using the given parameters and return it.
Description of argument(s):
file_name_dict A dictionary with keys/values which are to
appear as part of the file name.
dir_path The dir_path that is to appear as part of
the file name.
file_suffix A suffix to be included as part of the
file name.
"""
dir_path = gm.add_trailing_slash(dir_path)
return dir_path + join_dict(file_name_dict) + file_suffix
def parse_file_path(file_path):
r"""
Parse a file path created by create_file_path and return the result as a
dictionary.
This function is the complement to create_file_path.
Description of argument(s):
file_path The file_path.
Example use:
gp.print_var(boot_results_file_path)
file_path_data = parse_file_path(boot_results_file_path)
gp.print_var(file_path_data)
Program output.
boot_results_file_path:
/tmp/pgm_name.obmc_boot_test:openbmc_nickname.beye6:master_pid.2039:boot_re
sults
file_path_data:
file_path_data[dir_path]: /tmp/
file_path_data[pgm_name]: obmc_boot_test
file_path_data[openbmc_nickname]: beye6
file_path_data[master_pid]: 2039
file_path_data[boot_results]:
"""
try:
result_dict = collections.OrderedDict()
except AttributeError:
result_dict = DotDict()
dir_path = os.path.dirname(file_path) + os.sep
file_path = os.path.basename(file_path)
result_dict['dir_path'] = dir_path
result_dict.update(split_to_dict(file_path))
return result_dict
def parse_key_value(string,
delim=":",
strip=" ",
to_lower=1,
underscores=1):
r"""
Parse a key/value string and return as a key/value tuple.
This function is useful for parsing a line of program output or data that
is in the following form:
<key or variable name><delimiter><value>
An example of a key/value string would be as follows:
Current Limit State: No Active Power Limit
In the example shown, the delimiter is ":". The resulting key would be as
follows:
Current Limit State
Note: If one were to take the default values of to_lower=1 and
underscores=1, the resulting key would be as follows:
current_limit_state
The to_lower and underscores arguments are provided for those who wish to
have their key names have the look and feel of python variable names.
The resulting value for the example above would be as follows:
No Active Power Limit
Another example:
name=Mike
In this case, the delim would be "=", the key is "name" and the value is
"Mike".
Description of argument(s):
string The string to be parsed.
delim The delimiter which separates the key from
the value.
strip The characters (if any) to strip from the
beginning and end of both the key and the
value.
to_lower Change the key name to lower case.
underscores Change any blanks found in the key name to
underscores.
"""
pair = string.split(delim)
key = pair[0].strip(strip)
if len(pair) == 0:
value = ""
else:
value = delim.join(pair[1:]).strip(strip)
if to_lower:
key = key.lower()
if underscores:
key = re.sub(r" ", "_", key)
return key, value
def key_value_list_to_dict(list,
process_indent=0,
**args):
r"""
Convert a list containing key/value strings or tuples to a dictionary and
return it.
See docstring of parse_key_value function for details on key/value strings.
Example usage:
For the following value of list:
list:
list[0]: Current Limit State: No Active Power Limit
list[1]: Exception actions: Hard Power Off & Log Event to SEL
list[2]: Power Limit: 0 Watts
list[3]: Correction time: 0 milliseconds
list[4]: Sampling period: 0 seconds
And the following call in python:
power_limit = key_value_outbuf_to_dict(list)
The resulting power_limit directory would look like this:
power_limit:
[current_limit_state]: No Active Power Limit
[exception_actions]: Hard Power Off & Log Event to SEL
[power_limit]: 0 Watts
[correction_time]: 0 milliseconds
[sampling_period]: 0 seconds
For the following list:
headers:
headers[0]:
headers[0][0]: content-length
headers[0][1]: 559
headers[1]:
headers[1][0]: x-xss-protection
headers[1][1]: 1; mode=block
And the following call in python:
headers_dict = key_value_list_to_dict(headers)
The resulting headers_dict would look like this:
headers_dict:
[content-length]: 559
[x-xss-protection]: 1; mode=block
Another example containing a sub-list (see process_indent description
below):
Provides Device SDRs : yes
Additional Device Support :
Sensor Device
SEL Device
FRU Inventory Device
Chassis Device
Note that the 2 qualifications for containing a sub-list are met: 1)
'Additional Device Support' has no value and 2) The entries below it are
indented. In this case those entries contain no delimiters (":") so they
will be processed as a list rather than as a dictionary. The result would
be as follows:
mc_info:
mc_info[provides_device_sdrs]: yes
mc_info[additional_device_support]:
mc_info[additional_device_support][0]: Sensor Device
mc_info[additional_device_support][1]: SEL Device
mc_info[additional_device_support][2]: FRU Inventory Device
mc_info[additional_device_support][3]: Chassis Device
Description of argument(s):
list A list of key/value strings. (See
docstring of parse_key_value function for
details).
process_indent This indicates that indented
sub-dictionaries and sub-lists are to be
processed as such. An entry may have a
sub-dict or sub-list if 1) It has no value
other than blank 2) There are entries
below it that are indented. Note that
process_indent is not allowed for a list
of tuples (vs. a list of key/value
strings).
**args Arguments to be interpreted by
parse_key_value. (See docstring of
parse_key_value function for details).
"""
try:
result_dict = collections.OrderedDict()
except AttributeError:
result_dict = DotDict()
if not process_indent:
for entry in list:
if type(entry) is tuple:
key, value = entry
else:
key, value = parse_key_value(entry, **args)
result_dict[key] = value
return result_dict
# Process list while paying heed to indentation.
delim = args.get("delim", ":")
# Initialize "parent_" indentation level variables.
parent_indent = len(list[0]) - len(list[0].lstrip())
sub_list = []
for entry in list:
key, value = parse_key_value(entry, **args)
indent = len(entry) - len(entry.lstrip())
if indent > parent_indent and parent_value == "":
# This line is indented compared to the parent entry and the
# parent entry has no value.
# Append the entry to sub_list for later processing.
sub_list.append(str(entry))
continue
# Process any outstanding sub_list and add it to
# result_dict[parent_key].
if len(sub_list) > 0:
if any(delim in word for word in sub_list):
# If delim is found anywhere in the sub_list, we'll process
# as a sub-dictionary.
result_dict[parent_key] = key_value_list_to_dict(sub_list,
**args)
else:
result_dict[parent_key] = map(str.strip, sub_list)
del sub_list[:]
result_dict[key] = value
parent_key = key
parent_value = value
parent_indent = indent
# Any outstanding sub_list to be processed?
if len(sub_list) > 0:
if any(delim in word for word in sub_list):
# If delim is found anywhere in the sub_list, we'll process as a
# sub-dictionary.
result_dict[parent_key] = key_value_list_to_dict(sub_list, **args)
else:
result_dict[parent_key] = map(str.strip, sub_list)
return result_dict
def key_value_outbuf_to_dict(out_buf,
**args):
r"""
Convert a buffer with a key/value string on each line to a dictionary and
return it.
Each line in the out_buf should end with a \n.
See docstring of parse_key_value function for details on key/value strings.
Example usage:
For the following value of out_buf:
Current Limit State: No Active Power Limit
Exception actions: Hard Power Off & Log Event to SEL
Power Limit: 0 Watts
Correction time: 0 milliseconds
Sampling period: 0 seconds
And the following call in python:
power_limit = key_value_outbuf_to_dict(out_buf)
The resulting power_limit directory would look like this:
power_limit:
[current_limit_state]: No Active Power Limit
[exception_actions]: Hard Power Off & Log Event to SEL
[power_limit]: 0 Watts
[correction_time]: 0 milliseconds
[sampling_period]: 0 seconds
Description of argument(s):
out_buf A buffer with a key/value string on each
line. (See docstring of parse_key_value
function for details).
**args Arguments to be interpreted by
parse_key_value. (See docstring of
parse_key_value function for details).
"""
# Create key_var_list and remove null entries.
key_var_list = list(filter(None, out_buf.split("\n")))
return key_value_list_to_dict(key_var_list, **args)
def create_field_desc_regex(line):
r"""
Create a field descriptor regular expression based on the input line and
return it.
This function is designed for use by the list_to_report function (defined
below).
Example:
Given the following input line:
-------- ------------ ------------------ ------------------------
This function will return this regular expression:
(.{8}) (.{12}) (.{18}) (.{24})
This means that other report lines interpreted using the regular
expression are expected to have:
- An 8 character field
- 3 spaces
- A 12 character field
- One space
- An 18 character field
- One space
- A 24 character field
Description of argument(s):
line A line consisting of dashes to represent
fields and spaces to delimit fields.
"""
# Split the line into a descriptors list. Example:
# descriptors:
# descriptors[0]: --------
# descriptors[1]:
# descriptors[2]:
# descriptors[3]: ------------
# descriptors[4]: ------------------
# descriptors[5]: ------------------------
descriptors = line.split(" ")
# Create regexes list. Example:
# regexes:
# regexes[0]: (.{8})
# regexes[1]:
# regexes[2]:
# regexes[3]: (.{12})
# regexes[4]: (.{18})
# regexes[5]: (.{24})
regexes = []
for descriptor in descriptors:
if descriptor == "":
regexes.append("")
else:
regexes.append("(.{" + str(len(descriptor)) + "})")
# Join the regexes list into a regex string.
field_desc_regex = ' '.join(regexes)
return field_desc_regex
def list_to_report(report_list,
to_lower=1,
field_delim=None):
r"""
Convert a list containing report text lines to a report "object" and
return it.
The first entry in report_list must be a header line consisting of column
names delimited by white space. No column name may contain white space.
The remaining report_list entries should contain tabular data which
corresponds to the column names.
A report object is a list where each entry is a dictionary whose keys are
the field names from the first entry in report_list.
Example:
Given the following report_list as input:
rl:
rl[0]: Filesystem 1K-blocks Used Available Use% Mounted on
rl[1]: dev 247120 0 247120 0% /dev
rl[2]: tmpfs 248408 79792 168616 32% /run
This function will return a list of dictionaries as shown below:
df_report:
df_report[0]:
[filesystem]: dev
[1k-blocks]: 247120
[used]: 0
[available]: 247120
[use%]: 0%
[mounted]: /dev
df_report[1]:
[filesystem]: dev
[1k-blocks]: 247120
[used]: 0
[available]: 247120
[use%]: 0%
[mounted]: /dev
Notice that because "Mounted on" contains a space, "on" would be
considered the 7th field. In this case, there is never any data in field
7 so things work out nicely. A caller could do some pre-processing if
desired (e.g. change "Mounted on" to "Mounted_on").
Example 2:
If the 2nd line of report data is a series of dashes and spaces as in the
following example, that line will serve to delineate columns.
The 2nd line of data is like this:
ID status size
tool,clientid,userid
-------- ------------ ------------------ ------------------------
20000001 in progress 0x7D0 ,,
Description of argument(s):
report_list A list where each entry is one line of
output from a report. The first entry
must be a header line which contains
column names. Column names may not
contain spaces.
to_lower Change the resulting key names to lower
case.
field_delim Indicates that there are field delimiters
in report_list entries (which should be
removed).
"""
if len(report_list) <= 1:
# If we don't have at least a descriptor line and one line of data,
# return an empty array.
return []
if field_delim is not None:
report_list = [re.sub("\\|", "", line) for line in report_list]
header_line = report_list[0]
if to_lower:
header_line = header_line.lower()
field_desc_regex = ""
if re.match(r"^-[ -]*$", report_list[1]):
# We have a field descriptor line (as shown in example 2 above).
field_desc_regex = create_field_desc_regex(report_list[1])
field_desc_len = len(report_list[1])
pad_format_string = "%-" + str(field_desc_len) + "s"
# The field descriptor line has served its purpose. Deleting it.
del report_list[1]
# Process the header line by creating a list of column names.
if field_desc_regex == "":
columns = header_line.split()
else:
# Pad the line with spaces on the right to facilitate processing with
# field_desc_regex.
header_line = pad_format_string % header_line
columns = map(str.strip, re.findall(field_desc_regex, header_line)[0])
report_obj = []
for report_line in report_list[1:]:
if field_desc_regex == "":
line = report_line.split()
else:
# Pad the line with spaces on the right to facilitate processing
# with field_desc_regex.
report_line = pad_format_string % report_line
line = map(str.strip, re.findall(field_desc_regex, report_line)[0])
try:
line_dict = collections.OrderedDict(zip(columns, line))
except AttributeError:
line_dict = DotDict(zip(columns, line))
report_obj.append(line_dict)
return report_obj
def outbuf_to_report(out_buf,
**args):
r"""
Convert a text buffer containing report lines to a report "object" and
return it.
Refer to list_to_report (above) for more details.
Example:
Given the following out_buf:
Filesystem 1K-blocks Used Available Use% Mounted
on
dev 247120 0 247120 0% /dev
tmpfs 248408 79792 168616 32% /run
This function will return a list of dictionaries as shown below:
df_report:
df_report[0]:
[filesystem]: dev
[1k-blocks]: 247120
[used]: 0
[available]: 247120
[use%]: 0%
[mounted]: /dev
df_report[1]:
[filesystem]: dev
[1k-blocks]: 247120
[used]: 0
[available]: 247120
[use%]: 0%
[mounted]: /dev
Other possible uses:
- Process the output of a ps command.
- Process the output of an ls command (the caller would need to supply
column names)
Description of argument(s):
out_buf A text report. The first line must be a
header line which contains column names.
Column names may not contain spaces.
**args Arguments to be interpreted by
list_to_report. (See docstring of
list_to_report function for details).
"""
report_list = list(filter(None, out_buf.split("\n")))
return list_to_report(report_list, **args)
def nested_get(key_name, structure):
r"""
Return a list of all values from the nested structure that have the given
key name.
Example:
Given a dictionary structure named "personnel" with the following contents:
personnel:
[manager]:
[last_name]: Doe
[first_name]: John
[accountant]:
[last_name]: Smith
[first_name]: Will
The following code...
last_names = nested_get('last_name', personnel)
print_var(last_names)
Would result in the following data returned:
last_names:
last_names[0]: Doe
last_names[1]: Smith
Description of argument(s):
key_name The key name (e.g. 'last_name').
structure Any nested combination of lists or
dictionaries (e.g. a dictionary, a
dictionary of dictionaries, a list of
dictionaries, etc.). This function will
locate the given key at any level within
the structure and include its value in the
returned list.
"""
result = []
if type(structure) is list:
for entry in structure:
result += nested_get(key_name, entry)
return result
elif gp.is_dict(structure):
for key, value in structure.items():
result += nested_get(key_name, value)
if key == key_name:
result.append(value)
return result
def match_struct(structure, match_dict, regex=False):
r"""
Return True or False to indicate whether the structure matches the match
dictionary.
Example:
Given a dictionary structure named "personnel" with the following contents:
personnel:
[manager]:
[last_name]: Doe
[first_name]: John
[accountant]:
[last_name]: Smith
[first_name]: Will
The following call would return True.
match_struct(personnel, {'last_name': '^Doe$'}, regex=True)
Whereas the following call would return False.
match_struct(personnel, {'last_name': 'Johnson'}, regex=True)
Description of argument(s):
structure Any nested combination of lists or
dictionaries. See the prolog of
get_nested() for details.
match_dict Each key/value pair in match_dict must
exist somewhere in the structure for the
structure to be considered a match. A
match value of None is considered a
special case where the structure would be
considered a match only if the key in
question is found nowhere in the structure.
regex Indicates whether the values in the
match_dict should be interpreted as
regular expressions.
"""
# The structure must match for each match_dict entry to be considered a
# match. Therefore, any failure to match is grounds for returning False.
for match_key, match_value in match_dict.items():
struct_key_values = nested_get(match_key, structure)
if match_value is None:
# Handle this as special case.
if len(struct_key_values) != 0:
return False
else:
if len(struct_key_values) == 0:
return False
if regex:
matches = [x for x in struct_key_values
if re.search(match_value, str(x))]
if not matches:
return False
elif match_value not in struct_key_values:
return False
return True
def filter_struct(structure, filter_dict, regex=False, invert=False):
r"""
Filter the structure by removing any entries that do NOT contain the
keys/values specified in filter_dict and return the result.
The selection process is directed only at the first-level entries of the
structure.
Example:
Given a dictionary named "properties" that has the following structure:
properties:
[/redfish/v1/Systems/system/Processors]:
[Members]:
[0]:
[@odata.id]:
/redfish/v1/Systems/system/Processors/cpu0
[1]:
[@odata.id]:
/redfish/v1/Systems/system/Processors/cpu1
[/redfish/v1/Systems/system/Processors/cpu0]:
[Status]:
[State]: Enabled
[Health]: OK
[/redfish/v1/Systems/system/Processors/cpu1]:
[Status]:
[State]: Enabled
[Health]: Bad
The following call:
properties = filter_struct(properties, "[('Health', 'OK')]")
Would return a new properties dictionary that looks like this:
properties:
[/redfish/v1/Systems/system/Processors/cpu0]:
[Status]:
[State]: Enabled
[Health]: OK
Note that the first item in the original properties directory had no key
anywhere in the structure named "Health". Therefore, that item failed to
make the cut. The next item did have a key named "Health" whose value was
"OK" so it was included in the new structure. The third item had a key
named "Health" but its value was not "OK" so it also failed to make the
cut.
Description of argument(s):
structure Any nested combination of lists or
dictionaries. See the prolog of
get_nested() for details.
filter_dict For each key/value pair in filter_dict,
each entry in structure must contain the
same key/value pair at some level. A
filter_dict value of None is treated as a
special case. Taking the example shown
above, [('State', None)] would mean that
the result should only contain records
that have no State key at all.
regex Indicates whether the values in the
filter_dict should be interpreted as
regular expressions.
invert Invert the results. Instead of including
only matching entries in the results,
include only NON-matching entries in the
results.
"""
# Convert filter_dict from a string containing a python object definition
# to an actual python object (if warranted).
filter_dict = fa.source_to_object(filter_dict)
# Determine whether structure is a list or a dictionary and process
# accordingly. The result returned will be of the same type as the
# structure.
if type(structure) is list:
result = []
for element in structure:
if match_struct(element, filter_dict, regex) != invert:
result.append(element)
else:
try:
result = collections.OrderedDict()
except AttributeError:
result = DotDict()
for struct_key, struct_value in structure.items():
if match_struct(struct_value, filter_dict, regex) != invert:
result[struct_key] = struct_value
return result