lib/var_funcs.py

#!/usr/bin/env python

r"""
Define variable manipulation functions.
"""

import os
import re

try:
    from robot.utils import DotDict
except ImportError:
    pass

import collections

import gen_print as gp
import gen_misc as gm
import func_args as fa


def create_var_dict(*args):
    r"""
    Create a dictionary whose keys/values are the arg names/arg values passed to it and return it to the
    caller.

    Note: The resulting dictionary will be ordered.

    Description of argument(s):
    *args  An unlimited number of arguments to be processed.

    Example use:

    first_name = 'Steve'
    last_name = 'Smith'
    var_dict = create_var_dict(first_name, last_name)

    gp.print_var(var_dict)

    The print-out of the resulting var dictionary is:
    var_dict:
      var_dict[first_name]:                           Steve
      var_dict[last_name]:                            Smith
    """

    try:
        result_dict = collections.OrderedDict()
    except AttributeError:
        result_dict = DotDict()

    arg_num = 1
    for arg in args:
        arg_name = gp.get_arg_name(None, arg_num, stack_frame_ix=2)
        result_dict[arg_name] = arg
        arg_num += 1

    return result_dict


default_record_delim = ':'
default_key_val_delim = '.'


def join_dict(dict,
              record_delim=default_record_delim,
              key_val_delim=default_key_val_delim):
    r"""
    Join a dictionary's keys and values into a string and return the string.

    Description of argument(s):
    dict                            The dictionary whose keys and values are to be joined.
    record_delim                    The delimiter to be used to separate dictionary pairs in the resulting
                                    string.
    key_val_delim                   The delimiter to be used to separate keys from values in the resulting
                                    string.

    Example use:

    gp.print_var(var_dict)
    str1 = join_dict(var_dict)
    gp.print_var(str1)

    Program output.
    var_dict:
      var_dict[first_name]:                           Steve
      var_dict[last_name]:                            Smith
    str1:                                             first_name.Steve:last_name.Smith
    """

    format_str = '%s' + key_val_delim + '%s'
    return record_delim.join([format_str % (key, value) for (key, value) in
                              dict.items()])


def split_to_dict(string,
                  record_delim=default_record_delim,
                  key_val_delim=default_key_val_delim):
    r"""
    Split a string into a dictionary and return it.

    This function is the complement to join_dict.

    Description of argument(s):
    string                          The string to be split into a dictionary.  The string must have the
                                    proper delimiters in it.  A string created by join_dict would qualify.
    record_delim                    The delimiter to be used to separate dictionary pairs in the input string.
    key_val_delim                   The delimiter to be used to separate keys/values in the input string.

    Example use:

    gp.print_var(str1)
    new_dict = split_to_dict(str1)
    gp.print_var(new_dict)


    Program output.
    str1:                                             first_name.Steve:last_name.Smith
    new_dict:
      new_dict[first_name]:                           Steve
      new_dict[last_name]:                            Smith
    """

    try:
        result_dict = collections.OrderedDict()
    except AttributeError:
        result_dict = DotDict()

    raw_keys_values = string.split(record_delim)
    for key_value in raw_keys_values:
        key_value_list = key_value.split(key_val_delim)
        try:
            result_dict[key_value_list[0]] = key_value_list[1]
        except IndexError:
            result_dict[key_value_list[0]] = ""

    return result_dict


def create_file_path(file_name_dict,
                     dir_path="/tmp/",
                     file_suffix=""):
    r"""
    Create a file path using the given parameters and return it.

    Description of argument(s):
    file_name_dict                  A dictionary with keys/values which are to appear as part of the file
                                    name.
    dir_path                        The dir_path that is to appear as part of the file name.
    file_suffix                     A suffix to be included as part of the file name.
    """

    dir_path = gm.add_trailing_slash(dir_path)
    return dir_path + join_dict(file_name_dict) + file_suffix


def parse_file_path(file_path):
    r"""
    Parse a file path created by create_file_path and return the result as a dictionary.

    This function is the complement to create_file_path.

    Description of argument(s):
    file_path                       The file_path.

    Example use:
    gp.print_var(boot_results_file_path)
    file_path_data = parse_file_path(boot_results_file_path)
    gp.print_var(file_path_data)

    Program output.

    boot_results_file_path:
    /tmp/pgm_name.obmc_boot_test:openbmc_nickname.beye6:master_pid.2039:boot_results
    file_path_data:
      file_path_data[dir_path]:                       /tmp/
      file_path_data[pgm_name]:                       obmc_boot_test
      file_path_data[openbmc_nickname]:               beye6
      file_path_data[master_pid]:                     2039
      file_path_data[boot_results]:
    """

    try:
        result_dict = collections.OrderedDict()
    except AttributeError:
        result_dict = DotDict()

    dir_path = os.path.dirname(file_path) + os.sep
    file_path = os.path.basename(file_path)

    result_dict['dir_path'] = dir_path

    result_dict.update(split_to_dict(file_path))

    return result_dict


def parse_key_value(string,
                    delim=":",
                    strip=" ",
                    to_lower=1,
                    underscores=1):
    r"""
    Parse a key/value string and return as a key/value tuple.

    This function is useful for parsing a line of program output or data that is in the following form:
    <key or variable name><delimiter><value>

    An example of a key/value string would be as follows:

    Current Limit State: No Active Power Limit

    In the example shown, the delimiter is ":".  The resulting key would be as follows:
    Current Limit State

    Note: If one were to take the default values of to_lower=1 and underscores=1, the resulting key would be
    as follows:
    current_limit_state

    The to_lower and underscores arguments are provided for those who wish to have their key names have the
    look and feel of python variable names.

    The resulting value for the example above would be as follows:
    No Active Power Limit

    Another example:
    name=Mike

    In this case, the delim would be "=", the key is "name" and the value is "Mike".

    Description of argument(s):
    string                          The string to be parsed.
    delim                           The delimiter which separates the key from the value.
    strip                           The characters (if any) to strip from the beginning and end of both the
                                    key and the value.
    to_lower                        Change the key name to lower case.
    underscores                     Change any blanks found in the key name to underscores.
    """

    pair = string.split(delim)

    key = pair[0].strip(strip)
    if len(pair) == 0:
        value = ""
    else:
        value = delim.join(pair[1:]).strip(strip)

    if to_lower:
        key = key.lower()
    if underscores:
        key = re.sub(r" ", "_", key)

    return key, value


def key_value_list_to_dict(list,
                           process_indent=0,
                           **args):
    r"""
    Convert a list containing key/value strings or tuples to a dictionary and return it.

    See docstring of parse_key_value function for details on key/value strings.

    Example usage:

    For the following value of list:

    list:
      list[0]:          Current Limit State: No Active Power Limit
      list[1]:          Exception actions:   Hard Power Off & Log Event to SEL
      list[2]:          Power Limit:         0 Watts
      list[3]:          Correction time:     0 milliseconds
      list[4]:          Sampling period:     0 seconds

    And the following call in python:

    power_limit = key_value_outbuf_to_dict(list)

    The resulting power_limit directory would look like this:

    power_limit:
      [current_limit_state]:        No Active Power Limit
      [exception_actions]:          Hard Power Off & Log Event to SEL
      [power_limit]:                0 Watts
      [correction_time]:            0 milliseconds
      [sampling_period]:            0 seconds

    For the following list:

    headers:
      headers[0]:
        headers[0][0]:           content-length
        headers[0][1]:           559
      headers[1]:
        headers[1][0]:           x-xss-protection
        headers[1][1]:           1; mode=block

    And the following call in python:

    headers_dict = key_value_list_to_dict(headers)

    The resulting headers_dict would look like this:

    headers_dict:
      [content-length]:          559
      [x-xss-protection]:        1; mode=block

    Another example containing a sub-list (see process_indent description below):

    Provides Device SDRs      : yes
    Additional Device Support :
        Sensor Device
        SEL Device
        FRU Inventory Device
        Chassis Device

    Note that the 2 qualifications for containing a sub-list are met: 1) 'Additional Device Support' has no
    value and 2) The entries below it are indented.  In this case those entries contain no delimiters (":")
    so they will be processed as a list rather than as a dictionary.  The result would be as follows:

    mc_info:
      mc_info[provides_device_sdrs]:            yes
      mc_info[additional_device_support]:
        mc_info[additional_device_support][0]:  Sensor Device
        mc_info[additional_device_support][1]:  SEL Device
        mc_info[additional_device_support][2]:  FRU Inventory Device
        mc_info[additional_device_support][3]:  Chassis Device

    Description of argument(s):
    list                            A list of key/value strings.  (See docstring of parse_key_value function
                                    for details).
    process_indent                  This indicates that indented sub-dictionaries and sub-lists are to be
                                    processed as such.  An entry may have a sub-dict or sub-list if 1) It has
                                    no value other than blank 2) There are entries below it that are
                                    indented.  Note that process_indent is not allowed for a list of tuples
                                    (vs. a list of key/value strings).
    **args                          Arguments to be interpreted by parse_key_value.  (See docstring of
                                    parse_key_value function for details).
    """

    try:
        result_dict = collections.OrderedDict()
    except AttributeError:
        result_dict = DotDict()

    if not process_indent:
        for entry in list:
            if type(entry) is tuple:
                key, value = entry
            else:
                key, value = parse_key_value(entry, **args)
            result_dict[key] = value
        return result_dict

    # Process list while paying heed to indentation.
    delim = args.get("delim", ":")
    # Initialize "parent_" indentation level variables.
    parent_indent = len(list[0]) - len(list[0].lstrip())
    sub_list = []
    for entry in list:
        key, value = parse_key_value(entry, **args)

        indent = len(entry) - len(entry.lstrip())

        if indent > parent_indent and parent_value == "":
            # This line is indented compared to the parent entry and the parent entry has no value.
            # Append the entry to sub_list for later processing.
            sub_list.append(str(entry))
            continue

        # Process any outstanding sub_list and add it to result_dict[parent_key].
        if len(sub_list) > 0:
            if any(delim in word for word in sub_list):
                # If delim is found anywhere in the sub_list, we'll process as a sub-dictionary.
                result_dict[parent_key] = key_value_list_to_dict(sub_list,
                                                                 **args)
            else:
                result_dict[parent_key] = map(str.strip, sub_list)
            del sub_list[:]

        result_dict[key] = value

        parent_key = key
        parent_value = value
        parent_indent = indent

    # Any outstanding sub_list to be processed?
    if len(sub_list) > 0:
        if any(delim in word for word in sub_list):
            # If delim is found anywhere in the sub_list, we'll process as a sub-dictionary.
            result_dict[parent_key] = key_value_list_to_dict(sub_list, **args)
        else:
            result_dict[parent_key] = map(str.strip, sub_list)

    return result_dict


def key_value_outbuf_to_dict(out_buf,
                             **args):
    r"""
    Convert a buffer with a key/value string on each line to a dictionary and return it.

    Each line in the out_buf should end with a \n.

    See docstring of parse_key_value function for details on key/value strings.

    Example usage:

    For the following value of out_buf:

    Current Limit State: No Active Power Limit
    Exception actions:   Hard Power Off & Log Event to SEL
    Power Limit:         0 Watts
    Correction time:     0 milliseconds
    Sampling period:     0 seconds

    And the following call in python:

    power_limit = key_value_outbuf_to_dict(out_buf)

    The resulting power_limit directory would look like this:

    power_limit:
      [current_limit_state]:        No Active Power Limit
      [exception_actions]:          Hard Power Off & Log Event to SEL
      [power_limit]:                0 Watts
      [correction_time]:            0 milliseconds
      [sampling_period]:            0 seconds

    Description of argument(s):
    out_buf                         A buffer with a key/value string on each line. (See docstring of
                                    parse_key_value function for details).
    **args                          Arguments to be interpreted by parse_key_value.  (See docstring of
                                    parse_key_value function for details).
    """

    # Create key_var_list and remove null entries.
    key_var_list = list(filter(None, out_buf.split("\n")))
    return key_value_list_to_dict(key_var_list, **args)


def create_field_desc_regex(line):

    r"""
    Create a field descriptor regular expression based on the input line and return it.

    This function is designed for use by the list_to_report function (defined below).

    Example:

    Given the following input line:

    --------   ------------ ------------------ ------------------------

    This function will return this regular expression:

    (.{8})   (.{12}) (.{18}) (.{24})

    This means that other report lines interpreted using the regular expression are expected to have:
    - An 8 character field
    - 3 spaces
    - A 12 character field
    - One space
    - An 18 character field
    - One space
    - A 24 character field

    Description of argument(s):
    line                            A line consisting of dashes to represent fields and spaces to delimit
                                    fields.
    """

    # Split the line into a descriptors list.  Example:
    # descriptors:
    #  descriptors[0]:            --------
    #  descriptors[1]:
    #  descriptors[2]:
    #  descriptors[3]:            ------------
    #  descriptors[4]:            ------------------
    #  descriptors[5]:            ------------------------
    descriptors = line.split(" ")

    # Create regexes list.  Example:
    # regexes:
    #  regexes[0]:                (.{8})
    #  regexes[1]:
    #  regexes[2]:
    #  regexes[3]:                (.{12})
    #  regexes[4]:                (.{18})
    #  regexes[5]:                (.{24})
    regexes = []
    for descriptor in descriptors:
        if descriptor == "":
            regexes.append("")
        else:
            regexes.append("(.{" + str(len(descriptor)) + "})")

    # Join the regexes list into a regex string.
    field_desc_regex = ' '.join(regexes)

    return field_desc_regex


def list_to_report(report_list,
                   to_lower=1,
                   field_delim=None):
    r"""
    Convert a list containing report text lines to a report "object" and return it.

    The first entry in report_list must be a header line consisting of column names delimited by white space.
    No column name may contain white space.  The remaining report_list entries should contain tabular data
    which corresponds to the column names.

    A report object is a list where each entry is a dictionary whose keys are the field names from the first
    entry in report_list.

    Example:
    Given the following report_list as input:

    rl:
      rl[0]: Filesystem           1K-blocks      Used Available Use% Mounted on
      rl[1]: dev                     247120         0    247120   0% /dev
      rl[2]: tmpfs                   248408     79792    168616  32% /run

    This function will return a list of dictionaries as shown below:

    df_report:
      df_report[0]:
        [filesystem]:                  dev
        [1k-blocks]:                   247120
        [used]:                        0
        [available]:                   247120
        [use%]:                        0%
        [mounted]:                     /dev
      df_report[1]:
        [filesystem]:                  dev
        [1k-blocks]:                   247120
        [used]:                        0
        [available]:                   247120
        [use%]:                        0%
        [mounted]:                     /dev

    Notice that because "Mounted on" contains a space, "on" would be considered the 7th field.  In this case,
    there is never any data in field 7 so things work out nicely.  A caller could do some pre-processing if
    desired (e.g. change "Mounted on" to "Mounted_on").

    Example 2:

    If the 2nd line of report data is a series of dashes and spaces as in the following example, that line
    will serve to delineate columns.

    The 2nd line of data is like this:
    ID                              status       size               tool,clientid,userid
    -------- ------------ ------------------ ------------------------
    20000001 in progress  0x7D0              ,,

    Description of argument(s):
    report_list                     A list where each entry is one line of output from a report.  The first
                                    entry must be a header line which contains column names.  Column names
                                    may not contain spaces.
    to_lower                        Change the resulting key names to lower case.
    field_delim                     Indicates that there are field delimiters in report_list entries (which
                                    should be removed).
    """

    if len(report_list) <= 1:
        # If we don't have at least a descriptor line and one line of data, return an empty array.
        return []

    if field_delim is not None:
        report_list = [re.sub("\\|", "", line) for line in report_list]

    header_line = report_list[0]
    if to_lower:
        header_line = header_line.lower()

    field_desc_regex = ""
    if re.match(r"^-[ -]*$", report_list[1]):
        # We have a field descriptor line (as shown in example 2 above).
        field_desc_regex = create_field_desc_regex(report_list[1])
        field_desc_len = len(report_list[1])
        pad_format_string = "%-" + str(field_desc_len) + "s"
        # The field descriptor line has served its purpose.  Deleting it.
        del report_list[1]

    # Process the header line by creating a list of column names.
    if field_desc_regex == "":
        columns = header_line.split()
    else:
        # Pad the line with spaces on the right to facilitate processing with field_desc_regex.
        header_line = pad_format_string % header_line
        columns = list(map(str.strip,
                           re.findall(field_desc_regex, header_line)[0]))

    report_obj = []
    for report_line in report_list[1:]:
        if field_desc_regex == "":
            line = report_line.split()
        else:
            # Pad the line with spaces on the right to facilitate processing with field_desc_regex.
            report_line = pad_format_string % report_line
            line = list(map(str.strip,
                            re.findall(field_desc_regex, report_line)[0]))
        try:
            line_dict = collections.OrderedDict(zip(columns, line))
        except AttributeError:
            line_dict = DotDict(zip(columns, line))
        report_obj.append(line_dict)

    return report_obj


def outbuf_to_report(out_buf,
                     **args):
    r"""
    Convert a text buffer containing report lines to a report "object" and return it.

    Refer to list_to_report (above) for more details.

    Example:

    Given the following out_buf:

    Filesystem                      1K-blocks      Used Available Use% Mounted on
    dev                             247120         0    247120   0% /dev
    tmpfs                           248408     79792    168616  32% /run

    This function will return a list of dictionaries as shown below:

    df_report:
      df_report[0]:
        [filesystem]:                  dev
        [1k-blocks]:                   247120
        [used]:                        0
        [available]:                   247120
        [use%]:                        0%
        [mounted]:                     /dev
      df_report[1]:
        [filesystem]:                  dev
        [1k-blocks]:                   247120
        [used]:                        0
        [available]:                   247120
        [use%]:                        0%
        [mounted]:                     /dev

    Other possible uses:
    - Process the output of a ps command.
    - Process the output of an ls command (the caller would need to supply column names)

    Description of argument(s):
    out_buf                         A text report.  The first line must be a header line which contains
                                    column names.  Column names may not contain spaces.
    **args                          Arguments to be interpreted by list_to_report.  (See docstring of
                                    list_to_report function for details).
    """

    report_list = list(filter(None, out_buf.split("\n")))
    return list_to_report(report_list, **args)


def nested_get(key_name, structure):
    r"""
    Return a list of all values from the nested structure that have the given key name.

    Example:

    Given a dictionary structure named "personnel" with the following contents:

    personnel:
      [manager]:
        [last_name]:             Doe
        [first_name]:            John
      [accountant]:
        [last_name]:             Smith
        [first_name]:            Will

    The following code...

    last_names = nested_get('last_name', personnel)
    print_var(last_names)

    Would result in the following data returned:

    last_names:
      last_names[0]:             Doe
      last_names[1]:             Smith

    Description of argument(s):
    key_name                        The key name (e.g. 'last_name').
    structure                       Any nested combination of lists or dictionaries (e.g. a dictionary, a
                                    dictionary of dictionaries, a list of dictionaries, etc.).  This function
                                    will locate the given key at any level within the structure and include
                                    its value in the returned list.
    """

    result = []
    if type(structure) is list:
        for entry in structure:
            result += nested_get(key_name, entry)
        return result
    elif gp.is_dict(structure):
        for key, value in structure.items():
            result += nested_get(key_name, value)
            if key == key_name:
                result.append(value)

    return result


def match_struct(structure, match_dict, regex=False):
    r"""
    Return True or False to indicate whether the structure matches the match dictionary.

    Example:

    Given a dictionary structure named "personnel" with the following contents:

    personnel:
      [manager]:
        [last_name]:             Doe
        [first_name]:            John
      [accountant]:
        [last_name]:             Smith
        [first_name]:            Will

    The following call would return True.

    match_struct(personnel, {'last_name': '^Doe$'}, regex=True)

    Whereas the following call would return False.

    match_struct(personnel, {'last_name': 'Johnson'}, regex=True)

    Description of argument(s):
    structure                       Any nested combination of lists or dictionaries.  See the prolog of
                                    get_nested() for details.
    match_dict                      Each key/value pair in match_dict must exist somewhere in the structure
                                    for the structure to be considered a match.  A match value of None is
                                    considered a special case where the structure would be considered a match
                                    only if the key in question is found nowhere in the structure.
    regex                           Indicates whether the values in the match_dict should be interpreted as
                                    regular expressions.
    """

    # The structure must match for each match_dict entry to be considered a match.  Therefore, any failure
    # to match is grounds for returning False.
    for match_key, match_value in match_dict.items():
        struct_key_values = nested_get(match_key, structure)
        if match_value is None:
            # Handle this as special case.
            if len(struct_key_values) != 0:
                return False
        else:
            if len(struct_key_values) == 0:
                return False
            if regex:
                matches = [x for x in struct_key_values
                           if re.search(match_value, str(x))]
                if not matches:
                    return False
            elif match_value not in struct_key_values:
                return False

    return True


def filter_struct(structure, filter_dict, regex=False, invert=False):
    r"""
    Filter the structure by removing any entries that do NOT contain the keys/values specified in filter_dict
    and return the result.

    The selection process is directed only at the first-level entries of the structure.

    Example:

    Given a dictionary named "properties" that has the following structure:

    properties:
      [/redfish/v1/Systems/system/Processors]:
        [Members]:
          [0]:
            [@odata.id]:                              /redfish/v1/Systems/system/Processors/cpu0
          [1]:
            [@odata.id]:                              /redfish/v1/Systems/system/Processors/cpu1
      [/redfish/v1/Systems/system/Processors/cpu0]:
        [Status]:
          [State]:                                    Enabled
          [Health]:                                   OK
      [/redfish/v1/Systems/system/Processors/cpu1]:
        [Status]:
          [State]:                                    Enabled
          [Health]:                                   Bad

    The following call:

    properties = filter_struct(properties, "[('Health', 'OK')]")

    Would return a new properties dictionary that looks like this:

    properties:
      [/redfish/v1/Systems/system/Processors/cpu0]:
        [Status]:
          [State]:                                    Enabled
          [Health]:                                   OK

    Note that the first item in the original properties directory had no key anywhere in the structure named
    "Health".  Therefore, that item failed to make the cut.  The next item did have a key named "Health"
    whose value was "OK" so it was included in the new structure.  The third item had a key named "Health"
    but its value was not "OK" so it also failed to make the cut.

    Description of argument(s):
    structure                       Any nested combination of lists or dictionaries.  See the prolog of
                                    get_nested() for details.
    filter_dict                     For each key/value pair in filter_dict, each entry in structure must
                                    contain the same key/value pair at some level.  A filter_dict value of
                                    None is treated as a special case.  Taking the example shown above,
                                    [('State', None)] would mean that the result should only contain records
                                    that have no State key at all.
    regex                           Indicates whether the values in the filter_dict should be interpreted as
                                    regular expressions.
    invert                          Invert the results.  Instead of including only matching entries in the
                                    results, include only NON-matching entries in the results.
    """

    # Convert filter_dict from a string containing a python object definition to an actual python object (if
    # warranted).
    filter_dict = fa.source_to_object(filter_dict)

    # Determine whether structure is a list or a dictionary and process accordingly.  The result returned
    # will be of the same type as the structure.
    if type(structure) is list:
        result = []
        for element in structure:
            if match_struct(element, filter_dict, regex) != invert:
                result.append(element)
    else:
        try:
            result = collections.OrderedDict()
        except AttributeError:
            result = DotDict()
        for struct_key, struct_value in structure.items():
            if match_struct(struct_value, filter_dict, regex) != invert:
                result[struct_key] = struct_value

    return result