lib/var_funcs.py

#!/usr/bin/env python

r"""
Define variable manipulation functions.
"""

import os
import re

try:
    from robot.utils import DotDict
except ImportError:
    pass

import collections

import gen_print as gp
import gen_misc as gm
import func_args as fa


def create_var_dict(*args):
    r"""
    Create a dictionary whose keys/values are the arg names/arg values passed
    to it and return it to the caller.

    Note: The resulting dictionary will be ordered.

    Description of argument(s):
    *args  An unlimited number of arguments to be processed.

    Example use:

    first_name = 'Steve'
    last_name = 'Smith'
    var_dict = create_var_dict(first_name, last_name)

    gp.print_var(var_dict)

    The print-out of the resulting var dictionary is:
    var_dict:
      var_dict[first_name]:                           Steve
      var_dict[last_name]:                            Smith
    """

    try:
        result_dict = collections.OrderedDict()
    except AttributeError:
        result_dict = DotDict()

    arg_num = 1
    for arg in args:
        arg_name = gp.get_arg_name(None, arg_num, stack_frame_ix=2)
        result_dict[arg_name] = arg
        arg_num += 1

    return result_dict


default_record_delim = ':'
default_key_val_delim = '.'


def join_dict(dict,
              record_delim=default_record_delim,
              key_val_delim=default_key_val_delim):
    r"""
    Join a dictionary's keys and values into a string and return the string.

    Description of argument(s):
    dict                            The dictionary whose keys and values are
                                    to be joined.
    record_delim                    The delimiter to be used to separate
                                    dictionary pairs in the resulting string.
    key_val_delim                   The delimiter to be used to separate keys
                                    from values in the resulting string.

    Example use:

    gp.print_var(var_dict)
    str1 = join_dict(var_dict)
    gp.print_var(str1)

    Program output.
    var_dict:
      var_dict[first_name]:                           Steve
      var_dict[last_name]:                            Smith
    str1:
    first_name.Steve:last_name.Smith
    """

    format_str = '%s' + key_val_delim + '%s'
    return record_delim.join([format_str % (key, value) for (key, value) in
                              dict.items()])


def split_to_dict(string,
                  record_delim=default_record_delim,
                  key_val_delim=default_key_val_delim):
    r"""
    Split a string into a dictionary and return it.

    This function is the complement to join_dict.

    Description of argument(s):
    string                          The string to be split into a dictionary.
                                    The string must have the proper delimiters
                                    in it.  A string created by join_dict
                                    would qualify.
    record_delim                    The delimiter to be used to separate
                                    dictionary pairs in the input string.
    key_val_delim                   The delimiter to be used to separate
                                    keys/values in the input string.

    Example use:

    gp.print_var(str1)
    new_dict = split_to_dict(str1)
    gp.print_var(new_dict)


    Program output.
    str1:
    first_name.Steve:last_name.Smith
    new_dict:
      new_dict[first_name]:                           Steve
      new_dict[last_name]:                            Smith
    """

    try:
        result_dict = collections.OrderedDict()
    except AttributeError:
        result_dict = DotDict()

    raw_keys_values = string.split(record_delim)
    for key_value in raw_keys_values:
        key_value_list = key_value.split(key_val_delim)
        try:
            result_dict[key_value_list[0]] = key_value_list[1]
        except IndexError:
            result_dict[key_value_list[0]] = ""

    return result_dict


def create_file_path(file_name_dict,
                     dir_path="/tmp/",
                     file_suffix=""):
    r"""
    Create a file path using the given parameters and return it.

    Description of argument(s):
    file_name_dict                  A dictionary with keys/values which are to
                                    appear as part of the file name.
    dir_path                        The dir_path that is to appear as part of
                                    the file name.
    file_suffix                     A suffix to be included as part of the
                                    file name.
    """

    dir_path = gm.add_trailing_slash(dir_path)
    return dir_path + join_dict(file_name_dict) + file_suffix


def parse_file_path(file_path):
    r"""
    Parse a file path created by create_file_path and return the result as a
    dictionary.

    This function is the complement to create_file_path.

    Description of argument(s):
    file_path                       The file_path.

    Example use:
    gp.print_var(boot_results_file_path)
    file_path_data = parse_file_path(boot_results_file_path)
    gp.print_var(file_path_data)

    Program output.

    boot_results_file_path:
    /tmp/pgm_name.obmc_boot_test:openbmc_nickname.beye6:master_pid.2039:boot_re
    sults
    file_path_data:
      file_path_data[dir_path]:                       /tmp/
      file_path_data[pgm_name]:                       obmc_boot_test
      file_path_data[openbmc_nickname]:               beye6
      file_path_data[master_pid]:                     2039
      file_path_data[boot_results]:
    """

    try:
        result_dict = collections.OrderedDict()
    except AttributeError:
        result_dict = DotDict()

    dir_path = os.path.dirname(file_path) + os.sep
    file_path = os.path.basename(file_path)

    result_dict['dir_path'] = dir_path

    result_dict.update(split_to_dict(file_path))

    return result_dict


def parse_key_value(string,
                    delim=":",
                    strip=" ",
                    to_lower=1,
                    underscores=1):
    r"""
    Parse a key/value string and return as a key/value tuple.

    This function is useful for parsing a line of program output or data that
    is in the following form:
    <key or variable name><delimiter><value>

    An example of a key/value string would be as follows:

    Current Limit State: No Active Power Limit

    In the example shown, the delimiter is ":".  The resulting key would be as
    follows:
    Current Limit State

    Note: If one were to take the default values of to_lower=1 and
    underscores=1, the resulting key would be as follows:
    current_limit_state

    The to_lower and underscores arguments are provided for those who wish to
    have their key names have the look and feel of python variable names.

    The resulting value for the example above would be as follows:
    No Active Power Limit

    Another example:
    name=Mike

    In this case, the delim would be "=", the key is "name" and the value is
    "Mike".

    Description of argument(s):
    string                          The string to be parsed.
    delim                           The delimiter which separates the key from
                                    the value.
    strip                           The characters (if any) to strip from the
                                    beginning and end of both the key and the
                                    value.
    to_lower                        Change the key name to lower case.
    underscores                     Change any blanks found in the key name to
                                    underscores.
    """

    pair = string.split(delim)

    key = pair[0].strip(strip)
    if len(pair) == 0:
        value = ""
    else:
        value = delim.join(pair[1:]).strip(strip)

    if to_lower:
        key = key.lower()
    if underscores:
        key = re.sub(r" ", "_", key)

    return key, value


def key_value_list_to_dict(list,
                           process_indent=0,
                           **args):
    r"""
    Convert a list containing key/value strings or tuples to a dictionary and
    return it.

    See docstring of parse_key_value function for details on key/value strings.

    Example usage:

    For the following value of list:

    list:
      list[0]:          Current Limit State: No Active Power Limit
      list[1]:          Exception actions:   Hard Power Off & Log Event to SEL
      list[2]:          Power Limit:         0 Watts
      list[3]:          Correction time:     0 milliseconds
      list[4]:          Sampling period:     0 seconds

    And the following call in python:

    power_limit = key_value_outbuf_to_dict(list)

    The resulting power_limit directory would look like this:

    power_limit:
      [current_limit_state]:        No Active Power Limit
      [exception_actions]:          Hard Power Off & Log Event to SEL
      [power_limit]:                0 Watts
      [correction_time]:            0 milliseconds
      [sampling_period]:            0 seconds

    For the following list:

    headers:
      headers[0]:
        headers[0][0]:           content-length
        headers[0][1]:           559
      headers[1]:
        headers[1][0]:           x-xss-protection
        headers[1][1]:           1; mode=block

    And the following call in python:

    headers_dict = key_value_list_to_dict(headers)

    The resulting headers_dict would look like this:

    headers_dict:
      [content-length]:          559
      [x-xss-protection]:        1; mode=block

    Another example containing a sub-list (see process_indent description
    below):

    Provides Device SDRs      : yes
    Additional Device Support :
        Sensor Device
        SEL Device
        FRU Inventory Device
        Chassis Device

    Note that the 2 qualifications for containing a sub-list are met: 1)
    'Additional Device Support' has no value and 2) The entries below it are
    indented.  In this case those entries contain no delimiters (":") so they
    will be processed as a list rather than as a dictionary.  The result would
    be as follows:

    mc_info:
      mc_info[provides_device_sdrs]:            yes
      mc_info[additional_device_support]:
        mc_info[additional_device_support][0]:  Sensor Device
        mc_info[additional_device_support][1]:  SEL Device
        mc_info[additional_device_support][2]:  FRU Inventory Device
        mc_info[additional_device_support][3]:  Chassis Device

    Description of argument(s):
    list                            A list of key/value strings.  (See
                                    docstring of parse_key_value function for
                                    details).
    process_indent                  This indicates that indented
                                    sub-dictionaries and sub-lists are to be
                                    processed as such.  An entry may have a
                                    sub-dict or sub-list if 1) It has no value
                                    other than blank 2) There are entries
                                    below it that are indented.  Note that
                                    process_indent is not allowed for a list
                                    of tuples (vs. a list of key/value
                                    strings).
    **args                          Arguments to be interpreted by
                                    parse_key_value.  (See docstring of
                                    parse_key_value function for details).
    """

    try:
        result_dict = collections.OrderedDict()
    except AttributeError:
        result_dict = DotDict()

    if not process_indent:
        for entry in list:
            if type(entry) is tuple:
                key, value = entry
            else:
                key, value = parse_key_value(entry, **args)
            result_dict[key] = value
        return result_dict

    # Process list while paying heed to indentation.
    delim = args.get("delim", ":")
    # Initialize "parent_" indentation level variables.
    parent_indent = len(list[0]) - len(list[0].lstrip())
    sub_list = []
    for entry in list:
        key, value = parse_key_value(entry, **args)

        indent = len(entry) - len(entry.lstrip())

        if indent > parent_indent and parent_value == "":
            # This line is indented compared to the parent entry and the
            # parent entry has no value.
            # Append the entry to sub_list for later processing.
            sub_list.append(str(entry))
            continue

        # Process any outstanding sub_list and add it to
        # result_dict[parent_key].
        if len(sub_list) > 0:
            if any(delim in word for word in sub_list):
                # If delim is found anywhere in the sub_list, we'll process
                # as a sub-dictionary.
                result_dict[parent_key] = key_value_list_to_dict(sub_list,
                                                                 **args)
            else:
                result_dict[parent_key] = map(str.strip, sub_list)
            del sub_list[:]

        result_dict[key] = value

        parent_key = key
        parent_value = value
        parent_indent = indent

    # Any outstanding sub_list to be processed?
    if len(sub_list) > 0:
        if any(delim in word for word in sub_list):
            # If delim is found anywhere in the sub_list, we'll process as a
            # sub-dictionary.
            result_dict[parent_key] = key_value_list_to_dict(sub_list, **args)
        else:
            result_dict[parent_key] = map(str.strip, sub_list)

    return result_dict


def key_value_outbuf_to_dict(out_buf,
                             **args):
    r"""
    Convert a buffer with a key/value string on each line to a dictionary and
    return it.

    Each line in the out_buf should end with a \n.

    See docstring of parse_key_value function for details on key/value strings.

    Example usage:

    For the following value of out_buf:

    Current Limit State: No Active Power Limit
    Exception actions:   Hard Power Off & Log Event to SEL
    Power Limit:         0 Watts
    Correction time:     0 milliseconds
    Sampling period:     0 seconds

    And the following call in python:

    power_limit = key_value_outbuf_to_dict(out_buf)

    The resulting power_limit directory would look like this:

    power_limit:
      [current_limit_state]:        No Active Power Limit
      [exception_actions]:          Hard Power Off & Log Event to SEL
      [power_limit]:                0 Watts
      [correction_time]:            0 milliseconds
      [sampling_period]:            0 seconds

    Description of argument(s):
    out_buf                         A buffer with a key/value string on each
                                    line. (See docstring of parse_key_value
                                    function for details).
    **args                          Arguments to be interpreted by
                                    parse_key_value.  (See docstring of
                                    parse_key_value function for details).
    """

    # Create key_var_list and remove null entries.
    key_var_list = list(filter(None, out_buf.split("\n")))
    return key_value_list_to_dict(key_var_list, **args)


def create_field_desc_regex(line):

    r"""
    Create a field descriptor regular expression based on the input line and
    return it.

    This function is designed for use by the list_to_report function (defined
    below).

    Example:

    Given the following input line:

    --------   ------------ ------------------ ------------------------

    This function will return this regular expression:

    (.{8})   (.{12}) (.{18}) (.{24})

    This means that other report lines interpreted using the regular
    expression are expected to have:
    - An 8 character field
    - 3 spaces
    - A 12 character field
    - One space
    - An 18 character field
    - One space
    - A 24 character field

    Description of argument(s):
    line                            A line consisting of dashes to represent
                                    fields and spaces to delimit fields.
    """

    # Split the line into a descriptors list.  Example:
    # descriptors:
    #  descriptors[0]:            --------
    #  descriptors[1]:
    #  descriptors[2]:
    #  descriptors[3]:            ------------
    #  descriptors[4]:            ------------------
    #  descriptors[5]:            ------------------------
    descriptors = line.split(" ")

    # Create regexes list.  Example:
    # regexes:
    #  regexes[0]:                (.{8})
    #  regexes[1]:
    #  regexes[2]:
    #  regexes[3]:                (.{12})
    #  regexes[4]:                (.{18})
    #  regexes[5]:                (.{24})
    regexes = []
    for descriptor in descriptors:
        if descriptor == "":
            regexes.append("")
        else:
            regexes.append("(.{" + str(len(descriptor)) + "})")

    # Join the regexes list into a regex string.
    field_desc_regex = ' '.join(regexes)

    return field_desc_regex


def list_to_report(report_list,
                   to_lower=1,
                   field_delim=None):
    r"""
    Convert a list containing report text lines to a report "object" and
    return it.

    The first entry in report_list must be a header line consisting of column
    names delimited by white space.  No column name may contain white space.
    The remaining report_list entries should contain tabular data which
    corresponds to the column names.

    A report object is a list where each entry is a dictionary whose keys are
    the field names from the first entry in report_list.

    Example:
    Given the following report_list as input:

    rl:
      rl[0]: Filesystem           1K-blocks      Used Available Use% Mounted on
      rl[1]: dev                     247120         0    247120   0% /dev
      rl[2]: tmpfs                   248408     79792    168616  32% /run

    This function will return a list of dictionaries as shown below:

    df_report:
      df_report[0]:
        [filesystem]:                  dev
        [1k-blocks]:                   247120
        [used]:                        0
        [available]:                   247120
        [use%]:                        0%
        [mounted]:                     /dev
      df_report[1]:
        [filesystem]:                  dev
        [1k-blocks]:                   247120
        [used]:                        0
        [available]:                   247120
        [use%]:                        0%
        [mounted]:                     /dev

    Notice that because "Mounted on" contains a space, "on" would be
    considered the 7th field.  In this case, there is never any data in field
    7 so things work out nicely.  A caller could do some pre-processing if
    desired (e.g. change "Mounted on" to "Mounted_on").

    Example 2:

    If the 2nd line of report data is a series of dashes and spaces as in the
    following example, that line will serve to delineate columns.

    The 2nd line of data is like this:
    ID                              status       size
                                    tool,clientid,userid
    -------- ------------ ------------------ ------------------------
    20000001 in progress  0x7D0              ,,

    Description of argument(s):
    report_list                     A list where each entry is one line of
                                    output from a report.  The first entry
                                    must be a header line which contains
                                    column names.  Column names may not
                                    contain spaces.
    to_lower                        Change the resulting key names to lower
                                    case.
    field_delim                     Indicates that there are field delimiters
                                    in report_list entries (which should be
                                    removed).
    """

    if len(report_list) <= 1:
        # If we don't have at least a descriptor line and one line of data,
        # return an empty array.
        return []

    if field_delim is not None:
        report_list = [re.sub("\\|", "", line) for line in report_list]

    header_line = report_list[0]
    if to_lower:
        header_line = header_line.lower()

    field_desc_regex = ""
    if re.match(r"^-[ -]*$", report_list[1]):
        # We have a field descriptor line (as shown in example 2 above).
        field_desc_regex = create_field_desc_regex(report_list[1])
        field_desc_len = len(report_list[1])
        pad_format_string = "%-" + str(field_desc_len) + "s"
        # The field descriptor line has served its purpose.  Deleting it.
        del report_list[1]

    # Process the header line by creating a list of column names.
    if field_desc_regex == "":
        columns = header_line.split()
    else:
        # Pad the line with spaces on the right to facilitate processing with
        # field_desc_regex.
        header_line = pad_format_string % header_line
        columns = list(map(str.strip,
                           re.findall(field_desc_regex, header_line)[0]))

    report_obj = []
    for report_line in report_list[1:]:
        if field_desc_regex == "":
            line = report_line.split()
        else:
            # Pad the line with spaces on the right to facilitate processing
            # with field_desc_regex.
            report_line = pad_format_string % report_line
            line = list(map(str.strip,
                            re.findall(field_desc_regex, report_line)[0]))
        try:
            line_dict = collections.OrderedDict(zip(columns, line))
        except AttributeError:
            line_dict = DotDict(zip(columns, line))
        report_obj.append(line_dict)

    return report_obj


def outbuf_to_report(out_buf,
                     **args):
    r"""
    Convert a text buffer containing report lines to a report "object" and
    return it.

    Refer to list_to_report (above) for more details.

    Example:

    Given the following out_buf:

    Filesystem                      1K-blocks      Used Available Use% Mounted
                                    on
    dev                             247120         0    247120   0% /dev
    tmpfs                           248408     79792    168616  32% /run

    This function will return a list of dictionaries as shown below:

    df_report:
      df_report[0]:
        [filesystem]:                  dev
        [1k-blocks]:                   247120
        [used]:                        0
        [available]:                   247120
        [use%]:                        0%
        [mounted]:                     /dev
      df_report[1]:
        [filesystem]:                  dev
        [1k-blocks]:                   247120
        [used]:                        0
        [available]:                   247120
        [use%]:                        0%
        [mounted]:                     /dev

    Other possible uses:
    - Process the output of a ps command.
    - Process the output of an ls command (the caller would need to supply
      column names)

    Description of argument(s):
    out_buf                         A text report.  The first line must be a
                                    header line which contains column names.
                                    Column names may not contain spaces.
    **args                          Arguments to be interpreted by
                                    list_to_report.  (See docstring of
                                    list_to_report function for details).
    """

    report_list = list(filter(None, out_buf.split("\n")))
    return list_to_report(report_list, **args)


def nested_get(key_name, structure):
    r"""
    Return a list of all values from the nested structure that have the given
    key name.

    Example:

    Given a dictionary structure named "personnel" with the following contents:

    personnel:
      [manager]:
        [last_name]:             Doe
        [first_name]:            John
      [accountant]:
        [last_name]:             Smith
        [first_name]:            Will

    The following code...

    last_names = nested_get('last_name', personnel)
    print_var(last_names)

    Would result in the following data returned:

    last_names:
      last_names[0]:             Doe
      last_names[1]:             Smith

    Description of argument(s):
    key_name                        The key name (e.g. 'last_name').
    structure                       Any nested combination of lists or
                                    dictionaries (e.g. a dictionary, a
                                    dictionary of dictionaries, a list of
                                    dictionaries, etc.).  This function will
                                    locate the given key at any level within
                                    the structure and include its value in the
                                    returned list.
    """

    result = []
    if type(structure) is list:
        for entry in structure:
            result += nested_get(key_name, entry)
        return result
    elif gp.is_dict(structure):
        for key, value in structure.items():
            result += nested_get(key_name, value)
            if key == key_name:
                result.append(value)

    return result


def match_struct(structure, match_dict, regex=False):
    r"""
    Return True or False to indicate whether the structure matches the match
    dictionary.

    Example:

    Given a dictionary structure named "personnel" with the following contents:

    personnel:
      [manager]:
        [last_name]:             Doe
        [first_name]:            John
      [accountant]:
        [last_name]:             Smith
        [first_name]:            Will

    The following call would return True.

    match_struct(personnel, {'last_name': '^Doe$'}, regex=True)

    Whereas the following call would return False.

    match_struct(personnel, {'last_name': 'Johnson'}, regex=True)

    Description of argument(s):
    structure                       Any nested combination of lists or
                                    dictionaries.  See the prolog of
                                    get_nested() for details.
    match_dict                      Each key/value pair in match_dict must
                                    exist somewhere in the structure for the
                                    structure to be considered a match.  A
                                    match value of None is considered a
                                    special case where the structure would be
                                    considered a match only if the key in
                                    question is found nowhere in the structure.
    regex                           Indicates whether the values in the
                                    match_dict should be interpreted as
                                    regular expressions.
    """

    # The structure must match for each match_dict entry to be considered a
    # match.  Therefore, any failure to match is grounds for returning False.
    for match_key, match_value in match_dict.items():
        struct_key_values = nested_get(match_key, structure)
        if match_value is None:
            # Handle this as special case.
            if len(struct_key_values) != 0:
                return False
        else:
            if len(struct_key_values) == 0:
                return False
            if regex:
                matches = [x for x in struct_key_values
                           if re.search(match_value, str(x))]
                if not matches:
                    return False
            elif match_value not in struct_key_values:
                return False

    return True


def filter_struct(structure, filter_dict, regex=False, invert=False):
    r"""
    Filter the structure by removing any entries that do NOT contain the
    keys/values specified in filter_dict and return the result.

    The selection process is directed only at the first-level entries of the
    structure.

    Example:

    Given a dictionary named "properties" that has the following structure:

    properties:
      [/redfish/v1/Systems/system/Processors]:
        [Members]:
          [0]:
            [@odata.id]:
            /redfish/v1/Systems/system/Processors/cpu0
          [1]:
            [@odata.id]:
            /redfish/v1/Systems/system/Processors/cpu1
      [/redfish/v1/Systems/system/Processors/cpu0]:
        [Status]:
          [State]:                                    Enabled
          [Health]:                                   OK
      [/redfish/v1/Systems/system/Processors/cpu1]:
        [Status]:
          [State]:                                    Enabled
          [Health]:                                   Bad

    The following call:

    properties = filter_struct(properties, "[('Health', 'OK')]")

    Would return a new properties dictionary that looks like this:

    properties:
      [/redfish/v1/Systems/system/Processors/cpu0]:
        [Status]:
          [State]:                                    Enabled
          [Health]:                                   OK

    Note that the first item in the original properties directory had no key
    anywhere in the structure named "Health".  Therefore, that item failed to
    make the cut.  The next item did have a key named "Health" whose value was
    "OK" so it was included in the new structure.  The third item had a key
    named "Health" but its value was not "OK" so it also failed to make the
    cut.

    Description of argument(s):
    structure                       Any nested combination of lists or
                                    dictionaries.  See the prolog of
                                    get_nested() for details.
    filter_dict                     For each key/value pair in filter_dict,
                                    each entry in structure must contain the
                                    same key/value pair at some level.  A
                                    filter_dict value of None is treated as a
                                    special case.  Taking the example shown
                                    above, [('State', None)] would mean that
                                    the result should only contain records
                                    that have no State key at all.
    regex                           Indicates whether the values in the
                                    filter_dict should be interpreted as
                                    regular expressions.
    invert                          Invert the results.  Instead of including
                                    only matching entries in the results,
                                    include only NON-matching entries in the
                                    results.
    """

    # Convert filter_dict from a string containing a python object definition
    # to an actual python object (if warranted).
    filter_dict = fa.source_to_object(filter_dict)

    # Determine whether structure is a list or a dictionary and process
    # accordingly.  The result returned will be of the same type as the
    # structure.
    if type(structure) is list:
        result = []
        for element in structure:
            if match_struct(element, filter_dict, regex) != invert:
                result.append(element)
    else:
        try:
            result = collections.OrderedDict()
        except AttributeError:
            result = DotDict()
        for struct_key, struct_value in structure.items():
            if match_struct(struct_value, filter_dict, regex) != invert:
                result[struct_key] = struct_value

    return result