Add JSON inventory to hardbootme test

In utils_keywords.py: Minor change in comment block.
In htx_hardbootme_test:  Fail message to comply with standards.

test results:
/gsa/ausgsa/home/s/s/ssombar/bmctest/openbmc-test-automation
  file  out*xml

solves  openbmc-test-automation/issues/626
Change-Id: I0882ee4994230d743b22e40d21132182fe8bc95d
Signed-off-by: Steven Sombar <ssombar@us.ibm.com>
diff --git a/syslib/utils_keywords.py b/syslib/utils_keywords.py
index e6a4e0e..12d6c87 100644
--- a/syslib/utils_keywords.py
+++ b/syslib/utils_keywords.py
@@ -3,17 +3,237 @@
 r"""
 This module contains keyword functions to supplement robot's built in
 functions and use in test where generic robot keywords don't support.
-
 """
+
+try:
+    from robot.libraries.BuiltIn import BuiltIn
+    from robot.libraries import DateTime
+except ImportError:
+    pass
 import time
-from robot.libraries.BuiltIn import BuiltIn
-from robot.libraries import DateTime
-import re
+import os
+import difflib
+
+
+##########################################################################
+def json_inv_file_diff_check(file1_path,
+                             file2_path,
+                             diff_file_path,
+                             skip_dictionary):
+    r"""
+    Compare the contents of two files which contain inventory data in
+    JSON format.  The comparison is similar to the unix 'diff' command but
+    the output lists the hardware subsystem (category) where differences
+    are found, and some differences are selectively ignored.  The items
+    ignored are defined in the skip_dictionary dictionary.
+
+    Description of arguments:
+    file1_path       File containing JSON formatted data.
+    file2_path       File to compare to file1 to.
+    diff_file_path   File which will contain the resulting difference report.
+    skip_dictionary  Dictionary which defines what inventory items
+                     to ignore if there are differences in inventory
+                     files -- some differences are expected or
+                     immaterial.  Specify items in this dictionary
+                     if there are inventory items that should be
+                     ignored whenever a comparison between inventory
+                     files is done.  For example, assigned processor
+                     speed routinely varies depending upon the
+                     needs of OCC/tmgt/ondemand governor.
+                     Back-to-back inventory runs may show
+                     processor speed differences even if nothing
+                     else was run between them.
+                     Each dictionary entry is of the form
+                     category:leafname where category is a JSON
+                     hardware category such as "processor", "memory",
+                     "disk", "display", "network", etc.,
+                     and leafname is a leaf node (atribute name)
+                     within that category.
+                     For example: {'processor':'size'}, or
+                     {'processor':'size','memory':'claimed','display':'id'}.
+
+    Sample difference report:
+    Difference at line 102  (in section "memory":)
+     102 -   "slot": "UOPWR.BAR.1315ACA-DIMM0",
+     102 +   "slot": "0"
+    Difference at line 126  (in section "processor":)
+     126 -   "size": 2151000000,    +++ NOTE! This is an ignore item
+     126 +   "size": 2201000000,    +++ NOTE! This is an ignore item
+
+    Returns:
+    0 if both files contain the same information or they differ only in
+      items specified as those to ignore.
+    2 if FILES_DO_NOT_MATCH.
+    3 if INPUT_FILE_DOES_NOT_EXIST.
+    4 if IO_EXCEPTION_READING_FILE.
+    5 if IO_EXCEPTION_WRITING_FILE.
+    """
+
+    FILES_MATCH = 0
+    FILES_DO_NOT_MATCH = 2
+    INPUT_FILE_DOES_NOT_EXIST = 3
+    IO_EXCEPTION_READING_FILE = 4
+    IO_EXCEPTION_WRITING_FILE = 5
+
+    # Hardware categories which are reported in the JSON inventory files.
+    hardware_categories = ['\"processor\":', '\"memory\":', '\"disk\":',
+                           '\"I/O\":', '\"display\":', '\"generic\":',
+                           '\"network\":', '\"communication\":',
+                           '\"printer\":', '\"input\":', '\"multimedia\":',
+                           '\"tape\":']
+
+    # The minimum size in bytes a JSON file must be.
+    min_json_byte_size = 16
+
+    now = time.strftime("At %Y-%m-%d %H:%M:%S")
+
+    if (not os.path.exists(file1_path) or (not os.path.exists(file2_path))):
+        return INPUT_FILE_DOES_NOT_EXIST
+    try:
+        with open(file1_path, 'r') as file:
+            initial = file.readlines()
+        with open(file2_path, 'r') as file:
+            final = file.readlines()
+    except IOError:
+        file.close()
+        return IO_EXCEPTION_READING_FILE
+    except ValueError:
+        file.close()
+        return INPUT_FILE_MALFORMED
+    else:
+        file.close()
+
+    # Must have more than a trivial number of bytes.
+    if len(initial) <= min_json_byte_size:
+        return INPUT_FILE_MALFORMED
+
+    if (initial == final):
+        try:
+            file = open(diff_file_path, 'w')
+        except IOError:
+            file.close()
+        line_to_print = now + " found no difference between file " + \
+            file1_path + " and " + \
+            file2_path + "\n"
+        file.write(line_to_print)
+        file.close()
+        return FILES_MATCH
+
+    # Find the differences and write difference report to diff_file_path file.
+    try:
+        file = open(diff_file_path, 'w')
+    except IOError:
+        file.close()
+        return IO_EXCEPTION_WRITING_FILE
+
+    line_to_print = now + " compared files " + \
+        file1_path + " and " + \
+        file2_path + "\n"
+    file.write(line_to_print)
+
+    diff = difflib.ndiff(initial, final)
+    # The diff array contains all lines that match in the
+    # initial and final arrays, and also all lines that differ.
+    # The first two characters of each line
+    # are prefixed with two letters, defined as:
+    # '- ' This line is unique to initial
+    # '+ ' This line is line unique to final
+    # '  ' This line is common to both, and
+    # '? ' This line indicates approximate differences.
+    # For example,  comparing two three-line files:
+    #  This line is in both initial and final.
+    #  This line is too but the next line is different in each file.
+    # -                     "size": 2101000000,
+    # ?                              - ^
+    # +                     "size": 2002000000,
+    # ?                               ^^
+
+    print_header_flag = False
+    category = ""
+    row_num = 1
+    item_we_cannot_ignore = False
+
+    for my_line in diff:
+        diff_item = my_line.strip('\n')
+        # If it's a Category, such as processor or memory,
+        # save it.  We will print it out later.
+        # Most lines do not explicitly contain a category.  As such the
+        # category for that line is found in a previous line, which
+        # will be the category we last found.
+        for hdw_cat in hardware_categories:
+            if (hdw_cat in diff_item):
+                # If we don't have a match we will reuse
+                # the prvious category.
+                category = hdw_cat
+        # Lines beginning with minus or plus or q-mark are
+        # true difference items.
+        # We want to look at those in more detail.
+        if diff_item.startswith('? '):
+            # we can ignore these
+            continue
+        if (diff_item.startswith('- ') or diff_item.startswith('+ ')):
+            # If we have not printed the header line for this
+            # difference, print it now.
+            if print_header_flag is False:
+                line_to_print = "Difference at line " + \
+                 str(row_num) + "  (in section " + \
+                 category + ")\n"
+                file.write(line_to_print)
+            # If this is in the ignore dictionary, we'll print
+            # it but also add text that it is an ignore item.
+            skipitem = False
+            for key, value in skip_dictionary.iteritems():
+                if ((key in category.lower().strip()) and
+                   (value in diff_item.lower().strip())):
+                    line_to_print = "  " + \
+                        str(row_num) + " " + diff_item + \
+                        "    +++ NOTE! This difference is in" + \
+                        " the inventory ignore list and" + \
+                        " can be ignored. +++\n"
+                    # set flag indicating this item is a skip item
+                    skipitem = True
+                    break
+            if skipitem is False:
+                # Its not a skip item, that is,
+                # this is not on the ignore list.
+                # Print the item and set the item_we_canot_ignore flag
+                # indicating we have an item not on the ignore list.  The
+                # flag will determine the return code we
+                # pass back to the user at the end.
+                item_we_cannot_ignore = True
+                line_to_print = "  " + \
+                    str(row_num) + " " + diff_item + "\n"
+            file.write(line_to_print)
+            print_header_flag = True
+
+        else:
+            # Adjust row numbering as a difference is only one line
+            # but it takes several lines in the diff file.
+            if print_header_flag is True:
+                row_num = row_num + 1
+                print_header_flag = False
+            row_num = row_num + 1
+
+    # Make sure we end the file.
+    file.write("\n")
+    file.close()
+
+    if item_we_cannot_ignore:
+        # We have at least one diff_item not on the ignore list.
+        return FILES_DO_NOT_MATCH
+    else:
+        # Any differences were on the ignore list.
+        return FILES_MATCH
+###############################################################################
+
 
 ###############################################################################
 
 
-def run_until_keyword_fails(retry, retry_interval, name, *args):
+def run_until_keyword_fails(retry,
+                            retry_interval,
+                            name,
+                            *args):
     r"""
     Execute a robot keyword repeatedly until it either fails or the timeout
     value is exceeded.
diff --git a/syslib/utils_os.robot b/syslib/utils_os.robot
index a222791..0c62124 100755
--- a/syslib/utils_os.robot
+++ b/syslib/utils_os.robot
@@ -1,7 +1,5 @@
 *** Settings ***
-Documentation      Keywords for system related test. This is a subset of the
-...                utils.robot. This resource file keywords  is specifically
-...                define for system test use cases.
+Documentation      Keywords for system test.
 
 Library            ../lib/gen_robot_keyword.py
 Library            OperatingSystem
@@ -15,13 +13,23 @@
 
 *** Variables ***
 
-${htx_log_dir_path}   ${EXECDIR}${/}logs${/}
+${htx_log_dir_path}    ${EXECDIR}${/}logs${/}
 
 # Error strings to check from dmesg.
-${ERROR_REGEX}     error|GPU|NVRM|nvidia
+${ERROR_REGEX}         error|GPU|NVRM|nvidia
 
 # GPU specific error message from dmesg.
-${ERROR_DBE_MSG}   (DBE) has been detected on GPU
+${ERROR_DBE_MSG}       (DBE) has been detected on GPU
+
+# Inventory - List of I/O devices to collect for Inventory
+@{I/O}                 communication  disk  display  generic  input  multimedia
+...                    network  printer  tape
+
+# Inventory Paths of the JSON and YAML files
+${json_tmp_file_path}  ${EXECDIR}/inventory.json
+${yaml_file_path}      ${EXECDIR}/inventory.yaml
+
+
 
 *** Keywords ***
 
@@ -270,3 +278,107 @@
     Rprintn  ${shutdown}
     Should Contain  ${shutdown}  shutdown successfully
 
+
+Create JSON Inventory File
+    [Documentation]  Create a JSON inventory file, and make a YAML copy.
+    [Arguments]  ${json_file_path}
+    # Description of argument:
+    # json_file_path  Where the inventory file is wrtten to.
+
+    Login To OS
+    Compile Inventory JSON
+    Run  json2yaml ${json_tmp_file_path} ${yaml_file_path}
+    # Format to JSON pretty print to file.
+    Run  python -m json.tool ${json_tmp_file_path} > ${json_file_path}
+    OperatingSystem.File Should Exist  ${json_file_path}
+
+
+Compile Inventory JSON
+    [Documentation]  Compile the Inventory into a JSON file.
+    Create File  ${json_tmp_file_path}
+    Write New JSON List  ${json_tmp_file_path}  Inventory
+    Retrieve HW Info And Write  processor  ${json_tmp_file_path}
+    Retrieve HW Info And Write  memory  ${json_tmp_file_path}
+    Retrieve HW Info And Write List  ${I/O}  ${json_tmp_file_path}  I/O  last
+    Close New JSON List  ${json_tmp_file_path}
+
+Write New JSON List
+    [Documentation]  Start a new JSON list element in file.
+    [Arguments]  ${json_tmp_file_path}  ${json_field_name}
+    # Description of argument(s):
+    # json_tmp_file_path   Name of file to write to.
+    # json_field_name      Name to give json list element.
+    Append to File  ${json_tmp_file_path}  { "${json_field_name}" : [
+
+Close New JSON List
+    [Documentation]  Close JSON list element in file.
+    [Arguments]  ${json_tmp_file_path}
+    # Description of argument(s):
+    # json_tmp_file_path  Path of file to write to.
+    Append to File  ${json_tmp_file_path}  ]}
+
+Retrieve HW Info And Write
+    [Documentation]  Retrieve and write info, add a comma if not last item.
+    [Arguments]  ${class}  ${json_tmp_file_path}  ${last}=false
+    # Description of argument(s):
+    # class               Device class to retrieve with lshw.
+    # json_tmp_file_path  Path of file to write to.
+    # last                Is this the last element in the parent JSON?
+    Write New JSON List  ${json_tmp_file_path}  ${class}
+    ${output} =  Retrieve Hardware Info  ${class}
+    ${output} =  Clean Up String  ${output}
+    Run Keyword if  ${output.__class__ is not type(None)}
+    ...  Append To File  ${json_tmp_file_path}  ${output}
+    Close New JSON List  ${json_tmp_file_path}
+    Run Keyword if  '${last}' == 'false'
+    ...  Append to File  ${json_tmp_file_path}  ,
+
+Retrieve HW Info And Write List
+    [Documentation]  Does a Retrieve/Write with a list of classes and
+    ...              encapsulates them into one large JSON element.
+    [Arguments]  ${list}  ${json_tmp_file_path}  ${json_field_name}
+    ...          ${last}=false
+    # Description of argument(s):
+    # list                 The list of devices classes to retrieve with lshw.
+    # json_tmp_file_path   Path of file to write to.
+    # json_field_name      Name of the JSON element to encapsulate this list.
+    # last                 Is this the last element in the parent JSON?
+    Write New JSON List  ${json_tmp_file_path}  ${json_field_name}
+    : FOR  ${class}  IN  @{list}
+    \  ${tail}  Get From List  ${list}  -1
+    \  Run Keyword if  '${tail}' == '${class}'
+    \  ...  Retrieve HW Info And Write  ${class}  ${json_tmp_file_path}  true
+    \  ...  ELSE  Retrieve HW Info And Write  ${class}  ${json_tmp_file_path}
+    Close New JSON List  ${json_tmp_file_path}
+    Run Keyword if  '${last}' == 'false'
+    ...  Append to File  ${json_tmp_file_path}  ,
+
+Retrieve Hardware Info
+    [Documentation]  Retrieves the lshw output of the device class as JSON.
+    [Arguments]  ${class}
+    # Description of argument(s):
+    # class  Device class to retrieve with lshw.
+    ${output} =  Execute Command On OS  lshw -c ${class} -json
+    ${output} =  Verify JSON string  ${output}
+    [Return]  ${output}
+
+Verify JSON String
+    [Documentation]  Ensure the JSON string content is seperated by commas.
+    [Arguments]  ${unver_string}
+    # Description of argument(s):
+    # unver_string  JSON String we will be checking for lshw comma errors.
+    ${unver_string} =  Convert to String  ${unver_string}
+    ${ver_string} =  Replace String Using Regexp  ${unver_string}  }\\s*{  },{
+    [Return]  ${ver_string}
+
+Clean Up String
+    [Documentation]  Remove extra whitespace and trailing commas.
+    [Arguments]  ${dirty_string}
+    # Description of argument(s):
+    # dirty_string  String that will be space stripped and have comma removed.
+    ${clean_string} =  Strip String  ${dirty_string}
+    ${last_char} =  Get Substring  ${clean_string}  -1
+    ${trimmed_string} =  Get Substring  ${clean_string}  0  -1
+    ${clean_string} =  Set Variable If  '${last_char}' == ','
+    ...  ${trimmed_string}  ${clean_string}
+    [Return]  ${clean_string}
diff --git a/systest/Generate_OS_Inventory.robot b/systest/Generate_OS_Inventory.robot
old mode 100644
new mode 100755
index 733fabe..c63696d
--- a/systest/Generate_OS_Inventory.robot
+++ b/systest/Generate_OS_Inventory.robot
@@ -1,6 +1,6 @@
 ***Settings***
-Documentation      This module is for generating an inventory file using lshw
-...                commands. It will create a JSON file and a YAML file. it
+Documentation      This module generates an inventory file using lshw
+...                commands.  It will create a JSON file and a YAML file. It
 ...                will get the processor, memory and specified I/O devices.
 ...                Requires access to lshw, and json2yaml OS commands. This
 ...                robot file should be run as root or sudo for lshw.
@@ -11,114 +11,13 @@
 Resource           ../syslib/utils_os.robot
 
 ***Variables***
-# List of I/O Devices to Collect
-@{I/O}             communication  disk  display  generic  input  multimedia
-...                network  printer  tape
 
-# Paths of the JSON and YAML files
-${json_tmp_file_path}   ${EXECDIR}${/}inventory.json
-${json_file_path}       ${EXECDIR}${/}data${/}os_inventory.json
-${yaml_file_path}       ${EXECDIR}${/}inventory.yaml
+# Path of the JSON Inventory file
+${json_inventory_file_path}  ${EXECDIR}${/}data${/}os_inventory.json
 
 ***Test Case***
 
-Create YAML Inventory File
-    [Documentation]  Create a JSON inventory file, and make a YAML copy.
-    [Tags]  Create_YAML_Inventory_File
-    Login To OS
-    Compile Inventory JSON
-    Run  json2yaml ${json_tmp_file_path} ${yaml_file_path}
-    # Format to JSON pretty print to file.
-    Run  python -m json.tool ${json_tmp_file_path} > ${json_file_path}
-
-***Keywords***
-
-Compile Inventory JSON
-    [Documentation]  Compile the Inventory into a JSON file.
-    Create File  ${json_tmp_file_path}
-    Write New JSON List  ${json_tmp_file_path}  Inventory
-    Retrieve HW Info And Write  processor  ${json_tmp_file_path}
-    Retrieve HW Info And Write  memory  ${json_tmp_file_path}
-    Retrieve HW Info And Write List  ${I/O}  ${json_tmp_file_path}  I/O  last
-    Close New JSON List  ${json_tmp_file_path}
-
-Write New JSON List
-    [Documentation]  Start a new JSON list element in file.
-    [Arguments]  ${json_tmp_file_path}  ${json_field_name}
-    # Description of argument(s):
-    # json_tmp_file_path   Name of file to write to.
-    # json_field_name      Name to give json list element.
-    Append to File  ${json_tmp_file_path}  { "${json_field_name}" : [
-
-Close New JSON List
-    [Documentation]  Close JSON list element in file.
-    [Arguments]  ${json_tmp_file_path}
-    # Description of argument(s):
-    # json_tmp_file_path  Path of file to write to.
-    Append to File  ${json_tmp_file_path}  ]}
-
-Retrieve HW Info And Write
-    [Documentation]  Retrieve and write info, add a comma if not last item.
-    [Arguments]  ${class}  ${json_tmp_file_path}  ${last}=false
-    # Description of argument(s):
-    # class               Device class to retrieve with lshw.
-    # json_tmp_file_path  Path of file to write to.
-    # last                Is this the last element in the parent JSON?
-    Write New JSON List  ${json_tmp_file_path}  ${class}
-    ${output} =  Retrieve Hardware Info  ${class}
-    ${output} =  Clean Up String  ${output}
-    Run Keyword if  ${output.__class__ is not type(None)}
-    ...  Append To File  ${json_tmp_file_path}  ${output}
-    Close New JSON List  ${json_tmp_file_path}
-    Run Keyword if  '${last}' == 'false'
-    ...  Append to File  ${json_tmp_file_path}  ,
-
-Retrieve HW Info And Write List
-    [Documentation]  Does a Retrieve/Write with a list of classes and
-    ...              encapsulates them into one large JSON element.
-    [Arguments]  ${list}  ${json_tmp_file_path}  ${json_field_name}
-    ...          ${last}=false
-    # Description of argument(s):
-    # list                 The list of devices classes to retrieve with lshw.
-    # json_tmp_file_path   Path of file to write to.
-    # json_field_name      Name of the JSON element to encapsulate this list.
-    # last                 Is this the last element in the parent JSON?
-    Write New JSON List  ${json_tmp_file_path}  ${json_field_name}
-    : FOR  ${class}  IN  @{list}
-    \  ${tail}  Get From List  ${list}  -1
-    \  Run Keyword if  '${tail}' == '${class}'
-    \  ...  Retrieve HW Info And Write  ${class}  ${json_tmp_file_path}  true
-    \  ...  ELSE  Retrieve HW Info And Write  ${class}  ${json_tmp_file_path}
-    Close New JSON List  ${json_tmp_file_path}
-    Run Keyword if  '${last}' == 'false'
-    ...  Append to File  ${json_tmp_file_path}  ,
-
-Retrieve Hardware Info
-    [Documentation]  Retrieves the lshw output of the device class as JSON.
-    [Arguments]  ${class}
-    # Description of argument(s):
-    # class  Device class to retrieve with lshw.
-    ${output} =  Execute Command On OS  lshw -c ${class} -json
-    ${output} =  Verify JSON string  ${output}
-    [Return]  ${output}
-
-Verify JSON String
-    [Documentation]  Ensure the JSON string content is seperated by commas.
-    [Arguments]  ${unver_string}
-    # Description of argument(s):
-    # unver_string  JSON String we will be checking for lshw comma errors.
-    ${unver_string} =  Convert to String  ${unver_string}
-    ${ver_string} =  Replace String Using Regexp  ${unver_string}  }\\s*{  },{
-    [Return]  ${ver_string}
-
-Clean Up String
-    [Documentation]  Remove extra whitespace and trailing commas.
-    [Arguments]  ${dirty_string}
-    # Description of argument(s):
-    # dirty_string  String that will be space stripped and have comma removed.
-    ${clean_string} =  Strip String  ${dirty_string}
-    ${last_char} =  Get Substring  ${clean_string}  -1
-    ${trimmed_string} =  Get Substring  ${clean_string}  0  -1
-    ${clean_string} =  Set Variable If  '${last_char}' == ','
-    ...  ${trimmed_string}  ${clean_string}
-    [Return]  ${clean_string}
+Create An Inventory
+    [Documentation]  Snapshot system inventory to a JSON file.
+    [Tags]  Inventory Test
+    Create JSON Inventory File  ${json_inventory_file_path}
diff --git a/systest/htx_hardbootme_test.robot b/systest/htx_hardbootme_test.robot
old mode 100644
new mode 100755
index 428bf11..be2d920
--- a/systest/htx_hardbootme_test.robot
+++ b/systest/htx_hardbootme_test.robot
@@ -1,7 +1,32 @@
 *** Settings ***
-Documentation    Stress the system using HTX exerciser.
 
-Resource         ../syslib/utils_os.robot
+Documentation  Stress the system using HTX exerciser.
+
+# Test Parameters:
+# OPENBMC_HOST        The BMC host name or IP address.
+# OS_HOST             The OS host name or IP Address.
+# OS_USERNAME         The OS login userid (usually root).
+# OS_PASSWORD         The password for the OS login.
+# HTX_DURATION        Duration of HTX run, for example, 8 hours, or
+#                     30 minutes.
+# HTX_LOOP            The number of times to loop HTX.
+# HTX_INTERVAL        The time delay between consecutive checks of HTX
+#                     status, for example, 30s.
+#                     In summary: Run HTX for $HTX_DURATION, looping
+#                     $HTX_LOOP times checking every $HTX_INTERVAL.
+# HTX_KEEP_RUNNING    If set to 1, this indicates that the HTX is to
+#                     continue running after an error.
+# CHECK_INVENTORY     If set to 1, this enables OS inventory checking
+#                     before and after each HTX run.  This parameter
+#                     is optional.
+# PREV_INV_FILE_PATH  The file path and name of a previous inventory
+#                     snapshot file.  After HTX start the system inventory
+#                     is compared to the contents of this file.  Setting this
+#                     parameter is optional.  CHECK_INVENTORY does not
+#                     need to be set if PREV_INV_FILE_PATH is set.
+
+Resource        ../syslib/utils_os.robot
+Library         ../syslib/utils_keywords.py
 
 Suite Setup     Run Key  Start SOL Console Logging
 Test Setup      Pre Test Case Execution
@@ -9,7 +34,13 @@
 
 *** Variables ****
 
-${stack_mode}        skip
+${stack_mode}                skip
+${json_initial_file_path}    ${EXECDIR}/data/os_inventory_initial.json
+${json_final_file_path}      ${EXECDIR}/data/os_inventory_final.json
+${json_diff_file_path}       ${EXECDIR}/data/os_inventory_diff.json
+${last_inventory_file_path}  ${EMPTY}
+${run_the_inventory}         0
+&{ignore_dict}               processor=size
 
 *** Test Cases ***
 
@@ -20,26 +51,49 @@
     Rprintn
     Rpvars  HTX_DURATION  HTX_INTERVAL
 
-    Repeat Keyword  ${HTX_LOOP} times  Start HTX Exerciser
+    # Set last inventory file to PREV_INV_FILE_PATH otherwise set
+    # it to ${EMPTY}.
+    ${last_inventory_file_path}=  Get Variable Value  ${PREV_INV_FILE_PATH}
+    ...  ${EMPTY}
+
+    # Set ${run_the_inventory} if PREV_INV_FILE_PATH was specified,
+    # else set ${run_the_inventory} from the ${CHECK_INVENTORY} parameter.
+    ${run_the_inventory}=  Run Keyword If
+    ...  '${last_inventory_file_path}' != '${EMPTY}'  Set Variable  ${1}
+    ...  ELSE  Run Keyword If  '${last_inventory_file_path}' == '${EMPTY}'
+    ...  Get Variable Value  ${CHECK_INVENTORY}  ${EMPTY}
+
+    Set Suite Variable  ${run_the_inventory}  children=true
+    Set Suite Variable  ${last_inventory_file_path}  children=true
+
+    Repeat Keyword  ${HTX_LOOP} times  Run HTX Exerciser
 
 
 *** Keywords ***
 
-Start HTX Exerciser
-    [Documentation]  Start HTX exerciser.
+Run HTX Exerciser
+    [Documentation]  Run HTX exerciser.
     # Test Flow:
-    #              - Power on
-    #              - Establish SSH connection session
-    #              - Create HTX mdt profile
-    #              - Run HTX exerciser
-    #              - Check HTX status for errors
-    #              - Power off
+    # - Power on.
+    # - Establish SSH connection session.
+    # - Do inventory collection, compare with
+    #   previous inventory run if applicable.
+    # - Create HTX mdt profile.
+    # - Run HTX exerciser.
+    # - Check HTX status for errors.
+    # - Do inventory collection, compare with
+    #   previous inventory run.
+    # - Power off.
 
     Boot To OS
 
     # Post Power off and on, the OS SSH session needs to be established.
     Login To OS
 
+    Run Keyword If  '${run_the_inventory}' != '${EMPTY}'
+    ...  Do Inventory And Compare  ${json_initial_file_path}
+    ...  ${last_inventory_file_path}
+
     Run Keyword If  '${HTX_MDT_PROFILE}' == 'mdt.bu'
     ...  Create Default MDT Profile
 
@@ -49,6 +103,10 @@
 
     Shutdown HTX Exerciser
 
+    Run Keyword If  '${run_the_inventory}' != '${EMPTY}'
+    ...  Do Inventory And Compare  ${json_final_file_path}
+    ...  ${last_inventory_file_path}
+
     Power Off Host
 
     # Close all SSH and REST active sessions.
@@ -58,9 +116,51 @@
     Rprint Timen  HTX Test ran for: ${HTX_DURATION}
 
 
+Do Inventory and Compare
+    [Documentation]  Do inventory and compare.
+    [Arguments]  ${inventory_file_path}  ${last_inventory_file_path}
+    # Description of argument(s):
+    # inventory_file_path        The file to receive the inventory snapshot.
+    # last_inventory_file_path   The previous inventory to compare with.
+
+    Create JSON Inventory File  ${inventory_file_path}
+    Run Keyword If  '${last_inventory_file_path}' != '${EMPTY}'
+    ...  Compare Json Inventory Files  ${inventory_file_path}
+    ...  ${last_inventory_file_path}
+    ${last_inventory_file_path}=   Set Variable  ${inventory_file_path}
+    Set Suite Variable  ${last_inventory_file_path}  children=true
+
+
+Compare Json Inventory Files
+    [Documentation]  Compare JSON inventory files.
+    [Arguments]  ${file1}  ${file2}
+    # Description of argument(s):
+    # file1   A file that has an inventory snapshot in JSON format.
+    # file2   A file that has an inventory snapshot, to compare with file1.
+
+    ${diff_rc}=  JSON_Inv_File_Diff_Check  ${file1}
+     ...  ${file2}  ${json_diff_file_path}  ${ignore_dict}
+    Run Keyword If  '${diff_rc}' != '${0}'
+    ...  Report Inventory Mismatch  ${diff_rc}  ${json_diff_file_path}
+
+
+Report Inventory Mismatch
+    [Documentation]  Report inventory mismatch.
+    [Arguments]  ${diff_rc}  ${json_diff_file_path}
+    # Description of argument(s):
+    # diff_rc              The failing return code from the difference check.
+    # json_diff_file_path  The file that has the latest inventory snapshot.
+
+    Log To Console  Difference in inventory found, return code:
+    ...  no_newline=true
+    Log to Console  ${diff_rc}
+    Log to Console  Differences are listed in file:  no_newline=true
+    Log to Console  ${json_diff_file_path}
+    Fail  Inventory mismatch, rc=${diff_rc}
+
+
 Loop HTX Health Check
     [Documentation]  Run until HTX exerciser fails.
-
     Repeat Keyword  ${HTX_DURATION}
     ...  Run Keywords  Check HTX Run Status
     ...  AND  Sleep  ${HTX_INTERVAL}
@@ -73,7 +173,8 @@
     # 3. Close all open SSH connections.
 
     # Keep HTX running if user set HTX_KEEP_RUNNING to 1.
-    Run Keyword If  '${TEST_STATUS}' == 'FAIL' and ${HTX_KEEP_RUNNING} == ${0}
+    Run Keyword If
+    ...  '${TEST_STATUS}' == 'FAIL' and ${HTX_KEEP_RUNNING} == ${0}
     ...  Shutdown HTX Exerciser
 
     ${keyword_buf}=  Catenate  Stop SOL Console Logging