Re-working of obmc_boot_test.robot and added obmc_boot_test.py.

Change-Id: I35fa575c71a01c94fb32c9b837afad866d0a86e3
Signed-off-by: Michael Walsh <micwalsh@us.ibm.com>
diff --git a/lib/obmc_boot_test.py b/lib/obmc_boot_test.py
new file mode 100755
index 0000000..e8f44fe
--- /dev/null
+++ b/lib/obmc_boot_test.py
@@ -0,0 +1,192 @@
+#!/usr/bin/env python
+
+r"""
+This module is the python counterpart to obmc_boot_test.
+"""
+
+from tally_sheet import *
+import gen_robot_print as grp
+
+import os
+import time
+import subprocess
+
+from robot.utils import DotDict
+from robot.libraries.BuiltIn import BuiltIn
+from robot.libraries.OperatingSystem import OperatingSystem
+
+# Create boot_results_fields for use in creating boot_results.
+boot_results_fields = DotDict([('total', 0), ('pass', 0), ('fail', 0)])
+# Create boot_results which is global to this module.
+boot_results = tally_sheet('boot type',
+                           boot_results_fields,
+                           'boot_test_results')
+
+boot_results.set_sum_fields(['total', 'pass', 'fail'])
+boot_results.set_calc_fields(['total=pass+fail'])
+
+
+###############################################################################
+def add_trailing_slash(path):
+
+    r"""
+    Add a trailing slash to path if it doesn't already have one and return it.
+
+    """
+
+    return os.path.normpath(path) + os.path.sep
+
+###############################################################################
+
+
+###############################################################################
+def plug_in_setup():
+
+    r"""
+    Initialize all plug-in environment variables for use by the plug-in
+    programs.
+    """
+
+    boot_pass = int(BuiltIn().get_variable_value("${boot_pass}"))
+    if boot_pass > 1:
+        test_really_running = 1
+    else:
+        test_really_running = 0
+
+    BuiltIn().set_global_variable("${test_really_running}",
+                                  test_really_running)
+
+    next_boot = BuiltIn().get_variable_value("${next_boot}")
+    BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
+
+    # Setting master_pid correctly influences the behavior of plug-ins like
+    # DB_Logging
+    program_pid = BuiltIn().get_variable_value("${program_pid}")
+    try:
+        master_pid = OperatingSystem().get_environment_variable(
+            "AUTOBOOT_MASTER_PID")
+    except RuntimeError:
+        master_pid = program_pid
+    if master_pid == "":
+        master_pid = program_pid
+
+    BuiltIn().set_global_variable("${master_pid}", master_pid)
+
+    seconds = time.time()
+    loc_time = time.localtime(seconds)
+    time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
+
+    openbmc_nickname = BuiltIn().get_variable_value("${openbmc_nickname}")
+    if openbmc_nickname == "":
+        openbmc_host = BuiltIn().get_variable_value("${openbmc_host}")
+        ffdc_prefix = openbmc_host
+    else:
+        ffdc_prefix = openbmc_nickname
+
+    ffdc_prefix += "." + time_string
+
+    ffdc_dir_path = OperatingSystem().get_environment_variable(
+        "FFDC_DIR_PATH")
+    # Add trailing slash.
+    ffdc_dir_path = os.path.normpath(ffdc_dir_path) + os.sep
+    BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
+
+    # For each program parameter, set the corresponding AUTOBOOT_ environment
+    # variable value.  Also, set an AUTOBOOT_ environment variable for every
+    # element in additional_values.
+    additional_values = ["boot_type_desc", "boot_success", "boot_pass",
+                         "boot_fail", "test_really_running", "program_pid",
+                         "master_pid", "ffdc_prefix", "ffdc_dir_path"]
+    BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
+
+    parm_list = BuiltIn().get_variable_value("@{parm_list}")
+
+    plug_in_vars = parm_list + additional_values
+
+    for var_name in plug_in_vars:
+        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
+        var_name = var_name.upper()
+        if var_value is None:
+            var_value = ""
+        OperatingSystem().set_environment_variable(
+            "AUTOBOOT_" + var_name, var_value)
+
+    debug = int(BuiltIn().get_variable_value("${debug}"))
+    if debug:
+        cmd_buf = "printenv | egrep AUTOBOOT_ | sort -u"
+        grp.rpissuing(cmd_buf)
+        sub_proc = subprocess.Popen(cmd_buf, shell=True,
+                                    stdout=subprocess.PIPE,
+                                    stderr=subprocess.STDOUT)
+        out_buf, err_buf = sub_proc.communicate()
+        shell_rc = sub_proc.returncode
+        grp.rprint(out_buf)
+
+###############################################################################
+
+
+###############################################################################
+def create_boot_results_table():
+
+    r"""
+    Create our boot_results_table.
+    """
+
+    # At some point we'll want to change to reading in our boot types from
+    # some external source (e.g. file).
+
+    boot_results.add_row('BMC Power On')
+    boot_results.add_row('BMC Power Off')
+
+###############################################################################
+
+
+###############################################################################
+def update_boot_results_table(boot_type,
+                              boot_status):
+
+    r"""
+    Update our boot_results_table.  This includes:
+    - Updating the record for the given boot_type by incrementing the pass or
+      fail field.
+    - Calling the calc method to have the totals, etc. calculated.
+    - Updating global variables boot_pass/boot_fail.
+
+    Description of arguments:
+    boot_type    The type of boot just done (e.g. "BMC Power On").
+    boot_status  The status of the boot just done.  This should be equal to
+                 either "pass" or "fail" (case-insensitive).
+    """
+
+    boot_results.inc_row_field(boot_type, boot_status.lower())
+    totals_line = boot_results.calc()
+
+    # The caller of obmc_boot_test can pass boot_pass/boot_fail values because
+    # the caller may have already done some testing (e.g. "BMC OOB").  For the
+    # sake of DB logging done by plug-ins, we want to include these in our
+    # overall totals.
+    initial_boot_pass = int(BuiltIn().get_variable_value(
+        "${initial_boot_pass}"))
+    initial_boot_fail = int(BuiltIn().get_variable_value(
+        "${initial_boot_fail}"))
+
+    BuiltIn().set_global_variable("${boot_pass}",
+                                  totals_line['pass'] + initial_boot_pass)
+    BuiltIn().set_global_variable("${boot_fail}",
+                                  totals_line['fail'] + initial_boot_fail)
+
+###############################################################################
+
+
+###############################################################################
+def print_boot_results_table(header_footer="\n"):
+
+    r"""
+    Print the formatted boot_resuls_table to the console.
+    """
+
+    grp.rprint(header_footer)
+    grp.rprint(boot_results.sprint_report())
+    grp.rprint(header_footer)
+
+###############################################################################
diff --git a/tests/obmc_boot_test.robot b/tests/obmc_boot_test.robot
index 5accb0b..de7a4f6 100644
--- a/tests/obmc_boot_test.robot
+++ b/tests/obmc_boot_test.robot
@@ -1,187 +1,425 @@
 *** Settings ***
-Documentation  Does random repeated IPLs based on the state of the machine. The
-...  number of repetitions is designated by ${IPL_TOTAL}. Keyword names that are
-...  listed in @{AVAIL_IPLS} become the selection of possible IPLs for the test.
+Documentation  Do random repeated boots based on the state of the BMC machine.
+...  The number of repetitions is designated by ${max_num_tests}. Keyword
+...  names that are listed in @{AVAIL_BOOTS} become the selection of possible
+...  boots for the test.
 
-Resource  ../lib/boot/boot_resource_master.robot
-Resource  ../lib/dvt/obmc_call_points.robot
 Resource  ../lib/dvt/obmc_driver_vars.txt
 Resource  ../lib/list_utils.robot
+Resource  ../lib/openbmc_ffdc.robot
+
+Library   ../lib/gen_robot_print.py
+Library   ../lib/gen_robot_plug_in.py
+Library   ../lib/gen_robot_valid.py
+Library   ../lib/state.py
+Library   ../lib/boot/powerons.py
+Library   ../lib/boot/poweroffs.py
+Library   ../lib/obmc_boot_test.py
+
+#  WITH NAME  boot_results
+
+*** Variables ***
+# Initialize program parameters variables.
+# Create parm_list containing all of our program parameters.  This is used by
+# 'Rqprint Pgm Header'
+@{parm_list}                openbmc_nickname  openbmc_host  openbmc_username
+...  openbmc_password  os_host  os_username  os_password  pdu_host
+...  pdu_username  pdu_password  pdu_slot_no  openbmc_serial_host
+...  openbmc_serial_port  boot_stack  boot_list  max_num_tests
+...  plug_in_dir_paths  status_file_path  openbmc_model  boot_pass  boot_fail
+...  test_mode  quiet  debug
+
+# Initialize each program parameter.
+${openbmc_nickname}         ${EMPTY}
+${openbmc_host}             ${EMPTY}
+${openbmc_username}         root
+${openbmc_password}         0penBmc
+${os_host}                  ${EMPTY}
+${os_username}              root
+${os_password}              P@ssw0rd
+${pdu_host}                 ${EMPTY}
+${pdu_username}             admin
+${pdu_password}             admin
+${pdu_slot_no}              ${EMPTY}
+${openbmc_serial_host}      ${EMPTY}
+${openbmc_serial_port}      ${EMPTY}
+${boot_stack}               ${EMPTY}
+${boot_list}                ${EMPTY}
+${max_num_tests}            0
+${plug_in_dir_paths}        ${EMPTY}
+${status_file_path}         ${EMPTY}
+${openbmc_model}            ${EMPTY}
+# The reason boot_pass and boot_fail are parameters is that it is possible to
+# be called by a program that has already done some tests.  This allows us to
+# keep the grand total.
+${boot_pass}                ${0}
+${boot_fail}                ${0}
+${test_mode}                0
+${quiet}                    0
+${debug}                    0
+
+# Plug-in variables.
+${shell_rc}                 0x00000000
+${fail_on_plug_in_failure}  1
+${return_on_non_zero_rc}    0
+
+${next_boot}                ${EMPTY}
+# State dictionary.  Initializing to a realistic state for testing in
+# test_mode.
+&{default_state}            power=1
+...                         bmc=HOST_BOOTED
+...                         boot_progress=FW Progress, Starting OS
+...                         os_ping=1
+...                         os_login=1
+...                         os_run_cmd=1
+&{state}                    &{default_state}
+
+# Flag variables.
+${cp_setup_called}          ${0}
+# test_really_running is needed by DB_Logging plug-in.
+${test_really_running}      ${1}
 
 *** Test Cases ***
-Repeated Testing
-    [Documentation]  Performs random, repeated IPLs.
-    [Tags]  Repeated_Testing
+Randomized Boot Testing
+    [Documentation]  Performs random, repeated boots.
+    [Tags]  Randomized_boot_testing
 
     # Call the Main keyword to prevent any dots from appearing in the console
     # due to top level keywords.
     Main
 
 *** Keywords ***
+###############################################################################
 Main
-    Log to Console  ${SUITE NAME}
+    [Teardown]  Program Teardown
 
-    Do Test Setup
-    Call Point Setup
+    Setup
 
-    Log  Doing ${IPL_TOTAL} IPLs  console=True
+    :For  ${BOOT_COUNT}  IN RANGE  ${max_num_tests}
+    \  Test Loop Body  ${BOOT_COUNT}
 
-    :FOR  ${IPL_COUNT}  IN RANGE  ${IPL_TOTAL}
-    \  Log  ${\n}***Starting IPL ${IPL_COUNT+1} of ${IPL_TOTAL}***  console=True
-    \  Validate Or Open Connection  alias=${master_alias}
-    \  ${cur_state}=  Get Power State
-    \  ${next_IPL}=  Select IPL  ${cur_state}
-    \  Call Point Pre Boot
-    \  Log  We are doing a ${next_IPL}${\n}  console=True
-    \  Update Last Ten  ${next_IPL}
-    \  Run Keyword and Continue On Failure  Run IPL  ${next_IPL}
-    \  Call Point Post Boot
-    \  Run Keyword If  '${IPL_STATUS}' == 'PASS'
-    ...  Log  IPL_SUCCESS: "${next_IPL}" succeeded.  console=True
-    ...  ELSE  Log  IPL_FAILED: ${next_IPL} failed.  console=True
-    \  Update Run Table Values  ${next_IPL}
-    \  Log  FFDC Dump requested!  console=True
-    \  Log  ***Beginning dump of FFDC***  console=True
-    \  Call Point FFDC
-    \  Log Defect Information
-    \  Log Last Ten IPLs
-    \  Log FFDC Files
-    \  Log  ***Finished dumping of FFDC***  console=True
-    \  Call Point Stop Check
-    \  Log FFDC Summary
-    \  Log Run Table
-    \  Log  ${\n}***Finished IPL ${IPL_COUNT+1} of ${IPL_TOTAL}***  console=True
+    Rprint Timen  Completed all requested boot tests.
 
-Do Test Setup
-    [Documentation]  Do any setup that needs to be done before running a series
-    ...  of IPLs.
+###############################################################################
 
-    Should Not Be Empty  ${AVAIL_IPLS}
 
-    Setup Run Table
-    Log  ***Start of status file for ${OPENBMC_HOST}***  console=True
+###############################################################################
+Setup
+    [Documentation]  Do general program setup tasks.
 
-Setup Run Table
-    [Documentation]  For each available IPL, create a variable that stores the
-    ...  number of passes and fails for each IPL.
+    Rprintn
 
-    Log to Console  Setting up run table.
+    Validate Parms
 
-    :FOR  ${ipl}  IN  @{AVAIL_IPLS}
-    \  Set Global Variable  ${${ipl}_PASS}  ${0}
-    \  Set Global Variable  ${${ipl}_FAIL}  ${0}
+    Rqprint Pgm Header
 
-Select IPL
-    [Documentation]  Contains all of the logic for which IPLs can be chosen
-    ...  given the inputted state. Returns the chosen IPL.
-    [Arguments]  ${cur_state}
+    Create Boot Results Table
 
-    # cur_state      The power state of the machine, either zero or one.
+    # Preserve the values of boot_pass/boot_fail that were passed in.
+    Set Global Variable  ${initial_boot_pass}  ${boot_pass}
+    Set Global Variable  ${initial_boot_fail}  ${boot_fail}
 
-    ${ipl}=  Run Keyword If  ${cur_state} == ${0}  Select Power On
-    ...  ELSE  Run Keyword If  ${cur_state} == ${1}  Select Power Off
+    # Call "setup" plug-ins, if any.
+    Plug In Setup
+    ${rc}  ${shell_rc}  ${failed_plug_in_name}=  Rprocess Plug In Packages
+    ...  call_point=setup
+    Should Be Equal  '${rc}'  '${0}'
+
+    # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
+    Set Global Variable  ${TEST_MESSAGE}  ${EMPTY}
+
+    # Setting cp_setup_called lets our Teardown know that it needs to call
+    # the cleanup plug-in call point.
+    Set Global Variable  ${cp_setup_called}  ${1}
+
+    Rqprint Timen  Getting system state.
+    # The state dictionary must be primed before calling Test Loop Body.
+    ${temp_state}=  Run Keyword If  '${test_mode}' == '0'  Get State
+    ...  ELSE  Create Dictionary  &{default_state}
+    Set Global Variable  &{state}  &{temp_state}
+    rpvars  state
+
+###############################################################################
+
+
+###############################################################################
+Validate Parms
+    [Documentation]  Validate all program parameters.
+
+    rprintn
+
+    Rvalid Value  AVAIL_BOOTS
+    Rvalid Value  openbmc_host
+    Rvalid Value  openbmc_username
+    Rvalid Value  openbmc_password
+    # os_host is optional so no validation is being done.
+    Rvalid Value  os_username
+    Rvalid Value  os_password
+    Rvalid Value  pdu_host
+    Rvalid Value  pdu_username
+    Rvalid Value  pdu_password
+    Rvalid Integer  pdu_slot_no
+    Rvalid Value  openbmc_serial_host
+    Rvalid Integer  openbmc_serial_port
+    Rvalid Integer  max_num_tests
+    Rvalid Value  openbmc_model
+    Rvalid Integer  boot_pass
+    Rvalid Integer  boot_fail
+
+    ${boot_pass_temp}=  Convert To Integer  ${boot_pass}
+    Set Global Variable  ${boot_pass}  ${boot_pass_temp}
+    ${boot_fail_temp}=  Convert To Integer  ${boot_fail}
+    Set Global Variable  ${boot_fail}  ${boot_fail_temp}
+
+    ${temp_arr}=  Rvalidate Plug Ins  ${plug_in_dir_paths}
+    Set Global Variable  @{plug_in_packages_list}  @{temp_arr}
+
+###############################################################################
+
+
+###############################################################################
+Program Teardown
+    [Documentation]  Clean up after this program.
+
+    Run Keyword If  '${cp_setup_called}' == '1'  Run Keywords
+    ...  Plug In Setup  AND
+    ...  Rprocess Plug In Packages  call_point=cleanup
+    ...  stop_on_plug_in_failure=1
+
+    Rqprint Pgm Footer
+
+###############################################################################
+
+
+###############################################################################
+Test Loop Body
+    [Documentation]  The main loop body for the loop in "main".
+    [Arguments]  ${BOOT_COUNT}
+
+    Rqprintn
+    Rqprint Timen  Starting boot ${BOOT_COUNT+1} of ${max_num_tests}.
+
+    ${loc_next_boot}=  Select Boot  ${state['power']}
+    Set Global Variable  ${next_boot}  ${loc_next_boot}
+
+    ${status}  ${msg}=  Run Keyword And Ignore Error  Run Boot  ${next_boot}
+    Run Keyword If  '${status}' == 'FAIL'  rprint  ${msg}
+
+    rprintn
+    Run Keyword If  '${BOOT_STATUS}' == 'PASS'  Run Keywords
+    ...    Set Global Variable  ${boot_success}  ${1}  AND
+    ...    Rqprint Timen  BOOT_SUCCESS: "${next_boot}" succeeded.
+    ...  ELSE  Run Keywords
+    ...    Set Global Variable  ${boot_success}  ${0}  AND
+    ...      Rqprint Timen  BOOT_FAILED: ${next_boot} failed.
+
+    Update Boot Results Table  ${next_boot}  ${BOOT_STATUS}
+
+    # NOTE: A post_test_case call point failure is NOT counted as a boot
+    # failure.
+    Plug In Setup
+    ${rc}  ${shell_rc}  ${failed_plug_in_name}=  Rprocess Plug In Packages
+    ...  call_point=post_test_case  stop_on_plug_in_failure=1
+
+    Run Keyword If  '${BOOT_STATUS}' != 'PASS'
+    ...  Run Keyword and Continue On Failure  My FFDC
+
+    # Run plug-ins to see if we ought to stop execution.
+    Plug In Setup
+    ${rc}  ${shell_rc}  ${failed_plug_in_name}=  Rprocess Plug In Packages
+    ...  call_point=stop_check
+    Run Keyword If  '${rc}' != '${0}'  Run Keywords
+    ...  Rprint Error Report  Stopping as requested by user.
+    ...  Fail
+
+    Print Boot Results Table
+    Rqprint Timen  Finished boot ${BOOT_COUNT+1} of ${max_num_tests}.
+
+    Rqprint Timen  Getting system state.
+    # The state must be refreshed before calling Test Loop Body again.
+    ${temp_state}=  Run Keyword If  '${test_mode}' == '0'  Get State
+    ...  quiet=${1}
+    ...  ELSE  Create Dictionary  &{default_state}
+    Set Global Variable  &{state}  &{temp_state}
+    rpvars  state
+
+###############################################################################
+
+
+###############################################################################
+Select Boot
+    [Documentation]  Select a boot test to be run based on our current state.
+    ...  Return the chosen boot type.
+    [Arguments]  ${power}
+
+    # power      The power state of the machine, either zero or one.
+
+    ${boot}=  Run Keyword If  ${power} == ${0}  Select Power On
+    ...  ELSE  Run Keyword If  ${power} == ${1}  Select Power Off
     ...  ELSE  Run Keywords  Log to Console
-    ...  **ERROR** BMC not in state to power on or off: "${cur_state}"  AND
+    ...  **ERROR** BMC not in state to power on or off: "${power}"  AND
     ...  Fatal Error
 
-    [return]  ${ipl}
+    [return]  ${boot}
 
+###############################################################################
+
+
+###############################################################################
 Select Power On
-    [Documentation]  Randomly chooses an IPL from the list of Power On IPLs.
+    [Documentation]  Randomly chooses a boot from the list of Power On boots.
 
-    @{power_on_choices}=  Intersect Lists  ${VALID_POWER_ON}  ${AVAIL_IPLS}
+    @{power_on_choices}=  Intersect Lists  ${VALID_POWER_ON}  ${AVAIL_BOOTS}
 
     ${length}=  Get Length  ${power_on_choices}
 
-    # Currently selects the first IPL in the list of options, rather than
+    # Currently selects the first boot in the list of options, rather than
     # selecting randomly.
     ${chosen}=  Set Variable  @{power_on_choices}[0]
 
     [return]  ${chosen}
 
-Select Power Off
-    [Documentation]  Randomly chooses an IPL from the list of Power Off IPLs.
+###############################################################################
 
-    @{power_off_choices}=  Intersect Lists  ${VALID_POWER_OFF}  ${AVAIL_IPLS}
+
+###############################################################################
+Select Power Off
+    [Documentation]  Randomly chooses an boot from the list of Power Off boots.
+
+    @{power_off_choices}=  Intersect Lists  ${VALID_POWER_OFF}  ${AVAIL_BOOTS}
 
     ${length}=  Get Length  ${power_off_choices}
 
-    # Currently selects the first IPL in the list of options, rather than
+    # Currently selects the first boot in the list of options, rather than
     # selecting randomly.
     ${chosen}=  Set Variable  @{power_off_choices}[0]
 
     [return]  ${chosen}
 
-Run IPL
-    [Documentation]  Runs the selected IPL and marks the status when complete.
-    [Arguments]  ${ipl_keyword}
-    [Teardown]  Set Global Variable  ${IPL_STATUS}  ${KEYWORD STATUS}
+###############################################################################
 
-    # ipl_keyword      The name of the IPL to run, which corresponds to the
+
+###############################################################################
+Run Boot
+    [Documentation]  Run the selected boot and mark the status when complete.
+    [Arguments]  ${boot_keyword}
+    [Teardown]  Set Global Variable  ${BOOT_STATUS}  ${KEYWORD STATUS}
+
+    # boot_keyword     The name of the boot to run, which corresponds to the
     #                  keyword to run. (i.e "BMC Power On")
 
-    Run Keyword  ${ipl_keyword}
+    Print Test Start Message  ${boot_keyword}
 
+    Plug In Setup
+    ${rc}  ${shell_rc}  ${failed_plug_in_name}=  Rprocess Plug In Packages
+    ...  call_point=pre_boot
+    Should Be Equal  '${rc}'  '${0}'
+
+    @{cmd_buf}=  Create List  ${boot_keyword}
+    rqpissuing_keyword  ${cmd_buf}  ${test_mode}
+    Run Keyword If  '${test_mode}' == '0'  Run Keyword  @{cmd_buf}
+
+    Plug In Setup
+    ${rc}  ${shell_rc}  ${failed_plug_in_name}=  Rprocess Plug In Packages
+    ...  call_point=post_boot
+    Should Be Equal  '${rc}'  '${0}'
+
+###############################################################################
+
+
+###############################################################################
+Print Test Start Message
+    [Documentation]  Print a message indicating what boot test is about to run.
+    [Arguments]  ${boot_keyword}
+
+    ${doing_msg}=  Sprint Timen  Doing "${boot_keyword}".
+    rqprint  ${doing_msg}
+
+    Append to List  ${LAST_TEN}  ${doing_msg}
+    ${length}=  Get Length  ${LAST_TEN}
+
+    Run Keyword If  '${length}' > '${10}'  Remove From List  ${LAST_TEN}  0
+
+###############################################################################
+
+
+###############################################################################
+My FFDC
+    [Documentation]  Collect FFDC data.
+
+    Rqprint Timen  FFDC Dump requested.
+    Rqprint Timen  Starting dump of FFDC.
+    ${FFDC_DIR_PATH}=  Add Trailing Slash  ${FFDC_DIR_PATH}
+    # FFDC_LOG_PATH is used by "FFDC" keyword.
+    Set Global Variable  ${FFDC_LOG_PATH}  ${FFDC_DIR_PATH}
+
+    @{cmd_buf}=  Create List  FFDC
+    rqpissuing_keyword  ${cmd_buf}  ${test_mode}
+    Run Keyword If  '${test_mode}' == '0'  @{cmd_buf}
+
+    Plug In Setup
+    ${rc}  ${shell_rc}  ${failed_plug_in_name}=  Rprocess Plug In Packages
+    ...  call_point=ffdc  stop_on_plug_in_failure=1
+
+    Rqprint Timen  Finished dumping of FFDC.
+    Log FFDC Summary
+    Log Defect Information
+
+###############################################################################
+
+
+###############################################################################
 Log Defect Information
     [Documentation]  Logs information needed for a defect. This information
     ...  can also be found within the FFDC gathered.
 
-    Log  Copy this data to the defect:  console=True
+    Rqprintn
+    # indent=0, width=90, linefeed=1, char="="
+    Rqprint Dashes  ${0}  ${90}  ${1}  =
+    Rqprintn  Copy this data to the defect:
+    Rqprintn
 
-Log Last Ten IPLs
-    [Documentation]  Logs the last ten IPLs that were performed with their
+    Rqpvars  @{parm_list}
+    Print Last Ten Boots
+
+    ${rc}  ${output}=  Run Keyword If  '${test_mode}' == '0'
+    ...  Run and return RC and Output  ls ${LOG_PREFIX}*
+    ...  ELSE  Set Variable  ${0}  ${EMPTY}
+
+    Run Keyword If  '${rc}' != '${0}' and '${rc}' != 'None'  rqpvars  rc
+
+    Rqprintn
+    Rqprintn  FFDC data files:
+    Rqprintn  ${output}
+
+    Rqprintn
+    Rqprint Dashes  ${0}  ${90}  ${1}  =
+
+###############################################################################
+
+
+###############################################################################
+Print Last Ten Boots
+    [Documentation]  Logs the last ten boots that were performed with their
     ...  starting time stamp.
 
-    Log  ${\n}----------------------------------${\n}Last 10 IPLs${\n}
-    ...  console=True
-    :FOR  ${ipl}  IN  @{LAST_TEN}
-    \  Log  ${ipl}  console=True
-    Log  ----------------------------------${\n}  console=True
+    # indent 0, 90 chars wide, linefeed, char is "="
+    Rqprint Dashes  ${0}  ${90}
+    Rqprintn  Last 10 boots:
+    Rqprintn
+    :FOR  ${boot_entry}  IN  @{LAST_TEN}
+    \  rqprint  ${boot_entry}
+    Rqprint Dashes  ${0}  ${90}
 
-Log FFDC Files
-    [Documentation]  Logs the files outputted during FFDC gathering.
-    Log  This is where the list of FFDC files would be.  console=True
+###############################################################################
 
+
+###############################################################################
 Log FFDC Summary
     [Documentation]  Logs finding from within the FFDC files gathered.
-    Log  This is where the FFDC summary would go.  console=True
 
-Log Run Table
-    [Documentation]  Logs the table of IPLs that have passed and failed based on
-    ...  the available IPLs, as well as the total passes and failures.
+    Rqprint Timen  This is where the FFDC summary would go...
 
-    Log  ${\n}IPL type${space*14}Pass${space*3}Fail  console=True
-    Log  ==================================  console=True
-    :FOR  ${ipl}  IN  @{AVAIL_IPLS}
-    \  ${length}=  Get Length  ${ipl}
-    \  ${space_num}=  Set Variable  ${24-${length}}
-    \  Log  ${ipl}${space*${space_num}}${${ipl}_PASS}${space*5}${${ipl}_FAIL}
-    ...  console=True
-    Log  ==================================  console=True
-    Log  Totals:${space*17}${IPL_PASSED}${space*5}${IPL_FAILED}${\n}
-    ...  console=True
+###############################################################################
 
-Update Run Table Values
-    [Documentation]  Updates the table of IPLs that have passed and failed. See
-    ...  the "Log Run Table" keyword for more information.
-    [Arguments]  ${last_ipl}
 
-    # last_ipl      The name of the last IPL that ran (i.e "BMC Power On").
-
-    ${cur_value}=  Get Variable Value  ${${last_ipl}_${IPL_STATUS}}
-    Set Global Variable  ${${last_ipl}_${IPL_STATUS}}  ${cur_value+1}
-    ${total_value}=  Run Keyword If  '${IPL_STATUS}' == 'PASS'
-    ...              Get Variable Value  ${IPL_PASSED}
-    ...              ELSE  Get Variable Value  ${IPL_FAILED}
-    Run Keyword If  '${IPL_STATUS}' == 'PASS'
-    ...             Set Global Variable  ${IPL_PASSED}  ${total_value+1}
-    ...             ELSE  Set Global Variable  ${IPL_FAILED}  ${total_value+1}
-
-Update Last Ten
-    [Documentation]  Updates the list of last ten IPLs
-    [Arguments]  ${last_ipl}
-
-    # last_ipl      The name of the last IPL that ran (i.e. "BMC Power On")
-
-    ${time}=  Get Time
-    Append to List  ${LAST_TEN}  ${time} - Doing "${last_ipl}"