ffdc: Integrate ffdc script into robot framework flow.
This commit provides interface to enable automation tests (robot/jenkins)
to activate the new cli ffdc collector.
- Patchset 1: Initial commit for review.
- Patchset 2: Redesing and fixing issues
- Patchset 3: Fix build issue (pycodestyle)
- Patchset 4: Default ffdc store location running direct CLI
- Patchset 5: Addressing review comments and questions
- Patchset 6: Restore direct CLI default location to /tmp
- Patchset 7: Addressing issues from comments
Also extend socket timeout for long-running command peltool -a
- Patchset 8: Use Builtin().log_to_console("") method in the bridge code
- Patchset 9: Rebase
Tests:
- Standalone script: python3 ./ffdc/collect_ffdc.py
- Within robot env: python3 -m robot .... -v FFDC_DEFAULT:0 ./tools/myffdc.robot
Signed-off-by: Peter D Phan <peterp@us.ibm.com>
Change-Id: Ic892c48efe7dbdb3a4345ba89c3a0257f03a7ffb
diff --git a/ffdc/collect_ffdc.py b/ffdc/collect_ffdc.py
index 7711fd6..c1c9495 100644
--- a/ffdc/collect_ffdc.py
+++ b/ffdc/collect_ffdc.py
@@ -18,7 +18,7 @@
for found_dir in dirs:
sys.path.append(os.path.join(root, found_dir))
-from ffdc_collector import FFDCCollector
+from ffdc_collector import ffdc_collector
@click.command(context_settings=dict(help_option_names=['-h', '--help']))
@@ -61,26 +61,26 @@
click.echo("\n********** FFDC (First Failure Data Collection) Starts **********")
if input_options_ok(remote, username, password, config, type):
- thisFFDC = FFDCCollector(remote,
- username,
- password,
- config,
- location,
- type,
- protocol,
- env_vars,
- econfig,
- log_level)
- thisFFDC.collect_ffdc()
+ this_ffdc = ffdc_collector(remote,
+ username,
+ password,
+ config,
+ location,
+ type,
+ protocol,
+ env_vars,
+ econfig,
+ log_level)
+ this_ffdc.collect_ffdc()
- if len(os.listdir(thisFFDC.ffdc_dir_path)) == 0:
+ if len(os.listdir(this_ffdc.ffdc_dir_path)) == 0:
click.echo("\n\tFFDC Collection from " + remote + " has failed.\n\n")
else:
- click.echo(str("\n\t" + str(len(os.listdir(thisFFDC.ffdc_dir_path)))
+ click.echo(str("\n\t" + str(len(os.listdir(this_ffdc.ffdc_dir_path)))
+ " files were retrieved from " + remote))
- click.echo("\tFiles are stored in " + thisFFDC.ffdc_dir_path)
+ click.echo("\tFiles are stored in " + this_ffdc.ffdc_dir_path)
- click.echo("\tTotal elapsed time " + thisFFDC.elapsed_time + "\n\n")
+ click.echo("\tTotal elapsed time " + this_ffdc.elapsed_time + "\n\n")
click.echo("\n********** FFDC Finishes **********\n\n")
diff --git a/ffdc/ffdc_collector.py b/ffdc/ffdc_collector.py
index 4a8cfa4..61590a3 100644
--- a/ffdc/ffdc_collector.py
+++ b/ffdc/ffdc_collector.py
@@ -14,6 +14,8 @@
import platform
from errno import EACCES, EPERM
import subprocess
+
+sys.path.extend([f'./{name[0]}' for name in os.walk(".") if os.path.isdir(name[0])])
from ssh_utility import SSHRemoteclient
from telnet_utility import TelnetRemoteclient
@@ -33,8 +35,8 @@
- arg1
- arg2
"""
-abs_path = os.path.abspath(os.path.dirname(sys.argv[0]))
-plugin_dir = abs_path + '/plugins'
+plugin_dir = __file__.split(__file__.split("/")[-1])[0] + '/plugins'
+sys.path.append(plugin_dir)
try:
for module in os.listdir(plugin_dir):
if module == '__init__.py' or module[-3:] != '.py':
@@ -98,7 +100,7 @@
}
-class FFDCCollector:
+class ffdc_collector:
r"""
Execute commands from configuration file to collect log files.
@@ -154,9 +156,9 @@
# to be sure that all files for this run will have same timestamps
# and they will be saved in the same directory.
# self.location == local system for now
- self.set_ffdc_defaults()
+ self.set_ffdc_default_store_path()
- # Logger for this run. Need to be after set_ffdc_defaults()
+ # Logger for this run. Need to be after set_ffdc_default_store_path()
self.script_logging(getattr(logging, log_level.upper()))
# Verify top level directory exists for storage
@@ -181,7 +183,7 @@
# Load ENV vars from user.
self.logger.info("\n\tENV: User define input YAML variables")
self.env_dict = {}
- self. load_env()
+ self.load_env()
def verify_script_env(self):
@@ -686,7 +688,7 @@
progress_counter += 1
self.print_progress(progress_counter)
- def set_ffdc_defaults(self):
+ def set_ffdc_default_store_path(self):
r"""
Set a default value for self.ffdc_dir_path and self.ffdc_prefix.
Collected ffdc file will be stored in dir /self.location/hostname_timestr/.
@@ -704,7 +706,11 @@
self.ffdc_prefix = timestr + "_"
self.validate_local_store(self.ffdc_dir_path)
- def validate_local_store(self, dir_path):
+ # Need to verify local store path exists prior to instantiate this class.
+ # This class method is used to share the same code between CLI input parm
+ # and Robot Framework "${EXECDIR}/logs" before referencing this class.
+ @classmethod
+ def validate_local_store(cls, dir_path):
r"""
Ensure path exists to store FFDC files locally.
@@ -916,7 +922,7 @@
def execute_plugin_block(self, plugin_cmd_list):
r"""
- Pack the plugin command to quailifed python string object.
+ Pack the plugin command to qualifed python string object.
Description of argument(s):
plugin_list_dict Plugin block read from YAML
diff --git a/ffdc/ffdc_config.yaml b/ffdc/ffdc_config.yaml
index 1f7e041..b2bd403 100644
--- a/ffdc/ffdc_config.yaml
+++ b/ffdc/ffdc_config.yaml
@@ -61,7 +61,7 @@
- 'cat /var/log/obmc-console.log >/tmp/BMC_obmc_console.txt'
- 'cat /var/log/obmc-console1.log >/tmp/BMC_obmc_console1.txt'
- 'peltool -l >/tmp/PEL_logs_list.json'
- - 'peltool -a >/tmp/PEL_logs_display.json'
+ - {'peltool -a >/tmp/PEL_logs_display.json':240}
- 'hexdump -C /var/lib/phosphor-logging/extensions/pels/badPEL >/tmp/PEL_logs_badPEL.txt'
- 'guard -l >/tmp/GUARD_list.txt'
- 'killall -s SIGUSR1 pldmd; sleep 5'
diff --git a/ffdc/plugins/redfish.py b/ffdc/plugins/redfish.py
index 33a0669..0ea7a00 100644
--- a/ffdc/plugins/redfish.py
+++ b/ffdc/plugins/redfish.py
@@ -82,7 +82,7 @@
response = execute_redfish_cmd(parms + resource)
# Enumeration is done for available resources ignoring the
# ones for which response is not obtained.
- if 'Response Error' in response:
+ if 'Error getting response' in response:
continue
walk_nested_dict(response, url=resource)
diff --git a/lib/ffdc_cli_robot_script.py b/lib/ffdc_cli_robot_script.py
new file mode 100644
index 0000000..6780dd5
--- /dev/null
+++ b/lib/ffdc_cli_robot_script.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+
+sys.path.append(__file__.split(__file__.split("/")[-1])[0] + "../ffdc")
+from ffdc_collector import ffdc_collector
+
+from robot.libraries.BuiltIn import BuiltIn as robotBuildIn
+
+# (Sub) String constants used for input dictionary key search
+HOST = "HOST"
+USER = "USERNAME"
+PASSWD = "PASSWORD"
+CONFIG = "CONFIG"
+TYPE = "TYPE"
+LOC = "LOCATION"
+PROTOCOL = "PROTOCOL"
+ENV_VARS = "ENV_VARS"
+ECONFIG = "ECONFIG"
+LOGLEVEL = "LOG"
+
+
+def ffdc_robot_script_cli(**kwargs):
+ r"""
+
+ For the specified host, this method provide automation testcases the interface to
+ the new ffdc collector ../ffdc/ffdc_collector.py via robot variable FFDC_DEFAULT
+
+ variable FFDC_DEFAULT:1, by default use the existing ffdc collection method.
+ variable FFDC_DEFAULT:0 use the new ffdc method
+
+ Command examples:
+ (1) Legacy ffdc collection
+ python3 -m robot -v OPENBMC_HOST:<> -v OPENBMC_USERNAME:<> \
+ -v OPENBMC_PASSWORD:<> ./tools/myffdc.robot
+ (2) New ffdc collection
+ python3 -m robot -v OPENBMC_HOST:<> -v OPENBMC_USERNAME:<> \
+ -v OPENBMC_PASSWORD:<> -v FFDC_DEFAULT:0 ./tools/myffdc.robot
+
+ Description of argument(s)in dictionary: xx can be anything appropriate
+
+ xx_HOST:hostname name/ip of the targeted (remote) system
+ xx_USERNAME:username user on the targeted system with access to FFDC files
+ xx_PASSWORD:password password for user on targeted system
+ xx_CONFIG:ffdc_config configuration file listing commands and files for FFDC
+ xx_LOCATION:location where to store collected FFDC. Default: <current dir>/logs/
+ xx_TYPE:remote_type os type of the remote host.
+ xx_PROTOCOL:remote_protocol Protocol to use to collect data. Default: 'ALL'
+ ENV_VAR:env_vars User define CLI env vars '{"key : "value"}'. Default: ""
+ ECONFIG:econfig User define env vars YAML file. Default: ""
+ LOG_LEVEL:log_level CRITICAL, ERROR, WARNING, INFO, DEBUG. Default: INFO
+
+ Code examples:
+ (1) openbmc_ffdc.robot activate this method with no parm
+ Run Keyword If ${FFDC_DEFAULT} == ${1} FFDC
+ ... ELSE ffdc_robot_script_cli
+
+ (2) Method invocation with parms
+ ffdc_from = {'OS_HOST' : 'os host name or ip',
+ 'OS_USERNAME' : 'os username',
+ 'OS_PASSWORD' : 'password for os_username',
+ 'OS_TYPE' : 'os_type, ubuntu, rhel, aix, etc',
+ }
+ ffdc_robot_script_cli(ffdc_from)
+
+ """
+
+ robotBuildIn().log_to_console("Collecting FFDC - CLI log collector script")
+
+ if not kwargs:
+ dict_of_parms = {}
+ # When method is invoked with no parm,
+ # use robot variables
+ # OPENBMC_HOST, OPENBMC_USERNAME, OPENBMC_PASSWORD, OPENBMC (type)
+ dict_of_parms["OPENBMC_HOST"] = \
+ robotBuildIn().get_variable_value("${OPENBMC_HOST}", default=None)
+ dict_of_parms["OPENBMC_USERNAME"] = \
+ robotBuildIn().get_variable_value("${OPENBMC_USERNAME}", default=None)
+ dict_of_parms["OPENBMC_PASSWORD"] = \
+ robotBuildIn().get_variable_value("${OPENBMC_PASSWORD}", default=None)
+ dict_of_parms["REMOTE_TYPE"] = "OPENBMC"
+
+ run_ffdc_collector(dict_of_parms)
+
+ else:
+ if isinstance(kwargs, dict):
+ # When method is invoked with user defined dictionary,
+ # dictionary keys has the following format
+ # xx_HOST; xx_USERNAME, xx_PASSWORD, xx_TYPE
+ # where xx is one of OPENBMC, OS, or os_type LINUX/UBUNTU/AIX
+ run_ffdc_collector(**kwargs)
+
+
+def run_ffdc_collector(dict_of_parm):
+ r"""
+
+ Process input parameters and collect information
+
+ Description of argument(s)in dictionary: xx can be anything appropriate
+
+ xx_HOST:hostname name/ip of the targeted (remote) system
+ xx_USERNAME:username user on the targeted system with access to FFDC files
+ xx_PASSWORD:password password for user on targeted system
+ xx_CONFIG:ffdc_config configuration file listing commands and files for FFDC
+ xx_LOCATION:location where to store collected FFDC. Default: <current dir>/logs/
+ xx_TYPE:remote_type os type of the remote host.
+ xx_PROTOCOL:remote_protocol Protocol to use to collect data. Default: 'ALL'
+ ENV_VAR:env_vars User define CLI env vars '{"key : "value"}'. Default: ""
+ ECONFIG:econfig User define env vars YAML file. Default: ""
+ LOG_LEVEL:log_level CRITICAL, ERROR, WARNING, INFO, DEBUG. Default: INFO
+
+ """
+
+ # Clear local variables
+ remote = None
+ username = None
+ password = None
+ config = None
+ location = None
+ remote_type = None
+ protocol = None
+ env_vars = None
+ econfig = None
+ log_level = None
+
+ # Process input key/value pairs
+ for key in dict_of_parm.keys():
+ if HOST in key:
+ remote = dict_of_parm[key]
+ elif USER in key:
+ username = dict_of_parm[key]
+ elif PASSWD in key:
+ password = dict_of_parm[key]
+ elif CONFIG in key:
+ config = dict_of_parm[key]
+ elif LOC in key:
+ location = dict_of_parm[key]
+ elif TYPE in key:
+ remote_type = dict_of_parm[key]
+ elif PROTOCOL in key:
+ protocol = dict_of_parm[key]
+ elif ENV_VARS in key:
+ env_vars = dict_of_parm[key]
+ elif ECONFIG in key:
+ econfig = dict_of_parm[key]
+ elif LOGLEVEL in key:
+ log_level = dict_of_parm[key]
+
+ # Set defaults values for parms
+ # that are not specified with input and have acceptable defaults.
+ if not location:
+ # Default FFDC store location
+ location = robotBuildIn().get_variable_value("${EXECDIR}", default=None) + "/logs"
+ ffdc_collector.validate_local_store(location)
+
+ if not config:
+ # Default FFDC configuration
+ script_path = os.path.dirname(os.path.abspath(__file__))
+ config = script_path + "/../ffdc/ffdc_config.yaml"
+
+ if not protocol:
+ protocol = "ALL"
+
+ if not env_vars:
+ env_vars = ""
+
+ if not econfig:
+ econfig = ""
+
+ if not log_level:
+ log_level = "INFO"
+
+ # If minimum required inputs are met, go collect.
+ if (remote and username and password and remote_type):
+ # Execute data collection
+ this_ffdc = ffdc_collector(remote,
+ username,
+ password,
+ config,
+ location,
+ remote_type,
+ protocol,
+ env_vars,
+ econfig,
+ log_level)
+ this_ffdc.collect_ffdc()
diff --git a/lib/openbmc_ffdc.robot b/lib/openbmc_ffdc.robot
index de11927..f2dfc8f 100755
--- a/lib/openbmc_ffdc.robot
+++ b/lib/openbmc_ffdc.robot
@@ -33,6 +33,7 @@
Resource openbmc_ffdc_utils.robot
Resource dump_utils.robot
Library openbmc_ffdc.py
+Library ffdc_cli_robot_script.py
*** Keywords ***
@@ -51,6 +52,17 @@
${OVERRIDE_FFDC_ON_TEST_CASE_FAIL}= Convert To Integer ${OVERRIDE_FFDC_ON_TEST_CASE_FAIL}
Return From Keyword If ${OVERRIDE_FFDC_ON_TEST_CASE_FAIL}
- Run Keyword If '${TEST_STATUS}' == 'FAIL' FFDC
+ Run Keyword If '${TEST_STATUS}' == 'FAIL' Launch FFDC
Log Test Case Status
+
+
+Launch FFDC
+ [Documentation] Call point to call FFDC robot or FFDC script.
+ ... FFDC_DEFAULT set to 1, by default, in resource.robot
+ ... FFDC_DEFAULT:1 use legacy ffdc collector
+ ... FFDC_DEFAULT:0 use new ffdc collector.
+
+ Run Keyword If ${FFDC_DEFAULT} == ${1} FFDC # Keyword from openbmc_ffdc.py
+ ... ELSE ffdc_robot_script_cli # Keyword from ffdc_cli_robot_script.py
+
diff --git a/lib/resource.robot b/lib/resource.robot
index b3ad7ee..f5d0d81 100755
--- a/lib/resource.robot
+++ b/lib/resource.robot
@@ -142,6 +142,10 @@
${LDAP_USER} ${EMPTY}
${LDAP_USER_PASSWORD} ${EMPTY}
+# General tool variables
+# FFDC_DEFAULT == 1; use Default FFDC methods
+${FFDC_DEFAULT} ${1}
+
*** Keywords ***
Get Inventory Schema
[Documentation] Get inventory schema.
diff --git a/tools/myffdc.robot b/tools/myffdc.robot
index 77d5b78..1dff23b 100755
--- a/tools/myffdc.robot
+++ b/tools/myffdc.robot
@@ -42,5 +42,8 @@
** Keywords ***
Gather FFDC
- [Documentation] Collect FFDC.
- Run Keyword And Ignore Error FFDC
+ [Documentation] Call point to call FFDC robot or FFDC script..
+ Run Keyword If ${FFDC_DEFAULT} == ${1} FFDC # Keyword from openbmc_ffdc.py
+ ... ELSE ffdc_robot_script_cli # Keyword from ffdc_cli_robot_script.py
+
+