ffdc: Integrate ffdc script into robot framework flow.

This commit provides interface to enable automation tests (robot/jenkins)
to activate the new cli ffdc collector.

- Patchset 1: Initial commit for review.
- Patchset 2: Redesing and fixing issues
- Patchset 3: Fix build issue (pycodestyle)
- Patchset 4: Default ffdc store location running direct CLI
- Patchset 5: Addressing review comments and questions
- Patchset 6: Restore direct CLI default location to /tmp
- Patchset 7: Addressing issues from comments
              Also extend socket timeout for long-running command peltool -a
- Patchset 8: Use Builtin().log_to_console("") method in the bridge code
- Patchset 9: Rebase

Tests:
- Standalone script: python3 ./ffdc/collect_ffdc.py
- Within robot env: python3 -m robot .... -v FFDC_DEFAULT:0 ./tools/myffdc.robot

Signed-off-by: Peter D  Phan <peterp@us.ibm.com>
Change-Id: Ic892c48efe7dbdb3a4345ba89c3a0257f03a7ffb
diff --git a/ffdc/collect_ffdc.py b/ffdc/collect_ffdc.py
index 7711fd6..c1c9495 100644
--- a/ffdc/collect_ffdc.py
+++ b/ffdc/collect_ffdc.py
@@ -18,7 +18,7 @@
     for found_dir in dirs:
         sys.path.append(os.path.join(root, found_dir))
 
-from ffdc_collector import FFDCCollector
+from ffdc_collector import ffdc_collector
 
 
 @click.command(context_settings=dict(help_option_names=['-h', '--help']))
@@ -61,26 +61,26 @@
     click.echo("\n********** FFDC (First Failure Data Collection) Starts **********")
 
     if input_options_ok(remote, username, password, config, type):
-        thisFFDC = FFDCCollector(remote,
-                                 username,
-                                 password,
-                                 config,
-                                 location,
-                                 type,
-                                 protocol,
-                                 env_vars,
-                                 econfig,
-                                 log_level)
-        thisFFDC.collect_ffdc()
+        this_ffdc = ffdc_collector(remote,
+                                   username,
+                                   password,
+                                   config,
+                                   location,
+                                   type,
+                                   protocol,
+                                   env_vars,
+                                   econfig,
+                                   log_level)
+        this_ffdc.collect_ffdc()
 
-        if len(os.listdir(thisFFDC.ffdc_dir_path)) == 0:
+        if len(os.listdir(this_ffdc.ffdc_dir_path)) == 0:
             click.echo("\n\tFFDC Collection from " + remote + " has failed.\n\n")
         else:
-            click.echo(str("\n\t" + str(len(os.listdir(thisFFDC.ffdc_dir_path)))
+            click.echo(str("\n\t" + str(len(os.listdir(this_ffdc.ffdc_dir_path)))
                            + " files were retrieved from " + remote))
-            click.echo("\tFiles are stored in " + thisFFDC.ffdc_dir_path)
+            click.echo("\tFiles are stored in " + this_ffdc.ffdc_dir_path)
 
-        click.echo("\tTotal elapsed time " + thisFFDC.elapsed_time + "\n\n")
+        click.echo("\tTotal elapsed time " + this_ffdc.elapsed_time + "\n\n")
     click.echo("\n********** FFDC Finishes **********\n\n")
 
 
diff --git a/ffdc/ffdc_collector.py b/ffdc/ffdc_collector.py
index 4a8cfa4..61590a3 100644
--- a/ffdc/ffdc_collector.py
+++ b/ffdc/ffdc_collector.py
@@ -14,6 +14,8 @@
 import platform
 from errno import EACCES, EPERM
 import subprocess
+
+sys.path.extend([f'./{name[0]}' for name in os.walk(".") if os.path.isdir(name[0])])
 from ssh_utility import SSHRemoteclient
 from telnet_utility import TelnetRemoteclient
 
@@ -33,8 +35,8 @@
        - arg1
        - arg2
 """
-abs_path = os.path.abspath(os.path.dirname(sys.argv[0]))
-plugin_dir = abs_path + '/plugins'
+plugin_dir = __file__.split(__file__.split("/")[-1])[0] + '/plugins'
+sys.path.append(plugin_dir)
 try:
     for module in os.listdir(plugin_dir):
         if module == '__init__.py' or module[-3:] != '.py':
@@ -98,7 +100,7 @@
 }
 
 
-class FFDCCollector:
+class ffdc_collector:
 
     r"""
     Execute commands from configuration file to collect log files.
@@ -154,9 +156,9 @@
         # to be sure that all files for this run will have same timestamps
         # and they will be saved in the same directory.
         # self.location == local system for now
-        self.set_ffdc_defaults()
+        self.set_ffdc_default_store_path()
 
-        # Logger for this run.  Need to be after set_ffdc_defaults()
+        # Logger for this run.  Need to be after set_ffdc_default_store_path()
         self.script_logging(getattr(logging, log_level.upper()))
 
         # Verify top level directory exists for storage
@@ -181,7 +183,7 @@
         # Load ENV vars from user.
         self.logger.info("\n\tENV: User define input YAML variables")
         self.env_dict = {}
-        self. load_env()
+        self.load_env()
 
     def verify_script_env(self):
 
@@ -686,7 +688,7 @@
                 progress_counter += 1
                 self.print_progress(progress_counter)
 
-    def set_ffdc_defaults(self):
+    def set_ffdc_default_store_path(self):
         r"""
         Set a default value for self.ffdc_dir_path and self.ffdc_prefix.
         Collected ffdc file will be stored in dir /self.location/hostname_timestr/.
@@ -704,7 +706,11 @@
         self.ffdc_prefix = timestr + "_"
         self.validate_local_store(self.ffdc_dir_path)
 
-    def validate_local_store(self, dir_path):
+    # Need to verify local store path exists prior to instantiate this class.
+    # This class method is used to share the same code between CLI input parm
+    # and Robot Framework "${EXECDIR}/logs" before referencing this class.
+    @classmethod
+    def validate_local_store(cls, dir_path):
         r"""
         Ensure path exists to store FFDC files locally.
 
@@ -916,7 +922,7 @@
 
     def execute_plugin_block(self, plugin_cmd_list):
         r"""
-        Pack the plugin command to quailifed python string object.
+        Pack the plugin command to qualifed python string object.
 
         Description of argument(s):
         plugin_list_dict      Plugin block read from YAML
diff --git a/ffdc/ffdc_config.yaml b/ffdc/ffdc_config.yaml
index 1f7e041..b2bd403 100644
--- a/ffdc/ffdc_config.yaml
+++ b/ffdc/ffdc_config.yaml
@@ -61,7 +61,7 @@
             - 'cat /var/log/obmc-console.log >/tmp/BMC_obmc_console.txt'
             - 'cat /var/log/obmc-console1.log >/tmp/BMC_obmc_console1.txt'
             - 'peltool -l >/tmp/PEL_logs_list.json'
-            - 'peltool -a >/tmp/PEL_logs_display.json'
+            - {'peltool -a >/tmp/PEL_logs_display.json':240}
             - 'hexdump -C /var/lib/phosphor-logging/extensions/pels/badPEL >/tmp/PEL_logs_badPEL.txt'
             - 'guard -l >/tmp/GUARD_list.txt'
             - 'killall -s SIGUSR1 pldmd; sleep 5'
diff --git a/ffdc/plugins/redfish.py b/ffdc/plugins/redfish.py
index 33a0669..0ea7a00 100644
--- a/ffdc/plugins/redfish.py
+++ b/ffdc/plugins/redfish.py
@@ -82,7 +82,7 @@
             response = execute_redfish_cmd(parms + resource)
             # Enumeration is done for available resources ignoring the
             # ones for which response is not obtained.
-            if 'Response Error' in response:
+            if 'Error getting response' in response:
                 continue
 
             walk_nested_dict(response, url=resource)
diff --git a/lib/ffdc_cli_robot_script.py b/lib/ffdc_cli_robot_script.py
new file mode 100644
index 0000000..6780dd5
--- /dev/null
+++ b/lib/ffdc_cli_robot_script.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+
+sys.path.append(__file__.split(__file__.split("/")[-1])[0] + "../ffdc")
+from ffdc_collector import ffdc_collector
+
+from robot.libraries.BuiltIn import BuiltIn as robotBuildIn
+
+# (Sub) String constants used for input dictionary key search
+HOST = "HOST"
+USER = "USERNAME"
+PASSWD = "PASSWORD"
+CONFIG = "CONFIG"
+TYPE = "TYPE"
+LOC = "LOCATION"
+PROTOCOL = "PROTOCOL"
+ENV_VARS = "ENV_VARS"
+ECONFIG = "ECONFIG"
+LOGLEVEL = "LOG"
+
+
+def ffdc_robot_script_cli(**kwargs):
+    r"""
+
+    For the specified host, this method provide automation testcases the interface to
+    the new ffdc collector ../ffdc/ffdc_collector.py via robot variable FFDC_DEFAULT
+
+    variable FFDC_DEFAULT:1, by default use the existing ffdc collection method.
+    variable FFDC_DEFAULT:0 use the new ffdc method
+
+    Command examples:
+    (1) Legacy ffdc collection
+    python3 -m robot -v OPENBMC_HOST:<> -v OPENBMC_USERNAME:<> \
+                                        -v OPENBMC_PASSWORD:<> ./tools/myffdc.robot
+    (2) New ffdc collection
+    python3 -m robot -v OPENBMC_HOST:<> -v OPENBMC_USERNAME:<> \
+                        -v OPENBMC_PASSWORD:<> -v FFDC_DEFAULT:0  ./tools/myffdc.robot
+
+    Description of argument(s)in dictionary: xx can be anything appropriate
+
+        xx_HOST:hostname                name/ip of the targeted (remote) system
+        xx_USERNAME:username            user on the targeted system with access to FFDC files
+        xx_PASSWORD:password            password for user on targeted system
+        xx_CONFIG:ffdc_config           configuration file listing commands and files for FFDC
+        xx_LOCATION:location            where to store collected FFDC.  Default: <current dir>/logs/
+        xx_TYPE:remote_type             os type of the remote host.
+        xx_PROTOCOL:remote_protocol     Protocol to use to collect data. Default: 'ALL'
+        ENV_VAR:env_vars                User define CLI env vars '{"key : "value"}'. Default: ""
+        ECONFIG:econfig                 User define env vars YAML file. Default: ""
+        LOG_LEVEL:log_level             CRITICAL, ERROR, WARNING, INFO, DEBUG. Default: INFO
+
+    Code examples:
+    (1) openbmc_ffdc.robot activate this method with no parm
+        Run Keyword If  ${FFDC_DEFAULT} == ${1}  FFDC
+    ...    ELSE  ffdc_robot_script_cli
+
+    (2) Method invocation with parms
+        ffdc_from = {'OS_HOST' : 'os host name or ip',
+                     'OS_USERNAME' : 'os username',
+                     'OS_PASSWORD' : 'password for os_username',
+                     'OS_TYPE'     : 'os_type, ubuntu, rhel, aix, etc',
+                    }
+        ffdc_robot_script_cli(ffdc_from)
+
+    """
+
+    robotBuildIn().log_to_console("Collecting FFDC - CLI log collector script")
+
+    if not kwargs:
+        dict_of_parms = {}
+        # When method is invoked with no parm,
+        # use robot variables
+        # OPENBMC_HOST, OPENBMC_USERNAME, OPENBMC_PASSWORD, OPENBMC (type)
+        dict_of_parms["OPENBMC_HOST"] = \
+            robotBuildIn().get_variable_value("${OPENBMC_HOST}", default=None)
+        dict_of_parms["OPENBMC_USERNAME"] = \
+            robotBuildIn().get_variable_value("${OPENBMC_USERNAME}", default=None)
+        dict_of_parms["OPENBMC_PASSWORD"] = \
+            robotBuildIn().get_variable_value("${OPENBMC_PASSWORD}", default=None)
+        dict_of_parms["REMOTE_TYPE"] = "OPENBMC"
+
+        run_ffdc_collector(dict_of_parms)
+
+    else:
+        if isinstance(kwargs, dict):
+            # When method is invoked with user defined dictionary,
+            # dictionary keys has the following format
+            # xx_HOST; xx_USERNAME, xx_PASSWORD, xx_TYPE
+            # where xx is one of OPENBMC, OS, or os_type LINUX/UBUNTU/AIX
+            run_ffdc_collector(**kwargs)
+
+
+def run_ffdc_collector(dict_of_parm):
+    r"""
+
+    Process input parameters and collect information
+
+    Description of argument(s)in dictionary: xx can be anything appropriate
+
+        xx_HOST:hostname                name/ip of the targeted (remote) system
+        xx_USERNAME:username            user on the targeted system with access to FFDC files
+        xx_PASSWORD:password            password for user on targeted system
+        xx_CONFIG:ffdc_config           configuration file listing commands and files for FFDC
+        xx_LOCATION:location            where to store collected FFDC.  Default: <current dir>/logs/
+        xx_TYPE:remote_type             os type of the remote host.
+        xx_PROTOCOL:remote_protocol     Protocol to use to collect data. Default: 'ALL'
+        ENV_VAR:env_vars                User define CLI env vars '{"key : "value"}'. Default: ""
+        ECONFIG:econfig                 User define env vars YAML file. Default: ""
+        LOG_LEVEL:log_level             CRITICAL, ERROR, WARNING, INFO, DEBUG. Default: INFO
+
+    """
+
+    # Clear local variables
+    remote = None
+    username = None
+    password = None
+    config = None
+    location = None
+    remote_type = None
+    protocol = None
+    env_vars = None
+    econfig = None
+    log_level = None
+
+    # Process input key/value pairs
+    for key in dict_of_parm.keys():
+        if HOST in key:
+            remote = dict_of_parm[key]
+        elif USER in key:
+            username = dict_of_parm[key]
+        elif PASSWD in key:
+            password = dict_of_parm[key]
+        elif CONFIG in key:
+            config = dict_of_parm[key]
+        elif LOC in key:
+            location = dict_of_parm[key]
+        elif TYPE in key:
+            remote_type = dict_of_parm[key]
+        elif PROTOCOL in key:
+            protocol = dict_of_parm[key]
+        elif ENV_VARS in key:
+            env_vars = dict_of_parm[key]
+        elif ECONFIG in key:
+            econfig = dict_of_parm[key]
+        elif LOGLEVEL in key:
+            log_level = dict_of_parm[key]
+
+    # Set defaults values for parms
+    # that are not specified with input and have acceptable defaults.
+    if not location:
+        # Default FFDC store location
+        location = robotBuildIn().get_variable_value("${EXECDIR}", default=None) + "/logs"
+        ffdc_collector.validate_local_store(location)
+
+    if not config:
+        # Default FFDC configuration
+        script_path = os.path.dirname(os.path.abspath(__file__))
+        config = script_path + "/../ffdc/ffdc_config.yaml"
+
+    if not protocol:
+        protocol = "ALL"
+
+    if not env_vars:
+        env_vars = ""
+
+    if not econfig:
+        econfig = ""
+
+    if not log_level:
+        log_level = "INFO"
+
+    # If minimum required inputs are met, go collect.
+    if (remote and username and password and remote_type):
+        # Execute data collection
+        this_ffdc = ffdc_collector(remote,
+                                   username,
+                                   password,
+                                   config,
+                                   location,
+                                   remote_type,
+                                   protocol,
+                                   env_vars,
+                                   econfig,
+                                   log_level)
+        this_ffdc.collect_ffdc()
diff --git a/lib/openbmc_ffdc.robot b/lib/openbmc_ffdc.robot
index de11927..f2dfc8f 100755
--- a/lib/openbmc_ffdc.robot
+++ b/lib/openbmc_ffdc.robot
@@ -33,6 +33,7 @@
 Resource           openbmc_ffdc_utils.robot
 Resource           dump_utils.robot
 Library            openbmc_ffdc.py
+Library            ffdc_cli_robot_script.py
 
 *** Keywords ***
 
@@ -51,6 +52,17 @@
     ${OVERRIDE_FFDC_ON_TEST_CASE_FAIL}=  Convert To Integer  ${OVERRIDE_FFDC_ON_TEST_CASE_FAIL}
     Return From Keyword If  ${OVERRIDE_FFDC_ON_TEST_CASE_FAIL}
 
-    Run Keyword If  '${TEST_STATUS}' == 'FAIL'  FFDC
+    Run Keyword If  '${TEST_STATUS}' == 'FAIL'  Launch FFDC
 
     Log Test Case Status
+
+
+Launch FFDC
+    [Documentation]  Call point to call FFDC robot or FFDC script.
+    ...              FFDC_DEFAULT set to 1, by default, in resource.robot
+    ...              FFDC_DEFAULT:1  use legacy ffdc collector
+    ...              FFDC_DEFAULT:0  use new ffdc collector.
+
+    Run Keyword If  ${FFDC_DEFAULT} == ${1}  FFDC    # Keyword from openbmc_ffdc.py
+    ...    ELSE  ffdc_robot_script_cli               # Keyword from ffdc_cli_robot_script.py
+
diff --git a/lib/resource.robot b/lib/resource.robot
index b3ad7ee..f5d0d81 100755
--- a/lib/resource.robot
+++ b/lib/resource.robot
@@ -142,6 +142,10 @@
 ${LDAP_USER}                ${EMPTY}
 ${LDAP_USER_PASSWORD}       ${EMPTY}
 
+# General tool variables
+# FFDC_DEFAULT == 1; use Default FFDC methods
+${FFDC_DEFAULT}            ${1}
+
 *** Keywords ***
 Get Inventory Schema
     [Documentation]  Get inventory schema.
diff --git a/tools/myffdc.robot b/tools/myffdc.robot
index 77d5b78..1dff23b 100755
--- a/tools/myffdc.robot
+++ b/tools/myffdc.robot
@@ -42,5 +42,8 @@
 ** Keywords ***
 
 Gather FFDC
-    [Documentation]  Collect FFDC.
-    Run Keyword And Ignore Error   FFDC
+    [Documentation]  Call point to call FFDC robot or FFDC script..
+    Run Keyword If  ${FFDC_DEFAULT} == ${1}  FFDC    # Keyword from openbmc_ffdc.py
+    ...    ELSE  ffdc_robot_script_cli               # Keyword from ffdc_cli_robot_script.py
+
+