ffdc: Add standalone FFDC collector script
- Set 1: 1st Pass: default to OpenBMC and SSH.
- Set 2: Remove trailing spaces.
- Set 3: Remove trailing spaces - part 2.
- Set 4: Address pycodestyle,
openbmc Python Coding Guidelines and review feedback issues.
- Set 5: more correction to meet pycodestyle.
- Set 6: Address review comments. Test scenarios were (re)executed.
- Set 7: Relocate all FFDC Collector files in ffdc/ dir.
Test scenarios were (re)executed with newly relocated files.
- Set 8: Make script progress, status more informative.
Add error handling, making error message more informative.
Add test for error scenarios.
Tested: 7 cli scenarios
(1) python3 <sandbox>/openbmc-test-automation/ffdc/collect_ffdc.py \
-h <> -u <> -p <> -f <sandbox>/openbmc-test-automation/ffdc/ffdc_config.yaml
(2) cd <sandbox>/openbmc-test-automation/ffdc
python3 collect_ffdc.py -h <> -u <> -p <>
(3) export OPENBMC_HOST=<>
export OPENBMC_USERNAME=<>
export OPENBMC_PASSWORD=<>
cd <sandbox>/openbmc-test-automation/ffdc
python3 collect_ffdc.py
(4) python3 collect_ffdc.py -h <invalid host>
(5) python3 collect_ffdc.py -u <invalid userid>
(6) python3 collect_ffdc.py -p <invalid password>
(7) python3 collect_ffdc.py -l <path with no access (/var, /opt)>
Change-Id: Ia26630168856341e749ce73b5cced831fc470219
Signed-off-by: Peter D Phan <peterp@us.ibm.com>
diff --git a/ffdc/collect_ffdc.py b/ffdc/collect_ffdc.py
new file mode 100644
index 0000000..a12c993
--- /dev/null
+++ b/ffdc/collect_ffdc.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+
+r"""
+
+CLI FFDC Collector.
+"""
+
+import os
+import sys
+import click
+
+# ---------Set sys.path for cli command execution---------------------------------------
+# Absolute path to openbmc-test-automation/ffdc
+full_path = os.path.abspath(os.path.dirname(sys.argv[0])).split('ffdc')[0]
+sys.path.append(full_path)
+# Walk path and append to sys.path
+for root, dirs, files in os.walk(full_path):
+ for found_dir in dirs:
+ sys.path.append(os.path.join(root, found_dir))
+
+from ffdc_collector import FFDCCollector
+
+
+@click.command()
+@click.option('-h', '--hostname', envvar='OPENBMC_HOST',
+ help="ip or hostname of the target [default: OPENBMC_HOST]")
+@click.option('-u', '--username', envvar='OPENBMC_USERNAME',
+ help="username on targeted system [default: OPENBMC_USERNAME]")
+@click.option('-p', '--password', envvar='OPENBMC_PASSWORD',
+ help="password for username on targeted system [default: OPENBMC_PASSWORD]")
+@click.option('-f', '--ffdc_config', default="ffdc_config.yaml",
+ show_default=True, help="YAML FFDC configuration file")
+@click.option('-l', '--location', default="/tmp",
+ show_default=True, help="Location to store collected FFDC data")
+def cli_ffdc(hostname, username, password, ffdc_config, location):
+ r"""
+ Stand alone CLI to generate and collect FFDC from the selected target.
+
+ Description of argument(s):
+
+ hostname Name/IP of the remote (targetting) host.
+ username User on the remote host with access to FFDC files.
+ password Password for user on remote host.
+ ffdc_config Configuration file listing commands and files for FFDC.
+ location Where to store collected FFDC.
+
+ """
+ click.echo("\n********** FFDC Starts **********")
+
+ thisFFDC = FFDCCollector(hostname, username, password, ffdc_config, location)
+ thisFFDC.collect_ffdc()
+
+ if not thisFFDC.receive_file_list:
+ click.echo("\n\tFFDC Collection from " + hostname + " has failed\n\n.")
+ else:
+ click.echo(str("\t" + str(len(thisFFDC.receive_file_list)))
+ + " files were retrieved from " + hostname)
+ click.echo("\tFiles are stored in " + thisFFDC.ffdc_dir_path + "\n\n")
+
+ click.echo("\n********** FFDC Finishes **********\n\n")
+
+
+if __name__ == '__main__':
+ cli_ffdc()
diff --git a/ffdc/ffdc_collector.py b/ffdc/ffdc_collector.py
new file mode 100644
index 0000000..925ccfd
--- /dev/null
+++ b/ffdc/ffdc_collector.py
@@ -0,0 +1,249 @@
+#!/usr/bin/env python
+
+r"""
+See class prolog below for details.
+"""
+
+import os
+import sys
+import yaml
+import time
+import platform
+from errno import EACCES, EPERM
+from ssh_utility import SSHRemoteclient
+
+
+class FFDCCollector:
+
+ r"""
+ Sends commands from configuration file to the targeted system to collect log files.
+ Fetch and store generated files at the specified location.
+
+ """
+
+ def __init__(self, hostname, username, password, ffdc_config, location):
+
+ r"""
+ Description of argument(s):
+
+ hostname name/ip of the targeted (remote) system
+ username user on the targeted system with access to FFDC files
+ password password for user on targeted system
+ ffdc_config configuration file listing commands and files for FFDC
+ location Where to store collected FFDC
+
+ """
+ if self.verify_script_env():
+ self.hostname = hostname
+ self.username = username
+ self.password = password
+ self.ffdc_config = ffdc_config
+ self.location = location
+ self.remote_client = None
+ self.ffdc_dir_path = ""
+ self.ffdc_prefix = ""
+ self.receive_file_list = []
+ self.target_type = ""
+ else:
+ raise EnvironmentError("Python or python packages do not meet minimum version requirement.")
+
+ def verify_script_env(self):
+
+ # Import to log version
+ import click
+ import paramiko
+
+ run_env_ok = True
+ print("\n\t---- Script host environment ----")
+ print("\t{:<10} {:<10}".format('Script hostname', os.uname()[1]))
+ print("\t{:<10} {:<10}".format('Script host os', platform.platform()))
+ print("\t{:<10} {:>10}".format('Python', platform.python_version()))
+ print("\t{:<10} {:>10}".format('PyYAML', yaml.__version__))
+ print("\t{:<10} {:>10}".format('click', click.__version__))
+ print("\t{:<10} {:>10}".format('paramiko', paramiko.__version__))
+
+ if eval(yaml.__version__.replace('.', ',')) < (5, 4):
+ print("\n\tERROR: PyYAML version 5.4 or higher is needed.")
+ run_env_ok = False
+
+ print("\t---- End script host environment ----")
+ return run_env_ok
+
+ def target_is_pingable(self):
+
+ r"""
+ Check if target system is ping-able.
+
+ """
+ response = os.system("ping -c 1 -w 2 %s 2>&1 >/dev/null" % self.hostname)
+ if response == 0:
+ print("\n\t%s is ping-able." % self.hostname)
+ return True
+ else:
+ print("\n>>>>>\tERROR: %s is not ping-able. FFDC collection aborted.\n" % self.hostname)
+ sys.exit(-1)
+
+ def set_target_machine_type(self):
+ r"""
+ Determine and set target machine type.
+
+ """
+ # Default to openbmc for first few sprints
+ self.target_type = "OPENBMC"
+
+ def collect_ffdc(self):
+
+ r"""
+ Initiate FFDC Collection depending on requested protocol.
+
+ """
+
+ print("\n\n\t---- Start communicating with %s ----\n" % self.hostname)
+ if self.target_is_pingable():
+ # Verify top level directory exists for storage
+ self.validate_local_store(self.location)
+ self.set_target_machine_type()
+ self.generate_ffdc()
+
+ def ssh_to_target_system(self):
+ r"""
+ Open a ssh connection to targeted system.
+
+ """
+
+ self.remoteclient = SSHRemoteclient(self.hostname,
+ self.username,
+ self.password)
+
+ self.remoteclient.ssh_remoteclient_login()
+
+ def generate_ffdc(self):
+
+ r"""
+ Send commands in ffdc_config file to targeted system.
+
+ """
+
+ with open(self.ffdc_config, 'r') as file:
+ ffdc_actions = yaml.load(file, Loader=yaml.FullLoader)
+
+ for machine_type in ffdc_actions.keys():
+ if machine_type == self.target_type:
+
+ if (ffdc_actions[machine_type]['PROTOCOL'][0] == 'SSH'):
+
+ # Use SSH
+ self.ssh_to_target_system()
+
+ print("\n\tCollecting FFDC on " + self.hostname)
+ list_of_commands = ffdc_actions[machine_type]['COMMANDS']
+ progress_counter = 0
+ for command in list_of_commands:
+ self.remoteclient.execute_command(command)
+ progress_counter += 1
+ self.print_progress(progress_counter)
+
+ print("\n\n\tCopying FFDC from remote system %s \n\n" % self.hostname)
+ # Get default values for scp action.
+ # self.location == local system for now
+ self.set_ffdc_defaults()
+ # Retrieving files from target system
+ list_of_files = ffdc_actions[machine_type]['FILES']
+ self.scp_ffdc(self.ffdc_dir_path, self.ffdc_prefix, list_of_files)
+ else:
+ print("\n\tProtocol %s is not yet supported by this script.\n"
+ % ffdc_actions[machine_type]['PROTOCOL'][0])
+
+ def scp_ffdc(self,
+ targ_dir_path,
+ targ_file_prefix="",
+ file_list=None,
+ quiet=None):
+
+ r"""
+ SCP all files in file_dict to the indicated directory on the local system.
+
+ Description of argument(s):
+ targ_dir_path The path of the directory to receive the files.
+ targ_file_prefix Prefix which will be pre-pended to each
+ target file's name.
+ file_dict A dictionary of files to scp from targeted system to this system
+
+ """
+
+ self.remoteclient.scp_connection()
+
+ self.receive_file_list = []
+ progress_counter = 0
+ for filename in file_list:
+ source_file_path = filename
+ targ_file_path = targ_dir_path + targ_file_prefix + filename.split('/')[-1]
+
+ # self.remoteclient.scp_file_from_remote() completed without exception,
+ # add file to the receiving file list.
+ scp_result = self.remoteclient.scp_file_from_remote(source_file_path, targ_file_path)
+ if scp_result:
+ self.receive_file_list.append(targ_file_path)
+
+ if not quiet:
+ if scp_result:
+ print("\t\tSuccessfully copy from " + self.hostname + ':' + source_file_path + ".\n")
+ else:
+ print("\t\tFail to copy from " + self.hostname + ':' + source_file_path + ".\n")
+ else:
+ progress_counter += 1
+ self.print_progress(progress_counter)
+
+ self.remoteclient.ssh_remoteclient_disconnect()
+
+ def set_ffdc_defaults(self):
+
+ r"""
+ Set a default value for self.ffdc_dir_path and self.ffdc_prefix.
+ Collected ffdc file will be stored in dir /self.location/hostname_timestr/.
+ Individual ffdc file will have timestr_filename.
+
+ Description of class variables:
+ self.ffdc_dir_path The dir path where collected ffdc data files should be put.
+
+ self.ffdc_prefix The prefix to be given to each ffdc file name.
+
+ """
+
+ timestr = time.strftime("%Y%m%d-%H%M%S")
+ self.ffdc_dir_path = self.location + "/" + self.hostname + "_" + timestr + "/"
+ self.ffdc_prefix = timestr + "_"
+ self.validate_local_store(self.ffdc_dir_path)
+
+ def validate_local_store(self, dir_path):
+ r"""
+ Ensure path exists to store FFDC files locally.
+
+ Description of variable:
+ dir_path The dir path where collected ffdc data files will be stored.
+
+ """
+
+ if not os.path.exists(dir_path):
+ try:
+ os.mkdir(dir_path, 0o755)
+ except (IOError, OSError) as e:
+ # PermissionError
+ if e.errno == EPERM or e.errno == EACCES:
+ print('>>>>>\tERROR: os.mkdir %s failed with PermissionError.\n' % dir_path)
+ else:
+ print('>>>>>\tERROR: os.mkdir %s failed with %s.\n' % (dir_path, e.strerror))
+ sys.exit(-1)
+
+ def print_progress(self, progress):
+ r"""
+ Print activity progress +
+
+ Description of variable:
+ progress Progress counter.
+
+ """
+
+ sys.stdout.write("\r\t" + "+" * progress)
+ sys.stdout.flush()
+ time.sleep(.1)
diff --git a/ffdc/ffdc_config.yaml b/ffdc/ffdc_config.yaml
new file mode 100644
index 0000000..55f9de2
--- /dev/null
+++ b/ffdc/ffdc_config.yaml
@@ -0,0 +1,119 @@
+---
+# This yaml formatted file contains defaults items to collect FFDC for the targeted system
+# COMMANDS - List of commands to be run on the corresponding targeted system (inband)
+# to generate and/or collect data
+# FILES - List of files; with complete path; on the corresponding targeted system
+# to be copied to external destination
+# PROTOCOL - Protocol used to communicate with targeted system; ssh, Rest, redfish, etc
+#
+# Note: Items in COMMANDS and FILES are not necessarily one-to-one correspondence.
+# For example, a file could have been created by an internal process,
+# and is listed in FILES to be collected.
+
+# Commands and Files to collect for a given OpenBMC system.
+OPENBMC:
+ COMMANDS:
+ - 'cat /sys/class/watchdog/watchdog1/bootstatus >/tmp/BMC_flash_side.txt 2>&1'
+ - 'grep -r . /sys/class/hwmon/* >/tmp/BMC_hwmon.txt 2>&1'
+ - 'top -n 1 -b >/tmp/BMC_proc_list.txt 2>&1'
+ - 'ls -Al /proc/*/fd/ >/tmp/BMC_proc_fd_active_list.txt 2>&1'
+ - 'journalctl --no-pager >/tmp/BMC_journalctl_nopager.txt 2>&1'
+ - 'journalctl -o json-pretty >/tmp/BMC_journalctl_pretty.json 2>&1'
+ - 'dmesg >/tmp/BMC_dmesg.txt 2>&1'
+ - 'cat /proc/cpuinfo >/tmp/BMC_procinfo.txt 2>&1'
+ - 'cat /proc/meminfo >/tmp/BMC_meminfo.txt 2>&1'
+ - 'systemctl status --all >/tmp/BMC_systemd.txt 2>&1'
+ - 'systemctl list-units --failed >/tmp/BMC_failed_service.txt 2>&1'
+ - 'systemctl list-jobs >/tmp/BMC_list_service.txt 2>&1'
+ - 'cat /var/log/obmc-console.log >/tmp/BMC_obmc_console.txt 2>&1'
+ - 'cat /var/log/obmc-console1.log >/tmp/BMC_obmc_console1.txt 2>&1'
+ - 'peltool -l >/tmp/PEL_logs_list.json 2>&1'
+ - 'peltool -a >/tmp/PEL_logs_display.json 2>&1'
+ FILES:
+ - '/tmp/BMC_flash_side.txt'
+ - '/tmp/BMC_hwmon.txt'
+ - '/tmp/BMC_proc_list.txt'
+ - '/tmp/BMC_proc_fd_active_list.txt'
+ - '/tmp/BMC_journalctl_nopager.txt'
+ - '/tmp/BMC_journalctl_pretty.json'
+ - '/tmp/BMC_dmesg.txt'
+ - '/tmp/BMC_procinfo.txt'
+ - '/tmp/BMC_meminfo.txt'
+ - '/tmp/BMC_systemd.txt'
+ - '/tmp/BMC_failed_service.txt'
+ - '/tmp/BMC_list_service.txt'
+ - '/tmp/BMC_obmc_console.txt'
+ - '/tmp/BMC_obmc_console1.txt'
+ - '/tmp/PEL_logs_list.json'
+ - '/tmp/PEL_logs_display.json'
+ PROTOCOL:
+ - 'SSH'
+
+# Commands and Files to collect for all Linux distributions
+LINUX:
+ COMMANDS:
+ - 'cat /sys/firmware/opal/msglog >/tmp/OS_msglog.txt 2>&1'
+ - 'ppc64_cpu --frequency >/tmp/OS_cpufrequency.txt 2>&1'
+ - 'dmesg >/tmp/OS_dmesg.txt 2>&1'
+ - 'cat /var/log/opal-prd* >/tmp/OS_opal_prd.txt 2>&1'
+ - 'cat /var/log/boot.log >/tmp/OS_boot.txt 2>&1'
+ - 'cat /proc/cpuinfo >/tmp/OS_procinfo.txt 2>&1'
+ - 'cat /proc/meminfo >/tmp/OS_meminfo.txt 2>&1'
+ - 'netstat -a >/tmp/OS_netstat.txt 2>&1'
+ - 'lspci >/tmp/OS_lspci.txt 2>&1'
+ - 'lscpu >/tmp/OS_lscpu.txt 2>&1'
+ - 'lscfg >/tmp/OS_lscfg.txt 2>&1'
+ - 'journalctl --no-pager -b > /tmp/OS_journalctl_nopager.txt 2>&1'
+ FILES:
+ - '/tmp/OS_msglog.txt'
+ - '/tmp/OS_cpufrequency.txt'
+ - '/tmp/OS_dmesg.txt'
+ - '/tmp/OS_opal_prd.txt'
+ - '/tmp/OS_boot.txt'
+ - '/tmp/OS_procinfo.txt'
+ - '/tmp/OS_meminfo.txt'
+ - '/tmp/OS_netstat.txt'
+ - '/tmp/OS_lspci.txt'
+ - '/tmp/OS_lscpu.txt'
+ - '/tmp/OS_lscfg.txt'
+ - '/tmp/OS_journalctl_nopager.txt'
+ PROTOCOL:
+ - 'SSH'
+
+# Commands and Files to collect for Ubuntu Linux only
+UBUNTU_LINUX:
+ COMMANDS:
+ - '{ lsusb -t ; lsusb -v ; } >/tmp/OS_isub.txt 2>&1'
+ - 'tail -n 50000 /var/log/kern.log >/tmp/OS_kern.txt 2>&1'
+ - '{ cat /var/log/auth.log; cat /var/log/auth.log.1 ; } >/tmp/OS_authlog.txt 2>&1'
+ - 'tail -n 200000 /var/log/syslog >/tmp/OS_syslog.txt 2>&1'
+ - '{ uname -a; dpkg -s opal-prd; dpkg -s ipmitool ; } >/tmp/OS_info.txt 2>&1'
+ - 'rm -rf /tmp/sosreport*FFDC*'
+ - 'sosreport --batch --tmp-dir /tmp --ticket-number FFDC >/tmp/OS_sosreport.txt 2>&1'
+ FILES:
+ - '/tmp/OS_isub.txt'
+ - '/tmp/OS_kern.txt'
+ - '/tmp/OS_authlog.txt'
+ - '/tmp/OS_syslog.txt'
+ - '/tmp/OS_info.txt'
+ - '/tmp/OS_sosreport.txt'
+ PROTOCOL:
+ - 'SSH'
+
+# Commands and Files to collect for RHE Linux only
+RHEL_LINUX:
+ COMMANDS:
+ - '/usr/bin/ctversion -bv >/tmp/OS_rsct.txt 2>&1'
+ - 'cat /var/log/secure >/tmp/OS_secure.txt 2>&1'
+ - 'tail -n 200000 /var/log/messages >/tmp/OS_syslog.txt 2>&1'
+ - '{ lsb_release -a; cat /etc/redhat-release; uname -a; rpm -qa ; } >/tmp/OS_info.txt 2>&1'
+ - 'rm -rf /tmp/sosreport*FFDC*'
+ - 'sosreport --batch --tmp-dir /tmp --label FFDC >/tmp/OS_sosreport.txt 2>&1'
+ FILES:
+ - '/tmp/OS_rsct.txt'
+ - '/tmp/OS_secure.txt'
+ - '/tmp/OS_syslog.txt'
+ - '/tmp/OS_info.txt'
+ - '/tmp/OS_sosreport.txt'
+ PROTOCOL:
+ - 'SSH'
diff --git a/ffdc/setup.py b/ffdc/setup.py
new file mode 100644
index 0000000..9f06bc0
--- /dev/null
+++ b/ffdc/setup.py
@@ -0,0 +1,15 @@
+from setuptools import setup
+setup(
+ name='ffdc',
+ version='0.1',
+ description=("A standalone script to collect logs from a given system."),
+ py_modules=['install'],
+ install_requires=[
+ 'click',
+ 'PyYAML',
+ 'paramiko',
+ ],
+ entry_points={
+ 'console_scripts': ['collectFFDC=commands.install_cmd:main']
+ }
+)
diff --git a/ffdc/ssh_utility.py b/ffdc/ssh_utility.py
new file mode 100644
index 0000000..51e8209
--- /dev/null
+++ b/ffdc/ssh_utility.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+
+import paramiko
+from paramiko.ssh_exception import AuthenticationException, NoValidConnectionsError, SSHException
+from paramiko.buffered_pipe import PipeTimeout as PipeTimeout
+import scp
+import socket
+from socket import timeout as SocketTimeout
+
+
+class SSHRemoteclient:
+ r"""
+ Class to create ssh connection to remote host
+ for remote host command execution and scp.
+ """
+
+ def __init__(self, hostname, username, password):
+
+ r"""
+ Description of argument(s):
+
+ hostname name/ip of the remote (targetting) host
+ username user on the remote host with access to FFCD files
+ password password for user on remote host
+ """
+
+ self.ssh_output = None
+ self.ssh_error = None
+ self.sshclient = None
+ self.scpclient = None
+ self.hostname = hostname
+ self.username = username
+ self.password = password
+
+ def ssh_remoteclient_login(self):
+
+ r"""
+ Method to create a ssh connection to remote host.
+ """
+
+ try:
+ # SSHClient to make connections to the remote server
+ self.sshclient = paramiko.SSHClient()
+ # setting set_missing_host_key_policy() to allow any host.
+ self.sshclient.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+ # Connect to the server
+ self.sshclient.connect(hostname=self.hostname,
+ username=self.username,
+ password=self.password)
+
+ except AuthenticationException:
+ print("\n>>>>>\tERROR: Authentication failed, please verify your login credentials")
+ except SSHException:
+ print("\n>>>>>\tERROR: Failures in SSH2 protocol negotiation or logic errors.")
+ except NoValidConnectionsError:
+ print('\n>>>>>\tERROR: No Valid SSH Connection after multiple attempts.')
+ except socket.error:
+ print("\n>>>>>\tERROR: SSH Connection refused.")
+ except Exception:
+ raise Exception("\n>>>>>\tERROR: Unexpected Exception.")
+
+ def ssh_remoteclient_disconnect(self):
+
+ r"""
+ Clean up.
+ """
+
+ if self.sshclient:
+ self.sshclient.close()
+
+ if self.scpclient:
+ self.scpclient.close()
+
+ def execute_command(self, command):
+ """
+ Execute command on the remote host.
+
+ Description of argument(s):
+ command Command string sent to remote host
+
+ """
+
+ try:
+ stdin, stdout, stderr = self.sshclient.exec_command(command)
+ stdout.channel.recv_exit_status()
+ response = stdout.readlines()
+ return response
+ except (paramiko.AuthenticationException, paramiko.SSHException,
+ paramiko.ChannelException) as ex:
+ print("\n>>>>>\tERROR: Remote command execution fails.")
+
+ def scp_connection(self):
+
+ r"""
+ Create a scp connection for file transfer.
+ """
+ self.scpclient = scp.SCPClient(self.sshclient.get_transport())
+
+ def scp_file_from_remote(self, remote_file, local_file):
+
+ r"""
+ scp file in remote system to local with date-prefixed filename.
+
+ Description of argument(s):
+ remote_file Full path filename on the remote host
+
+ local_file Full path filename on the local host
+ local filename = date-time_remote filename
+
+ """
+
+ try:
+ self.scpclient.get(remote_file, local_file)
+ except scp.SCPException:
+ print("scp.SCPException scp %s from remotehost" % remote_file)
+ return False
+ except (SocketTimeout, PipeTimeout) as ex:
+ # Future enhancement: multiple retries on these exceptions due to bad ssh connection
+ print("Timeout scp %s from remotehost" % remote_file)
+ return False
+
+ # Return True for file accounting
+ return True