ffdc: Use paramiko non blocking APIs
- Set 1: Use paramiko non blocking APIs to better manage user input timeout values.
Remove duplicate entries in OPENBMC DUMP_LOGS.
- Set 2: Restore OPENBMC:DUMP_LOGS:COMMANDS with new layout design.
- Set 3: Restore 1 second delay.
- Set 4: Additional info for Timeout Error message.
- Set 5: Restore standalone LINUX block. Keep UBUNTU and RHEL as they were.
Tests:
- Set 1: Regression tests to all supported remote systems.
- Set 2: Regression tests to all supported remote systems.
- Set 3: Regression tests to all supported remote systems.
- Set 4: Regression tests to all supported remote systems.
Signed-off-by: Peter D Phan <peterp@us.ibm.com>
Change-Id: I3fafaefcb58dba500755f914facd95ce08095e32
diff --git a/ffdc/ffdc_collector.py b/ffdc/ffdc_collector.py
index 3657136..3ef0072 100644
--- a/ffdc/ffdc_collector.py
+++ b/ffdc/ffdc_collector.py
@@ -197,11 +197,13 @@
and self.remote_protocol != 'ALL':
continue
- if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'SSH':
- if 'SSH' in working_protocol_list:
+ if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'SSH' \
+ or ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'SCP':
+ if 'SSH' in working_protocol_list \
+ or 'SCP' in working_protocol_list:
self.protocol_ssh(ffdc_actions, machine_type, k)
else:
- print("\n\tERROR: SSH is not available for %s." % self.hostname)
+ print("\n\tERROR: SSH or SCP is not available for %s." % self.hostname)
if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'REDFISH':
if 'REDFISH' in working_protocol_list:
@@ -261,7 +263,7 @@
result = self.run_redfishtool(redfish_parm)
if result:
try:
- targ_file = ffdc_actions[machine_type][sub_type]['FILES'][index]
+ targ_file = self.get_file_list(ffdc_actions[machine_type][sub_type])[index]
except IndexError:
targ_file = each_url.split('/')[-1]
print("\n\t[WARN] Missing filename to store data from redfish URL %s." % each_url)
@@ -301,7 +303,7 @@
print("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'IPMI'))
ipmi_files_saved = []
progress_counter = 0
- list_of_cmd = ffdc_actions[machine_type][sub_type]['COMMANDS']
+ list_of_cmd = self.get_command_list(ffdc_actions[machine_type][sub_type])
for index, each_cmd in enumerate(list_of_cmd, start=0):
ipmi_parm = '-U ' + self.username + ' -P ' + self.password + ' -H ' \
+ self.hostname + ' ' + each_cmd
@@ -309,7 +311,7 @@
result = self.run_ipmitool(ipmi_parm)
if result:
try:
- targ_file = ffdc_actions[machine_type][sub_type]['FILES'][index]
+ targ_file = self.get_file_list(ffdc_actions[machine_type][sub_type])[index]
except IndexError:
targ_file = each_cmd.split('/')[-1]
print("\n\t[WARN] Missing filename to store data from IPMI %s." % each_cmd)
@@ -353,11 +355,39 @@
print("\n\n\tCopying FFDC files from remote system %s.\n" % self.hostname)
# Retrieving files from target system
- list_of_files = ffdc_actions_for_machine_type['FILES']
+ list_of_files = self.get_file_list(ffdc_actions_for_machine_type)
self.scp_ffdc(self.ffdc_dir_path, self.ffdc_prefix, form_filename, list_of_files)
else:
print("\n\n\tSkip copying FFDC files from remote system %s.\n" % self.hostname)
+ def get_command_list(self,
+ ffdc_actions_for_machine_type):
+ r"""
+ Fetch list of commands from configuration file
+
+ Description of argument(s):
+ ffdc_actions_for_machine_type commands and files for the selected remote host type.
+ """
+ try:
+ list_of_commands = ffdc_actions_for_machine_type['COMMANDS']
+ except KeyError:
+ list_of_commands = []
+ return list_of_commands
+
+ def get_file_list(self,
+ ffdc_actions_for_machine_type):
+ r"""
+ Fetch list of commands from configuration file
+
+ Description of argument(s):
+ ffdc_actions_for_machine_type commands and files for the selected remote host type.
+ """
+ try:
+ list_of_files = ffdc_actions_for_machine_type['FILES']
+ except KeyError:
+ list_of_files = []
+ return list_of_files
+
def ssh_execute_ffdc_commands(self,
ffdc_actions_for_machine_type,
form_filename=False):
@@ -370,8 +400,8 @@
"""
print("\n\t[Run] Executing commands on %s using %s"
% (self.hostname, ffdc_actions_for_machine_type['PROTOCOL'][0]))
- list_of_commands = ffdc_actions_for_machine_type['COMMANDS']
+ list_of_commands = self.get_command_list(ffdc_actions_for_machine_type)
# If command list is empty, returns
if not list_of_commands:
return
@@ -389,7 +419,8 @@
if form_filename:
command_txt = str(command_txt % self.target_type)
- self.remoteclient.execute_command(command_txt, command_timeout)
+ err, response = self.remoteclient.execute_command(command_txt, command_timeout)
+
progress_counter += 1
self.print_progress(progress_counter)
@@ -404,26 +435,29 @@
ffdc_actions_for_machine_type commands and files for the selected remote host type.
"""
- # Executing commands, if any
- self.ssh_execute_ffdc_commands(ffdc_actions_for_machine_type)
-
if self.remoteclient.scpclient:
print("\n\tCopying DUMP files from remote system %s.\n" % self.hostname)
- # Retrieving files from target system, if any
- list_of_files = ffdc_actions_for_machine_type['FILES']
+ list_of_commands = self.get_command_list(ffdc_actions_for_machine_type)
+ # If command list is empty, returns
+ if not list_of_commands:
+ return
- for filename in list_of_files:
- command = 'ls -AX ' + filename
- response = self.remoteclient.execute_command(command)
- # self.remoteclient.scp_file_from_remote() completed without exception,
- # if any
+ for command in list_of_commands:
+ try:
+ filename = command.split(' ')[2]
+ except IndexError:
+ print("\t\tInvalid command %s for DUMP_LOGS block." % command)
+ continue
+
+ err, response = self.remoteclient.execute_command(command)
+
if response:
scp_result = self.remoteclient.scp_file_from_remote(filename, self.ffdc_dir_path)
if scp_result:
print("\t\tSuccessfully copied from " + self.hostname + ':' + filename)
else:
- print("\t\tThere is no " + filename)
+ print("\t\tThere is no " + filename)
else:
print("\n\n\tSkip copying files from remote system %s.\n" % self.hostname)
@@ -452,9 +486,11 @@
source_file_path = filename
targ_file_path = targ_dir_path + targ_file_prefix + filename.split('/')[-1]
- # self.remoteclient.scp_file_from_remote() completed without exception,
- # add file to the receiving file list.
- scp_result = self.remoteclient.scp_file_from_remote(source_file_path, targ_file_path)
+ # If source file name contains wild card, copy filename as is.
+ if '*' in source_file_path:
+ scp_result = self.remoteclient.scp_file_from_remote(source_file_path, self.ffdc_dir_path)
+ else:
+ scp_result = self.remoteclient.scp_file_from_remote(source_file_path, targ_file_path)
if not quiet:
if scp_result:
diff --git a/ffdc/ffdc_config.yaml b/ffdc/ffdc_config.yaml
index 6f5497a..80ee559 100644
--- a/ffdc/ffdc_config.yaml
+++ b/ffdc/ffdc_config.yaml
@@ -10,8 +10,6 @@
# For example, a file could have been created by an internal process,
# and is listed in FILES to be collected.
#
-# Note: When a new remote type is added to this configuration file,
-# it is also need to be added the list of supported OSes in ffdc_collector.py
# Commands and Files to collect for a given OpenBMC system.
OPENBMC:
@@ -82,7 +80,9 @@
PROTOCOL:
- 'SSH'
- # Commands and Files to collect OPENBMC dumps
+ # DUMP_LOGS: This section provides option to 'SCP if file exist'.
+ # COMMANDS: filename is preceeded by ls -AX '.
+ # FILES: is not needed and is ignored if exists.
DUMP_LOGS:
COMMANDS:
- 'ls -AX /var/lib/systemd/coredump/core.*'
@@ -93,7 +93,7 @@
- '/var/lib/phosphor-debug-collector/dumps/*/*.tar.xz'
- '/var/lib/phosphor-debug-collector/hostbootdump/*/*.tar.gz'
PROTOCOL:
- - 'SSH'
+ - 'SCP'
# URLs and Files for OPENBMC redfish
# URLs and Files are one-to-one corresponding.
diff --git a/ffdc/ssh_utility.py b/ffdc/ssh_utility.py
index 44a76ff..56d4dad 100644
--- a/ffdc/ssh_utility.py
+++ b/ffdc/ssh_utility.py
@@ -8,6 +8,7 @@
from paramiko.buffered_pipe import PipeTimeout as PipeTimeout
from scp import SCPClient, SCPException
import sys
+import time
import socket
from socket import timeout as SocketTimeout
@@ -79,15 +80,26 @@
"""
+ empty = ''
+ cmd_start = time.time()
try:
- stdin, stdout, stderr = self.sshclient.exec_command(command, timeout=default_timeout)
- stdout.channel.recv_exit_status()
- response = stdout.readlines()
- return response
+ stdin, stdout, stderr = \
+ self.sshclient.exec_command(command, timeout=default_timeout)
+ start = time.time()
+ while time.time() < start + default_timeout:
+ if stdout.channel.exit_status_ready():
+ break
+ time.sleep(1)
+
+ return stderr.readlines(), stdout.readlines()
+
except (paramiko.AuthenticationException, paramiko.SSHException,
- paramiko.ChannelException) as e:
+ paramiko.ChannelException, SocketTimeout) as e:
# Log command with error. Return to caller for next command, if any.
- print("\n>>>>>\tERROR: Fail remote command %s %s %s\n\n" % (command, e.__class__, e))
+ print("\n>>>>>\tERROR: Fail remote command %s %s" % (e.__class__, e))
+ print(">>>>>\tCommand '%s' Elapsed Time %s" %
+ (command, time.strftime("%H:%M:%S", time.gmtime(time.time() - cmd_start))))
+ return empty, empty
def scp_connection(self):