blob: 3ef007237b3c5415ff57f66d12ea25f4619f4fb4 [file] [log] [blame]
#!/usr/bin/env python
r"""
See class prolog below for details.
"""
import os
import sys
import yaml
import time
import platform
from errno import EACCES, EPERM
import subprocess
from ssh_utility import SSHRemoteclient
class FFDCCollector:
r"""
Sends commands from configuration file to the targeted system to collect log files.
Fetch and store generated files at the specified location.
"""
def __init__(self,
hostname,
username,
password,
ffdc_config,
location,
remote_type,
remote_protocol):
r"""
Description of argument(s):
hostname name/ip of the targeted (remote) system
username user on the targeted system with access to FFDC files
password password for user on targeted system
ffdc_config configuration file listing commands and files for FFDC
location where to store collected FFDC
remote_type os type of the remote host
"""
if self.verify_script_env():
self.hostname = hostname
self.username = username
self.password = password
self.ffdc_config = ffdc_config
self.location = location
self.remote_client = None
self.ffdc_dir_path = ""
self.ffdc_prefix = ""
self.target_type = remote_type.upper()
self.remote_protocol = remote_protocol.upper()
self.start_time = 0
self.elapsed_time = ''
else:
sys.exit(-1)
# Load default or user define YAML configuration file.
with open(self.ffdc_config, 'r') as file:
self.ffdc_actions = yaml.load(file, Loader=yaml.FullLoader)
if self.target_type not in self.ffdc_actions.keys():
print("\n\tERROR: %s is not listed in %s.\n\n" % (self.target_type, self.ffdc_config))
sys.exit(-1)
def verify_script_env(self):
# Import to log version
import click
import paramiko
run_env_ok = True
redfishtool_version = self.run_redfishtool('-V').split(' ')[2].strip('\n')
ipmitool_version = self.run_ipmitool('-V').split(' ')[2]
print("\n\t---- Script host environment ----")
print("\t{:<10} {:<10}".format('Script hostname', os.uname()[1]))
print("\t{:<10} {:<10}".format('Script host os', platform.platform()))
print("\t{:<10} {:>10}".format('Python', platform.python_version()))
print("\t{:<10} {:>10}".format('PyYAML', yaml.__version__))
print("\t{:<10} {:>10}".format('click', click.__version__))
print("\t{:<10} {:>10}".format('paramiko', paramiko.__version__))
print("\t{:<10} {:>9}".format('redfishtool', redfishtool_version))
print("\t{:<10} {:>12}".format('ipmitool', ipmitool_version))
if eval(yaml.__version__.replace('.', ',')) < (5, 4, 1):
print("\n\tERROR: Python or python packages do not meet minimum version requirement.")
print("\tERROR: PyYAML version 5.4.1 or higher is needed.\n")
run_env_ok = False
print("\t---- End script host environment ----")
return run_env_ok
def target_is_pingable(self):
r"""
Check if target system is ping-able.
"""
response = os.system("ping -c 1 -w 2 %s 2>&1 >/dev/null" % self.hostname)
if response == 0:
print("\n\t[Check] %s is ping-able.\t\t [OK]" % self.hostname)
return True
else:
print("\n>>>>>\tERROR: %s is not ping-able. FFDC collection aborted.\n" % self.hostname)
sys.exit(-1)
def collect_ffdc(self):
r"""
Initiate FFDC Collection depending on requested protocol.
"""
print("\n\t---- Start communicating with %s ----" % self.hostname)
self.start_time = time.time()
working_protocol_list = []
if self.target_is_pingable():
# Check supported protocol ping,ssh, redfish are working.
if self.ssh_to_target_system():
working_protocol_list.append("SSH")
working_protocol_list.append("SCP")
# Redfish
if self.verify_redfish():
working_protocol_list.append("REDFISH")
print("\n\t[Check] %s Redfish Service.\t\t [OK]" % self.hostname)
else:
print("\n\t[Check] %s Redfish Service.\t\t [NOT AVAILABLE]" % self.hostname)
if self.verify_ipmi():
working_protocol_list.append("IPMI")
print("\n\t[Check] %s IPMI LAN Service.\t\t [OK]" % self.hostname)
else:
print("\n\t[Check] %s IPMI LAN Service.\t\t [NOT AVAILABLE]" % self.hostname)
# Verify top level directory exists for storage
self.validate_local_store(self.location)
print("\n\t---- Completed protocol pre-requisite check ----\n")
if ((self.remote_protocol not in working_protocol_list) and (self.remote_protocol != 'ALL')):
print("\n\tWorking protocol list: %s" % working_protocol_list)
print(
'>>>>>\tERROR: Requested protocol %s is not in working protocol list.\n'
% self.remote_protocol)
sys.exit(-1)
else:
self.generate_ffdc(working_protocol_list)
def ssh_to_target_system(self):
r"""
Open a ssh connection to targeted system.
"""
self.remoteclient = SSHRemoteclient(self.hostname,
self.username,
self.password)
self.remoteclient.ssh_remoteclient_login()
print("\n\t[Check] %s SSH connection established.\t [OK]" % self.hostname)
# Check scp connection.
# If scp connection fails,
# continue with FFDC generation but skip scp files to local host.
self.remoteclient.scp_connection()
return True
def generate_ffdc(self, working_protocol_list):
r"""
Determine actions based on remote host type
Description of argument(s):
working_protocol_list list of confirmed working protocols to connect to remote host.
"""
print("\n\t---- Executing commands on " + self.hostname + " ----")
print("\n\tWorking protocol list: %s" % working_protocol_list)
# Set prefix values for scp files and directory.
# Since the time stamp is at second granularity, these values are set here
# to be sure that all files for this run will have same timestamps
# and they will be saved in the same directory.
# self.location == local system for now
self.set_ffdc_defaults()
ffdc_actions = self.ffdc_actions
for machine_type in ffdc_actions.keys():
if self.target_type != machine_type:
continue
print("\tSystem Type: %s" % machine_type)
for k, v in ffdc_actions[machine_type].items():
if self.remote_protocol != ffdc_actions[machine_type][k]['PROTOCOL'][0] \
and self.remote_protocol != 'ALL':
continue
if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'SSH' \
or ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'SCP':
if 'SSH' in working_protocol_list \
or 'SCP' in working_protocol_list:
self.protocol_ssh(ffdc_actions, machine_type, k)
else:
print("\n\tERROR: SSH or SCP is not available for %s." % self.hostname)
if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'REDFISH':
if 'REDFISH' in working_protocol_list:
self.protocol_redfish(ffdc_actions, machine_type, k)
else:
print("\n\tERROR: REDFISH is not available for %s." % self.hostname)
if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'IPMI':
if 'IPMI' in working_protocol_list:
self.protocol_ipmi(ffdc_actions, machine_type, k)
else:
print("\n\tERROR: IMPI is not available for %s." % self.hostname)
# Close network connection after collecting all files
self.elapsed_time = time.strftime("%H:%M:%S", time.gmtime(time.time() - self.start_time))
self.remoteclient.ssh_remoteclient_disconnect()
def protocol_ssh(self,
ffdc_actions,
machine_type,
sub_type):
r"""
Perform actions using SSH and SCP protocols.
Description of argument(s):
ffdc_actions List of actions from ffdc_config.yaml.
machine_type OS Type of remote host.
sub_type Group type of commands.
"""
if sub_type == 'DUMP_LOGS':
self.group_copy(ffdc_actions[machine_type][sub_type])
else:
self.collect_and_copy_ffdc(ffdc_actions[machine_type][sub_type])
def protocol_redfish(self,
ffdc_actions,
machine_type,
sub_type):
r"""
Perform actions using Redfish protocol.
Description of argument(s):
ffdc_actions List of actions from ffdc_config.yaml.
machine_type OS Type of remote host.
sub_type Group type of commands.
"""
print("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'REDFISH'))
redfish_files_saved = []
progress_counter = 0
list_of_URL = ffdc_actions[machine_type][sub_type]['URL']
for index, each_url in enumerate(list_of_URL, start=0):
redfish_parm = '-u ' + self.username + ' -p ' + self.password + ' -r ' \
+ self.hostname + ' -S Always raw GET ' + each_url
result = self.run_redfishtool(redfish_parm)
if result:
try:
targ_file = self.get_file_list(ffdc_actions[machine_type][sub_type])[index]
except IndexError:
targ_file = each_url.split('/')[-1]
print("\n\t[WARN] Missing filename to store data from redfish URL %s." % each_url)
print("\t[WARN] Data will be stored in %s." % targ_file)
targ_file_with_path = (self.ffdc_dir_path
+ self.ffdc_prefix
+ targ_file)
# Creates a new file
with open(targ_file_with_path, 'w') as fp:
fp.write(result)
fp.close
redfish_files_saved.append(targ_file)
progress_counter += 1
self.print_progress(progress_counter)
print("\n\t[Run] Commands execution completed.\t\t [OK]")
for file in redfish_files_saved:
print("\n\t\tSuccessfully save file " + file + ".")
def protocol_ipmi(self,
ffdc_actions,
machine_type,
sub_type):
r"""
Perform actions using ipmitool over LAN protocol.
Description of argument(s):
ffdc_actions List of actions from ffdc_config.yaml.
machine_type OS Type of remote host.
sub_type Group type of commands.
"""
print("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'IPMI'))
ipmi_files_saved = []
progress_counter = 0
list_of_cmd = self.get_command_list(ffdc_actions[machine_type][sub_type])
for index, each_cmd in enumerate(list_of_cmd, start=0):
ipmi_parm = '-U ' + self.username + ' -P ' + self.password + ' -H ' \
+ self.hostname + ' ' + each_cmd
result = self.run_ipmitool(ipmi_parm)
if result:
try:
targ_file = self.get_file_list(ffdc_actions[machine_type][sub_type])[index]
except IndexError:
targ_file = each_cmd.split('/')[-1]
print("\n\t[WARN] Missing filename to store data from IPMI %s." % each_cmd)
print("\t[WARN] Data will be stored in %s." % targ_file)
targ_file_with_path = (self.ffdc_dir_path
+ self.ffdc_prefix
+ targ_file)
# Creates a new file
with open(targ_file_with_path, 'w') as fp:
fp.write(result)
fp.close
ipmi_files_saved.append(targ_file)
progress_counter += 1
self.print_progress(progress_counter)
print("\n\t[Run] Commands execution completed.\t\t [OK]")
for file in ipmi_files_saved:
print("\n\t\tSuccessfully save file " + file + ".")
def collect_and_copy_ffdc(self,
ffdc_actions_for_machine_type,
form_filename=False):
r"""
Send commands in ffdc_config file to targeted system.
Description of argument(s):
ffdc_actions_for_machine_type commands and files for the selected remote host type.
form_filename if true, pre-pend self.target_type to filename
"""
# Executing commands, , if any
self.ssh_execute_ffdc_commands(ffdc_actions_for_machine_type,
form_filename)
# Copying files
if self.remoteclient.scpclient:
print("\n\n\tCopying FFDC files from remote system %s.\n" % self.hostname)
# Retrieving files from target system
list_of_files = self.get_file_list(ffdc_actions_for_machine_type)
self.scp_ffdc(self.ffdc_dir_path, self.ffdc_prefix, form_filename, list_of_files)
else:
print("\n\n\tSkip copying FFDC files from remote system %s.\n" % self.hostname)
def get_command_list(self,
ffdc_actions_for_machine_type):
r"""
Fetch list of commands from configuration file
Description of argument(s):
ffdc_actions_for_machine_type commands and files for the selected remote host type.
"""
try:
list_of_commands = ffdc_actions_for_machine_type['COMMANDS']
except KeyError:
list_of_commands = []
return list_of_commands
def get_file_list(self,
ffdc_actions_for_machine_type):
r"""
Fetch list of commands from configuration file
Description of argument(s):
ffdc_actions_for_machine_type commands and files for the selected remote host type.
"""
try:
list_of_files = ffdc_actions_for_machine_type['FILES']
except KeyError:
list_of_files = []
return list_of_files
def ssh_execute_ffdc_commands(self,
ffdc_actions_for_machine_type,
form_filename=False):
r"""
Send commands in ffdc_config file to targeted system.
Description of argument(s):
ffdc_actions_for_machine_type commands and files for the selected remote host type.
form_filename if true, pre-pend self.target_type to filename
"""
print("\n\t[Run] Executing commands on %s using %s"
% (self.hostname, ffdc_actions_for_machine_type['PROTOCOL'][0]))
list_of_commands = self.get_command_list(ffdc_actions_for_machine_type)
# If command list is empty, returns
if not list_of_commands:
return
progress_counter = 0
for command in list_of_commands:
if isinstance(command, dict):
command_txt = next(iter(command))
command_timeout = next(iter(command.values()))
elif isinstance(command, str):
command_txt = command
# Default ssh command timeout 60 seconds
command_timeout = 60
if form_filename:
command_txt = str(command_txt % self.target_type)
err, response = self.remoteclient.execute_command(command_txt, command_timeout)
progress_counter += 1
self.print_progress(progress_counter)
print("\n\t[Run] Commands execution completed.\t\t [OK]")
def group_copy(self,
ffdc_actions_for_machine_type):
r"""
scp group of files (wild card) from remote host.
Description of argument(s):
ffdc_actions_for_machine_type commands and files for the selected remote host type.
"""
if self.remoteclient.scpclient:
print("\n\tCopying DUMP files from remote system %s.\n" % self.hostname)
list_of_commands = self.get_command_list(ffdc_actions_for_machine_type)
# If command list is empty, returns
if not list_of_commands:
return
for command in list_of_commands:
try:
filename = command.split(' ')[2]
except IndexError:
print("\t\tInvalid command %s for DUMP_LOGS block." % command)
continue
err, response = self.remoteclient.execute_command(command)
if response:
scp_result = self.remoteclient.scp_file_from_remote(filename, self.ffdc_dir_path)
if scp_result:
print("\t\tSuccessfully copied from " + self.hostname + ':' + filename)
else:
print("\t\tThere is no " + filename)
else:
print("\n\n\tSkip copying files from remote system %s.\n" % self.hostname)
def scp_ffdc(self,
targ_dir_path,
targ_file_prefix,
form_filename,
file_list=None,
quiet=None):
r"""
SCP all files in file_dict to the indicated directory on the local system.
Description of argument(s):
targ_dir_path The path of the directory to receive the files.
targ_file_prefix Prefix which will be pre-pended to each
target file's name.
file_dict A dictionary of files to scp from targeted system to this system
"""
progress_counter = 0
for filename in file_list:
if form_filename:
filename = str(filename % self.target_type)
source_file_path = filename
targ_file_path = targ_dir_path + targ_file_prefix + filename.split('/')[-1]
# If source file name contains wild card, copy filename as is.
if '*' in source_file_path:
scp_result = self.remoteclient.scp_file_from_remote(source_file_path, self.ffdc_dir_path)
else:
scp_result = self.remoteclient.scp_file_from_remote(source_file_path, targ_file_path)
if not quiet:
if scp_result:
print("\t\tSuccessfully copied from " + self.hostname + ':' + source_file_path + ".\n")
else:
print("\t\tFail to copy from " + self.hostname + ':' + source_file_path + ".\n")
else:
progress_counter += 1
self.print_progress(progress_counter)
def set_ffdc_defaults(self):
r"""
Set a default value for self.ffdc_dir_path and self.ffdc_prefix.
Collected ffdc file will be stored in dir /self.location/hostname_timestr/.
Individual ffdc file will have timestr_filename.
Description of class variables:
self.ffdc_dir_path The dir path where collected ffdc data files should be put.
self.ffdc_prefix The prefix to be given to each ffdc file name.
"""
timestr = time.strftime("%Y%m%d-%H%M%S")
self.ffdc_dir_path = self.location + "/" + self.hostname + "_" + timestr + "/"
self.ffdc_prefix = timestr + "_"
self.validate_local_store(self.ffdc_dir_path)
def validate_local_store(self, dir_path):
r"""
Ensure path exists to store FFDC files locally.
Description of variable:
dir_path The dir path where collected ffdc data files will be stored.
"""
if not os.path.exists(dir_path):
try:
os.mkdir(dir_path, 0o755)
except (IOError, OSError) as e:
# PermissionError
if e.errno == EPERM or e.errno == EACCES:
print('>>>>>\tERROR: os.mkdir %s failed with PermissionError.\n' % dir_path)
else:
print('>>>>>\tERROR: os.mkdir %s failed with %s.\n' % (dir_path, e.strerror))
sys.exit(-1)
def print_progress(self, progress):
r"""
Print activity progress +
Description of variable:
progress Progress counter.
"""
sys.stdout.write("\r\t" + "+" * progress)
sys.stdout.flush()
time.sleep(.1)
def verify_redfish(self):
r"""
Verify remote host has redfish service active
"""
redfish_parm = '-u ' + self.username + ' -p ' + self.password + ' -r ' \
+ self.hostname + ' -S Always raw GET /redfish/v1/'
return(self.run_redfishtool(redfish_parm, True))
def verify_ipmi(self):
r"""
Verify remote host has IPMI LAN service active
"""
ipmi_parm = '-U ' + self.username + ' -P ' + self.password + ' -H ' \
+ self.hostname + ' power status'
return(self.run_ipmitool(ipmi_parm, True))
def run_redfishtool(self,
parms_string,
quiet=False):
r"""
Run CLI redfishtool
Description of variable:
parms_string redfishtool subcommand and options.
quiet do not print redfishtool error message if True
"""
result = subprocess.run(['redfishtool ' + parms_string],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
universal_newlines=True)
if result.stderr and not quiet:
print('\n\t\tERROR with redfishtool ' + parms_string)
print('\t\t' + result.stderr)
return result.stdout
def run_ipmitool(self,
parms_string,
quiet=False):
r"""
Run CLI IPMI tool.
Description of variable:
parms_string ipmitool subcommand and options.
quiet do not print redfishtool error message if True
"""
result = subprocess.run(['ipmitool -I lanplus -C 17 ' + parms_string],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
universal_newlines=True)
if result.stderr and not quiet:
print('\n\t\tERROR with ipmitool -I lanplus -C 17 ' + parms_string)
print('\t\t' + result.stderr)
return result.stdout