blob: f196e0dc146816be4f22334dc1307f5bfc8123c4 [file] [log] [blame]
Peter D Phan72ce6b82021-06-03 06:18:26 -05001#!/usr/bin/env python
2
3r"""
4See class prolog below for details.
5"""
6
7import os
8import sys
9import yaml
10import time
Peter D Phane86d9a52021-07-15 10:42:25 -050011import logging
Peter D Phan72ce6b82021-06-03 06:18:26 -050012import platform
13from errno import EACCES, EPERM
Peter D Phan0c669772021-06-24 13:52:42 -050014import subprocess
Peter D Phan72ce6b82021-06-03 06:18:26 -050015from ssh_utility import SSHRemoteclient
Peter D Phan5963d632021-07-12 09:58:55 -050016from telnet_utility import TelnetRemoteclient
Peter D Phan72ce6b82021-06-03 06:18:26 -050017
18
19class FFDCCollector:
20
21 r"""
22 Sends commands from configuration file to the targeted system to collect log files.
23 Fetch and store generated files at the specified location.
24
25 """
26
Peter D Phan0c669772021-06-24 13:52:42 -050027 def __init__(self,
28 hostname,
29 username,
30 password,
31 ffdc_config,
32 location,
33 remote_type,
Peter D Phane86d9a52021-07-15 10:42:25 -050034 remote_protocol,
35 log_level):
Peter D Phan72ce6b82021-06-03 06:18:26 -050036 r"""
37 Description of argument(s):
38
39 hostname name/ip of the targeted (remote) system
40 username user on the targeted system with access to FFDC files
41 password password for user on targeted system
42 ffdc_config configuration file listing commands and files for FFDC
Peter D Phan04aca3b2021-06-21 10:37:18 -050043 location where to store collected FFDC
44 remote_type os type of the remote host
Peter D Phan72ce6b82021-06-03 06:18:26 -050045
46 """
Peter D Phane86d9a52021-07-15 10:42:25 -050047
48 self.hostname = hostname
49 self.username = username
50 self.password = password
George Keishing04d29102021-07-16 02:05:57 -050051 # This is for the env vars a user can use in YAML to load it at runtime.
52 # Example YAML:
53 # -COMMANDS:
54 # - my_command ${hostname} ${username} ${password}
55 os.environ['hostname'] = hostname
56 os.environ['username'] = username
57 os.environ['password'] = password
58
Peter D Phane86d9a52021-07-15 10:42:25 -050059 self.ffdc_config = ffdc_config
60 self.location = location + "/" + remote_type.upper()
61 self.ssh_remoteclient = None
62 self.telnet_remoteclient = None
63 self.ffdc_dir_path = ""
64 self.ffdc_prefix = ""
65 self.target_type = remote_type.upper()
66 self.remote_protocol = remote_protocol.upper()
67 self.start_time = 0
68 self.elapsed_time = ''
69 self.logger = None
70
71 # Set prefix values for scp files and directory.
72 # Since the time stamp is at second granularity, these values are set here
73 # to be sure that all files for this run will have same timestamps
74 # and they will be saved in the same directory.
75 # self.location == local system for now
76 self.set_ffdc_defaults()
77
78 # Logger for this run. Need to be after set_ffdc_defaults()
79 self.script_logging(getattr(logging, log_level.upper()))
80
81 # Verify top level directory exists for storage
82 self.validate_local_store(self.location)
83
Peter D Phan72ce6b82021-06-03 06:18:26 -050084 if self.verify_script_env():
Peter D Phane86d9a52021-07-15 10:42:25 -050085 # Load default or user define YAML configuration file.
86 with open(self.ffdc_config, 'r') as file:
87 self.ffdc_actions = yaml.load(file, Loader=yaml.FullLoader)
88
89 if self.target_type not in self.ffdc_actions.keys():
90 self.logger.error(
91 "\n\tERROR: %s is not listed in %s.\n\n" % (self.target_type, self.ffdc_config))
92 sys.exit(-1)
Peter D Phan72ce6b82021-06-03 06:18:26 -050093 else:
Peter D Phan8462faf2021-06-16 12:24:15 -050094 sys.exit(-1)
Peter D Phan72ce6b82021-06-03 06:18:26 -050095
96 def verify_script_env(self):
97
98 # Import to log version
99 import click
100 import paramiko
101
102 run_env_ok = True
Peter D Phan0c669772021-06-24 13:52:42 -0500103
George Keishingeafba182021-06-29 13:44:58 -0500104 redfishtool_version = self.run_redfishtool('-V').split(' ')[2].strip('\n')
105 ipmitool_version = self.run_ipmitool('-V').split(' ')[2]
Peter D Phan0c669772021-06-24 13:52:42 -0500106
Peter D Phane86d9a52021-07-15 10:42:25 -0500107 self.logger.info("\n\t---- Script host environment ----")
108 self.logger.info("\t{:<10} {:<10}".format('Script hostname', os.uname()[1]))
109 self.logger.info("\t{:<10} {:<10}".format('Script host os', platform.platform()))
110 self.logger.info("\t{:<10} {:>10}".format('Python', platform.python_version()))
111 self.logger.info("\t{:<10} {:>10}".format('PyYAML', yaml.__version__))
112 self.logger.info("\t{:<10} {:>10}".format('click', click.__version__))
113 self.logger.info("\t{:<10} {:>10}".format('paramiko', paramiko.__version__))
114 self.logger.info("\t{:<10} {:>9}".format('redfishtool', redfishtool_version))
115 self.logger.info("\t{:<10} {:>12}".format('ipmitool', ipmitool_version))
Peter D Phan72ce6b82021-06-03 06:18:26 -0500116
Peter D Phan8462faf2021-06-16 12:24:15 -0500117 if eval(yaml.__version__.replace('.', ',')) < (5, 4, 1):
Peter D Phane86d9a52021-07-15 10:42:25 -0500118 self.logger.error("\n\tERROR: Python or python packages do not meet minimum version requirement.")
119 self.logger.error("\tERROR: PyYAML version 5.4.1 or higher is needed.\n")
Peter D Phan72ce6b82021-06-03 06:18:26 -0500120 run_env_ok = False
121
Peter D Phane86d9a52021-07-15 10:42:25 -0500122 self.logger.info("\t---- End script host environment ----")
Peter D Phan72ce6b82021-06-03 06:18:26 -0500123 return run_env_ok
124
Peter D Phane86d9a52021-07-15 10:42:25 -0500125 def script_logging(self,
126 log_level_attr):
127 r"""
128 Create logger
129
130 """
131 self.logger = logging.getLogger()
132 self.logger.setLevel(log_level_attr)
133 log_file_handler = logging.FileHandler(self.ffdc_dir_path + "collector.log")
134
135 stdout_handler = logging.StreamHandler(sys.stdout)
136 self.logger.addHandler(log_file_handler)
137 self.logger.addHandler(stdout_handler)
138
139 # Turn off paramiko INFO logging
140 logging.getLogger("paramiko").setLevel(logging.WARNING)
141
Peter D Phan72ce6b82021-06-03 06:18:26 -0500142 def target_is_pingable(self):
Peter D Phan72ce6b82021-06-03 06:18:26 -0500143 r"""
144 Check if target system is ping-able.
145
146 """
George Keishing0662e942021-07-13 05:12:20 -0500147 response = os.system("ping -c 1 %s 2>&1 >/dev/null" % self.hostname)
Peter D Phan72ce6b82021-06-03 06:18:26 -0500148 if response == 0:
Peter D Phane86d9a52021-07-15 10:42:25 -0500149 self.logger.info("\n\t[Check] %s is ping-able.\t\t [OK]" % self.hostname)
Peter D Phan72ce6b82021-06-03 06:18:26 -0500150 return True
151 else:
Peter D Phane86d9a52021-07-15 10:42:25 -0500152 self.logger.error(
153 "\n>>>>>\tERROR: %s is not ping-able. FFDC collection aborted.\n" % self.hostname)
Peter D Phan72ce6b82021-06-03 06:18:26 -0500154 sys.exit(-1)
155
Peter D Phan72ce6b82021-06-03 06:18:26 -0500156 def collect_ffdc(self):
Peter D Phan72ce6b82021-06-03 06:18:26 -0500157 r"""
158 Initiate FFDC Collection depending on requested protocol.
159
160 """
161
Peter D Phane86d9a52021-07-15 10:42:25 -0500162 self.logger.info("\n\t---- Start communicating with %s ----" % self.hostname)
Peter D Phan7610bc42021-07-06 06:31:05 -0500163 self.start_time = time.time()
George Keishing772c9772021-06-16 23:23:42 -0500164 working_protocol_list = []
Peter D Phan72ce6b82021-06-03 06:18:26 -0500165 if self.target_is_pingable():
George Keishing04d29102021-07-16 02:05:57 -0500166 working_protocol_list.append("SHELL")
George Keishing772c9772021-06-16 23:23:42 -0500167 # Check supported protocol ping,ssh, redfish are working.
168 if self.ssh_to_target_system():
169 working_protocol_list.append("SSH")
George Keishing615fe322021-06-24 01:24:36 -0500170 working_protocol_list.append("SCP")
Peter D Phan0c669772021-06-24 13:52:42 -0500171
172 # Redfish
173 if self.verify_redfish():
174 working_protocol_list.append("REDFISH")
Peter D Phane86d9a52021-07-15 10:42:25 -0500175 self.logger.info("\n\t[Check] %s Redfish Service.\t\t [OK]" % self.hostname)
Peter D Phan0c669772021-06-24 13:52:42 -0500176 else:
Peter D Phane86d9a52021-07-15 10:42:25 -0500177 self.logger.info("\n\t[Check] %s Redfish Service.\t\t [NOT AVAILABLE]" % self.hostname)
Peter D Phan0c669772021-06-24 13:52:42 -0500178
Peter D Phan5963d632021-07-12 09:58:55 -0500179 # IPMI
George Keishingeafba182021-06-29 13:44:58 -0500180 if self.verify_ipmi():
181 working_protocol_list.append("IPMI")
Peter D Phane86d9a52021-07-15 10:42:25 -0500182 self.logger.info("\n\t[Check] %s IPMI LAN Service.\t\t [OK]" % self.hostname)
George Keishingeafba182021-06-29 13:44:58 -0500183 else:
Peter D Phane86d9a52021-07-15 10:42:25 -0500184 self.logger.info("\n\t[Check] %s IPMI LAN Service.\t\t [NOT AVAILABLE]" % self.hostname)
George Keishingeafba182021-06-29 13:44:58 -0500185
Peter D Phan5963d632021-07-12 09:58:55 -0500186 # Telnet
187 if self.telnet_to_target_system():
188 working_protocol_list.append("TELNET")
189
Peter D Phan72ce6b82021-06-03 06:18:26 -0500190 # Verify top level directory exists for storage
191 self.validate_local_store(self.location)
Peter D Phane86d9a52021-07-15 10:42:25 -0500192 self.logger.info("\n\t---- Completed protocol pre-requisite check ----\n")
Peter D Phan0c669772021-06-24 13:52:42 -0500193
194 if ((self.remote_protocol not in working_protocol_list) and (self.remote_protocol != 'ALL')):
Peter D Phane86d9a52021-07-15 10:42:25 -0500195 self.logger.info("\n\tWorking protocol list: %s" % working_protocol_list)
196 self.logger.error(
Peter D Phan0c669772021-06-24 13:52:42 -0500197 '>>>>>\tERROR: Requested protocol %s is not in working protocol list.\n'
198 % self.remote_protocol)
199 sys.exit(-1)
200 else:
201 self.generate_ffdc(working_protocol_list)
Peter D Phan72ce6b82021-06-03 06:18:26 -0500202
203 def ssh_to_target_system(self):
204 r"""
205 Open a ssh connection to targeted system.
206
207 """
208
Peter D Phan5963d632021-07-12 09:58:55 -0500209 self.ssh_remoteclient = SSHRemoteclient(self.hostname,
210 self.username,
211 self.password)
Peter D Phan72ce6b82021-06-03 06:18:26 -0500212
Peter D Phan5963d632021-07-12 09:58:55 -0500213 if self.ssh_remoteclient.ssh_remoteclient_login():
Peter D Phane86d9a52021-07-15 10:42:25 -0500214 self.logger.info("\n\t[Check] %s SSH connection established.\t [OK]" % self.hostname)
Peter D Phan733df632021-06-17 13:13:36 -0500215
Peter D Phan5963d632021-07-12 09:58:55 -0500216 # Check scp connection.
217 # If scp connection fails,
218 # continue with FFDC generation but skip scp files to local host.
219 self.ssh_remoteclient.scp_connection()
220 return True
221 else:
Peter D Phane86d9a52021-07-15 10:42:25 -0500222 self.logger.info("\n\t[Check] %s SSH connection.\t [NOT AVAILABLE]" % self.hostname)
Peter D Phan5963d632021-07-12 09:58:55 -0500223 return False
224
225 def telnet_to_target_system(self):
226 r"""
227 Open a telnet connection to targeted system.
228 """
229 self.telnet_remoteclient = TelnetRemoteclient(self.hostname,
230 self.username,
231 self.password)
232 if self.telnet_remoteclient.tn_remoteclient_login():
Peter D Phane86d9a52021-07-15 10:42:25 -0500233 self.logger.info("\n\t[Check] %s Telnet connection established.\t [OK]" % self.hostname)
Peter D Phan5963d632021-07-12 09:58:55 -0500234 return True
235 else:
Peter D Phane86d9a52021-07-15 10:42:25 -0500236 self.logger.info("\n\t[Check] %s Telnet connection.\t [NOT AVAILABLE]" % self.hostname)
Peter D Phan5963d632021-07-12 09:58:55 -0500237 return False
Peter D Phan72ce6b82021-06-03 06:18:26 -0500238
George Keishing772c9772021-06-16 23:23:42 -0500239 def generate_ffdc(self, working_protocol_list):
Peter D Phan72ce6b82021-06-03 06:18:26 -0500240 r"""
Peter D Phan04aca3b2021-06-21 10:37:18 -0500241 Determine actions based on remote host type
Peter D Phan72ce6b82021-06-03 06:18:26 -0500242
Peter D Phan04aca3b2021-06-21 10:37:18 -0500243 Description of argument(s):
244 working_protocol_list list of confirmed working protocols to connect to remote host.
Peter D Phan72ce6b82021-06-03 06:18:26 -0500245 """
246
Peter D Phane86d9a52021-07-15 10:42:25 -0500247 self.logger.info("\n\t---- Executing commands on " + self.hostname + " ----")
248 self.logger.info("\n\tWorking protocol list: %s" % working_protocol_list)
Peter D Phan72ce6b82021-06-03 06:18:26 -0500249
George Keishing86764b52021-07-01 04:32:03 -0500250 ffdc_actions = self.ffdc_actions
Peter D Phan2b8052d2021-06-22 10:55:41 -0500251
Peter D Phan72ce6b82021-06-03 06:18:26 -0500252 for machine_type in ffdc_actions.keys():
George Keishing6ea92b02021-07-01 11:20:50 -0500253 if self.target_type != machine_type:
254 continue
Peter D Phan72ce6b82021-06-03 06:18:26 -0500255
Peter D Phane86d9a52021-07-15 10:42:25 -0500256 self.logger.info("\n\tFFDC Path: %s " % self.ffdc_dir_path)
257 self.logger.info("\tSystem Type: %s" % machine_type)
George Keishing6ea92b02021-07-01 11:20:50 -0500258 for k, v in ffdc_actions[machine_type].items():
Peter D Phan72ce6b82021-06-03 06:18:26 -0500259
George Keishing6ea92b02021-07-01 11:20:50 -0500260 if self.remote_protocol != ffdc_actions[machine_type][k]['PROTOCOL'][0] \
261 and self.remote_protocol != 'ALL':
262 continue
Peter D Phan72ce6b82021-06-03 06:18:26 -0500263
Peter D Phanbabf2962021-07-07 11:24:40 -0500264 if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'SSH' \
265 or ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'SCP':
266 if 'SSH' in working_protocol_list \
267 or 'SCP' in working_protocol_list:
Peter D Phan3beb02e2021-07-06 13:25:17 -0500268 self.protocol_ssh(ffdc_actions, machine_type, k)
269 else:
Peter D Phane86d9a52021-07-15 10:42:25 -0500270 self.logger.error("\n\tERROR: SSH or SCP is not available for %s." % self.hostname)
George Keishing6ea92b02021-07-01 11:20:50 -0500271
Peter D Phan5963d632021-07-12 09:58:55 -0500272 if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'TELNET':
273 if 'TELNET' in working_protocol_list:
274 self.protocol_telnet(ffdc_actions, machine_type, k)
275 else:
Peter D Phane86d9a52021-07-15 10:42:25 -0500276 self.logger.error("\n\tERROR: TELNET is not available for %s." % self.hostname)
Peter D Phan5963d632021-07-12 09:58:55 -0500277
George Keishing6ea92b02021-07-01 11:20:50 -0500278 if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'REDFISH':
Peter D Phan3beb02e2021-07-06 13:25:17 -0500279 if 'REDFISH' in working_protocol_list:
280 self.protocol_redfish(ffdc_actions, machine_type, k)
281 else:
Peter D Phane86d9a52021-07-15 10:42:25 -0500282 self.logger.error("\n\tERROR: REDFISH is not available for %s." % self.hostname)
George Keishing6ea92b02021-07-01 11:20:50 -0500283
284 if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'IPMI':
Peter D Phan3beb02e2021-07-06 13:25:17 -0500285 if 'IPMI' in working_protocol_list:
286 self.protocol_ipmi(ffdc_actions, machine_type, k)
287 else:
George Keishingcce505b2021-07-16 02:23:25 -0500288 self.logger.error("\n\tERROR: IMPI is not available for %s." % self.hostname)
George Keishing04d29102021-07-16 02:05:57 -0500289
290 if ffdc_actions[machine_type][k]['PROTOCOL'][0] == 'SHELL':
291 if 'SHELL' in working_protocol_list:
292 self.protocol_shell_script(ffdc_actions, machine_type, k)
293 else:
294 self.logger.error("\n\tERROR: can't execute SHELL script")
George Keishingeafba182021-06-29 13:44:58 -0500295
Peter D Phan04aca3b2021-06-21 10:37:18 -0500296 # Close network connection after collecting all files
Peter D Phan7610bc42021-07-06 06:31:05 -0500297 self.elapsed_time = time.strftime("%H:%M:%S", time.gmtime(time.time() - self.start_time))
Peter D Phan5963d632021-07-12 09:58:55 -0500298 self.ssh_remoteclient.ssh_remoteclient_disconnect()
299 self.telnet_remoteclient.tn_remoteclient_disconnect()
Peter D Phan04aca3b2021-06-21 10:37:18 -0500300
Peter D Phan0c669772021-06-24 13:52:42 -0500301 def protocol_ssh(self,
302 ffdc_actions,
George Keishing6ea92b02021-07-01 11:20:50 -0500303 machine_type,
304 sub_type):
Peter D Phan0c669772021-06-24 13:52:42 -0500305 r"""
306 Perform actions using SSH and SCP protocols.
307
308 Description of argument(s):
309 ffdc_actions List of actions from ffdc_config.yaml.
310 machine_type OS Type of remote host.
George Keishing6ea92b02021-07-01 11:20:50 -0500311 sub_type Group type of commands.
Peter D Phan0c669772021-06-24 13:52:42 -0500312 """
313
George Keishing6ea92b02021-07-01 11:20:50 -0500314 if sub_type == 'DUMP_LOGS':
315 self.group_copy(ffdc_actions[machine_type][sub_type])
316 else:
317 self.collect_and_copy_ffdc(ffdc_actions[machine_type][sub_type])
Peter D Phan0c669772021-06-24 13:52:42 -0500318
Peter D Phan5963d632021-07-12 09:58:55 -0500319 def protocol_telnet(self,
320 ffdc_actions,
321 machine_type,
322 sub_type):
323 r"""
324 Perform actions using telnet protocol.
325 Description of argument(s):
326 ffdc_actions List of actions from ffdc_config.yaml.
327 machine_type OS Type of remote host.
328 """
Peter D Phane86d9a52021-07-15 10:42:25 -0500329 self.logger.info("\n\t[Run] Executing commands on %s using %s" % (self.hostname, 'TELNET'))
Peter D Phan5963d632021-07-12 09:58:55 -0500330 telnet_files_saved = []
331 progress_counter = 0
332 list_of_commands = ffdc_actions[machine_type][sub_type]['COMMANDS']
333 for index, each_cmd in enumerate(list_of_commands, start=0):
334 command_txt, command_timeout = self.unpack_command(each_cmd)
335 result = self.telnet_remoteclient.execute_command(command_txt, command_timeout)
336 if result:
337 try:
338 targ_file = ffdc_actions[machine_type][sub_type]['FILES'][index]
339 except IndexError:
Peter D Phane86d9a52021-07-15 10:42:25 -0500340 targ_file = command_txt
341 self.logger.warning(
342 "\n\t[WARN] Missing filename to store data from telnet %s." % each_cmd)
343 self.logger.warning("\t[WARN] Data will be stored in %s." % targ_file)
Peter D Phan5963d632021-07-12 09:58:55 -0500344 targ_file_with_path = (self.ffdc_dir_path
345 + self.ffdc_prefix
346 + targ_file)
347 # Creates a new file
348 with open(targ_file_with_path, 'wb') as fp:
349 fp.write(result)
350 fp.close
351 telnet_files_saved.append(targ_file)
352 progress_counter += 1
353 self.print_progress(progress_counter)
Peter D Phane86d9a52021-07-15 10:42:25 -0500354 self.logger.info("\n\t[Run] Commands execution completed.\t\t [OK]")
Peter D Phan5963d632021-07-12 09:58:55 -0500355 for file in telnet_files_saved:
Peter D Phane86d9a52021-07-15 10:42:25 -0500356 self.logger.info("\n\t\tSuccessfully save file " + file + ".")
Peter D Phan5963d632021-07-12 09:58:55 -0500357
Peter D Phan0c669772021-06-24 13:52:42 -0500358 def protocol_redfish(self,
359 ffdc_actions,
George Keishing6ea92b02021-07-01 11:20:50 -0500360 machine_type,
361 sub_type):
Peter D Phan0c669772021-06-24 13:52:42 -0500362 r"""
363 Perform actions using Redfish protocol.
364
365 Description of argument(s):
366 ffdc_actions List of actions from ffdc_config.yaml.
367 machine_type OS Type of remote host.
George Keishing6ea92b02021-07-01 11:20:50 -0500368 sub_type Group type of commands.
Peter D Phan0c669772021-06-24 13:52:42 -0500369 """
370
Peter D Phane86d9a52021-07-15 10:42:25 -0500371 self.logger.info("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'REDFISH'))
Peter D Phan0c669772021-06-24 13:52:42 -0500372 redfish_files_saved = []
373 progress_counter = 0
George Keishing6ea92b02021-07-01 11:20:50 -0500374 list_of_URL = ffdc_actions[machine_type][sub_type]['URL']
Peter D Phan0c669772021-06-24 13:52:42 -0500375 for index, each_url in enumerate(list_of_URL, start=0):
376 redfish_parm = '-u ' + self.username + ' -p ' + self.password + ' -r ' \
377 + self.hostname + ' -S Always raw GET ' + each_url
378
379 result = self.run_redfishtool(redfish_parm)
380 if result:
381 try:
Peter D Phanbabf2962021-07-07 11:24:40 -0500382 targ_file = self.get_file_list(ffdc_actions[machine_type][sub_type])[index]
Peter D Phan0c669772021-06-24 13:52:42 -0500383 except IndexError:
384 targ_file = each_url.split('/')[-1]
Peter D Phane86d9a52021-07-15 10:42:25 -0500385 self.logger.warning(
386 "\n\t[WARN] Missing filename to store data from redfish URL %s." % each_url)
387 self.logger.warning("\t[WARN] Data will be stored in %s." % targ_file)
Peter D Phan0c669772021-06-24 13:52:42 -0500388
389 targ_file_with_path = (self.ffdc_dir_path
390 + self.ffdc_prefix
391 + targ_file)
392
393 # Creates a new file
394 with open(targ_file_with_path, 'w') as fp:
395 fp.write(result)
396 fp.close
397 redfish_files_saved.append(targ_file)
398
399 progress_counter += 1
400 self.print_progress(progress_counter)
401
Peter D Phane86d9a52021-07-15 10:42:25 -0500402 self.logger.info("\n\t[Run] Commands execution completed.\t\t [OK]")
Peter D Phan0c669772021-06-24 13:52:42 -0500403
404 for file in redfish_files_saved:
Peter D Phane86d9a52021-07-15 10:42:25 -0500405 self.logger.info("\n\t\tSuccessfully save file " + file + ".")
Peter D Phan0c669772021-06-24 13:52:42 -0500406
George Keishingeafba182021-06-29 13:44:58 -0500407 def protocol_ipmi(self,
408 ffdc_actions,
George Keishing6ea92b02021-07-01 11:20:50 -0500409 machine_type,
410 sub_type):
George Keishingeafba182021-06-29 13:44:58 -0500411 r"""
412 Perform actions using ipmitool over LAN protocol.
413
414 Description of argument(s):
415 ffdc_actions List of actions from ffdc_config.yaml.
416 machine_type OS Type of remote host.
George Keishing6ea92b02021-07-01 11:20:50 -0500417 sub_type Group type of commands.
George Keishingeafba182021-06-29 13:44:58 -0500418 """
419
Peter D Phane86d9a52021-07-15 10:42:25 -0500420 self.logger.info("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'IPMI'))
George Keishingeafba182021-06-29 13:44:58 -0500421 ipmi_files_saved = []
422 progress_counter = 0
Peter D Phanbabf2962021-07-07 11:24:40 -0500423 list_of_cmd = self.get_command_list(ffdc_actions[machine_type][sub_type])
George Keishingeafba182021-06-29 13:44:58 -0500424 for index, each_cmd in enumerate(list_of_cmd, start=0):
425 ipmi_parm = '-U ' + self.username + ' -P ' + self.password + ' -H ' \
426 + self.hostname + ' ' + each_cmd
427
428 result = self.run_ipmitool(ipmi_parm)
429 if result:
430 try:
Peter D Phanbabf2962021-07-07 11:24:40 -0500431 targ_file = self.get_file_list(ffdc_actions[machine_type][sub_type])[index]
George Keishingeafba182021-06-29 13:44:58 -0500432 except IndexError:
George Keishing6ea92b02021-07-01 11:20:50 -0500433 targ_file = each_cmd.split('/')[-1]
Peter D Phane86d9a52021-07-15 10:42:25 -0500434 self.logger.warning("\n\t[WARN] Missing filename to store data from IPMI %s." % each_cmd)
435 self.logger.warning("\t[WARN] Data will be stored in %s." % targ_file)
George Keishingeafba182021-06-29 13:44:58 -0500436
437 targ_file_with_path = (self.ffdc_dir_path
438 + self.ffdc_prefix
439 + targ_file)
440
441 # Creates a new file
442 with open(targ_file_with_path, 'w') as fp:
443 fp.write(result)
444 fp.close
445 ipmi_files_saved.append(targ_file)
446
447 progress_counter += 1
448 self.print_progress(progress_counter)
449
Peter D Phane86d9a52021-07-15 10:42:25 -0500450 self.logger.info("\n\t[Run] Commands execution completed.\t\t [OK]")
George Keishingeafba182021-06-29 13:44:58 -0500451
452 for file in ipmi_files_saved:
Peter D Phane86d9a52021-07-15 10:42:25 -0500453 self.logger.info("\n\t\tSuccessfully save file " + file + ".")
George Keishingeafba182021-06-29 13:44:58 -0500454
Peter D Phan04aca3b2021-06-21 10:37:18 -0500455 def collect_and_copy_ffdc(self,
Peter D Phan2b8052d2021-06-22 10:55:41 -0500456 ffdc_actions_for_machine_type,
457 form_filename=False):
Peter D Phan04aca3b2021-06-21 10:37:18 -0500458 r"""
459 Send commands in ffdc_config file to targeted system.
460
461 Description of argument(s):
462 ffdc_actions_for_machine_type commands and files for the selected remote host type.
Peter D Phan2b8052d2021-06-22 10:55:41 -0500463 form_filename if true, pre-pend self.target_type to filename
Peter D Phan04aca3b2021-06-21 10:37:18 -0500464 """
465
Peter D Phan2b6cb3a2021-07-19 06:55:42 -0500466 # Executing commands, if any
Peter D Phan3beb02e2021-07-06 13:25:17 -0500467 self.ssh_execute_ffdc_commands(ffdc_actions_for_machine_type,
468 form_filename)
Peter D Phan04aca3b2021-06-21 10:37:18 -0500469
Peter D Phan3beb02e2021-07-06 13:25:17 -0500470 # Copying files
Peter D Phan5963d632021-07-12 09:58:55 -0500471 if self.ssh_remoteclient.scpclient:
Peter D Phane86d9a52021-07-15 10:42:25 -0500472 self.logger.info("\n\n\tCopying FFDC files from remote system %s.\n" % self.hostname)
Peter D Phan2b8052d2021-06-22 10:55:41 -0500473
Peter D Phan04aca3b2021-06-21 10:37:18 -0500474 # Retrieving files from target system
Peter D Phanbabf2962021-07-07 11:24:40 -0500475 list_of_files = self.get_file_list(ffdc_actions_for_machine_type)
Peter D Phan2b8052d2021-06-22 10:55:41 -0500476 self.scp_ffdc(self.ffdc_dir_path, self.ffdc_prefix, form_filename, list_of_files)
Peter D Phan04aca3b2021-06-21 10:37:18 -0500477 else:
Peter D Phane86d9a52021-07-15 10:42:25 -0500478 self.logger.info("\n\n\tSkip copying FFDC files from remote system %s.\n" % self.hostname)
Peter D Phan04aca3b2021-06-21 10:37:18 -0500479
Peter D Phanbabf2962021-07-07 11:24:40 -0500480 def get_command_list(self,
481 ffdc_actions_for_machine_type):
482 r"""
483 Fetch list of commands from configuration file
484
485 Description of argument(s):
486 ffdc_actions_for_machine_type commands and files for the selected remote host type.
487 """
488 try:
489 list_of_commands = ffdc_actions_for_machine_type['COMMANDS']
490 except KeyError:
491 list_of_commands = []
492 return list_of_commands
493
494 def get_file_list(self,
495 ffdc_actions_for_machine_type):
496 r"""
497 Fetch list of commands from configuration file
498
499 Description of argument(s):
500 ffdc_actions_for_machine_type commands and files for the selected remote host type.
501 """
502 try:
503 list_of_files = ffdc_actions_for_machine_type['FILES']
504 except KeyError:
505 list_of_files = []
506 return list_of_files
507
Peter D Phan5963d632021-07-12 09:58:55 -0500508 def unpack_command(self,
509 command):
510 r"""
511 Unpack command from config file
512
513 Description of argument(s):
514 command Command from config file.
515 """
516 if isinstance(command, dict):
517 command_txt = next(iter(command))
518 command_timeout = next(iter(command.values()))
519 elif isinstance(command, str):
520 command_txt = command
521 # Default command timeout 60 seconds
522 command_timeout = 60
523
524 return command_txt, command_timeout
525
Peter D Phan3beb02e2021-07-06 13:25:17 -0500526 def ssh_execute_ffdc_commands(self,
527 ffdc_actions_for_machine_type,
528 form_filename=False):
529 r"""
530 Send commands in ffdc_config file to targeted system.
531
532 Description of argument(s):
533 ffdc_actions_for_machine_type commands and files for the selected remote host type.
534 form_filename if true, pre-pend self.target_type to filename
535 """
Peter D Phane86d9a52021-07-15 10:42:25 -0500536 self.logger.info("\n\t[Run] Executing commands on %s using %s"
537 % (self.hostname, ffdc_actions_for_machine_type['PROTOCOL'][0]))
Peter D Phan3beb02e2021-07-06 13:25:17 -0500538
Peter D Phanbabf2962021-07-07 11:24:40 -0500539 list_of_commands = self.get_command_list(ffdc_actions_for_machine_type)
Peter D Phan3beb02e2021-07-06 13:25:17 -0500540 # If command list is empty, returns
541 if not list_of_commands:
542 return
543
544 progress_counter = 0
545 for command in list_of_commands:
Peter D Phan5963d632021-07-12 09:58:55 -0500546 command_txt, command_timeout = self.unpack_command(command)
Peter D Phan3beb02e2021-07-06 13:25:17 -0500547
548 if form_filename:
549 command_txt = str(command_txt % self.target_type)
550
Peter D Phan2b6cb3a2021-07-19 06:55:42 -0500551 cmd_exit_code, err, response = \
552 self.ssh_remoteclient.execute_command(command_txt, command_timeout)
553
554 if cmd_exit_code:
555 self.logger.warning(
556 "\n\t\t[WARN] %s exits with code %s." % (command_txt, str(cmd_exit_code)))
557 self.logger.warning("\t\t[WARN] %s " % err)
Peter D Phanbabf2962021-07-07 11:24:40 -0500558
Peter D Phan3beb02e2021-07-06 13:25:17 -0500559 progress_counter += 1
560 self.print_progress(progress_counter)
561
Peter D Phane86d9a52021-07-15 10:42:25 -0500562 self.logger.info("\n\t[Run] Commands execution completed.\t\t [OK]")
Peter D Phan3beb02e2021-07-06 13:25:17 -0500563
Peter D Phan56429a62021-06-23 08:38:29 -0500564 def group_copy(self,
565 ffdc_actions_for_machine_type):
Peter D Phan56429a62021-06-23 08:38:29 -0500566 r"""
567 scp group of files (wild card) from remote host.
568
569 Description of argument(s):
570 ffdc_actions_for_machine_type commands and files for the selected remote host type.
571 """
Peter D Phan3beb02e2021-07-06 13:25:17 -0500572
Peter D Phan5963d632021-07-12 09:58:55 -0500573 if self.ssh_remoteclient.scpclient:
Peter D Phane86d9a52021-07-15 10:42:25 -0500574 self.logger.info("\n\tCopying DUMP files from remote system %s.\n" % self.hostname)
Peter D Phan56429a62021-06-23 08:38:29 -0500575
Peter D Phanbabf2962021-07-07 11:24:40 -0500576 list_of_commands = self.get_command_list(ffdc_actions_for_machine_type)
577 # If command list is empty, returns
578 if not list_of_commands:
579 return
Peter D Phan56429a62021-06-23 08:38:29 -0500580
Peter D Phanbabf2962021-07-07 11:24:40 -0500581 for command in list_of_commands:
582 try:
583 filename = command.split(' ')[2]
584 except IndexError:
Peter D Phane86d9a52021-07-15 10:42:25 -0500585 self.logger.info("\t\tInvalid command %s for DUMP_LOGS block." % command)
Peter D Phanbabf2962021-07-07 11:24:40 -0500586 continue
587
Peter D Phan2b6cb3a2021-07-19 06:55:42 -0500588 cmd_exit_code, err, response = \
589 self.ssh_remoteclient.execute_command(command)
Peter D Phanbabf2962021-07-07 11:24:40 -0500590
Peter D Phan2b6cb3a2021-07-19 06:55:42 -0500591 # If file does not exist, code take no action.
592 # cmd_exit_code is ignored for this scenario.
Peter D Phan56429a62021-06-23 08:38:29 -0500593 if response:
Peter D Phan5963d632021-07-12 09:58:55 -0500594 scp_result = self.ssh_remoteclient.scp_file_from_remote(filename, self.ffdc_dir_path)
Peter D Phan56429a62021-06-23 08:38:29 -0500595 if scp_result:
Peter D Phane86d9a52021-07-15 10:42:25 -0500596 self.logger.info("\t\tSuccessfully copied from " + self.hostname + ':' + filename)
Peter D Phan56429a62021-06-23 08:38:29 -0500597 else:
Peter D Phane86d9a52021-07-15 10:42:25 -0500598 self.logger.info("\t\tThere is no " + filename)
Peter D Phan56429a62021-06-23 08:38:29 -0500599
600 else:
Peter D Phane86d9a52021-07-15 10:42:25 -0500601 self.logger.info("\n\n\tSkip copying files from remote system %s.\n" % self.hostname)
Peter D Phan56429a62021-06-23 08:38:29 -0500602
Peter D Phan72ce6b82021-06-03 06:18:26 -0500603 def scp_ffdc(self,
604 targ_dir_path,
Peter D Phan2b8052d2021-06-22 10:55:41 -0500605 targ_file_prefix,
606 form_filename,
Peter D Phan72ce6b82021-06-03 06:18:26 -0500607 file_list=None,
608 quiet=None):
Peter D Phan72ce6b82021-06-03 06:18:26 -0500609 r"""
610 SCP all files in file_dict to the indicated directory on the local system.
611
612 Description of argument(s):
613 targ_dir_path The path of the directory to receive the files.
614 targ_file_prefix Prefix which will be pre-pended to each
615 target file's name.
616 file_dict A dictionary of files to scp from targeted system to this system
617
618 """
619
Peter D Phan72ce6b82021-06-03 06:18:26 -0500620 progress_counter = 0
621 for filename in file_list:
Peter D Phan2b8052d2021-06-22 10:55:41 -0500622 if form_filename:
623 filename = str(filename % self.target_type)
Peter D Phan72ce6b82021-06-03 06:18:26 -0500624 source_file_path = filename
625 targ_file_path = targ_dir_path + targ_file_prefix + filename.split('/')[-1]
626
Peter D Phanbabf2962021-07-07 11:24:40 -0500627 # If source file name contains wild card, copy filename as is.
628 if '*' in source_file_path:
Peter D Phan5963d632021-07-12 09:58:55 -0500629 scp_result = self.ssh_remoteclient.scp_file_from_remote(source_file_path, self.ffdc_dir_path)
Peter D Phanbabf2962021-07-07 11:24:40 -0500630 else:
Peter D Phan5963d632021-07-12 09:58:55 -0500631 scp_result = self.ssh_remoteclient.scp_file_from_remote(source_file_path, targ_file_path)
Peter D Phan72ce6b82021-06-03 06:18:26 -0500632
633 if not quiet:
634 if scp_result:
Peter D Phane86d9a52021-07-15 10:42:25 -0500635 self.logger.info(
636 "\t\tSuccessfully copied from " + self.hostname + ':' + source_file_path + ".\n")
Peter D Phan72ce6b82021-06-03 06:18:26 -0500637 else:
Peter D Phane86d9a52021-07-15 10:42:25 -0500638 self.logger.info(
639 "\t\tFail to copy from " + self.hostname + ':' + source_file_path + ".\n")
Peter D Phan72ce6b82021-06-03 06:18:26 -0500640 else:
641 progress_counter += 1
642 self.print_progress(progress_counter)
643
Peter D Phan72ce6b82021-06-03 06:18:26 -0500644 def set_ffdc_defaults(self):
Peter D Phan72ce6b82021-06-03 06:18:26 -0500645 r"""
646 Set a default value for self.ffdc_dir_path and self.ffdc_prefix.
647 Collected ffdc file will be stored in dir /self.location/hostname_timestr/.
648 Individual ffdc file will have timestr_filename.
649
650 Description of class variables:
651 self.ffdc_dir_path The dir path where collected ffdc data files should be put.
652
653 self.ffdc_prefix The prefix to be given to each ffdc file name.
654
655 """
656
657 timestr = time.strftime("%Y%m%d-%H%M%S")
658 self.ffdc_dir_path = self.location + "/" + self.hostname + "_" + timestr + "/"
659 self.ffdc_prefix = timestr + "_"
660 self.validate_local_store(self.ffdc_dir_path)
661
662 def validate_local_store(self, dir_path):
663 r"""
664 Ensure path exists to store FFDC files locally.
665
666 Description of variable:
667 dir_path The dir path where collected ffdc data files will be stored.
668
669 """
670
671 if not os.path.exists(dir_path):
672 try:
George Keishing7b3a5132021-07-13 09:24:02 -0500673 os.makedirs(dir_path, 0o755)
Peter D Phan72ce6b82021-06-03 06:18:26 -0500674 except (IOError, OSError) as e:
675 # PermissionError
676 if e.errno == EPERM or e.errno == EACCES:
Peter D Phane86d9a52021-07-15 10:42:25 -0500677 self.logger.error(
678 '>>>>>\tERROR: os.makedirs %s failed with PermissionError.\n' % dir_path)
Peter D Phan72ce6b82021-06-03 06:18:26 -0500679 else:
Peter D Phane86d9a52021-07-15 10:42:25 -0500680 self.logger.error(
681 '>>>>>\tERROR: os.makedirs %s failed with %s.\n' % (dir_path, e.strerror))
Peter D Phan72ce6b82021-06-03 06:18:26 -0500682 sys.exit(-1)
683
684 def print_progress(self, progress):
685 r"""
686 Print activity progress +
687
688 Description of variable:
689 progress Progress counter.
690
691 """
692
693 sys.stdout.write("\r\t" + "+" * progress)
694 sys.stdout.flush()
695 time.sleep(.1)
Peter D Phan0c669772021-06-24 13:52:42 -0500696
697 def verify_redfish(self):
698 r"""
699 Verify remote host has redfish service active
700
701 """
702 redfish_parm = '-u ' + self.username + ' -p ' + self.password + ' -r ' \
703 + self.hostname + ' -S Always raw GET /redfish/v1/'
704 return(self.run_redfishtool(redfish_parm, True))
705
George Keishingeafba182021-06-29 13:44:58 -0500706 def verify_ipmi(self):
707 r"""
708 Verify remote host has IPMI LAN service active
709
710 """
711 ipmi_parm = '-U ' + self.username + ' -P ' + self.password + ' -H ' \
712 + self.hostname + ' power status'
713 return(self.run_ipmitool(ipmi_parm, True))
714
Peter D Phan0c669772021-06-24 13:52:42 -0500715 def run_redfishtool(self,
716 parms_string,
717 quiet=False):
718 r"""
719 Run CLI redfishtool
720
721 Description of variable:
722 parms_string redfishtool subcommand and options.
723 quiet do not print redfishtool error message if True
724 """
725
726 result = subprocess.run(['redfishtool ' + parms_string],
727 stdout=subprocess.PIPE,
728 stderr=subprocess.PIPE,
729 shell=True,
730 universal_newlines=True)
731
732 if result.stderr and not quiet:
Peter D Phane86d9a52021-07-15 10:42:25 -0500733 self.logger.error('\n\t\tERROR with redfishtool ' + parms_string)
734 self.logger.error('\t\t' + result.stderr)
Peter D Phan0c669772021-06-24 13:52:42 -0500735
736 return result.stdout
George Keishingeafba182021-06-29 13:44:58 -0500737
738 def run_ipmitool(self,
739 parms_string,
740 quiet=False):
741 r"""
742 Run CLI IPMI tool.
743
744 Description of variable:
745 parms_string ipmitool subcommand and options.
746 quiet do not print redfishtool error message if True
747 """
748
749 result = subprocess.run(['ipmitool -I lanplus -C 17 ' + parms_string],
750 stdout=subprocess.PIPE,
751 stderr=subprocess.PIPE,
752 shell=True,
753 universal_newlines=True)
754
755 if result.stderr and not quiet:
Peter D Phane86d9a52021-07-15 10:42:25 -0500756 self.logger.error('\n\t\tERROR with ipmitool -I lanplus -C 17 ' + parms_string)
757 self.logger.error('\t\t' + result.stderr)
George Keishingeafba182021-06-29 13:44:58 -0500758
759 return result.stdout
George Keishing04d29102021-07-16 02:05:57 -0500760
761 def run_shell_script(self,
762 parms_string,
763 quiet=False):
764 r"""
765 Run CLI shell script tool.
766
767 Description of variable:
768 parms_string script command options.
769 quiet do not print redfishtool error message if True
770 """
771
772 result = subprocess.run([parms_string],
773 stdout=subprocess.PIPE,
774 stderr=subprocess.PIPE,
775 shell=True,
776 universal_newlines=True)
777
778 if result.stderr and not quiet:
779 self.logger.error('\n\t\tERROR executing %s' % parms_string)
780 self.logger.error('\t\t' + result.stderr)
781
782 return result.stdout
783
784 def protocol_shell_script(self,
785 ffdc_actions,
786 machine_type,
787 sub_type):
788 r"""
789 Perform SHELL script execution locally.
790
791 Description of argument(s):
792 ffdc_actions List of actions from ffdc_config.yaml.
793 machine_type OS Type of remote host.
794 sub_type Group type of commands.
795 """
796
797 self.logger.info("\n\t[Run] Executing commands to %s using %s" % (self.hostname, 'SHELL'))
798 shell_files_saved = []
799 progress_counter = 0
800 list_of_cmd = self.get_command_list(ffdc_actions[machine_type][sub_type])
801 for index, each_cmd in enumerate(list_of_cmd, start=0):
802
803 result = self.run_shell_script(each_cmd)
804 if result:
805 try:
806 targ_file = self.get_file_list(ffdc_actions[machine_type][sub_type])[index]
807 except IndexError:
808 targ_file = each_cmd.split('/')[-1]
809 self.logger.warning("\n\t[WARN] Missing filename to store data %s." % each_cmd)
810 self.logger.warning("\t[WARN] Data will be stored in %s." % targ_file)
811
812 targ_file_with_path = (self.ffdc_dir_path
813 + self.ffdc_prefix
814 + targ_file)
815
816 # Creates a new file
817 with open(targ_file_with_path, 'w') as fp:
818 fp.write(result)
819 fp.close
820 shell_files_saved.append(targ_file)
821
822 progress_counter += 1
823 self.print_progress(progress_counter)
824
825 self.logger.info("\n\t[Run] Commands execution completed.\t\t [OK]")
826
827 for file in shell_files_saved:
828 self.logger.info("\n\t\tSuccessfully save file " + file + ".")