ffdc/ffdc_collector.py

#!/usr/bin/env python

r"""
See class prolog below for details.
"""

import os
import sys
import yaml
import time
import platform
from errno import EACCES, EPERM
from ssh_utility import SSHRemoteclient


class FFDCCollector:

    r"""
    Sends commands from configuration file to the targeted system to collect log files.
    Fetch and store generated files at the specified location.

    """

    # List of supported OSes.
    supported_oses = ['OPENBMC', 'RHEL', 'AIX', 'UBUNTU']

    def __init__(self, hostname, username, password, ffdc_config, location, remote_type):
        r"""
        Description of argument(s):

        hostname                name/ip of the targeted (remote) system
        username                user on the targeted system with access to FFDC files
        password                password for user on targeted system
        ffdc_config             configuration file listing commands and files for FFDC
        location                where to store collected FFDC
        remote_type             os type of the remote host

        """
        if self.verify_script_env():
            self.hostname = hostname
            self.username = username
            self.password = password
            self.ffdc_config = ffdc_config
            self.location = location
            self.remote_client = None
            self.ffdc_dir_path = ""
            self.ffdc_prefix = ""
            self.target_type = remote_type.upper()
        else:
            sys.exit(-1)

    def verify_script_env(self):

        # Import to log version
        import click
        import paramiko

        run_env_ok = True
        print("\n\t---- Script host environment ----")
        print("\t{:<10}  {:<10}".format('Script hostname', os.uname()[1]))
        print("\t{:<10}  {:<10}".format('Script host os', platform.platform()))
        print("\t{:<10}  {:>10}".format('Python', platform.python_version()))
        print("\t{:<10}  {:>10}".format('PyYAML', yaml.__version__))
        print("\t{:<10}  {:>10}".format('click', click.__version__))
        print("\t{:<10}  {:>10}".format('paramiko', paramiko.__version__))

        if eval(yaml.__version__.replace('.', ',')) < (5, 4, 1):
            print("\n\tERROR: Python or python packages do not meet minimum version requirement.")
            print("\tERROR: PyYAML version 5.4.1 or higher is needed.\n")
            run_env_ok = False

        print("\t---- End script host environment ----")
        return run_env_ok

    def target_is_pingable(self):
        r"""
        Check if target system is ping-able.

        """
        response = os.system("ping -c 1 -w 2 %s  2>&1 >/dev/null" % self.hostname)
        if response == 0:
            print("\n\t[Check] %s is ping-able.\t\t [OK]" % self.hostname)
            return True
        else:
            print("\n>>>>>\tERROR: %s is not ping-able. FFDC collection aborted.\n" % self.hostname)
            sys.exit(-1)

    def inspect_target_machine_type(self):
        r"""
        Inspect remote host os-release or uname.

        """
        command = "cat /etc/os-release"
        response = self.remoteclient.execute_command(command)
        if response:
            print("\n\t[INFO] %s /etc/os-release\n" % self.hostname)
            print("\t\t %s" % self.find_os_type(response, 'PRETTY_NAME'))
            identity = self.find_os_type(response, 'ID').split('=')[1].upper()
        else:
            response = self.remoteclient.execute_command('uname -a')
            print("\n\t[INFO] %s uname -a\n" % self.hostname)
            print("\t\t %s" % ' '.join(response))
            identity = self.find_os_type(response, 'AIX').split(' ')[0].upper()

            # If OS does not have /etc/os-release and is not AIX,
            # script does not yet know what to do.
            if not identity:
                print(">>>>>\tERROR: Script does not yet know about %s" % ' '.join(response))
                sys.exit(-1)

        if self.target_type not in identity:
            user_target_type = self.target_type
            self.target_type = ""
            for each_os in FFDCCollector.supported_oses:
                if each_os in identity:
                    self.target_type = each_os
                    break

            # If OS in not one of ['OPENBMC', 'RHEL', 'AIX', 'UBUNTU']
            # script does not yet know what to do.
            if not self.target_type:
                print(">>>>>\tERROR: Script does not yet know about %s" % identity)
                sys.exit(-1)

            print("\n\t[WARN] user request %s does not match remote host type %s.\n"
                  % (user_target_type, self.target_type))
            print("\t[WARN] FFDC collection continues for %s.\n" % self.target_type)

    def find_os_type(self,
                     listing_from_os,
                     key):

        r"""
        Return OS information with the requested key

        Description of argument(s):

        listing_from_os    list of information returns from OS command
        key                key of the desired data

        """

        for each_item in listing_from_os:
            if key in each_item:
                return each_item
        return ''

    def collect_ffdc(self):
        r"""
        Initiate FFDC Collection depending on requested protocol.

        """

        print("\n\t---- Start communicating with %s ----" % self.hostname)
        working_protocol_list = []
        if self.target_is_pingable():
            # Check supported protocol ping,ssh, redfish are working.
            if self.ssh_to_target_system():
                working_protocol_list.append("SSH")
                working_protocol_list.append("SCP")
            # Verify top level directory exists for storage
            self.validate_local_store(self.location)
            self.inspect_target_machine_type()
            print("\n\t---- Completed protocol pre-requisite check ----\n")
            self.generate_ffdc(working_protocol_list)

    def ssh_to_target_system(self):
        r"""
        Open a ssh connection to targeted system.

        """

        self.remoteclient = SSHRemoteclient(self.hostname,
                                            self.username,
                                            self.password)

        self.remoteclient.ssh_remoteclient_login()
        print("\n\t[Check] %s SSH connection established.\t [OK]" % self.hostname)

        # Check scp connection.
        # If scp connection fails,
        # continue with FFDC generation but skip scp files to local host.
        self.remoteclient.scp_connection()
        return True

    def generate_ffdc(self, working_protocol_list):
        r"""
        Determine actions based on remote host type

        Description of argument(s):
        working_protocol_list    list of confirmed working protocols to connect to remote host.
        """

        print("\n\t---- Executing commands on " + self.hostname + " ----")
        print("\n\tWorking protocol list: %s" % working_protocol_list)
        with open(self.ffdc_config, 'r') as file:
            ffdc_actions = yaml.load(file, Loader=yaml.FullLoader)

        # Set prefix values for scp files and directory.
        # Since the time stamp is at second granularity, these values are set here
        # to be sure that all files for this run will have same timestamps
        # and they will be saved in the same directory.
        # self.location == local system for now
        self.set_ffdc_defaults()

        for machine_type in ffdc_actions.keys():

            if machine_type == self.target_type:
                if (ffdc_actions[machine_type]['PROTOCOL'][0] in working_protocol_list):

                    # For OPENBMC collect general system info.
                    if self.target_type == 'OPENBMC':

                        self.collect_and_copy_ffdc(ffdc_actions['GENERAL'],
                                                   form_filename=True)
                        self.group_copy(ffdc_actions['OPENBMC_DUMPS'])

                    # For RHEL and UBUNTU, collect common Linux OS FFDC.
                    if self.target_type == 'RHEL' \
                            or self.target_type == 'UBUNTU':

                        self.collect_and_copy_ffdc(ffdc_actions['LINUX'])

                    # Collect remote host specific FFDC.
                    self.collect_and_copy_ffdc(ffdc_actions[machine_type])
                else:
                    print("\n\tProtocol %s is not yet supported by this script.\n"
                          % ffdc_actions[machine_type]['PROTOCOL'][0])

        # Close network connection after collecting all files
        self.remoteclient.ssh_remoteclient_disconnect()

    def collect_and_copy_ffdc(self,
                              ffdc_actions_for_machine_type,
                              form_filename=False):
        r"""
        Send commands in ffdc_config file to targeted system.

        Description of argument(s):
        ffdc_actions_for_machine_type    commands and files for the selected remote host type.
        form_filename                    if true, pre-pend self.target_type to filename
        """

        print("\n\t[Run] Executing commands on %s using %s"
              % (self.hostname, ffdc_actions_for_machine_type['PROTOCOL'][0]))
        list_of_commands = ffdc_actions_for_machine_type['COMMANDS']
        progress_counter = 0
        for command in list_of_commands:
            if form_filename:
                command = str(command % self.target_type)
            self.remoteclient.execute_command(command)
            progress_counter += 1
            self.print_progress(progress_counter)

        print("\n\t[Run] Commands execution completed.\t\t [OK]")

        if self.remoteclient.scpclient:
            print("\n\n\tCopying FFDC files from remote system %s.\n" % self.hostname)

            # Retrieving files from target system
            list_of_files = ffdc_actions_for_machine_type['FILES']
            self.scp_ffdc(self.ffdc_dir_path, self.ffdc_prefix, form_filename, list_of_files)
        else:
            print("\n\n\tSkip copying FFDC files from remote system %s.\n" % self.hostname)

    def group_copy(self,
                   ffdc_actions_for_machine_type):

        r"""
        scp group of files (wild card) from remote host.

        Description of argument(s):
        ffdc_actions_for_machine_type    commands and files for the selected remote host type.
        """
        if self.remoteclient.scpclient:
            print("\n\tCopying DUMP files from remote system %s.\n" % self.hostname)

            # Retrieving files from target system, if any
            list_of_files = ffdc_actions_for_machine_type['FILES']

            for filename in list_of_files:
                command = 'ls -AX ' + filename
                response = self.remoteclient.execute_command(command)
                # self.remoteclient.scp_file_from_remote() completed without exception,
                # if any
                if response:
                    scp_result = self.remoteclient.scp_file_from_remote(filename, self.ffdc_dir_path)
                    if scp_result:
                        print("\t\tSuccessfully copied from " + self.hostname + ':' + filename)
                else:
                    print("\t\tThere is no  " + filename)

        else:
            print("\n\n\tSkip copying files from remote system %s.\n" % self.hostname)

    def scp_ffdc(self,
                 targ_dir_path,
                 targ_file_prefix,
                 form_filename,
                 file_list=None,
                 quiet=None):
        r"""
        SCP all files in file_dict to the indicated directory on the local system.

        Description of argument(s):
        targ_dir_path                   The path of the directory to receive the files.
        targ_file_prefix                Prefix which will be pre-pended to each
                                        target file's name.
        file_dict                       A dictionary of files to scp from targeted system to this system

        """

        progress_counter = 0
        for filename in file_list:
            if form_filename:
                filename = str(filename % self.target_type)
            source_file_path = filename
            targ_file_path = targ_dir_path + targ_file_prefix + filename.split('/')[-1]

            # self.remoteclient.scp_file_from_remote() completed without exception,
            # add file to the receiving file list.
            scp_result = self.remoteclient.scp_file_from_remote(source_file_path, targ_file_path)

            if not quiet:
                if scp_result:
                    print("\t\tSuccessfully copied from " + self.hostname + ':' + source_file_path + ".\n")
                else:
                    print("\t\tFail to copy from " + self.hostname + ':' + source_file_path + ".\n")
            else:
                progress_counter += 1
                self.print_progress(progress_counter)

    def set_ffdc_defaults(self):
        r"""
        Set a default value for self.ffdc_dir_path and self.ffdc_prefix.
        Collected ffdc file will be stored in dir /self.location/hostname_timestr/.
        Individual ffdc file will have timestr_filename.

        Description of class variables:
        self.ffdc_dir_path  The dir path where collected ffdc data files should be put.

        self.ffdc_prefix    The prefix to be given to each ffdc file name.

        """

        timestr = time.strftime("%Y%m%d-%H%M%S")
        self.ffdc_dir_path = self.location + "/" + self.hostname + "_" + timestr + "/"
        self.ffdc_prefix = timestr + "_"
        self.validate_local_store(self.ffdc_dir_path)

    def validate_local_store(self, dir_path):
        r"""
        Ensure path exists to store FFDC files locally.

        Description of variable:
        dir_path  The dir path where collected ffdc data files will be stored.

        """

        if not os.path.exists(dir_path):
            try:
                os.mkdir(dir_path, 0o755)
            except (IOError, OSError) as e:
                # PermissionError
                if e.errno == EPERM or e.errno == EACCES:
                    print('>>>>>\tERROR: os.mkdir %s failed with PermissionError.\n' % dir_path)
                else:
                    print('>>>>>\tERROR: os.mkdir %s failed with %s.\n' % (dir_path, e.strerror))
                sys.exit(-1)

    def print_progress(self, progress):
        r"""
        Print activity progress +

        Description of variable:
        progress  Progress counter.

        """

        sys.stdout.write("\r\t" + "+" * progress)
        sys.stdout.flush()
        time.sleep(.1)