lib/gen_arg.py

#!/usr/bin/env python

r"""
This module provides valuable argument processing functions like gen_get_options and sprint_args.
"""

import sys
import os
import re
try:
    import psutil
    psutil_imported = True
except ImportError:
    psutil_imported = False
try:
    import __builtin__
except ImportError:
    import builtins as __builtin__
import atexit
import signal
import argparse
import textwrap as textwrap

import gen_print as gp
import gen_valid as gv
import gen_cmd as gc
import gen_misc as gm


class MultilineFormatter(argparse.HelpFormatter):
    def _fill_text(self, text, width, indent):
        r"""
        Split text into formatted lines for every "%%n" encountered in the text and return the result.
        """
        lines = self._whitespace_matcher.sub(' ', text).strip().split('%n')
        formatted_lines = \
            [textwrap.fill(x, width, initial_indent=indent, subsequent_indent=indent) + '\n' for x in lines]
        return ''.join(formatted_lines)


class ArgumentDefaultsHelpMultilineFormatter(MultilineFormatter, argparse.ArgumentDefaultsHelpFormatter):
    pass


default_string = '  The default value is "%(default)s".'
module = sys.modules["__main__"]


def gen_get_options(parser,
                    stock_list=[]):
    r"""
    Parse the command line arguments using the parser object passed and return True/False (i.e. pass/fail).
    However, if gv.exit_on_error is set, simply exit the program on failure.  Also set the following built in
    values:

    __builtin__.quiet      This value is used by the qprint functions.
    __builtin__.test_mode  This value is used by command processing functions.
    __builtin__.debug      This value is used by the dprint functions.
    __builtin__.arg_obj    This value is used by print_program_header, etc.
    __builtin__.parser     This value is used by print_program_header, etc.

    Description of arguments:
    parser                          A parser object.  See argparse module documentation for details.
    stock_list                      The caller can use this parameter to request certain stock parameters
                                    offered by this function.  For example, this function will define a
                                    "quiet" option upon request.  This includes stop help text and parm
                                    checking.  The stock_list is a list of tuples each of which consists of
                                    an arg_name and a default value.  Example: stock_list = [("test_mode",
                                    0), ("quiet", 1), ("debug", 0)]
    """

    # This is a list of stock parms that we support.
    master_stock_list = ["quiet", "test_mode", "debug", "loglevel"]

    # Process stock_list.
    for ix in range(0, len(stock_list)):
        if len(stock_list[ix]) < 1:
            error_message = "Programmer error - stock_list[" + str(ix) +\
                            "] is supposed to be a tuple containing at" +\
                            " least one element which is the name of" +\
                            " the desired stock parameter:\n" +\
                            gp.sprint_var(stock_list)
            return gv.process_error_message(error_message)
        if isinstance(stock_list[ix], tuple):
            arg_name = stock_list[ix][0]
            default = stock_list[ix][1]
        else:
            arg_name = stock_list[ix]
            default = None

        if arg_name not in master_stock_list:
            error_message = "Programmer error - arg_name \"" + arg_name +\
                            "\" not found found in stock list:\n" +\
                            gp.sprint_var(master_stock_list)
            return gv.process_error_message(error_message)

        if arg_name == "quiet":
            if default is None:
                default = 0
            parser.add_argument(
                '--quiet',
                default=default,
                type=int,
                choices=[1, 0],
                help='If this parameter is set to "1", %(prog)s'
                     + ' will print only essential information, i.e. it will'
                     + ' not echo parameters, echo commands, print the total'
                     + ' run time, etc.' + default_string)
        elif arg_name == "test_mode":
            if default is None:
                default = 0
            parser.add_argument(
                '--test_mode',
                default=default,
                type=int,
                choices=[1, 0],
                help='This means that %(prog)s should go through all the'
                     + ' motions but not actually do anything substantial.'
                     + '  This is mainly to be used by the developer of'
                     + ' %(prog)s.' + default_string)
        elif arg_name == "debug":
            if default is None:
                default = 0
            parser.add_argument(
                '--debug',
                default=default,
                type=int,
                choices=[1, 0],
                help='If this parameter is set to "1", %(prog)s will print'
                     + ' additional debug information.  This is mainly to be'
                     + ' used by the developer of %(prog)s.' + default_string)
        elif arg_name == "loglevel":
            if default is None:
                default = "info"
            parser.add_argument(
                '--loglevel',
                default=default,
                type=str,
                choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL',
                         'debug', 'info', 'warning', 'error', 'critical'],
                help='If this parameter is set to "1", %(prog)s will print'
                     + ' additional debug information.  This is mainly to be'
                     + ' used by the developer of %(prog)s.' + default_string)

    arg_obj = parser.parse_args()

    __builtin__.quiet = 0
    __builtin__.test_mode = 0
    __builtin__.debug = 0
    __builtin__.loglevel = 'WARNING'
    for ix in range(0, len(stock_list)):
        if isinstance(stock_list[ix], tuple):
            arg_name = stock_list[ix][0]
            default = stock_list[ix][1]
        else:
            arg_name = stock_list[ix]
            default = None
        if arg_name == "quiet":
            __builtin__.quiet = arg_obj.quiet
        elif arg_name == "test_mode":
            __builtin__.test_mode = arg_obj.test_mode
        elif arg_name == "debug":
            __builtin__.debug = arg_obj.debug
        elif arg_name == "loglevel":
            __builtin__.loglevel = arg_obj.loglevel

    __builtin__.arg_obj = arg_obj
    __builtin__.parser = parser

    # For each command line parameter, create a corresponding global variable and assign it the appropriate
    # value.  For example, if the command line contained "--last_name='Smith', we'll create a global variable
    # named "last_name" with the value "Smith".
    module = sys.modules['__main__']
    for key in arg_obj.__dict__:
        setattr(module, key, getattr(__builtin__.arg_obj, key))

    return True


def set_pgm_arg(var_value,
                var_name=None):
    r"""
    Set the value of the arg_obj.__dict__ entry named in var_name with the var_value provided.  Also, set
    corresponding global variable.

    Description of arguments:
    var_value                       The value to set in the variable.
    var_name                        The name of the variable to set.  This defaults to the name of the
                                    variable used for var_value when calling this function.
    """

    if var_name is None:
        var_name = gp.get_arg_name(None, 1, 2)

    arg_obj.__dict__[var_name] = var_value
    module = sys.modules['__main__']
    setattr(module, var_name, var_value)
    if var_name == "quiet":
        __builtin__.quiet = var_value
    elif var_name == "debug":
        __builtin__.debug = var_value
    elif var_name == "test_mode":
        __builtin__.test_mode = var_value


def sprint_args(arg_obj,
                indent=0):
    r"""
    sprint_var all of the arguments found in arg_obj and return the result as a string.

    Description of arguments:
    arg_obj                         An argument object such as is returned by the argparse parse_args()
                                    method.
    indent                          The number of spaces to indent each line of output.
    """

    col1_width = gp.dft_col1_width + indent

    buffer = ""
    for key in arg_obj.__dict__:
        buffer += gp.sprint_varx(key, getattr(arg_obj, key), 0, indent,
                                 col1_width)
    return buffer


def sync_args():
    r"""
    Synchronize the argument values to match their corresponding global variable values.

    The user's validate_parms() function may manipulate global variables that correspond to program
    arguments.  After validate_parms() is called, sync_args is called to set the altered values back into the
    arg_obj.  This will ensure that the print-out of program arguments reflects the updated values.

    Example:

    def validate_parms():

        # Set a default value for dir_path argument.
        dir_path = gm.add_trailing_slash(gm.dft(dir_path, os.getcwd()))
    """
    module = sys.modules['__main__']
    for key in arg_obj.__dict__:
        arg_obj.__dict__[key] = getattr(module, key)


term_options = None


def set_term_options(**kwargs):
    r"""
    Set the global term_options.

    If the global term_options is not None, gen_exit_function() will call terminate_descendants().

    Description of arguments():
    kwargs                          Supported keyword options follow:
        term_requests               Requests to terminate specified descendants of this program.  The
                                    following values for term_requests are supported:
            children                Terminate the direct children of this program.
            descendants             Terminate all descendants of this program.
            <dictionary>            A dictionary with support for the following keys:
                pgm_names           A list of program names which will be used to identify which descendant
                                    processes should be terminated.
    """

    global term_options
    # Validation:
    arg_names = list(kwargs.keys())
    gv.valid_list(arg_names, ['term_requests'])
    if type(kwargs['term_requests']) is dict:
        keys = list(kwargs['term_requests'].keys())
        gv.valid_list(keys, ['pgm_names'])
    else:
        gv.valid_value(kwargs['term_requests'], ['children', 'descendants'])
    term_options = kwargs


if psutil_imported:
    def match_process_by_pgm_name(process, pgm_name):
        r"""
        Return True or False to indicate whether the process matches the program name.

        Description of argument(s):
        process                     A psutil process object such as the one returned by psutil.Process().
        pgm_name                    The name of a program to look for in the cmdline field of the process
                                    object.
        """

        # This function will examine elements 0 and 1 of the cmdline field of the process object.  The
        # following examples will illustrate the reasons for this:

        # Example 1: Suppose a process was started like this:

        # shell_cmd('python_pgm_template --quiet=0', fork=1)

        # And then this function is called as follows:

        # match_process_by_pgm_name(process, "python_pgm_template")

        # The process object might contain the following for its cmdline field:

        # cmdline:
        #   [0]:                       /usr/bin/python
        #   [1]:                       /my_path/python_pgm_template
        #   [2]:                       --quiet=0

        # Because "python_pgm_template" is a python program, the python interpreter (e.g. "/usr/bin/python")
        # will appear in entry 0 of cmdline and the python_pgm_template will appear in entry 1 (with a
        # qualifying dir path).

        # Example 2: Suppose a process was started like this:

        # shell_cmd('sleep 5', fork=1)

        # And then this function is called as follows:

        # match_process_by_pgm_name(process, "sleep")

        # The process object might contain the following for its cmdline field:

        # cmdline:
        #   [0]:                       sleep
        #   [1]:                       5

        # Because "sleep" is a compiled executable, it will appear in entry 0.

        optional_dir_path_regex = "(.*/)?"
        cmdline = process.as_dict()['cmdline']
        return re.match(optional_dir_path_regex + pgm_name + '( |$)', cmdline[0]) \
            or re.match(optional_dir_path_regex + pgm_name + '( |$)', cmdline[1])

    def select_processes_by_pgm_name(processes, pgm_name):
        r"""
        Select the processes that match pgm_name and return the result as a list of process objects.

        Description of argument(s):
        processes                   A list of psutil process objects such as the one returned by
                                    psutil.Process().
        pgm_name                    The name of a program to look for in the cmdline field of each process
                                    object.
        """

        return [process for process in processes if match_process_by_pgm_name(process, pgm_name)]

    def sprint_process_report(pids):
        r"""
        Create a process report for the given pids and return it as a string.

        Description of argument(s):
        pids                        A list of process IDs for processes to be included in the report.
        """
        report = "\n"
        cmd_buf = "echo ; ps wwo user,pgrp,pid,ppid,lstart,cmd --forest " + ' '.join(pids)
        report += gp.sprint_issuing(cmd_buf)
        rc, outbuf = gc.shell_cmd(cmd_buf, quiet=1)
        report += outbuf + "\n"

        return report

    def get_descendant_info(process=psutil.Process()):
        r"""
        Get info about the descendants of the given process and return as a tuple of descendants,
        descendant_pids and process_report.

        descendants will be a list of process objects.  descendant_pids will be a list of pids (in str form)
        and process_report will be a report produced by a call to sprint_process_report().

        Description of argument(s):
        process                     A psutil process object such as the one returned by psutil.Process().
        """
        descendants = process.children(recursive=True)
        descendant_pids = [str(process.pid) for process in descendants]
        if descendants:
            process_report = sprint_process_report([str(process.pid)] + descendant_pids)
        else:
            process_report = ""
        return descendants, descendant_pids, process_report

    def terminate_descendants():
        r"""
        Terminate descendants of the current process according to the requirements layed out in global
        term_options variable.

        Note: If term_options is not null, gen_exit_function() will automatically call this function.

        When this function gets called, descendant processes may be running and may be printing to the same
        stdout stream being used by this process.  If this function writes directly to stdout, its output can
        be interspersed with any output generated by descendant processes.  This makes it very difficult to
        interpret the output.  In order solve this problem, the activity of this process will be logged to a
        temporary file.  After descendant processes have been terminated successfully, the temporary file
        will be printed to stdout and then deleted.  However, if this function should fail to complete (i.e.
        get hung waiting for descendants to terminate gracefully), the temporary file will not be deleted and
        can be used by the developer for debugging.  If no descendant processes are found, this function will
        return before creating the temporary file.

        Note that a general principal being observed here is that each process is responsible for the
        children it produces.
        """

        message = "\n" + gp.sprint_dashes(width=120) \
            + gp.sprint_executing() + "\n"

        current_process = psutil.Process()

        descendants, descendant_pids, process_report = get_descendant_info(current_process)
        if not descendants:
            # If there are no descendants, then we have nothing to do.
            return

        terminate_descendants_temp_file_path = gm.create_temp_file_path()
        gp.print_vars(terminate_descendants_temp_file_path)

        message += gp.sprint_varx("pgm_name", gp.pgm_name) \
            + gp.sprint_vars(term_options) \
            + process_report

        # Process the termination requests:
        if term_options['term_requests'] == 'children':
            term_processes = current_process.children(recursive=False)
            term_pids = [str(process.pid) for process in term_processes]
        elif term_options['term_requests'] == 'descendants':
            term_processes = descendants
            term_pids = descendant_pids
        else:
            # Process term requests by pgm_names.
            term_processes = []
            for pgm_name in term_options['term_requests']['pgm_names']:
                term_processes.extend(select_processes_by_pgm_name(descendants, pgm_name))
            term_pids = [str(process.pid) for process in term_processes]

        message += gp.sprint_timen("Processes to be terminated:") \
            + gp.sprint_var(term_pids)
        for process in term_processes:
            process.terminate()
        message += gp.sprint_timen("Waiting on the following pids: " + ' '.join(descendant_pids))
        gm.append_file(terminate_descendants_temp_file_path, message)
        psutil.wait_procs(descendants)

        # Checking after the fact to see whether any descendant processes are still alive.  If so, a process
        # report showing this will be included in the output.
        descendants, descendant_pids, process_report = get_descendant_info(current_process)
        if descendants:
            message = "\n" + gp.sprint_timen("Not all of the processes terminated:") \
                + process_report
            gm.append_file(terminate_descendants_temp_file_path, message)

        message = gp.sprint_dashes(width=120)
        gm.append_file(terminate_descendants_temp_file_path, message)
        gp.print_file(terminate_descendants_temp_file_path)
        os.remove(terminate_descendants_temp_file_path)


def gen_exit_function():
    r"""
    Execute whenever the program ends normally or with the signals that we catch (i.e. TERM, INT).
    """

    # ignore_err influences the way shell_cmd processes errors.  Since we're doing exit processing, we don't
    # want to stop the program due to a shell_cmd failure.
    ignore_err = 1

    if psutil_imported and term_options:
        terminate_descendants()

    # Call the main module's exit_function if it is defined.
    exit_function = getattr(module, "exit_function", None)
    if exit_function:
        exit_function()

    gp.qprint_pgm_footer()


def gen_signal_handler(signal_number,
                       frame):
    r"""
    Handle signals.  Without a function to catch a SIGTERM or SIGINT, the program would terminate immediately
    with return code 143 and without calling the exit_function.
    """

    # The convention is to set up exit_function with atexit.register() so there is no need to explicitly
    # call exit_function from here.

    gp.qprint_executing()

    # Calling exit prevents control from returning to the code that was running when the signal was received.
    exit(0)


def gen_post_validation(exit_function=None,
                        signal_handler=None):
    r"""
    Do generic post-validation processing.  By "post", we mean that this is to be called from a validation
    function after the caller has done any validation desired.  If the calling program passes exit_function
    and signal_handler parms, this function will register them.  In other words, it will make the
    signal_handler functions get called for SIGINT and SIGTERM and will make the exit_function function run
    prior to the termination of the program.

    Description of arguments:
    exit_function                   A function object pointing to the caller's exit function.  This defaults
                                    to this module's gen_exit_function.
    signal_handler                  A function object pointing to the caller's signal_handler function.  This
                                    defaults to this module's gen_signal_handler.
    """

    # Get defaults.
    exit_function = exit_function or gen_exit_function
    signal_handler = signal_handler or gen_signal_handler

    atexit.register(exit_function)
    signal.signal(signal.SIGINT, signal_handler)
    signal.signal(signal.SIGTERM, signal_handler)


def gen_setup():
    r"""
    Do general setup for a program.
    """

    # Set exit_on_error for gen_valid functions.
    gv.set_exit_on_error(True)

    # Get main module variable values.
    parser = getattr(module, "parser")
    stock_list = getattr(module, "stock_list")
    validate_parms = getattr(module, "validate_parms", None)

    gen_get_options(parser, stock_list)

    if validate_parms:
        validate_parms()
        sync_args()
    gen_post_validation()

    gp.qprint_pgm_header()