FFDC replace eval with importlib code
Changes:
- Remove eval functions
- Add importlib code to replace eval
- Update plugin documentation and YAML
- Tweak code to handle new changes
Tested:
- Ran with the changes from sandbox.
- Tested YAML with different plugin examples
- Tested bad path YAML
Change-Id: Iea0d4b879c1afc4fd25ab3a3c3ccb0e0875f86bc
Signed-off-by: George Keishing <gkeishin@in.ibm.com>
diff --git a/ffdc/docs/plugin.md b/ffdc/docs/plugin.md
index f70fd91..0680ac2 100644
--- a/ffdc/docs/plugin.md
+++ b/ffdc/docs/plugin.md
@@ -92,7 +92,8 @@
```
- plugin:
- - plugin_name: plugin.foo_func.print_vars
+ - plugin_name: plugins.foo_func
+ - plugin_function: print_vars
- plugin_args:
- "Hello plugin"
```
@@ -101,7 +102,8 @@
```
- plugin:
- - plugin_name: return_value = plugin.foo_func.return_vars
+ - plugin_name: plugins.foo_func
+ - plugin_function: return_value = func_return_vars
- plugin_args:
```
@@ -113,7 +115,8 @@
```
- plugin:
- - plugin_name: plugin.foo_func.print_vars
+ - plugin_name: plugins.foo_func
+ - plugin_function: print_vars
- plugin_args:
- return_value
```
@@ -121,7 +124,7 @@
To accept multiple return values by using coma "," separated statement
```
- - plugin_name: return_value1,return_value2 = plugin.foo_func.print_vars
+ - plugin_function: return_value1,return_value2 = print_vars
```
Accessing a class method:
@@ -130,14 +133,16 @@
syntax
```
- - plugin_name: plugin.<file_name>.<class_object>.<class_method>
+ - plugin_name: plugins.<file_name>.<class_object>
+ - plugin_function: <class_method>
```
Example: (from the class example previously mentioned)
```
- plugin:
- - plugin_name: plugin.plugin_class.plugin_class.plugin_print_msg
+ - plugin_name: plugins.plugin_class.plugin_class
+ - plugin_function: plugin_print_msg
- plugin_args:
- self
- "Hello Plugin call"
@@ -191,7 +196,8 @@
```
- plugin:
- - plugin_name: plugin.foo_func.print_vars
+ - plugin_name: plugins.foo_func
+ - plugin_function: print_vars
- plugin_args:
- return_value
- plugin_error: exit_on_error
@@ -215,7 +221,8 @@
```
- plugin:
- plugin:
- - plugin_name: plugin.ssh_execution.ssh_execute_cmd
+ - plugin_name: plugin.ssh_execution
+ - plugin_function: ssh_execute_cmd
- plugin_args:
- ${hostname}
- ${username}
diff --git a/ffdc/docs/yaml_syntax_rules.md b/ffdc/docs/yaml_syntax_rules.md
index 546da52..415048d 100644
--- a/ffdc/docs/yaml_syntax_rules.md
+++ b/ffdc/docs/yaml_syntax_rules.md
@@ -82,7 +82,8 @@
REDFISH_LOGS:
COMMANDS
- plugin:
- - plugin_name: plugin.redfish.enumerate_request
+ - plugin_name: plugin.redfish
+ - plugin_function: enumerate_request
- plugin_args:
- ${hostname}
- ${username}
diff --git a/ffdc/ffdc_collector.py b/ffdc/ffdc_collector.py
index 1467bfa..f8708ad 100644
--- a/ffdc/ffdc_collector.py
+++ b/ffdc/ffdc_collector.py
@@ -4,6 +4,7 @@
See class prolog below for details.
"""
+import importlib
import json
import logging
import os
@@ -13,6 +14,7 @@
import sys
import time
from errno import EACCES, EPERM
+from typing import Any
import yaml
@@ -30,51 +32,22 @@
from telnet_utility import TelnetRemoteclient # NOQA
r"""
-User define plugins python functions.
-
-It will imports files from directory plugins
-
-plugins
-├── file1.py
-└── file2.py
-
-Example how to define in YAML:
- - plugin:
- - plugin_name: plugin.foo_func.foo_func_yaml
- - plugin_args:
- - arg1
- - arg2
-"""
-plugin_dir = os.path.join(os.path.dirname(__file__), "plugins")
-sys.path.append(plugin_dir)
-
-for module in os.listdir(plugin_dir):
- if module == "__init__.py" or not module.endswith(".py"):
- continue
-
- plugin_module = f"plugins.{module[:-3]}"
- try:
- plugin = __import__(plugin_module, globals(), locals(), [], 0)
- except Exception as e:
- print(f"PLUGIN: Exception: {e}")
- print(f"PLUGIN: Module import failed: {module}")
- continue
-
-r"""
This is for plugin functions returning data or responses to the caller
in YAML plugin setup.
Example:
- plugin:
- - plugin_name: version = plugin.ssh_execution.ssh_execute_cmd
+ - plugin_name: plugin.ssh_execution
+ - plugin_function: version = ssh_execute_cmd
- plugin_args:
- ${hostname}
- ${username}
- ${password}
- "cat /etc/os-release | grep VERSION_ID | awk -F'=' '{print $2}'"
- plugin:
- - plugin_name: plugin.print_vars.print_vars
+ - plugin_name: plugin.print_vars
+ - plugin_function: print_vars
- plugin_args:
- version
@@ -82,37 +55,90 @@
block or plugin
"""
+# Global variables for storing plugin return values, plugin return variables,
+# and log storage path.
global global_log_store_path
global global_plugin_dict
global global_plugin_list
+global global_plugin_type_list
+global global_plugin_error_dict
-# Hold the plugin return values in dict and plugin return vars in list.
-# Dict is to reference and update vars processing in parser where as
-# list is for current vars from the plugin block which needs processing.
+# Hold the plugin return values in a dictionary and plugin return variables in
+# a list. The dictionary is used for referencing and updating variables during
+# parsing in the parser, while the list is used for storing current variables
+# from the plugin block that need processing.
global_plugin_dict = {}
global_plugin_list = []
-# Hold the plugin return named declared if function returned values are
-# list,dict.
-# Refer this name list to look up the plugin dict for eval() args function
-# Example ['version']
+# Hold the plugin return named variables if the function returned values are
+# lists or dictionaries. This list is used to reference the plugin dictionary
+# for python function execute arguments.
+# Example: ['version']
global_plugin_type_list = []
# Path where logs are to be stored or written.
global_log_store_path = ""
# Plugin error state defaults.
-plugin_error_dict = {
+global_plugin_error_dict = {
"exit_on_error": False,
"continue_on_error": False,
}
+def execute_python_function(module_name, function_name, *args, **kwargs):
+ r"""
+ Execute a Python function from a module dynamically.
+
+ This function dynamically imports a module and executes a specified
+ function from that module with the provided arguments. The function takes
+ the module name, function name, and arguments as input. The function
+ returns the result of the executed function.
+
+ If an ImportError or AttributeError occurs, the function prints an error
+ message and returns None.
+
+ Parameters:
+ module_name (str): The name of the module containing the function.
+ function_name (str): The name of the function to execute.
+ *args: Positional arguments to pass to the function.
+ **kwargs: Keyword arguments to pass to the function.
+
+ Returns:
+ Any: The result of the executed function or None if an error occurs.
+ """
+ try:
+ # Dynamically import the module.
+ module = importlib.import_module(module_name)
+
+ # Get the function from the module.
+ func = getattr(module, function_name)
+
+ # Call the function with the provided arguments.
+ result = func(*args, **kwargs)
+
+ except (ImportError, AttributeError) as e:
+ print(f"\tERROR: execute_python_function: {e}")
+ # Set the plugin error state.
+ global_plugin_error_dict["exit_on_error"] = True
+ print("\treturn: PLUGIN_EXEC_ERROR")
+ return "PLUGIN_EXEC_ERROR"
+
+ return result
+
+
class ffdc_collector:
r"""
- Execute commands from configuration file to collect log files.
- Fetch and store generated files at the specified location.
+ Execute commands from a configuration file to collect log files and store
+ the generated files at the specified location.
+ This class is designed to execute commands specified in a configuration
+ YAML file to collect log files from a remote host.
+
+ The class establishes connections using SSH, Telnet, or other protocols
+ based on the configuration. It fetches and stores the generated files at
+ the specified location. The class provides methods for initializing the
+ collector, executing commands, and handling errors.
"""
def __init__(
@@ -215,7 +241,7 @@
sys.exit(-1)
self.logger.info("\n\tENV: User define input YAML variables")
- self.env_dict = self.load_env()
+ self.load_env()
else:
sys.exit(-1)
@@ -652,12 +678,12 @@
if "plugin" in each_cmd:
# If the error is set and plugin explicitly
# requested to skip execution on error..
- if plugin_error_dict[
+ if global_plugin_error_dict[
"exit_on_error"
] and self.plugin_error_check(each_cmd["plugin"]):
self.logger.info(
"\n\t[PLUGIN-ERROR] exit_on_error: %s"
- % plugin_error_dict["exit_on_error"]
+ % global_plugin_error_dict["exit_on_error"]
)
self.logger.info(
"\t[PLUGIN-SKIP] %s" % each_cmd["plugin"][0]
@@ -1307,20 +1333,20 @@
self.env attribute for later use.
"""
- os.environ["hostname"] = self.hostname
- os.environ["username"] = self.username
- os.environ["password"] = self.password
- os.environ["port_ssh"] = self.port_ssh
- os.environ["port_https"] = self.port_https
- os.environ["port_ipmi"] = self.port_ipmi
+ tmp_env_vars = {
+ "hostname": self.hostname,
+ "username": self.username,
+ "password": self.password,
+ "port_ssh": self.port_ssh,
+ "port_https": self.port_https,
+ "port_ipmi": self.port_ipmi,
+ }
- # Append default Env.
- self.env_dict["hostname"] = self.hostname
- self.env_dict["username"] = self.username
- self.env_dict["password"] = self.password
- self.env_dict["port_ssh"] = self.port_ssh
- self.env_dict["port_https"] = self.port_https
- self.env_dict["port_ipmi"] = self.port_ipmi
+ # Updatae default Env and Dict var for both so that it can be
+ # verified when referencing it throughout the code.
+ for key, value in tmp_env_vars.items():
+ os.environ[key] = value
+ self.env_dict[key] = value
try:
tmp_env_dict = {}
@@ -1363,48 +1389,6 @@
self.logger.info(json.dumps(mask_dict, indent=8, sort_keys=False))
- def execute_python_eval(self, eval_string):
- r"""
- Execute a qualified Python function string using the eval() function.
-
- This method executes a provided Python function string using the
- eval() function.
-
- The method takes the eval_string as an argument, which is expected to
- be a valid Python function call.
-
- The method returns the result of the executed function.
-
- Example:
- eval(plugin.foo_func.foo_func(10))
-
- Parameters:
- eval_string (str): A valid Python function call string.
-
- Returns:
- str: The result of the executed function and on failure return
- PLUGIN_EVAL_ERROR.
- """
- try:
- self.logger.info("\tExecuting plugin func()")
- self.logger.debug("\tCall func: %s" % eval_string)
- result = eval(eval_string)
- self.logger.info("\treturn: %s" % str(result))
- except (
- ValueError,
- SyntaxError,
- NameError,
- AttributeError,
- TypeError,
- ) as e:
- self.logger.error("\tERROR: execute_python_eval: %s" % e)
- # Set the plugin error state.
- plugin_error_dict["exit_on_error"] = True
- self.logger.info("\treturn: PLUGIN_EVAL_ERROR")
- return "PLUGIN_EVAL_ERROR"
-
- return result
-
def execute_plugin_block(self, plugin_cmd_list):
r"""
Pack the plugin commands into qualified Python string objects.
@@ -1449,17 +1433,28 @@
str: Execute and not response or a string value(s) responses,
"""
+
+ # Declare a variable plugin resp that can accept any data type.
+ resp: Any = ""
+ args_string = ""
+
try:
idx = self.key_index_list_dict("plugin_name", plugin_cmd_list)
+ # Get plugin module name
plugin_name = plugin_cmd_list[idx]["plugin_name"]
+
+ # Get plugin function name
+ idx = self.key_index_list_dict("plugin_function", plugin_cmd_list)
+ plugin_function = plugin_cmd_list[idx]["plugin_function"]
+
# Equal separator means plugin function returns result.
- if " = " in plugin_name:
+ if " = " in plugin_function:
# Ex. ['result', 'plugin.foo_func.my_func']
- plugin_name_args = plugin_name.split(" = ")
+ plugin_function_args = plugin_function.split(" = ")
# plugin func return data.
- for arg in plugin_name_args:
- if arg == plugin_name_args[-1]:
- plugin_name = arg
+ for arg in plugin_function_args:
+ if arg == plugin_function_args[-1]:
+ plugin_function = arg
else:
plugin_resp = arg.split(",")
# ['result1','result2']
@@ -1478,8 +1473,31 @@
else:
plugin_args = self.yaml_args_populate([])
- # Pack the args list to string parameters for plugin function.
- parm_args_str = self.yaml_args_string(plugin_args)
+ # Replace keys in the string with their corresponding
+ # values from the dictionary.
+ for key, value in global_plugin_dict.items():
+ # Iterate through the list and check if each element matched
+ # exact or in the string. If matches update the plugin element
+ # in the list.
+ for index, element in enumerate(plugin_args):
+ try:
+ if isinstance(element, str):
+ # If the key is not in the list element sting,
+ # then continue for the next element in the list.
+ if str(key) not in str(element):
+ continue
+ if isinstance(value, str):
+ plugin_args[index] = element.replace(
+ key, value
+ )
+ else:
+ plugin_args[index] = global_plugin_dict[
+ element
+ ]
+ # break
+ except KeyError as e:
+ print(f"Exception {e}")
+ pass
"""
Example of plugin_func:
@@ -1490,27 +1508,38 @@
"/redfish/v1/",
"json")
"""
- if parm_args_str:
- plugin_func = f"{plugin_name}({parm_args_str})"
- else:
- plugin_func = f"{plugin_name}()"
+ # For logging purpose to mask password.
+ # List should be string element to join else gives TypeError
+ args_string = self.print_plugin_args_string(plugin_args)
- # Execute plugin function.
- if global_plugin_dict:
- resp = self.execute_python_eval(plugin_func)
- # Update plugin vars dict if there is any.
- if resp != "PLUGIN_EVAL_ERROR":
- self.response_args_data(resp)
- else:
- resp = self.execute_python_eval(plugin_func)
+ # If user wants to debug plugins.
+ self.logger.debug(
+ f"\tDebug Plugin function: \n\t\t{plugin_name}."
+ f"{plugin_function}{args_string}"
+ )
+
+ # For generic logging plugin info.
+ self.logger.info(
+ f"\tPlugin function: \n\t\t{plugin_name}."
+ f"{plugin_function}()"
+ )
+
+ # Execute the plugins function with args.
+ resp = execute_python_function(
+ plugin_name, plugin_function, *plugin_args
+ )
+ self.logger.info(f"\tPlugin response = {resp}")
+ # Update plugin vars dict if there is any.
+ if resp != "PLUGIN_EXEC_ERROR":
+ self.process_response_args_data(resp)
except Exception as e:
# Set the plugin error state.
- plugin_error_dict["exit_on_error"] = True
+ global_plugin_error_dict["exit_on_error"] = True
self.logger.error("\tERROR: execute_plugin_block: %s" % e)
pass
# There is a real error executing the plugin function.
- if resp == "PLUGIN_EVAL_ERROR":
+ if resp == "PLUGIN_EXEC_ERROR":
return resp
# Check if plugin_expects_return (int, string, list,dict etc)
@@ -1531,20 +1560,66 @@
"\tERROR: Plugin expects return data: %s"
% plugin_expects
)
- plugin_error_dict["exit_on_error"] = True
+ global_plugin_error_dict["exit_on_error"] = True
elif not resp:
self.logger.error(
"\tERROR: Plugin func failed to return data"
)
- plugin_error_dict["exit_on_error"] = True
+ global_plugin_error_dict["exit_on_error"] = True
return resp
- def response_args_data(self, plugin_resp):
+ def print_plugin_args_string(self, plugin_args):
r"""
- Parse the plugin function response and update plugin return variable.
+ Generate a string representation of plugin arguments, replacing the
+ password if necessary.
- plugin_resp Response data from plugin function.
+ This method generates a string representation of the provided plugin
+ arguments, joining them with commas. If the password is present in the
+ arguments, it is replaced with "********".
+ The method returns the generated string. If an exception occurs during
+ the process, the method logs a debug log and returns "(None)".
+
+ Parameters:
+ plugin_args (list): A list of plugin arguments.
+
+ Returns:
+ str: The generated string representation of the plugin arguments.
+ """
+ try:
+ plugin_args_str = "(" + ", ".join(map(str, plugin_args)) + ")"
+ if self.password in plugin_args_str:
+ args_string = plugin_args_str.replace(
+ self.password, "********"
+ )
+ else:
+ args_string = plugin_args_str
+ except Exception as e:
+ self.logger.debug("\tWARN:Print args string : %s" % e)
+ return "(None)"
+
+ return args_string
+
+ def process_response_args_data(self, plugin_resp):
+ r"""
+ Parse the plugin function response and update plugin return variables.
+
+ This method parses the response data from a plugin function and
+ updates the plugin return variables accordingly. The method takes the
+ plugin_resp argument, which is expected to be the response data from a
+ plugin function.
+
+ The method handles various data types (string, bytes,
+ tuple, list, int, float) and updates the global global_plugin_dict
+ dictionary with the parsed response data. If there is an error during
+ the process, the method logs a warning and continues with the next
+ plugin block execution.
+
+ Parameters:
+ plugin_resp (Any): The response data from the plugin function.
+
+ Returns:
+ None
"""
resp_list = []
resp_data = ""
@@ -1561,10 +1636,10 @@
resp_list.append(resp_data)
elif isinstance(plugin_resp, tuple):
if len(global_plugin_list) == 1:
- resp_list.append(plugin_resp)
+ resp_list.append(list(plugin_resp))
else:
resp_list = list(plugin_resp)
- resp_list = [x.strip("\r\n\t") for x in resp_list]
+ resp_list = [x for x in resp_list]
elif isinstance(plugin_resp, list):
if len(global_plugin_list) == 1:
resp_list.append([x.strip("\r\n\t") for x in plugin_resp])
@@ -1584,53 +1659,13 @@
dict_idx = global_plugin_list[idx]
global_plugin_dict[dict_idx] = item
except (IndexError, ValueError) as e:
- self.logger.warn("\tWARN: response_args_data: %s" % e)
+ self.logger.warn("\tWARN: process_response_args_data: %s" % e)
pass
# Done updating plugin dict irrespective of pass or failed,
# clear all the list element for next plugin block execute.
global_plugin_list.clear()
- def yaml_args_string(self, plugin_args):
- r"""
- Pack the arguments into a string representation.
-
- This method processes the plugin_arg argument, which is expected to
- contain a list of arguments. The method iterates through the list,
- converts each argument to a string, and concatenates them into a
- single string. Special handling is applied for integer, float, and
- predefined plugin variable types.
-
- Ecample:
- From
- ['xx.xx.xx.xx:443', 'root', '********', '/redfish/v1/', 'json']
- to
- "xx.xx.xx.xx:443","root","********","/redfish/v1/","json"
-
- Parameters:
- plugin_args (list): A list of arguments to be packed into
- a string.
-
- Returns:
- str: A string representation of the arguments.
- """
- args_str = ""
-
- for i, arg in enumerate(plugin_args):
- if arg:
- if isinstance(arg, (int, float)):
- args_str += str(arg)
- elif arg in global_plugin_type_list:
- args_str += str(global_plugin_dict[arg])
- else:
- args_str += f'"{arg.strip("\r\n\t")}"'
-
- # Skip last list element.
- if i != len(plugin_args) - 1:
- args_str += ","
-
- return args_str
-
def yaml_args_populate(self, yaml_arg_list):
r"""
Decode environment and plugin variables and populate the argument list.
@@ -1665,7 +1700,7 @@
if isinstance(arg, (int, float)):
populated_list.append(arg)
elif isinstance(arg, str):
- arg_str = self.yaml_env_and_plugin_vars_populate(str(arg))
+ arg_str = self.yaml_env_and_plugin_vars_populate(arg)
populated_list.append(arg_str)
else:
populated_list.append(arg)
@@ -1708,6 +1743,9 @@
env_var_regex = r"\$\{([^\}]+)\}"
env_var_names_list = re.findall(env_var_regex, yaml_arg_str)
+ # If the list in empty [] nothing to update.
+ if not len(env_var_names_list):
+ return yaml_arg_str
for var in env_var_names_list:
env_var = os.environ.get(var)
if env_var:
@@ -1744,7 +1782,7 @@
if isinstance(plugin_var_value, (list, dict)):
"""
List data type or dict can't be replaced, use
- directly in eval function call.
+ directly in plugin function call.
"""
global_plugin_type_list.append(var)
else:
@@ -1768,7 +1806,8 @@
This method checks if any dictionary in the plugin_dict list contains
a "plugin_error" key. If such a dictionary is found, it retrieves the
value associated with the "plugin_error" key and returns the
- corresponding error message from the plugin_error_dict attribute.
+ corresponding error message from the global_plugin_error_dict
+ attribute.
Parameters:
plugin_dict (list of dict): A list of dictionaries containing
@@ -1782,7 +1821,7 @@
for d in plugin_dict:
if "plugin_error" in d:
value = d["plugin_error"]
- return self.plugin_error_dict.get(value, None)
+ return global_plugin_error_dict.get(value, None)
return None
def key_index_list_dict(self, key, list_dict):
diff --git a/ffdc/ffdc_config.yaml b/ffdc/ffdc_config.yaml
index 1214a67..7290a7f 100644
--- a/ffdc/ffdc_config.yaml
+++ b/ffdc/ffdc_config.yaml
@@ -147,7 +147,8 @@
${hostname}:${port_https} -S Always raw GET
/redfish/v1/Systems/system/LogServices/EventLog/Entries
- plugin:
- - plugin_name: plugin.redfish.enumerate_request
+ - plugin_name: plugins.redfish
+ - plugin_function: enumerate_request
- plugin_args:
- ${hostname}:${port_https}
- ${username}
diff --git a/ffdc/plugins/scp_execution.py b/ffdc/plugins/scp_execution.py
index c33d61f..ee6b343 100644
--- a/ffdc/plugins/scp_execution.py
+++ b/ffdc/plugins/scp_execution.py
@@ -19,7 +19,9 @@
from ssh_utility import SSHRemoteclient # NOQA
-def scp_remote_file(hostname, username, password, filename, local_dir_path):
+def scp_remote_file(
+ hostname, username, password, port_ssh, filename, local_dir_path
+):
r"""
Copy a file from a remote host to the local host using SCP.
@@ -34,6 +36,7 @@
hostname (str): Name or IP address of the remote host.
username (str): User on the remote host with access to files.
password (str): Password for the user on the remote host.
+ port_ssh (int): SSH/SCP port value. By default, 22.
filename (str): Filename with full path on the remote host.
Can contain wildcards for multiple files.
local_dir_path (str): Location to store the file on the local host.
@@ -41,7 +44,7 @@
Returns:
None
"""
- ssh_remoteclient = SSHRemoteclient(hostname, username, password)
+ ssh_remoteclient = SSHRemoteclient(hostname, username, password, port_ssh)
if ssh_remoteclient.ssh_remoteclient_login():
# Obtain scp connection.
diff --git a/ffdc/templates/log_collector_config_template.yaml b/ffdc/templates/log_collector_config_template.yaml
index 6ccd4fb..3d467f4 100644
--- a/ffdc/templates/log_collector_config_template.yaml
+++ b/ffdc/templates/log_collector_config_template.yaml
@@ -34,7 +34,8 @@
SHELL_LOGS:
COMMANDS:
- plugin:
- - plugin_name: plugin.ssh_execution.ssh_execute_cmd
+ - plugin_name: plugins.ssh_execution
+ - plugin_function: ssh_execute_cmd
- plugin_args:
- ${hostname}
- ${username}