blob: 03e8a113296f13eabddc7d4477a9d0b3b3b88eca [file] [log] [blame]
George Keishinge7e91712021-09-03 11:28:44 -05001#!/usr/bin/env python3
Michael Walshc3b512e2017-02-20 15:59:01 -06002
3r"""
4This module provides command execution functions such as cmd_fnc and cmd_fnc_u.
5"""
6
George Keishinge635ddc2022-12-08 07:38:02 -06007import collections
George Keishinge635ddc2022-12-08 07:38:02 -06008import inspect
Patrick Williams20f38712022-12-08 06:18:26 -06009import os
10import re
11import signal
12import subprocess
13import sys
14import time
Michael Walshc3b512e2017-02-20 15:59:01 -060015
Patrick Williams20f38712022-12-08 06:18:26 -060016import func_args as fa
17import gen_misc as gm
Michael Walshc3b512e2017-02-20 15:59:01 -060018import gen_print as gp
19import gen_valid as gv
Michael Walshd6901502017-11-14 12:58:37 -060020
21robot_env = gp.robot_env
22
Michael Walshafc53a22017-04-12 15:52:28 -050023if robot_env:
Michael Walsha3e2f532018-01-10 13:43:42 -060024 from robot.libraries.BuiltIn import BuiltIn
Michael Walshc3b512e2017-02-20 15:59:01 -060025
26
Michael Walsh9e042ad2019-10-16 17:14:31 -050027# cmd_fnc and cmd_fnc_u should now be considered deprecated. shell_cmd and t_shell_cmd should be used
28# instead.
Patrick Williams20f38712022-12-08 06:18:26 -060029def cmd_fnc(
30 cmd_buf,
31 quiet=None,
32 test_mode=None,
33 debug=0,
34 print_output=1,
35 show_err=1,
36 return_stderr=0,
37 ignore_err=1,
38):
Michael Walshc3b512e2017-02-20 15:59:01 -060039 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -050040 Run the given command in a shell and return the shell return code and the output.
Michael Walshc3b512e2017-02-20 15:59:01 -060041
42 Description of arguments:
43 cmd_buf The command string to be run in a shell.
Michael Walsh9e042ad2019-10-16 17:14:31 -050044 quiet Indicates whether this function should run the print_issuing() function
45 which prints "Issuing: <cmd string>" to stdout.
46 test_mode If test_mode is set, this function will not actually run the command. If
47 print_output is set, it will print "(test_mode) Issuing: <cmd string>" to
Michael Walshcfe9fed2017-09-12 17:13:10 -050048 stdout.
Michael Walsh9e042ad2019-10-16 17:14:31 -050049 debug If debug is set, this function will print extra debug info.
50 print_output If this is set, this function will print the stdout/stderr generated by
51 the shell command.
52 show_err If show_err is set, this function will print a standardized error report
53 if the shell command returns non-zero.
54 return_stderr If return_stderr is set, this function will process the stdout and stderr
55 streams from the shell command separately. It will also return stderr in
56 addition to the return code and the stdout.
Michael Walshc3b512e2017-02-20 15:59:01 -060057 """
58
Michael Walshcfe9fed2017-09-12 17:13:10 -050059 # Determine default values.
Michael Walshc3b512e2017-02-20 15:59:01 -060060 quiet = int(gm.global_default(quiet, 0))
61 test_mode = int(gm.global_default(test_mode, 0))
Michael Walshc3b512e2017-02-20 15:59:01 -060062
63 if debug:
Michael Walshafc53a22017-04-12 15:52:28 -050064 gp.print_vars(cmd_buf, quiet, test_mode, debug)
Michael Walshc3b512e2017-02-20 15:59:01 -060065
Michael Walshec01a6f2019-08-01 12:43:20 -050066 err_msg = gv.valid_value(cmd_buf)
Michael Walshc3b512e2017-02-20 15:59:01 -060067 if err_msg != "":
68 raise ValueError(err_msg)
69
70 if not quiet:
Michael Walshafc53a22017-04-12 15:52:28 -050071 gp.pissuing(cmd_buf, test_mode)
Michael Walshc3b512e2017-02-20 15:59:01 -060072
73 if test_mode:
Michael Walshcfe9fed2017-09-12 17:13:10 -050074 if return_stderr:
75 return 0, "", ""
76 else:
77 return 0, ""
78
79 if return_stderr:
80 err_buf = ""
81 stderr = subprocess.PIPE
82 else:
83 stderr = subprocess.STDOUT
Michael Walshc3b512e2017-02-20 15:59:01 -060084
Patrick Williams20f38712022-12-08 06:18:26 -060085 sub_proc = subprocess.Popen(
86 cmd_buf,
87 bufsize=1,
88 shell=True,
89 universal_newlines=True,
90 executable="/bin/bash",
91 stdout=subprocess.PIPE,
92 stderr=stderr,
93 )
Michael Walshc3b512e2017-02-20 15:59:01 -060094 out_buf = ""
Michael Walshcfe9fed2017-09-12 17:13:10 -050095 if return_stderr:
96 for line in sub_proc.stderr:
George Keishing36efbc02018-12-12 10:18:23 -060097 try:
98 err_buf += line
99 except TypeError:
100 line = line.decode("utf-8")
101 err_buf += line
Michael Walshcfe9fed2017-09-12 17:13:10 -0500102 if not print_output:
103 continue
Michael Walshc108e422019-03-28 12:27:18 -0500104 gp.gp_print(line)
Michael Walshc3b512e2017-02-20 15:59:01 -0600105 for line in sub_proc.stdout:
George Keishing36efbc02018-12-12 10:18:23 -0600106 try:
107 out_buf += line
108 except TypeError:
109 line = line.decode("utf-8")
110 out_buf += line
Michael Walshc3b512e2017-02-20 15:59:01 -0600111 if not print_output:
112 continue
Michael Walshc108e422019-03-28 12:27:18 -0500113 gp.gp_print(line)
Michael Walshc3b512e2017-02-20 15:59:01 -0600114 if print_output and not robot_env:
115 sys.stdout.flush()
116 sub_proc.communicate()
117 shell_rc = sub_proc.returncode
Michael Walsha3e2f532018-01-10 13:43:42 -0600118 if shell_rc != 0:
119 err_msg = "The prior shell command failed.\n"
Michael Walsh1429e122019-05-20 10:06:18 -0500120 err_msg += gp.sprint_var(shell_rc, gp.hexa())
Michael Walshcfe9fed2017-09-12 17:13:10 -0500121 if not print_output:
122 err_msg += "out_buf:\n" + out_buf
Michael Walshc3b512e2017-02-20 15:59:01 -0600123
Michael Walsha3e2f532018-01-10 13:43:42 -0600124 if show_err:
Michael Walshc108e422019-03-28 12:27:18 -0500125 gp.print_error_report(err_msg)
Michael Walsha3e2f532018-01-10 13:43:42 -0600126 if not ignore_err:
127 if robot_env:
128 BuiltIn().fail(err_msg)
129 else:
130 raise ValueError(err_msg)
Michael Walshcfe9fed2017-09-12 17:13:10 -0500131
132 if return_stderr:
133 return shell_rc, out_buf, err_buf
134 else:
135 return shell_rc, out_buf
Michael Walshc3b512e2017-02-20 15:59:01 -0600136
Michael Walshc3b512e2017-02-20 15:59:01 -0600137
Patrick Williams20f38712022-12-08 06:18:26 -0600138def cmd_fnc_u(
139 cmd_buf,
140 quiet=None,
141 debug=None,
142 print_output=1,
143 show_err=1,
144 return_stderr=0,
145 ignore_err=1,
146):
Michael Walshc3b512e2017-02-20 15:59:01 -0600147 r"""
148 Call cmd_fnc with test_mode=0. See cmd_fnc (above) for details.
149
150 Note the "u" in "cmd_fnc_u" stands for "unconditional".
151 """
152
Patrick Williams20f38712022-12-08 06:18:26 -0600153 return cmd_fnc(
154 cmd_buf,
155 test_mode=0,
156 quiet=quiet,
157 debug=debug,
158 print_output=print_output,
159 show_err=show_err,
160 return_stderr=return_stderr,
161 ignore_err=ignore_err,
162 )
Michael Walshc3b512e2017-02-20 15:59:01 -0600163
Michael Walshf41fac82017-08-02 15:05:24 -0500164
Michael Walshf41fac82017-08-02 15:05:24 -0500165def parse_command_string(command_string):
Michael Walshf41fac82017-08-02 15:05:24 -0500166 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500167 Parse a bash command-line command string and return the result as a dictionary of parms.
Michael Walshf41fac82017-08-02 15:05:24 -0500168
Michael Walsh9e042ad2019-10-16 17:14:31 -0500169 This can be useful for answering questions like "What did the user specify as the value for parm x in the
170 command string?".
Michael Walshf41fac82017-08-02 15:05:24 -0500171
Michael Walsh9e042ad2019-10-16 17:14:31 -0500172 This function expects the command string to follow the following posix conventions:
Michael Walshf41fac82017-08-02 15:05:24 -0500173 - Short parameters:
174 -<parm name><space><arg value>
175 - Long parameters:
176 --<parm name>=<arg value>
177
Michael Walsh9e042ad2019-10-16 17:14:31 -0500178 The first item in the string will be considered to be the command. All values not conforming to the
179 specifications above will be considered positional parms. If there are multiple parms with the same
180 name, they will be put into a list (see illustration below where "-v" is specified multiple times).
Michael Walshf41fac82017-08-02 15:05:24 -0500181
182 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500183 command_string The complete command string including all parameters and arguments.
Michael Walshf41fac82017-08-02 15:05:24 -0500184
185 Sample input:
186
Michael Walsh9e042ad2019-10-16 17:14:31 -0500187 robot_cmd_buf: robot -v OPENBMC_HOST:dummy1 -v keyword_string:'Set
188 Auto Reboot no' -v lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot -v quiet:0 -v
189 test_mode:0 -v debug:0 --outputdir='/home/user1/status/children/'
190 --output=dummy1.Auto_reboot.170802.124544.output.xml --log=dummy1.Auto_reboot.170802.124544.log.html
Michael Walshf41fac82017-08-02 15:05:24 -0500191 --report=dummy1.Auto_reboot.170802.124544.report.html
192 /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
193
194 Sample output:
195
196 robot_cmd_buf_dict:
197 robot_cmd_buf_dict[command]: robot
198 robot_cmd_buf_dict[v]:
199 robot_cmd_buf_dict[v][0]: OPENBMC_HOST:dummy1
Michael Walsh9e042ad2019-10-16 17:14:31 -0500200 robot_cmd_buf_dict[v][1]: keyword_string:Set Auto Reboot no
Michael Walshf41fac82017-08-02 15:05:24 -0500201 robot_cmd_buf_dict[v][2]:
202 lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot
203 robot_cmd_buf_dict[v][3]: quiet:0
204 robot_cmd_buf_dict[v][4]: test_mode:0
205 robot_cmd_buf_dict[v][5]: debug:0
Michael Walsh9e042ad2019-10-16 17:14:31 -0500206 robot_cmd_buf_dict[outputdir]: /home/user1/status/children/
207 robot_cmd_buf_dict[output]: dummy1.Auto_reboot.170802.124544.output.xml
208 robot_cmd_buf_dict[log]: dummy1.Auto_reboot.170802.124544.log.html
209 robot_cmd_buf_dict[report]: dummy1.Auto_reboot.170802.124544.report.html
Michael Walshf41fac82017-08-02 15:05:24 -0500210 robot_cmd_buf_dict[positional]:
211 /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
212 """
213
Michael Walsh9e042ad2019-10-16 17:14:31 -0500214 # We want the parms in the string broken down the way bash would do it, so we'll call upon bash to do
215 # that by creating a simple inline bash function.
Patrick Williams20f38712022-12-08 06:18:26 -0600216 bash_func_def = (
217 'function parse { for parm in "${@}" ; do' + " echo $parm ; done ; }"
218 )
Michael Walshf41fac82017-08-02 15:05:24 -0500219
Patrick Williams20f38712022-12-08 06:18:26 -0600220 rc, outbuf = cmd_fnc_u(
221 bash_func_def + " ; parse " + command_string, quiet=1, print_output=0
222 )
Michael Walshf41fac82017-08-02 15:05:24 -0500223 command_string_list = outbuf.rstrip("\n").split("\n")
224
225 command_string_dict = collections.OrderedDict()
226 ix = 1
Patrick Williams20f38712022-12-08 06:18:26 -0600227 command_string_dict["command"] = command_string_list[0]
Michael Walshf41fac82017-08-02 15:05:24 -0500228 while ix < len(command_string_list):
229 if command_string_list[ix].startswith("--"):
230 key, value = command_string_list[ix].split("=")
231 key = key.lstrip("-")
232 elif command_string_list[ix].startswith("-"):
233 key = command_string_list[ix].lstrip("-")
234 ix += 1
235 try:
236 value = command_string_list[ix]
237 except IndexError:
238 value = ""
239 else:
Patrick Williams20f38712022-12-08 06:18:26 -0600240 key = "positional"
Michael Walshf41fac82017-08-02 15:05:24 -0500241 value = command_string_list[ix]
242 if key in command_string_dict:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500243 if isinstance(command_string_dict[key], str):
Michael Walshf41fac82017-08-02 15:05:24 -0500244 command_string_dict[key] = [command_string_dict[key]]
245 command_string_dict[key].append(value)
246 else:
247 command_string_dict[key] = value
248 ix += 1
249
250 return command_string_dict
Michael Walsh21083d22018-06-01 14:19:32 -0500251
252
253# Save the original SIGALRM handler for later restoration by shell_cmd.
254original_sigalrm_handler = signal.getsignal(signal.SIGALRM)
255
256
Patrick Williams20f38712022-12-08 06:18:26 -0600257def shell_cmd_timed_out(signal_number, frame):
Michael Walsh21083d22018-06-01 14:19:32 -0500258 r"""
259 Handle an alarm signal generated during the shell_cmd function.
260 """
261
262 gp.dprint_executing()
Michael Walsh3fb26182019-08-28 16:51:05 -0500263 global command_timed_out
264 command_timed_out = True
Michael Walsh21083d22018-06-01 14:19:32 -0500265 # Get subprocess pid from shell_cmd's call stack.
Patrick Williams20f38712022-12-08 06:18:26 -0600266 sub_proc = gp.get_stack_var("sub_proc", 0)
Michael Walsh21083d22018-06-01 14:19:32 -0500267 pid = sub_proc.pid
Michael Walsh3fb26182019-08-28 16:51:05 -0500268 gp.dprint_var(pid)
269 # Terminate the child process group.
270 os.killpg(pid, signal.SIGKILL)
Michael Walsh21083d22018-06-01 14:19:32 -0500271 # Restore the original SIGALRM handler.
272 signal.signal(signal.SIGALRM, original_sigalrm_handler)
273
274 return
275
276
Patrick Williams20f38712022-12-08 06:18:26 -0600277def shell_cmd(
278 command_string,
279 quiet=None,
280 print_output=None,
281 show_err=1,
282 test_mode=0,
283 time_out=None,
284 max_attempts=1,
285 retry_sleep_time=5,
286 valid_rcs=[0],
287 ignore_err=None,
288 return_stderr=0,
289 fork=0,
290 error_regexes=None,
291):
Michael Walsh21083d22018-06-01 14:19:32 -0500292 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500293 Run the given command string in a shell and return a tuple consisting of the shell return code and the
294 output.
Michael Walsh21083d22018-06-01 14:19:32 -0500295
296 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500297 command_string The command string to be run in a shell (e.g. "ls /tmp").
298 quiet If set to 0, this function will print "Issuing: <cmd string>" to stdout.
299 When the quiet argument is set to None, this function will assign a
300 default value by searching upward in the stack for the quiet variable
301 value. If no such value is found, quiet is set to 0.
302 print_output If this is set, this function will print the stdout/stderr generated by
303 the shell command to stdout.
304 show_err If show_err is set, this function will print a standardized error report
305 if the shell command fails (i.e. if the shell command returns a shell_rc
306 that is not in valid_rcs). Note: Error text is only printed if ALL
307 attempts to run the command_string fail. In other words, if the command
308 execution is ultimately successful, initial failures are hidden.
309 test_mode If test_mode is set, this function will not actually run the command. If
310 print_output is also set, this function will print "(test_mode) Issuing:
311 <cmd string>" to stdout. A caller should call shell_cmd directly if they
312 wish to have the command string run unconditionally. They should call
313 the t_shell_cmd wrapper (defined below) if they wish to run the command
314 string only if the prevailing test_mode variable is set to 0.
315 time_out A time-out value expressed in seconds. If the command string has not
316 finished executing within <time_out> seconds, it will be halted and
317 counted as an error.
318 max_attempts The max number of attempts that should be made to run the command string.
319 retry_sleep_time The number of seconds to sleep between attempts.
320 valid_rcs A list of integers indicating which shell_rc values are not to be
321 considered errors.
322 ignore_err Ignore error means that a failure encountered by running the command
323 string will not be raised as a python exception. When the ignore_err
324 argument is set to None, this function will assign a default value by
325 searching upward in the stack for the ignore_err variable value. If no
326 such value is found, ignore_err is set to 1.
327 return_stderr If return_stderr is set, this function will process the stdout and stderr
328 streams from the shell command separately. In such a case, the tuple
329 returned by this function will consist of three values rather than just
330 two: rc, stdout, stderr.
331 fork Run the command string asynchronously (i.e. don't wait for status of the
332 child process and don't try to get stdout/stderr) and return the Popen
333 object created by the subprocess.popen() function. See the kill_cmd
334 function for details on how to process the popen object.
335 error_regexes A list of regular expressions to be used to identify errors in the
336 command output. If there is a match for any of these regular
337 expressions, the command will be considered a failure and the shell_rc
338 will be set to -1. For example, if error_regexes = ['ERROR:'] and the
339 command output contains 'ERROR: Unrecognized option', it will be counted
340 as an error even if the command returned 0. This is useful when running
341 commands that do not always return non-zero on error.
Michael Walsh21083d22018-06-01 14:19:32 -0500342 """
343
Michael Walsh3fb26182019-08-28 16:51:05 -0500344 err_msg = gv.valid_value(command_string)
345 if err_msg:
346 raise ValueError(err_msg)
347
Michael Walsh21083d22018-06-01 14:19:32 -0500348 # Assign default values to some of the arguments to this function.
Patrick Williams20f38712022-12-08 06:18:26 -0600349 quiet = int(gm.dft(quiet, gp.get_stack_var("quiet", 0)))
Michael Walsh55abd1b2018-08-31 13:00:50 -0500350 print_output = int(gm.dft(print_output, not quiet))
Michael Walsh45fead42018-09-26 17:20:48 -0500351 show_err = int(show_err)
Patrick Williams20f38712022-12-08 06:18:26 -0600352 ignore_err = int(gm.dft(ignore_err, gp.get_stack_var("ignore_err", 1)))
Michael Walsh21083d22018-06-01 14:19:32 -0500353
Michael Walsh3fb26182019-08-28 16:51:05 -0500354 gp.qprint_issuing(command_string, test_mode)
Michael Walsh21083d22018-06-01 14:19:32 -0500355 if test_mode:
Michael Walsh3fb26182019-08-28 16:51:05 -0500356 return (0, "", "") if return_stderr else (0, "")
Michael Walsh21083d22018-06-01 14:19:32 -0500357
Michael Walsh3fb26182019-08-28 16:51:05 -0500358 # Convert a string python dictionary definition to a dictionary.
359 valid_rcs = fa.source_to_object(valid_rcs)
Michael Walsh21083d22018-06-01 14:19:32 -0500360 # Convert each list entry to a signed value.
Michael Walsh3fb26182019-08-28 16:51:05 -0500361 valid_rcs = [gm.to_signed(x) for x in valid_rcs]
Michael Walsh21083d22018-06-01 14:19:32 -0500362
Michael Walsh3fb26182019-08-28 16:51:05 -0500363 stderr = subprocess.PIPE if return_stderr else subprocess.STDOUT
Michael Walsh21083d22018-06-01 14:19:32 -0500364
Michael Walsh9e042ad2019-10-16 17:14:31 -0500365 # Write all output to func_out_history_buf rather than directly to stdout. This allows us to decide
366 # what to print after all attempts to run the command string have been made. func_out_history_buf will
367 # contain the complete history from the current invocation of this function.
Michael Walsh3fb26182019-08-28 16:51:05 -0500368 global command_timed_out
369 command_timed_out = False
370 func_out_history_buf = ""
Michael Walsh21083d22018-06-01 14:19:32 -0500371 for attempt_num in range(1, max_attempts + 1):
Patrick Williams20f38712022-12-08 06:18:26 -0600372 sub_proc = subprocess.Popen(
373 command_string,
374 bufsize=1,
375 shell=True,
376 universal_newlines=True,
377 executable="/bin/bash",
378 stdin=subprocess.PIPE,
379 stdout=subprocess.PIPE,
380 stderr=stderr,
381 )
Michael Walshfaafa9c2018-06-27 16:39:31 -0500382 if fork:
Michael Walsh4d5f6682019-09-24 14:23:40 -0500383 return sub_proc
Michael Walsh3fb26182019-08-28 16:51:05 -0500384
385 if time_out:
386 command_timed_out = False
Michael Walsh21083d22018-06-01 14:19:32 -0500387 # Designate a SIGALRM handling function and set alarm.
388 signal.signal(signal.SIGALRM, shell_cmd_timed_out)
389 signal.alarm(time_out)
390 try:
Michael Walsh3fb26182019-08-28 16:51:05 -0500391 stdout_buf, stderr_buf = sub_proc.communicate()
Michael Walsh21083d22018-06-01 14:19:32 -0500392 except IOError:
393 command_timed_out = True
Michael Walsh21083d22018-06-01 14:19:32 -0500394 # Restore the original SIGALRM handler and clear the alarm.
395 signal.signal(signal.SIGALRM, original_sigalrm_handler)
396 signal.alarm(0)
Michael Walsh3fb26182019-08-28 16:51:05 -0500397
Michael Walsh9e042ad2019-10-16 17:14:31 -0500398 # Output from this loop iteration is written to func_out_buf for later processing. This can include
399 # stdout, stderr and our own error messages.
Michael Walsh3fb26182019-08-28 16:51:05 -0500400 func_out_buf = ""
401 if print_output:
402 if return_stderr:
403 func_out_buf += stderr_buf
404 func_out_buf += stdout_buf
405 shell_rc = sub_proc.returncode
406 if shell_rc in valid_rcs:
Michael Walsh9e042ad2019-10-16 17:14:31 -0500407 # Check output for text indicating there is an error.
Patrick Williams20f38712022-12-08 06:18:26 -0600408 if error_regexes and re.match("|".join(error_regexes), stdout_buf):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500409 shell_rc = -1
410 else:
411 break
Michael Walsh21083d22018-06-01 14:19:32 -0500412 err_msg = "The prior shell command failed.\n"
Michael Walsh21083d22018-06-01 14:19:32 -0500413 err_msg += gp.sprint_var(attempt_num)
Michael Walsh3fb26182019-08-28 16:51:05 -0500414 err_msg += gp.sprint_vars(command_string, command_timed_out, time_out)
415 err_msg += gp.sprint_varx("child_pid", sub_proc.pid)
416 err_msg += gp.sprint_vars(shell_rc, valid_rcs, fmt=gp.hexa())
Michael Walsh9e042ad2019-10-16 17:14:31 -0500417 if error_regexes:
418 err_msg += gp.sprint_vars(error_regexes)
Michael Walsh21083d22018-06-01 14:19:32 -0500419 if not print_output:
420 if return_stderr:
Michael Walsh3fb26182019-08-28 16:51:05 -0500421 err_msg += "stderr_buf:\n" + stderr_buf
422 err_msg += "stdout_buf:\n" + stdout_buf
Michael Walsh21083d22018-06-01 14:19:32 -0500423 if show_err:
Michael Walsh3fb26182019-08-28 16:51:05 -0500424 func_out_buf += gp.sprint_error_report(err_msg)
Michael Walsh21083d22018-06-01 14:19:32 -0500425 if attempt_num < max_attempts:
Michael Walsh3fb26182019-08-28 16:51:05 -0500426 cmd_buf = "time.sleep(" + str(retry_sleep_time) + ")"
427 if show_err:
428 func_out_buf += gp.sprint_issuing(cmd_buf)
429 exec(cmd_buf)
430 func_out_history_buf += func_out_buf
Michael Walsh21083d22018-06-01 14:19:32 -0500431
Michael Walsh3fb26182019-08-28 16:51:05 -0500432 if shell_rc in valid_rcs:
433 gp.gp_print(func_out_buf)
434 else:
435 if show_err:
Patrick Williams20f38712022-12-08 06:18:26 -0600436 gp.gp_print(func_out_history_buf, stream="stderr")
Michael Walsh3fb26182019-08-28 16:51:05 -0500437 else:
Michael Walsh9e042ad2019-10-16 17:14:31 -0500438 # There is no error information to show so just print output from last loop iteration.
Michael Walsh3fb26182019-08-28 16:51:05 -0500439 gp.gp_print(func_out_buf)
Michael Walsh21083d22018-06-01 14:19:32 -0500440 if not ignore_err:
Michael Walsh9e042ad2019-10-16 17:14:31 -0500441 # If the caller has already asked to show error info, avoid repeating that in the failure message.
Patrick Williams20f38712022-12-08 06:18:26 -0600442 err_msg = (
443 "The prior shell command failed.\n" if show_err else err_msg
444 )
Michael Walsh21083d22018-06-01 14:19:32 -0500445 if robot_env:
446 BuiltIn().fail(err_msg)
447 else:
Michael Walsh3fb26182019-08-28 16:51:05 -0500448 raise ValueError(err_msg)
Michael Walsh21083d22018-06-01 14:19:32 -0500449
Patrick Williams20f38712022-12-08 06:18:26 -0600450 return (
451 (shell_rc, stdout_buf, stderr_buf)
452 if return_stderr
Michael Walsh3fb26182019-08-28 16:51:05 -0500453 else (shell_rc, stdout_buf)
Patrick Williams20f38712022-12-08 06:18:26 -0600454 )
Michael Walsh21083d22018-06-01 14:19:32 -0500455
456
457def t_shell_cmd(command_string, **kwargs):
458 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500459 Search upward in the the call stack to obtain the test_mode argument, add it to kwargs and then call
460 shell_cmd and return the result.
Michael Walsh21083d22018-06-01 14:19:32 -0500461
462 See shell_cmd prolog for details on all arguments.
463 """
464
Patrick Williams20f38712022-12-08 06:18:26 -0600465 if "test_mode" in kwargs:
466 error_message = (
467 "Programmer error - test_mode is not a valid"
468 + " argument to this function."
469 )
Michael Walsh21083d22018-06-01 14:19:32 -0500470 gp.print_error_report(error_message)
471 exit(1)
472
Patrick Williams20f38712022-12-08 06:18:26 -0600473 test_mode = int(gp.get_stack_var("test_mode", 0))
474 kwargs["test_mode"] = test_mode
Michael Walsh21083d22018-06-01 14:19:32 -0500475
476 return shell_cmd(command_string, **kwargs)
George Keishing16244c22019-01-31 16:16:14 +0000477
478
Michael Walsh4d5f6682019-09-24 14:23:40 -0500479def kill_cmd(popen, sig=signal.SIGTERM):
480 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500481 Kill the subprocess represented by the Popen object and return a tuple consisting of the shell return
482 code and the output.
Michael Walsh4d5f6682019-09-24 14:23:40 -0500483
Michael Walsh9e042ad2019-10-16 17:14:31 -0500484 This function is meant to be used as the follow-up for a call to shell_cmd(..., fork=1).
Michael Walsh4d5f6682019-09-24 14:23:40 -0500485
486 Example:
487 popen = shell_cmd("some_pgm.py", fork=1)
488 ...
489 shell_rc, output = kill_cmd(popen)
490
491 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500492 popen A Popen object returned by the subprocess.Popen() command.
Michael Walsh4d5f6682019-09-24 14:23:40 -0500493 sig The signal to be sent to the child process.
494 """
495
496 gp.dprint_var(popen.pid)
497 os.killpg(popen.pid, sig)
498 stdout, stderr = popen.communicate()
499 shell_rc = popen.returncode
500 return (shell_rc, stdout, stderr) if stderr else (shell_rc, stdout)
501
502
George Keishing16244c22019-01-31 16:16:14 +0000503def re_order_kwargs(stack_frame_ix, **kwargs):
504 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500505 Re-order the kwargs to match the order in which they were specified on a function invocation and return
506 as an ordered dictionary.
George Keishing16244c22019-01-31 16:16:14 +0000507
Michael Walsh9e042ad2019-10-16 17:14:31 -0500508 Note that this re_order_kwargs function should not be necessary in python versions 3.6 and beyond.
George Keishing16244c22019-01-31 16:16:14 +0000509
510 Example:
511
512 The caller calls func1 like this:
513
514 func1('mike', arg1='one', arg2='two', arg3='three')
515
516 And func1 is defined as follows:
517
518 def func1(first_arg, **kwargs):
519
520 kwargs = re_order_kwargs(first_arg_num=2, stack_frame_ix=3, **kwargs)
521
Michael Walsh9e042ad2019-10-16 17:14:31 -0500522 The kwargs dictionary before calling re_order_kwargs (where order is not guaranteed):
George Keishing16244c22019-01-31 16:16:14 +0000523
524 kwargs:
525 kwargs[arg3]: three
526 kwargs[arg2]: two
527 kwargs[arg1]: one
528
529 The kwargs dictionary after calling re_order_kwargs:
530
531 kwargs:
532 kwargs[arg1]: one
533 kwargs[arg2]: two
534 kwargs[arg3]: three
535
Michael Walsh9e042ad2019-10-16 17:14:31 -0500536 Note that the re-ordered kwargs match the order specified on the call to func1.
George Keishing16244c22019-01-31 16:16:14 +0000537
538 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500539 stack_frame_ix The stack frame of the function whose kwargs values must be re-ordered.
540 0 is the stack frame of re_order_kwargs, 1 is the stack from of its
541 caller and so on.
542 kwargs The keyword argument dictionary which is to be re-ordered.
George Keishing16244c22019-01-31 16:16:14 +0000543 """
544
545 new_kwargs = collections.OrderedDict()
546
547 # Get position number of first keyword on the calling line of code.
Patrick Williams20f38712022-12-08 06:18:26 -0600548 (args, varargs, keywords, locals) = inspect.getargvalues(
549 inspect.stack()[stack_frame_ix][0]
550 )
George Keishing16244c22019-01-31 16:16:14 +0000551 first_kwarg_pos = 1 + len(args)
552 if varargs is not None:
553 first_kwarg_pos += len(locals[varargs])
554 for arg_num in range(first_kwarg_pos, first_kwarg_pos + len(kwargs)):
555 # This will result in an arg_name value such as "arg1='one'".
556 arg_name = gp.get_arg_name(None, arg_num, stack_frame_ix + 2)
557 # Continuing with the prior example, the following line will result
558 # in key being set to 'arg1'.
Patrick Williams20f38712022-12-08 06:18:26 -0600559 key = arg_name.split("=")[0]
George Keishing16244c22019-01-31 16:16:14 +0000560 new_kwargs[key] = kwargs[key]
561
562 return new_kwargs
563
564
565def default_arg_delim(arg_dashes):
566 r"""
567 Return the default argument delimiter value for the given arg_dashes value.
568
Michael Walsh9e042ad2019-10-16 17:14:31 -0500569 Note: this function is useful for functions that manipulate bash command line arguments (e.g. --parm=1 or
570 -parm 1).
George Keishing16244c22019-01-31 16:16:14 +0000571
572 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500573 arg_dashes The argument dashes specifier (usually, "-" or "--").
George Keishing16244c22019-01-31 16:16:14 +0000574 """
575
576 if arg_dashes == "--":
577 return "="
578
579 return " "
580
581
582def create_command_string(command, *pos_parms, **options):
583 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500584 Create and return a bash command string consisting of the given arguments formatted as text.
George Keishing16244c22019-01-31 16:16:14 +0000585
586 The default formatting of options is as follows:
587
588 <single dash><option name><space delim><option value>
589
590 Example:
591
592 -parm value
593
Michael Walsh9e042ad2019-10-16 17:14:31 -0500594 The caller can change the kind of dashes/delimiters used by specifying "arg_dashes" and/or "arg_delims"
595 as options. These options are processed specially by the create_command_string function and do NOT get
596 inserted into the resulting command string. All options following the arg_dashes/arg_delims options will
597 then use the specified values for dashes/delims. In the special case of arg_dashes equal to "--", the
George Keishing16244c22019-01-31 16:16:14 +0000598 arg_delim will automatically be changed to "=". See examples below.
599
600 Quoting rules:
601
Michael Walsh9e042ad2019-10-16 17:14:31 -0500602 The create_command_string function will single quote option values as needed to prevent bash expansion.
603 If the caller wishes to defeat this action, they may single or double quote the option value themselves.
604 See examples below.
George Keishing16244c22019-01-31 16:16:14 +0000605
Michael Walsh9e042ad2019-10-16 17:14:31 -0500606 pos_parms are NOT automatically quoted. The caller is advised to either explicitly add quotes or to use
607 the quote_bash_parm functions to quote any pos_parms.
George Keishing16244c22019-01-31 16:16:14 +0000608
609 Examples:
610
611 command_string = create_command_string('cd', '~')
612
613 Result:
614 cd ~
615
Michael Walsh9e042ad2019-10-16 17:14:31 -0500616 Note that the pos_parm ("~") does NOT get quoted, as per the aforementioned rules. If quotes are
617 desired, they may be added explicitly by the caller:
George Keishing16244c22019-01-31 16:16:14 +0000618
619 command_string = create_command_string('cd', '\'~\'')
620
621 Result:
622 cd '~'
623
624 command_string = create_command_string('grep', '\'^[^ ]*=\'',
625 '/tmp/myfile', i=None, m='1', arg_dashes='--', color='always')
626
627 Result:
628 grep -i -m 1 --color=always '^[^ ]*=' /tmp/myfile
629
Michael Walsh9e042ad2019-10-16 17:14:31 -0500630 In the preceding example, note the use of None to cause the "i" parm to be treated as a flag (i.e. no
631 argument value is generated). Also, note the use of arg_dashes to change the type of dashes used on all
632 subsequent options. The following example is equivalent to the prior. Note that quote_bash_parm is used
633 instead of including the quotes explicitly.
George Keishing16244c22019-01-31 16:16:14 +0000634
635 command_string = create_command_string('grep', quote_bash_parm('^[^ ]*='),
636 '/tmp/myfile', i=None, m='1', arg_dashes='--', color='always')
637
638 Result:
639 grep -i -m 1 --color=always '^[^ ]*=' /tmp/myfile
640
Michael Walsh9e042ad2019-10-16 17:14:31 -0500641 In the following example, note the automatic quoting of the password option, as per the aforementioned
642 rules.
George Keishing16244c22019-01-31 16:16:14 +0000643
644 command_string = create_command_string('my_pgm', '/tmp/myfile', i=None,
645 m='1', arg_dashes='--', password='${my_pw}')
646
Michael Walsh9e042ad2019-10-16 17:14:31 -0500647 However, let's say that the caller wishes to have bash expand the password value. To achieve this, the
648 caller can use double quotes:
George Keishing16244c22019-01-31 16:16:14 +0000649
650 command_string = create_command_string('my_pgm', '/tmp/myfile', i=None,
651 m='1', arg_dashes='--', password='"${my_pw}"')
652
653 Result:
654 my_pgm -i -m 1 --password="${my_pw}" /tmp/myfile
655
656 command_string = create_command_string('ipmitool', 'power status',
George Keishinge33ad1d2019-12-09 11:17:36 -0600657 I='lanplus', C='3', 'p=623', U='root', P='0penBmc', H='xx.xx.xx.xx')
George Keishing16244c22019-01-31 16:16:14 +0000658
659 Result:
George Keishinge33ad1d2019-12-09 11:17:36 -0600660 ipmitool -I lanplus -C 3 -p 623 -U root -P 0penBmc -H xx.xx.xx.xx power status
George Keishing16244c22019-01-31 16:16:14 +0000661
Michael Walsh9e042ad2019-10-16 17:14:31 -0500662 By default create_command_string will take measures to preserve the order of the callers options. In
663 some cases, this effort may fail (as when calling directly from a robot program). In this case, the
664 caller can accept the responsibility of keeping an ordered list of options by calling this function with
665 the last positional parm as some kind of dictionary (preferably an OrderedDict) and avoiding the use of
666 any actual option args.
George Keishing16244c22019-01-31 16:16:14 +0000667
668 Example:
669 kwargs = collections.OrderedDict([('pass', 0), ('fail', 0)])
670 command_string = create_command_string('my program', 'pos_parm1', kwargs)
671
672 Result:
673
674 my program -pass 0 -fail 0 pos_parm1
675
Michael Walsh9e042ad2019-10-16 17:14:31 -0500676 Note to programmers who wish to write a wrapper to this function: If the python version is less than
677 3.6, to get the options to be processed correctly, the wrapper function must include a _stack_frame_ix_
678 keyword argument to allow this function to properly re-order options:
George Keishing16244c22019-01-31 16:16:14 +0000679
680 def create_ipmi_ext_command_string(command, **kwargs):
681
682 return create_command_string('ipmitool', command, _stack_frame_ix_=2,
683 **kwargs)
684
685 Example call of wrapper function:
686
Michael Walsh9e042ad2019-10-16 17:14:31 -0500687 command_string = create_ipmi_ext_command_string('power status', I='lanplus')
George Keishing16244c22019-01-31 16:16:14 +0000688
689 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500690 command The command (e.g. "cat", "sort", "ipmitool", etc.).
691 pos_parms The positional parms for the command (e.g. PATTERN, FILENAME, etc.).
692 These will be placed at the end of the resulting command string.
693 options The command options (e.g. "-m 1", "--max-count=NUM", etc.). Note that if
694 the value of any option is None, then it will be understood to be a flag
695 (for which no value is required).
George Keishing16244c22019-01-31 16:16:14 +0000696 """
697
698 arg_dashes = "-"
699 delim = default_arg_delim(arg_dashes)
700
701 command_string = command
702
Michael Walsh23ac9b42019-02-20 16:46:04 -0600703 if len(pos_parms) > 0 and gp.is_dict(pos_parms[-1]):
George Keishing16244c22019-01-31 16:16:14 +0000704 # Convert pos_parms from tuple to list.
705 pos_parms = list(pos_parms)
Michael Walsh9e042ad2019-10-16 17:14:31 -0500706 # Re-assign options to be the last pos_parm value (which is a dictionary).
George Keishing16244c22019-01-31 16:16:14 +0000707 options = pos_parms[-1]
708 # Now delete the last pos_parm.
709 del pos_parms[-1]
710 else:
Michael Walsh9e042ad2019-10-16 17:14:31 -0500711 # Either get stack_frame_ix from the caller via options or set it to the default value.
Patrick Williams20f38712022-12-08 06:18:26 -0600712 stack_frame_ix = options.pop("_stack_frame_ix_", 1)
Michael Walshfffddca2019-08-20 17:19:13 +0000713 if gm.python_version < gm.ordered_dict_version:
Michael Walsh9e042ad2019-10-16 17:14:31 -0500714 # Re-establish the original options order as specified on the original line of code. This
715 # function depends on correct order.
Michael Walshfffddca2019-08-20 17:19:13 +0000716 options = re_order_kwargs(stack_frame_ix, **options)
George Keishing16244c22019-01-31 16:16:14 +0000717 for key, value in options.items():
718 # Check for special values in options and process them.
719 if key == "arg_dashes":
720 arg_dashes = str(value)
721 delim = default_arg_delim(arg_dashes)
722 continue
723 if key == "arg_delim":
724 delim = str(value)
725 continue
726 # Format the options elements into the command string.
727 command_string += " " + arg_dashes + key
728 if value is not None:
729 command_string += delim
730 if re.match(r'^(["].*["]|[\'].*[\'])$', str(value)):
731 # Already quoted.
732 command_string += str(value)
733 else:
734 command_string += gm.quote_bash_parm(str(value))
Michael Walsh9e042ad2019-10-16 17:14:31 -0500735 # Finally, append the pos_parms to the end of the command_string. Use filter to eliminate blank pos
736 # parms.
Patrick Williams20f38712022-12-08 06:18:26 -0600737 command_string = " ".join([command_string] + list(filter(None, pos_parms)))
George Keishing16244c22019-01-31 16:16:14 +0000738
739 return command_string