blob: cac5ba41319b7fbe01714b5895dc6298cca10a9e [file] [log] [blame]
George Keishinge7e91712021-09-03 11:28:44 -05001#!/usr/bin/env python3
Michael Walshc3b512e2017-02-20 15:59:01 -06002
3r"""
4This module provides command execution functions such as cmd_fnc and cmd_fnc_u.
5"""
6
Michael Walsh21083d22018-06-01 14:19:32 -05007import os
Michael Walshc3b512e2017-02-20 15:59:01 -06008import sys
9import subprocess
Michael Walshf41fac82017-08-02 15:05:24 -050010import collections
Michael Walsh21083d22018-06-01 14:19:32 -050011import signal
12import time
George Keishing16244c22019-01-31 16:16:14 +000013import re
14import inspect
Michael Walshc3b512e2017-02-20 15:59:01 -060015
Michael Walshc3b512e2017-02-20 15:59:01 -060016import gen_print as gp
17import gen_valid as gv
18import gen_misc as gm
Michael Walsh1429e122019-05-20 10:06:18 -050019import func_args as fa
Michael Walshd6901502017-11-14 12:58:37 -060020
21robot_env = gp.robot_env
22
Michael Walshafc53a22017-04-12 15:52:28 -050023if robot_env:
Michael Walsha3e2f532018-01-10 13:43:42 -060024 from robot.libraries.BuiltIn import BuiltIn
Michael Walshc3b512e2017-02-20 15:59:01 -060025
26
Michael Walsh9e042ad2019-10-16 17:14:31 -050027# cmd_fnc and cmd_fnc_u should now be considered deprecated. shell_cmd and t_shell_cmd should be used
28# instead.
Michael Walshc3b512e2017-02-20 15:59:01 -060029def cmd_fnc(cmd_buf,
30 quiet=None,
31 test_mode=None,
Michael Walshafc53a22017-04-12 15:52:28 -050032 debug=0,
Michael Walshc3b512e2017-02-20 15:59:01 -060033 print_output=1,
Michael Walshcfe9fed2017-09-12 17:13:10 -050034 show_err=1,
Michael Walsha3e2f532018-01-10 13:43:42 -060035 return_stderr=0,
36 ignore_err=1):
Michael Walshc3b512e2017-02-20 15:59:01 -060037 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -050038 Run the given command in a shell and return the shell return code and the output.
Michael Walshc3b512e2017-02-20 15:59:01 -060039
40 Description of arguments:
41 cmd_buf The command string to be run in a shell.
Michael Walsh9e042ad2019-10-16 17:14:31 -050042 quiet Indicates whether this function should run the print_issuing() function
43 which prints "Issuing: <cmd string>" to stdout.
44 test_mode If test_mode is set, this function will not actually run the command. If
45 print_output is set, it will print "(test_mode) Issuing: <cmd string>" to
Michael Walshcfe9fed2017-09-12 17:13:10 -050046 stdout.
Michael Walsh9e042ad2019-10-16 17:14:31 -050047 debug If debug is set, this function will print extra debug info.
48 print_output If this is set, this function will print the stdout/stderr generated by
49 the shell command.
50 show_err If show_err is set, this function will print a standardized error report
51 if the shell command returns non-zero.
52 return_stderr If return_stderr is set, this function will process the stdout and stderr
53 streams from the shell command separately. It will also return stderr in
54 addition to the return code and the stdout.
Michael Walshc3b512e2017-02-20 15:59:01 -060055 """
56
Michael Walshcfe9fed2017-09-12 17:13:10 -050057 # Determine default values.
Michael Walshc3b512e2017-02-20 15:59:01 -060058 quiet = int(gm.global_default(quiet, 0))
59 test_mode = int(gm.global_default(test_mode, 0))
Michael Walshc3b512e2017-02-20 15:59:01 -060060
61 if debug:
Michael Walshafc53a22017-04-12 15:52:28 -050062 gp.print_vars(cmd_buf, quiet, test_mode, debug)
Michael Walshc3b512e2017-02-20 15:59:01 -060063
Michael Walshec01a6f2019-08-01 12:43:20 -050064 err_msg = gv.valid_value(cmd_buf)
Michael Walshc3b512e2017-02-20 15:59:01 -060065 if err_msg != "":
66 raise ValueError(err_msg)
67
68 if not quiet:
Michael Walshafc53a22017-04-12 15:52:28 -050069 gp.pissuing(cmd_buf, test_mode)
Michael Walshc3b512e2017-02-20 15:59:01 -060070
71 if test_mode:
Michael Walshcfe9fed2017-09-12 17:13:10 -050072 if return_stderr:
73 return 0, "", ""
74 else:
75 return 0, ""
76
77 if return_stderr:
78 err_buf = ""
79 stderr = subprocess.PIPE
80 else:
81 stderr = subprocess.STDOUT
Michael Walshc3b512e2017-02-20 15:59:01 -060082
83 sub_proc = subprocess.Popen(cmd_buf,
84 bufsize=1,
85 shell=True,
George Keishinge7e91712021-09-03 11:28:44 -050086 universal_newlines=True,
Michael Walsh3ba8ecd2018-04-24 11:33:25 -050087 executable='/bin/bash',
Michael Walshc3b512e2017-02-20 15:59:01 -060088 stdout=subprocess.PIPE,
Michael Walshcfe9fed2017-09-12 17:13:10 -050089 stderr=stderr)
Michael Walshc3b512e2017-02-20 15:59:01 -060090 out_buf = ""
Michael Walshcfe9fed2017-09-12 17:13:10 -050091 if return_stderr:
92 for line in sub_proc.stderr:
George Keishing36efbc02018-12-12 10:18:23 -060093 try:
94 err_buf += line
95 except TypeError:
96 line = line.decode("utf-8")
97 err_buf += line
Michael Walshcfe9fed2017-09-12 17:13:10 -050098 if not print_output:
99 continue
Michael Walshc108e422019-03-28 12:27:18 -0500100 gp.gp_print(line)
Michael Walshc3b512e2017-02-20 15:59:01 -0600101 for line in sub_proc.stdout:
George Keishing36efbc02018-12-12 10:18:23 -0600102 try:
103 out_buf += line
104 except TypeError:
105 line = line.decode("utf-8")
106 out_buf += line
Michael Walshc3b512e2017-02-20 15:59:01 -0600107 if not print_output:
108 continue
Michael Walshc108e422019-03-28 12:27:18 -0500109 gp.gp_print(line)
Michael Walshc3b512e2017-02-20 15:59:01 -0600110 if print_output and not robot_env:
111 sys.stdout.flush()
112 sub_proc.communicate()
113 shell_rc = sub_proc.returncode
Michael Walsha3e2f532018-01-10 13:43:42 -0600114 if shell_rc != 0:
115 err_msg = "The prior shell command failed.\n"
Michael Walsh1429e122019-05-20 10:06:18 -0500116 err_msg += gp.sprint_var(shell_rc, gp.hexa())
Michael Walshcfe9fed2017-09-12 17:13:10 -0500117 if not print_output:
118 err_msg += "out_buf:\n" + out_buf
Michael Walshc3b512e2017-02-20 15:59:01 -0600119
Michael Walsha3e2f532018-01-10 13:43:42 -0600120 if show_err:
Michael Walshc108e422019-03-28 12:27:18 -0500121 gp.print_error_report(err_msg)
Michael Walsha3e2f532018-01-10 13:43:42 -0600122 if not ignore_err:
123 if robot_env:
124 BuiltIn().fail(err_msg)
125 else:
126 raise ValueError(err_msg)
Michael Walshcfe9fed2017-09-12 17:13:10 -0500127
128 if return_stderr:
129 return shell_rc, out_buf, err_buf
130 else:
131 return shell_rc, out_buf
Michael Walshc3b512e2017-02-20 15:59:01 -0600132
Michael Walshc3b512e2017-02-20 15:59:01 -0600133
Michael Walshc3b512e2017-02-20 15:59:01 -0600134def cmd_fnc_u(cmd_buf,
135 quiet=None,
136 debug=None,
137 print_output=1,
Michael Walshcfe9fed2017-09-12 17:13:10 -0500138 show_err=1,
Michael Walsha3e2f532018-01-10 13:43:42 -0600139 return_stderr=0,
140 ignore_err=1):
Michael Walshc3b512e2017-02-20 15:59:01 -0600141 r"""
142 Call cmd_fnc with test_mode=0. See cmd_fnc (above) for details.
143
144 Note the "u" in "cmd_fnc_u" stands for "unconditional".
145 """
146
147 return cmd_fnc(cmd_buf, test_mode=0, quiet=quiet, debug=debug,
Michael Walshcfe9fed2017-09-12 17:13:10 -0500148 print_output=print_output, show_err=show_err,
Michael Walsha3e2f532018-01-10 13:43:42 -0600149 return_stderr=return_stderr, ignore_err=ignore_err)
Michael Walshc3b512e2017-02-20 15:59:01 -0600150
Michael Walshf41fac82017-08-02 15:05:24 -0500151
Michael Walshf41fac82017-08-02 15:05:24 -0500152def parse_command_string(command_string):
Michael Walshf41fac82017-08-02 15:05:24 -0500153 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500154 Parse a bash command-line command string and return the result as a dictionary of parms.
Michael Walshf41fac82017-08-02 15:05:24 -0500155
Michael Walsh9e042ad2019-10-16 17:14:31 -0500156 This can be useful for answering questions like "What did the user specify as the value for parm x in the
157 command string?".
Michael Walshf41fac82017-08-02 15:05:24 -0500158
Michael Walsh9e042ad2019-10-16 17:14:31 -0500159 This function expects the command string to follow the following posix conventions:
Michael Walshf41fac82017-08-02 15:05:24 -0500160 - Short parameters:
161 -<parm name><space><arg value>
162 - Long parameters:
163 --<parm name>=<arg value>
164
Michael Walsh9e042ad2019-10-16 17:14:31 -0500165 The first item in the string will be considered to be the command. All values not conforming to the
166 specifications above will be considered positional parms. If there are multiple parms with the same
167 name, they will be put into a list (see illustration below where "-v" is specified multiple times).
Michael Walshf41fac82017-08-02 15:05:24 -0500168
169 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500170 command_string The complete command string including all parameters and arguments.
Michael Walshf41fac82017-08-02 15:05:24 -0500171
172 Sample input:
173
Michael Walsh9e042ad2019-10-16 17:14:31 -0500174 robot_cmd_buf: robot -v OPENBMC_HOST:dummy1 -v keyword_string:'Set
175 Auto Reboot no' -v lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot -v quiet:0 -v
176 test_mode:0 -v debug:0 --outputdir='/home/user1/status/children/'
177 --output=dummy1.Auto_reboot.170802.124544.output.xml --log=dummy1.Auto_reboot.170802.124544.log.html
Michael Walshf41fac82017-08-02 15:05:24 -0500178 --report=dummy1.Auto_reboot.170802.124544.report.html
179 /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
180
181 Sample output:
182
183 robot_cmd_buf_dict:
184 robot_cmd_buf_dict[command]: robot
185 robot_cmd_buf_dict[v]:
186 robot_cmd_buf_dict[v][0]: OPENBMC_HOST:dummy1
Michael Walsh9e042ad2019-10-16 17:14:31 -0500187 robot_cmd_buf_dict[v][1]: keyword_string:Set Auto Reboot no
Michael Walshf41fac82017-08-02 15:05:24 -0500188 robot_cmd_buf_dict[v][2]:
189 lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot
190 robot_cmd_buf_dict[v][3]: quiet:0
191 robot_cmd_buf_dict[v][4]: test_mode:0
192 robot_cmd_buf_dict[v][5]: debug:0
Michael Walsh9e042ad2019-10-16 17:14:31 -0500193 robot_cmd_buf_dict[outputdir]: /home/user1/status/children/
194 robot_cmd_buf_dict[output]: dummy1.Auto_reboot.170802.124544.output.xml
195 robot_cmd_buf_dict[log]: dummy1.Auto_reboot.170802.124544.log.html
196 robot_cmd_buf_dict[report]: dummy1.Auto_reboot.170802.124544.report.html
Michael Walshf41fac82017-08-02 15:05:24 -0500197 robot_cmd_buf_dict[positional]:
198 /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
199 """
200
Michael Walsh9e042ad2019-10-16 17:14:31 -0500201 # We want the parms in the string broken down the way bash would do it, so we'll call upon bash to do
202 # that by creating a simple inline bash function.
Michael Walshf41fac82017-08-02 15:05:24 -0500203 bash_func_def = "function parse { for parm in \"${@}\" ; do" +\
204 " echo $parm ; done ; }"
205
206 rc, outbuf = cmd_fnc_u(bash_func_def + " ; parse " + command_string,
207 quiet=1, print_output=0)
208 command_string_list = outbuf.rstrip("\n").split("\n")
209
210 command_string_dict = collections.OrderedDict()
211 ix = 1
212 command_string_dict['command'] = command_string_list[0]
213 while ix < len(command_string_list):
214 if command_string_list[ix].startswith("--"):
215 key, value = command_string_list[ix].split("=")
216 key = key.lstrip("-")
217 elif command_string_list[ix].startswith("-"):
218 key = command_string_list[ix].lstrip("-")
219 ix += 1
220 try:
221 value = command_string_list[ix]
222 except IndexError:
223 value = ""
224 else:
225 key = 'positional'
226 value = command_string_list[ix]
227 if key in command_string_dict:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500228 if isinstance(command_string_dict[key], str):
Michael Walshf41fac82017-08-02 15:05:24 -0500229 command_string_dict[key] = [command_string_dict[key]]
230 command_string_dict[key].append(value)
231 else:
232 command_string_dict[key] = value
233 ix += 1
234
235 return command_string_dict
Michael Walsh21083d22018-06-01 14:19:32 -0500236
237
238# Save the original SIGALRM handler for later restoration by shell_cmd.
239original_sigalrm_handler = signal.getsignal(signal.SIGALRM)
240
241
242def shell_cmd_timed_out(signal_number,
243 frame):
244 r"""
245 Handle an alarm signal generated during the shell_cmd function.
246 """
247
248 gp.dprint_executing()
Michael Walsh3fb26182019-08-28 16:51:05 -0500249 global command_timed_out
250 command_timed_out = True
Michael Walsh21083d22018-06-01 14:19:32 -0500251 # Get subprocess pid from shell_cmd's call stack.
252 sub_proc = gp.get_stack_var('sub_proc', 0)
253 pid = sub_proc.pid
Michael Walsh3fb26182019-08-28 16:51:05 -0500254 gp.dprint_var(pid)
255 # Terminate the child process group.
256 os.killpg(pid, signal.SIGKILL)
Michael Walsh21083d22018-06-01 14:19:32 -0500257 # Restore the original SIGALRM handler.
258 signal.signal(signal.SIGALRM, original_sigalrm_handler)
259
260 return
261
262
263def shell_cmd(command_string,
264 quiet=None,
Michael Walsh55abd1b2018-08-31 13:00:50 -0500265 print_output=None,
Michael Walsh21083d22018-06-01 14:19:32 -0500266 show_err=1,
267 test_mode=0,
268 time_out=None,
269 max_attempts=1,
270 retry_sleep_time=5,
Michael Walsh3fb26182019-08-28 16:51:05 -0500271 valid_rcs=[0],
Michael Walsh21083d22018-06-01 14:19:32 -0500272 ignore_err=None,
Michael Walshfaafa9c2018-06-27 16:39:31 -0500273 return_stderr=0,
Michael Walsh9e042ad2019-10-16 17:14:31 -0500274 fork=0,
275 error_regexes=None):
Michael Walsh21083d22018-06-01 14:19:32 -0500276 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500277 Run the given command string in a shell and return a tuple consisting of the shell return code and the
278 output.
Michael Walsh21083d22018-06-01 14:19:32 -0500279
280 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500281 command_string The command string to be run in a shell (e.g. "ls /tmp").
282 quiet If set to 0, this function will print "Issuing: <cmd string>" to stdout.
283 When the quiet argument is set to None, this function will assign a
284 default value by searching upward in the stack for the quiet variable
285 value. If no such value is found, quiet is set to 0.
286 print_output If this is set, this function will print the stdout/stderr generated by
287 the shell command to stdout.
288 show_err If show_err is set, this function will print a standardized error report
289 if the shell command fails (i.e. if the shell command returns a shell_rc
290 that is not in valid_rcs). Note: Error text is only printed if ALL
291 attempts to run the command_string fail. In other words, if the command
292 execution is ultimately successful, initial failures are hidden.
293 test_mode If test_mode is set, this function will not actually run the command. If
294 print_output is also set, this function will print "(test_mode) Issuing:
295 <cmd string>" to stdout. A caller should call shell_cmd directly if they
296 wish to have the command string run unconditionally. They should call
297 the t_shell_cmd wrapper (defined below) if they wish to run the command
298 string only if the prevailing test_mode variable is set to 0.
299 time_out A time-out value expressed in seconds. If the command string has not
300 finished executing within <time_out> seconds, it will be halted and
301 counted as an error.
302 max_attempts The max number of attempts that should be made to run the command string.
303 retry_sleep_time The number of seconds to sleep between attempts.
304 valid_rcs A list of integers indicating which shell_rc values are not to be
305 considered errors.
306 ignore_err Ignore error means that a failure encountered by running the command
307 string will not be raised as a python exception. When the ignore_err
308 argument is set to None, this function will assign a default value by
309 searching upward in the stack for the ignore_err variable value. If no
310 such value is found, ignore_err is set to 1.
311 return_stderr If return_stderr is set, this function will process the stdout and stderr
312 streams from the shell command separately. In such a case, the tuple
313 returned by this function will consist of three values rather than just
314 two: rc, stdout, stderr.
315 fork Run the command string asynchronously (i.e. don't wait for status of the
316 child process and don't try to get stdout/stderr) and return the Popen
317 object created by the subprocess.popen() function. See the kill_cmd
318 function for details on how to process the popen object.
319 error_regexes A list of regular expressions to be used to identify errors in the
320 command output. If there is a match for any of these regular
321 expressions, the command will be considered a failure and the shell_rc
322 will be set to -1. For example, if error_regexes = ['ERROR:'] and the
323 command output contains 'ERROR: Unrecognized option', it will be counted
324 as an error even if the command returned 0. This is useful when running
325 commands that do not always return non-zero on error.
Michael Walsh21083d22018-06-01 14:19:32 -0500326 """
327
Michael Walsh3fb26182019-08-28 16:51:05 -0500328 err_msg = gv.valid_value(command_string)
329 if err_msg:
330 raise ValueError(err_msg)
331
Michael Walsh21083d22018-06-01 14:19:32 -0500332 # Assign default values to some of the arguments to this function.
Michael Walsh4cb9b2a2018-06-08 17:57:00 -0500333 quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0)))
Michael Walsh55abd1b2018-08-31 13:00:50 -0500334 print_output = int(gm.dft(print_output, not quiet))
Michael Walsh45fead42018-09-26 17:20:48 -0500335 show_err = int(show_err)
Michael Walsh3fb26182019-08-28 16:51:05 -0500336 ignore_err = int(gm.dft(ignore_err, gp.get_stack_var('ignore_err', 1)))
Michael Walsh21083d22018-06-01 14:19:32 -0500337
Michael Walsh3fb26182019-08-28 16:51:05 -0500338 gp.qprint_issuing(command_string, test_mode)
Michael Walsh21083d22018-06-01 14:19:32 -0500339 if test_mode:
Michael Walsh3fb26182019-08-28 16:51:05 -0500340 return (0, "", "") if return_stderr else (0, "")
Michael Walsh21083d22018-06-01 14:19:32 -0500341
Michael Walsh3fb26182019-08-28 16:51:05 -0500342 # Convert a string python dictionary definition to a dictionary.
343 valid_rcs = fa.source_to_object(valid_rcs)
Michael Walsh21083d22018-06-01 14:19:32 -0500344 # Convert each list entry to a signed value.
Michael Walsh3fb26182019-08-28 16:51:05 -0500345 valid_rcs = [gm.to_signed(x) for x in valid_rcs]
Michael Walsh21083d22018-06-01 14:19:32 -0500346
Michael Walsh3fb26182019-08-28 16:51:05 -0500347 stderr = subprocess.PIPE if return_stderr else subprocess.STDOUT
Michael Walsh21083d22018-06-01 14:19:32 -0500348
Michael Walsh9e042ad2019-10-16 17:14:31 -0500349 # Write all output to func_out_history_buf rather than directly to stdout. This allows us to decide
350 # what to print after all attempts to run the command string have been made. func_out_history_buf will
351 # contain the complete history from the current invocation of this function.
Michael Walsh3fb26182019-08-28 16:51:05 -0500352 global command_timed_out
353 command_timed_out = False
354 func_out_history_buf = ""
Michael Walsh21083d22018-06-01 14:19:32 -0500355 for attempt_num in range(1, max_attempts + 1):
356 sub_proc = subprocess.Popen(command_string,
357 bufsize=1,
358 shell=True,
Michael Walsh3fb26182019-08-28 16:51:05 -0500359 universal_newlines=True,
Michael Walsh21083d22018-06-01 14:19:32 -0500360 executable='/bin/bash',
George Keishinge7e91712021-09-03 11:28:44 -0500361 stdin=subprocess.PIPE,
Michael Walsh21083d22018-06-01 14:19:32 -0500362 stdout=subprocess.PIPE,
363 stderr=stderr)
Michael Walshfaafa9c2018-06-27 16:39:31 -0500364 if fork:
Michael Walsh4d5f6682019-09-24 14:23:40 -0500365 return sub_proc
Michael Walsh3fb26182019-08-28 16:51:05 -0500366
367 if time_out:
368 command_timed_out = False
Michael Walsh21083d22018-06-01 14:19:32 -0500369 # Designate a SIGALRM handling function and set alarm.
370 signal.signal(signal.SIGALRM, shell_cmd_timed_out)
371 signal.alarm(time_out)
372 try:
Michael Walsh3fb26182019-08-28 16:51:05 -0500373 stdout_buf, stderr_buf = sub_proc.communicate()
Michael Walsh21083d22018-06-01 14:19:32 -0500374 except IOError:
375 command_timed_out = True
Michael Walsh21083d22018-06-01 14:19:32 -0500376 # Restore the original SIGALRM handler and clear the alarm.
377 signal.signal(signal.SIGALRM, original_sigalrm_handler)
378 signal.alarm(0)
Michael Walsh3fb26182019-08-28 16:51:05 -0500379
Michael Walsh9e042ad2019-10-16 17:14:31 -0500380 # Output from this loop iteration is written to func_out_buf for later processing. This can include
381 # stdout, stderr and our own error messages.
Michael Walsh3fb26182019-08-28 16:51:05 -0500382 func_out_buf = ""
383 if print_output:
384 if return_stderr:
385 func_out_buf += stderr_buf
386 func_out_buf += stdout_buf
387 shell_rc = sub_proc.returncode
388 if shell_rc in valid_rcs:
Michael Walsh9e042ad2019-10-16 17:14:31 -0500389 # Check output for text indicating there is an error.
390 if error_regexes and re.match('|'.join(error_regexes), stdout_buf):
391 shell_rc = -1
392 else:
393 break
Michael Walsh21083d22018-06-01 14:19:32 -0500394 err_msg = "The prior shell command failed.\n"
Michael Walsh21083d22018-06-01 14:19:32 -0500395 err_msg += gp.sprint_var(attempt_num)
Michael Walsh3fb26182019-08-28 16:51:05 -0500396 err_msg += gp.sprint_vars(command_string, command_timed_out, time_out)
397 err_msg += gp.sprint_varx("child_pid", sub_proc.pid)
398 err_msg += gp.sprint_vars(shell_rc, valid_rcs, fmt=gp.hexa())
Michael Walsh9e042ad2019-10-16 17:14:31 -0500399 if error_regexes:
400 err_msg += gp.sprint_vars(error_regexes)
Michael Walsh21083d22018-06-01 14:19:32 -0500401 if not print_output:
402 if return_stderr:
Michael Walsh3fb26182019-08-28 16:51:05 -0500403 err_msg += "stderr_buf:\n" + stderr_buf
404 err_msg += "stdout_buf:\n" + stdout_buf
Michael Walsh21083d22018-06-01 14:19:32 -0500405 if show_err:
Michael Walsh3fb26182019-08-28 16:51:05 -0500406 func_out_buf += gp.sprint_error_report(err_msg)
Michael Walsh21083d22018-06-01 14:19:32 -0500407 if attempt_num < max_attempts:
Michael Walsh3fb26182019-08-28 16:51:05 -0500408 cmd_buf = "time.sleep(" + str(retry_sleep_time) + ")"
409 if show_err:
410 func_out_buf += gp.sprint_issuing(cmd_buf)
411 exec(cmd_buf)
412 func_out_history_buf += func_out_buf
Michael Walsh21083d22018-06-01 14:19:32 -0500413
Michael Walsh3fb26182019-08-28 16:51:05 -0500414 if shell_rc in valid_rcs:
415 gp.gp_print(func_out_buf)
416 else:
417 if show_err:
418 gp.gp_print(func_out_history_buf, stream='stderr')
419 else:
Michael Walsh9e042ad2019-10-16 17:14:31 -0500420 # There is no error information to show so just print output from last loop iteration.
Michael Walsh3fb26182019-08-28 16:51:05 -0500421 gp.gp_print(func_out_buf)
Michael Walsh21083d22018-06-01 14:19:32 -0500422 if not ignore_err:
Michael Walsh9e042ad2019-10-16 17:14:31 -0500423 # If the caller has already asked to show error info, avoid repeating that in the failure message.
Michael Walsh3fb26182019-08-28 16:51:05 -0500424 err_msg = "The prior shell command failed.\n" if show_err \
425 else err_msg
Michael Walsh21083d22018-06-01 14:19:32 -0500426 if robot_env:
427 BuiltIn().fail(err_msg)
428 else:
Michael Walsh3fb26182019-08-28 16:51:05 -0500429 raise ValueError(err_msg)
Michael Walsh21083d22018-06-01 14:19:32 -0500430
Michael Walsh3fb26182019-08-28 16:51:05 -0500431 return (shell_rc, stdout_buf, stderr_buf) if return_stderr \
432 else (shell_rc, stdout_buf)
Michael Walsh21083d22018-06-01 14:19:32 -0500433
434
435def t_shell_cmd(command_string, **kwargs):
436 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500437 Search upward in the the call stack to obtain the test_mode argument, add it to kwargs and then call
438 shell_cmd and return the result.
Michael Walsh21083d22018-06-01 14:19:32 -0500439
440 See shell_cmd prolog for details on all arguments.
441 """
442
443 if 'test_mode' in kwargs:
444 error_message = "Programmer error - test_mode is not a valid" +\
445 " argument to this function."
446 gp.print_error_report(error_message)
447 exit(1)
448
Michael Walsh3fb26182019-08-28 16:51:05 -0500449 test_mode = int(gp.get_stack_var('test_mode', 0))
Michael Walsh21083d22018-06-01 14:19:32 -0500450 kwargs['test_mode'] = test_mode
451
452 return shell_cmd(command_string, **kwargs)
George Keishing16244c22019-01-31 16:16:14 +0000453
454
Michael Walsh4d5f6682019-09-24 14:23:40 -0500455def kill_cmd(popen, sig=signal.SIGTERM):
456 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500457 Kill the subprocess represented by the Popen object and return a tuple consisting of the shell return
458 code and the output.
Michael Walsh4d5f6682019-09-24 14:23:40 -0500459
Michael Walsh9e042ad2019-10-16 17:14:31 -0500460 This function is meant to be used as the follow-up for a call to shell_cmd(..., fork=1).
Michael Walsh4d5f6682019-09-24 14:23:40 -0500461
462 Example:
463 popen = shell_cmd("some_pgm.py", fork=1)
464 ...
465 shell_rc, output = kill_cmd(popen)
466
467 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500468 popen A Popen object returned by the subprocess.Popen() command.
Michael Walsh4d5f6682019-09-24 14:23:40 -0500469 sig The signal to be sent to the child process.
470 """
471
472 gp.dprint_var(popen.pid)
473 os.killpg(popen.pid, sig)
474 stdout, stderr = popen.communicate()
475 shell_rc = popen.returncode
476 return (shell_rc, stdout, stderr) if stderr else (shell_rc, stdout)
477
478
George Keishing16244c22019-01-31 16:16:14 +0000479def re_order_kwargs(stack_frame_ix, **kwargs):
480 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500481 Re-order the kwargs to match the order in which they were specified on a function invocation and return
482 as an ordered dictionary.
George Keishing16244c22019-01-31 16:16:14 +0000483
Michael Walsh9e042ad2019-10-16 17:14:31 -0500484 Note that this re_order_kwargs function should not be necessary in python versions 3.6 and beyond.
George Keishing16244c22019-01-31 16:16:14 +0000485
486 Example:
487
488 The caller calls func1 like this:
489
490 func1('mike', arg1='one', arg2='two', arg3='three')
491
492 And func1 is defined as follows:
493
494 def func1(first_arg, **kwargs):
495
496 kwargs = re_order_kwargs(first_arg_num=2, stack_frame_ix=3, **kwargs)
497
Michael Walsh9e042ad2019-10-16 17:14:31 -0500498 The kwargs dictionary before calling re_order_kwargs (where order is not guaranteed):
George Keishing16244c22019-01-31 16:16:14 +0000499
500 kwargs:
501 kwargs[arg3]: three
502 kwargs[arg2]: two
503 kwargs[arg1]: one
504
505 The kwargs dictionary after calling re_order_kwargs:
506
507 kwargs:
508 kwargs[arg1]: one
509 kwargs[arg2]: two
510 kwargs[arg3]: three
511
Michael Walsh9e042ad2019-10-16 17:14:31 -0500512 Note that the re-ordered kwargs match the order specified on the call to func1.
George Keishing16244c22019-01-31 16:16:14 +0000513
514 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500515 stack_frame_ix The stack frame of the function whose kwargs values must be re-ordered.
516 0 is the stack frame of re_order_kwargs, 1 is the stack from of its
517 caller and so on.
518 kwargs The keyword argument dictionary which is to be re-ordered.
George Keishing16244c22019-01-31 16:16:14 +0000519 """
520
521 new_kwargs = collections.OrderedDict()
522
523 # Get position number of first keyword on the calling line of code.
524 (args, varargs, keywords, locals) =\
525 inspect.getargvalues(inspect.stack()[stack_frame_ix][0])
526 first_kwarg_pos = 1 + len(args)
527 if varargs is not None:
528 first_kwarg_pos += len(locals[varargs])
529 for arg_num in range(first_kwarg_pos, first_kwarg_pos + len(kwargs)):
530 # This will result in an arg_name value such as "arg1='one'".
531 arg_name = gp.get_arg_name(None, arg_num, stack_frame_ix + 2)
532 # Continuing with the prior example, the following line will result
533 # in key being set to 'arg1'.
534 key = arg_name.split('=')[0]
535 new_kwargs[key] = kwargs[key]
536
537 return new_kwargs
538
539
540def default_arg_delim(arg_dashes):
541 r"""
542 Return the default argument delimiter value for the given arg_dashes value.
543
Michael Walsh9e042ad2019-10-16 17:14:31 -0500544 Note: this function is useful for functions that manipulate bash command line arguments (e.g. --parm=1 or
545 -parm 1).
George Keishing16244c22019-01-31 16:16:14 +0000546
547 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500548 arg_dashes The argument dashes specifier (usually, "-" or "--").
George Keishing16244c22019-01-31 16:16:14 +0000549 """
550
551 if arg_dashes == "--":
552 return "="
553
554 return " "
555
556
557def create_command_string(command, *pos_parms, **options):
558 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500559 Create and return a bash command string consisting of the given arguments formatted as text.
George Keishing16244c22019-01-31 16:16:14 +0000560
561 The default formatting of options is as follows:
562
563 <single dash><option name><space delim><option value>
564
565 Example:
566
567 -parm value
568
Michael Walsh9e042ad2019-10-16 17:14:31 -0500569 The caller can change the kind of dashes/delimiters used by specifying "arg_dashes" and/or "arg_delims"
570 as options. These options are processed specially by the create_command_string function and do NOT get
571 inserted into the resulting command string. All options following the arg_dashes/arg_delims options will
572 then use the specified values for dashes/delims. In the special case of arg_dashes equal to "--", the
George Keishing16244c22019-01-31 16:16:14 +0000573 arg_delim will automatically be changed to "=". See examples below.
574
575 Quoting rules:
576
Michael Walsh9e042ad2019-10-16 17:14:31 -0500577 The create_command_string function will single quote option values as needed to prevent bash expansion.
578 If the caller wishes to defeat this action, they may single or double quote the option value themselves.
579 See examples below.
George Keishing16244c22019-01-31 16:16:14 +0000580
Michael Walsh9e042ad2019-10-16 17:14:31 -0500581 pos_parms are NOT automatically quoted. The caller is advised to either explicitly add quotes or to use
582 the quote_bash_parm functions to quote any pos_parms.
George Keishing16244c22019-01-31 16:16:14 +0000583
584 Examples:
585
586 command_string = create_command_string('cd', '~')
587
588 Result:
589 cd ~
590
Michael Walsh9e042ad2019-10-16 17:14:31 -0500591 Note that the pos_parm ("~") does NOT get quoted, as per the aforementioned rules. If quotes are
592 desired, they may be added explicitly by the caller:
George Keishing16244c22019-01-31 16:16:14 +0000593
594 command_string = create_command_string('cd', '\'~\'')
595
596 Result:
597 cd '~'
598
599 command_string = create_command_string('grep', '\'^[^ ]*=\'',
600 '/tmp/myfile', i=None, m='1', arg_dashes='--', color='always')
601
602 Result:
603 grep -i -m 1 --color=always '^[^ ]*=' /tmp/myfile
604
Michael Walsh9e042ad2019-10-16 17:14:31 -0500605 In the preceding example, note the use of None to cause the "i" parm to be treated as a flag (i.e. no
606 argument value is generated). Also, note the use of arg_dashes to change the type of dashes used on all
607 subsequent options. The following example is equivalent to the prior. Note that quote_bash_parm is used
608 instead of including the quotes explicitly.
George Keishing16244c22019-01-31 16:16:14 +0000609
610 command_string = create_command_string('grep', quote_bash_parm('^[^ ]*='),
611 '/tmp/myfile', i=None, m='1', arg_dashes='--', color='always')
612
613 Result:
614 grep -i -m 1 --color=always '^[^ ]*=' /tmp/myfile
615
Michael Walsh9e042ad2019-10-16 17:14:31 -0500616 In the following example, note the automatic quoting of the password option, as per the aforementioned
617 rules.
George Keishing16244c22019-01-31 16:16:14 +0000618
619 command_string = create_command_string('my_pgm', '/tmp/myfile', i=None,
620 m='1', arg_dashes='--', password='${my_pw}')
621
Michael Walsh9e042ad2019-10-16 17:14:31 -0500622 However, let's say that the caller wishes to have bash expand the password value. To achieve this, the
623 caller can use double quotes:
George Keishing16244c22019-01-31 16:16:14 +0000624
625 command_string = create_command_string('my_pgm', '/tmp/myfile', i=None,
626 m='1', arg_dashes='--', password='"${my_pw}"')
627
628 Result:
629 my_pgm -i -m 1 --password="${my_pw}" /tmp/myfile
630
631 command_string = create_command_string('ipmitool', 'power status',
George Keishinge33ad1d2019-12-09 11:17:36 -0600632 I='lanplus', C='3', 'p=623', U='root', P='0penBmc', H='xx.xx.xx.xx')
George Keishing16244c22019-01-31 16:16:14 +0000633
634 Result:
George Keishinge33ad1d2019-12-09 11:17:36 -0600635 ipmitool -I lanplus -C 3 -p 623 -U root -P 0penBmc -H xx.xx.xx.xx power status
George Keishing16244c22019-01-31 16:16:14 +0000636
Michael Walsh9e042ad2019-10-16 17:14:31 -0500637 By default create_command_string will take measures to preserve the order of the callers options. In
638 some cases, this effort may fail (as when calling directly from a robot program). In this case, the
639 caller can accept the responsibility of keeping an ordered list of options by calling this function with
640 the last positional parm as some kind of dictionary (preferably an OrderedDict) and avoiding the use of
641 any actual option args.
George Keishing16244c22019-01-31 16:16:14 +0000642
643 Example:
644 kwargs = collections.OrderedDict([('pass', 0), ('fail', 0)])
645 command_string = create_command_string('my program', 'pos_parm1', kwargs)
646
647 Result:
648
649 my program -pass 0 -fail 0 pos_parm1
650
Michael Walsh9e042ad2019-10-16 17:14:31 -0500651 Note to programmers who wish to write a wrapper to this function: If the python version is less than
652 3.6, to get the options to be processed correctly, the wrapper function must include a _stack_frame_ix_
653 keyword argument to allow this function to properly re-order options:
George Keishing16244c22019-01-31 16:16:14 +0000654
655 def create_ipmi_ext_command_string(command, **kwargs):
656
657 return create_command_string('ipmitool', command, _stack_frame_ix_=2,
658 **kwargs)
659
660 Example call of wrapper function:
661
Michael Walsh9e042ad2019-10-16 17:14:31 -0500662 command_string = create_ipmi_ext_command_string('power status', I='lanplus')
George Keishing16244c22019-01-31 16:16:14 +0000663
664 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500665 command The command (e.g. "cat", "sort", "ipmitool", etc.).
666 pos_parms The positional parms for the command (e.g. PATTERN, FILENAME, etc.).
667 These will be placed at the end of the resulting command string.
668 options The command options (e.g. "-m 1", "--max-count=NUM", etc.). Note that if
669 the value of any option is None, then it will be understood to be a flag
670 (for which no value is required).
George Keishing16244c22019-01-31 16:16:14 +0000671 """
672
673 arg_dashes = "-"
674 delim = default_arg_delim(arg_dashes)
675
676 command_string = command
677
Michael Walsh23ac9b42019-02-20 16:46:04 -0600678 if len(pos_parms) > 0 and gp.is_dict(pos_parms[-1]):
George Keishing16244c22019-01-31 16:16:14 +0000679 # Convert pos_parms from tuple to list.
680 pos_parms = list(pos_parms)
Michael Walsh9e042ad2019-10-16 17:14:31 -0500681 # Re-assign options to be the last pos_parm value (which is a dictionary).
George Keishing16244c22019-01-31 16:16:14 +0000682 options = pos_parms[-1]
683 # Now delete the last pos_parm.
684 del pos_parms[-1]
685 else:
Michael Walsh9e042ad2019-10-16 17:14:31 -0500686 # Either get stack_frame_ix from the caller via options or set it to the default value.
Michael Walshfffddca2019-08-20 17:19:13 +0000687 stack_frame_ix = options.pop('_stack_frame_ix_', 1)
688 if gm.python_version < gm.ordered_dict_version:
Michael Walsh9e042ad2019-10-16 17:14:31 -0500689 # Re-establish the original options order as specified on the original line of code. This
690 # function depends on correct order.
Michael Walshfffddca2019-08-20 17:19:13 +0000691 options = re_order_kwargs(stack_frame_ix, **options)
George Keishing16244c22019-01-31 16:16:14 +0000692 for key, value in options.items():
693 # Check for special values in options and process them.
694 if key == "arg_dashes":
695 arg_dashes = str(value)
696 delim = default_arg_delim(arg_dashes)
697 continue
698 if key == "arg_delim":
699 delim = str(value)
700 continue
701 # Format the options elements into the command string.
702 command_string += " " + arg_dashes + key
703 if value is not None:
704 command_string += delim
705 if re.match(r'^(["].*["]|[\'].*[\'])$', str(value)):
706 # Already quoted.
707 command_string += str(value)
708 else:
709 command_string += gm.quote_bash_parm(str(value))
Michael Walsh9e042ad2019-10-16 17:14:31 -0500710 # Finally, append the pos_parms to the end of the command_string. Use filter to eliminate blank pos
711 # parms.
Michael Walsh23ac9b42019-02-20 16:46:04 -0600712 command_string = ' '.join([command_string] + list(filter(None, pos_parms)))
George Keishing16244c22019-01-31 16:16:14 +0000713
714 return command_string