blob: 18b79e9b93411f0afbaf45e22c2b97773a341720 [file] [log] [blame]
Michael Walshc3b512e2017-02-20 15:59:01 -06001#!/usr/bin/env python
2
3r"""
4This module provides command execution functions such as cmd_fnc and cmd_fnc_u.
5"""
6
Michael Walsh21083d22018-06-01 14:19:32 -05007import os
Michael Walshc3b512e2017-02-20 15:59:01 -06008import sys
9import subprocess
Michael Walshf41fac82017-08-02 15:05:24 -050010import collections
Michael Walsh21083d22018-06-01 14:19:32 -050011import signal
12import time
George Keishing16244c22019-01-31 16:16:14 +000013import re
14import inspect
Michael Walshc3b512e2017-02-20 15:59:01 -060015
Michael Walshc3b512e2017-02-20 15:59:01 -060016import gen_print as gp
17import gen_valid as gv
18import gen_misc as gm
Michael Walsh1429e122019-05-20 10:06:18 -050019import func_args as fa
Michael Walshd6901502017-11-14 12:58:37 -060020
21robot_env = gp.robot_env
22
Michael Walshafc53a22017-04-12 15:52:28 -050023if robot_env:
Michael Walsha3e2f532018-01-10 13:43:42 -060024 from robot.libraries.BuiltIn import BuiltIn
Michael Walshc3b512e2017-02-20 15:59:01 -060025
26
Michael Walsh9e042ad2019-10-16 17:14:31 -050027# cmd_fnc and cmd_fnc_u should now be considered deprecated. shell_cmd and t_shell_cmd should be used
28# instead.
Michael Walshc3b512e2017-02-20 15:59:01 -060029def cmd_fnc(cmd_buf,
30 quiet=None,
31 test_mode=None,
Michael Walshafc53a22017-04-12 15:52:28 -050032 debug=0,
Michael Walshc3b512e2017-02-20 15:59:01 -060033 print_output=1,
Michael Walshcfe9fed2017-09-12 17:13:10 -050034 show_err=1,
Michael Walsha3e2f532018-01-10 13:43:42 -060035 return_stderr=0,
36 ignore_err=1):
Michael Walshc3b512e2017-02-20 15:59:01 -060037 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -050038 Run the given command in a shell and return the shell return code and the output.
Michael Walshc3b512e2017-02-20 15:59:01 -060039
40 Description of arguments:
41 cmd_buf The command string to be run in a shell.
Michael Walsh9e042ad2019-10-16 17:14:31 -050042 quiet Indicates whether this function should run the print_issuing() function
43 which prints "Issuing: <cmd string>" to stdout.
44 test_mode If test_mode is set, this function will not actually run the command. If
45 print_output is set, it will print "(test_mode) Issuing: <cmd string>" to
Michael Walshcfe9fed2017-09-12 17:13:10 -050046 stdout.
Michael Walsh9e042ad2019-10-16 17:14:31 -050047 debug If debug is set, this function will print extra debug info.
48 print_output If this is set, this function will print the stdout/stderr generated by
49 the shell command.
50 show_err If show_err is set, this function will print a standardized error report
51 if the shell command returns non-zero.
52 return_stderr If return_stderr is set, this function will process the stdout and stderr
53 streams from the shell command separately. It will also return stderr in
54 addition to the return code and the stdout.
Michael Walshc3b512e2017-02-20 15:59:01 -060055 """
56
Michael Walshcfe9fed2017-09-12 17:13:10 -050057 # Determine default values.
Michael Walshc3b512e2017-02-20 15:59:01 -060058 quiet = int(gm.global_default(quiet, 0))
59 test_mode = int(gm.global_default(test_mode, 0))
Michael Walshc3b512e2017-02-20 15:59:01 -060060
61 if debug:
Michael Walshafc53a22017-04-12 15:52:28 -050062 gp.print_vars(cmd_buf, quiet, test_mode, debug)
Michael Walshc3b512e2017-02-20 15:59:01 -060063
Michael Walshec01a6f2019-08-01 12:43:20 -050064 err_msg = gv.valid_value(cmd_buf)
Michael Walshc3b512e2017-02-20 15:59:01 -060065 if err_msg != "":
66 raise ValueError(err_msg)
67
68 if not quiet:
Michael Walshafc53a22017-04-12 15:52:28 -050069 gp.pissuing(cmd_buf, test_mode)
Michael Walshc3b512e2017-02-20 15:59:01 -060070
71 if test_mode:
Michael Walshcfe9fed2017-09-12 17:13:10 -050072 if return_stderr:
73 return 0, "", ""
74 else:
75 return 0, ""
76
77 if return_stderr:
78 err_buf = ""
79 stderr = subprocess.PIPE
80 else:
81 stderr = subprocess.STDOUT
Michael Walshc3b512e2017-02-20 15:59:01 -060082
83 sub_proc = subprocess.Popen(cmd_buf,
84 bufsize=1,
85 shell=True,
Michael Walsh3ba8ecd2018-04-24 11:33:25 -050086 executable='/bin/bash',
Michael Walshc3b512e2017-02-20 15:59:01 -060087 stdout=subprocess.PIPE,
Michael Walshcfe9fed2017-09-12 17:13:10 -050088 stderr=stderr)
Michael Walshc3b512e2017-02-20 15:59:01 -060089 out_buf = ""
Michael Walshcfe9fed2017-09-12 17:13:10 -050090 if return_stderr:
91 for line in sub_proc.stderr:
George Keishing36efbc02018-12-12 10:18:23 -060092 try:
93 err_buf += line
94 except TypeError:
95 line = line.decode("utf-8")
96 err_buf += line
Michael Walshcfe9fed2017-09-12 17:13:10 -050097 if not print_output:
98 continue
Michael Walshc108e422019-03-28 12:27:18 -050099 gp.gp_print(line)
Michael Walshc3b512e2017-02-20 15:59:01 -0600100 for line in sub_proc.stdout:
George Keishing36efbc02018-12-12 10:18:23 -0600101 try:
102 out_buf += line
103 except TypeError:
104 line = line.decode("utf-8")
105 out_buf += line
Michael Walshc3b512e2017-02-20 15:59:01 -0600106 if not print_output:
107 continue
Michael Walshc108e422019-03-28 12:27:18 -0500108 gp.gp_print(line)
Michael Walshc3b512e2017-02-20 15:59:01 -0600109 if print_output and not robot_env:
110 sys.stdout.flush()
111 sub_proc.communicate()
112 shell_rc = sub_proc.returncode
Michael Walsha3e2f532018-01-10 13:43:42 -0600113 if shell_rc != 0:
114 err_msg = "The prior shell command failed.\n"
Michael Walsh1429e122019-05-20 10:06:18 -0500115 err_msg += gp.sprint_var(shell_rc, gp.hexa())
Michael Walshcfe9fed2017-09-12 17:13:10 -0500116 if not print_output:
117 err_msg += "out_buf:\n" + out_buf
Michael Walshc3b512e2017-02-20 15:59:01 -0600118
Michael Walsha3e2f532018-01-10 13:43:42 -0600119 if show_err:
Michael Walshc108e422019-03-28 12:27:18 -0500120 gp.print_error_report(err_msg)
Michael Walsha3e2f532018-01-10 13:43:42 -0600121 if not ignore_err:
122 if robot_env:
123 BuiltIn().fail(err_msg)
124 else:
125 raise ValueError(err_msg)
Michael Walshcfe9fed2017-09-12 17:13:10 -0500126
127 if return_stderr:
128 return shell_rc, out_buf, err_buf
129 else:
130 return shell_rc, out_buf
Michael Walshc3b512e2017-02-20 15:59:01 -0600131
Michael Walshc3b512e2017-02-20 15:59:01 -0600132
Michael Walshc3b512e2017-02-20 15:59:01 -0600133def cmd_fnc_u(cmd_buf,
134 quiet=None,
135 debug=None,
136 print_output=1,
Michael Walshcfe9fed2017-09-12 17:13:10 -0500137 show_err=1,
Michael Walsha3e2f532018-01-10 13:43:42 -0600138 return_stderr=0,
139 ignore_err=1):
Michael Walshc3b512e2017-02-20 15:59:01 -0600140 r"""
141 Call cmd_fnc with test_mode=0. See cmd_fnc (above) for details.
142
143 Note the "u" in "cmd_fnc_u" stands for "unconditional".
144 """
145
146 return cmd_fnc(cmd_buf, test_mode=0, quiet=quiet, debug=debug,
Michael Walshcfe9fed2017-09-12 17:13:10 -0500147 print_output=print_output, show_err=show_err,
Michael Walsha3e2f532018-01-10 13:43:42 -0600148 return_stderr=return_stderr, ignore_err=ignore_err)
Michael Walshc3b512e2017-02-20 15:59:01 -0600149
Michael Walshf41fac82017-08-02 15:05:24 -0500150
Michael Walshf41fac82017-08-02 15:05:24 -0500151def parse_command_string(command_string):
Michael Walshf41fac82017-08-02 15:05:24 -0500152 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500153 Parse a bash command-line command string and return the result as a dictionary of parms.
Michael Walshf41fac82017-08-02 15:05:24 -0500154
Michael Walsh9e042ad2019-10-16 17:14:31 -0500155 This can be useful for answering questions like "What did the user specify as the value for parm x in the
156 command string?".
Michael Walshf41fac82017-08-02 15:05:24 -0500157
Michael Walsh9e042ad2019-10-16 17:14:31 -0500158 This function expects the command string to follow the following posix conventions:
Michael Walshf41fac82017-08-02 15:05:24 -0500159 - Short parameters:
160 -<parm name><space><arg value>
161 - Long parameters:
162 --<parm name>=<arg value>
163
Michael Walsh9e042ad2019-10-16 17:14:31 -0500164 The first item in the string will be considered to be the command. All values not conforming to the
165 specifications above will be considered positional parms. If there are multiple parms with the same
166 name, they will be put into a list (see illustration below where "-v" is specified multiple times).
Michael Walshf41fac82017-08-02 15:05:24 -0500167
168 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500169 command_string The complete command string including all parameters and arguments.
Michael Walshf41fac82017-08-02 15:05:24 -0500170
171 Sample input:
172
Michael Walsh9e042ad2019-10-16 17:14:31 -0500173 robot_cmd_buf: robot -v OPENBMC_HOST:dummy1 -v keyword_string:'Set
174 Auto Reboot no' -v lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot -v quiet:0 -v
175 test_mode:0 -v debug:0 --outputdir='/home/user1/status/children/'
176 --output=dummy1.Auto_reboot.170802.124544.output.xml --log=dummy1.Auto_reboot.170802.124544.log.html
Michael Walshf41fac82017-08-02 15:05:24 -0500177 --report=dummy1.Auto_reboot.170802.124544.report.html
178 /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
179
180 Sample output:
181
182 robot_cmd_buf_dict:
183 robot_cmd_buf_dict[command]: robot
184 robot_cmd_buf_dict[v]:
185 robot_cmd_buf_dict[v][0]: OPENBMC_HOST:dummy1
Michael Walsh9e042ad2019-10-16 17:14:31 -0500186 robot_cmd_buf_dict[v][1]: keyword_string:Set Auto Reboot no
Michael Walshf41fac82017-08-02 15:05:24 -0500187 robot_cmd_buf_dict[v][2]:
188 lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot
189 robot_cmd_buf_dict[v][3]: quiet:0
190 robot_cmd_buf_dict[v][4]: test_mode:0
191 robot_cmd_buf_dict[v][5]: debug:0
Michael Walsh9e042ad2019-10-16 17:14:31 -0500192 robot_cmd_buf_dict[outputdir]: /home/user1/status/children/
193 robot_cmd_buf_dict[output]: dummy1.Auto_reboot.170802.124544.output.xml
194 robot_cmd_buf_dict[log]: dummy1.Auto_reboot.170802.124544.log.html
195 robot_cmd_buf_dict[report]: dummy1.Auto_reboot.170802.124544.report.html
Michael Walshf41fac82017-08-02 15:05:24 -0500196 robot_cmd_buf_dict[positional]:
197 /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
198 """
199
Michael Walsh9e042ad2019-10-16 17:14:31 -0500200 # We want the parms in the string broken down the way bash would do it, so we'll call upon bash to do
201 # that by creating a simple inline bash function.
Michael Walshf41fac82017-08-02 15:05:24 -0500202 bash_func_def = "function parse { for parm in \"${@}\" ; do" +\
203 " echo $parm ; done ; }"
204
205 rc, outbuf = cmd_fnc_u(bash_func_def + " ; parse " + command_string,
206 quiet=1, print_output=0)
207 command_string_list = outbuf.rstrip("\n").split("\n")
208
209 command_string_dict = collections.OrderedDict()
210 ix = 1
211 command_string_dict['command'] = command_string_list[0]
212 while ix < len(command_string_list):
213 if command_string_list[ix].startswith("--"):
214 key, value = command_string_list[ix].split("=")
215 key = key.lstrip("-")
216 elif command_string_list[ix].startswith("-"):
217 key = command_string_list[ix].lstrip("-")
218 ix += 1
219 try:
220 value = command_string_list[ix]
221 except IndexError:
222 value = ""
223 else:
224 key = 'positional'
225 value = command_string_list[ix]
226 if key in command_string_dict:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500227 if isinstance(command_string_dict[key], str):
Michael Walshf41fac82017-08-02 15:05:24 -0500228 command_string_dict[key] = [command_string_dict[key]]
229 command_string_dict[key].append(value)
230 else:
231 command_string_dict[key] = value
232 ix += 1
233
234 return command_string_dict
Michael Walsh21083d22018-06-01 14:19:32 -0500235
236
237# Save the original SIGALRM handler for later restoration by shell_cmd.
238original_sigalrm_handler = signal.getsignal(signal.SIGALRM)
239
240
241def shell_cmd_timed_out(signal_number,
242 frame):
243 r"""
244 Handle an alarm signal generated during the shell_cmd function.
245 """
246
247 gp.dprint_executing()
Michael Walsh3fb26182019-08-28 16:51:05 -0500248 global command_timed_out
249 command_timed_out = True
Michael Walsh21083d22018-06-01 14:19:32 -0500250 # Get subprocess pid from shell_cmd's call stack.
251 sub_proc = gp.get_stack_var('sub_proc', 0)
252 pid = sub_proc.pid
Michael Walsh3fb26182019-08-28 16:51:05 -0500253 gp.dprint_var(pid)
254 # Terminate the child process group.
255 os.killpg(pid, signal.SIGKILL)
Michael Walsh21083d22018-06-01 14:19:32 -0500256 # Restore the original SIGALRM handler.
257 signal.signal(signal.SIGALRM, original_sigalrm_handler)
258
259 return
260
261
262def shell_cmd(command_string,
263 quiet=None,
Michael Walsh55abd1b2018-08-31 13:00:50 -0500264 print_output=None,
Michael Walsh21083d22018-06-01 14:19:32 -0500265 show_err=1,
266 test_mode=0,
267 time_out=None,
268 max_attempts=1,
269 retry_sleep_time=5,
Michael Walsh3fb26182019-08-28 16:51:05 -0500270 valid_rcs=[0],
Michael Walsh21083d22018-06-01 14:19:32 -0500271 ignore_err=None,
Michael Walshfaafa9c2018-06-27 16:39:31 -0500272 return_stderr=0,
Michael Walsh9e042ad2019-10-16 17:14:31 -0500273 fork=0,
274 error_regexes=None):
Michael Walsh21083d22018-06-01 14:19:32 -0500275 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500276 Run the given command string in a shell and return a tuple consisting of the shell return code and the
277 output.
Michael Walsh21083d22018-06-01 14:19:32 -0500278
279 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500280 command_string The command string to be run in a shell (e.g. "ls /tmp").
281 quiet If set to 0, this function will print "Issuing: <cmd string>" to stdout.
282 When the quiet argument is set to None, this function will assign a
283 default value by searching upward in the stack for the quiet variable
284 value. If no such value is found, quiet is set to 0.
285 print_output If this is set, this function will print the stdout/stderr generated by
286 the shell command to stdout.
287 show_err If show_err is set, this function will print a standardized error report
288 if the shell command fails (i.e. if the shell command returns a shell_rc
289 that is not in valid_rcs). Note: Error text is only printed if ALL
290 attempts to run the command_string fail. In other words, if the command
291 execution is ultimately successful, initial failures are hidden.
292 test_mode If test_mode is set, this function will not actually run the command. If
293 print_output is also set, this function will print "(test_mode) Issuing:
294 <cmd string>" to stdout. A caller should call shell_cmd directly if they
295 wish to have the command string run unconditionally. They should call
296 the t_shell_cmd wrapper (defined below) if they wish to run the command
297 string only if the prevailing test_mode variable is set to 0.
298 time_out A time-out value expressed in seconds. If the command string has not
299 finished executing within <time_out> seconds, it will be halted and
300 counted as an error.
301 max_attempts The max number of attempts that should be made to run the command string.
302 retry_sleep_time The number of seconds to sleep between attempts.
303 valid_rcs A list of integers indicating which shell_rc values are not to be
304 considered errors.
305 ignore_err Ignore error means that a failure encountered by running the command
306 string will not be raised as a python exception. When the ignore_err
307 argument is set to None, this function will assign a default value by
308 searching upward in the stack for the ignore_err variable value. If no
309 such value is found, ignore_err is set to 1.
310 return_stderr If return_stderr is set, this function will process the stdout and stderr
311 streams from the shell command separately. In such a case, the tuple
312 returned by this function will consist of three values rather than just
313 two: rc, stdout, stderr.
314 fork Run the command string asynchronously (i.e. don't wait for status of the
315 child process and don't try to get stdout/stderr) and return the Popen
316 object created by the subprocess.popen() function. See the kill_cmd
317 function for details on how to process the popen object.
318 error_regexes A list of regular expressions to be used to identify errors in the
319 command output. If there is a match for any of these regular
320 expressions, the command will be considered a failure and the shell_rc
321 will be set to -1. For example, if error_regexes = ['ERROR:'] and the
322 command output contains 'ERROR: Unrecognized option', it will be counted
323 as an error even if the command returned 0. This is useful when running
324 commands that do not always return non-zero on error.
Michael Walsh21083d22018-06-01 14:19:32 -0500325 """
326
Michael Walsh3fb26182019-08-28 16:51:05 -0500327 err_msg = gv.valid_value(command_string)
328 if err_msg:
329 raise ValueError(err_msg)
330
Michael Walsh21083d22018-06-01 14:19:32 -0500331 # Assign default values to some of the arguments to this function.
Michael Walsh4cb9b2a2018-06-08 17:57:00 -0500332 quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0)))
Michael Walsh55abd1b2018-08-31 13:00:50 -0500333 print_output = int(gm.dft(print_output, not quiet))
Michael Walsh45fead42018-09-26 17:20:48 -0500334 show_err = int(show_err)
Michael Walsh3fb26182019-08-28 16:51:05 -0500335 ignore_err = int(gm.dft(ignore_err, gp.get_stack_var('ignore_err', 1)))
Michael Walsh21083d22018-06-01 14:19:32 -0500336
Michael Walsh3fb26182019-08-28 16:51:05 -0500337 gp.qprint_issuing(command_string, test_mode)
Michael Walsh21083d22018-06-01 14:19:32 -0500338 if test_mode:
Michael Walsh3fb26182019-08-28 16:51:05 -0500339 return (0, "", "") if return_stderr else (0, "")
Michael Walsh21083d22018-06-01 14:19:32 -0500340
Michael Walsh3fb26182019-08-28 16:51:05 -0500341 # Convert a string python dictionary definition to a dictionary.
342 valid_rcs = fa.source_to_object(valid_rcs)
Michael Walsh21083d22018-06-01 14:19:32 -0500343 # Convert each list entry to a signed value.
Michael Walsh3fb26182019-08-28 16:51:05 -0500344 valid_rcs = [gm.to_signed(x) for x in valid_rcs]
Michael Walsh21083d22018-06-01 14:19:32 -0500345
Michael Walsh3fb26182019-08-28 16:51:05 -0500346 stderr = subprocess.PIPE if return_stderr else subprocess.STDOUT
Michael Walsh21083d22018-06-01 14:19:32 -0500347
Michael Walsh9e042ad2019-10-16 17:14:31 -0500348 # Write all output to func_out_history_buf rather than directly to stdout. This allows us to decide
349 # what to print after all attempts to run the command string have been made. func_out_history_buf will
350 # contain the complete history from the current invocation of this function.
Michael Walsh3fb26182019-08-28 16:51:05 -0500351 global command_timed_out
352 command_timed_out = False
353 func_out_history_buf = ""
Michael Walsh21083d22018-06-01 14:19:32 -0500354 for attempt_num in range(1, max_attempts + 1):
355 sub_proc = subprocess.Popen(command_string,
Michael Walsh3fb26182019-08-28 16:51:05 -0500356 preexec_fn=os.setsid,
Michael Walsh21083d22018-06-01 14:19:32 -0500357 bufsize=1,
358 shell=True,
Michael Walsh3fb26182019-08-28 16:51:05 -0500359 universal_newlines=True,
Michael Walsh21083d22018-06-01 14:19:32 -0500360 executable='/bin/bash',
361 stdout=subprocess.PIPE,
362 stderr=stderr)
Michael Walshfaafa9c2018-06-27 16:39:31 -0500363 if fork:
Michael Walsh4d5f6682019-09-24 14:23:40 -0500364 return sub_proc
Michael Walsh3fb26182019-08-28 16:51:05 -0500365
366 if time_out:
367 command_timed_out = False
Michael Walsh21083d22018-06-01 14:19:32 -0500368 # Designate a SIGALRM handling function and set alarm.
369 signal.signal(signal.SIGALRM, shell_cmd_timed_out)
370 signal.alarm(time_out)
371 try:
Michael Walsh3fb26182019-08-28 16:51:05 -0500372 stdout_buf, stderr_buf = sub_proc.communicate()
Michael Walsh21083d22018-06-01 14:19:32 -0500373 except IOError:
374 command_timed_out = True
Michael Walsh21083d22018-06-01 14:19:32 -0500375 # Restore the original SIGALRM handler and clear the alarm.
376 signal.signal(signal.SIGALRM, original_sigalrm_handler)
377 signal.alarm(0)
Michael Walsh3fb26182019-08-28 16:51:05 -0500378
Michael Walsh9e042ad2019-10-16 17:14:31 -0500379 # Output from this loop iteration is written to func_out_buf for later processing. This can include
380 # stdout, stderr and our own error messages.
Michael Walsh3fb26182019-08-28 16:51:05 -0500381 func_out_buf = ""
382 if print_output:
383 if return_stderr:
384 func_out_buf += stderr_buf
385 func_out_buf += stdout_buf
386 shell_rc = sub_proc.returncode
387 if shell_rc in valid_rcs:
Michael Walsh9e042ad2019-10-16 17:14:31 -0500388 # Check output for text indicating there is an error.
389 if error_regexes and re.match('|'.join(error_regexes), stdout_buf):
390 shell_rc = -1
391 else:
392 break
Michael Walsh21083d22018-06-01 14:19:32 -0500393 err_msg = "The prior shell command failed.\n"
Michael Walsh21083d22018-06-01 14:19:32 -0500394 err_msg += gp.sprint_var(attempt_num)
Michael Walsh3fb26182019-08-28 16:51:05 -0500395 err_msg += gp.sprint_vars(command_string, command_timed_out, time_out)
396 err_msg += gp.sprint_varx("child_pid", sub_proc.pid)
397 err_msg += gp.sprint_vars(shell_rc, valid_rcs, fmt=gp.hexa())
Michael Walsh9e042ad2019-10-16 17:14:31 -0500398 if error_regexes:
399 err_msg += gp.sprint_vars(error_regexes)
Michael Walsh21083d22018-06-01 14:19:32 -0500400 if not print_output:
401 if return_stderr:
Michael Walsh3fb26182019-08-28 16:51:05 -0500402 err_msg += "stderr_buf:\n" + stderr_buf
403 err_msg += "stdout_buf:\n" + stdout_buf
Michael Walsh21083d22018-06-01 14:19:32 -0500404 if show_err:
Michael Walsh3fb26182019-08-28 16:51:05 -0500405 func_out_buf += gp.sprint_error_report(err_msg)
Michael Walsh21083d22018-06-01 14:19:32 -0500406 if attempt_num < max_attempts:
Michael Walsh3fb26182019-08-28 16:51:05 -0500407 cmd_buf = "time.sleep(" + str(retry_sleep_time) + ")"
408 if show_err:
409 func_out_buf += gp.sprint_issuing(cmd_buf)
410 exec(cmd_buf)
411 func_out_history_buf += func_out_buf
Michael Walsh21083d22018-06-01 14:19:32 -0500412
Michael Walsh3fb26182019-08-28 16:51:05 -0500413 if shell_rc in valid_rcs:
414 gp.gp_print(func_out_buf)
415 else:
416 if show_err:
417 gp.gp_print(func_out_history_buf, stream='stderr')
418 else:
Michael Walsh9e042ad2019-10-16 17:14:31 -0500419 # There is no error information to show so just print output from last loop iteration.
Michael Walsh3fb26182019-08-28 16:51:05 -0500420 gp.gp_print(func_out_buf)
Michael Walsh21083d22018-06-01 14:19:32 -0500421 if not ignore_err:
Michael Walsh9e042ad2019-10-16 17:14:31 -0500422 # If the caller has already asked to show error info, avoid repeating that in the failure message.
Michael Walsh3fb26182019-08-28 16:51:05 -0500423 err_msg = "The prior shell command failed.\n" if show_err \
424 else err_msg
Michael Walsh21083d22018-06-01 14:19:32 -0500425 if robot_env:
426 BuiltIn().fail(err_msg)
427 else:
Michael Walsh3fb26182019-08-28 16:51:05 -0500428 raise ValueError(err_msg)
Michael Walsh21083d22018-06-01 14:19:32 -0500429
Michael Walsh3fb26182019-08-28 16:51:05 -0500430 return (shell_rc, stdout_buf, stderr_buf) if return_stderr \
431 else (shell_rc, stdout_buf)
Michael Walsh21083d22018-06-01 14:19:32 -0500432
433
434def t_shell_cmd(command_string, **kwargs):
435 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500436 Search upward in the the call stack to obtain the test_mode argument, add it to kwargs and then call
437 shell_cmd and return the result.
Michael Walsh21083d22018-06-01 14:19:32 -0500438
439 See shell_cmd prolog for details on all arguments.
440 """
441
442 if 'test_mode' in kwargs:
443 error_message = "Programmer error - test_mode is not a valid" +\
444 " argument to this function."
445 gp.print_error_report(error_message)
446 exit(1)
447
Michael Walsh3fb26182019-08-28 16:51:05 -0500448 test_mode = int(gp.get_stack_var('test_mode', 0))
Michael Walsh21083d22018-06-01 14:19:32 -0500449 kwargs['test_mode'] = test_mode
450
451 return shell_cmd(command_string, **kwargs)
George Keishing16244c22019-01-31 16:16:14 +0000452
453
Michael Walsh4d5f6682019-09-24 14:23:40 -0500454def kill_cmd(popen, sig=signal.SIGTERM):
455 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500456 Kill the subprocess represented by the Popen object and return a tuple consisting of the shell return
457 code and the output.
Michael Walsh4d5f6682019-09-24 14:23:40 -0500458
Michael Walsh9e042ad2019-10-16 17:14:31 -0500459 This function is meant to be used as the follow-up for a call to shell_cmd(..., fork=1).
Michael Walsh4d5f6682019-09-24 14:23:40 -0500460
461 Example:
462 popen = shell_cmd("some_pgm.py", fork=1)
463 ...
464 shell_rc, output = kill_cmd(popen)
465
466 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500467 popen A Popen object returned by the subprocess.Popen() command.
Michael Walsh4d5f6682019-09-24 14:23:40 -0500468 sig The signal to be sent to the child process.
469 """
470
471 gp.dprint_var(popen.pid)
472 os.killpg(popen.pid, sig)
473 stdout, stderr = popen.communicate()
474 shell_rc = popen.returncode
475 return (shell_rc, stdout, stderr) if stderr else (shell_rc, stdout)
476
477
George Keishing16244c22019-01-31 16:16:14 +0000478def re_order_kwargs(stack_frame_ix, **kwargs):
479 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500480 Re-order the kwargs to match the order in which they were specified on a function invocation and return
481 as an ordered dictionary.
George Keishing16244c22019-01-31 16:16:14 +0000482
Michael Walsh9e042ad2019-10-16 17:14:31 -0500483 Note that this re_order_kwargs function should not be necessary in python versions 3.6 and beyond.
George Keishing16244c22019-01-31 16:16:14 +0000484
485 Example:
486
487 The caller calls func1 like this:
488
489 func1('mike', arg1='one', arg2='two', arg3='three')
490
491 And func1 is defined as follows:
492
493 def func1(first_arg, **kwargs):
494
495 kwargs = re_order_kwargs(first_arg_num=2, stack_frame_ix=3, **kwargs)
496
Michael Walsh9e042ad2019-10-16 17:14:31 -0500497 The kwargs dictionary before calling re_order_kwargs (where order is not guaranteed):
George Keishing16244c22019-01-31 16:16:14 +0000498
499 kwargs:
500 kwargs[arg3]: three
501 kwargs[arg2]: two
502 kwargs[arg1]: one
503
504 The kwargs dictionary after calling re_order_kwargs:
505
506 kwargs:
507 kwargs[arg1]: one
508 kwargs[arg2]: two
509 kwargs[arg3]: three
510
Michael Walsh9e042ad2019-10-16 17:14:31 -0500511 Note that the re-ordered kwargs match the order specified on the call to func1.
George Keishing16244c22019-01-31 16:16:14 +0000512
513 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500514 stack_frame_ix The stack frame of the function whose kwargs values must be re-ordered.
515 0 is the stack frame of re_order_kwargs, 1 is the stack from of its
516 caller and so on.
517 kwargs The keyword argument dictionary which is to be re-ordered.
George Keishing16244c22019-01-31 16:16:14 +0000518 """
519
520 new_kwargs = collections.OrderedDict()
521
522 # Get position number of first keyword on the calling line of code.
523 (args, varargs, keywords, locals) =\
524 inspect.getargvalues(inspect.stack()[stack_frame_ix][0])
525 first_kwarg_pos = 1 + len(args)
526 if varargs is not None:
527 first_kwarg_pos += len(locals[varargs])
528 for arg_num in range(first_kwarg_pos, first_kwarg_pos + len(kwargs)):
529 # This will result in an arg_name value such as "arg1='one'".
530 arg_name = gp.get_arg_name(None, arg_num, stack_frame_ix + 2)
531 # Continuing with the prior example, the following line will result
532 # in key being set to 'arg1'.
533 key = arg_name.split('=')[0]
534 new_kwargs[key] = kwargs[key]
535
536 return new_kwargs
537
538
539def default_arg_delim(arg_dashes):
540 r"""
541 Return the default argument delimiter value for the given arg_dashes value.
542
Michael Walsh9e042ad2019-10-16 17:14:31 -0500543 Note: this function is useful for functions that manipulate bash command line arguments (e.g. --parm=1 or
544 -parm 1).
George Keishing16244c22019-01-31 16:16:14 +0000545
546 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500547 arg_dashes The argument dashes specifier (usually, "-" or "--").
George Keishing16244c22019-01-31 16:16:14 +0000548 """
549
550 if arg_dashes == "--":
551 return "="
552
553 return " "
554
555
556def create_command_string(command, *pos_parms, **options):
557 r"""
Michael Walsh9e042ad2019-10-16 17:14:31 -0500558 Create and return a bash command string consisting of the given arguments formatted as text.
George Keishing16244c22019-01-31 16:16:14 +0000559
560 The default formatting of options is as follows:
561
562 <single dash><option name><space delim><option value>
563
564 Example:
565
566 -parm value
567
Michael Walsh9e042ad2019-10-16 17:14:31 -0500568 The caller can change the kind of dashes/delimiters used by specifying "arg_dashes" and/or "arg_delims"
569 as options. These options are processed specially by the create_command_string function and do NOT get
570 inserted into the resulting command string. All options following the arg_dashes/arg_delims options will
571 then use the specified values for dashes/delims. In the special case of arg_dashes equal to "--", the
George Keishing16244c22019-01-31 16:16:14 +0000572 arg_delim will automatically be changed to "=". See examples below.
573
574 Quoting rules:
575
Michael Walsh9e042ad2019-10-16 17:14:31 -0500576 The create_command_string function will single quote option values as needed to prevent bash expansion.
577 If the caller wishes to defeat this action, they may single or double quote the option value themselves.
578 See examples below.
George Keishing16244c22019-01-31 16:16:14 +0000579
Michael Walsh9e042ad2019-10-16 17:14:31 -0500580 pos_parms are NOT automatically quoted. The caller is advised to either explicitly add quotes or to use
581 the quote_bash_parm functions to quote any pos_parms.
George Keishing16244c22019-01-31 16:16:14 +0000582
583 Examples:
584
585 command_string = create_command_string('cd', '~')
586
587 Result:
588 cd ~
589
Michael Walsh9e042ad2019-10-16 17:14:31 -0500590 Note that the pos_parm ("~") does NOT get quoted, as per the aforementioned rules. If quotes are
591 desired, they may be added explicitly by the caller:
George Keishing16244c22019-01-31 16:16:14 +0000592
593 command_string = create_command_string('cd', '\'~\'')
594
595 Result:
596 cd '~'
597
598 command_string = create_command_string('grep', '\'^[^ ]*=\'',
599 '/tmp/myfile', i=None, m='1', arg_dashes='--', color='always')
600
601 Result:
602 grep -i -m 1 --color=always '^[^ ]*=' /tmp/myfile
603
Michael Walsh9e042ad2019-10-16 17:14:31 -0500604 In the preceding example, note the use of None to cause the "i" parm to be treated as a flag (i.e. no
605 argument value is generated). Also, note the use of arg_dashes to change the type of dashes used on all
606 subsequent options. The following example is equivalent to the prior. Note that quote_bash_parm is used
607 instead of including the quotes explicitly.
George Keishing16244c22019-01-31 16:16:14 +0000608
609 command_string = create_command_string('grep', quote_bash_parm('^[^ ]*='),
610 '/tmp/myfile', i=None, m='1', arg_dashes='--', color='always')
611
612 Result:
613 grep -i -m 1 --color=always '^[^ ]*=' /tmp/myfile
614
Michael Walsh9e042ad2019-10-16 17:14:31 -0500615 In the following example, note the automatic quoting of the password option, as per the aforementioned
616 rules.
George Keishing16244c22019-01-31 16:16:14 +0000617
618 command_string = create_command_string('my_pgm', '/tmp/myfile', i=None,
619 m='1', arg_dashes='--', password='${my_pw}')
620
Michael Walsh9e042ad2019-10-16 17:14:31 -0500621 However, let's say that the caller wishes to have bash expand the password value. To achieve this, the
622 caller can use double quotes:
George Keishing16244c22019-01-31 16:16:14 +0000623
624 command_string = create_command_string('my_pgm', '/tmp/myfile', i=None,
625 m='1', arg_dashes='--', password='"${my_pw}"')
626
627 Result:
628 my_pgm -i -m 1 --password="${my_pw}" /tmp/myfile
629
630 command_string = create_command_string('ipmitool', 'power status',
George Keishinge33ad1d2019-12-09 11:17:36 -0600631 I='lanplus', C='3', 'p=623', U='root', P='0penBmc', H='xx.xx.xx.xx')
George Keishing16244c22019-01-31 16:16:14 +0000632
633 Result:
George Keishinge33ad1d2019-12-09 11:17:36 -0600634 ipmitool -I lanplus -C 3 -p 623 -U root -P 0penBmc -H xx.xx.xx.xx power status
George Keishing16244c22019-01-31 16:16:14 +0000635
Michael Walsh9e042ad2019-10-16 17:14:31 -0500636 By default create_command_string will take measures to preserve the order of the callers options. In
637 some cases, this effort may fail (as when calling directly from a robot program). In this case, the
638 caller can accept the responsibility of keeping an ordered list of options by calling this function with
639 the last positional parm as some kind of dictionary (preferably an OrderedDict) and avoiding the use of
640 any actual option args.
George Keishing16244c22019-01-31 16:16:14 +0000641
642 Example:
643 kwargs = collections.OrderedDict([('pass', 0), ('fail', 0)])
644 command_string = create_command_string('my program', 'pos_parm1', kwargs)
645
646 Result:
647
648 my program -pass 0 -fail 0 pos_parm1
649
Michael Walsh9e042ad2019-10-16 17:14:31 -0500650 Note to programmers who wish to write a wrapper to this function: If the python version is less than
651 3.6, to get the options to be processed correctly, the wrapper function must include a _stack_frame_ix_
652 keyword argument to allow this function to properly re-order options:
George Keishing16244c22019-01-31 16:16:14 +0000653
654 def create_ipmi_ext_command_string(command, **kwargs):
655
656 return create_command_string('ipmitool', command, _stack_frame_ix_=2,
657 **kwargs)
658
659 Example call of wrapper function:
660
Michael Walsh9e042ad2019-10-16 17:14:31 -0500661 command_string = create_ipmi_ext_command_string('power status', I='lanplus')
George Keishing16244c22019-01-31 16:16:14 +0000662
663 Description of argument(s):
Michael Walsh9e042ad2019-10-16 17:14:31 -0500664 command The command (e.g. "cat", "sort", "ipmitool", etc.).
665 pos_parms The positional parms for the command (e.g. PATTERN, FILENAME, etc.).
666 These will be placed at the end of the resulting command string.
667 options The command options (e.g. "-m 1", "--max-count=NUM", etc.). Note that if
668 the value of any option is None, then it will be understood to be a flag
669 (for which no value is required).
George Keishing16244c22019-01-31 16:16:14 +0000670 """
671
672 arg_dashes = "-"
673 delim = default_arg_delim(arg_dashes)
674
675 command_string = command
676
Michael Walsh23ac9b42019-02-20 16:46:04 -0600677 if len(pos_parms) > 0 and gp.is_dict(pos_parms[-1]):
George Keishing16244c22019-01-31 16:16:14 +0000678 # Convert pos_parms from tuple to list.
679 pos_parms = list(pos_parms)
Michael Walsh9e042ad2019-10-16 17:14:31 -0500680 # Re-assign options to be the last pos_parm value (which is a dictionary).
George Keishing16244c22019-01-31 16:16:14 +0000681 options = pos_parms[-1]
682 # Now delete the last pos_parm.
683 del pos_parms[-1]
684 else:
Michael Walsh9e042ad2019-10-16 17:14:31 -0500685 # Either get stack_frame_ix from the caller via options or set it to the default value.
Michael Walshfffddca2019-08-20 17:19:13 +0000686 stack_frame_ix = options.pop('_stack_frame_ix_', 1)
687 if gm.python_version < gm.ordered_dict_version:
Michael Walsh9e042ad2019-10-16 17:14:31 -0500688 # Re-establish the original options order as specified on the original line of code. This
689 # function depends on correct order.
Michael Walshfffddca2019-08-20 17:19:13 +0000690 options = re_order_kwargs(stack_frame_ix, **options)
George Keishing16244c22019-01-31 16:16:14 +0000691 for key, value in options.items():
692 # Check for special values in options and process them.
693 if key == "arg_dashes":
694 arg_dashes = str(value)
695 delim = default_arg_delim(arg_dashes)
696 continue
697 if key == "arg_delim":
698 delim = str(value)
699 continue
700 # Format the options elements into the command string.
701 command_string += " " + arg_dashes + key
702 if value is not None:
703 command_string += delim
704 if re.match(r'^(["].*["]|[\'].*[\'])$', str(value)):
705 # Already quoted.
706 command_string += str(value)
707 else:
708 command_string += gm.quote_bash_parm(str(value))
Michael Walsh9e042ad2019-10-16 17:14:31 -0500709 # Finally, append the pos_parms to the end of the command_string. Use filter to eliminate blank pos
710 # parms.
Michael Walsh23ac9b42019-02-20 16:46:04 -0600711 command_string = ' '.join([command_string] + list(filter(None, pos_parms)))
George Keishing16244c22019-01-31 16:16:14 +0000712
713 return command_string