blob: 5e004c142fcf2bec76a7bb95b812b0f1b489fd04 [file] [log] [blame]
Michael Walshc3b512e2017-02-20 15:59:01 -06001#!/usr/bin/env python
2
3r"""
4This module provides command execution functions such as cmd_fnc and cmd_fnc_u.
5"""
6
Michael Walsh21083d22018-06-01 14:19:32 -05007import os
Michael Walshc3b512e2017-02-20 15:59:01 -06008import sys
9import subprocess
Michael Walshf41fac82017-08-02 15:05:24 -050010import collections
Michael Walsh21083d22018-06-01 14:19:32 -050011import signal
12import time
Michael Walshc3b512e2017-02-20 15:59:01 -060013
Michael Walshc3b512e2017-02-20 15:59:01 -060014import gen_print as gp
15import gen_valid as gv
16import gen_misc as gm
Michael Walshd6901502017-11-14 12:58:37 -060017
18robot_env = gp.robot_env
19
Michael Walshafc53a22017-04-12 15:52:28 -050020if robot_env:
21 import gen_robot_print as grp
Michael Walsha3e2f532018-01-10 13:43:42 -060022 from robot.libraries.BuiltIn import BuiltIn
Michael Walshc3b512e2017-02-20 15:59:01 -060023
24
Michael Walsh21083d22018-06-01 14:19:32 -050025# cmd_fnc and cmd_fnc_u should now be considered deprecated. shell_cmd and
26# t_shell_cmd should be used instead.
Michael Walshc3b512e2017-02-20 15:59:01 -060027def cmd_fnc(cmd_buf,
28 quiet=None,
29 test_mode=None,
Michael Walshafc53a22017-04-12 15:52:28 -050030 debug=0,
Michael Walshc3b512e2017-02-20 15:59:01 -060031 print_output=1,
Michael Walshcfe9fed2017-09-12 17:13:10 -050032 show_err=1,
Michael Walsha3e2f532018-01-10 13:43:42 -060033 return_stderr=0,
34 ignore_err=1):
Michael Walshc3b512e2017-02-20 15:59:01 -060035 r"""
Michael Walshcfe9fed2017-09-12 17:13:10 -050036 Run the given command in a shell and return the shell return code and the
37 output.
Michael Walshc3b512e2017-02-20 15:59:01 -060038
39 Description of arguments:
40 cmd_buf The command string to be run in a shell.
41 quiet Indicates whether this function should run
Michael Walshcfe9fed2017-09-12 17:13:10 -050042 the print_issuing() function which prints
43 "Issuing: <cmd string>" to stdout.
Michael Walshc3b512e2017-02-20 15:59:01 -060044 test_mode If test_mode is set, this function will
Michael Walshcfe9fed2017-09-12 17:13:10 -050045 not actually run the command. If
46 print_output is set, it will print
47 "(test_mode) Issuing: <cmd string>" to
48 stdout.
Michael Walshc3b512e2017-02-20 15:59:01 -060049 debug If debug is set, this function will print
50 extra debug info.
51 print_output If this is set, this function will print
Michael Walshcfe9fed2017-09-12 17:13:10 -050052 the stdout/stderr generated by the shell
53 command.
Michael Walshc3b512e2017-02-20 15:59:01 -060054 show_err If show_err is set, this function will
Michael Walshcfe9fed2017-09-12 17:13:10 -050055 print a standardized error report if the
56 shell command returns non-zero.
57 return_stderr If return_stderr is set, this function
58 will process the stdout and stderr streams
59 from the shell command separately. It
60 will also return stderr in addition to the
61 return code and the stdout.
Michael Walshc3b512e2017-02-20 15:59:01 -060062 """
63
Michael Walshcfe9fed2017-09-12 17:13:10 -050064 # Determine default values.
Michael Walshc3b512e2017-02-20 15:59:01 -060065 quiet = int(gm.global_default(quiet, 0))
66 test_mode = int(gm.global_default(test_mode, 0))
Michael Walshc3b512e2017-02-20 15:59:01 -060067
68 if debug:
Michael Walshafc53a22017-04-12 15:52:28 -050069 gp.print_vars(cmd_buf, quiet, test_mode, debug)
Michael Walshc3b512e2017-02-20 15:59:01 -060070
71 err_msg = gv.svalid_value(cmd_buf)
72 if err_msg != "":
73 raise ValueError(err_msg)
74
75 if not quiet:
Michael Walshafc53a22017-04-12 15:52:28 -050076 gp.pissuing(cmd_buf, test_mode)
Michael Walshc3b512e2017-02-20 15:59:01 -060077
78 if test_mode:
Michael Walshcfe9fed2017-09-12 17:13:10 -050079 if return_stderr:
80 return 0, "", ""
81 else:
82 return 0, ""
83
84 if return_stderr:
85 err_buf = ""
86 stderr = subprocess.PIPE
87 else:
88 stderr = subprocess.STDOUT
Michael Walshc3b512e2017-02-20 15:59:01 -060089
90 sub_proc = subprocess.Popen(cmd_buf,
91 bufsize=1,
92 shell=True,
Michael Walsh3ba8ecd2018-04-24 11:33:25 -050093 executable='/bin/bash',
Michael Walshc3b512e2017-02-20 15:59:01 -060094 stdout=subprocess.PIPE,
Michael Walshcfe9fed2017-09-12 17:13:10 -050095 stderr=stderr)
Michael Walshc3b512e2017-02-20 15:59:01 -060096 out_buf = ""
Michael Walshcfe9fed2017-09-12 17:13:10 -050097 if return_stderr:
98 for line in sub_proc.stderr:
99 err_buf += line
100 if not print_output:
101 continue
102 if robot_env:
103 grp.rprint(line)
104 else:
105 sys.stdout.write(line)
Michael Walshc3b512e2017-02-20 15:59:01 -0600106 for line in sub_proc.stdout:
107 out_buf += line
108 if not print_output:
109 continue
110 if robot_env:
111 grp.rprint(line)
112 else:
113 sys.stdout.write(line)
114 if print_output and not robot_env:
115 sys.stdout.flush()
116 sub_proc.communicate()
117 shell_rc = sub_proc.returncode
Michael Walsha3e2f532018-01-10 13:43:42 -0600118 if shell_rc != 0:
119 err_msg = "The prior shell command failed.\n"
120 err_msg += gp.sprint_var(shell_rc, 1)
Michael Walshcfe9fed2017-09-12 17:13:10 -0500121 if not print_output:
122 err_msg += "out_buf:\n" + out_buf
Michael Walshc3b512e2017-02-20 15:59:01 -0600123
Michael Walsha3e2f532018-01-10 13:43:42 -0600124 if show_err:
125 if robot_env:
126 grp.rprint_error_report(err_msg)
127 else:
128 gp.print_error_report(err_msg)
129 if not ignore_err:
130 if robot_env:
131 BuiltIn().fail(err_msg)
132 else:
133 raise ValueError(err_msg)
Michael Walshcfe9fed2017-09-12 17:13:10 -0500134
135 if return_stderr:
136 return shell_rc, out_buf, err_buf
137 else:
138 return shell_rc, out_buf
Michael Walshc3b512e2017-02-20 15:59:01 -0600139
Michael Walshc3b512e2017-02-20 15:59:01 -0600140
Michael Walshc3b512e2017-02-20 15:59:01 -0600141def cmd_fnc_u(cmd_buf,
142 quiet=None,
143 debug=None,
144 print_output=1,
Michael Walshcfe9fed2017-09-12 17:13:10 -0500145 show_err=1,
Michael Walsha3e2f532018-01-10 13:43:42 -0600146 return_stderr=0,
147 ignore_err=1):
Michael Walshc3b512e2017-02-20 15:59:01 -0600148 r"""
149 Call cmd_fnc with test_mode=0. See cmd_fnc (above) for details.
150
151 Note the "u" in "cmd_fnc_u" stands for "unconditional".
152 """
153
154 return cmd_fnc(cmd_buf, test_mode=0, quiet=quiet, debug=debug,
Michael Walshcfe9fed2017-09-12 17:13:10 -0500155 print_output=print_output, show_err=show_err,
Michael Walsha3e2f532018-01-10 13:43:42 -0600156 return_stderr=return_stderr, ignore_err=ignore_err)
Michael Walshc3b512e2017-02-20 15:59:01 -0600157
Michael Walshf41fac82017-08-02 15:05:24 -0500158
Michael Walshf41fac82017-08-02 15:05:24 -0500159def parse_command_string(command_string):
Michael Walshf41fac82017-08-02 15:05:24 -0500160 r"""
161 Parse a bash command-line command string and return the result as a
162 dictionary of parms.
163
164 This can be useful for answering questions like "What did the user specify
165 as the value for parm x in the command string?".
166
167 This function expects the command string to follow the following posix
168 conventions:
169 - Short parameters:
170 -<parm name><space><arg value>
171 - Long parameters:
172 --<parm name>=<arg value>
173
174 The first item in the string will be considered to be the command. All
175 values not conforming to the specifications above will be considered
176 positional parms. If there are multiple parms with the same name, they
177 will be put into a list (see illustration below where "-v" is specified
178 multiple times).
179
180 Description of argument(s):
181 command_string The complete command string including all
182 parameters and arguments.
183
184 Sample input:
185
186 robot_cmd_buf: robot -v
187 OPENBMC_HOST:dummy1 -v keyword_string:'Set Auto Reboot no' -v
188 lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot -v
189 quiet:0 -v test_mode:0 -v debug:0
190 --outputdir='/home/user1/status/children/'
191 --output=dummy1.Auto_reboot.170802.124544.output.xml
192 --log=dummy1.Auto_reboot.170802.124544.log.html
193 --report=dummy1.Auto_reboot.170802.124544.report.html
194 /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
195
196 Sample output:
197
198 robot_cmd_buf_dict:
199 robot_cmd_buf_dict[command]: robot
200 robot_cmd_buf_dict[v]:
201 robot_cmd_buf_dict[v][0]: OPENBMC_HOST:dummy1
202 robot_cmd_buf_dict[v][1]: keyword_string:Set Auto
203 Reboot no
204 robot_cmd_buf_dict[v][2]:
205 lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot
206 robot_cmd_buf_dict[v][3]: quiet:0
207 robot_cmd_buf_dict[v][4]: test_mode:0
208 robot_cmd_buf_dict[v][5]: debug:0
209 robot_cmd_buf_dict[outputdir]:
210 /home/user1/status/children/
211 robot_cmd_buf_dict[output]:
212 dummy1.Auto_reboot.170802.124544.output.xml
213 robot_cmd_buf_dict[log]:
214 dummy1.Auto_reboot.170802.124544.log.html
215 robot_cmd_buf_dict[report]:
216 dummy1.Auto_reboot.170802.124544.report.html
217 robot_cmd_buf_dict[positional]:
218 /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
219 """
220
221 # We want the parms in the string broken down the way bash would do it,
222 # so we'll call upon bash to do that by creating a simple inline bash
223 # function.
224 bash_func_def = "function parse { for parm in \"${@}\" ; do" +\
225 " echo $parm ; done ; }"
226
227 rc, outbuf = cmd_fnc_u(bash_func_def + " ; parse " + command_string,
228 quiet=1, print_output=0)
229 command_string_list = outbuf.rstrip("\n").split("\n")
230
231 command_string_dict = collections.OrderedDict()
232 ix = 1
233 command_string_dict['command'] = command_string_list[0]
234 while ix < len(command_string_list):
235 if command_string_list[ix].startswith("--"):
236 key, value = command_string_list[ix].split("=")
237 key = key.lstrip("-")
238 elif command_string_list[ix].startswith("-"):
239 key = command_string_list[ix].lstrip("-")
240 ix += 1
241 try:
242 value = command_string_list[ix]
243 except IndexError:
244 value = ""
245 else:
246 key = 'positional'
247 value = command_string_list[ix]
248 if key in command_string_dict:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500249 if isinstance(command_string_dict[key], str):
Michael Walshf41fac82017-08-02 15:05:24 -0500250 command_string_dict[key] = [command_string_dict[key]]
251 command_string_dict[key].append(value)
252 else:
253 command_string_dict[key] = value
254 ix += 1
255
256 return command_string_dict
Michael Walsh21083d22018-06-01 14:19:32 -0500257
258
259# Save the original SIGALRM handler for later restoration by shell_cmd.
260original_sigalrm_handler = signal.getsignal(signal.SIGALRM)
261
262
263def shell_cmd_timed_out(signal_number,
264 frame):
265 r"""
266 Handle an alarm signal generated during the shell_cmd function.
267 """
268
269 gp.dprint_executing()
270 # Get subprocess pid from shell_cmd's call stack.
271 sub_proc = gp.get_stack_var('sub_proc', 0)
272 pid = sub_proc.pid
273 # Terminate the child process.
274 os.kill(pid, signal.SIGTERM)
275 # Restore the original SIGALRM handler.
276 signal.signal(signal.SIGALRM, original_sigalrm_handler)
277
278 return
279
280
281def shell_cmd(command_string,
282 quiet=None,
Michael Walsh55abd1b2018-08-31 13:00:50 -0500283 print_output=None,
Michael Walsh21083d22018-06-01 14:19:32 -0500284 show_err=1,
285 test_mode=0,
286 time_out=None,
287 max_attempts=1,
288 retry_sleep_time=5,
289 allowed_shell_rcs=[0],
290 ignore_err=None,
Michael Walshfaafa9c2018-06-27 16:39:31 -0500291 return_stderr=0,
292 fork=0):
Michael Walsh21083d22018-06-01 14:19:32 -0500293 r"""
294 Run the given command string in a shell and return a tuple consisting of
295 the shell return code and the output.
296
297 Description of argument(s):
298 command_string The command string to be run in a shell
299 (e.g. "ls /tmp").
300 quiet If set to 0, this function will print
Michael Walsh4cb9b2a2018-06-08 17:57:00 -0500301 "Issuing: <cmd string>" to stdout. When
302 the quiet argument is set to None, this
303 function will assign a default value by
304 searching upward in the stack for the
305 quiet variable value. If no such value is
306 found, quiet is set to 0.
Michael Walsh21083d22018-06-01 14:19:32 -0500307 print_output If this is set, this function will print
308 the stdout/stderr generated by the shell
309 command to stdout.
310 show_err If show_err is set, this function will
311 print a standardized error report if the
312 shell command fails (i.e. if the shell
313 command returns a shell_rc that is not in
314 allowed_shell_rcs). Note: Error text is
315 only printed if ALL attempts to run the
316 command_string fail. In other words, if
317 the command execution is ultimately
318 successful, initial failures are hidden.
319 test_mode If test_mode is set, this function will
320 not actually run the command. If
321 print_output is also set, this function
322 will print "(test_mode) Issuing: <cmd
323 string>" to stdout. A caller should call
324 shell_cmd directly if they wish to have
325 the command string run unconditionally.
326 They should call the t_shell_cmd wrapper
327 (defined below) if they wish to run the
328 command string only if the prevailing
329 test_mode variable is set to 0.
330 time_out A time-out value expressed in seconds. If
331 the command string has not finished
332 executing within <time_out> seconds, it
333 will be halted and counted as an error.
334 max_attempts The max number of attempts that should be
335 made to run the command string.
336 retry_sleep_time The number of seconds to sleep between
337 attempts.
338 allowed_shell_rcs A list of integers indicating which
339 shell_rc values are not to be considered
340 errors.
341 ignore_err Ignore error means that a failure
342 encountered by running the command string
343 will not be raised as a python exception.
344 When the ignore_err argument is set to
345 None, this function will assign a default
346 value by searching upward in the stack for
347 the ignore_err variable value. If no such
348 value is found, ignore_err is set to 1.
349 return_stderr If return_stderr is set, this function
350 will process the stdout and stderr streams
351 from the shell command separately. In
352 such a case, the tuple returned by this
353 function will consist of three values
354 rather than just two: rc, stdout, stderr.
Michael Walshfaafa9c2018-06-27 16:39:31 -0500355 fork Run the command string asynchronously
356 (i.e. don't wait for status of the child
357 process and don't try to get
358 stdout/stderr).
Michael Walsh21083d22018-06-01 14:19:32 -0500359 """
360
361 # Assign default values to some of the arguments to this function.
Michael Walsh4cb9b2a2018-06-08 17:57:00 -0500362 quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0)))
Michael Walsh55abd1b2018-08-31 13:00:50 -0500363 print_output = int(gm.dft(print_output, not quiet))
Michael Walsh45fead42018-09-26 17:20:48 -0500364 show_err = int(show_err)
Michael Walsh22c3b6d2018-11-16 15:26:05 -0600365 global_ignore_err = gp.get_var_value(ignore_err, 1)
366 stack_ignore_err = gp.get_stack_var('ignore_err', global_ignore_err)
367 ignore_err = int(gm.dft(ignore_err, gm.dft(stack_ignore_err, 1)))
Michael Walsh21083d22018-06-01 14:19:32 -0500368
369 err_msg = gv.svalid_value(command_string)
370 if err_msg != "":
371 raise ValueError(err_msg)
372
373 if not quiet:
374 gp.print_issuing(command_string, test_mode)
375
376 if test_mode:
377 if return_stderr:
378 return 0, "", ""
379 else:
380 return 0, ""
381
382 # Convert each list entry to a signed value.
383 allowed_shell_rcs = [gm.to_signed(x) for x in allowed_shell_rcs]
384
385 if return_stderr:
386 stderr = subprocess.PIPE
387 else:
388 stderr = subprocess.STDOUT
389
390 shell_rc = 0
391 out_buf = ""
392 err_buf = ""
393 # Write all output to func_history_stdout rather than directly to stdout.
394 # This allows us to decide what to print after all attempts to run the
395 # command string have been made. func_history_stdout will contain the
396 # complete stdout history from the current invocation of this function.
397 func_history_stdout = ""
398 for attempt_num in range(1, max_attempts + 1):
399 sub_proc = subprocess.Popen(command_string,
400 bufsize=1,
401 shell=True,
402 executable='/bin/bash',
403 stdout=subprocess.PIPE,
404 stderr=stderr)
405 out_buf = ""
406 err_buf = ""
407 # Output from this loop iteration is written to func_stdout for later
408 # processing.
409 func_stdout = ""
Michael Walshfaafa9c2018-06-27 16:39:31 -0500410 if fork:
411 break
Michael Walsh21083d22018-06-01 14:19:32 -0500412 command_timed_out = False
413 if time_out is not None:
414 # Designate a SIGALRM handling function and set alarm.
415 signal.signal(signal.SIGALRM, shell_cmd_timed_out)
416 signal.alarm(time_out)
417 try:
418 if return_stderr:
419 for line in sub_proc.stderr:
420 err_buf += line
421 if not print_output:
422 continue
423 func_stdout += line
424 for line in sub_proc.stdout:
425 out_buf += line
426 if not print_output:
427 continue
428 func_stdout += line
429 except IOError:
430 command_timed_out = True
431 sub_proc.communicate()
432 shell_rc = sub_proc.returncode
433 # Restore the original SIGALRM handler and clear the alarm.
434 signal.signal(signal.SIGALRM, original_sigalrm_handler)
435 signal.alarm(0)
436 if shell_rc in allowed_shell_rcs:
437 break
438 err_msg = "The prior shell command failed.\n"
Michael Walsha750ed72018-07-24 16:18:09 -0500439 if quiet:
440 err_msg += gp.sprint_var(command_string)
Michael Walsh21083d22018-06-01 14:19:32 -0500441 if command_timed_out:
442 err_msg += gp.sprint_var(command_timed_out)
443 err_msg += gp.sprint_var(time_out)
444 err_msg += gp.sprint_varx("child_pid", sub_proc.pid)
445 err_msg += gp.sprint_var(attempt_num)
446 err_msg += gp.sprint_var(shell_rc, 1)
447 err_msg += gp.sprint_var(allowed_shell_rcs, 1)
448 if not print_output:
449 if return_stderr:
450 err_msg += "err_buf:\n" + err_buf
451 err_msg += "out_buf:\n" + out_buf
452 if show_err:
453 if robot_env:
454 func_stdout += grp.sprint_error_report(err_msg)
455 else:
456 func_stdout += gp.sprint_error_report(err_msg)
457 func_history_stdout += func_stdout
458 if attempt_num < max_attempts:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500459 func_history_stdout += gp.sprint_issuing("time.sleep("
460 + str(retry_sleep_time)
461 + ")")
Michael Walsh21083d22018-06-01 14:19:32 -0500462 time.sleep(retry_sleep_time)
463
464 if shell_rc not in allowed_shell_rcs:
465 func_stdout = func_history_stdout
466
467 if robot_env:
468 grp.rprint(func_stdout)
469 else:
470 sys.stdout.write(func_stdout)
471 sys.stdout.flush()
472
473 if shell_rc not in allowed_shell_rcs:
474 if not ignore_err:
475 if robot_env:
476 BuiltIn().fail(err_msg)
477 else:
478 raise ValueError("The prior shell command failed.\n")
479
480 if return_stderr:
481 return shell_rc, out_buf, err_buf
482 else:
483 return shell_rc, out_buf
484
485
486def t_shell_cmd(command_string, **kwargs):
487 r"""
488 Search upward in the the call stack to obtain the test_mode argument, add
489 it to kwargs and then call shell_cmd and return the result.
490
491 See shell_cmd prolog for details on all arguments.
492 """
493
494 if 'test_mode' in kwargs:
495 error_message = "Programmer error - test_mode is not a valid" +\
496 " argument to this function."
497 gp.print_error_report(error_message)
498 exit(1)
499
500 test_mode = gp.get_stack_var('test_mode',
501 int(gp.get_var_value(None, 0, "test_mode")))
502 kwargs['test_mode'] = test_mode
503
504 return shell_cmd(command_string, **kwargs)