blob: 52f255aeab1e72c7ed20df06f1de15e847c3c902 [file] [log] [blame]
Michael Walshc3b512e2017-02-20 15:59:01 -06001#!/usr/bin/env python
2
3r"""
4This module provides command execution functions such as cmd_fnc and cmd_fnc_u.
5"""
6
Michael Walsh21083d22018-06-01 14:19:32 -05007import os
Michael Walshc3b512e2017-02-20 15:59:01 -06008import sys
9import subprocess
Michael Walshf41fac82017-08-02 15:05:24 -050010import collections
Michael Walsh21083d22018-06-01 14:19:32 -050011import signal
12import time
Michael Walshc3b512e2017-02-20 15:59:01 -060013
Michael Walshc3b512e2017-02-20 15:59:01 -060014import gen_print as gp
15import gen_valid as gv
16import gen_misc as gm
Michael Walshd6901502017-11-14 12:58:37 -060017
18robot_env = gp.robot_env
19
Michael Walshafc53a22017-04-12 15:52:28 -050020if robot_env:
21 import gen_robot_print as grp
Michael Walsha3e2f532018-01-10 13:43:42 -060022 from robot.libraries.BuiltIn import BuiltIn
Michael Walshc3b512e2017-02-20 15:59:01 -060023
24
Michael Walsh21083d22018-06-01 14:19:32 -050025# cmd_fnc and cmd_fnc_u should now be considered deprecated. shell_cmd and
26# t_shell_cmd should be used instead.
Michael Walshc3b512e2017-02-20 15:59:01 -060027def cmd_fnc(cmd_buf,
28 quiet=None,
29 test_mode=None,
Michael Walshafc53a22017-04-12 15:52:28 -050030 debug=0,
Michael Walshc3b512e2017-02-20 15:59:01 -060031 print_output=1,
Michael Walshcfe9fed2017-09-12 17:13:10 -050032 show_err=1,
Michael Walsha3e2f532018-01-10 13:43:42 -060033 return_stderr=0,
34 ignore_err=1):
Michael Walshc3b512e2017-02-20 15:59:01 -060035 r"""
Michael Walshcfe9fed2017-09-12 17:13:10 -050036 Run the given command in a shell and return the shell return code and the
37 output.
Michael Walshc3b512e2017-02-20 15:59:01 -060038
39 Description of arguments:
40 cmd_buf The command string to be run in a shell.
41 quiet Indicates whether this function should run
Michael Walshcfe9fed2017-09-12 17:13:10 -050042 the print_issuing() function which prints
43 "Issuing: <cmd string>" to stdout.
Michael Walshc3b512e2017-02-20 15:59:01 -060044 test_mode If test_mode is set, this function will
Michael Walshcfe9fed2017-09-12 17:13:10 -050045 not actually run the command. If
46 print_output is set, it will print
47 "(test_mode) Issuing: <cmd string>" to
48 stdout.
Michael Walshc3b512e2017-02-20 15:59:01 -060049 debug If debug is set, this function will print
50 extra debug info.
51 print_output If this is set, this function will print
Michael Walshcfe9fed2017-09-12 17:13:10 -050052 the stdout/stderr generated by the shell
53 command.
Michael Walshc3b512e2017-02-20 15:59:01 -060054 show_err If show_err is set, this function will
Michael Walshcfe9fed2017-09-12 17:13:10 -050055 print a standardized error report if the
56 shell command returns non-zero.
57 return_stderr If return_stderr is set, this function
58 will process the stdout and stderr streams
59 from the shell command separately. It
60 will also return stderr in addition to the
61 return code and the stdout.
Michael Walshc3b512e2017-02-20 15:59:01 -060062 """
63
Michael Walshcfe9fed2017-09-12 17:13:10 -050064 # Determine default values.
Michael Walshc3b512e2017-02-20 15:59:01 -060065 quiet = int(gm.global_default(quiet, 0))
66 test_mode = int(gm.global_default(test_mode, 0))
Michael Walshc3b512e2017-02-20 15:59:01 -060067
68 if debug:
Michael Walshafc53a22017-04-12 15:52:28 -050069 gp.print_vars(cmd_buf, quiet, test_mode, debug)
Michael Walshc3b512e2017-02-20 15:59:01 -060070
71 err_msg = gv.svalid_value(cmd_buf)
72 if err_msg != "":
73 raise ValueError(err_msg)
74
75 if not quiet:
Michael Walshafc53a22017-04-12 15:52:28 -050076 gp.pissuing(cmd_buf, test_mode)
Michael Walshc3b512e2017-02-20 15:59:01 -060077
78 if test_mode:
Michael Walshcfe9fed2017-09-12 17:13:10 -050079 if return_stderr:
80 return 0, "", ""
81 else:
82 return 0, ""
83
84 if return_stderr:
85 err_buf = ""
86 stderr = subprocess.PIPE
87 else:
88 stderr = subprocess.STDOUT
Michael Walshc3b512e2017-02-20 15:59:01 -060089
90 sub_proc = subprocess.Popen(cmd_buf,
91 bufsize=1,
92 shell=True,
Michael Walsh3ba8ecd2018-04-24 11:33:25 -050093 executable='/bin/bash',
Michael Walshc3b512e2017-02-20 15:59:01 -060094 stdout=subprocess.PIPE,
Michael Walshcfe9fed2017-09-12 17:13:10 -050095 stderr=stderr)
Michael Walshc3b512e2017-02-20 15:59:01 -060096 out_buf = ""
Michael Walshcfe9fed2017-09-12 17:13:10 -050097 if return_stderr:
98 for line in sub_proc.stderr:
99 err_buf += line
100 if not print_output:
101 continue
102 if robot_env:
103 grp.rprint(line)
104 else:
105 sys.stdout.write(line)
Michael Walshc3b512e2017-02-20 15:59:01 -0600106 for line in sub_proc.stdout:
107 out_buf += line
108 if not print_output:
109 continue
110 if robot_env:
111 grp.rprint(line)
112 else:
113 sys.stdout.write(line)
114 if print_output and not robot_env:
115 sys.stdout.flush()
116 sub_proc.communicate()
117 shell_rc = sub_proc.returncode
Michael Walsha3e2f532018-01-10 13:43:42 -0600118 if shell_rc != 0:
119 err_msg = "The prior shell command failed.\n"
120 err_msg += gp.sprint_var(shell_rc, 1)
Michael Walshcfe9fed2017-09-12 17:13:10 -0500121 if not print_output:
122 err_msg += "out_buf:\n" + out_buf
Michael Walshc3b512e2017-02-20 15:59:01 -0600123
Michael Walsha3e2f532018-01-10 13:43:42 -0600124 if show_err:
125 if robot_env:
126 grp.rprint_error_report(err_msg)
127 else:
128 gp.print_error_report(err_msg)
129 if not ignore_err:
130 if robot_env:
131 BuiltIn().fail(err_msg)
132 else:
133 raise ValueError(err_msg)
Michael Walshcfe9fed2017-09-12 17:13:10 -0500134
135 if return_stderr:
136 return shell_rc, out_buf, err_buf
137 else:
138 return shell_rc, out_buf
Michael Walshc3b512e2017-02-20 15:59:01 -0600139
Michael Walshc3b512e2017-02-20 15:59:01 -0600140
Michael Walshc3b512e2017-02-20 15:59:01 -0600141def cmd_fnc_u(cmd_buf,
142 quiet=None,
143 debug=None,
144 print_output=1,
Michael Walshcfe9fed2017-09-12 17:13:10 -0500145 show_err=1,
Michael Walsha3e2f532018-01-10 13:43:42 -0600146 return_stderr=0,
147 ignore_err=1):
Michael Walshc3b512e2017-02-20 15:59:01 -0600148 r"""
149 Call cmd_fnc with test_mode=0. See cmd_fnc (above) for details.
150
151 Note the "u" in "cmd_fnc_u" stands for "unconditional".
152 """
153
154 return cmd_fnc(cmd_buf, test_mode=0, quiet=quiet, debug=debug,
Michael Walshcfe9fed2017-09-12 17:13:10 -0500155 print_output=print_output, show_err=show_err,
Michael Walsha3e2f532018-01-10 13:43:42 -0600156 return_stderr=return_stderr, ignore_err=ignore_err)
Michael Walshc3b512e2017-02-20 15:59:01 -0600157
Michael Walshf41fac82017-08-02 15:05:24 -0500158
Michael Walshf41fac82017-08-02 15:05:24 -0500159def parse_command_string(command_string):
Michael Walshf41fac82017-08-02 15:05:24 -0500160 r"""
161 Parse a bash command-line command string and return the result as a
162 dictionary of parms.
163
164 This can be useful for answering questions like "What did the user specify
165 as the value for parm x in the command string?".
166
167 This function expects the command string to follow the following posix
168 conventions:
169 - Short parameters:
170 -<parm name><space><arg value>
171 - Long parameters:
172 --<parm name>=<arg value>
173
174 The first item in the string will be considered to be the command. All
175 values not conforming to the specifications above will be considered
176 positional parms. If there are multiple parms with the same name, they
177 will be put into a list (see illustration below where "-v" is specified
178 multiple times).
179
180 Description of argument(s):
181 command_string The complete command string including all
182 parameters and arguments.
183
184 Sample input:
185
186 robot_cmd_buf: robot -v
187 OPENBMC_HOST:dummy1 -v keyword_string:'Set Auto Reboot no' -v
188 lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot -v
189 quiet:0 -v test_mode:0 -v debug:0
190 --outputdir='/home/user1/status/children/'
191 --output=dummy1.Auto_reboot.170802.124544.output.xml
192 --log=dummy1.Auto_reboot.170802.124544.log.html
193 --report=dummy1.Auto_reboot.170802.124544.report.html
194 /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
195
196 Sample output:
197
198 robot_cmd_buf_dict:
199 robot_cmd_buf_dict[command]: robot
200 robot_cmd_buf_dict[v]:
201 robot_cmd_buf_dict[v][0]: OPENBMC_HOST:dummy1
202 robot_cmd_buf_dict[v][1]: keyword_string:Set Auto
203 Reboot no
204 robot_cmd_buf_dict[v][2]:
205 lib_file_path:/home/user1/git/openbmc-test-automation/lib/utils.robot
206 robot_cmd_buf_dict[v][3]: quiet:0
207 robot_cmd_buf_dict[v][4]: test_mode:0
208 robot_cmd_buf_dict[v][5]: debug:0
209 robot_cmd_buf_dict[outputdir]:
210 /home/user1/status/children/
211 robot_cmd_buf_dict[output]:
212 dummy1.Auto_reboot.170802.124544.output.xml
213 robot_cmd_buf_dict[log]:
214 dummy1.Auto_reboot.170802.124544.log.html
215 robot_cmd_buf_dict[report]:
216 dummy1.Auto_reboot.170802.124544.report.html
217 robot_cmd_buf_dict[positional]:
218 /home/user1/git/openbmc-test-automation/extended/run_keyword.robot
219 """
220
221 # We want the parms in the string broken down the way bash would do it,
222 # so we'll call upon bash to do that by creating a simple inline bash
223 # function.
224 bash_func_def = "function parse { for parm in \"${@}\" ; do" +\
225 " echo $parm ; done ; }"
226
227 rc, outbuf = cmd_fnc_u(bash_func_def + " ; parse " + command_string,
228 quiet=1, print_output=0)
229 command_string_list = outbuf.rstrip("\n").split("\n")
230
231 command_string_dict = collections.OrderedDict()
232 ix = 1
233 command_string_dict['command'] = command_string_list[0]
234 while ix < len(command_string_list):
235 if command_string_list[ix].startswith("--"):
236 key, value = command_string_list[ix].split("=")
237 key = key.lstrip("-")
238 elif command_string_list[ix].startswith("-"):
239 key = command_string_list[ix].lstrip("-")
240 ix += 1
241 try:
242 value = command_string_list[ix]
243 except IndexError:
244 value = ""
245 else:
246 key = 'positional'
247 value = command_string_list[ix]
248 if key in command_string_dict:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500249 if isinstance(command_string_dict[key], str):
Michael Walshf41fac82017-08-02 15:05:24 -0500250 command_string_dict[key] = [command_string_dict[key]]
251 command_string_dict[key].append(value)
252 else:
253 command_string_dict[key] = value
254 ix += 1
255
256 return command_string_dict
Michael Walsh21083d22018-06-01 14:19:32 -0500257
258
259# Save the original SIGALRM handler for later restoration by shell_cmd.
260original_sigalrm_handler = signal.getsignal(signal.SIGALRM)
261
262
263def shell_cmd_timed_out(signal_number,
264 frame):
265 r"""
266 Handle an alarm signal generated during the shell_cmd function.
267 """
268
269 gp.dprint_executing()
270 # Get subprocess pid from shell_cmd's call stack.
271 sub_proc = gp.get_stack_var('sub_proc', 0)
272 pid = sub_proc.pid
273 # Terminate the child process.
274 os.kill(pid, signal.SIGTERM)
275 # Restore the original SIGALRM handler.
276 signal.signal(signal.SIGALRM, original_sigalrm_handler)
277
278 return
279
280
281def shell_cmd(command_string,
282 quiet=None,
283 print_output=1,
284 show_err=1,
285 test_mode=0,
286 time_out=None,
287 max_attempts=1,
288 retry_sleep_time=5,
289 allowed_shell_rcs=[0],
290 ignore_err=None,
Michael Walshfaafa9c2018-06-27 16:39:31 -0500291 return_stderr=0,
292 fork=0):
Michael Walsh21083d22018-06-01 14:19:32 -0500293 r"""
294 Run the given command string in a shell and return a tuple consisting of
295 the shell return code and the output.
296
297 Description of argument(s):
298 command_string The command string to be run in a shell
299 (e.g. "ls /tmp").
300 quiet If set to 0, this function will print
Michael Walsh4cb9b2a2018-06-08 17:57:00 -0500301 "Issuing: <cmd string>" to stdout. When
302 the quiet argument is set to None, this
303 function will assign a default value by
304 searching upward in the stack for the
305 quiet variable value. If no such value is
306 found, quiet is set to 0.
Michael Walsh21083d22018-06-01 14:19:32 -0500307 print_output If this is set, this function will print
308 the stdout/stderr generated by the shell
309 command to stdout.
310 show_err If show_err is set, this function will
311 print a standardized error report if the
312 shell command fails (i.e. if the shell
313 command returns a shell_rc that is not in
314 allowed_shell_rcs). Note: Error text is
315 only printed if ALL attempts to run the
316 command_string fail. In other words, if
317 the command execution is ultimately
318 successful, initial failures are hidden.
319 test_mode If test_mode is set, this function will
320 not actually run the command. If
321 print_output is also set, this function
322 will print "(test_mode) Issuing: <cmd
323 string>" to stdout. A caller should call
324 shell_cmd directly if they wish to have
325 the command string run unconditionally.
326 They should call the t_shell_cmd wrapper
327 (defined below) if they wish to run the
328 command string only if the prevailing
329 test_mode variable is set to 0.
330 time_out A time-out value expressed in seconds. If
331 the command string has not finished
332 executing within <time_out> seconds, it
333 will be halted and counted as an error.
334 max_attempts The max number of attempts that should be
335 made to run the command string.
336 retry_sleep_time The number of seconds to sleep between
337 attempts.
338 allowed_shell_rcs A list of integers indicating which
339 shell_rc values are not to be considered
340 errors.
341 ignore_err Ignore error means that a failure
342 encountered by running the command string
343 will not be raised as a python exception.
344 When the ignore_err argument is set to
345 None, this function will assign a default
346 value by searching upward in the stack for
347 the ignore_err variable value. If no such
348 value is found, ignore_err is set to 1.
349 return_stderr If return_stderr is set, this function
350 will process the stdout and stderr streams
351 from the shell command separately. In
352 such a case, the tuple returned by this
353 function will consist of three values
354 rather than just two: rc, stdout, stderr.
Michael Walshfaafa9c2018-06-27 16:39:31 -0500355 fork Run the command string asynchronously
356 (i.e. don't wait for status of the child
357 process and don't try to get
358 stdout/stderr).
Michael Walsh21083d22018-06-01 14:19:32 -0500359 """
360
361 # Assign default values to some of the arguments to this function.
Michael Walsh4cb9b2a2018-06-08 17:57:00 -0500362 quiet = int(gm.dft(quiet, gp.get_stack_var('quiet', 0)))
363 ignore_err = int(gm.dft(ignore_err, gp.get_stack_var('ignore_err', 1)))
Michael Walsh21083d22018-06-01 14:19:32 -0500364
365 err_msg = gv.svalid_value(command_string)
366 if err_msg != "":
367 raise ValueError(err_msg)
368
369 if not quiet:
370 gp.print_issuing(command_string, test_mode)
371
372 if test_mode:
373 if return_stderr:
374 return 0, "", ""
375 else:
376 return 0, ""
377
378 # Convert each list entry to a signed value.
379 allowed_shell_rcs = [gm.to_signed(x) for x in allowed_shell_rcs]
380
381 if return_stderr:
382 stderr = subprocess.PIPE
383 else:
384 stderr = subprocess.STDOUT
385
386 shell_rc = 0
387 out_buf = ""
388 err_buf = ""
389 # Write all output to func_history_stdout rather than directly to stdout.
390 # This allows us to decide what to print after all attempts to run the
391 # command string have been made. func_history_stdout will contain the
392 # complete stdout history from the current invocation of this function.
393 func_history_stdout = ""
394 for attempt_num in range(1, max_attempts + 1):
395 sub_proc = subprocess.Popen(command_string,
396 bufsize=1,
397 shell=True,
398 executable='/bin/bash',
399 stdout=subprocess.PIPE,
400 stderr=stderr)
401 out_buf = ""
402 err_buf = ""
403 # Output from this loop iteration is written to func_stdout for later
404 # processing.
405 func_stdout = ""
Michael Walshfaafa9c2018-06-27 16:39:31 -0500406 if fork:
407 break
Michael Walsh21083d22018-06-01 14:19:32 -0500408 command_timed_out = False
409 if time_out is not None:
410 # Designate a SIGALRM handling function and set alarm.
411 signal.signal(signal.SIGALRM, shell_cmd_timed_out)
412 signal.alarm(time_out)
413 try:
414 if return_stderr:
415 for line in sub_proc.stderr:
416 err_buf += line
417 if not print_output:
418 continue
419 func_stdout += line
420 for line in sub_proc.stdout:
421 out_buf += line
422 if not print_output:
423 continue
424 func_stdout += line
425 except IOError:
426 command_timed_out = True
427 sub_proc.communicate()
428 shell_rc = sub_proc.returncode
429 # Restore the original SIGALRM handler and clear the alarm.
430 signal.signal(signal.SIGALRM, original_sigalrm_handler)
431 signal.alarm(0)
432 if shell_rc in allowed_shell_rcs:
433 break
434 err_msg = "The prior shell command failed.\n"
Michael Walsha750ed72018-07-24 16:18:09 -0500435 if quiet:
436 err_msg += gp.sprint_var(command_string)
Michael Walsh21083d22018-06-01 14:19:32 -0500437 if command_timed_out:
438 err_msg += gp.sprint_var(command_timed_out)
439 err_msg += gp.sprint_var(time_out)
440 err_msg += gp.sprint_varx("child_pid", sub_proc.pid)
441 err_msg += gp.sprint_var(attempt_num)
442 err_msg += gp.sprint_var(shell_rc, 1)
443 err_msg += gp.sprint_var(allowed_shell_rcs, 1)
444 if not print_output:
445 if return_stderr:
446 err_msg += "err_buf:\n" + err_buf
447 err_msg += "out_buf:\n" + out_buf
448 if show_err:
449 if robot_env:
450 func_stdout += grp.sprint_error_report(err_msg)
451 else:
452 func_stdout += gp.sprint_error_report(err_msg)
453 func_history_stdout += func_stdout
454 if attempt_num < max_attempts:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500455 func_history_stdout += gp.sprint_issuing("time.sleep("
456 + str(retry_sleep_time)
457 + ")")
Michael Walsh21083d22018-06-01 14:19:32 -0500458 time.sleep(retry_sleep_time)
459
460 if shell_rc not in allowed_shell_rcs:
461 func_stdout = func_history_stdout
462
463 if robot_env:
464 grp.rprint(func_stdout)
465 else:
466 sys.stdout.write(func_stdout)
467 sys.stdout.flush()
468
469 if shell_rc not in allowed_shell_rcs:
470 if not ignore_err:
471 if robot_env:
472 BuiltIn().fail(err_msg)
473 else:
474 raise ValueError("The prior shell command failed.\n")
475
476 if return_stderr:
477 return shell_rc, out_buf, err_buf
478 else:
479 return shell_rc, out_buf
480
481
482def t_shell_cmd(command_string, **kwargs):
483 r"""
484 Search upward in the the call stack to obtain the test_mode argument, add
485 it to kwargs and then call shell_cmd and return the result.
486
487 See shell_cmd prolog for details on all arguments.
488 """
489
490 if 'test_mode' in kwargs:
491 error_message = "Programmer error - test_mode is not a valid" +\
492 " argument to this function."
493 gp.print_error_report(error_message)
494 exit(1)
495
496 test_mode = gp.get_stack_var('test_mode',
497 int(gp.get_var_value(None, 0, "test_mode")))
498 kwargs['test_mode'] = test_mode
499
500 return shell_cmd(command_string, **kwargs)