blob: 9631182b8013fc5a679759ee79cb65621286ded2 [file] [log] [blame]
Michael Walshde791732016-09-06 14:25:24 -05001#!/usr/bin/env python
2
Michael Walsh7423c012016-10-04 10:27:21 -05003r"""
4This module provides many valuable functions such as my_parm_file.
5"""
Michael Walshde791732016-09-06 14:25:24 -05006
7# sys and os are needed to get the program dir path and program name.
8import sys
Michael Walsheaa16852017-09-19 16:30:43 -05009import errno
Michael Walshde791732016-09-06 14:25:24 -050010import os
Michael Walshdece16c2018-08-07 15:01:05 -050011import collections
Michael Walsh6aa69802018-09-21 16:38:34 -050012import json
Michael Walshc9cb9722018-10-01 17:56:20 -050013import time
Michael Walshf5293d22019-02-01 14:23:02 -060014import inspect
Michael Walshdece16c2018-08-07 15:01:05 -050015try:
16 import ConfigParser
17except ImportError:
18 import configparser
George Keishing3b7115a2018-08-02 10:48:17 -050019try:
Michael Walsh61f5e8f2018-08-03 11:16:00 -050020 import StringIO
George Keishing3b7115a2018-08-02 10:48:17 -050021except ImportError:
Michael Walshdece16c2018-08-07 15:01:05 -050022 import io
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060023import re
Michael Walsh1c85bab2017-05-04 14:29:24 -050024import socket
Michael Walsh3b621fe2018-07-24 16:27:53 -050025import tempfile
26try:
27 import psutil
28 psutil_imported = True
29except ImportError:
30 psutil_imported = False
Michael Walshde791732016-09-06 14:25:24 -050031
Michael Walsh7423c012016-10-04 10:27:21 -050032import gen_print as gp
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060033import gen_cmd as gc
34
Michael Walsh93a09f22017-11-13 15:34:46 -060035robot_env = gp.robot_env
36if robot_env:
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060037 from robot.libraries.BuiltIn import BuiltIn
Michael Walshdece16c2018-08-07 15:01:05 -050038 from robot.utils import DotDict
Michael Walsh7423c012016-10-04 10:27:21 -050039
Michael Walshde791732016-09-06 14:25:24 -050040
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060041def add_trailing_slash(dir_path):
Michael Walsh7db77942017-01-10 11:37:06 -060042 r"""
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060043 Add a trailing slash to the directory path if it doesn't already have one
44 and return it.
Michael Walsh7db77942017-01-10 11:37:06 -060045
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060046 Description of arguments:
47 dir_path A directory path.
Michael Walsh7db77942017-01-10 11:37:06 -060048 """
49
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060050 return os.path.normpath(dir_path) + os.path.sep
51
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060052
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060053def which(file_path):
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060054 r"""
55 Find the full path of an executable file and return it.
56
57 The PATH environment variable dictates the results of this function.
58
59 Description of arguments:
60 file_path The relative file path (e.g. "my_file" or
61 "lib/my_file").
62 """
63
64 shell_rc, out_buf = gc.cmd_fnc_u("which " + file_path, quiet=1,
65 print_output=0, show_err=0)
66 if shell_rc != 0:
67 error_message = "Failed to find complete path for file \"" +\
68 file_path + "\".\n"
Michael Walsh0d5f96a2019-05-20 10:09:57 -050069 error_message += gp.sprint_var(shell_rc, gp.hexa())
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060070 error_message += out_buf
71 if robot_env:
72 BuiltIn().fail(gp.sprint_error(error_message))
73 else:
74 gp.print_error_report(error_message)
75 return False
76
77 file_path = out_buf.rstrip("\n")
78
79 return file_path
80
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060081
Michael Walshf7400f32018-09-26 17:13:43 -050082def add_path(new_path,
83 path,
84 position=0):
85 r"""
86 Add new_path to path, provided that path doesn't already contain new_path,
87 and return the result.
88
89 Example:
90 If PATH has a value of "/bin/user:/lib/user". The following code:
91
92 PATH = add_path("/tmp/new_path", PATH)
93
94 will change PATH to "/tmp/new_path:/bin/user:/lib/user".
95
96 Description of argument(s):
97 new_path The path to be added. This function will
98 strip the trailing slash.
99 path The path value to which the new_path
100 should be added.
101 position The position in path where the new_path
102 should be added. 0 means it should be
103 added to the beginning, 1 means add it as
104 the 2nd item, etc. sys.maxsize means it
105 should be added to the end.
106 """
107
108 path_list = list(filter(None, path.split(":")))
109 new_path = new_path.rstrip("/")
110 if new_path not in path_list:
111 path_list.insert(int(position), new_path)
112 return ":".join(path_list)
113
114
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600115def dft(value, default):
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600116 r"""
117 Return default if value is None. Otherwise, return value.
118
119 This is really just shorthand as shown below.
120
121 dft(value, default)
122
123 vs
124
125 default if value is None else value
126
127 Description of arguments:
128 value The value to be returned.
129 default The default value to return if value is
130 None.
131 """
132
133 return default if value is None else value
134
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600135
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600136def get_mod_global(var_name,
137 default=None,
138 mod_name="__main__"):
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600139 r"""
140 Get module global variable value and return it.
141
142 If we are running in a robot environment, the behavior will default to
143 calling get_variable_value.
144
145 Description of arguments:
146 var_name The name of the variable whose value is
147 sought.
148 default The value to return if the global does not
149 exist.
150 mod_name The name of the module containing the
151 global variable.
152 """
153
154 if robot_env:
155 return BuiltIn().get_variable_value("${" + var_name + "}", default)
156
157 try:
158 module = sys.modules[mod_name]
159 except KeyError:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500160 gp.print_error_report("Programmer error - The mod_name passed to"
161 + " this function is invalid:\n"
162 + gp.sprint_var(mod_name))
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600163 raise ValueError('Programmer error.')
164
165 if default is None:
166 return getattr(module, var_name)
167 else:
168 return getattr(module, var_name, default)
169
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600170
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600171def global_default(var_value,
172 default=0):
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600173 r"""
174 If var_value is not None, return it. Otherwise, return the global
175 variable of the same name, if it exists. If not, return default.
176
177 This is meant for use by functions needing help assigning dynamic default
178 values to their parms. Example:
179
180 def func1(parm1=None):
181
182 parm1 = global_default(parm1, 0)
183
184 Description of arguments:
185 var_value The value being evaluated.
186 default The value to be returned if var_value is
187 None AND the global variable of the same
188 name does not exist.
189 """
190
191 var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
192
193 return dft(var_value, get_mod_global(var_name, 0))
Michael Walsh7db77942017-01-10 11:37:06 -0600194
Michael Walsh7db77942017-01-10 11:37:06 -0600195
Michael Walsh7db77942017-01-10 11:37:06 -0600196def set_mod_global(var_value,
197 mod_name="__main__",
198 var_name=None):
Michael Walsh7db77942017-01-10 11:37:06 -0600199 r"""
200 Set a global variable for a given module.
201
202 Description of arguments:
203 var_value The value to set in the variable.
204 mod_name The name of the module whose variable is
205 to be set.
206 var_name The name of the variable to set. This
207 defaults to the name of the variable used
208 for var_value when calling this function.
209 """
210
211 try:
212 module = sys.modules[mod_name]
213 except KeyError:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500214 gp.print_error_report("Programmer error - The mod_name passed to"
215 + " this function is invalid:\n"
216 + gp.sprint_var(mod_name))
Michael Walsh7db77942017-01-10 11:37:06 -0600217 raise ValueError('Programmer error.')
218
219 if var_name is None:
220 var_name = gp.get_arg_name(None, 1, 2)
221
222 setattr(module, var_name, var_value)
223
Michael Walsh7db77942017-01-10 11:37:06 -0600224
Michael Walshde791732016-09-06 14:25:24 -0500225def my_parm_file(prop_file_path):
Michael Walshde791732016-09-06 14:25:24 -0500226 r"""
227 Read a properties file, put the keys/values into a dictionary and return
228 the dictionary.
229
230 The properties file must have the following format:
231 var_name<= or :>var_value
232 Comment lines (those beginning with a "#") and blank lines are allowed and
233 will be ignored. Leading and trailing single or double quotes will be
234 stripped from the value. E.g.
235 var1="This one"
236 Quotes are stripped so the resulting value for var1 is:
237 This one
238
239 Description of arguments:
Michael Walsh7423c012016-10-04 10:27:21 -0500240 prop_file_path The caller should pass the path to the
241 properties file.
Michael Walshde791732016-09-06 14:25:24 -0500242 """
243
244 # ConfigParser expects at least one section header in the file (or you
245 # get ConfigParser.MissingSectionHeaderError). Properties files don't
246 # need those so I'll write a dummy section header.
247
Michael Walshdece16c2018-08-07 15:01:05 -0500248 try:
249 string_file = StringIO.StringIO()
250 except NameError:
251 string_file = io.StringIO()
252
Michael Walshde791732016-09-06 14:25:24 -0500253 # Write the dummy section header to the string file.
254 string_file.write('[dummysection]\n')
255 # Write the entire contents of the properties file to the string file.
256 string_file.write(open(prop_file_path).read())
257 # Rewind the string file.
258 string_file.seek(0, os.SEEK_SET)
259
260 # Create the ConfigParser object.
Michael Walshdece16c2018-08-07 15:01:05 -0500261 try:
262 config_parser = ConfigParser.ConfigParser()
263 except NameError:
George Keishing36efbc02018-12-12 10:18:23 -0600264 config_parser = configparser.ConfigParser(strict=False)
Michael Walshde791732016-09-06 14:25:24 -0500265 # Make the property names case-sensitive.
266 config_parser.optionxform = str
267 # Read the properties from the string file.
268 config_parser.readfp(string_file)
269 # Return the properties as a dictionary.
Michael Walshdece16c2018-08-07 15:01:05 -0500270 if robot_env:
271 return DotDict(config_parser.items('dummysection'))
272 else:
273 return collections.OrderedDict(config_parser.items('dummysection'))
Michael Walshde791732016-09-06 14:25:24 -0500274
Michael Walsh7423c012016-10-04 10:27:21 -0500275
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600276def file_to_list(file_path,
277 newlines=0,
278 comments=1,
279 trim=0):
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600280 r"""
281 Return the contents of a file as a list. Each element of the resulting
282 list is one line from the file.
283
284 Description of arguments:
285 file_path The path to the file (relative or
286 absolute).
287 newlines Include newlines from the file in the
288 results.
289 comments Include comment lines and blank lines in
290 the results. Comment lines are any that
291 begin with 0 or more spaces followed by
292 the pound sign ("#").
293 trim Trim white space from the beginning and
294 end of each line.
295 """
296
297 lines = []
298 file = open(file_path)
299 for line in file:
300 if not comments:
301 if re.match(r"[ ]*#|^$", line):
302 continue
303 if not newlines:
304 line = line.rstrip("\n")
305 if trim:
306 line = line.strip()
307 lines.append(line)
Michael Walsh1383f352018-09-27 16:25:54 -0500308 file.close()
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600309
310 return lines
311
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600312
Michael Walsh1383f352018-09-27 16:25:54 -0500313def file_to_str(*args, **kwargs):
314 r"""
315 Return the contents of a file as a string.
316
317 Description of arguments:
318 See file_to_list defined above for description of arguments.
319 """
320
321 return '\n'.join(file_to_list(*args, **kwargs))
322
323
Michael Walsh7423c012016-10-04 10:27:21 -0500324def return_path_list():
Michael Walsh7423c012016-10-04 10:27:21 -0500325 r"""
326 This function will split the PATH environment variable into a PATH_LIST
327 and return it. Each element in the list will be normalized and have a
328 trailing slash added.
329 """
330
331 PATH_LIST = os.environ['PATH'].split(":")
332 PATH_LIST = [os.path.normpath(path) + os.sep for path in PATH_LIST]
333
334 return PATH_LIST
335
Michael Walsh7db77942017-01-10 11:37:06 -0600336
Michael Walsh9fac55c2017-09-29 16:53:56 -0500337def escape_bash_quotes(buffer):
Michael Walsh9fac55c2017-09-29 16:53:56 -0500338 r"""
339 Escape quotes in string and return it.
340
341 The escape style implemented will be for use on the bash command line.
342
343 Example:
344 That's all.
345
346 Result:
347 That'\''s all.
348
349 The result may then be single quoted on a bash command. Example:
350
351 echo 'That'\''s all.'
352
353 Description of argument(s):
354 buffer The string whose quotes are to be escaped.
355 """
356
357 return re.sub("\'", "\'\\\'\'", buffer)
358
359
Michael Walsh7db77942017-01-10 11:37:06 -0600360def quote_bash_parm(parm):
Michael Walsh7db77942017-01-10 11:37:06 -0600361 r"""
362 Return the bash command line parm with single quotes if they are needed.
363
364 Description of arguments:
365 parm The string to be quoted.
366 """
367
368 # If any of these characters are found in the parm string, then the
369 # string should be quoted. This list is by no means complete and should
370 # be expanded as needed by the developer of this function.
Michael Walsh9fc17c32019-01-21 14:49:10 -0600371 # Spaces
372 # Single or double quotes.
373 # Bash variables (therefore, any string with a "$" may need quoting).
374 # Glob characters: *, ?, []
375 # Extended Glob characters: +, @, !
376 # Bash brace expansion: {}
377 # Tilde expansion: ~
378 # Piped commands: |
379 # Bash re-direction: >, <
380 bash_special_chars = set(' \'"$*?[]+@!{}~|><')
Michael Walsh7db77942017-01-10 11:37:06 -0600381
382 if any((char in bash_special_chars) for char in parm):
Michael Walsh9fc17c32019-01-21 14:49:10 -0600383 return "'" + escape_bash_quotes(parm) + "'"
384
385 if parm == '':
386 parm = "''"
Michael Walsh7db77942017-01-10 11:37:06 -0600387
388 return parm
389
Michael Walsh1c85bab2017-05-04 14:29:24 -0500390
Michael Walsh74427232018-08-31 12:54:24 -0500391def get_host_name_ip(host=None,
Michael Walshf74b3e42018-01-10 11:11:54 -0600392 short_name=0):
Michael Walsh1c85bab2017-05-04 14:29:24 -0500393 r"""
394 Get the host name and the IP address for the given host and return them as
395 a tuple.
396
397 Description of argument(s):
Michael Walshd1b6c702017-05-30 17:54:30 -0500398 host The host name or IP address to be obtained.
Michael Walshf74b3e42018-01-10 11:11:54 -0600399 short_name Include the short host name in the
400 returned tuple, i.e. return host, ip and
401 short_host.
Michael Walsh1c85bab2017-05-04 14:29:24 -0500402 """
403
Michael Walsh74427232018-08-31 12:54:24 -0500404 host = dft(host, socket.gethostname())
Michael Walshf74b3e42018-01-10 11:11:54 -0600405 host_name = socket.getfqdn(host)
Michael Walshd1b6c702017-05-30 17:54:30 -0500406 try:
407 host_ip = socket.gethostbyname(host)
408 except socket.gaierror as my_gaierror:
409 message = "Unable to obtain the host name for the following host:" +\
410 "\n" + gp.sprint_var(host)
411 gp.print_error_report(message)
412 raise my_gaierror
Michael Walsh1c85bab2017-05-04 14:29:24 -0500413
Michael Walshf74b3e42018-01-10 11:11:54 -0600414 if short_name:
415 host_short_name = host_name.split(".")[0]
416 return host_name, host_ip, host_short_name
417 else:
418 return host_name, host_ip
Michael Walsh1c85bab2017-05-04 14:29:24 -0500419
Michael Walsheaa16852017-09-19 16:30:43 -0500420
421def pid_active(pid):
Michael Walsheaa16852017-09-19 16:30:43 -0500422 r"""
423 Return true if pid represents an active pid and false otherwise.
424
425 Description of argument(s):
426 pid The pid whose status is being sought.
427 """
428
429 try:
430 os.kill(int(pid), 0)
431 except OSError as err:
432 if err.errno == errno.ESRCH:
433 # ESRCH == No such process
434 return False
435 elif err.errno == errno.EPERM:
436 # EPERM clearly means there's a process to deny access to
437 return True
438 else:
439 # According to "man 2 kill" possible error values are
440 # (EINVAL, EPERM, ESRCH)
441 raise
442
443 return True
Michael Walsh112c3592018-06-01 14:15:58 -0500444
445
446def to_signed(number,
Michael Walshdece16c2018-08-07 15:01:05 -0500447 bit_width=None):
Michael Walsh112c3592018-06-01 14:15:58 -0500448 r"""
449 Convert number to a signed number and return the result.
450
451 Examples:
452
453 With the following code:
454
455 var1 = 0xfffffffffffffff1
456 print_var(var1)
Michael Walsh0d5f96a2019-05-20 10:09:57 -0500457 print_var(var1, hexa())
Michael Walsh112c3592018-06-01 14:15:58 -0500458 var1 = to_signed(var1)
459 print_var(var1)
Michael Walsh0d5f96a2019-05-20 10:09:57 -0500460 print_var(var1, hexa())
Michael Walsh112c3592018-06-01 14:15:58 -0500461
462 The following is written to stdout:
463 var1: 18446744073709551601
464 var1: 0x00000000fffffffffffffff1
465 var1: -15
466 var1: 0xfffffffffffffff1
467
468 The same code but with var1 set to 0x000000000000007f produces the
469 following:
470 var1: 127
471 var1: 0x000000000000007f
472 var1: 127
473 var1: 0x000000000000007f
474
475 Description of argument(s):
476 number The number to be converted.
477 bit_width The number of bits that defines a complete
478 hex value. Typically, this would be a
479 multiple of 32.
480 """
481
Michael Walshdece16c2018-08-07 15:01:05 -0500482 if bit_width is None:
483 try:
484 bit_width = gp.bit_length(long(sys.maxsize)) + 1
485 except NameError:
486 bit_width = gp.bit_length(int(sys.maxsize)) + 1
487
Michael Walsh112c3592018-06-01 14:15:58 -0500488 if number < 0:
489 return number
490 neg_bit_mask = 2**(bit_width - 1)
491 if number & neg_bit_mask:
492 return ((2**bit_width) - number) * -1
493 else:
494 return number
Michael Walsh3b621fe2018-07-24 16:27:53 -0500495
496
497def get_child_pids(quiet=1):
498
499 r"""
500 Get and return a list of pids representing all first-generation processes
501 that are the children of the current process.
502
503 Example:
504
505 children = get_child_pids()
506 print_var(children)
507
508 Output:
509 children:
510 children[0]: 9123
511
512 Description of argument(s):
513 quiet Display output to stdout detailing how
514 this child pids are obtained.
515 """
516
517 if psutil_imported:
518 # If "import psutil" worked, find child pids using psutil.
519 current_process = psutil.Process()
520 return [x.pid for x in current_process.children(recursive=False)]
521 else:
522 # Otherwise, find child pids using shell commands.
523 print_output = not quiet
524
525 ps_cmd_buf = "ps --no-headers --ppid " + str(os.getpid()) +\
526 " -o pid,args"
527 # Route the output of ps to a temporary file for later grepping.
528 # Avoid using " | grep" in the ps command string because it creates
529 # yet another process which is of no interest to the caller.
530 temp = tempfile.NamedTemporaryFile()
531 temp_file_path = temp.name
532 gc.shell_cmd(ps_cmd_buf + " > " + temp_file_path,
533 print_output=print_output)
534 # Sample contents of the temporary file:
535 # 30703 sleep 2
536 # 30795 /bin/bash -c ps --no-headers --ppid 30672 -o pid,args >
537 # /tmp/tmpqqorWY
538 # Use egrep to exclude the "ps" process itself from the results
539 # collected with the prior shell_cmd invocation. Only the other
540 # children are of interest to the caller. Use cut on the grep results
541 # to obtain only the pid column.
542 rc, output = \
543 gc.shell_cmd("egrep -v '" + re.escape(ps_cmd_buf) + "' "
544 + temp_file_path + " | cut -c1-5",
545 print_output=print_output)
546 # Split the output buffer by line into a list. Strip each element of
547 # extra spaces and convert each element to an integer.
548 return map(int, map(str.strip, filter(None, output.split("\n"))))
Michael Walsh6aa69802018-09-21 16:38:34 -0500549
550
551def json_loads_multiple(buffer):
552 r"""
553 Convert the contents of the buffer to a JSON array, run json.loads() on it
554 and return the result.
555
556 The buffer is expected to contain one or more JSON objects.
557
558 Description of argument(s):
559 buffer A string containing several JSON objects.
560 """
561
562 # Any line consisting of just "}", which indicates the end of an object,
563 # should have a comma appended.
564 regex = "([\\r\\n])[\\}]([\\r\\n])"
565 buffer = re.sub(regex, "\\1},\\2", buffer, 1)
566 # Remove the comma from after the final object and place the whole buffer
567 # inside square brackets.
568 buffer = "[" + re.sub(",([\r\n])$", "\\1}", buffer, 1) + "]"
569 if gp.robot_env:
570 return json.loads(buffer, object_pairs_hook=DotDict)
571 else:
572 return json.loads(buffer, object_pairs_hook=collections.OrderedDict)
Michael Walshc9cb9722018-10-01 17:56:20 -0500573
574
575def file_date_time_stamp():
576 r"""
577 Return a date/time stamp in the following format: yymmdd.HHMMSS
578
579 This value is suitable for including in file names. Example
580 file1.181001.171716.status
581 """
582
583 return time.strftime("%y%m%d.%H%M%S", time.localtime(time.time()))
Michael Walshf5293d22019-02-01 14:23:02 -0600584
585
586def get_function_stack():
587 r"""
588 Return a list of all the function names currently in the call stack.
589
590 This function's name will be at offset 0. This function's caller's name
591 will be at offset 1 and so on.
592 """
593
594 return [str(stack_frame[3]) for stack_frame in inspect.stack()]
Michael Walsh2ce1dba2019-02-05 19:29:28 +0000595
596
597def username():
598 r"""
599 Return the username for the current process.
600 """
601
602 username = os.environ.get("USER", "")
603 if username != "":
604 return username
605 user_num = str(os.geteuid())
606 try:
607 username = os.getlogin()
608 except OSError:
609 if user_num == "0":
610 username = "root"
611 else:
612 username = "?"
613
614 return username