blob: db7ecf4611ae14afd1b909f173ee621aa690e3ad [file] [log] [blame]
Michael Walshde791732016-09-06 14:25:24 -05001#!/usr/bin/env python
2
Michael Walsh7423c012016-10-04 10:27:21 -05003r"""
4This module provides many valuable functions such as my_parm_file.
5"""
Michael Walshde791732016-09-06 14:25:24 -05006
7# sys and os are needed to get the program dir path and program name.
8import sys
Michael Walsheaa16852017-09-19 16:30:43 -05009import errno
Michael Walshde791732016-09-06 14:25:24 -050010import os
Michael Walshdece16c2018-08-07 15:01:05 -050011import collections
Michael Walsh6aa69802018-09-21 16:38:34 -050012import json
Michael Walshc9cb9722018-10-01 17:56:20 -050013import time
Michael Walshdece16c2018-08-07 15:01:05 -050014try:
15 import ConfigParser
16except ImportError:
17 import configparser
George Keishing3b7115a2018-08-02 10:48:17 -050018try:
Michael Walsh61f5e8f2018-08-03 11:16:00 -050019 import StringIO
George Keishing3b7115a2018-08-02 10:48:17 -050020except ImportError:
Michael Walshdece16c2018-08-07 15:01:05 -050021 import io
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060022import re
Michael Walsh1c85bab2017-05-04 14:29:24 -050023import socket
Michael Walsh3b621fe2018-07-24 16:27:53 -050024import tempfile
25try:
26 import psutil
27 psutil_imported = True
28except ImportError:
29 psutil_imported = False
Michael Walshde791732016-09-06 14:25:24 -050030
Michael Walsh7423c012016-10-04 10:27:21 -050031import gen_print as gp
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060032import gen_cmd as gc
33
Michael Walsh93a09f22017-11-13 15:34:46 -060034robot_env = gp.robot_env
35if robot_env:
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060036 from robot.libraries.BuiltIn import BuiltIn
Michael Walshdece16c2018-08-07 15:01:05 -050037 from robot.utils import DotDict
Michael Walsh7423c012016-10-04 10:27:21 -050038
Michael Walshde791732016-09-06 14:25:24 -050039
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060040def add_trailing_slash(dir_path):
Michael Walsh7db77942017-01-10 11:37:06 -060041 r"""
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060042 Add a trailing slash to the directory path if it doesn't already have one
43 and return it.
Michael Walsh7db77942017-01-10 11:37:06 -060044
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060045 Description of arguments:
46 dir_path A directory path.
Michael Walsh7db77942017-01-10 11:37:06 -060047 """
48
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060049 return os.path.normpath(dir_path) + os.path.sep
50
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060051
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060052def which(file_path):
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060053 r"""
54 Find the full path of an executable file and return it.
55
56 The PATH environment variable dictates the results of this function.
57
58 Description of arguments:
59 file_path The relative file path (e.g. "my_file" or
60 "lib/my_file").
61 """
62
63 shell_rc, out_buf = gc.cmd_fnc_u("which " + file_path, quiet=1,
64 print_output=0, show_err=0)
65 if shell_rc != 0:
66 error_message = "Failed to find complete path for file \"" +\
67 file_path + "\".\n"
68 error_message += gp.sprint_var(shell_rc, 1)
69 error_message += out_buf
70 if robot_env:
71 BuiltIn().fail(gp.sprint_error(error_message))
72 else:
73 gp.print_error_report(error_message)
74 return False
75
76 file_path = out_buf.rstrip("\n")
77
78 return file_path
79
Michael Walsh0f2ea5f2017-02-20 15:55:00 -060080
Michael Walshf7400f32018-09-26 17:13:43 -050081def add_path(new_path,
82 path,
83 position=0):
84 r"""
85 Add new_path to path, provided that path doesn't already contain new_path,
86 and return the result.
87
88 Example:
89 If PATH has a value of "/bin/user:/lib/user". The following code:
90
91 PATH = add_path("/tmp/new_path", PATH)
92
93 will change PATH to "/tmp/new_path:/bin/user:/lib/user".
94
95 Description of argument(s):
96 new_path The path to be added. This function will
97 strip the trailing slash.
98 path The path value to which the new_path
99 should be added.
100 position The position in path where the new_path
101 should be added. 0 means it should be
102 added to the beginning, 1 means add it as
103 the 2nd item, etc. sys.maxsize means it
104 should be added to the end.
105 """
106
107 path_list = list(filter(None, path.split(":")))
108 new_path = new_path.rstrip("/")
109 if new_path not in path_list:
110 path_list.insert(int(position), new_path)
111 return ":".join(path_list)
112
113
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600114def dft(value, default):
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600115 r"""
116 Return default if value is None. Otherwise, return value.
117
118 This is really just shorthand as shown below.
119
120 dft(value, default)
121
122 vs
123
124 default if value is None else value
125
126 Description of arguments:
127 value The value to be returned.
128 default The default value to return if value is
129 None.
130 """
131
132 return default if value is None else value
133
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600134
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600135def get_mod_global(var_name,
136 default=None,
137 mod_name="__main__"):
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600138 r"""
139 Get module global variable value and return it.
140
141 If we are running in a robot environment, the behavior will default to
142 calling get_variable_value.
143
144 Description of arguments:
145 var_name The name of the variable whose value is
146 sought.
147 default The value to return if the global does not
148 exist.
149 mod_name The name of the module containing the
150 global variable.
151 """
152
153 if robot_env:
154 return BuiltIn().get_variable_value("${" + var_name + "}", default)
155
156 try:
157 module = sys.modules[mod_name]
158 except KeyError:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500159 gp.print_error_report("Programmer error - The mod_name passed to"
160 + " this function is invalid:\n"
161 + gp.sprint_var(mod_name))
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600162 raise ValueError('Programmer error.')
163
164 if default is None:
165 return getattr(module, var_name)
166 else:
167 return getattr(module, var_name, default)
168
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600169
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600170def global_default(var_value,
171 default=0):
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600172 r"""
173 If var_value is not None, return it. Otherwise, return the global
174 variable of the same name, if it exists. If not, return default.
175
176 This is meant for use by functions needing help assigning dynamic default
177 values to their parms. Example:
178
179 def func1(parm1=None):
180
181 parm1 = global_default(parm1, 0)
182
183 Description of arguments:
184 var_value The value being evaluated.
185 default The value to be returned if var_value is
186 None AND the global variable of the same
187 name does not exist.
188 """
189
190 var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
191
192 return dft(var_value, get_mod_global(var_name, 0))
Michael Walsh7db77942017-01-10 11:37:06 -0600193
Michael Walsh7db77942017-01-10 11:37:06 -0600194
Michael Walsh7db77942017-01-10 11:37:06 -0600195def set_mod_global(var_value,
196 mod_name="__main__",
197 var_name=None):
Michael Walsh7db77942017-01-10 11:37:06 -0600198 r"""
199 Set a global variable for a given module.
200
201 Description of arguments:
202 var_value The value to set in the variable.
203 mod_name The name of the module whose variable is
204 to be set.
205 var_name The name of the variable to set. This
206 defaults to the name of the variable used
207 for var_value when calling this function.
208 """
209
210 try:
211 module = sys.modules[mod_name]
212 except KeyError:
Joy Onyerikwu004ad3c2018-06-11 16:29:56 -0500213 gp.print_error_report("Programmer error - The mod_name passed to"
214 + " this function is invalid:\n"
215 + gp.sprint_var(mod_name))
Michael Walsh7db77942017-01-10 11:37:06 -0600216 raise ValueError('Programmer error.')
217
218 if var_name is None:
219 var_name = gp.get_arg_name(None, 1, 2)
220
221 setattr(module, var_name, var_value)
222
Michael Walsh7db77942017-01-10 11:37:06 -0600223
Michael Walshde791732016-09-06 14:25:24 -0500224def my_parm_file(prop_file_path):
Michael Walshde791732016-09-06 14:25:24 -0500225 r"""
226 Read a properties file, put the keys/values into a dictionary and return
227 the dictionary.
228
229 The properties file must have the following format:
230 var_name<= or :>var_value
231 Comment lines (those beginning with a "#") and blank lines are allowed and
232 will be ignored. Leading and trailing single or double quotes will be
233 stripped from the value. E.g.
234 var1="This one"
235 Quotes are stripped so the resulting value for var1 is:
236 This one
237
238 Description of arguments:
Michael Walsh7423c012016-10-04 10:27:21 -0500239 prop_file_path The caller should pass the path to the
240 properties file.
Michael Walshde791732016-09-06 14:25:24 -0500241 """
242
243 # ConfigParser expects at least one section header in the file (or you
244 # get ConfigParser.MissingSectionHeaderError). Properties files don't
245 # need those so I'll write a dummy section header.
246
Michael Walshdece16c2018-08-07 15:01:05 -0500247 try:
248 string_file = StringIO.StringIO()
249 except NameError:
250 string_file = io.StringIO()
251
Michael Walshde791732016-09-06 14:25:24 -0500252 # Write the dummy section header to the string file.
253 string_file.write('[dummysection]\n')
254 # Write the entire contents of the properties file to the string file.
255 string_file.write(open(prop_file_path).read())
256 # Rewind the string file.
257 string_file.seek(0, os.SEEK_SET)
258
259 # Create the ConfigParser object.
Michael Walshdece16c2018-08-07 15:01:05 -0500260 try:
261 config_parser = ConfigParser.ConfigParser()
262 except NameError:
George Keishing36efbc02018-12-12 10:18:23 -0600263 config_parser = configparser.ConfigParser(strict=False)
Michael Walshde791732016-09-06 14:25:24 -0500264 # Make the property names case-sensitive.
265 config_parser.optionxform = str
266 # Read the properties from the string file.
267 config_parser.readfp(string_file)
268 # Return the properties as a dictionary.
Michael Walshdece16c2018-08-07 15:01:05 -0500269 if robot_env:
270 return DotDict(config_parser.items('dummysection'))
271 else:
272 return collections.OrderedDict(config_parser.items('dummysection'))
Michael Walshde791732016-09-06 14:25:24 -0500273
Michael Walsh7423c012016-10-04 10:27:21 -0500274
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600275def file_to_list(file_path,
276 newlines=0,
277 comments=1,
278 trim=0):
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600279 r"""
280 Return the contents of a file as a list. Each element of the resulting
281 list is one line from the file.
282
283 Description of arguments:
284 file_path The path to the file (relative or
285 absolute).
286 newlines Include newlines from the file in the
287 results.
288 comments Include comment lines and blank lines in
289 the results. Comment lines are any that
290 begin with 0 or more spaces followed by
291 the pound sign ("#").
292 trim Trim white space from the beginning and
293 end of each line.
294 """
295
296 lines = []
297 file = open(file_path)
298 for line in file:
299 if not comments:
300 if re.match(r"[ ]*#|^$", line):
301 continue
302 if not newlines:
303 line = line.rstrip("\n")
304 if trim:
305 line = line.strip()
306 lines.append(line)
Michael Walsh1383f352018-09-27 16:25:54 -0500307 file.close()
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600308
309 return lines
310
Michael Walsh0f2ea5f2017-02-20 15:55:00 -0600311
Michael Walsh1383f352018-09-27 16:25:54 -0500312def file_to_str(*args, **kwargs):
313 r"""
314 Return the contents of a file as a string.
315
316 Description of arguments:
317 See file_to_list defined above for description of arguments.
318 """
319
320 return '\n'.join(file_to_list(*args, **kwargs))
321
322
Michael Walsh7423c012016-10-04 10:27:21 -0500323def return_path_list():
Michael Walsh7423c012016-10-04 10:27:21 -0500324 r"""
325 This function will split the PATH environment variable into a PATH_LIST
326 and return it. Each element in the list will be normalized and have a
327 trailing slash added.
328 """
329
330 PATH_LIST = os.environ['PATH'].split(":")
331 PATH_LIST = [os.path.normpath(path) + os.sep for path in PATH_LIST]
332
333 return PATH_LIST
334
Michael Walsh7db77942017-01-10 11:37:06 -0600335
Michael Walsh9fac55c2017-09-29 16:53:56 -0500336def escape_bash_quotes(buffer):
Michael Walsh9fac55c2017-09-29 16:53:56 -0500337 r"""
338 Escape quotes in string and return it.
339
340 The escape style implemented will be for use on the bash command line.
341
342 Example:
343 That's all.
344
345 Result:
346 That'\''s all.
347
348 The result may then be single quoted on a bash command. Example:
349
350 echo 'That'\''s all.'
351
352 Description of argument(s):
353 buffer The string whose quotes are to be escaped.
354 """
355
356 return re.sub("\'", "\'\\\'\'", buffer)
357
358
Michael Walsh7db77942017-01-10 11:37:06 -0600359def quote_bash_parm(parm):
Michael Walsh7db77942017-01-10 11:37:06 -0600360 r"""
361 Return the bash command line parm with single quotes if they are needed.
362
363 Description of arguments:
364 parm The string to be quoted.
365 """
366
367 # If any of these characters are found in the parm string, then the
368 # string should be quoted. This list is by no means complete and should
369 # be expanded as needed by the developer of this function.
370 bash_special_chars = set(' $')
371
372 if any((char in bash_special_chars) for char in parm):
373 return "'" + parm + "'"
374
375 return parm
376
Michael Walsh1c85bab2017-05-04 14:29:24 -0500377
Michael Walsh74427232018-08-31 12:54:24 -0500378def get_host_name_ip(host=None,
Michael Walshf74b3e42018-01-10 11:11:54 -0600379 short_name=0):
Michael Walsh1c85bab2017-05-04 14:29:24 -0500380 r"""
381 Get the host name and the IP address for the given host and return them as
382 a tuple.
383
384 Description of argument(s):
Michael Walshd1b6c702017-05-30 17:54:30 -0500385 host The host name or IP address to be obtained.
Michael Walshf74b3e42018-01-10 11:11:54 -0600386 short_name Include the short host name in the
387 returned tuple, i.e. return host, ip and
388 short_host.
Michael Walsh1c85bab2017-05-04 14:29:24 -0500389 """
390
Michael Walsh74427232018-08-31 12:54:24 -0500391 host = dft(host, socket.gethostname())
Michael Walshf74b3e42018-01-10 11:11:54 -0600392 host_name = socket.getfqdn(host)
Michael Walshd1b6c702017-05-30 17:54:30 -0500393 try:
394 host_ip = socket.gethostbyname(host)
395 except socket.gaierror as my_gaierror:
396 message = "Unable to obtain the host name for the following host:" +\
397 "\n" + gp.sprint_var(host)
398 gp.print_error_report(message)
399 raise my_gaierror
Michael Walsh1c85bab2017-05-04 14:29:24 -0500400
Michael Walshf74b3e42018-01-10 11:11:54 -0600401 if short_name:
402 host_short_name = host_name.split(".")[0]
403 return host_name, host_ip, host_short_name
404 else:
405 return host_name, host_ip
Michael Walsh1c85bab2017-05-04 14:29:24 -0500406
Michael Walsheaa16852017-09-19 16:30:43 -0500407
408def pid_active(pid):
Michael Walsheaa16852017-09-19 16:30:43 -0500409 r"""
410 Return true if pid represents an active pid and false otherwise.
411
412 Description of argument(s):
413 pid The pid whose status is being sought.
414 """
415
416 try:
417 os.kill(int(pid), 0)
418 except OSError as err:
419 if err.errno == errno.ESRCH:
420 # ESRCH == No such process
421 return False
422 elif err.errno == errno.EPERM:
423 # EPERM clearly means there's a process to deny access to
424 return True
425 else:
426 # According to "man 2 kill" possible error values are
427 # (EINVAL, EPERM, ESRCH)
428 raise
429
430 return True
Michael Walsh112c3592018-06-01 14:15:58 -0500431
432
433def to_signed(number,
Michael Walshdece16c2018-08-07 15:01:05 -0500434 bit_width=None):
Michael Walsh112c3592018-06-01 14:15:58 -0500435 r"""
436 Convert number to a signed number and return the result.
437
438 Examples:
439
440 With the following code:
441
442 var1 = 0xfffffffffffffff1
443 print_var(var1)
444 print_var(var1, 1)
445 var1 = to_signed(var1)
446 print_var(var1)
447 print_var(var1, 1)
448
449 The following is written to stdout:
450 var1: 18446744073709551601
451 var1: 0x00000000fffffffffffffff1
452 var1: -15
453 var1: 0xfffffffffffffff1
454
455 The same code but with var1 set to 0x000000000000007f produces the
456 following:
457 var1: 127
458 var1: 0x000000000000007f
459 var1: 127
460 var1: 0x000000000000007f
461
462 Description of argument(s):
463 number The number to be converted.
464 bit_width The number of bits that defines a complete
465 hex value. Typically, this would be a
466 multiple of 32.
467 """
468
Michael Walshdece16c2018-08-07 15:01:05 -0500469 if bit_width is None:
470 try:
471 bit_width = gp.bit_length(long(sys.maxsize)) + 1
472 except NameError:
473 bit_width = gp.bit_length(int(sys.maxsize)) + 1
474
Michael Walsh112c3592018-06-01 14:15:58 -0500475 if number < 0:
476 return number
477 neg_bit_mask = 2**(bit_width - 1)
478 if number & neg_bit_mask:
479 return ((2**bit_width) - number) * -1
480 else:
481 return number
Michael Walsh3b621fe2018-07-24 16:27:53 -0500482
483
484def get_child_pids(quiet=1):
485
486 r"""
487 Get and return a list of pids representing all first-generation processes
488 that are the children of the current process.
489
490 Example:
491
492 children = get_child_pids()
493 print_var(children)
494
495 Output:
496 children:
497 children[0]: 9123
498
499 Description of argument(s):
500 quiet Display output to stdout detailing how
501 this child pids are obtained.
502 """
503
504 if psutil_imported:
505 # If "import psutil" worked, find child pids using psutil.
506 current_process = psutil.Process()
507 return [x.pid for x in current_process.children(recursive=False)]
508 else:
509 # Otherwise, find child pids using shell commands.
510 print_output = not quiet
511
512 ps_cmd_buf = "ps --no-headers --ppid " + str(os.getpid()) +\
513 " -o pid,args"
514 # Route the output of ps to a temporary file for later grepping.
515 # Avoid using " | grep" in the ps command string because it creates
516 # yet another process which is of no interest to the caller.
517 temp = tempfile.NamedTemporaryFile()
518 temp_file_path = temp.name
519 gc.shell_cmd(ps_cmd_buf + " > " + temp_file_path,
520 print_output=print_output)
521 # Sample contents of the temporary file:
522 # 30703 sleep 2
523 # 30795 /bin/bash -c ps --no-headers --ppid 30672 -o pid,args >
524 # /tmp/tmpqqorWY
525 # Use egrep to exclude the "ps" process itself from the results
526 # collected with the prior shell_cmd invocation. Only the other
527 # children are of interest to the caller. Use cut on the grep results
528 # to obtain only the pid column.
529 rc, output = \
530 gc.shell_cmd("egrep -v '" + re.escape(ps_cmd_buf) + "' "
531 + temp_file_path + " | cut -c1-5",
532 print_output=print_output)
533 # Split the output buffer by line into a list. Strip each element of
534 # extra spaces and convert each element to an integer.
535 return map(int, map(str.strip, filter(None, output.split("\n"))))
Michael Walsh6aa69802018-09-21 16:38:34 -0500536
537
538def json_loads_multiple(buffer):
539 r"""
540 Convert the contents of the buffer to a JSON array, run json.loads() on it
541 and return the result.
542
543 The buffer is expected to contain one or more JSON objects.
544
545 Description of argument(s):
546 buffer A string containing several JSON objects.
547 """
548
549 # Any line consisting of just "}", which indicates the end of an object,
550 # should have a comma appended.
551 regex = "([\\r\\n])[\\}]([\\r\\n])"
552 buffer = re.sub(regex, "\\1},\\2", buffer, 1)
553 # Remove the comma from after the final object and place the whole buffer
554 # inside square brackets.
555 buffer = "[" + re.sub(",([\r\n])$", "\\1}", buffer, 1) + "]"
556 if gp.robot_env:
557 return json.loads(buffer, object_pairs_hook=DotDict)
558 else:
559 return json.loads(buffer, object_pairs_hook=collections.OrderedDict)
Michael Walshc9cb9722018-10-01 17:56:20 -0500560
561
562def file_date_time_stamp():
563 r"""
564 Return a date/time stamp in the following format: yymmdd.HHMMSS
565
566 This value is suitable for including in file names. Example
567 file1.181001.171716.status
568 """
569
570 return time.strftime("%y%m%d.%H%M%S", time.localtime(time.time()))