blob: a8814dc330513ca2ee73a0f89f81aae2591e8914 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# builtin.py - builtins and utilities definitions for pysh.
2#
3# Copyright 2007 Patrick Mezard
4#
5# This software may be used and distributed according to the terms
6# of the GNU General Public License, incorporated herein by reference.
7
8"""Builtin and internal utilities implementations.
9
10- Beware not to use python interpreter environment as if it were the shell
11environment. For instance, commands working directory must be explicitely handled
12through env['PWD'] instead of relying on python working directory.
13"""
14import errno
15import optparse
16import os
17import re
18import subprocess
19import sys
20import time
21
22def has_subprocess_bug():
23 return getattr(subprocess, 'list2cmdline') and \
24 ( subprocess.list2cmdline(['']) == '' or \
25 subprocess.list2cmdline(['foo|bar']) == 'foo|bar')
26
27# Detect python bug 1634343: "subprocess swallows empty arguments under win32"
28# <http://sourceforge.net/tracker/index.php?func=detail&aid=1634343&group_id=5470&atid=105470>
29# Also detect: "[ 1710802 ] subprocess must escape redirection characters under win32"
30# <http://sourceforge.net/tracker/index.php?func=detail&aid=1710802&group_id=5470&atid=105470>
31if has_subprocess_bug():
32 import subprocess_fix
33 subprocess.list2cmdline = subprocess_fix.list2cmdline
34
35from sherrors import *
36
37class NonExitingParser(optparse.OptionParser):
38 """OptionParser default behaviour upon error is to print the error message and
39 exit. Raise a utility error instead.
40 """
41 def error(self, msg):
42 raise UtilityError(msg)
43
44#-------------------------------------------------------------------------------
45# set special builtin
46#-------------------------------------------------------------------------------
47OPT_SET = NonExitingParser(usage="set - set or unset options and positional parameters")
48OPT_SET.add_option( '-f', action='store_true', dest='has_f', default=False,
49 help='The shell shall disable pathname expansion.')
50OPT_SET.add_option('-e', action='store_true', dest='has_e', default=False,
51 help="""When this option is on, if a simple command fails for any of the \
52 reasons listed in Consequences of Shell Errors or returns an exit status \
53 value >0, and is not part of the compound list following a while, until, \
54 or if keyword, and is not a part of an AND or OR list, and is not a \
55 pipeline preceded by the ! reserved word, then the shell shall immediately \
56 exit.""")
57OPT_SET.add_option('-x', action='store_true', dest='has_x', default=False,
58 help="""The shell shall write to standard error a trace for each command \
59 after it expands the command and before it executes it. It is unspecified \
60 whether the command that turns tracing off is traced.""")
61
62def builtin_set(name, args, interp, env, stdin, stdout, stderr, debugflags):
63 if 'debug-utility' in debugflags:
64 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
65
66 option, args = OPT_SET.parse_args(args)
67 env = interp.get_env()
68
69 if option.has_f:
70 env.set_opt('-f')
71 if option.has_e:
72 env.set_opt('-e')
73 if option.has_x:
74 env.set_opt('-x')
75 return 0
76
77#-------------------------------------------------------------------------------
78# shift special builtin
79#-------------------------------------------------------------------------------
80def builtin_shift(name, args, interp, env, stdin, stdout, stderr, debugflags):
81 if 'debug-utility' in debugflags:
82 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
83
84 params = interp.get_env().get_positional_args()
85 if args:
86 try:
87 n = int(args[0])
88 if n > len(params):
89 raise ValueError()
90 except ValueError:
91 return 1
92 else:
93 n = 1
94
95 params[:n] = []
96 interp.get_env().set_positional_args(params)
97 return 0
98
99#-------------------------------------------------------------------------------
100# export special builtin
101#-------------------------------------------------------------------------------
102OPT_EXPORT = NonExitingParser(usage="set - set or unset options and positional parameters")
103OPT_EXPORT.add_option('-p', action='store_true', dest='has_p', default=False)
104
105def builtin_export(name, args, interp, env, stdin, stdout, stderr, debugflags):
106 if 'debug-utility' in debugflags:
107 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
108
109 option, args = OPT_EXPORT.parse_args(args)
110 if option.has_p:
111 raise NotImplementedError()
112
113 for arg in args:
114 try:
115 name, value = arg.split('=', 1)
116 except ValueError:
117 name, value = arg, None
118 env = interp.get_env().export(name, value)
119
120 return 0
121
122#-------------------------------------------------------------------------------
123# return special builtin
124#-------------------------------------------------------------------------------
125def builtin_return(name, args, interp, env, stdin, stdout, stderr, debugflags):
126 if 'debug-utility' in debugflags:
127 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
128 res = 0
129 if args:
130 try:
131 res = int(args[0])
132 except ValueError:
133 res = 0
134 if not 0<=res<=255:
135 res = 0
136
137 # BUG: should be last executed command exit code
138 raise ReturnSignal(res)
139
140#-------------------------------------------------------------------------------
141# trap special builtin
142#-------------------------------------------------------------------------------
143def builtin_trap(name, args, interp, env, stdin, stdout, stderr, debugflags):
144 if 'debug-utility' in debugflags:
145 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
146 if len(args) < 2:
147 stderr.write('trap: usage: trap [[arg] signal_spec ...]\n')
148 return 2
149
150 action = args[0]
151 for sig in args[1:]:
152 try:
153 env.traps[sig] = action
154 except Exception as e:
155 stderr.write('trap: %s\n' % str(e))
156 return 0
157
158#-------------------------------------------------------------------------------
159# unset special builtin
160#-------------------------------------------------------------------------------
161OPT_UNSET = NonExitingParser("unset - unset values and attributes of variables and functions")
162OPT_UNSET.add_option( '-f', action='store_true', dest='has_f', default=False)
163OPT_UNSET.add_option( '-v', action='store_true', dest='has_v', default=False)
164
165def builtin_unset(name, args, interp, env, stdin, stdout, stderr, debugflags):
166 if 'debug-utility' in debugflags:
167 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
168
169 option, args = OPT_UNSET.parse_args(args)
170
171 status = 0
172 env = interp.get_env()
173 for arg in args:
174 try:
175 if option.has_f:
176 env.remove_function(arg)
177 else:
178 del env[arg]
179 except KeyError:
180 pass
181 except VarAssignmentError:
182 status = 1
183
184 return status
185
186#-------------------------------------------------------------------------------
187# wait special builtin
188#-------------------------------------------------------------------------------
189def builtin_wait(name, args, interp, env, stdin, stdout, stderr, debugflags):
190 if 'debug-utility' in debugflags:
191 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
192
193 return interp.wait([int(arg) for arg in args])
194
195#-------------------------------------------------------------------------------
196# cat utility
197#-------------------------------------------------------------------------------
198def utility_cat(name, args, interp, env, stdin, stdout, stderr, debugflags):
199 if 'debug-utility' in debugflags:
200 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
201
202 if not args:
203 args = ['-']
204
205 status = 0
206 for arg in args:
207 if arg == '-':
208 data = stdin.read()
209 else:
210 path = os.path.join(env['PWD'], arg)
211 try:
212 f = file(path, 'rb')
213 try:
214 data = f.read()
215 finally:
216 f.close()
217 except IOError as e:
218 if e.errno != errno.ENOENT:
219 raise
220 status = 1
221 continue
222 stdout.write(data)
223 stdout.flush()
224 return status
225
226#-------------------------------------------------------------------------------
227# cd utility
228#-------------------------------------------------------------------------------
229OPT_CD = NonExitingParser("cd - change the working directory")
230
231def utility_cd(name, args, interp, env, stdin, stdout, stderr, debugflags):
232 if 'debug-utility' in debugflags:
233 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
234
235 option, args = OPT_CD.parse_args(args)
236 env = interp.get_env()
237
238 directory = None
239 printdir = False
240 if not args:
241 home = env.get('HOME')
242 if home:
243 # Unspecified, do nothing
244 return 0
245 else:
246 directory = home
247 elif len(args)==1:
248 directory = args[0]
249 if directory=='-':
250 if 'OLDPWD' not in env:
251 raise UtilityError("OLDPWD not set")
252 printdir = True
253 directory = env['OLDPWD']
254 else:
255 raise UtilityError("too many arguments")
256
257 curpath = None
258 # Absolute directories will be handled correctly by the os.path.join call.
259 if not directory.startswith('.') and not directory.startswith('..'):
260 cdpaths = env.get('CDPATH', '.').split(';')
261 for cdpath in cdpaths:
262 p = os.path.join(cdpath, directory)
263 if os.path.isdir(p):
264 curpath = p
265 break
266
267 if curpath is None:
268 curpath = directory
269 curpath = os.path.join(env['PWD'], directory)
270
271 env['OLDPWD'] = env['PWD']
272 env['PWD'] = curpath
273 if printdir:
274 stdout.write('%s\n' % curpath)
275 return 0
276
277#-------------------------------------------------------------------------------
278# colon utility
279#-------------------------------------------------------------------------------
280def utility_colon(name, args, interp, env, stdin, stdout, stderr, debugflags):
281 if 'debug-utility' in debugflags:
282 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
283 return 0
284
285#-------------------------------------------------------------------------------
286# echo utility
287#-------------------------------------------------------------------------------
288def utility_echo(name, args, interp, env, stdin, stdout, stderr, debugflags):
289 if 'debug-utility' in debugflags:
290 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
291
292 # Echo only takes arguments, no options. Use printf if you need fancy stuff.
293 output = ' '.join(args) + '\n'
294 stdout.write(output)
295 stdout.flush()
296 return 0
297
298#-------------------------------------------------------------------------------
299# egrep utility
300#-------------------------------------------------------------------------------
301# egrep is usually a shell script.
302# Unfortunately, pysh does not support shell scripts *with arguments* right now,
303# so the redirection is implemented here, assuming grep is available.
304def utility_egrep(name, args, interp, env, stdin, stdout, stderr, debugflags):
305 if 'debug-utility' in debugflags:
306 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
307
308 return run_command('grep', ['-E'] + args, interp, env, stdin, stdout,
309 stderr, debugflags)
310
311#-------------------------------------------------------------------------------
312# env utility
313#-------------------------------------------------------------------------------
314def utility_env(name, args, interp, env, stdin, stdout, stderr, debugflags):
315 if 'debug-utility' in debugflags:
316 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
317
318 if args and args[0]=='-i':
319 raise NotImplementedError('env: -i option is not implemented')
320
321 i = 0
322 for arg in args:
323 if '=' not in arg:
324 break
325 # Update the current environment
326 name, value = arg.split('=', 1)
327 env[name] = value
328 i += 1
329
330 if args[i:]:
331 # Find then execute the specified interpreter
332 utility = env.find_in_path(args[i])
333 if not utility:
334 return 127
335 args[i:i+1] = utility
336 name = args[i]
337 args = args[i+1:]
338 try:
339 return run_command(name, args, interp, env, stdin, stdout, stderr,
340 debugflags)
341 except UtilityError:
342 stderr.write('env: failed to execute %s' % ' '.join([name]+args))
343 return 126
344 else:
345 for pair in env.get_variables().iteritems():
346 stdout.write('%s=%s\n' % pair)
347 return 0
348
349#-------------------------------------------------------------------------------
350# exit utility
351#-------------------------------------------------------------------------------
352def utility_exit(name, args, interp, env, stdin, stdout, stderr, debugflags):
353 if 'debug-utility' in debugflags:
354 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
355
356 res = None
357 if args:
358 try:
359 res = int(args[0])
360 except ValueError:
361 res = None
362 if not 0<=res<=255:
363 res = None
364
365 if res is None:
366 # BUG: should be last executed command exit code
367 res = 0
368
369 raise ExitSignal(res)
370
371#-------------------------------------------------------------------------------
372# fgrep utility
373#-------------------------------------------------------------------------------
374# see egrep
375def utility_fgrep(name, args, interp, env, stdin, stdout, stderr, debugflags):
376 if 'debug-utility' in debugflags:
377 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
378
379 return run_command('grep', ['-F'] + args, interp, env, stdin, stdout,
380 stderr, debugflags)
381
382#-------------------------------------------------------------------------------
383# gunzip utility
384#-------------------------------------------------------------------------------
385# see egrep
386def utility_gunzip(name, args, interp, env, stdin, stdout, stderr, debugflags):
387 if 'debug-utility' in debugflags:
388 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
389
390 return run_command('gzip', ['-d'] + args, interp, env, stdin, stdout,
391 stderr, debugflags)
392
393#-------------------------------------------------------------------------------
394# kill utility
395#-------------------------------------------------------------------------------
396def utility_kill(name, args, interp, env, stdin, stdout, stderr, debugflags):
397 if 'debug-utility' in debugflags:
398 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
399
400 for arg in args:
401 pid = int(arg)
402 status = subprocess.call(['pskill', '/T', str(pid)],
403 shell=True,
404 stdout=subprocess.PIPE,
405 stderr=subprocess.PIPE)
406 # pskill is asynchronous, hence the stupid polling loop
407 while 1:
408 p = subprocess.Popen(['pslist', str(pid)],
409 shell=True,
410 stdout=subprocess.PIPE,
411 stderr=subprocess.STDOUT)
412 output = p.communicate()[0]
413 if ('process %d was not' % pid) in output:
414 break
415 time.sleep(1)
416 return status
417
418#-------------------------------------------------------------------------------
419# mkdir utility
420#-------------------------------------------------------------------------------
421OPT_MKDIR = NonExitingParser("mkdir - make directories.")
422OPT_MKDIR.add_option('-p', action='store_true', dest='has_p', default=False)
423
424def utility_mkdir(name, args, interp, env, stdin, stdout, stderr, debugflags):
425 if 'debug-utility' in debugflags:
426 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
427
428 # TODO: implement umask
429 # TODO: implement proper utility error report
430 option, args = OPT_MKDIR.parse_args(args)
431 for arg in args:
432 path = os.path.join(env['PWD'], arg)
433 if option.has_p:
434 try:
435 os.makedirs(path)
436 except IOError as e:
437 if e.errno != errno.EEXIST:
438 raise
439 else:
440 os.mkdir(path)
441 return 0
442
443#-------------------------------------------------------------------------------
444# netstat utility
445#-------------------------------------------------------------------------------
446def utility_netstat(name, args, interp, env, stdin, stdout, stderr, debugflags):
447 # Do you really expect me to implement netstat ?
448 # This empty form is enough for Mercurial tests since it's
449 # supposed to generate nothing upon success. Faking this test
450 # is not a big deal either.
451 if 'debug-utility' in debugflags:
452 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
453 return 0
454
455#-------------------------------------------------------------------------------
456# pwd utility
457#-------------------------------------------------------------------------------
458OPT_PWD = NonExitingParser("pwd - return working directory name")
459OPT_PWD.add_option('-L', action='store_true', dest='has_L', default=True,
460 help="""If the PWD environment variable contains an absolute pathname of \
461 the current directory that does not contain the filenames dot or dot-dot, \
462 pwd shall write this pathname to standard output. Otherwise, the -L option \
463 shall behave as the -P option.""")
464OPT_PWD.add_option('-P', action='store_true', dest='has_L', default=False,
465 help="""The absolute pathname written shall not contain filenames that, in \
466 the context of the pathname, refer to files of type symbolic link.""")
467
468def utility_pwd(name, args, interp, env, stdin, stdout, stderr, debugflags):
469 if 'debug-utility' in debugflags:
470 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
471
472 option, args = OPT_PWD.parse_args(args)
473 stdout.write('%s\n' % env['PWD'])
474 return 0
475
476#-------------------------------------------------------------------------------
477# printf utility
478#-------------------------------------------------------------------------------
479RE_UNESCAPE = re.compile(r'(\\x[a-zA-Z0-9]{2}|\\[0-7]{1,3}|\\.)')
480
481def utility_printf(name, args, interp, env, stdin, stdout, stderr, debugflags):
482 if 'debug-utility' in debugflags:
483 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
484
485 def replace(m):
486 assert m.group()
487 g = m.group()[1:]
488 if g.startswith('x'):
489 return chr(int(g[1:], 16))
490 if len(g) <= 3 and len([c for c in g if c in '01234567']) == len(g):
491 # Yay, an octal number
492 return chr(int(g, 8))
493 return {
494 'a': '\a',
495 'b': '\b',
496 'f': '\f',
497 'n': '\n',
498 'r': '\r',
499 't': '\t',
500 'v': '\v',
501 '\\': '\\',
502 }.get(g)
503
504 # Convert escape sequences
505 format = re.sub(RE_UNESCAPE, replace, args[0])
506 stdout.write(format % tuple(args[1:]))
507 return 0
508
509#-------------------------------------------------------------------------------
510# true utility
511#-------------------------------------------------------------------------------
512def utility_true(name, args, interp, env, stdin, stdout, stderr, debugflags):
513 if 'debug-utility' in debugflags:
514 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
515 return 0
516
517#-------------------------------------------------------------------------------
518# sed utility
519#-------------------------------------------------------------------------------
520RE_SED = re.compile(r'^s(.).*\1[a-zA-Z]*$')
521
522# cygwin sed fails with some expressions when they do not end with a single space.
523# see unit tests for details. Interestingly, the same expressions works perfectly
524# in cygwin shell.
525def utility_sed(name, args, interp, env, stdin, stdout, stderr, debugflags):
526 if 'debug-utility' in debugflags:
527 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
528
529 # Scan pattern arguments and append a space if necessary
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600530 for i in range(len(args)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500531 if not RE_SED.search(args[i]):
532 continue
533 args[i] = args[i] + ' '
534
535 return run_command(name, args, interp, env, stdin, stdout,
536 stderr, debugflags)
537
538#-------------------------------------------------------------------------------
539# sleep utility
540#-------------------------------------------------------------------------------
541def utility_sleep(name, args, interp, env, stdin, stdout, stderr, debugflags):
542 if 'debug-utility' in debugflags:
543 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
544 time.sleep(int(args[0]))
545 return 0
546
547#-------------------------------------------------------------------------------
548# sort utility
549#-------------------------------------------------------------------------------
550OPT_SORT = NonExitingParser("sort - sort, merge, or sequence check text files")
551
552def utility_sort(name, args, interp, env, stdin, stdout, stderr, debugflags):
553
554 def sort(path):
555 if path == '-':
556 lines = stdin.readlines()
557 else:
558 try:
559 f = file(path)
560 try:
561 lines = f.readlines()
562 finally:
563 f.close()
564 except IOError as e:
565 stderr.write(str(e) + '\n')
566 return 1
567
568 if lines and lines[-1][-1]!='\n':
569 lines[-1] = lines[-1] + '\n'
570 return lines
571
572 if 'debug-utility' in debugflags:
573 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
574
575 option, args = OPT_SORT.parse_args(args)
576 alllines = []
577
578 if len(args)<=0:
579 args += ['-']
580
581 # Load all files lines
582 curdir = os.getcwd()
583 try:
584 os.chdir(env['PWD'])
585 for path in args:
586 alllines += sort(path)
587 finally:
588 os.chdir(curdir)
589
590 alllines.sort()
591 for line in alllines:
592 stdout.write(line)
593 return 0
594
595#-------------------------------------------------------------------------------
596# hg utility
597#-------------------------------------------------------------------------------
598
599hgcommands = [
600 'add',
601 'addremove',
602 'commit', 'ci',
603 'debugrename',
604 'debugwalk',
605 'falabala', # Dummy command used in a mercurial test
606 'incoming',
607 'locate',
608 'pull',
609 'push',
610 'qinit',
611 'remove', 'rm',
612 'rename', 'mv',
613 'revert',
614 'showconfig',
615 'status', 'st',
616 'strip',
617 ]
618
619def rewriteslashes(name, args):
620 # Several hg commands output file paths, rewrite the separators
621 if len(args) > 1 and name.lower().endswith('python') \
622 and args[0].endswith('hg'):
623 for cmd in hgcommands:
624 if cmd in args[1:]:
625 return True
626
627 # svn output contains many paths with OS specific separators.
628 # Normalize these to unix paths.
629 base = os.path.basename(name)
630 if base.startswith('svn'):
631 return True
632
633 return False
634
635def rewritehg(output):
636 if not output:
637 return output
638 # Rewrite os specific messages
639 output = output.replace(': The system cannot find the file specified',
640 ': No such file or directory')
641 output = re.sub(': Access is denied.*$', ': Permission denied', output)
642 output = output.replace(': No connection could be made because the target machine actively refused it',
643 ': Connection refused')
644 return output
645
646
647def run_command(name, args, interp, env, stdin, stdout,
648 stderr, debugflags):
649 # Execute the command
650 if 'debug-utility' in debugflags:
651 print interp.log(' '.join([name, str(args), interp['PWD']]) + '\n')
652
653 hgbin = interp.options().hgbinary
654 ishg = hgbin and ('hg' in name or args and 'hg' in args[0])
655 unixoutput = 'cygwin' in name or ishg
656
657 exec_env = env.get_variables()
658 try:
659 # BUG: comparing file descriptor is clearly not a reliable way to tell
660 # whether they point on the same underlying object. But in pysh limited
661 # scope this is usually right, we do not expect complicated redirections
662 # besides usual 2>&1.
663 # Still there is one case we have but cannot deal with is when stdout
664 # and stderr are redirected *by pysh caller*. This the reason for the
665 # --redirect pysh() option.
666 # Now, we want to know they are the same because we sometimes need to
667 # transform the command output, mostly remove CR-LF to ensure that
668 # command output is unix-like. Cygwin utilies are a special case because
669 # they explicitely set their output streams to binary mode, so we have
670 # nothing to do. For all others commands, we have to guess whether they
671 # are sending text data, in which case the transformation must be done.
672 # Again, the NUL character test is unreliable but should be enough for
673 # hg tests.
674 redirected = stdout.fileno()==stderr.fileno()
675 if not redirected:
676 p = subprocess.Popen([name] + args, cwd=env['PWD'], env=exec_env,
677 stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
678 else:
679 p = subprocess.Popen([name] + args, cwd=env['PWD'], env=exec_env,
680 stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
681 out, err = p.communicate()
682 except WindowsError as e:
683 raise UtilityError(str(e))
684
685 if not unixoutput:
686 def encode(s):
687 if '\0' in s:
688 return s
689 return s.replace('\r\n', '\n')
690 else:
691 encode = lambda s: s
692
693 if rewriteslashes(name, args):
694 encode1_ = encode
695 def encode(s):
696 s = encode1_(s)
697 s = s.replace('\\\\', '\\')
698 s = s.replace('\\', '/')
699 return s
700
701 if ishg:
702 encode2_ = encode
703 def encode(s):
704 return rewritehg(encode2_(s))
705
706 stdout.write(encode(out))
707 if not redirected:
708 stderr.write(encode(err))
709 return p.returncode
710