Squashed 'yocto-poky/' content from commit ea562de
git-subtree-dir: yocto-poky
git-subtree-split: ea562de57590c966cd5a75fda8defecd397e6436
diff --git a/bitbake/lib/bb/pysh/interp.py b/bitbake/lib/bb/pysh/interp.py
new file mode 100644
index 0000000..25d8c92
--- /dev/null
+++ b/bitbake/lib/bb/pysh/interp.py
@@ -0,0 +1,1367 @@
+# interp.py - shell interpreter for pysh.
+#
+# Copyright 2007 Patrick Mezard
+#
+# This software may be used and distributed according to the terms
+# of the GNU General Public License, incorporated herein by reference.
+
+"""Implement the shell interpreter.
+
+Most references are made to "The Open Group Base Specifications Issue 6".
+<http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html>
+"""
+# TODO: document the fact input streams must implement fileno() so Popen will work correctly.
+# it requires non-stdin stream to be implemented as files. Still to be tested...
+# DOC: pathsep is used in PATH instead of ':'. Clearly, there are path syntax issues here.
+# TODO: stop command execution upon error.
+# TODO: sort out the filename/io_number mess. It should be possible to use filenames only.
+# TODO: review subshell implementation
+# TODO: test environment cloning for non-special builtins
+# TODO: set -x should not rebuild commands from tokens, assignments/redirections are lost
+# TODO: unit test for variable assignment
+# TODO: test error management wrt error type/utility type
+# TODO: test for binary output everywhere
+# BUG: debug-parsing does not pass log file to PLY. Maybe a PLY upgrade is necessary.
+import base64
+import cPickle as pickle
+import errno
+import glob
+import os
+import re
+import subprocess
+import sys
+import tempfile
+
+try:
+ s = set()
+ del s
+except NameError:
+ from Set import Set as set
+
+import builtin
+from sherrors import *
+import pyshlex
+import pyshyacc
+
+def mappend(func, *args, **kargs):
+ """Like map but assume func returns a list. Returned lists are merged into
+ a single one.
+ """
+ return reduce(lambda a,b: a+b, map(func, *args, **kargs), [])
+
+class FileWrapper:
+ """File object wrapper to ease debugging.
+
+ Allow mode checking and implement file duplication through a simple
+ reference counting scheme. Not sure the latter is really useful since
+ only real file descriptors can be used.
+ """
+ def __init__(self, mode, file, close=True):
+ if mode not in ('r', 'w', 'a'):
+ raise IOError('invalid mode: %s' % mode)
+ self._mode = mode
+ self._close = close
+ if isinstance(file, FileWrapper):
+ if file._refcount[0] <= 0:
+ raise IOError(0, 'Error')
+ self._refcount = file._refcount
+ self._refcount[0] += 1
+ self._file = file._file
+ else:
+ self._refcount = [1]
+ self._file = file
+
+ def dup(self):
+ return FileWrapper(self._mode, self, self._close)
+
+ def fileno(self):
+ """fileno() should be only necessary for input streams."""
+ return self._file.fileno()
+
+ def read(self, size=-1):
+ if self._mode!='r':
+ raise IOError(0, 'Error')
+ return self._file.read(size)
+
+ def readlines(self, *args, **kwargs):
+ return self._file.readlines(*args, **kwargs)
+
+ def write(self, s):
+ if self._mode not in ('w', 'a'):
+ raise IOError(0, 'Error')
+ return self._file.write(s)
+
+ def flush(self):
+ self._file.flush()
+
+ def close(self):
+ if not self._refcount:
+ return
+ assert self._refcount[0] > 0
+
+ self._refcount[0] -= 1
+ if self._refcount[0] == 0:
+ self._mode = 'c'
+ if self._close:
+ self._file.close()
+ self._refcount = None
+
+ def mode(self):
+ return self._mode
+
+ def __getattr__(self, name):
+ if name == 'name':
+ self.name = getattr(self._file, name)
+ return self.name
+ else:
+ raise AttributeError(name)
+
+ def __del__(self):
+ self.close()
+
+
+def win32_open_devnull(mode):
+ return open('NUL', mode)
+
+
+class Redirections:
+ """Stores open files and their mapping to pseudo-sh file descriptor.
+ """
+ # BUG: redirections are not handled correctly: 1>&3 2>&3 3>&4 does
+ # not make 1 to redirect to 4
+ def __init__(self, stdin=None, stdout=None, stderr=None):
+ self._descriptors = {}
+ if stdin is not None:
+ self._add_descriptor(0, stdin)
+ if stdout is not None:
+ self._add_descriptor(1, stdout)
+ if stderr is not None:
+ self._add_descriptor(2, stderr)
+
+ def add_here_document(self, interp, name, content, io_number=None):
+ if io_number is None:
+ io_number = 0
+
+ if name==pyshlex.unquote_wordtree(name):
+ content = interp.expand_here_document(('TOKEN', content))
+
+ # Write document content in a temporary file
+ tmp = tempfile.TemporaryFile()
+ try:
+ tmp.write(content)
+ tmp.flush()
+ tmp.seek(0)
+ self._add_descriptor(io_number, FileWrapper('r', tmp))
+ except:
+ tmp.close()
+ raise
+
+ def add(self, interp, op, filename, io_number=None):
+ if op not in ('<', '>', '>|', '>>', '>&'):
+ # TODO: add descriptor duplication and here_documents
+ raise RedirectionError('Unsupported redirection operator "%s"' % op)
+
+ if io_number is not None:
+ io_number = int(io_number)
+
+ if (op == '>&' and filename.isdigit()) or filename=='-':
+ # No expansion for file descriptors, quote them if you want a filename
+ fullname = filename
+ else:
+ if filename.startswith('/'):
+ # TODO: win32 kludge
+ if filename=='/dev/null':
+ fullname = 'NUL'
+ else:
+ # TODO: handle absolute pathnames, they are unlikely to exist on the
+ # current platform (win32 for instance).
+ raise NotImplementedError()
+ else:
+ fullname = interp.expand_redirection(('TOKEN', filename))
+ if not fullname:
+ raise RedirectionError('%s: ambiguous redirect' % filename)
+ # Build absolute path based on PWD
+ fullname = os.path.join(interp.get_env()['PWD'], fullname)
+
+ if op=='<':
+ return self._add_input_redirection(interp, fullname, io_number)
+ elif op in ('>', '>|'):
+ clobber = ('>|'==op)
+ return self._add_output_redirection(interp, fullname, io_number, clobber)
+ elif op=='>>':
+ return self._add_output_appending(interp, fullname, io_number)
+ elif op=='>&':
+ return self._dup_output_descriptor(fullname, io_number)
+
+ def close(self):
+ if self._descriptors is not None:
+ for desc in self._descriptors.itervalues():
+ desc.flush()
+ desc.close()
+ self._descriptors = None
+
+ def stdin(self):
+ return self._descriptors[0]
+
+ def stdout(self):
+ return self._descriptors[1]
+
+ def stderr(self):
+ return self._descriptors[2]
+
+ def clone(self):
+ clone = Redirections()
+ for desc, fileobj in self._descriptors.iteritems():
+ clone._descriptors[desc] = fileobj.dup()
+ return clone
+
+ def _add_output_redirection(self, interp, filename, io_number, clobber):
+ if io_number is None:
+ # io_number default to standard output
+ io_number = 1
+
+ if not clobber and interp.get_env().has_opt('-C') and os.path.isfile(filename):
+ # File already exist in no-clobber mode, bail out
+ raise RedirectionError('File "%s" already exists' % filename)
+
+ # Open and register
+ self._add_file_descriptor(io_number, filename, 'w')
+
+ def _add_output_appending(self, interp, filename, io_number):
+ if io_number is None:
+ io_number = 1
+ self._add_file_descriptor(io_number, filename, 'a')
+
+ def _add_input_redirection(self, interp, filename, io_number):
+ if io_number is None:
+ io_number = 0
+ self._add_file_descriptor(io_number, filename, 'r')
+
+ def _add_file_descriptor(self, io_number, filename, mode):
+ try:
+ if filename.startswith('/'):
+ if filename=='/dev/null':
+ f = win32_open_devnull(mode+'b')
+ else:
+ # TODO: handle absolute pathnames, they are unlikely to exist on the
+ # current platform (win32 for instance).
+ raise NotImplementedError('cannot open absolute path %s' % repr(filename))
+ else:
+ f = file(filename, mode+'b')
+ except IOError as e:
+ raise RedirectionError(str(e))
+
+ wrapper = None
+ try:
+ wrapper = FileWrapper(mode, f)
+ f = None
+ self._add_descriptor(io_number, wrapper)
+ except:
+ if f: f.close()
+ if wrapper: wrapper.close()
+ raise
+
+ def _dup_output_descriptor(self, source_fd, dest_fd):
+ if source_fd is None:
+ source_fd = 1
+ self._dup_file_descriptor(source_fd, dest_fd, 'w')
+
+ def _dup_file_descriptor(self, source_fd, dest_fd, mode):
+ source_fd = int(source_fd)
+ if source_fd not in self._descriptors:
+ raise RedirectionError('"%s" is not a valid file descriptor' % str(source_fd))
+ source = self._descriptors[source_fd]
+
+ if source.mode()!=mode:
+ raise RedirectionError('Descriptor %s cannot be duplicated in mode "%s"' % (str(source), mode))
+
+ if dest_fd=='-':
+ # Close the source descriptor
+ del self._descriptors[source_fd]
+ source.close()
+ else:
+ dest_fd = int(dest_fd)
+ if dest_fd not in self._descriptors:
+ raise RedirectionError('Cannot replace file descriptor %s' % str(dest_fd))
+
+ dest = self._descriptors[dest_fd]
+ if dest.mode()!=mode:
+ raise RedirectionError('Descriptor %s cannot be cannot be redirected in mode "%s"' % (str(dest), mode))
+
+ self._descriptors[dest_fd] = source.dup()
+ dest.close()
+
+ def _add_descriptor(self, io_number, file):
+ io_number = int(io_number)
+
+ if io_number in self._descriptors:
+ # Close the current descriptor
+ d = self._descriptors[io_number]
+ del self._descriptors[io_number]
+ d.close()
+
+ self._descriptors[io_number] = file
+
+ def __str__(self):
+ names = [('%d=%r' % (k, getattr(v, 'name', None))) for k,v
+ in self._descriptors.iteritems()]
+ names = ','.join(names)
+ return 'Redirections(%s)' % names
+
+ def __del__(self):
+ self.close()
+
+def cygwin_to_windows_path(path):
+ """Turn /cygdrive/c/foo into c:/foo, or return path if it
+ is not a cygwin path.
+ """
+ if not path.startswith('/cygdrive/'):
+ return path
+ path = path[len('/cygdrive/'):]
+ path = path[:1] + ':' + path[1:]
+ return path
+
+def win32_to_unix_path(path):
+ if path is not None:
+ path = path.replace('\\', '/')
+ return path
+
+_RE_SHEBANG = re.compile(r'^\#!\s?([^\s]+)(?:\s([^\s]+))?')
+_SHEBANG_CMDS = {
+ '/usr/bin/env': 'env',
+ '/bin/sh': 'pysh',
+ 'python': 'python',
+}
+
+def resolve_shebang(path, ignoreshell=False):
+ """Return a list of arguments as shebang interpreter call or an empty list
+ if path does not refer to an executable script.
+ See <http://www.opengroup.org/austin/docs/austin_51r2.txt>.
+
+ ignoreshell - set to True to ignore sh shebangs. Return an empty list instead.
+ """
+ try:
+ f = file(path)
+ try:
+ # At most 80 characters in the first line
+ header = f.read(80).splitlines()[0]
+ finally:
+ f.close()
+
+ m = _RE_SHEBANG.search(header)
+ if not m:
+ return []
+ cmd, arg = m.group(1,2)
+ if os.path.isfile(cmd):
+ # Keep this one, the hg script for instance contains a weird windows
+ # shebang referencing the current python install.
+ cmdfile = os.path.basename(cmd).lower()
+ if cmdfile == 'python.exe':
+ cmd = 'python'
+ pass
+ elif cmd not in _SHEBANG_CMDS:
+ raise CommandNotFound('Unknown interpreter "%s" referenced in '\
+ 'shebang' % header)
+ cmd = _SHEBANG_CMDS.get(cmd)
+ if cmd is None or (ignoreshell and cmd == 'pysh'):
+ return []
+ if arg is None:
+ return [cmd, win32_to_unix_path(path)]
+ return [cmd, arg, win32_to_unix_path(path)]
+ except IOError as e:
+ if e.errno!=errno.ENOENT and \
+ (e.errno!=errno.EPERM and not os.path.isdir(path)): # Opening a directory raises EPERM
+ raise
+ return []
+
+def win32_find_in_path(name, path):
+ if isinstance(path, str):
+ path = path.split(os.pathsep)
+
+ exts = os.environ.get('PATHEXT', '').lower().split(os.pathsep)
+ for p in path:
+ p_name = os.path.join(p, name)
+
+ prefix = resolve_shebang(p_name)
+ if prefix:
+ return prefix
+
+ for ext in exts:
+ p_name_ext = p_name + ext
+ if os.path.exists(p_name_ext):
+ return [win32_to_unix_path(p_name_ext)]
+ return []
+
+class Traps(dict):
+ def __setitem__(self, key, value):
+ if key not in ('EXIT',):
+ raise NotImplementedError()
+ super(Traps, self).__setitem__(key, value)
+
+# IFS white spaces character class
+_IFS_WHITESPACES = (' ', '\t', '\n')
+
+class Environment:
+ """Environment holds environment variables, export table, function
+ definitions and whatever is defined in 2.12 "Shell Execution Environment",
+ redirection excepted.
+ """
+ def __init__(self, pwd):
+ self._opt = set() #Shell options
+
+ self._functions = {}
+ self._env = {'?': '0', '#': '0'}
+ self._exported = set([
+ 'HOME', 'IFS', 'PATH'
+ ])
+
+ # Set environment vars with side-effects
+ self._ifs_ws = None # Set of IFS whitespace characters
+ self._ifs_re = None # Regular expression used to split between words using IFS classes
+ self['IFS'] = ''.join(_IFS_WHITESPACES) #Default environment values
+ self['PWD'] = pwd
+ self.traps = Traps()
+
+ def clone(self, subshell=False):
+ env = Environment(self['PWD'])
+ env._opt = set(self._opt)
+ for k,v in self.get_variables().iteritems():
+ if k in self._exported:
+ env.export(k,v)
+ elif subshell:
+ env[k] = v
+
+ if subshell:
+ env._functions = dict(self._functions)
+
+ return env
+
+ def __getitem__(self, key):
+ if key in ('@', '*', '-', '$'):
+ raise NotImplementedError('%s is not implemented' % repr(key))
+ return self._env[key]
+
+ def get(self, key, defval=None):
+ try:
+ return self[key]
+ except KeyError:
+ return defval
+
+ def __setitem__(self, key, value):
+ if key=='IFS':
+ # Update the whitespace/non-whitespace classes
+ self._update_ifs(value)
+ elif key=='PWD':
+ pwd = os.path.abspath(value)
+ if not os.path.isdir(pwd):
+ raise VarAssignmentError('Invalid directory %s' % value)
+ value = pwd
+ elif key in ('?', '!'):
+ value = str(int(value))
+ self._env[key] = value
+
+ def __delitem__(self, key):
+ if key in ('IFS', 'PWD', '?'):
+ raise VarAssignmentError('%s cannot be unset' % key)
+ del self._env[key]
+
+ def __contains__(self, item):
+ return item in self._env
+
+ def set_positional_args(self, args):
+ """Set the content of 'args' as positional argument from 1 to len(args).
+ Return previous argument as a list of strings.
+ """
+ # Save and remove previous arguments
+ prevargs = []
+ for i in xrange(int(self._env['#'])):
+ i = str(i+1)
+ prevargs.append(self._env[i])
+ del self._env[i]
+ self._env['#'] = '0'
+
+ #Set new ones
+ for i,arg in enumerate(args):
+ self._env[str(i+1)] = str(arg)
+ self._env['#'] = str(len(args))
+
+ return prevargs
+
+ def get_positional_args(self):
+ return [self._env[str(i+1)] for i in xrange(int(self._env['#']))]
+
+ def get_variables(self):
+ return dict(self._env)
+
+ def export(self, key, value=None):
+ if value is not None:
+ self[key] = value
+ self._exported.add(key)
+
+ def get_exported(self):
+ return [(k,self._env.get(k)) for k in self._exported]
+
+ def split_fields(self, word):
+ if not self._ifs_ws or not word:
+ return [word]
+ return re.split(self._ifs_re, word)
+
+ def _update_ifs(self, value):
+ """Update the split_fields related variables when IFS character set is
+ changed.
+ """
+ # TODO: handle NULL IFS
+
+ # Separate characters in whitespace and non-whitespace
+ chars = set(value)
+ ws = [c for c in chars if c in _IFS_WHITESPACES]
+ nws = [c for c in chars if c not in _IFS_WHITESPACES]
+
+ # Keep whitespaces in a string for left and right stripping
+ self._ifs_ws = ''.join(ws)
+
+ # Build a regexp to split fields
+ trailing = '[' + ''.join([re.escape(c) for c in ws]) + ']'
+ if nws:
+ # First, the single non-whitespace occurence.
+ nws = '[' + ''.join([re.escape(c) for c in nws]) + ']'
+ nws = '(?:' + trailing + '*' + nws + trailing + '*' + '|' + trailing + '+)'
+ else:
+ # Then mix all parts with quantifiers
+ nws = trailing + '+'
+ self._ifs_re = re.compile(nws)
+
+ def has_opt(self, opt, val=None):
+ return (opt, val) in self._opt
+
+ def set_opt(self, opt, val=None):
+ self._opt.add((opt, val))
+
+ def find_in_path(self, name, pwd=False):
+ path = self._env.get('PATH', '').split(os.pathsep)
+ if pwd:
+ path[:0] = [self['PWD']]
+ if os.name == 'nt':
+ return win32_find_in_path(name, self._env.get('PATH', ''))
+ else:
+ raise NotImplementedError()
+
+ def define_function(self, name, body):
+ if not is_name(name):
+ raise ShellSyntaxError('%s is not a valid function name' % repr(name))
+ self._functions[name] = body
+
+ def remove_function(self, name):
+ del self._functions[name]
+
+ def is_function(self, name):
+ return name in self._functions
+
+ def get_function(self, name):
+ return self._functions.get(name)
+
+
+name_charset = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
+name_charset = dict(zip(name_charset,name_charset))
+
+def match_name(s):
+ """Return the length in characters of the longest prefix made of name
+ allowed characters in s.
+ """
+ for i,c in enumerate(s):
+ if c not in name_charset:
+ return s[:i]
+ return s
+
+def is_name(s):
+ return len([c for c in s if c not in name_charset])<=0
+
+def is_special_param(c):
+ return len(c)==1 and c in ('@','*','#','?','-','$','!','0')
+
+def utility_not_implemented(name, *args, **kwargs):
+ raise NotImplementedError('%s utility is not implemented' % name)
+
+
+class Utility:
+ """Define utilities properties:
+ func -- utility callable. See builtin module for utility samples.
+ is_special -- see XCU 2.8.
+ """
+ def __init__(self, func, is_special=0):
+ self.func = func
+ self.is_special = bool(is_special)
+
+
+def encodeargs(args):
+ def encodearg(s):
+ lines = base64.encodestring(s)
+ lines = [l.splitlines()[0] for l in lines]
+ return ''.join(lines)
+
+ s = pickle.dumps(args)
+ return encodearg(s)
+
+def decodeargs(s):
+ s = base64.decodestring(s)
+ return pickle.loads(s)
+
+
+class GlobError(Exception):
+ pass
+
+class Options:
+ def __init__(self):
+ # True if Mercurial operates with binary streams
+ self.hgbinary = True
+
+class Interpreter:
+ # Implementation is very basic: the execute() method just makes a DFS on the
+ # AST and execute nodes one by one. Nodes are tuple (name,obj) where name
+ # is a string identifier and obj the AST element returned by the parser.
+ #
+ # Handler are named after the node identifiers.
+ # TODO: check node names and remove the switch in execute with some
+ # dynamic getattr() call to find node handlers.
+ """Shell interpreter.
+
+ The following debugging flags can be passed:
+ debug-parsing - enable PLY debugging.
+ debug-tree - print the generated AST.
+ debug-cmd - trace command execution before word expansion, plus exit status.
+ debug-utility - trace utility execution.
+ """
+
+ # List supported commands.
+ COMMANDS = {
+ 'cat': Utility(builtin.utility_cat,),
+ 'cd': Utility(builtin.utility_cd,),
+ ':': Utility(builtin.utility_colon,),
+ 'echo': Utility(builtin.utility_echo),
+ 'env': Utility(builtin.utility_env),
+ 'exit': Utility(builtin.utility_exit),
+ 'export': Utility(builtin.builtin_export, is_special=1),
+ 'egrep': Utility(builtin.utility_egrep),
+ 'fgrep': Utility(builtin.utility_fgrep),
+ 'gunzip': Utility(builtin.utility_gunzip),
+ 'kill': Utility(builtin.utility_kill),
+ 'mkdir': Utility(builtin.utility_mkdir),
+ 'netstat': Utility(builtin.utility_netstat),
+ 'printf': Utility(builtin.utility_printf),
+ 'pwd': Utility(builtin.utility_pwd),
+ 'return': Utility(builtin.builtin_return, is_special=1),
+ 'sed': Utility(builtin.utility_sed,),
+ 'set': Utility(builtin.builtin_set,),
+ 'shift': Utility(builtin.builtin_shift,),
+ 'sleep': Utility(builtin.utility_sleep,),
+ 'sort': Utility(builtin.utility_sort,),
+ 'trap': Utility(builtin.builtin_trap, is_special=1),
+ 'true': Utility(builtin.utility_true),
+ 'unset': Utility(builtin.builtin_unset, is_special=1),
+ 'wait': Utility(builtin.builtin_wait, is_special=1),
+ }
+
+ def __init__(self, pwd, debugflags = [], env=None, redirs=None, stdin=None,
+ stdout=None, stderr=None, opts=Options()):
+ self._env = env
+ if self._env is None:
+ self._env = Environment(pwd)
+ self._children = {}
+
+ self._redirs = redirs
+ self._close_redirs = False
+
+ if self._redirs is None:
+ if stdin is None:
+ stdin = sys.stdin
+ if stdout is None:
+ stdout = sys.stdout
+ if stderr is None:
+ stderr = sys.stderr
+ stdin = FileWrapper('r', stdin, False)
+ stdout = FileWrapper('w', stdout, False)
+ stderr = FileWrapper('w', stderr, False)
+ self._redirs = Redirections(stdin, stdout, stderr)
+ self._close_redirs = True
+
+ self._debugflags = list(debugflags)
+ self._logfile = sys.stderr
+ self._options = opts
+
+ def close(self):
+ """Must be called when the interpreter is no longer used."""
+ script = self._env.traps.get('EXIT')
+ if script:
+ try:
+ self.execute_script(script=script)
+ except:
+ pass
+
+ if self._redirs is not None and self._close_redirs:
+ self._redirs.close()
+ self._redirs = None
+
+ def log(self, s):
+ self._logfile.write(s)
+ self._logfile.flush()
+
+ def __getitem__(self, key):
+ return self._env[key]
+
+ def __setitem__(self, key, value):
+ self._env[key] = value
+
+ def options(self):
+ return self._options
+
+ def redirect(self, redirs, ios):
+ def add_redir(io):
+ if isinstance(io, pyshyacc.IORedirect):
+ redirs.add(self, io.op, io.filename, io.io_number)
+ else:
+ redirs.add_here_document(self, io.name, io.content, io.io_number)
+
+ map(add_redir, ios)
+ return redirs
+
+ def execute_script(self, script=None, ast=None, sourced=False,
+ scriptpath=None):
+ """If script is not None, parse the input. Otherwise takes the supplied
+ AST. Then execute the AST.
+ Return the script exit status.
+ """
+ try:
+ if scriptpath is not None:
+ self._env['0'] = os.path.abspath(scriptpath)
+
+ if script is not None:
+ debug_parsing = ('debug-parsing' in self._debugflags)
+ cmds, script = pyshyacc.parse(script, True, debug_parsing)
+ if 'debug-tree' in self._debugflags:
+ pyshyacc.print_commands(cmds, self._logfile)
+ self._logfile.flush()
+ else:
+ cmds, script = ast, ''
+
+ status = 0
+ for cmd in cmds:
+ try:
+ status = self.execute(cmd)
+ except ExitSignal as e:
+ if sourced:
+ raise
+ status = int(e.args[0])
+ return status
+ except ShellError:
+ self._env['?'] = 1
+ raise
+ if 'debug-utility' in self._debugflags or 'debug-cmd' in self._debugflags:
+ self.log('returncode ' + str(status)+ '\n')
+ return status
+ except CommandNotFound as e:
+ print >>self._redirs.stderr, str(e)
+ self._redirs.stderr.flush()
+ # Command not found by non-interactive shell
+ # return 127
+ raise
+ except RedirectionError as e:
+ # TODO: should be handled depending on the utility status
+ print >>self._redirs.stderr, str(e)
+ self._redirs.stderr.flush()
+ # Command not found by non-interactive shell
+ # return 127
+ raise
+
+ def dotcommand(self, env, args):
+ if len(args) < 1:
+ raise ShellError('. expects at least one argument')
+ path = args[0]
+ if '/' not in path:
+ found = env.find_in_path(args[0], True)
+ if found:
+ path = found[0]
+ script = file(path).read()
+ return self.execute_script(script=script, sourced=True)
+
+ def execute(self, token, redirs=None):
+ """Execute and AST subtree with supplied redirections overriding default
+ interpreter ones.
+ Return the exit status.
+ """
+ if not token:
+ return 0
+
+ if redirs is None:
+ redirs = self._redirs
+
+ if isinstance(token, list):
+ # Commands sequence
+ res = 0
+ for t in token:
+ res = self.execute(t, redirs)
+ return res
+
+ type, value = token
+ status = 0
+ if type=='simple_command':
+ redirs_copy = redirs.clone()
+ try:
+ # TODO: define and handle command return values
+ # TODO: implement set -e
+ status = self._execute_simple_command(value, redirs_copy)
+ finally:
+ redirs_copy.close()
+ elif type=='pipeline':
+ status = self._execute_pipeline(value, redirs)
+ elif type=='and_or':
+ status = self._execute_and_or(value, redirs)
+ elif type=='for_clause':
+ status = self._execute_for_clause(value, redirs)
+ elif type=='while_clause':
+ status = self._execute_while_clause(value, redirs)
+ elif type=='function_definition':
+ status = self._execute_function_definition(value, redirs)
+ elif type=='brace_group':
+ status = self._execute_brace_group(value, redirs)
+ elif type=='if_clause':
+ status = self._execute_if_clause(value, redirs)
+ elif type=='subshell':
+ status = self.subshell(ast=value.cmds, redirs=redirs)
+ elif type=='async':
+ status = self._asynclist(value)
+ elif type=='redirect_list':
+ redirs_copy = self.redirect(redirs.clone(), value.redirs)
+ try:
+ status = self.execute(value.cmd, redirs_copy)
+ finally:
+ redirs_copy.close()
+ else:
+ raise NotImplementedError('Unsupported token type ' + type)
+
+ if status < 0:
+ status = 255
+ return status
+
+ def _execute_if_clause(self, if_clause, redirs):
+ cond_status = self.execute(if_clause.cond, redirs)
+ if cond_status==0:
+ return self.execute(if_clause.if_cmds, redirs)
+ else:
+ return self.execute(if_clause.else_cmds, redirs)
+
+ def _execute_brace_group(self, group, redirs):
+ status = 0
+ for cmd in group.cmds:
+ status = self.execute(cmd, redirs)
+ return status
+
+ def _execute_function_definition(self, fundef, redirs):
+ self._env.define_function(fundef.name, fundef.body)
+ return 0
+
+ def _execute_while_clause(self, while_clause, redirs):
+ status = 0
+ while 1:
+ cond_status = 0
+ for cond in while_clause.condition:
+ cond_status = self.execute(cond, redirs)
+
+ if cond_status:
+ break
+
+ for cmd in while_clause.cmds:
+ status = self.execute(cmd, redirs)
+
+ return status
+
+ def _execute_for_clause(self, for_clause, redirs):
+ if not is_name(for_clause.name):
+ raise ShellSyntaxError('%s is not a valid name' % repr(for_clause.name))
+ items = mappend(self.expand_token, for_clause.items)
+
+ status = 0
+ for item in items:
+ self._env[for_clause.name] = item
+ for cmd in for_clause.cmds:
+ status = self.execute(cmd, redirs)
+ return status
+
+ def _execute_and_or(self, or_and, redirs):
+ res = self.execute(or_and.left, redirs)
+ if (or_and.op=='&&' and res==0) or (or_and.op!='&&' and res!=0):
+ res = self.execute(or_and.right, redirs)
+ return res
+
+ def _execute_pipeline(self, pipeline, redirs):
+ if len(pipeline.commands)==1:
+ status = self.execute(pipeline.commands[0], redirs)
+ else:
+ # Execute all commands one after the other
+ status = 0
+ inpath, outpath = None, None
+ try:
+ # Commands inputs and outputs cannot really be plugged as done
+ # by a real shell. Run commands sequentially and chain their
+ # input/output throught temporary files.
+ tmpfd, inpath = tempfile.mkstemp()
+ os.close(tmpfd)
+ tmpfd, outpath = tempfile.mkstemp()
+ os.close(tmpfd)
+
+ inpath = win32_to_unix_path(inpath)
+ outpath = win32_to_unix_path(outpath)
+
+ for i, cmd in enumerate(pipeline.commands):
+ call_redirs = redirs.clone()
+ try:
+ if i!=0:
+ call_redirs.add(self, '<', inpath)
+ if i!=len(pipeline.commands)-1:
+ call_redirs.add(self, '>', outpath)
+
+ status = self.execute(cmd, call_redirs)
+
+ # Chain inputs/outputs
+ inpath, outpath = outpath, inpath
+ finally:
+ call_redirs.close()
+ finally:
+ if inpath: os.remove(inpath)
+ if outpath: os.remove(outpath)
+
+ if pipeline.reverse_status:
+ status = int(not status)
+ self._env['?'] = status
+ return status
+
+ def _execute_function(self, name, args, interp, env, stdin, stdout, stderr, *others):
+ assert interp is self
+
+ func = env.get_function(name)
+ #Set positional parameters
+ prevargs = None
+ try:
+ prevargs = env.set_positional_args(args)
+ try:
+ redirs = Redirections(stdin.dup(), stdout.dup(), stderr.dup())
+ try:
+ status = self.execute(func, redirs)
+ finally:
+ redirs.close()
+ except ReturnSignal as e:
+ status = int(e.args[0])
+ env['?'] = status
+ return status
+ finally:
+ #Reset positional parameters
+ if prevargs is not None:
+ env.set_positional_args(prevargs)
+
+ def _execute_simple_command(self, token, redirs):
+ """Can raise ReturnSignal when return builtin is called, ExitSignal when
+ exit is called, and other shell exceptions upon builtin failures.
+ """
+ debug_command = 'debug-cmd' in self._debugflags
+ if debug_command:
+ self.log('word' + repr(token.words) + '\n')
+ self.log('assigns' + repr(token.assigns) + '\n')
+ self.log('redirs' + repr(token.redirs) + '\n')
+
+ is_special = None
+ env = self._env
+
+ try:
+ # Word expansion
+ args = []
+ for word in token.words:
+ args += self.expand_token(word)
+ if is_special is None and args:
+ is_special = env.is_function(args[0]) or \
+ (args[0] in self.COMMANDS and self.COMMANDS[args[0]].is_special)
+
+ if debug_command:
+ self.log('_execute_simple_command' + str(args) + '\n')
+
+ if not args:
+ # Redirections happen is a subshell
+ redirs = redirs.clone()
+ elif not is_special:
+ env = self._env.clone()
+
+ # Redirections
+ self.redirect(redirs, token.redirs)
+
+ # Variables assignments
+ res = 0
+ for type,(k,v) in token.assigns:
+ status, expanded = self.expand_variable((k,v))
+ if status is not None:
+ res = status
+ if args:
+ env.export(k, expanded)
+ else:
+ env[k] = expanded
+
+ if args and args[0] in ('.', 'source'):
+ res = self.dotcommand(env, args[1:])
+ elif args:
+ if args[0] in self.COMMANDS:
+ command = self.COMMANDS[args[0]]
+ elif env.is_function(args[0]):
+ command = Utility(self._execute_function, is_special=True)
+ else:
+ if not '/' in args[0].replace('\\', '/'):
+ cmd = env.find_in_path(args[0])
+ if not cmd:
+ # TODO: test error code on unknown command => 127
+ raise CommandNotFound('Unknown command: "%s"' % args[0])
+ else:
+ # Handle commands like '/cygdrive/c/foo.bat'
+ cmd = cygwin_to_windows_path(args[0])
+ if not os.path.exists(cmd):
+ raise CommandNotFound('%s: No such file or directory' % args[0])
+ shebang = resolve_shebang(cmd)
+ if shebang:
+ cmd = shebang
+ else:
+ cmd = [cmd]
+ args[0:1] = cmd
+ command = Utility(builtin.run_command)
+
+ # Command execution
+ if 'debug-cmd' in self._debugflags:
+ self.log('redirections ' + str(redirs) + '\n')
+
+ res = command.func(args[0], args[1:], self, env,
+ redirs.stdin(), redirs.stdout(),
+ redirs.stderr(), self._debugflags)
+
+ if self._env.has_opt('-x'):
+ # Trace command execution in shell environment
+ # BUG: would be hard to reproduce a real shell behaviour since
+ # the AST is not annotated with source lines/tokens.
+ self._redirs.stdout().write(' '.join(args))
+
+ except ReturnSignal:
+ raise
+ except ShellError as e:
+ if is_special or isinstance(e, (ExitSignal,
+ ShellSyntaxError, ExpansionError)):
+ raise e
+ self._redirs.stderr().write(str(e)+'\n')
+ return 1
+
+ return res
+
+ def expand_token(self, word):
+ """Expand a word as specified in [2.6 Word Expansions]. Return the list
+ of expanded words.
+ """
+ status, wtrees = self._expand_word(word)
+ return map(pyshlex.wordtree_as_string, wtrees)
+
+ def expand_variable(self, word):
+ """Return a status code (or None if no command expansion occurred)
+ and a single word.
+ """
+ status, wtrees = self._expand_word(word, pathname=False, split=False)
+ words = map(pyshlex.wordtree_as_string, wtrees)
+ assert len(words)==1
+ return status, words[0]
+
+ def expand_here_document(self, word):
+ """Return the expanded document as a single word. The here document is
+ assumed to be unquoted.
+ """
+ status, wtrees = self._expand_word(word, pathname=False,
+ split=False, here_document=True)
+ words = map(pyshlex.wordtree_as_string, wtrees)
+ assert len(words)==1
+ return words[0]
+
+ def expand_redirection(self, word):
+ """Return a single word."""
+ return self.expand_variable(word)[1]
+
+ def get_env(self):
+ return self._env
+
+ def _expand_word(self, token, pathname=True, split=True, here_document=False):
+ wtree = pyshlex.make_wordtree(token[1], here_document=here_document)
+
+ # TODO: implement tilde expansion
+ def expand(wtree):
+ """Return a pseudo wordtree: the tree or its subelements can be empty
+ lists when no value result from the expansion.
+ """
+ status = None
+ for part in wtree:
+ if not isinstance(part, list):
+ continue
+ if part[0]in ("'", '\\'):
+ continue
+ elif part[0] in ('`', '$('):
+ status, result = self._expand_command(part)
+ part[:] = result
+ elif part[0] in ('$', '${'):
+ part[:] = self._expand_parameter(part, wtree[0]=='"', split)
+ elif part[0] in ('', '"'):
+ status, result = expand(part)
+ part[:] = result
+ else:
+ raise NotImplementedError('%s expansion is not implemented'
+ % part[0])
+ # [] is returned when an expansion result in no-field,
+ # like an empty $@
+ wtree = [p for p in wtree if p != []]
+ if len(wtree) < 3:
+ return status, []
+ return status, wtree
+
+ status, wtree = expand(wtree)
+ if len(wtree) == 0:
+ return status, wtree
+ wtree = pyshlex.normalize_wordtree(wtree)
+
+ if split:
+ wtrees = self._split_fields(wtree)
+ else:
+ wtrees = [wtree]
+
+ if pathname:
+ wtrees = mappend(self._expand_pathname, wtrees)
+
+ wtrees = map(self._remove_quotes, wtrees)
+ return status, wtrees
+
+ def _expand_command(self, wtree):
+ # BUG: there is something to do with backslashes and quoted
+ # characters here
+ command = pyshlex.wordtree_as_string(wtree[1:-1])
+ status, output = self.subshell_output(command)
+ return status, ['', output, '']
+
+ def _expand_parameter(self, wtree, quoted=False, split=False):
+ """Return a valid wtree or an empty list when no parameter results."""
+ # Get the parameter name
+ # TODO: implement weird expansion rules with ':'
+ name = pyshlex.wordtree_as_string(wtree[1:-1])
+ if not is_name(name) and not is_special_param(name):
+ raise ExpansionError('Bad substitution "%s"' % name)
+ # TODO: implement special parameters
+ if name in ('@', '*'):
+ args = self._env.get_positional_args()
+ if len(args) == 0:
+ return []
+ if len(args)<2:
+ return ['', ''.join(args), '']
+
+ sep = self._env.get('IFS', '')[:1]
+ if split and quoted and name=='@':
+ # Introduce a new token to tell the caller that these parameters
+ # cause a split as specified in 2.5.2
+ return ['@'] + args + ['']
+ else:
+ return ['', sep.join(args), '']
+
+ return ['', self._env.get(name, ''), '']
+
+ def _split_fields(self, wtree):
+ def is_empty(split):
+ return split==['', '', '']
+
+ def split_positional(quoted):
+ # Return a list of wtree split according positional parameters rules.
+ # All remaining '@' groups are removed.
+ assert quoted[0]=='"'
+
+ splits = [[]]
+ for part in quoted:
+ if not isinstance(part, list) or part[0]!='@':
+ splits[-1].append(part)
+ else:
+ # Empty or single argument list were dealt with already
+ assert len(part)>3
+ # First argument must join with the beginning part of the original word
+ splits[-1].append(part[1])
+ # Create double-quotes expressions for every argument after the first
+ for arg in part[2:-1]:
+ splits[-1].append('"')
+ splits.append(['"', arg])
+ return splits
+
+ # At this point, all expansions but pathnames have occured. Only quoted
+ # and positional sequences remain. Thus, all candidates for field splitting
+ # are in the tree root, or are positional splits ('@') and lie in root
+ # children.
+ if not wtree or wtree[0] not in ('', '"'):
+ # The whole token is quoted or empty, nothing to split
+ return [wtree]
+
+ if wtree[0]=='"':
+ wtree = ['', wtree, '']
+
+ result = [['', '']]
+ for part in wtree[1:-1]:
+ if isinstance(part, list):
+ if part[0]=='"':
+ splits = split_positional(part)
+ if len(splits)<=1:
+ result[-1] += [part, '']
+ else:
+ # Terminate the current split
+ result[-1] += [splits[0], '']
+ result += splits[1:-1]
+ # Create a new split
+ result += [['', splits[-1], '']]
+ else:
+ result[-1] += [part, '']
+ else:
+ splits = self._env.split_fields(part)
+ if len(splits)<=1:
+ # No split
+ result[-1][-1] += part
+ else:
+ # Terminate the current resulting part and create a new one
+ result[-1][-1] += splits[0]
+ result[-1].append('')
+ result += [['', r, ''] for r in splits[1:-1]]
+ result += [['', splits[-1]]]
+ result[-1].append('')
+
+ # Leading and trailing empty groups come from leading/trailing blanks
+ if result and is_empty(result[-1]):
+ result[-1:] = []
+ if result and is_empty(result[0]):
+ result[:1] = []
+ return result
+
+ def _expand_pathname(self, wtree):
+ """See [2.6.6 Pathname Expansion]."""
+ if self._env.has_opt('-f'):
+ return [wtree]
+
+ # All expansions have been performed, only quoted sequences should remain
+ # in the tree. Generate the pattern by folding the tree, escaping special
+ # characters when appear quoted
+ special_chars = '*?[]'
+
+ def make_pattern(wtree):
+ subpattern = []
+ for part in wtree[1:-1]:
+ if isinstance(part, list):
+ part = make_pattern(part)
+ elif wtree[0]!='':
+ for c in part:
+ # Meta-characters cannot be quoted
+ if c in special_chars:
+ raise GlobError()
+ subpattern.append(part)
+ return ''.join(subpattern)
+
+ def pwd_glob(pattern):
+ cwd = os.getcwd()
+ os.chdir(self._env['PWD'])
+ try:
+ return glob.glob(pattern)
+ finally:
+ os.chdir(cwd)
+
+ #TODO: check working directory issues here wrt relative patterns
+ try:
+ pattern = make_pattern(wtree)
+ paths = pwd_glob(pattern)
+ except GlobError:
+ # BUG: Meta-characters were found in quoted sequences. The should
+ # have been used literally but this is unsupported in current glob module.
+ # Instead we consider the whole tree must be used literally and
+ # therefore there is no point in globbing. This is wrong when meta
+ # characters are mixed with quoted meta in the same pattern like:
+ # < foo*"py*" >
+ paths = []
+
+ if not paths:
+ return [wtree]
+ return [['', path, ''] for path in paths]
+
+ def _remove_quotes(self, wtree):
+ """See [2.6.7 Quote Removal]."""
+
+ def unquote(wtree):
+ unquoted = []
+ for part in wtree[1:-1]:
+ if isinstance(part, list):
+ part = unquote(part)
+ unquoted.append(part)
+ return ''.join(unquoted)
+
+ return ['', unquote(wtree), '']
+
+ def subshell(self, script=None, ast=None, redirs=None):
+ """Execute the script or AST in a subshell, with inherited redirections
+ if redirs is not None.
+ """
+ if redirs:
+ sub_redirs = redirs
+ else:
+ sub_redirs = redirs.clone()
+
+ subshell = None
+ try:
+ subshell = Interpreter(None, self._debugflags, self._env.clone(True),
+ sub_redirs, opts=self._options)
+ return subshell.execute_script(script, ast)
+ finally:
+ if not redirs: sub_redirs.close()
+ if subshell: subshell.close()
+
+ def subshell_output(self, script):
+ """Execute the script in a subshell and return the captured output."""
+ # Create temporary file to capture subshell output
+ tmpfd, tmppath = tempfile.mkstemp()
+ try:
+ tmpfile = os.fdopen(tmpfd, 'wb')
+ stdout = FileWrapper('w', tmpfile)
+
+ redirs = Redirections(self._redirs.stdin().dup(),
+ stdout,
+ self._redirs.stderr().dup())
+ try:
+ status = self.subshell(script=script, redirs=redirs)
+ finally:
+ redirs.close()
+ redirs = None
+
+ # Extract subshell standard output
+ tmpfile = open(tmppath, 'rb')
+ try:
+ output = tmpfile.read()
+ return status, output.rstrip('\n')
+ finally:
+ tmpfile.close()
+ finally:
+ os.remove(tmppath)
+
+ def _asynclist(self, cmd):
+ args = (self._env.get_variables(), cmd)
+ arg = encodeargs(args)
+ assert len(args) < 30*1024
+ cmd = ['pysh.bat', '--ast', '-c', arg]
+ p = subprocess.Popen(cmd, cwd=self._env['PWD'])
+ self._children[p.pid] = p
+ self._env['!'] = p.pid
+ return 0
+
+ def wait(self, pids=None):
+ if not pids:
+ pids = self._children.keys()
+
+ status = 127
+ for pid in pids:
+ if pid not in self._children:
+ continue
+ p = self._children.pop(pid)
+ status = p.wait()
+
+ return status
+