blob: 31e48ea4dc10480a7e8a35889676cad5d42fbd58 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Script utility functions
2#
3# Copyright (C) 2014 Intel Corporation
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050018import argparse
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080019import glob
20import logging
21import os
Brad Bishopd7bf8c12018-02-25 22:55:05 -050022import random
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080023import shlex
24import shutil
Brad Bishopd7bf8c12018-02-25 22:55:05 -050025import string
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080026import subprocess
27import sys
28import tempfile
Patrick Williamsc124f4f2015-09-15 14:41:29 -050029
Brad Bishop6e60e8b2018-02-01 10:27:11 -050030def logger_create(name, stream=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050031 logger = logging.getLogger(name)
Brad Bishop6e60e8b2018-02-01 10:27:11 -050032 loggerhandler = logging.StreamHandler(stream=stream)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050033 loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
34 logger.addHandler(loggerhandler)
35 logger.setLevel(logging.INFO)
36 return logger
37
38def logger_setup_color(logger, color='auto'):
39 from bb.msg import BBLogFormatter
40 console = logging.StreamHandler(sys.stdout)
41 formatter = BBLogFormatter("%(levelname)s: %(message)s")
42 console.setFormatter(formatter)
43 logger.handlers = [console]
44 if color == 'always' or (color=='auto' and console.stream.isatty()):
45 formatter.enable_color()
46
47
48def load_plugins(logger, plugins, pluginpath):
49 import imp
50
51 def load_plugin(name):
52 logger.debug('Loading plugin %s' % name)
53 fp, pathname, description = imp.find_module(name, [pluginpath])
54 try:
55 return imp.load_module(name, fp, pathname, description)
56 finally:
57 if fp:
58 fp.close()
59
Brad Bishop6e60e8b2018-02-01 10:27:11 -050060 def plugin_name(filename):
61 return os.path.splitext(os.path.basename(filename))[0]
62
63 known_plugins = [plugin_name(p.__name__) for p in plugins]
Patrick Williamsc124f4f2015-09-15 14:41:29 -050064 logger.debug('Loading plugins from %s...' % pluginpath)
65 for fn in glob.glob(os.path.join(pluginpath, '*.py')):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050066 name = plugin_name(fn)
67 if name != '__init__' and name not in known_plugins:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050068 plugin = load_plugin(name)
69 if hasattr(plugin, 'plugin_init'):
70 plugin.plugin_init(plugins)
71 plugins.append(plugin)
72
73def git_convert_standalone_clone(repodir):
74 """If specified directory is a git repository, ensure it's a standalone clone"""
75 import bb.process
76 if os.path.exists(os.path.join(repodir, '.git')):
77 alternatesfile = os.path.join(repodir, '.git', 'objects', 'info', 'alternates')
78 if os.path.exists(alternatesfile):
79 # This will have been cloned with -s, so we need to convert it so none
80 # of the contents is shared
81 bb.process.run('git repack -a', cwd=repodir)
82 os.remove(alternatesfile)
83
Brad Bishopd7bf8c12018-02-25 22:55:05 -050084def _get_temp_recipe_dir(d):
85 # This is a little bit hacky but we need to find a place where we can put
86 # the recipe so that bitbake can find it. We're going to delete it at the
87 # end so it doesn't really matter where we put it.
88 bbfiles = d.getVar('BBFILES').split()
89 fetchrecipedir = None
90 for pth in bbfiles:
91 if pth.endswith('.bb'):
92 pthdir = os.path.dirname(pth)
93 if os.access(os.path.dirname(os.path.dirname(pthdir)), os.W_OK):
94 fetchrecipedir = pthdir.replace('*', 'recipetool')
95 if pthdir.endswith('workspace/recipes/*'):
96 # Prefer the workspace
97 break
98 return fetchrecipedir
99
100class FetchUrlFailure(Exception):
101 def __init__(self, url):
102 self.url = url
103 def __str__(self):
104 return "Failed to fetch URL %s" % self.url
105
106def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False, mirrors=False):
107 """
108 Fetch the specified URL using normal do_fetch and do_unpack tasks, i.e.
109 any dependencies that need to be satisfied in order to support the fetch
110 operation will be taken care of
111 """
112
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500113 import bb
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500114
115 checksums = {}
116 fetchrecipepn = None
117
118 # We need to put our temp directory under ${BASE_WORKDIR} otherwise
119 # we may have problems with the recipe-specific sysroot population
120 tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500121 bb.utils.mkdirhier(tmpparent)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500122 tmpdir = tempfile.mkdtemp(prefix='recipetool-', dir=tmpparent)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500123 try:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500124 tmpworkdir = os.path.join(tmpdir, 'work')
125 logger.debug('fetch_url: temp dir is %s' % tmpdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500126
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500127 fetchrecipedir = _get_temp_recipe_dir(tinfoil.config_data)
128 if not fetchrecipedir:
129 logger.error('Searched BBFILES but unable to find a writeable place to put temporary recipe')
130 sys.exit(1)
131 fetchrecipe = None
132 bb.utils.mkdirhier(fetchrecipedir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500133 try:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500134 # Generate a dummy recipe so we can follow more or less normal paths
135 # for do_fetch and do_unpack
136 # I'd use tempfile functions here but underscores can be produced by that and those
137 # aren't allowed in recipe file names except to separate the version
138 rndstring = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8))
139 fetchrecipe = os.path.join(fetchrecipedir, 'tmp-recipetool-%s.bb' % rndstring)
140 fetchrecipepn = os.path.splitext(os.path.basename(fetchrecipe))[0]
141 logger.debug('Generating initial recipe %s for fetching' % fetchrecipe)
142 with open(fetchrecipe, 'w') as f:
143 # We don't want to have to specify LIC_FILES_CHKSUM
144 f.write('LICENSE = "CLOSED"\n')
145 # We don't need the cross-compiler
146 f.write('INHIBIT_DEFAULT_DEPS = "1"\n')
147 # We don't have the checksums yet so we can't require them
148 f.write('BB_STRICT_CHECKSUM = "ignore"\n')
149 f.write('SRC_URI = "%s"\n' % srcuri)
150 f.write('SRCREV = "%s"\n' % srcrev)
151 f.write('WORKDIR = "%s"\n' % tmpworkdir)
152 # Set S out of the way so it doesn't get created under the workdir
153 f.write('S = "%s"\n' % os.path.join(tmpdir, 'emptysrc'))
154 if not mirrors:
155 # We do not need PREMIRRORS since we are almost certainly
156 # fetching new source rather than something that has already
157 # been fetched. Hence, we disable them by default.
158 # However, we provide an option for users to enable it.
159 f.write('PREMIRRORS = ""\n')
160 f.write('MIRRORS = ""\n')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500161
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500162 logger.info('Fetching %s...' % srcuri)
163
164 # FIXME this is too noisy at the moment
165
166 # Parse recipes so our new recipe gets picked up
167 tinfoil.parse_recipes()
168
169 def eventhandler(event):
170 if isinstance(event, bb.fetch2.MissingChecksumEvent):
171 checksums.update(event.checksums)
172 return True
173 return False
174
175 # Run the fetch + unpack tasks
176 res = tinfoil.build_targets(fetchrecipepn,
177 'do_unpack',
178 handle_events=True,
179 extra_events=['bb.fetch2.MissingChecksumEvent'],
180 event_callback=eventhandler)
181 if not res:
182 raise FetchUrlFailure(srcuri)
183
184 # Remove unneeded directories
185 rd = tinfoil.parse_recipe(fetchrecipepn)
186 if rd:
187 pathvars = ['T', 'RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE']
188 for pathvar in pathvars:
189 path = rd.getVar(pathvar)
190 shutil.rmtree(path)
191 finally:
192 if fetchrecipe:
193 try:
194 os.remove(fetchrecipe)
195 except FileNotFoundError:
196 pass
197 try:
198 os.rmdir(fetchrecipedir)
199 except OSError as e:
200 import errno
201 if e.errno != errno.ENOTEMPTY:
202 raise
203
204 bb.utils.mkdirhier(destdir)
205 for fn in os.listdir(tmpworkdir):
206 shutil.move(os.path.join(tmpworkdir, fn), destdir)
207
208 finally:
209 if not preserve_tmp:
210 shutil.rmtree(tmpdir)
211 tmpdir = None
212
213 return checksums, tmpdir
214
215
216def run_editor(fn, logger=None):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600217 if isinstance(fn, str):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800218 files = [fn]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500219 else:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800220 files = fn
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500221
222 editor = os.getenv('VISUAL', os.getenv('EDITOR', 'vi'))
223 try:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800224 #print(shlex.split(editor) + files)
225 return subprocess.check_call(shlex.split(editor) + files)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500226 except subprocess.CalledProcessError as exc:
227 logger.error("Execution of '%s' failed: %s" % (editor, exc))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500228 return 1
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600229
230def is_src_url(param):
231 """
232 Check if a parameter is a URL and return True if so
233 NOTE: be careful about changing this as it will influence how devtool/recipetool command line handling works
234 """
235 if not param:
236 return False
237 elif '://' in param:
238 return True
239 elif param.startswith('git@') or ('@' in param and param.endswith('.git')):
240 return True
241 return False