blob: f433106c9a88b15e421b13c48133dc098d6b6e79 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Script utility functions
2#
3# Copyright (C) 2014 Intel Corporation
4#
Brad Bishopc342db32019-05-15 21:57:59 -04005# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08008import glob
9import logging
10import os
Brad Bishopd7bf8c12018-02-25 22:55:05 -050011import random
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080012import shlex
13import shutil
Brad Bishopd7bf8c12018-02-25 22:55:05 -050014import string
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080015import subprocess
16import sys
17import tempfile
Brad Bishop79641f22019-09-10 07:20:22 -040018import threading
Brad Bishop19323692019-04-05 15:28:33 -040019import importlib
Andrew Geissler595f6302022-01-24 19:11:47 +000020import importlib.machinery
21import importlib.util
Patrick Williamsc124f4f2015-09-15 14:41:29 -050022
Brad Bishop79641f22019-09-10 07:20:22 -040023class KeepAliveStreamHandler(logging.StreamHandler):
24 def __init__(self, keepalive=True, **kwargs):
25 super().__init__(**kwargs)
26 if keepalive is True:
Andrew Geissler615f2f12022-07-15 14:00:58 -050027 keepalive = 5000 # default timeout
Brad Bishop79641f22019-09-10 07:20:22 -040028 self._timeout = threading.Condition()
29 self._stop = False
30
31 # background thread waits on condition, if the condition does not
32 # happen emit a keep alive message
33 def thread():
34 while not self._stop:
35 with self._timeout:
36 if not self._timeout.wait(keepalive):
37 self.emit(logging.LogRecord("keepalive", logging.INFO,
Andrew Geissler615f2f12022-07-15 14:00:58 -050038 None, None, "Keepalive message", None, None))
Brad Bishop79641f22019-09-10 07:20:22 -040039
Andrew Geissler615f2f12022-07-15 14:00:58 -050040 self._thread = threading.Thread(target=thread, daemon=True)
Brad Bishop79641f22019-09-10 07:20:22 -040041 self._thread.start()
42
43 def close(self):
44 # mark the thread to stop and notify it
45 self._stop = True
46 with self._timeout:
47 self._timeout.notify()
48 # wait for it to join
49 self._thread.join()
50 super().close()
51
52 def emit(self, record):
53 super().emit(record)
54 # trigger timer reset
55 with self._timeout:
56 self._timeout.notify()
57
58def logger_create(name, stream=None, keepalive=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059 logger = logging.getLogger(name)
Brad Bishop79641f22019-09-10 07:20:22 -040060 if keepalive is not None:
61 loggerhandler = KeepAliveStreamHandler(stream=stream, keepalive=keepalive)
62 else:
63 loggerhandler = logging.StreamHandler(stream=stream)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050064 loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
65 logger.addHandler(loggerhandler)
66 logger.setLevel(logging.INFO)
67 return logger
68
69def logger_setup_color(logger, color='auto'):
70 from bb.msg import BBLogFormatter
Brad Bishop19323692019-04-05 15:28:33 -040071
72 for handler in logger.handlers:
73 if (isinstance(handler, logging.StreamHandler) and
Andrew Geissler615f2f12022-07-15 14:00:58 -050074 isinstance(handler.formatter, BBLogFormatter)):
Brad Bishop19323692019-04-05 15:28:33 -040075 if color == 'always' or (color == 'auto' and handler.stream.isatty()):
76 handler.formatter.enable_color()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050077
78
79def load_plugins(logger, plugins, pluginpath):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050080 def load_plugin(name):
81 logger.debug('Loading plugin %s' % name)
Andrew Geissler615f2f12022-07-15 14:00:58 -050082 spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath])
Brad Bishop19323692019-04-05 15:28:33 -040083 if spec:
Andrew Geissler595f6302022-01-24 19:11:47 +000084 mod = importlib.util.module_from_spec(spec)
85 spec.loader.exec_module(mod)
86 return mod
Patrick Williamsc124f4f2015-09-15 14:41:29 -050087
Brad Bishop6e60e8b2018-02-01 10:27:11 -050088 def plugin_name(filename):
89 return os.path.splitext(os.path.basename(filename))[0]
90
91 known_plugins = [plugin_name(p.__name__) for p in plugins]
Patrick Williamsc124f4f2015-09-15 14:41:29 -050092 logger.debug('Loading plugins from %s...' % pluginpath)
93 for fn in glob.glob(os.path.join(pluginpath, '*.py')):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050094 name = plugin_name(fn)
95 if name != '__init__' and name not in known_plugins:
Patrick Williamsc124f4f2015-09-15 14:41:29 -050096 plugin = load_plugin(name)
97 if hasattr(plugin, 'plugin_init'):
98 plugin.plugin_init(plugins)
99 plugins.append(plugin)
100
Brad Bishop19323692019-04-05 15:28:33 -0400101
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500102def git_convert_standalone_clone(repodir):
103 """If specified directory is a git repository, ensure it's a standalone clone"""
104 import bb.process
105 if os.path.exists(os.path.join(repodir, '.git')):
106 alternatesfile = os.path.join(repodir, '.git', 'objects', 'info', 'alternates')
107 if os.path.exists(alternatesfile):
108 # This will have been cloned with -s, so we need to convert it so none
109 # of the contents is shared
110 bb.process.run('git repack -a', cwd=repodir)
111 os.remove(alternatesfile)
112
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500113def _get_temp_recipe_dir(d):
114 # This is a little bit hacky but we need to find a place where we can put
115 # the recipe so that bitbake can find it. We're going to delete it at the
116 # end so it doesn't really matter where we put it.
117 bbfiles = d.getVar('BBFILES').split()
118 fetchrecipedir = None
119 for pth in bbfiles:
120 if pth.endswith('.bb'):
121 pthdir = os.path.dirname(pth)
122 if os.access(os.path.dirname(os.path.dirname(pthdir)), os.W_OK):
123 fetchrecipedir = pthdir.replace('*', 'recipetool')
124 if pthdir.endswith('workspace/recipes/*'):
125 # Prefer the workspace
126 break
127 return fetchrecipedir
128
129class FetchUrlFailure(Exception):
130 def __init__(self, url):
131 self.url = url
132 def __str__(self):
133 return "Failed to fetch URL %s" % self.url
134
135def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False, mirrors=False):
136 """
137 Fetch the specified URL using normal do_fetch and do_unpack tasks, i.e.
138 any dependencies that need to be satisfied in order to support the fetch
139 operation will be taken care of
140 """
141
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500142 import bb
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500143
144 checksums = {}
145 fetchrecipepn = None
146
147 # We need to put our temp directory under ${BASE_WORKDIR} otherwise
148 # we may have problems with the recipe-specific sysroot population
149 tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500150 bb.utils.mkdirhier(tmpparent)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500151 tmpdir = tempfile.mkdtemp(prefix='recipetool-', dir=tmpparent)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500152 try:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500153 tmpworkdir = os.path.join(tmpdir, 'work')
154 logger.debug('fetch_url: temp dir is %s' % tmpdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500155
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500156 fetchrecipedir = _get_temp_recipe_dir(tinfoil.config_data)
157 if not fetchrecipedir:
158 logger.error('Searched BBFILES but unable to find a writeable place to put temporary recipe')
159 sys.exit(1)
160 fetchrecipe = None
161 bb.utils.mkdirhier(fetchrecipedir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500162 try:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500163 # Generate a dummy recipe so we can follow more or less normal paths
164 # for do_fetch and do_unpack
165 # I'd use tempfile functions here but underscores can be produced by that and those
166 # aren't allowed in recipe file names except to separate the version
167 rndstring = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8))
168 fetchrecipe = os.path.join(fetchrecipedir, 'tmp-recipetool-%s.bb' % rndstring)
169 fetchrecipepn = os.path.splitext(os.path.basename(fetchrecipe))[0]
170 logger.debug('Generating initial recipe %s for fetching' % fetchrecipe)
171 with open(fetchrecipe, 'w') as f:
172 # We don't want to have to specify LIC_FILES_CHKSUM
173 f.write('LICENSE = "CLOSED"\n')
174 # We don't need the cross-compiler
175 f.write('INHIBIT_DEFAULT_DEPS = "1"\n')
176 # We don't have the checksums yet so we can't require them
177 f.write('BB_STRICT_CHECKSUM = "ignore"\n')
178 f.write('SRC_URI = "%s"\n' % srcuri)
179 f.write('SRCREV = "%s"\n' % srcrev)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000180 f.write('PV = "0.0+${SRCPV}"\n')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500181 f.write('WORKDIR = "%s"\n' % tmpworkdir)
182 # Set S out of the way so it doesn't get created under the workdir
183 f.write('S = "%s"\n' % os.path.join(tmpdir, 'emptysrc'))
184 if not mirrors:
185 # We do not need PREMIRRORS since we are almost certainly
186 # fetching new source rather than something that has already
187 # been fetched. Hence, we disable them by default.
188 # However, we provide an option for users to enable it.
189 f.write('PREMIRRORS = ""\n')
190 f.write('MIRRORS = ""\n')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500191
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500192 logger.info('Fetching %s...' % srcuri)
193
194 # FIXME this is too noisy at the moment
195
196 # Parse recipes so our new recipe gets picked up
197 tinfoil.parse_recipes()
198
199 def eventhandler(event):
200 if isinstance(event, bb.fetch2.MissingChecksumEvent):
201 checksums.update(event.checksums)
202 return True
203 return False
204
205 # Run the fetch + unpack tasks
206 res = tinfoil.build_targets(fetchrecipepn,
207 'do_unpack',
208 handle_events=True,
209 extra_events=['bb.fetch2.MissingChecksumEvent'],
210 event_callback=eventhandler)
211 if not res:
212 raise FetchUrlFailure(srcuri)
213
214 # Remove unneeded directories
215 rd = tinfoil.parse_recipe(fetchrecipepn)
216 if rd:
217 pathvars = ['T', 'RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE']
218 for pathvar in pathvars:
219 path = rd.getVar(pathvar)
Patrick Williams93c203f2021-10-06 16:15:23 -0500220 if os.path.exists(path):
221 shutil.rmtree(path)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500222 finally:
223 if fetchrecipe:
224 try:
225 os.remove(fetchrecipe)
226 except FileNotFoundError:
227 pass
228 try:
229 os.rmdir(fetchrecipedir)
230 except OSError as e:
231 import errno
232 if e.errno != errno.ENOTEMPTY:
233 raise
234
235 bb.utils.mkdirhier(destdir)
236 for fn in os.listdir(tmpworkdir):
237 shutil.move(os.path.join(tmpworkdir, fn), destdir)
238
239 finally:
240 if not preserve_tmp:
241 shutil.rmtree(tmpdir)
242 tmpdir = None
243
244 return checksums, tmpdir
245
246
247def run_editor(fn, logger=None):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600248 if isinstance(fn, str):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800249 files = [fn]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500250 else:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800251 files = fn
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500252
253 editor = os.getenv('VISUAL', os.getenv('EDITOR', 'vi'))
254 try:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800255 #print(shlex.split(editor) + files)
256 return subprocess.check_call(shlex.split(editor) + files)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500257 except subprocess.CalledProcessError as exc:
258 logger.error("Execution of '%s' failed: %s" % (editor, exc))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500259 return 1
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600260
261def is_src_url(param):
262 """
263 Check if a parameter is a URL and return True if so
264 NOTE: be careful about changing this as it will influence how devtool/recipetool command line handling works
265 """
266 if not param:
267 return False
268 elif '://' in param:
269 return True
270 elif param.startswith('git@') or ('@' in param and param.endswith('.git')):
271 return True
272 return False
Brad Bishop6dbb3162019-11-25 09:41:34 -0500273
274def filter_src_subdirs(pth):
275 """
276 Filter out subdirectories of initial unpacked source trees that we do not care about.
277 Used by devtool and recipetool.
278 """
279 dirlist = os.listdir(pth)
Patrick Williams2a254922023-08-11 09:48:11 -0500280 filterout = ['git.indirectionsymlink', 'source-date-epoch', 'sstate-install-recipe_qa']
Brad Bishop6dbb3162019-11-25 09:41:34 -0500281 dirlist = [x for x in dirlist if x not in filterout]
282 return dirlist