blob: abfe24bacec98fb1e1850e648dbebe8d29e17ab5 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Copyright (C) 2012 Linux Foundation
2# Author: Richard Purdie
3# Some code and influence taken from srctree.bbclass:
4# Copyright (C) 2009 Chris Larson <clarson@kergoth.com>
5# Released under the MIT license (see COPYING.MIT for the terms)
6#
Brad Bishop6e60e8b2018-02-01 10:27:11 -05007# externalsrc.bbclass enables use of an existing source tree, usually external to
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008# the build system to build a piece of software rather than the usual fetch/unpack/patch
9# process.
10#
11# To use, add externalsrc to the global inherit and set EXTERNALSRC to point at the
12# directory you want to use containing the sources e.g. from local.conf for a recipe
13# called "myrecipe" you would do:
14#
15# INHERIT += "externalsrc"
Patrick Williams213cb262021-08-07 19:21:33 -050016# EXTERNALSRC:pn-myrecipe = "/path/to/my/source/tree"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050017#
18# In order to make this class work for both target and native versions (or with
19# multilibs/cross or other BBCLASSEXTEND variants), B is set to point to a separate
20# directory under the work directory (split source and build directories). This is
21# the default, but the build directory can be set to the source directory if
22# circumstances dictate by setting EXTERNALSRC_BUILD to the same value, e.g.:
23#
Patrick Williams213cb262021-08-07 19:21:33 -050024# EXTERNALSRC_BUILD:pn-myrecipe = "/path/to/my/source/tree"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050025#
26
27SRCTREECOVEREDTASKS ?= "do_patch do_unpack do_fetch"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050028EXTERNALSRC_SYMLINKS ?= "oe-workdir:${WORKDIR} oe-logs:${T}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050029
30python () {
Brad Bishop6e60e8b2018-02-01 10:27:11 -050031 externalsrc = d.getVar('EXTERNALSRC')
Brad Bishopd7bf8c12018-02-25 22:55:05 -050032 externalsrcbuild = d.getVar('EXTERNALSRC_BUILD')
33
34 if externalsrc and not externalsrc.startswith("/"):
35 bb.error("EXTERNALSRC must be an absolute path")
36 if externalsrcbuild and not externalsrcbuild.startswith("/"):
37 bb.error("EXTERNALSRC_BUILD must be an absolute path")
Patrick Williamsc0f7c042017-02-23 20:41:17 -060038
39 # If this is the base recipe and EXTERNALSRC is set for it or any of its
40 # derivatives, then enable BB_DONT_CACHE to force the recipe to always be
41 # re-parsed so that the file-checksums function for do_compile is run every
42 # time.
Brad Bishop6e60e8b2018-02-01 10:27:11 -050043 bpn = d.getVar('BPN')
Brad Bishop004d4992018-10-02 23:54:45 +020044 classextend = (d.getVar('BBCLASSEXTEND') or '').split()
45 if bpn == d.getVar('PN') or not classextend:
Patrick Williamsc0f7c042017-02-23 20:41:17 -060046 if (externalsrc or
47 ('native' in classextend and
Patrick Williams213cb262021-08-07 19:21:33 -050048 d.getVar('EXTERNALSRC:pn-%s-native' % bpn)) or
Patrick Williamsc0f7c042017-02-23 20:41:17 -060049 ('nativesdk' in classextend and
Patrick Williams213cb262021-08-07 19:21:33 -050050 d.getVar('EXTERNALSRC:pn-nativesdk-%s' % bpn)) or
Patrick Williamsc0f7c042017-02-23 20:41:17 -060051 ('cross' in classextend and
Patrick Williams213cb262021-08-07 19:21:33 -050052 d.getVar('EXTERNALSRC:pn-%s-cross' % bpn))):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060053 d.setVar('BB_DONT_CACHE', '1')
54
Patrick Williamsc124f4f2015-09-15 14:41:29 -050055 if externalsrc:
Brad Bishop316dfdd2018-06-25 12:45:53 -040056 import oe.recipeutils
57 import oe.path
58
Patrick Williamsc124f4f2015-09-15 14:41:29 -050059 d.setVar('S', externalsrc)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050060 if externalsrcbuild:
61 d.setVar('B', externalsrcbuild)
62 else:
63 d.setVar('B', '${WORKDIR}/${BPN}-${PV}/')
64
65 local_srcuri = []
Brad Bishop6e60e8b2018-02-01 10:27:11 -050066 fetch = bb.fetch2.Fetch((d.getVar('SRC_URI') or '').split(), d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050067 for url in fetch.urls:
68 url_data = fetch.ud[url]
69 parm = url_data.parm
70 if (url_data.type == 'file' or
Andrew Geissler90fd73c2021-03-05 15:25:55 -060071 url_data.type == 'npmsw' or
Patrick Williamsc124f4f2015-09-15 14:41:29 -050072 'type' in parm and parm['type'] == 'kmeta'):
73 local_srcuri.append(url)
74
75 d.setVar('SRC_URI', ' '.join(local_srcuri))
76
Andrew Geissler82c905d2020-04-13 13:39:40 -050077 # Dummy value because the default function can't be called with blank SRC_URI
78 d.setVar('SRCPV', '999')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050079
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080080 if d.getVar('CONFIGUREOPT_DEPTRACK') == '--disable-dependency-tracking':
81 d.setVar('CONFIGUREOPT_DEPTRACK', '')
82
Brad Bishop6e60e8b2018-02-01 10:27:11 -050083 tasks = filter(lambda k: d.getVarFlag(k, "task"), d.keys())
Patrick Williamsc124f4f2015-09-15 14:41:29 -050084
85 for task in tasks:
86 if task.endswith("_setscene"):
87 # sstate is never going to work for external source trees, disable it
88 bb.build.deltask(task, d)
Andrew Geissler4c19ea12020-10-27 13:52:24 -050089 elif os.path.realpath(d.getVar('S')) == os.path.realpath(d.getVar('B')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050090 # Since configure will likely touch ${S}, ensure only we lock so one task has access at a time
91 d.appendVarFlag(task, "lockfiles", " ${S}/singletask.lock")
92
93 # We do not want our source to be wiped out, ever (kernel.bbclass does this for do_clean)
Brad Bishop316dfdd2018-06-25 12:45:53 -040094 cleandirs = oe.recipeutils.split_var_value(d.getVarFlag(task, 'cleandirs', False) or '')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050095 setvalue = False
96 for cleandir in cleandirs[:]:
Brad Bishop316dfdd2018-06-25 12:45:53 -040097 if oe.path.is_path_parent(externalsrc, d.expand(cleandir)):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050098 cleandirs.remove(cleandir)
99 setvalue = True
100 if setvalue:
101 d.setVarFlag(task, 'cleandirs', ' '.join(cleandirs))
102
103 fetch_tasks = ['do_fetch', 'do_unpack']
104 # If we deltask do_patch, there's no dependency to ensure do_unpack gets run, so add one
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500105 # Note that we cannot use d.appendVarFlag() here because deps is expected to be a list object, not a string
106 d.setVarFlag('do_configure', 'deps', (d.getVarFlag('do_configure', 'deps', False) or []) + ['do_unpack'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500107
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500108 for task in d.getVar("SRCTREECOVEREDTASKS").split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500109 if local_srcuri and task in fetch_tasks:
110 continue
111 bb.build.deltask(task, d)
Andrew Geisslereff27472021-10-29 15:35:00 -0500112 if task == 'do_unpack':
113 # The reproducible build create_source_date_epoch_stamp function must
Patrick Williams93c203f2021-10-06 16:15:23 -0500114 # be run after the source is available and before the
115 # do_deploy_source_date_epoch task. In the normal case, it's attached
116 # to do_unpack as a postfuncs, but since we removed do_unpack (above)
117 # we need to move the function elsewhere. The easiest thing to do is
118 # move it into the prefuncs of the do_deploy_source_date_epoch task.
119 # This is safe, as externalsrc runs with the source already unpacked.
120 d.prependVarFlag('do_deploy_source_date_epoch', 'prefuncs', 'create_source_date_epoch_stamp ')
Andrew Geissler5199d832021-09-24 16:47:35 -0500121
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500122 d.prependVarFlag('do_compile', 'prefuncs', "externalsrc_compile_prefunc ")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500123 d.prependVarFlag('do_configure', 'prefuncs', "externalsrc_configure_prefunc ")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500124
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500125 d.setVarFlag('do_compile', 'file-checksums', '${@srctree_hash_files(d)}')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600126 d.setVarFlag('do_configure', 'file-checksums', '${@srctree_configure_hash_files(d)}')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500127
128 # We don't want the workdir to go away
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500129 d.appendVar('RM_WORK_EXCLUDE', ' ' + d.getVar('PN'))
130
131 bb.build.addtask('do_buildclean',
132 'do_clean' if d.getVar('S') == d.getVar('B') else None,
133 None, d)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500134
135 # If B=S the same builddir is used even for different architectures.
136 # Thus, use a shared CONFIGURESTAMPFILE and STAMP directory so that
137 # change of do_configure task hash is correctly detected and stamps are
138 # invalidated if e.g. MACHINE changes.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500139 if d.getVar('S') == d.getVar('B'):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500140 configstamp = '${TMPDIR}/work-shared/${PN}/${EXTENDPE}${PV}-${PR}/configure.sstate'
141 d.setVar('CONFIGURESTAMPFILE', configstamp)
142 d.setVar('STAMP', '${STAMPS_DIR}/work-shared/${PN}/${EXTENDPE}${PV}-${PR}')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500143 d.setVar('STAMPCLEAN', '${STAMPS_DIR}/work-shared/${PN}/*-*')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500144}
145
146python externalsrc_configure_prefunc() {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500147 s_dir = d.getVar('S')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500148 # Create desired symlinks
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500149 symlinks = (d.getVar('EXTERNALSRC_SYMLINKS') or '').split()
150 newlinks = []
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500151 for symlink in symlinks:
152 symsplit = symlink.split(':', 1)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500153 lnkfile = os.path.join(s_dir, symsplit[0])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500154 target = d.expand(symsplit[1])
155 if len(symsplit) > 1:
156 if os.path.islink(lnkfile):
157 # Link already exists, leave it if it points to the right location already
158 if os.readlink(lnkfile) == target:
159 continue
160 os.unlink(lnkfile)
161 elif os.path.exists(lnkfile):
162 # File/dir exists with same name as link, just leave it alone
163 continue
164 os.symlink(target, lnkfile)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500165 newlinks.append(symsplit[0])
166 # Hide the symlinks from git
167 try:
168 git_exclude_file = os.path.join(s_dir, '.git/info/exclude')
169 if os.path.exists(git_exclude_file):
170 with open(git_exclude_file, 'r+') as efile:
171 elines = efile.readlines()
172 for link in newlinks:
173 if link in elines or '/'+link in elines:
174 continue
175 efile.write('/' + link + '\n')
176 except IOError as ioe:
177 bb.note('Failed to hide EXTERNALSRC_SYMLINKS from git')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500178}
179
180python externalsrc_compile_prefunc() {
181 # Make it obvious that this is happening, since forgetting about it could lead to much confusion
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500182 bb.plain('NOTE: %s: compiling from external source tree %s' % (d.getVar('PN'), d.getVar('EXTERNALSRC')))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500183}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500184
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500185do_buildclean[dirs] = "${S} ${B}"
186do_buildclean[nostamp] = "1"
187do_buildclean[doc] = "Call 'make clean' or equivalent in ${B}"
188externalsrc_do_buildclean() {
189 if [ -e Makefile -o -e makefile -o -e GNUmakefile ]; then
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500190 rm -f ${@' '.join([x.split(':')[0] for x in (d.getVar('EXTERNALSRC_SYMLINKS') or '').split()])}
Brad Bishop316dfdd2018-06-25 12:45:53 -0400191 if [ "${CLEANBROKEN}" != "1" ]; then
192 oe_runmake clean || die "make failed"
193 fi
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500194 else
195 bbnote "nothing to do - no makefile found"
196 fi
197}
198
199def srctree_hash_files(d, srcdir=None):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500200 import shutil
201 import subprocess
202 import tempfile
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600203 import hashlib
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500204
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500205 s_dir = srcdir or d.getVar('EXTERNALSRC')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500206 git_dir = None
207
208 try:
209 git_dir = os.path.join(s_dir,
Brad Bishop316dfdd2018-06-25 12:45:53 -0400210 subprocess.check_output(['git', '-C', s_dir, 'rev-parse', '--git-dir'], stderr=subprocess.DEVNULL).decode("utf-8").rstrip())
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600211 top_git_dir = os.path.join(s_dir, subprocess.check_output(['git', '-C', d.getVar("TOPDIR"), 'rev-parse', '--git-dir'],
212 stderr=subprocess.DEVNULL).decode("utf-8").rstrip())
213 if git_dir == top_git_dir:
214 git_dir = None
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500215 except subprocess.CalledProcessError:
216 pass
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500217
218 ret = " "
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500219 if git_dir is not None:
Brad Bishopa34c0302019-09-23 22:34:48 -0400220 oe_hash_file = os.path.join(git_dir, 'oe-devtool-tree-sha1-%s' % d.getVar('PN'))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500221 with tempfile.NamedTemporaryFile(prefix='oe-devtool-index') as tmp_index:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500222 # Clone index
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500223 shutil.copyfile(os.path.join(git_dir, 'index'), tmp_index.name)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500224 # Update our custom index
225 env = os.environ.copy()
226 env['GIT_INDEX_FILE'] = tmp_index.name
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500227 subprocess.check_output(['git', 'add', '-A', '.'], cwd=s_dir, env=env)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600228 git_sha1 = subprocess.check_output(['git', 'write-tree'], cwd=s_dir, env=env).decode("utf-8")
Andrew Geisslerc926e172021-05-07 16:11:35 -0500229 submodule_helper = subprocess.check_output(['git', 'submodule--helper', 'list'], cwd=s_dir, env=env).decode("utf-8")
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600230 for line in submodule_helper.splitlines():
Andrew Geisslerc926e172021-05-07 16:11:35 -0500231 module_dir = os.path.join(s_dir, line.rsplit(maxsplit=1)[1])
232 if os.path.isdir(module_dir):
Andrew Geissler95ac1b82021-03-31 14:34:31 -0500233 proc = subprocess.Popen(['git', 'add', '-A', '.'], cwd=module_dir, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
234 proc.communicate()
235 proc = subprocess.Popen(['git', 'write-tree'], cwd=module_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
236 stdout, _ = proc.communicate()
237 git_sha1 += stdout.decode("utf-8")
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600238 sha1 = hashlib.sha1(git_sha1.encode("utf-8")).hexdigest()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500239 with open(oe_hash_file, 'w') as fobj:
240 fobj.write(sha1)
241 ret = oe_hash_file + ':True'
242 else:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500243 ret = s_dir + '/*:True'
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500244 return ret
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600245
246def srctree_configure_hash_files(d):
247 """
248 Get the list of files that should trigger do_configure to re-execute,
249 based on the value of CONFIGURE_FILES
250 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500251 in_files = (d.getVar('CONFIGURE_FILES') or '').split()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600252 out_items = []
253 search_files = []
254 for entry in in_files:
255 if entry.startswith('/'):
256 out_items.append('%s:%s' % (entry, os.path.exists(entry)))
257 else:
258 search_files.append(entry)
259 if search_files:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500260 s_dir = d.getVar('EXTERNALSRC')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600261 for root, _, files in os.walk(s_dir):
262 for f in files:
263 if f in search_files:
264 out_items.append('%s:True' % os.path.join(root, f))
265 return ' '.join(out_items)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500266
267EXPORT_FUNCTIONS do_buildclean