blob: 33070cd17f96dff0e08c792f573a7ad95c6e43da [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3#
4# This bbclass is used for creating archive for:
Andrew Geissler82c905d2020-04-13 13:39:40 -05005# 1) original (or unpacked) source: ARCHIVER_MODE[src] = "original"
6# 2) patched source: ARCHIVER_MODE[src] = "patched" (default)
7# 3) configured source: ARCHIVER_MODE[src] = "configured"
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00008# 4) source mirror: ARCHIVER_MODE[src] = "mirror"
Andrew Geissler82c905d2020-04-13 13:39:40 -05009# 5) The patches between do_unpack and do_patch:
10# ARCHIVER_MODE[diff] = "1"
11# And you can set the one that you'd like to exclude from the diff:
12# ARCHIVER_MODE[diff-exclude] ?= ".pc autom4te.cache patches"
13# 6) The environment data, similar to 'bitbake -e recipe':
14# ARCHIVER_MODE[dumpdata] = "1"
15# 7) The recipe (.bb and .inc): ARCHIVER_MODE[recipe] = "1"
16# 8) Whether output the .src.rpm package:
17# ARCHIVER_MODE[srpm] = "1"
18# 9) Filter the license, the recipe whose license in
19# COPYLEFT_LICENSE_INCLUDE will be included, and in
20# COPYLEFT_LICENSE_EXCLUDE will be excluded.
21# COPYLEFT_LICENSE_INCLUDE = 'GPL* LGPL*'
22# COPYLEFT_LICENSE_EXCLUDE = 'CLOSED Proprietary'
23# 10) The recipe type that will be archived:
24# COPYLEFT_RECIPE_TYPES = 'target'
25# 11) The source mirror mode:
26# ARCHIVER_MODE[mirror] = "split" (default): Sources are split into
27# per-recipe directories in a similar way to other archiver modes.
28# Post-processing may be required to produce a single mirror directory.
29# This does however allow inspection of duplicate sources and more
30# intelligent handling.
31# ARCHIVER_MODE[mirror] = "combined": All sources are placed into a single
32# directory suitable for direct use as a mirror. Duplicate sources are
33# ignored.
34# 12) Source mirror exclusions:
35# ARCHIVER_MIRROR_EXCLUDE is a list of prefixes to exclude from the mirror.
36# This may be used for sources which you are already publishing yourself
37# (e.g. if the URI starts with 'https://mysite.com/' and your mirror is
38# going to be published to the same site). It may also be used to exclude
39# local files (with the prefix 'file://') if these will be provided as part
40# of an archive of the layers themselves.
Patrick Williamsc124f4f2015-09-15 14:41:29 -050041#
42
Patrick Williamsc124f4f2015-09-15 14:41:29 -050043# Create archive for all the recipe types
44COPYLEFT_RECIPE_TYPES ?= 'target native nativesdk cross crosssdk cross-canadian'
45inherit copyleft_filter
46
47ARCHIVER_MODE[srpm] ?= "0"
48ARCHIVER_MODE[src] ?= "patched"
49ARCHIVER_MODE[diff] ?= "0"
50ARCHIVER_MODE[diff-exclude] ?= ".pc autom4te.cache patches"
51ARCHIVER_MODE[dumpdata] ?= "0"
52ARCHIVER_MODE[recipe] ?= "0"
Andrew Geissler82c905d2020-04-13 13:39:40 -050053ARCHIVER_MODE[mirror] ?= "split"
Andrew Geissler595f6302022-01-24 19:11:47 +000054ARCHIVER_MODE[compression] ?= "xz"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050055
56DEPLOY_DIR_SRC ?= "${DEPLOY_DIR}/sources"
Andrew Geisslerf0343792020-11-18 10:42:21 -060057ARCHIVER_TOPDIR ?= "${WORKDIR}/archiver-sources"
Andrew Geissler615f2f12022-07-15 14:00:58 -050058ARCHIVER_ARCH = "${TARGET_SYS}"
59ARCHIVER_OUTDIR = "${ARCHIVER_TOPDIR}/${ARCHIVER_ARCH}/${PF}/"
Brad Bishop19323692019-04-05 15:28:33 -040060ARCHIVER_RPMTOPDIR ?= "${WORKDIR}/deploy-sources-rpm"
Andrew Geissler615f2f12022-07-15 14:00:58 -050061ARCHIVER_RPMOUTDIR = "${ARCHIVER_RPMTOPDIR}/${ARCHIVER_ARCH}/${PF}/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050062ARCHIVER_WORKDIR = "${WORKDIR}/archiver-work/"
63
Andrew Geissler82c905d2020-04-13 13:39:40 -050064# When producing a combined mirror directory, allow duplicates for the case
65# where multiple recipes use the same SRC_URI.
66ARCHIVER_COMBINED_MIRRORDIR = "${ARCHIVER_TOPDIR}/mirror"
Andrew Geissler7e0e3c02022-02-25 20:34:39 +000067SSTATE_ALLOW_OVERLAP_FILES += "${DEPLOY_DIR_SRC}/mirror"
Brad Bishop19323692019-04-05 15:28:33 -040068
Patrick Williamsc124f4f2015-09-15 14:41:29 -050069do_dumpdata[dirs] = "${ARCHIVER_OUTDIR}"
70do_ar_recipe[dirs] = "${ARCHIVER_OUTDIR}"
71do_ar_original[dirs] = "${ARCHIVER_OUTDIR} ${ARCHIVER_WORKDIR}"
72do_deploy_archives[dirs] = "${WORKDIR}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050073
74# This is a convenience for the shell script to use it
75
76
77python () {
Brad Bishop6e60e8b2018-02-01 10:27:11 -050078 pn = d.getVar('PN')
79 assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050080 if pn in assume_provided:
Brad Bishop6e60e8b2018-02-01 10:27:11 -050081 for p in d.getVar("PROVIDES").split():
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050082 if p != pn:
83 pn = p
84 break
Patrick Williamsc124f4f2015-09-15 14:41:29 -050085
86 included, reason = copyleft_should_include(d)
87 if not included:
88 bb.debug(1, 'archiver: %s is excluded: %s' % (pn, reason))
89 return
90 else:
91 bb.debug(1, 'archiver: %s is included: %s' % (pn, reason))
92
Brad Bishop6e60e8b2018-02-01 10:27:11 -050093
94 # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted,
95 # so avoid archiving source here.
96 if pn.startswith('glibc-locale'):
97 return
98
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050099 # We just archive gcc-source for all the gcc related recipes
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500100 if d.getVar('BPN') in ['gcc', 'libgcc'] \
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500101 and not pn.startswith('gcc-source'):
102 bb.debug(1, 'archiver: %s is excluded, covered by gcc-source' % pn)
103 return
104
Andrew Geissler615f2f12022-07-15 14:00:58 -0500105 # TARGET_SYS in ARCHIVER_ARCH will break the stamp for gcc-source in multiconfig
106 if pn.startswith('gcc-source'):
107 d.setVar('ARCHIVER_ARCH', "allarch")
108
Brad Bishop79641f22019-09-10 07:20:22 -0400109 def hasTask(task):
110 return bool(d.getVarFlag(task, "task", False)) and not bool(d.getVarFlag(task, "noexec", False))
111
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500112 ar_src = d.getVarFlag('ARCHIVER_MODE', 'src')
113 ar_dumpdata = d.getVarFlag('ARCHIVER_MODE', 'dumpdata')
114 ar_recipe = d.getVarFlag('ARCHIVER_MODE', 'recipe')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500115
116 if ar_src == "original":
117 d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_original' % pn)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500118 # 'patched' and 'configured' invoke do_unpack_and_patch because
119 # do_ar_patched resp. do_ar_configured depend on it, but for 'original'
120 # we have to add it explicitly.
121 if d.getVarFlag('ARCHIVER_MODE', 'diff') == '1':
122 d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_unpack_and_patch' % pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500123 elif ar_src == "patched":
124 d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_patched' % pn)
125 elif ar_src == "configured":
126 # We can't use "addtask do_ar_configured after do_configure" since it
Andrew Geisslerc926e172021-05-07 16:11:35 -0500127 # will cause the deptask of do_populate_sysroot to run no matter what
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128 # archives we need, so we add the depends here.
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500129
130 # There is a corner case with "gcc-source-${PV}" recipes, they don't have
131 # the "do_configure" task, so we need to use "do_preconfigure"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800132 if hasTask("do_preconfigure"):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500133 d.appendVarFlag('do_ar_configured', 'depends', ' %s:do_preconfigure' % pn)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800134 elif hasTask("do_configure"):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500135 d.appendVarFlag('do_ar_configured', 'depends', ' %s:do_configure' % pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500136 d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_configured' % pn)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500137 elif ar_src == "mirror":
138 d.appendVarFlag('do_deploy_archives', 'depends', '%s:do_ar_mirror' % pn)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500139
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500140 elif ar_src:
141 bb.fatal("Invalid ARCHIVER_MODE[src]: %s" % ar_src)
142
143 if ar_dumpdata == "1":
144 d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_dumpdata' % pn)
145
146 if ar_recipe == "1":
147 d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_recipe' % pn)
148
Brad Bishop316dfdd2018-06-25 12:45:53 -0400149 # Output the SRPM package
150 if d.getVarFlag('ARCHIVER_MODE', 'srpm') == "1" and d.getVar('PACKAGES'):
Brad Bishop79641f22019-09-10 07:20:22 -0400151 if "package_rpm" not in d.getVar('PACKAGE_CLASSES'):
152 bb.fatal("ARCHIVER_MODE[srpm] needs package_rpm in PACKAGE_CLASSES")
153
154 # Some recipes do not have any packaging tasks
155 if hasTask("do_package_write_rpm"):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500156 d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_package_write_rpm' % pn)
Brad Bishop19323692019-04-05 15:28:33 -0400157 d.appendVarFlag('do_package_write_rpm', 'dirs', ' ${ARCHIVER_RPMTOPDIR}')
158 d.appendVarFlag('do_package_write_rpm', 'sstate-inputdirs', ' ${ARCHIVER_RPMTOPDIR}')
159 d.appendVarFlag('do_package_write_rpm', 'sstate-outputdirs', ' ${DEPLOY_DIR_SRC}')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500160 if ar_dumpdata == "1":
161 d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_dumpdata' % pn)
162 if ar_recipe == "1":
163 d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_recipe' % pn)
164 if ar_src == "original":
165 d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_original' % pn)
166 elif ar_src == "patched":
167 d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_patched' % pn)
168 elif ar_src == "configured":
169 d.appendVarFlag('do_package_write_rpm', 'depends', ' %s:do_ar_configured' % pn)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500170}
171
Andrew Geisslerc926e172021-05-07 16:11:35 -0500172# Take all the sources for a recipe and put them in WORKDIR/archiver-work/.
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500173# Files in SRC_URI are copied directly, anything that's a directory
174# (e.g. git repositories) is "unpacked" and then put into a tarball.
175python do_ar_original() {
176
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500177 import shutil, tempfile
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500178
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500179 if d.getVarFlag('ARCHIVER_MODE', 'src') != "original":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500180 return
181
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500182 ar_outdir = d.getVar('ARCHIVER_OUTDIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500183 bb.note('Archiving the original source...')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500184 urls = d.getVar("SRC_URI").split()
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600185 # destsuffix (git fetcher) and subdir (everything else) are allowed to be
186 # absolute paths (for example, destsuffix=${S}/foobar).
187 # That messes with unpacking inside our tmpdir below, because the fetchers
188 # will then unpack in that directory and completely ignore the tmpdir.
189 # That breaks parallel tasks relying on ${S}, like do_compile.
190 #
191 # To solve this, we remove these parameters from all URLs.
192 # We do this even for relative paths because it makes the content of the
193 # archives more useful (no extra paths that are only used during
194 # compilation).
195 for i, url in enumerate(urls):
196 decoded = bb.fetch2.decodeurl(url)
197 for param in ('destsuffix', 'subdir'):
198 if param in decoded[5]:
199 del decoded[5][param]
200 encoded = bb.fetch2.encodeurl(decoded)
201 urls[i] = encoded
Andrew Geissler1e34c2d2020-05-29 16:02:59 -0500202
203 # Cleanup SRC_URI before call bb.fetch2.Fetch() since now SRC_URI is in the
204 # variable "urls", otherwise there might be errors like:
205 # The SRCREV_FORMAT variable must be set when multiple SCMs are used
206 ld = bb.data.createCopy(d)
207 ld.setVar('SRC_URI', '')
208 fetch = bb.fetch2.Fetch(urls, ld)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600209 tarball_suffix = {}
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500210 for url in fetch.urls:
211 local = fetch.localpath(url).rstrip("/");
212 if os.path.isfile(local):
213 shutil.copy(local, ar_outdir)
214 elif os.path.isdir(local):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500215 tmpdir = tempfile.mkdtemp(dir=d.getVar('ARCHIVER_WORKDIR'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500216 fetch.unpack(tmpdir, (url,))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600217 # To handle recipes with more than one source, we add the "name"
218 # URL parameter as suffix. We treat it as an error when
219 # there's more than one URL without a name, or a name gets reused.
220 # This is an additional safety net, in practice the name has
221 # to be set when using the git fetcher, otherwise SRCREV cannot
222 # be set separately for each URL.
223 params = bb.fetch2.decodeurl(url)[5]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500224 type = bb.fetch2.decodeurl(url)[0]
225 location = bb.fetch2.decodeurl(url)[2]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600226 name = params.get('name', '')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500227 if type.lower() == 'file':
228 name_tmp = location.rstrip("*").rstrip("/")
229 name = os.path.basename(name_tmp)
230 else:
231 if name in tarball_suffix:
232 if not name:
233 bb.fatal("Cannot determine archive names for original source because 'name' URL parameter is unset in more than one URL. Add it to at least one of these: %s %s" % (tarball_suffix[name], url))
234 else:
235 bb.fatal("Cannot determine archive names for original source because 'name=' URL parameter '%s' is used twice. Make it unique in: %s %s" % (tarball_suffix[name], url))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600236 tarball_suffix[name] = url
237 create_tarball(d, tmpdir + '/.', name, ar_outdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500238
239 # Emit patch series files for 'original'
240 bb.note('Writing patch series files...')
241 for patch in src_patches(d):
242 _, _, local, _, _, parm = bb.fetch.decodeurl(patch)
243 patchdir = parm.get('patchdir')
244 if patchdir:
245 series = os.path.join(ar_outdir, 'series.subdir.%s' % patchdir.replace('/', '_'))
246 else:
247 series = os.path.join(ar_outdir, 'series')
248
249 with open(series, 'a') as s:
250 s.write('%s -p%s\n' % (os.path.basename(local), parm['striplevel']))
251}
252
253python do_ar_patched() {
254
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500255 if d.getVarFlag('ARCHIVER_MODE', 'src') != 'patched':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500256 return
257
258 # Get the ARCHIVER_OUTDIR before we reset the WORKDIR
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500259 ar_outdir = d.getVar('ARCHIVER_OUTDIR')
Brad Bishopa34c0302019-09-23 22:34:48 -0400260 if not is_work_shared(d):
261 ar_workdir = d.getVar('ARCHIVER_WORKDIR')
262 d.setVar('WORKDIR', ar_workdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500263 bb.note('Archiving the patched source...')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500264 create_tarball(d, d.getVar('S'), 'patched', ar_outdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500265}
266
267python do_ar_configured() {
268 import shutil
269
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500270 # Forcibly expand the sysroot paths as we're about to change WORKDIR
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500271 d.setVar('STAGING_DIR_HOST', d.getVar('STAGING_DIR_HOST'))
272 d.setVar('STAGING_DIR_TARGET', d.getVar('STAGING_DIR_TARGET'))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500273 d.setVar('RECIPE_SYSROOT', d.getVar('RECIPE_SYSROOT'))
274 d.setVar('RECIPE_SYSROOT_NATIVE', d.getVar('RECIPE_SYSROOT_NATIVE'))
275
276 ar_outdir = d.getVar('ARCHIVER_OUTDIR')
277 if d.getVarFlag('ARCHIVER_MODE', 'src') == 'configured':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500278 bb.note('Archiving the configured source...')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500279 pn = d.getVar('PN')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500280 # "gcc-source-${PV}" recipes don't have "do_configure"
281 # task, so we need to run "do_preconfigure" instead
282 if pn.startswith("gcc-source-"):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500283 d.setVar('WORKDIR', d.getVar('ARCHIVER_WORKDIR'))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500284 bb.build.exec_func('do_preconfigure', d)
285
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500286 # The libtool-native's do_configure will remove the
287 # ${STAGING_DATADIR}/aclocal/libtool.m4, so we can't re-run the
288 # do_configure, we archive the already configured ${S} to
289 # instead of.
Patrick Williams213cb262021-08-07 19:21:33 -0500290 # The kernel class functions require it to be on work-shared, we
291 # don't unpack, patch, configure again, just archive the already
292 # configured ${S}
293 elif not (pn == 'libtool-native' or is_work_shared(d)):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800294 def runTask(task):
295 prefuncs = d.getVarFlag(task, 'prefuncs') or ''
296 for func in prefuncs.split():
297 if func != "sysroot_cleansstate":
298 bb.build.exec_func(func, d)
299 bb.build.exec_func(task, d)
300 postfuncs = d.getVarFlag(task, 'postfuncs') or ''
301 for func in postfuncs.split():
302 if func != 'do_qa_configure':
303 bb.build.exec_func(func, d)
304
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500305 # Change the WORKDIR to make do_configure run in another dir.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500306 d.setVar('WORKDIR', d.getVar('ARCHIVER_WORKDIR'))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800307
308 preceeds = bb.build.preceedtask('do_configure', False, d)
309 for task in preceeds:
310 if task != 'do_patch' and task != 'do_prepare_recipe_sysroot':
311 runTask(task)
312 runTask('do_configure')
313
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500314 srcdir = d.getVar('S')
315 builddir = d.getVar('B')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500316 if srcdir != builddir:
317 if os.path.exists(builddir):
318 oe.path.copytree(builddir, os.path.join(srcdir, \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500319 'build.%s.ar_configured' % d.getVar('PF')))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320 create_tarball(d, srcdir, 'configured', ar_outdir)
321}
322
Andrew Geissler82c905d2020-04-13 13:39:40 -0500323python do_ar_mirror() {
324 import subprocess
325
326 src_uri = (d.getVar('SRC_URI') or '').split()
327 if len(src_uri) == 0:
328 return
329
330 dl_dir = d.getVar('DL_DIR')
331 mirror_exclusions = (d.getVar('ARCHIVER_MIRROR_EXCLUDE') or '').split()
332 mirror_mode = d.getVarFlag('ARCHIVER_MODE', 'mirror')
333 have_mirror_tarballs = d.getVar('BB_GENERATE_MIRROR_TARBALLS')
334
335 if mirror_mode == 'combined':
336 destdir = d.getVar('ARCHIVER_COMBINED_MIRRORDIR')
337 elif mirror_mode == 'split':
338 destdir = d.getVar('ARCHIVER_OUTDIR')
339 else:
340 bb.fatal('Invalid ARCHIVER_MODE[mirror]: %s' % (mirror_mode))
341
342 if not have_mirror_tarballs:
343 bb.fatal('Using `ARCHIVER_MODE[src] = "mirror"` depends on setting `BB_GENERATE_MIRROR_TARBALLS = "1"`')
344
345 def is_excluded(url):
346 for prefix in mirror_exclusions:
347 if url.startswith(prefix):
348 return True
349 return False
350
351 bb.note('Archiving the source as a mirror...')
352
353 bb.utils.mkdirhier(destdir)
354
355 fetcher = bb.fetch2.Fetch(src_uri, d)
356
Andrew Geissler5a43b432020-06-13 10:46:56 -0500357 for ud in fetcher.expanded_urldata():
358 if is_excluded(ud.url):
359 bb.note('Skipping excluded url: %s' % (ud.url))
Andrew Geissler82c905d2020-04-13 13:39:40 -0500360 continue
361
Andrew Geissler5a43b432020-06-13 10:46:56 -0500362 bb.note('Archiving url: %s' % (ud.url))
Andrew Geissler82c905d2020-04-13 13:39:40 -0500363 ud.setup_localpath(d)
364 localpath = None
365
366 # Check for mirror tarballs first. We will archive the first mirror
367 # tarball that we find as it's assumed that we just need one.
368 for mirror_fname in ud.mirrortarballs:
369 mirror_path = os.path.join(dl_dir, mirror_fname)
370 if os.path.exists(mirror_path):
371 bb.note('Found mirror tarball: %s' % (mirror_path))
372 localpath = mirror_path
373 break
374
375 if len(ud.mirrortarballs) and not localpath:
376 bb.warn('Mirror tarballs are listed for a source but none are present. ' \
377 'Falling back to original download.\n' \
Andrew Geissler5a43b432020-06-13 10:46:56 -0500378 'SRC_URI = %s' % (ud.url))
Andrew Geissler82c905d2020-04-13 13:39:40 -0500379
380 # Check original download
381 if not localpath:
382 bb.note('Using original download: %s' % (ud.localpath))
383 localpath = ud.localpath
384
385 if not localpath or not os.path.exists(localpath):
386 bb.fatal('Original download is missing for a source.\n' \
Andrew Geissler5a43b432020-06-13 10:46:56 -0500387 'SRC_URI = %s' % (ud.url))
Andrew Geissler82c905d2020-04-13 13:39:40 -0500388
389 # We now have an appropriate localpath
390 bb.note('Copying source mirror')
391 cmd = 'cp -fpPRH %s %s' % (localpath, destdir)
392 subprocess.check_call(cmd, shell=True)
393}
394
Brad Bishopc4ea0752018-11-15 14:30:15 -0800395def exclude_useless_paths(tarinfo):
396 if tarinfo.isdir():
397 if tarinfo.name.endswith('/temp') or tarinfo.name.endswith('/patches') or tarinfo.name.endswith('/.pc'):
398 return None
399 elif tarinfo.name == 'temp' or tarinfo.name == 'patches' or tarinfo.name == '.pc':
400 return None
401 return tarinfo
402
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500403def create_tarball(d, srcdir, suffix, ar_outdir):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500404 """
405 create the tarball from srcdir
406 """
407 import tarfile
408
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500409 # Make sure we are only creating a single tarball for gcc sources
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500410 if (d.getVar('SRC_URI') == ""):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500411 return
412
Brad Bishop316dfdd2018-06-25 12:45:53 -0400413 # For the kernel archive, srcdir may just be a link to the
414 # work-shared location. Use os.path.realpath to make sure
415 # that we archive the actual directory and not just the link.
416 srcdir = os.path.realpath(srcdir)
417
Andrew Geisslereff27472021-10-29 15:35:00 -0500418 compression_method = d.getVarFlag('ARCHIVER_MODE', 'compression')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500419 bb.utils.mkdirhier(ar_outdir)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500420 if suffix:
Andrew Geisslereff27472021-10-29 15:35:00 -0500421 filename = '%s-%s.tar.%s' % (d.getVar('PF'), suffix, compression_method)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500422 else:
Andrew Geisslereff27472021-10-29 15:35:00 -0500423 filename = '%s.tar.%s' % (d.getVar('PF'), compression_method)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500424 tarname = os.path.join(ar_outdir, filename)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500425
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500426 bb.note('Creating %s' % tarname)
Andrew Geisslereff27472021-10-29 15:35:00 -0500427 tar = tarfile.open(tarname, 'w:%s' % compression_method)
Brad Bishopc4ea0752018-11-15 14:30:15 -0800428 tar.add(srcdir, arcname=os.path.basename(srcdir), filter=exclude_useless_paths)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500429 tar.close()
430
431# creating .diff.gz between source.orig and source
432def create_diff_gz(d, src_orig, src, ar_outdir):
433
434 import subprocess
435
436 if not os.path.isdir(src) or not os.path.isdir(src_orig):
437 return
438
439 # The diff --exclude can't exclude the file with path, so we copy
440 # the patched source, and remove the files that we'd like to
441 # exclude.
442 src_patched = src + '.patched'
443 oe.path.copyhardlinktree(src, src_patched)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500444 for i in d.getVarFlag('ARCHIVER_MODE', 'diff-exclude').split():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500445 bb.utils.remove(os.path.join(src_orig, i), recurse=True)
446 bb.utils.remove(os.path.join(src_patched, i), recurse=True)
447
448 dirname = os.path.dirname(src)
449 basename = os.path.basename(src)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500450 bb.utils.mkdirhier(ar_outdir)
451 cwd = os.getcwd()
452 try:
453 os.chdir(dirname)
454 out_file = os.path.join(ar_outdir, '%s-diff.gz' % d.getVar('PF'))
455 diff_cmd = 'diff -Naur %s.orig %s.patched | gzip -c > %s' % (basename, basename, out_file)
456 subprocess.check_call(diff_cmd, shell=True)
457 bb.utils.remove(src_patched, recurse=True)
458 finally:
459 os.chdir(cwd)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500460
Brad Bishop316dfdd2018-06-25 12:45:53 -0400461def is_work_shared(d):
462 pn = d.getVar('PN')
463 return bb.data.inherits_class('kernel', d) or pn.startswith('gcc-source')
464
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500465# Run do_unpack and do_patch
466python do_unpack_and_patch() {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500467 if d.getVarFlag('ARCHIVER_MODE', 'src') not in \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500468 [ 'patched', 'configured'] and \
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500469 d.getVarFlag('ARCHIVER_MODE', 'diff') != '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500470 return
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500471 ar_outdir = d.getVar('ARCHIVER_OUTDIR')
472 ar_workdir = d.getVar('ARCHIVER_WORKDIR')
473 ar_sysroot_native = d.getVar('STAGING_DIR_NATIVE')
474 pn = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500475
Andrew Geisslerc926e172021-05-07 16:11:35 -0500476 # The kernel class functions require it to be on work-shared, so we don't change WORKDIR
Brad Bishop316dfdd2018-06-25 12:45:53 -0400477 if not is_work_shared(d):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500478 # Change the WORKDIR to make do_unpack do_patch run in another dir.
479 d.setVar('WORKDIR', ar_workdir)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500480 # Restore the original path to recipe's native sysroot (it's relative to WORKDIR).
481 d.setVar('STAGING_DIR_NATIVE', ar_sysroot_native)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500482
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500483 # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the
484 # possibly requiring of the following tasks (such as some recipes's
485 # do_patch required 'B' existed).
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500486 bb.utils.mkdirhier(d.getVar('B'))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500487
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500488 bb.build.exec_func('do_unpack', d)
489
490 # Save the original source for creating the patches
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500491 if d.getVarFlag('ARCHIVER_MODE', 'diff') == '1':
492 src = d.getVar('S').rstrip('/')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500493 src_orig = '%s.orig' % src
494 oe.path.copytree(src, src_orig)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500495
Patrick Williams213cb262021-08-07 19:21:33 -0500496 if bb.data.inherits_class('dos2unix', d):
497 bb.build.exec_func('do_convert_crlf_to_lf', d)
498
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500499 # Make sure gcc and kernel sources are patched only once
Brad Bishop316dfdd2018-06-25 12:45:53 -0400500 if not (d.getVar('SRC_URI') == "" or is_work_shared(d)):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500501 bb.build.exec_func('do_patch', d)
502
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500503 # Create the patches
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500504 if d.getVarFlag('ARCHIVER_MODE', 'diff') == '1':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500505 bb.note('Creating diff gz...')
506 create_diff_gz(d, src_orig, src, ar_outdir)
507 bb.utils.remove(src_orig, recurse=True)
508}
509
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500510# BBINCLUDED is special (excluded from basehash signature
511# calculation). Using it in a task signature can cause "basehash
512# changed" errors.
513#
514# Depending on BBINCLUDED also causes do_ar_recipe to run again
515# for unrelated changes, like adding or removing buildhistory.bbclass.
516#
517# For these reasons we ignore the dependency completely. The versioning
518# of the output file ensures that we create it each time the recipe
519# gets rebuilt, at least as long as a PR server is used. We also rely
520# on that mechanism to catch changes in the file content, because the
Andrew Geisslerc926e172021-05-07 16:11:35 -0500521# file content is not part of the task signature either.
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500522do_ar_recipe[vardepsexclude] += "BBINCLUDED"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500523python do_ar_recipe () {
524 """
525 archive the recipe, including .bb and .inc.
526 """
527 import re
528 import shutil
529
530 require_re = re.compile( r"require\s+(.+)" )
531 include_re = re.compile( r"include\s+(.+)" )
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500532 bbfile = d.getVar('FILE')
533 outdir = os.path.join(d.getVar('WORKDIR'), \
534 '%s-recipe' % d.getVar('PF'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500535 bb.utils.mkdirhier(outdir)
536 shutil.copy(bbfile, outdir)
537
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500538 pn = d.getVar('PN')
539 bbappend_files = d.getVar('BBINCLUDED').split()
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500540 # If recipe name is aa, we need to match files like aa.bbappend and aa_1.1.bbappend
541 # Files like aa1.bbappend or aa1_1.1.bbappend must be excluded.
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500542 bbappend_re = re.compile( r".*/%s_[^/]*\.bbappend$" % re.escape(pn))
543 bbappend_re1 = re.compile( r".*/%s\.bbappend$" % re.escape(pn))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500544 for file in bbappend_files:
545 if bbappend_re.match(file) or bbappend_re1.match(file):
546 shutil.copy(file, outdir)
547
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500548 dirname = os.path.dirname(bbfile)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500549 bbpath = '%s:%s' % (dirname, d.getVar('BBPATH'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500550 f = open(bbfile, 'r')
551 for line in f.readlines():
552 incfile = None
553 if require_re.match(line):
554 incfile = require_re.match(line).group(1)
555 elif include_re.match(line):
556 incfile = include_re.match(line).group(1)
557 if incfile:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500558 incfile = d.expand(incfile)
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500559 if incfile:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500560 incfile = bb.utils.which(bbpath, incfile)
Brad Bishop1d80a2e2019-11-15 16:35:03 -0500561 if incfile:
562 shutil.copy(incfile, outdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500563
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500564 create_tarball(d, outdir, 'recipe', d.getVar('ARCHIVER_OUTDIR'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500565 bb.utils.remove(outdir, recurse=True)
566}
567
568python do_dumpdata () {
569 """
570 dump environment data to ${PF}-showdata.dump
571 """
572
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500573 dumpfile = os.path.join(d.getVar('ARCHIVER_OUTDIR'), \
574 '%s-showdata.dump' % d.getVar('PF'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500575 bb.note('Dumping metadata into %s' % dumpfile)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500576 with open(dumpfile, "w") as f:
577 # emit variables and shell functions
578 bb.data.emit_env(f, d, True)
579 # emit the metadata which isn't valid shell
580 for e in d.keys():
581 if d.getVarFlag(e, "python", False):
582 f.write("\npython %s () {\n%s}\n" % (e, d.getVar(e, False)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500583}
584
585SSTATETASKS += "do_deploy_archives"
586do_deploy_archives () {
Andrew Geissler78b72792022-06-14 06:47:25 -0500587 bbnote "Deploying source archive files from ${ARCHIVER_TOPDIR} to ${DEPLOY_DIR_SRC}."
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500588}
589python do_deploy_archives_setscene () {
590 sstate_setscene(d)
591}
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500592do_deploy_archives[dirs] = "${ARCHIVER_TOPDIR}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500593do_deploy_archives[sstate-inputdirs] = "${ARCHIVER_TOPDIR}"
594do_deploy_archives[sstate-outputdirs] = "${DEPLOY_DIR_SRC}"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500595addtask do_deploy_archives_setscene
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500596
597addtask do_ar_original after do_unpack
Andrew Geissler475cb722020-07-10 16:00:51 -0500598addtask do_unpack_and_patch after do_patch do_preconfigure
599addtask do_ar_patched after do_unpack_and_patch
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500600addtask do_ar_configured after do_unpack_and_patch
Andrew Geissler82c905d2020-04-13 13:39:40 -0500601addtask do_ar_mirror after do_fetch
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500602addtask do_dumpdata
603addtask do_ar_recipe
Andrew Geissler1e34c2d2020-05-29 16:02:59 -0500604addtask do_deploy_archives
605do_build[recrdeptask] += "do_deploy_archives"
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600606do_rootfs[recrdeptask] += "do_deploy_archives"
Andrew Geissler1e34c2d2020-05-29 16:02:59 -0500607do_populate_sdk[recrdeptask] += "do_deploy_archives"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500608
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500609python () {
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500610 # Add tasks in the correct order, specifically for linux-yocto to avoid race condition.
611 # sstatesig.py:sstate_rundepfilter has special support that excludes this dependency
612 # so that do_kernel_configme does not need to run again when do_unpack_and_patch
613 # gets added or removed (by adding or removing archiver.bbclass).
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500614 if bb.data.inherits_class('kernel-yocto', d):
615 bb.build.addtask('do_kernel_configme', 'do_configure', 'do_unpack_and_patch', d)
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500616}