blob: 476876e03d0f8c744057c0e84546343ec85d3fcf [file] [log] [blame]
Patrick Williams92b42cb2022-09-03 06:53:57 -05001#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: MIT
5#
6
7# Extensible SDK
8
9inherit populate_sdk_base
10
11# Used to override TOOLCHAIN_HOST_TASK in the eSDK case
12TOOLCHAIN_HOST_TASK_ESDK = " \
13 meta-environment-extsdk-${MACHINE} \
14 "
15
16SDK_RELOCATE_AFTER_INSTALL:task-populate-sdk-ext = "0"
17
18SDK_EXT = ""
19SDK_EXT:task-populate-sdk-ext = "-ext"
20
21# Options are full or minimal
22SDK_EXT_TYPE ?= "full"
23SDK_INCLUDE_PKGDATA ?= "0"
24SDK_INCLUDE_TOOLCHAIN ?= "${@'1' if d.getVar('SDK_EXT_TYPE') == 'full' else '0'}"
25SDK_INCLUDE_NATIVESDK ?= "0"
26SDK_INCLUDE_BUILDTOOLS ?= '1'
27
28SDK_RECRDEP_TASKS ?= ""
29SDK_CUSTOM_TEMPLATECONF ?= "0"
30
31ESDK_LOCALCONF_ALLOW ?= ""
32ESDK_LOCALCONF_REMOVE ?= "CONF_VERSION \
33 BB_NUMBER_THREADS \
34 BB_NUMBER_PARSE_THREADS \
35 PARALLEL_MAKE \
36 PRSERV_HOST \
37 SSTATE_MIRRORS \
38 DL_DIR \
39 SSTATE_DIR \
40 TMPDIR \
41 BB_SERVER_TIMEOUT \
42 "
43ESDK_CLASS_INHERIT_DISABLE ?= "buildhistory icecc"
44SDK_UPDATE_URL ?= ""
45
46SDK_TARGETS ?= "${PN}"
47
48def get_sdk_install_targets(d, images_only=False):
49 sdk_install_targets = ''
50 if images_only or d.getVar('SDK_EXT_TYPE') != 'minimal':
51 sdk_install_targets = d.getVar('SDK_TARGETS')
52
53 depd = d.getVar('BB_TASKDEPDATA', False)
54 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d)
55 tasklist.remove('do_build')
56 for v in depd.values():
57 if v[1] in tasklist:
58 if v[0] not in sdk_install_targets:
59 sdk_install_targets += ' {}'.format(v[0])
60
61 if not images_only:
62 if d.getVar('SDK_INCLUDE_PKGDATA') == '1':
63 sdk_install_targets += ' meta-world-pkgdata:do_allpackagedata'
64 if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1':
65 sdk_install_targets += ' meta-extsdk-toolchain:do_populate_sysroot'
66
67 return sdk_install_targets
68
69get_sdk_install_targets[vardepsexclude] = "BB_TASKDEPDATA"
70
71OE_INIT_ENV_SCRIPT ?= "oe-init-build-env"
72
73# The files from COREBASE that you want preserved in the COREBASE copied
74# into the sdk. This allows someone to have their own setup scripts in
75# COREBASE be preserved as well as untracked files.
76COREBASE_FILES ?= " \
77 oe-init-build-env \
78 scripts \
79 LICENSE \
80 .templateconf \
81"
82
83SDK_DIR:task-populate-sdk-ext = "${WORKDIR}/sdk-ext"
84B:task-populate-sdk-ext = "${SDK_DIR}"
85TOOLCHAINEXT_OUTPUTNAME ?= "${SDK_NAME}-toolchain-ext-${SDK_VERSION}"
86TOOLCHAIN_OUTPUTNAME:task-populate-sdk-ext = "${TOOLCHAINEXT_OUTPUTNAME}"
87
88SDK_EXT_TARGET_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest"
89SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest"
90
91python write_target_sdk_ext_manifest () {
92 from oe.sdk import get_extra_sdkinfo
93 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
94 extra_info = get_extra_sdkinfo(sstate_dir)
95
96 target = d.getVar('TARGET_SYS')
97 target_multimach = d.getVar('MULTIMACH_TARGET_SYS')
98 real_target_multimach = d.getVar('REAL_MULTIMACH_TARGET_SYS')
99
100 pkgs = {}
101 os.makedirs(os.path.dirname(d.getVar('SDK_EXT_TARGET_MANIFEST')), exist_ok=True)
102 with open(d.getVar('SDK_EXT_TARGET_MANIFEST'), 'w') as f:
103 for fn in extra_info['filesizes']:
104 info = fn.split(':')
105 if info[2] in (target, target_multimach, real_target_multimach) \
106 or info[5] == 'allarch':
107 if not info[1] in pkgs:
108 f.write("%s %s %s\n" % (info[1], info[2], info[3]))
109 pkgs[info[1]] = {}
110}
111python write_host_sdk_ext_manifest () {
112 from oe.sdk import get_extra_sdkinfo
113 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
114 extra_info = get_extra_sdkinfo(sstate_dir)
115 host = d.getVar('BUILD_SYS')
116 with open(d.getVar('SDK_EXT_HOST_MANIFEST'), 'w') as f:
117 for fn in extra_info['filesizes']:
118 info = fn.split(':')
119 if info[2] == host:
120 f.write("%s %s %s\n" % (info[1], info[2], info[3]))
121}
122
Andrew Geisslerc5535c92023-01-27 16:10:19 -0600123SDK_POSTPROCESS_COMMAND:append:task-populate-sdk-ext = " write_target_sdk_ext_manifest; write_host_sdk_ext_manifest; "
Patrick Williams92b42cb2022-09-03 06:53:57 -0500124
125SDK_TITLE:task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME') or d.getVar('DISTRO')} Extensible SDK"
126
127def clean_esdk_builddir(d, sdkbasepath):
128 """Clean up traces of the fake build for create_filtered_tasklist()"""
129 import shutil
130 cleanpaths = ['cache', 'tmp']
131 for pth in cleanpaths:
132 fullpth = os.path.join(sdkbasepath, pth)
133 if os.path.isdir(fullpth):
134 shutil.rmtree(fullpth)
135 elif os.path.isfile(fullpth):
136 os.remove(fullpth)
137
138def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath):
139 """
140 Create a filtered list of tasks. Also double-checks that the build system
141 within the SDK basically works and required sstate artifacts are available.
142 """
143 import tempfile
144 import shutil
145 import oe.copy_buildsystem
146
147 # Create a temporary build directory that we can pass to the env setup script
148 shutil.copyfile(sdkbasepath + '/conf/local.conf', sdkbasepath + '/conf/local.conf.bak')
149 try:
150 with open(sdkbasepath + '/conf/local.conf', 'a') as f:
151 # Force the use of sstate from the build system
152 f.write('\nSSTATE_DIR:forcevariable = "%s"\n' % d.getVar('SSTATE_DIR'))
153 f.write('SSTATE_MIRRORS:forcevariable = "file://universal/(.*) file://universal-4.9/\\1 file://universal-4.9/(.*) file://universal-4.8/\\1"\n')
154 # Ensure TMPDIR is the default so that clean_esdk_builddir() can delete it
155 f.write('TMPDIR:forcevariable = "${TOPDIR}/tmp"\n')
156 f.write('TCLIBCAPPEND:forcevariable = ""\n')
157 # Drop uninative if the build isn't using it (or else NATIVELSBSTRING will
158 # be different and we won't be able to find our native sstate)
159 if not bb.data.inherits_class('uninative', d):
160 f.write('INHERIT:remove = "uninative"\n')
161
162 # Unfortunately the default SDKPATH (or even a custom value) may contain characters that bitbake
163 # will not allow in its COREBASE path, so we need to rename the directory temporarily
164 temp_sdkbasepath = d.getVar('SDK_OUTPUT') + '/tmp-renamed-sdk'
165 # Delete any existing temp dir
166 try:
167 shutil.rmtree(temp_sdkbasepath)
168 except FileNotFoundError:
169 pass
170 bb.utils.rename(sdkbasepath, temp_sdkbasepath)
171 cmdprefix = '. %s .; ' % conf_initpath
172 logfile = d.getVar('WORKDIR') + '/tasklist_bb_log.txt'
173 try:
174 oe.copy_buildsystem.check_sstate_task_list(d, get_sdk_install_targets(d), tasklistfile, cmdprefix=cmdprefix, cwd=temp_sdkbasepath, logfile=logfile)
175 except bb.process.ExecutionError as e:
176 msg = 'Failed to generate filtered task list for extensible SDK:\n%s' % e.stdout.rstrip()
177 if 'attempted to execute unexpectedly and should have been setscened' in e.stdout:
178 msg += '\n----------\n\nNOTE: "attempted to execute unexpectedly and should have been setscened" errors indicate this may be caused by missing sstate artifacts that were likely produced in earlier builds, but have been subsequently deleted for some reason.\n'
179 bb.fatal(msg)
180 bb.utils.rename(temp_sdkbasepath, sdkbasepath)
181 # Clean out residue of running bitbake, which check_sstate_task_list()
182 # will effectively do
183 clean_esdk_builddir(d, sdkbasepath)
184 finally:
185 localconf = sdkbasepath + '/conf/local.conf'
186 if os.path.exists(localconf + '.bak'):
187 os.replace(localconf + '.bak', localconf)
188
189python copy_buildsystem () {
190 import re
191 import shutil
192 import glob
193 import oe.copy_buildsystem
194
195 oe_init_env_script = d.getVar('OE_INIT_ENV_SCRIPT')
196
197 conf_bbpath = ''
198 conf_initpath = ''
199 core_meta_subdir = ''
200
201 # Copy in all metadata layers + bitbake (as repositories)
202 buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d)
203 baseoutpath = d.getVar('SDK_OUTPUT') + '/' + d.getVar('SDKPATH')
204
205 #check if custome templateconf path is set
206 use_custom_templateconf = d.getVar('SDK_CUSTOM_TEMPLATECONF')
207
208 # Determine if we're building a derivative extensible SDK (from devtool build-sdk)
209 derivative = (d.getVar('SDK_DERIVATIVE') or '') == '1'
210 if derivative:
211 workspace_name = 'orig-workspace'
212 else:
213 workspace_name = None
214
215 corebase, sdkbblayers = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers', workspace_name)
216 conf_bbpath = os.path.join('layers', corebase, 'bitbake')
217
218 for path in os.listdir(baseoutpath + '/layers'):
219 relpath = os.path.join('layers', path, oe_init_env_script)
220 if os.path.exists(os.path.join(baseoutpath, relpath)):
221 conf_initpath = relpath
222
223 relpath = os.path.join('layers', path, 'scripts', 'devtool')
224 if os.path.exists(os.path.join(baseoutpath, relpath)):
225 scriptrelpath = os.path.dirname(relpath)
226
227 relpath = os.path.join('layers', path, 'meta')
228 if os.path.exists(os.path.join(baseoutpath, relpath, 'lib', 'oe')):
229 core_meta_subdir = relpath
230
231 d.setVar('oe_init_build_env_path', conf_initpath)
232 d.setVar('scriptrelpath', scriptrelpath)
233
234 # Write out config file for devtool
235 import configparser
Patrick Williams7784c422022-11-17 07:29:11 -0600236 config = configparser.ConfigParser()
Patrick Williams92b42cb2022-09-03 06:53:57 -0500237 config.add_section('General')
238 config.set('General', 'bitbake_subdir', conf_bbpath)
239 config.set('General', 'init_path', conf_initpath)
240 config.set('General', 'core_meta_subdir', core_meta_subdir)
241 config.add_section('SDK')
242 config.set('SDK', 'sdk_targets', d.getVar('SDK_TARGETS'))
243 updateurl = d.getVar('SDK_UPDATE_URL')
244 if updateurl:
245 config.set('SDK', 'updateserver', updateurl)
246 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf'))
247 with open(os.path.join(baseoutpath, 'conf', 'devtool.conf'), 'w') as f:
248 config.write(f)
249
250 unlockedsigs = os.path.join(baseoutpath, 'conf', 'unlocked-sigs.inc')
251 with open(unlockedsigs, 'w') as f:
252 pass
253
254 # Create a layer for new recipes / appends
255 bbpath = d.getVar('BBPATH')
256 env = os.environ.copy()
257 env['PYTHONDONTWRITEBYTECODE'] = '1'
Andrew Geissler517393d2023-01-13 08:55:19 -0600258 bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--layerseries', d.getVar("LAYERSERIES_CORENAMES"), '--create-only', os.path.join(baseoutpath, 'workspace')], env=env)
Patrick Williams92b42cb2022-09-03 06:53:57 -0500259
260 # Create bblayers.conf
261 bb.utils.mkdirhier(baseoutpath + '/conf')
262 with open(baseoutpath + '/conf/bblayers.conf', 'w') as f:
263 f.write('# WARNING: this configuration has been automatically generated and in\n')
264 f.write('# most cases should not be edited. If you need more flexibility than\n')
265 f.write('# this configuration provides, it is strongly suggested that you set\n')
266 f.write('# up a proper instance of the full build system and use that instead.\n\n')
267
268 # LCONF_VERSION may not be set, for example when using meta-poky
269 # so don't error if it isn't found
270 lconf_version = d.getVar('LCONF_VERSION', False)
271 if lconf_version is not None:
272 f.write('LCONF_VERSION = "%s"\n\n' % lconf_version)
273
274 f.write('BBPATH = "$' + '{TOPDIR}"\n')
275 f.write('SDKBASEMETAPATH = "$' + '{TOPDIR}"\n')
276 f.write('BBLAYERS := " \\\n')
277 for layerrelpath in sdkbblayers:
278 f.write(' $' + '{SDKBASEMETAPATH}/layers/%s \\\n' % layerrelpath)
279 f.write(' $' + '{SDKBASEMETAPATH}/workspace \\\n')
280 f.write(' "\n')
281
282 # Copy uninative tarball
283 # For now this is where uninative.bbclass expects the tarball
284 if bb.data.inherits_class('uninative', d):
285 uninative_file = d.expand('${UNINATIVE_DLDIR}/' + d.getVarFlag("UNINATIVE_CHECKSUM", d.getVar("BUILD_ARCH")) + '/${UNINATIVE_TARBALL}')
286 uninative_checksum = bb.utils.sha256_file(uninative_file)
287 uninative_outdir = '%s/downloads/uninative/%s' % (baseoutpath, uninative_checksum)
288 bb.utils.mkdirhier(uninative_outdir)
289 shutil.copy(uninative_file, uninative_outdir)
290
291 env_passthrough = (d.getVar('BB_ENV_PASSTHROUGH_ADDITIONS') or '').split()
292 env_passthrough_values = {}
293
294 # Create local.conf
295 builddir = d.getVar('TOPDIR')
296 if derivative and os.path.exists(builddir + '/conf/site.conf'):
297 shutil.copyfile(builddir + '/conf/site.conf', baseoutpath + '/conf/site.conf')
298 if derivative and os.path.exists(builddir + '/conf/auto.conf'):
299 shutil.copyfile(builddir + '/conf/auto.conf', baseoutpath + '/conf/auto.conf')
300 if derivative:
301 shutil.copyfile(builddir + '/conf/local.conf', baseoutpath + '/conf/local.conf')
302 else:
303 local_conf_allowed = (d.getVar('ESDK_LOCALCONF_ALLOW') or '').split()
304 local_conf_remove = (d.getVar('ESDK_LOCALCONF_REMOVE') or '').split()
305 def handle_var(varname, origvalue, op, newlines):
306 if varname in local_conf_remove or (origvalue.strip().startswith('/') and not varname in local_conf_allowed):
307 newlines.append('# Removed original setting of %s\n' % varname)
308 return None, op, 0, True
309 else:
310 if varname in env_passthrough:
311 env_passthrough_values[varname] = origvalue
312 return origvalue, op, 0, True
313 varlist = ['[^#=+ ]*']
314 oldlines = []
315 if os.path.exists(builddir + '/conf/site.conf'):
316 with open(builddir + '/conf/site.conf', 'r') as f:
317 oldlines += f.readlines()
318 if os.path.exists(builddir + '/conf/auto.conf'):
319 with open(builddir + '/conf/auto.conf', 'r') as f:
320 oldlines += f.readlines()
321 if os.path.exists(builddir + '/conf/local.conf'):
322 with open(builddir + '/conf/local.conf', 'r') as f:
323 oldlines += f.readlines()
324 (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var)
325
326 with open(baseoutpath + '/conf/local.conf', 'w') as f:
327 f.write('# WARNING: this configuration has been automatically generated and in\n')
328 f.write('# most cases should not be edited. If you need more flexibility than\n')
329 f.write('# this configuration provides, it is strongly suggested that you set\n')
330 f.write('# up a proper instance of the full build system and use that instead.\n\n')
331 for line in newlines:
332 if line.strip() and not line.startswith('#'):
333 f.write(line)
334 # Write a newline just in case there's none at the end of the original
335 f.write('\n')
336
337 f.write('TMPDIR = "${TOPDIR}/tmp"\n')
338 f.write('TCLIBCAPPEND = ""\n')
339 f.write('DL_DIR = "${TOPDIR}/downloads"\n')
340
341 if bb.data.inherits_class('uninative', d):
342 f.write('INHERIT += "%s"\n' % 'uninative')
343 f.write('UNINATIVE_CHECKSUM[%s] = "%s"\n\n' % (d.getVar('BUILD_ARCH'), uninative_checksum))
344 f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False))
345
346 # Some classes are not suitable for SDK, remove them from INHERIT
347 f.write('INHERIT:remove = "%s"\n' % d.getVar('ESDK_CLASS_INHERIT_DISABLE', False))
348
349 # Bypass the default connectivity check if any
350 f.write('CONNECTIVITY_CHECK_URIS = ""\n\n')
351
352 # This warning will come out if reverse dependencies for a task
353 # don't have sstate as well as the task itself. We already know
354 # this will be the case for the extensible sdk, so turn off the
355 # warning.
356 f.write('SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK = "none"\n\n')
357
358 # Warn if the sigs in the locked-signature file don't match
359 # the sig computed from the metadata.
360 f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n\n')
361
362 # We want to be able to set this without a full reparse
363 f.write('BB_HASHCONFIG_IGNORE_VARS:append = " SIGGEN_UNLOCKED_RECIPES"\n\n')
364
365 # Set up which tasks are ignored for run on install
366 f.write('BB_SETSCENE_ENFORCE_IGNORE_TASKS = "%:* *:do_shared_workdir *:do_rm_work wic-tools:* *:do_addto_recipe_sysroot"\n\n')
367
368 # Hide the config information from bitbake output (since it's fixed within the SDK)
369 f.write('BUILDCFG_HEADER = ""\n\n')
370
371 # Write METADATA_REVISION
Patrick Williams8e7b46e2023-05-01 14:19:06 -0500372 # Needs distro override so it can override the value set in the bbclass code (later than local.conf)
373 f.write('METADATA_REVISION:%s = "%s"\n\n' % (d.getVar('DISTRO'), d.getVar('METADATA_REVISION')))
Patrick Williams92b42cb2022-09-03 06:53:57 -0500374
375 f.write('# Provide a flag to indicate we are in the EXT_SDK Context\n')
376 f.write('WITHIN_EXT_SDK = "1"\n\n')
377
378 # Map gcc-dependent uninative sstate cache for installer usage
379 f.write('SSTATE_MIRRORS += " file://universal/(.*) file://universal-4.9/\\1 file://universal-4.9/(.*) file://universal-4.8/\\1"\n\n')
380
381 if d.getVar("PRSERV_HOST"):
382 # Override this, we now include PR data, so it should only point ot the local database
383 f.write('PRSERV_HOST = "localhost:0"\n\n')
384
385 # Allow additional config through sdk-extra.conf
386 fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d)
387 if fn:
388 with open(fn, 'r') as xf:
389 for line in xf:
390 f.write(line)
391
392 # If you define a sdk_extraconf() function then it can contain additional config
393 # (Though this is awkward; sdk-extra.conf should probably be used instead)
394 extraconf = (d.getVar('sdk_extraconf') or '').strip()
395 if extraconf:
396 # Strip off any leading / trailing spaces
397 for line in extraconf.splitlines():
398 f.write(line.strip() + '\n')
399
400 f.write('require conf/locked-sigs.inc\n')
401 f.write('require conf/unlocked-sigs.inc\n')
402
403 # Copy multiple configurations if they exist in the users config directory
404 if d.getVar('BBMULTICONFIG') is not None:
405 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf', 'multiconfig'))
406 for mc in d.getVar('BBMULTICONFIG').split():
407 dest_stub = "/conf/multiconfig/%s.conf" % (mc,)
408 if os.path.exists(builddir + dest_stub):
409 shutil.copyfile(builddir + dest_stub, baseoutpath + dest_stub)
410
411 cachedir = os.path.join(baseoutpath, 'cache')
412 bb.utils.mkdirhier(cachedir)
413 bb.parse.siggen.copy_unitaskhashes(cachedir)
414
415 # If PR Service is in use, we need to export this as well
416 bb.note('Do we have a pr database?')
417 if d.getVar("PRSERV_HOST"):
418 bb.note('Writing PR database...')
419 # Based on the code in classes/prexport.bbclass
420 import oe.prservice
421 #dump meta info of tables
422 localdata = d.createCopy()
423 localdata.setVar('PRSERV_DUMPOPT_COL', "1")
424 localdata.setVar('PRSERV_DUMPDIR', os.path.join(baseoutpath, 'conf'))
425 localdata.setVar('PRSERV_DUMPFILE', '${PRSERV_DUMPDIR}/prserv.inc')
426
427 bb.note('PR Database write to %s' % (localdata.getVar('PRSERV_DUMPFILE')))
428
429 retval = oe.prservice.prserv_dump_db(localdata)
430 if not retval:
431 bb.error("prexport_handler: export failed!")
432 return
433 (metainfo, datainfo) = retval
434 oe.prservice.prserv_export_tofile(localdata, metainfo, datainfo, True)
435
436 # Use templateconf.cfg file from builddir if exists
437 if os.path.exists(builddir + '/conf/templateconf.cfg') and use_custom_templateconf == '1':
438 shutil.copyfile(builddir + '/conf/templateconf.cfg', baseoutpath + '/conf/templateconf.cfg')
439 else:
440 # Write a templateconf.cfg
441 with open(baseoutpath + '/conf/templateconf.cfg', 'w') as f:
442 f.write('meta/conf/templates/default\n')
443 os.makedirs(os.path.join(baseoutpath, core_meta_subdir, 'conf/templates/default'), exist_ok=True)
444
445 # Ensure any variables set from the external environment (by way of
446 # BB_ENV_PASSTHROUGH_ADDITIONS) are set in the SDK's configuration
447 extralines = []
448 for name, value in env_passthrough_values.items():
449 actualvalue = d.getVar(name) or ''
450 if value != actualvalue:
451 extralines.append('%s = "%s"\n' % (name, actualvalue))
452 if extralines:
453 with open(baseoutpath + '/conf/local.conf', 'a') as f:
454 f.write('\n')
455 f.write('# Extra settings from environment:\n')
456 for line in extralines:
457 f.write(line)
458 f.write('\n')
459
460 # Filter the locked signatures file to just the sstate tasks we are interested in
461 excluded_targets = get_sdk_install_targets(d, images_only=True)
462 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc'
463 lockedsigs_pruned = baseoutpath + '/conf/locked-sigs.inc'
464 #nativesdk-only sigfile to merge into locked-sigs.inc
465 sdk_include_nativesdk = (d.getVar("SDK_INCLUDE_NATIVESDK") == '1')
466 nativesigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc'
467 nativesigfile_pruned = d.getVar('WORKDIR') + '/locked-sigs_nativesdk_pruned.inc'
468
469 if sdk_include_nativesdk:
470 oe.copy_buildsystem.prune_lockedsigs([],
471 excluded_targets.split(),
472 nativesigfile,
473 True,
474 nativesigfile_pruned)
475
476 oe.copy_buildsystem.merge_lockedsigs([],
477 sigfile,
478 nativesigfile_pruned,
479 sigfile)
480
481 oe.copy_buildsystem.prune_lockedsigs([],
482 excluded_targets.split(),
483 sigfile,
484 False,
485 lockedsigs_pruned)
486
487 sstate_out = baseoutpath + '/sstate-cache'
488 bb.utils.remove(sstate_out, True)
489
490 # uninative.bbclass sets NATIVELSBSTRING to 'universal%s' % oe.utils.host_gcc_version(d)
491 fixedlsbstring = "universal%s" % oe.utils.host_gcc_version(d)
492
493 sdk_include_toolchain = (d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1')
494 sdk_ext_type = d.getVar('SDK_EXT_TYPE')
495 if (sdk_ext_type != 'minimal' or sdk_include_toolchain or derivative) and not sdk_include_nativesdk:
496 # Create the filtered task list used to generate the sstate cache shipped with the SDK
497 tasklistfn = d.getVar('WORKDIR') + '/tasklist.txt'
498 create_filtered_tasklist(d, baseoutpath, tasklistfn, conf_initpath)
499 else:
500 tasklistfn = None
501
Patrick Williams92b42cb2022-09-03 06:53:57 -0500502 cachedir = os.path.join(baseoutpath, 'cache')
503 bb.utils.mkdirhier(cachedir)
504 bb.parse.siggen.copy_unitaskhashes(cachedir)
505
506 # Add packagedata if enabled
507 if d.getVar('SDK_INCLUDE_PKGDATA') == '1':
508 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base.inc'
509 lockedsigs_copy = d.getVar('WORKDIR') + '/locked-sigs-copy.inc'
510 shutil.move(lockedsigs_pruned, lockedsigs_base)
511 oe.copy_buildsystem.merge_lockedsigs(['do_packagedata'],
512 lockedsigs_base,
513 d.getVar('STAGING_DIR_HOST') + '/world-pkgdata/locked-sigs-pkgdata.inc',
514 lockedsigs_pruned,
515 lockedsigs_copy)
516
517 if sdk_include_toolchain:
518 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base2.inc'
519 lockedsigs_toolchain = d.expand("${STAGING_DIR}/${TUNE_PKGARCH}/meta-extsdk-toolchain/locked-sigs/locked-sigs-extsdk-toolchain.inc")
520 shutil.move(lockedsigs_pruned, lockedsigs_base)
521 oe.copy_buildsystem.merge_lockedsigs([],
522 lockedsigs_base,
523 lockedsigs_toolchain,
524 lockedsigs_pruned)
525 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_toolchain,
526 d.getVar('SSTATE_DIR'),
527 sstate_out, d,
528 fixedlsbstring,
529 filterfile=tasklistfn)
530
531 if sdk_ext_type == 'minimal':
532 if derivative:
533 # Assume the user is not going to set up an additional sstate
534 # mirror, thus we need to copy the additional artifacts (from
535 # workspace recipes) into the derivative SDK
536 lockedsigs_orig = d.getVar('TOPDIR') + '/conf/locked-sigs.inc'
537 if os.path.exists(lockedsigs_orig):
538 lockedsigs_extra = d.getVar('WORKDIR') + '/locked-sigs-extra.inc'
539 oe.copy_buildsystem.merge_lockedsigs(None,
540 lockedsigs_orig,
541 lockedsigs_pruned,
542 None,
543 lockedsigs_extra)
544 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra,
545 d.getVar('SSTATE_DIR'),
546 sstate_out, d,
547 fixedlsbstring,
548 filterfile=tasklistfn)
549 else:
550 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned,
551 d.getVar('SSTATE_DIR'),
552 sstate_out, d,
553 fixedlsbstring,
554 filterfile=tasklistfn)
555
556 # We don't need sstate do_package files
557 for root, dirs, files in os.walk(sstate_out):
558 for name in files:
559 if name.endswith("_package.tar.zst"):
560 f = os.path.join(root, name)
561 os.remove(f)
562
563 # Write manifest file
564 # Note: at the moment we cannot include the env setup script here to keep
565 # it updated, since it gets modified during SDK installation (see
566 # sdk_ext_postinst() below) thus the checksum we take here would always
567 # be different.
568 manifest_file_list = ['conf/*']
569 if d.getVar('BBMULTICONFIG') is not None:
570 manifest_file_list.append('conf/multiconfig/*')
571
572 esdk_manifest_excludes = (d.getVar('ESDK_MANIFEST_EXCLUDES') or '').split()
573 esdk_manifest_excludes_list = []
574 for exclude_item in esdk_manifest_excludes:
575 esdk_manifest_excludes_list += glob.glob(os.path.join(baseoutpath, exclude_item))
576 manifest_file = os.path.join(baseoutpath, 'conf', 'sdk-conf-manifest')
577 with open(manifest_file, 'w') as f:
578 for item in manifest_file_list:
579 for fn in glob.glob(os.path.join(baseoutpath, item)):
580 if fn == manifest_file or os.path.isdir(fn):
581 continue
582 if fn in esdk_manifest_excludes_list:
583 continue
584 chksum = bb.utils.sha256_file(fn)
585 f.write('%s\t%s\n' % (chksum, os.path.relpath(fn, baseoutpath)))
586}
587
588def get_current_buildtools(d):
589 """Get the file name of the current buildtools installer"""
590 import glob
591 btfiles = glob.glob(os.path.join(d.getVar('SDK_DEPLOY'), '*-buildtools-nativesdk-standalone-*.sh'))
592 btfiles.sort(key=os.path.getctime)
593 return os.path.basename(btfiles[-1])
594
595def get_sdk_required_utilities(buildtools_fn, d):
596 """Find required utilities that aren't provided by the buildtools"""
597 sanity_required_utilities = (d.getVar('SANITY_REQUIRED_UTILITIES') or '').split()
598 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}gcc'))
599 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}g++'))
600 if buildtools_fn:
601 buildtools_installer = os.path.join(d.getVar('SDK_DEPLOY'), buildtools_fn)
602 filelist, _ = bb.process.run('%s -l' % buildtools_installer)
603 else:
604 buildtools_installer = None
605 filelist = ""
606 localdata = bb.data.createCopy(d)
607 localdata.setVar('SDKPATH', '.')
608 sdkpathnative = localdata.getVar('SDKPATHNATIVE')
609 sdkbindirs = [localdata.getVar('bindir_nativesdk'),
610 localdata.getVar('sbindir_nativesdk'),
611 localdata.getVar('base_bindir_nativesdk'),
612 localdata.getVar('base_sbindir_nativesdk')]
613 for line in filelist.splitlines():
614 splitline = line.split()
615 if len(splitline) > 5:
616 fn = splitline[5]
617 if not fn.startswith('./'):
618 fn = './%s' % fn
619 if fn.startswith(sdkpathnative):
620 relpth = '/' + os.path.relpath(fn, sdkpathnative)
621 for bindir in sdkbindirs:
622 if relpth.startswith(bindir):
623 relpth = os.path.relpath(relpth, bindir)
624 if relpth in sanity_required_utilities:
625 sanity_required_utilities.remove(relpth)
626 break
627 return ' '.join(sanity_required_utilities)
628
629install_tools() {
630 install -d ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}
631 scripts="devtool recipetool oe-find-native-sysroot runqemu* wic"
632 for script in $scripts; do
633 for scriptfn in `find ${SDK_OUTPUT}/${SDKPATH}/${scriptrelpath} -maxdepth 1 -executable -name "$script"`; do
634 targetscriptfn="${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/$(basename $scriptfn)"
635 test -e ${targetscriptfn} || ln -rs ${scriptfn} ${targetscriptfn}
636 done
637 done
638 # We can't use the same method as above because files in the sysroot won't exist at this point
639 # (they get populated from sstate on installation)
640 unfsd_path="${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/unfsd"
641 if [ "${SDK_INCLUDE_TOOLCHAIN}" = "1" -a ! -e $unfsd_path ] ; then
642 binrelpath=${@os.path.relpath(d.getVar('STAGING_BINDIR_NATIVE'), d.getVar('TMPDIR'))}
643 ln -rs ${SDK_OUTPUT}/${SDKPATH}/tmp/$binrelpath/unfsd $unfsd_path
644 fi
645 touch ${SDK_OUTPUT}/${SDKPATH}/.devtoolbase
646
647 # find latest buildtools-tarball and install it
648 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then
649 install ${SDK_DEPLOY}/${SDK_BUILDTOOLS_INSTALLER} ${SDK_OUTPUT}/${SDKPATH}
650 fi
651
652 install -m 0644 ${COREBASE}/meta/files/ext-sdk-prepare.py ${SDK_OUTPUT}/${SDKPATH}
653}
654do_populate_sdk_ext[file-checksums] += "${COREBASE}/meta/files/ext-sdk-prepare.py:True"
655
656sdk_ext_preinst() {
657 # Since bitbake won't run as root it doesn't make sense to try and install
658 # the extensible sdk as root.
659 if [ "`id -u`" = "0" ]; then
660 echo "ERROR: The extensible sdk cannot be installed as root."
661 exit 1
662 fi
663 if ! command -v locale > /dev/null; then
664 echo "ERROR: The installer requires the locale command, please install it first"
665 exit 1
666 fi
667 # Check setting of LC_ALL set above
668 canonicalised_locale=`echo $LC_ALL | sed 's/UTF-8/utf8/'`
669 if ! locale -a | grep -q $canonicalised_locale ; then
670 echo "ERROR: the installer requires the $LC_ALL locale to be installed (but not selected), please install it first"
671 exit 1
672 fi
673 # The relocation script used by buildtools installer requires python
674 if ! command -v python3 > /dev/null; then
675 echo "ERROR: The installer requires python3, please install it first"
676 exit 1
677 fi
678 missing_utils=""
679 for util in ${SDK_REQUIRED_UTILITIES}; do
680 if ! command -v $util > /dev/null; then
681 missing_utils="$missing_utils $util"
682 fi
683 done
684 if [ -n "$missing_utils" ] ; then
685 echo "ERROR: the SDK requires the following missing utilities, please install them: $missing_utils"
686 exit 1
687 fi
688 SDK_EXTENSIBLE="1"
689 if [ "$publish" = "1" ] && [ "${SDK_EXT_TYPE}" = "minimal" ] ; then
690 EXTRA_TAR_OPTIONS="$EXTRA_TAR_OPTIONS --exclude=sstate-cache"
691 fi
692}
693SDK_PRE_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_preinst}"
694
695# FIXME this preparation should be done as part of the SDK construction
696sdk_ext_postinst() {
697 printf "\nExtracting buildtools...\n"
698 cd $target_sdk_dir
699 env_setup_script="$target_sdk_dir/environment-setup-${REAL_MULTIMACH_TARGET_SYS}"
700 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then
701 printf "buildtools\ny" | ./${SDK_BUILDTOOLS_INSTALLER} > buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
702
703 # Delete the buildtools tar file since it won't be used again
704 rm -f ./${SDK_BUILDTOOLS_INSTALLER}
705 # We don't need the log either since it succeeded
706 rm -f buildtools.log
707
708 # Make sure when the user sets up the environment, they also get
709 # the buildtools-tarball tools in their path.
710 echo "# Save and reset OECORE_NATIVE_SYSROOT as buildtools may change it" >> $env_setup_script
711 echo "SAVED=\"\$OECORE_NATIVE_SYSROOT\"" >> $env_setup_script
712 echo ". $target_sdk_dir/buildtools/environment-setup*" >> $env_setup_script
713 echo "OECORE_NATIVE_SYSROOT=\"\$SAVED\"" >> $env_setup_script
714 fi
715
716 # Allow bitbake environment setup to be ran as part of this sdk.
717 echo "export OE_SKIP_SDK_CHECK=1" >> $env_setup_script
718 # Work around runqemu not knowing how to get this information within the eSDK
719 echo "export DEPLOY_DIR_IMAGE=$target_sdk_dir/tmp/${@os.path.relpath(d.getVar('DEPLOY_DIR_IMAGE'), d.getVar('TMPDIR'))}" >> $env_setup_script
720
721 # A bit of another hack, but we need this in the path only for devtool
722 # so put it at the end of $PATH.
Andrew Geisslerfc113ea2023-03-31 09:59:46 -0500723 echo "export PATH=\"$target_sdk_dir/sysroots/${SDK_SYS}${bindir_nativesdk}:\$PATH\"" >> $env_setup_script
Patrick Williams92b42cb2022-09-03 06:53:57 -0500724
725 echo "printf 'SDK environment now set up; additionally you may now run devtool to perform development tasks.\nRun devtool --help for further details.\n'" >> $env_setup_script
726
727 # Warn if trying to use external bitbake and the ext SDK together
728 echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING: attempting to use the extensible SDK in an environment set up to run bitbake - this may lead to unexpected results. Please source this script in a new shell session instead.') || true" >> $env_setup_script
729
730 if [ "$prepare_buildsystem" != "no" -a -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then
731 printf "Preparing build system...\n"
732 # dash which is /bin/sh on Ubuntu will not preserve the
733 # current working directory when first ran, nor will it set $1 when
734 # sourcing a script. That is why this has to look so ugly.
735 LOGFILE="$target_sdk_dir/preparing_build_system.log"
736 sh -c ". buildtools/environment-setup* > $LOGFILE && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE && python3 $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
737 fi
738 if [ -e $target_sdk_dir/ext-sdk-prepare.py ]; then
739 rm $target_sdk_dir/ext-sdk-prepare.py
740 fi
741 echo done
742}
743
744SDK_POST_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_postinst}"
745
746SDK_POSTPROCESS_COMMAND:prepend:task-populate-sdk-ext = "copy_buildsystem; install_tools; "
747
748SDK_INSTALL_TARGETS = ""
749fakeroot python do_populate_sdk_ext() {
750 # FIXME hopefully we can remove this restriction at some point, but uninative
751 # currently forces this upon us
752 if d.getVar('SDK_ARCH') != d.getVar('BUILD_ARCH'):
753 bb.fatal('The extensible SDK can currently only be built for the same architecture as the machine being built on - SDK_ARCH is set to %s (likely via setting SDKMACHINE) which is different from the architecture of the build machine (%s). Unable to continue.' % (d.getVar('SDK_ARCH'), d.getVar('BUILD_ARCH')))
754
755 # FIXME hopefully we can remove this restriction at some point, but the eSDK
756 # can only be built for the primary (default) multiconfig
757 if d.getVar('BB_CURRENT_MC') != 'default':
758 bb.fatal('The extensible SDK can currently only be built for the default multiconfig. Currently trying to build for %s.' % d.getVar('BB_CURRENT_MC'))
759
760 # eSDK dependencies don't use the traditional variables and things don't work properly if they are set
761 d.setVar("TOOLCHAIN_HOST_TASK", "${TOOLCHAIN_HOST_TASK_ESDK}")
762 d.setVar("TOOLCHAIN_TARGET_TASK", "")
763
764 d.setVar('SDK_INSTALL_TARGETS', get_sdk_install_targets(d))
765 if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1':
766 buildtools_fn = get_current_buildtools(d)
767 else:
768 buildtools_fn = None
769 d.setVar('SDK_REQUIRED_UTILITIES', get_sdk_required_utilities(buildtools_fn, d))
770 d.setVar('SDK_BUILDTOOLS_INSTALLER', buildtools_fn)
771 d.setVar('SDKDEPLOYDIR', '${SDKEXTDEPLOYDIR}')
772 # ESDKs have a libc from the buildtools so ensure we don't ship linguas twice
773 d.delVar('SDKIMAGE_LINGUAS')
774 if d.getVar("SDK_INCLUDE_NATIVESDK") == '1':
775 generate_nativesdk_lockedsigs(d)
776 populate_sdk_common(d)
777}
778
779def generate_nativesdk_lockedsigs(d):
780 import oe.copy_buildsystem
781 sigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc'
782 oe.copy_buildsystem.generate_locked_sigs(sigfile, d)
783
784def get_ext_sdk_depends(d):
785 # Note: the deps varflag is a list not a string, so we need to specify expand=False
786 deps = d.getVarFlag('do_image_complete', 'deps', False)
787 pn = d.getVar('PN')
788 deplist = ['%s:%s' % (pn, dep) for dep in deps]
789 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d)
790 tasklist.append('do_rootfs')
791 for task in tasklist:
792 deplist.extend((d.getVarFlag(task, 'depends') or '').split())
793 return ' '.join(deplist)
794
795python do_sdk_depends() {
796 # We have to do this separately in its own task so we avoid recursing into
797 # dependencies we don't need to (e.g. buildtools-tarball) and bringing those
798 # into the SDK's sstate-cache
799 import oe.copy_buildsystem
800 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc'
801 oe.copy_buildsystem.generate_locked_sigs(sigfile, d)
802}
803addtask sdk_depends
804
805do_sdk_depends[dirs] = "${WORKDIR}"
806do_sdk_depends[depends] = "${@get_ext_sdk_depends(d)} meta-extsdk-toolchain:do_populate_sysroot"
807do_sdk_depends[recrdeptask] = "${@d.getVarFlag('do_populate_sdk', 'recrdeptask', False)}"
808do_sdk_depends[recrdeptask] += "do_populate_lic do_package_qa do_populate_sysroot do_deploy ${SDK_RECRDEP_TASKS}"
809do_sdk_depends[rdepends] = "${@' '.join([x + ':do_package_write_${IMAGE_PKGTYPE} ' + x + ':do_packagedata' for x in d.getVar('TOOLCHAIN_HOST_TASK_ESDK').split()])}"
810
811do_populate_sdk_ext[dirs] = "${@d.getVarFlag('do_populate_sdk', 'dirs', False)}"
812
813do_populate_sdk_ext[depends] = "${@d.getVarFlag('do_populate_sdk', 'depends', False)} \
814 ${@'buildtools-tarball:do_populate_sdk' if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1' else ''} \
815 ${@'meta-world-pkgdata:do_collect_packagedata' if d.getVar('SDK_INCLUDE_PKGDATA') == '1' else ''} \
816 ${@'meta-extsdk-toolchain:do_locked_sigs' if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1' else ''}"
817
818# We must avoid depending on do_build here if rm_work.bbclass is active,
819# because otherwise do_rm_work may run before do_populate_sdk_ext itself.
820# We can't mark do_populate_sdk_ext and do_sdk_depends as having to
821# run before do_rm_work, because then they would also run as part
822# of normal builds.
823do_populate_sdk_ext[rdepends] += "${@' '.join([x + ':' + (d.getVar('RM_WORK_BUILD_WITHOUT') or 'do_build') for x in d.getVar('SDK_TARGETS').split()])}"
824
825# Make sure code changes can result in rebuild
826do_populate_sdk_ext[vardeps] += "copy_buildsystem \
827 sdk_ext_postinst"
828
829# Since any change in the metadata of any layer should cause a rebuild of the
830# sdk(since the layers are put in the sdk) set the task to nostamp so it
831# always runs.
832do_populate_sdk_ext[nostamp] = "1"
833
834SDKEXTDEPLOYDIR = "${WORKDIR}/deploy-${PN}-populate-sdk-ext"
835
836SSTATETASKS += "do_populate_sdk_ext"
837SSTATE_SKIP_CREATION:task-populate-sdk-ext = '1'
838do_populate_sdk_ext[cleandirs] = "${SDKEXTDEPLOYDIR}"
839do_populate_sdk_ext[sstate-inputdirs] = "${SDKEXTDEPLOYDIR}"
840do_populate_sdk_ext[sstate-outputdirs] = "${SDK_DEPLOY}"
841do_populate_sdk_ext[stamp-extra-info] = "${MACHINE_ARCH}"
842
843addtask populate_sdk_ext after do_sdk_depends