blob: 2e6673bcd9358d9329b614706d8550ed274d0c64 [file] [log] [blame]
Patrick Williams92b42cb2022-09-03 06:53:57 -05001#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: MIT
5#
6
7# Extensible SDK
8
9inherit populate_sdk_base
10
11# Used to override TOOLCHAIN_HOST_TASK in the eSDK case
12TOOLCHAIN_HOST_TASK_ESDK = " \
13 meta-environment-extsdk-${MACHINE} \
14 "
15
16SDK_RELOCATE_AFTER_INSTALL:task-populate-sdk-ext = "0"
17
18SDK_EXT = ""
19SDK_EXT:task-populate-sdk-ext = "-ext"
20
21# Options are full or minimal
22SDK_EXT_TYPE ?= "full"
23SDK_INCLUDE_PKGDATA ?= "0"
24SDK_INCLUDE_TOOLCHAIN ?= "${@'1' if d.getVar('SDK_EXT_TYPE') == 'full' else '0'}"
25SDK_INCLUDE_NATIVESDK ?= "0"
26SDK_INCLUDE_BUILDTOOLS ?= '1'
27
28SDK_RECRDEP_TASKS ?= ""
29SDK_CUSTOM_TEMPLATECONF ?= "0"
30
31ESDK_LOCALCONF_ALLOW ?= ""
32ESDK_LOCALCONF_REMOVE ?= "CONF_VERSION \
33 BB_NUMBER_THREADS \
34 BB_NUMBER_PARSE_THREADS \
35 PARALLEL_MAKE \
36 PRSERV_HOST \
37 SSTATE_MIRRORS \
38 DL_DIR \
39 SSTATE_DIR \
40 TMPDIR \
41 BB_SERVER_TIMEOUT \
42 "
43ESDK_CLASS_INHERIT_DISABLE ?= "buildhistory icecc"
44SDK_UPDATE_URL ?= ""
45
46SDK_TARGETS ?= "${PN}"
47
48def get_sdk_install_targets(d, images_only=False):
49 sdk_install_targets = ''
50 if images_only or d.getVar('SDK_EXT_TYPE') != 'minimal':
51 sdk_install_targets = d.getVar('SDK_TARGETS')
52
53 depd = d.getVar('BB_TASKDEPDATA', False)
54 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d)
55 tasklist.remove('do_build')
56 for v in depd.values():
57 if v[1] in tasklist:
58 if v[0] not in sdk_install_targets:
59 sdk_install_targets += ' {}'.format(v[0])
60
61 if not images_only:
62 if d.getVar('SDK_INCLUDE_PKGDATA') == '1':
63 sdk_install_targets += ' meta-world-pkgdata:do_allpackagedata'
64 if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1':
65 sdk_install_targets += ' meta-extsdk-toolchain:do_populate_sysroot'
66
67 return sdk_install_targets
68
69get_sdk_install_targets[vardepsexclude] = "BB_TASKDEPDATA"
70
71OE_INIT_ENV_SCRIPT ?= "oe-init-build-env"
72
73# The files from COREBASE that you want preserved in the COREBASE copied
74# into the sdk. This allows someone to have their own setup scripts in
75# COREBASE be preserved as well as untracked files.
76COREBASE_FILES ?= " \
77 oe-init-build-env \
78 scripts \
79 LICENSE \
80 .templateconf \
81"
82
83SDK_DIR:task-populate-sdk-ext = "${WORKDIR}/sdk-ext"
84B:task-populate-sdk-ext = "${SDK_DIR}"
85TOOLCHAINEXT_OUTPUTNAME ?= "${SDK_NAME}-toolchain-ext-${SDK_VERSION}"
86TOOLCHAIN_OUTPUTNAME:task-populate-sdk-ext = "${TOOLCHAINEXT_OUTPUTNAME}"
87
88SDK_EXT_TARGET_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest"
89SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest"
90
91python write_target_sdk_ext_manifest () {
92 from oe.sdk import get_extra_sdkinfo
93 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
94 extra_info = get_extra_sdkinfo(sstate_dir)
95
96 target = d.getVar('TARGET_SYS')
97 target_multimach = d.getVar('MULTIMACH_TARGET_SYS')
98 real_target_multimach = d.getVar('REAL_MULTIMACH_TARGET_SYS')
99
100 pkgs = {}
101 os.makedirs(os.path.dirname(d.getVar('SDK_EXT_TARGET_MANIFEST')), exist_ok=True)
102 with open(d.getVar('SDK_EXT_TARGET_MANIFEST'), 'w') as f:
103 for fn in extra_info['filesizes']:
104 info = fn.split(':')
105 if info[2] in (target, target_multimach, real_target_multimach) \
106 or info[5] == 'allarch':
107 if not info[1] in pkgs:
108 f.write("%s %s %s\n" % (info[1], info[2], info[3]))
109 pkgs[info[1]] = {}
110}
111python write_host_sdk_ext_manifest () {
112 from oe.sdk import get_extra_sdkinfo
113 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
114 extra_info = get_extra_sdkinfo(sstate_dir)
115 host = d.getVar('BUILD_SYS')
116 with open(d.getVar('SDK_EXT_HOST_MANIFEST'), 'w') as f:
117 for fn in extra_info['filesizes']:
118 info = fn.split(':')
119 if info[2] == host:
120 f.write("%s %s %s\n" % (info[1], info[2], info[3]))
121}
122
Andrew Geisslerc5535c92023-01-27 16:10:19 -0600123SDK_POSTPROCESS_COMMAND:append:task-populate-sdk-ext = " write_target_sdk_ext_manifest; write_host_sdk_ext_manifest; "
Patrick Williams92b42cb2022-09-03 06:53:57 -0500124
125SDK_TITLE:task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME') or d.getVar('DISTRO')} Extensible SDK"
126
127def clean_esdk_builddir(d, sdkbasepath):
128 """Clean up traces of the fake build for create_filtered_tasklist()"""
129 import shutil
130 cleanpaths = ['cache', 'tmp']
131 for pth in cleanpaths:
132 fullpth = os.path.join(sdkbasepath, pth)
133 if os.path.isdir(fullpth):
134 shutil.rmtree(fullpth)
135 elif os.path.isfile(fullpth):
136 os.remove(fullpth)
137
138def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath):
139 """
140 Create a filtered list of tasks. Also double-checks that the build system
141 within the SDK basically works and required sstate artifacts are available.
142 """
143 import tempfile
144 import shutil
145 import oe.copy_buildsystem
146
147 # Create a temporary build directory that we can pass to the env setup script
148 shutil.copyfile(sdkbasepath + '/conf/local.conf', sdkbasepath + '/conf/local.conf.bak')
149 try:
150 with open(sdkbasepath + '/conf/local.conf', 'a') as f:
151 # Force the use of sstate from the build system
152 f.write('\nSSTATE_DIR:forcevariable = "%s"\n' % d.getVar('SSTATE_DIR'))
153 f.write('SSTATE_MIRRORS:forcevariable = "file://universal/(.*) file://universal-4.9/\\1 file://universal-4.9/(.*) file://universal-4.8/\\1"\n')
154 # Ensure TMPDIR is the default so that clean_esdk_builddir() can delete it
155 f.write('TMPDIR:forcevariable = "${TOPDIR}/tmp"\n')
156 f.write('TCLIBCAPPEND:forcevariable = ""\n')
157 # Drop uninative if the build isn't using it (or else NATIVELSBSTRING will
158 # be different and we won't be able to find our native sstate)
159 if not bb.data.inherits_class('uninative', d):
160 f.write('INHERIT:remove = "uninative"\n')
161
162 # Unfortunately the default SDKPATH (or even a custom value) may contain characters that bitbake
163 # will not allow in its COREBASE path, so we need to rename the directory temporarily
164 temp_sdkbasepath = d.getVar('SDK_OUTPUT') + '/tmp-renamed-sdk'
165 # Delete any existing temp dir
166 try:
167 shutil.rmtree(temp_sdkbasepath)
168 except FileNotFoundError:
169 pass
170 bb.utils.rename(sdkbasepath, temp_sdkbasepath)
171 cmdprefix = '. %s .; ' % conf_initpath
172 logfile = d.getVar('WORKDIR') + '/tasklist_bb_log.txt'
173 try:
174 oe.copy_buildsystem.check_sstate_task_list(d, get_sdk_install_targets(d), tasklistfile, cmdprefix=cmdprefix, cwd=temp_sdkbasepath, logfile=logfile)
175 except bb.process.ExecutionError as e:
176 msg = 'Failed to generate filtered task list for extensible SDK:\n%s' % e.stdout.rstrip()
177 if 'attempted to execute unexpectedly and should have been setscened' in e.stdout:
178 msg += '\n----------\n\nNOTE: "attempted to execute unexpectedly and should have been setscened" errors indicate this may be caused by missing sstate artifacts that were likely produced in earlier builds, but have been subsequently deleted for some reason.\n'
179 bb.fatal(msg)
180 bb.utils.rename(temp_sdkbasepath, sdkbasepath)
181 # Clean out residue of running bitbake, which check_sstate_task_list()
182 # will effectively do
183 clean_esdk_builddir(d, sdkbasepath)
184 finally:
185 localconf = sdkbasepath + '/conf/local.conf'
186 if os.path.exists(localconf + '.bak'):
187 os.replace(localconf + '.bak', localconf)
188
189python copy_buildsystem () {
190 import re
191 import shutil
192 import glob
193 import oe.copy_buildsystem
194
195 oe_init_env_script = d.getVar('OE_INIT_ENV_SCRIPT')
196
197 conf_bbpath = ''
198 conf_initpath = ''
199 core_meta_subdir = ''
200
201 # Copy in all metadata layers + bitbake (as repositories)
202 buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d)
203 baseoutpath = d.getVar('SDK_OUTPUT') + '/' + d.getVar('SDKPATH')
204
205 #check if custome templateconf path is set
206 use_custom_templateconf = d.getVar('SDK_CUSTOM_TEMPLATECONF')
207
208 # Determine if we're building a derivative extensible SDK (from devtool build-sdk)
209 derivative = (d.getVar('SDK_DERIVATIVE') or '') == '1'
210 if derivative:
211 workspace_name = 'orig-workspace'
212 else:
213 workspace_name = None
214
215 corebase, sdkbblayers = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers', workspace_name)
216 conf_bbpath = os.path.join('layers', corebase, 'bitbake')
217
218 for path in os.listdir(baseoutpath + '/layers'):
219 relpath = os.path.join('layers', path, oe_init_env_script)
220 if os.path.exists(os.path.join(baseoutpath, relpath)):
221 conf_initpath = relpath
222
223 relpath = os.path.join('layers', path, 'scripts', 'devtool')
224 if os.path.exists(os.path.join(baseoutpath, relpath)):
225 scriptrelpath = os.path.dirname(relpath)
226
227 relpath = os.path.join('layers', path, 'meta')
228 if os.path.exists(os.path.join(baseoutpath, relpath, 'lib', 'oe')):
229 core_meta_subdir = relpath
230
231 d.setVar('oe_init_build_env_path', conf_initpath)
232 d.setVar('scriptrelpath', scriptrelpath)
233
234 # Write out config file for devtool
235 import configparser
Patrick Williams7784c422022-11-17 07:29:11 -0600236 config = configparser.ConfigParser()
Patrick Williams92b42cb2022-09-03 06:53:57 -0500237 config.add_section('General')
238 config.set('General', 'bitbake_subdir', conf_bbpath)
239 config.set('General', 'init_path', conf_initpath)
240 config.set('General', 'core_meta_subdir', core_meta_subdir)
241 config.add_section('SDK')
242 config.set('SDK', 'sdk_targets', d.getVar('SDK_TARGETS'))
243 updateurl = d.getVar('SDK_UPDATE_URL')
244 if updateurl:
245 config.set('SDK', 'updateserver', updateurl)
246 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf'))
247 with open(os.path.join(baseoutpath, 'conf', 'devtool.conf'), 'w') as f:
248 config.write(f)
249
250 unlockedsigs = os.path.join(baseoutpath, 'conf', 'unlocked-sigs.inc')
251 with open(unlockedsigs, 'w') as f:
252 pass
253
254 # Create a layer for new recipes / appends
255 bbpath = d.getVar('BBPATH')
256 env = os.environ.copy()
257 env['PYTHONDONTWRITEBYTECODE'] = '1'
Andrew Geissler517393d2023-01-13 08:55:19 -0600258 bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--layerseries', d.getVar("LAYERSERIES_CORENAMES"), '--create-only', os.path.join(baseoutpath, 'workspace')], env=env)
Patrick Williams92b42cb2022-09-03 06:53:57 -0500259
260 # Create bblayers.conf
261 bb.utils.mkdirhier(baseoutpath + '/conf')
262 with open(baseoutpath + '/conf/bblayers.conf', 'w') as f:
263 f.write('# WARNING: this configuration has been automatically generated and in\n')
264 f.write('# most cases should not be edited. If you need more flexibility than\n')
265 f.write('# this configuration provides, it is strongly suggested that you set\n')
266 f.write('# up a proper instance of the full build system and use that instead.\n\n')
267
268 # LCONF_VERSION may not be set, for example when using meta-poky
269 # so don't error if it isn't found
270 lconf_version = d.getVar('LCONF_VERSION', False)
271 if lconf_version is not None:
272 f.write('LCONF_VERSION = "%s"\n\n' % lconf_version)
273
274 f.write('BBPATH = "$' + '{TOPDIR}"\n')
275 f.write('SDKBASEMETAPATH = "$' + '{TOPDIR}"\n')
276 f.write('BBLAYERS := " \\\n')
277 for layerrelpath in sdkbblayers:
278 f.write(' $' + '{SDKBASEMETAPATH}/layers/%s \\\n' % layerrelpath)
279 f.write(' $' + '{SDKBASEMETAPATH}/workspace \\\n')
280 f.write(' "\n')
281
282 # Copy uninative tarball
283 # For now this is where uninative.bbclass expects the tarball
284 if bb.data.inherits_class('uninative', d):
285 uninative_file = d.expand('${UNINATIVE_DLDIR}/' + d.getVarFlag("UNINATIVE_CHECKSUM", d.getVar("BUILD_ARCH")) + '/${UNINATIVE_TARBALL}')
286 uninative_checksum = bb.utils.sha256_file(uninative_file)
287 uninative_outdir = '%s/downloads/uninative/%s' % (baseoutpath, uninative_checksum)
288 bb.utils.mkdirhier(uninative_outdir)
289 shutil.copy(uninative_file, uninative_outdir)
290
291 env_passthrough = (d.getVar('BB_ENV_PASSTHROUGH_ADDITIONS') or '').split()
292 env_passthrough_values = {}
293
294 # Create local.conf
295 builddir = d.getVar('TOPDIR')
296 if derivative and os.path.exists(builddir + '/conf/site.conf'):
297 shutil.copyfile(builddir + '/conf/site.conf', baseoutpath + '/conf/site.conf')
298 if derivative and os.path.exists(builddir + '/conf/auto.conf'):
299 shutil.copyfile(builddir + '/conf/auto.conf', baseoutpath + '/conf/auto.conf')
300 if derivative:
301 shutil.copyfile(builddir + '/conf/local.conf', baseoutpath + '/conf/local.conf')
302 else:
303 local_conf_allowed = (d.getVar('ESDK_LOCALCONF_ALLOW') or '').split()
304 local_conf_remove = (d.getVar('ESDK_LOCALCONF_REMOVE') or '').split()
305 def handle_var(varname, origvalue, op, newlines):
306 if varname in local_conf_remove or (origvalue.strip().startswith('/') and not varname in local_conf_allowed):
307 newlines.append('# Removed original setting of %s\n' % varname)
308 return None, op, 0, True
309 else:
310 if varname in env_passthrough:
311 env_passthrough_values[varname] = origvalue
312 return origvalue, op, 0, True
313 varlist = ['[^#=+ ]*']
314 oldlines = []
315 if os.path.exists(builddir + '/conf/site.conf'):
316 with open(builddir + '/conf/site.conf', 'r') as f:
317 oldlines += f.readlines()
318 if os.path.exists(builddir + '/conf/auto.conf'):
319 with open(builddir + '/conf/auto.conf', 'r') as f:
320 oldlines += f.readlines()
321 if os.path.exists(builddir + '/conf/local.conf'):
322 with open(builddir + '/conf/local.conf', 'r') as f:
323 oldlines += f.readlines()
324 (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var)
325
326 with open(baseoutpath + '/conf/local.conf', 'w') as f:
327 f.write('# WARNING: this configuration has been automatically generated and in\n')
328 f.write('# most cases should not be edited. If you need more flexibility than\n')
329 f.write('# this configuration provides, it is strongly suggested that you set\n')
330 f.write('# up a proper instance of the full build system and use that instead.\n\n')
331 for line in newlines:
332 if line.strip() and not line.startswith('#'):
333 f.write(line)
334 # Write a newline just in case there's none at the end of the original
335 f.write('\n')
336
337 f.write('TMPDIR = "${TOPDIR}/tmp"\n')
338 f.write('TCLIBCAPPEND = ""\n')
339 f.write('DL_DIR = "${TOPDIR}/downloads"\n')
340
341 if bb.data.inherits_class('uninative', d):
342 f.write('INHERIT += "%s"\n' % 'uninative')
343 f.write('UNINATIVE_CHECKSUM[%s] = "%s"\n\n' % (d.getVar('BUILD_ARCH'), uninative_checksum))
344 f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False))
345
346 # Some classes are not suitable for SDK, remove them from INHERIT
347 f.write('INHERIT:remove = "%s"\n' % d.getVar('ESDK_CLASS_INHERIT_DISABLE', False))
348
349 # Bypass the default connectivity check if any
350 f.write('CONNECTIVITY_CHECK_URIS = ""\n\n')
351
352 # This warning will come out if reverse dependencies for a task
353 # don't have sstate as well as the task itself. We already know
354 # this will be the case for the extensible sdk, so turn off the
355 # warning.
356 f.write('SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK = "none"\n\n')
357
358 # Warn if the sigs in the locked-signature file don't match
359 # the sig computed from the metadata.
360 f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n\n')
361
362 # We want to be able to set this without a full reparse
363 f.write('BB_HASHCONFIG_IGNORE_VARS:append = " SIGGEN_UNLOCKED_RECIPES"\n\n')
364
365 # Set up which tasks are ignored for run on install
366 f.write('BB_SETSCENE_ENFORCE_IGNORE_TASKS = "%:* *:do_shared_workdir *:do_rm_work wic-tools:* *:do_addto_recipe_sysroot"\n\n')
367
368 # Hide the config information from bitbake output (since it's fixed within the SDK)
369 f.write('BUILDCFG_HEADER = ""\n\n')
370
371 # Write METADATA_REVISION
372 f.write('METADATA_REVISION = "%s"\n\n' % d.getVar('METADATA_REVISION'))
373
374 f.write('# Provide a flag to indicate we are in the EXT_SDK Context\n')
375 f.write('WITHIN_EXT_SDK = "1"\n\n')
376
377 # Map gcc-dependent uninative sstate cache for installer usage
378 f.write('SSTATE_MIRRORS += " file://universal/(.*) file://universal-4.9/\\1 file://universal-4.9/(.*) file://universal-4.8/\\1"\n\n')
379
380 if d.getVar("PRSERV_HOST"):
381 # Override this, we now include PR data, so it should only point ot the local database
382 f.write('PRSERV_HOST = "localhost:0"\n\n')
383
384 # Allow additional config through sdk-extra.conf
385 fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d)
386 if fn:
387 with open(fn, 'r') as xf:
388 for line in xf:
389 f.write(line)
390
391 # If you define a sdk_extraconf() function then it can contain additional config
392 # (Though this is awkward; sdk-extra.conf should probably be used instead)
393 extraconf = (d.getVar('sdk_extraconf') or '').strip()
394 if extraconf:
395 # Strip off any leading / trailing spaces
396 for line in extraconf.splitlines():
397 f.write(line.strip() + '\n')
398
399 f.write('require conf/locked-sigs.inc\n')
400 f.write('require conf/unlocked-sigs.inc\n')
401
402 # Copy multiple configurations if they exist in the users config directory
403 if d.getVar('BBMULTICONFIG') is not None:
404 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf', 'multiconfig'))
405 for mc in d.getVar('BBMULTICONFIG').split():
406 dest_stub = "/conf/multiconfig/%s.conf" % (mc,)
407 if os.path.exists(builddir + dest_stub):
408 shutil.copyfile(builddir + dest_stub, baseoutpath + dest_stub)
409
410 cachedir = os.path.join(baseoutpath, 'cache')
411 bb.utils.mkdirhier(cachedir)
412 bb.parse.siggen.copy_unitaskhashes(cachedir)
413
414 # If PR Service is in use, we need to export this as well
415 bb.note('Do we have a pr database?')
416 if d.getVar("PRSERV_HOST"):
417 bb.note('Writing PR database...')
418 # Based on the code in classes/prexport.bbclass
419 import oe.prservice
420 #dump meta info of tables
421 localdata = d.createCopy()
422 localdata.setVar('PRSERV_DUMPOPT_COL', "1")
423 localdata.setVar('PRSERV_DUMPDIR', os.path.join(baseoutpath, 'conf'))
424 localdata.setVar('PRSERV_DUMPFILE', '${PRSERV_DUMPDIR}/prserv.inc')
425
426 bb.note('PR Database write to %s' % (localdata.getVar('PRSERV_DUMPFILE')))
427
428 retval = oe.prservice.prserv_dump_db(localdata)
429 if not retval:
430 bb.error("prexport_handler: export failed!")
431 return
432 (metainfo, datainfo) = retval
433 oe.prservice.prserv_export_tofile(localdata, metainfo, datainfo, True)
434
435 # Use templateconf.cfg file from builddir if exists
436 if os.path.exists(builddir + '/conf/templateconf.cfg') and use_custom_templateconf == '1':
437 shutil.copyfile(builddir + '/conf/templateconf.cfg', baseoutpath + '/conf/templateconf.cfg')
438 else:
439 # Write a templateconf.cfg
440 with open(baseoutpath + '/conf/templateconf.cfg', 'w') as f:
441 f.write('meta/conf/templates/default\n')
442 os.makedirs(os.path.join(baseoutpath, core_meta_subdir, 'conf/templates/default'), exist_ok=True)
443
444 # Ensure any variables set from the external environment (by way of
445 # BB_ENV_PASSTHROUGH_ADDITIONS) are set in the SDK's configuration
446 extralines = []
447 for name, value in env_passthrough_values.items():
448 actualvalue = d.getVar(name) or ''
449 if value != actualvalue:
450 extralines.append('%s = "%s"\n' % (name, actualvalue))
451 if extralines:
452 with open(baseoutpath + '/conf/local.conf', 'a') as f:
453 f.write('\n')
454 f.write('# Extra settings from environment:\n')
455 for line in extralines:
456 f.write(line)
457 f.write('\n')
458
459 # Filter the locked signatures file to just the sstate tasks we are interested in
460 excluded_targets = get_sdk_install_targets(d, images_only=True)
461 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc'
462 lockedsigs_pruned = baseoutpath + '/conf/locked-sigs.inc'
463 #nativesdk-only sigfile to merge into locked-sigs.inc
464 sdk_include_nativesdk = (d.getVar("SDK_INCLUDE_NATIVESDK") == '1')
465 nativesigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc'
466 nativesigfile_pruned = d.getVar('WORKDIR') + '/locked-sigs_nativesdk_pruned.inc'
467
468 if sdk_include_nativesdk:
469 oe.copy_buildsystem.prune_lockedsigs([],
470 excluded_targets.split(),
471 nativesigfile,
472 True,
473 nativesigfile_pruned)
474
475 oe.copy_buildsystem.merge_lockedsigs([],
476 sigfile,
477 nativesigfile_pruned,
478 sigfile)
479
480 oe.copy_buildsystem.prune_lockedsigs([],
481 excluded_targets.split(),
482 sigfile,
483 False,
484 lockedsigs_pruned)
485
486 sstate_out = baseoutpath + '/sstate-cache'
487 bb.utils.remove(sstate_out, True)
488
489 # uninative.bbclass sets NATIVELSBSTRING to 'universal%s' % oe.utils.host_gcc_version(d)
490 fixedlsbstring = "universal%s" % oe.utils.host_gcc_version(d)
491
492 sdk_include_toolchain = (d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1')
493 sdk_ext_type = d.getVar('SDK_EXT_TYPE')
494 if (sdk_ext_type != 'minimal' or sdk_include_toolchain or derivative) and not sdk_include_nativesdk:
495 # Create the filtered task list used to generate the sstate cache shipped with the SDK
496 tasklistfn = d.getVar('WORKDIR') + '/tasklist.txt'
497 create_filtered_tasklist(d, baseoutpath, tasklistfn, conf_initpath)
498 else:
499 tasklistfn = None
500
Patrick Williams92b42cb2022-09-03 06:53:57 -0500501 cachedir = os.path.join(baseoutpath, 'cache')
502 bb.utils.mkdirhier(cachedir)
503 bb.parse.siggen.copy_unitaskhashes(cachedir)
504
505 # Add packagedata if enabled
506 if d.getVar('SDK_INCLUDE_PKGDATA') == '1':
507 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base.inc'
508 lockedsigs_copy = d.getVar('WORKDIR') + '/locked-sigs-copy.inc'
509 shutil.move(lockedsigs_pruned, lockedsigs_base)
510 oe.copy_buildsystem.merge_lockedsigs(['do_packagedata'],
511 lockedsigs_base,
512 d.getVar('STAGING_DIR_HOST') + '/world-pkgdata/locked-sigs-pkgdata.inc',
513 lockedsigs_pruned,
514 lockedsigs_copy)
515
516 if sdk_include_toolchain:
517 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base2.inc'
518 lockedsigs_toolchain = d.expand("${STAGING_DIR}/${TUNE_PKGARCH}/meta-extsdk-toolchain/locked-sigs/locked-sigs-extsdk-toolchain.inc")
519 shutil.move(lockedsigs_pruned, lockedsigs_base)
520 oe.copy_buildsystem.merge_lockedsigs([],
521 lockedsigs_base,
522 lockedsigs_toolchain,
523 lockedsigs_pruned)
524 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_toolchain,
525 d.getVar('SSTATE_DIR'),
526 sstate_out, d,
527 fixedlsbstring,
528 filterfile=tasklistfn)
529
530 if sdk_ext_type == 'minimal':
531 if derivative:
532 # Assume the user is not going to set up an additional sstate
533 # mirror, thus we need to copy the additional artifacts (from
534 # workspace recipes) into the derivative SDK
535 lockedsigs_orig = d.getVar('TOPDIR') + '/conf/locked-sigs.inc'
536 if os.path.exists(lockedsigs_orig):
537 lockedsigs_extra = d.getVar('WORKDIR') + '/locked-sigs-extra.inc'
538 oe.copy_buildsystem.merge_lockedsigs(None,
539 lockedsigs_orig,
540 lockedsigs_pruned,
541 None,
542 lockedsigs_extra)
543 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra,
544 d.getVar('SSTATE_DIR'),
545 sstate_out, d,
546 fixedlsbstring,
547 filterfile=tasklistfn)
548 else:
549 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned,
550 d.getVar('SSTATE_DIR'),
551 sstate_out, d,
552 fixedlsbstring,
553 filterfile=tasklistfn)
554
555 # We don't need sstate do_package files
556 for root, dirs, files in os.walk(sstate_out):
557 for name in files:
558 if name.endswith("_package.tar.zst"):
559 f = os.path.join(root, name)
560 os.remove(f)
561
562 # Write manifest file
563 # Note: at the moment we cannot include the env setup script here to keep
564 # it updated, since it gets modified during SDK installation (see
565 # sdk_ext_postinst() below) thus the checksum we take here would always
566 # be different.
567 manifest_file_list = ['conf/*']
568 if d.getVar('BBMULTICONFIG') is not None:
569 manifest_file_list.append('conf/multiconfig/*')
570
571 esdk_manifest_excludes = (d.getVar('ESDK_MANIFEST_EXCLUDES') or '').split()
572 esdk_manifest_excludes_list = []
573 for exclude_item in esdk_manifest_excludes:
574 esdk_manifest_excludes_list += glob.glob(os.path.join(baseoutpath, exclude_item))
575 manifest_file = os.path.join(baseoutpath, 'conf', 'sdk-conf-manifest')
576 with open(manifest_file, 'w') as f:
577 for item in manifest_file_list:
578 for fn in glob.glob(os.path.join(baseoutpath, item)):
579 if fn == manifest_file or os.path.isdir(fn):
580 continue
581 if fn in esdk_manifest_excludes_list:
582 continue
583 chksum = bb.utils.sha256_file(fn)
584 f.write('%s\t%s\n' % (chksum, os.path.relpath(fn, baseoutpath)))
585}
586
587def get_current_buildtools(d):
588 """Get the file name of the current buildtools installer"""
589 import glob
590 btfiles = glob.glob(os.path.join(d.getVar('SDK_DEPLOY'), '*-buildtools-nativesdk-standalone-*.sh'))
591 btfiles.sort(key=os.path.getctime)
592 return os.path.basename(btfiles[-1])
593
594def get_sdk_required_utilities(buildtools_fn, d):
595 """Find required utilities that aren't provided by the buildtools"""
596 sanity_required_utilities = (d.getVar('SANITY_REQUIRED_UTILITIES') or '').split()
597 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}gcc'))
598 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}g++'))
599 if buildtools_fn:
600 buildtools_installer = os.path.join(d.getVar('SDK_DEPLOY'), buildtools_fn)
601 filelist, _ = bb.process.run('%s -l' % buildtools_installer)
602 else:
603 buildtools_installer = None
604 filelist = ""
605 localdata = bb.data.createCopy(d)
606 localdata.setVar('SDKPATH', '.')
607 sdkpathnative = localdata.getVar('SDKPATHNATIVE')
608 sdkbindirs = [localdata.getVar('bindir_nativesdk'),
609 localdata.getVar('sbindir_nativesdk'),
610 localdata.getVar('base_bindir_nativesdk'),
611 localdata.getVar('base_sbindir_nativesdk')]
612 for line in filelist.splitlines():
613 splitline = line.split()
614 if len(splitline) > 5:
615 fn = splitline[5]
616 if not fn.startswith('./'):
617 fn = './%s' % fn
618 if fn.startswith(sdkpathnative):
619 relpth = '/' + os.path.relpath(fn, sdkpathnative)
620 for bindir in sdkbindirs:
621 if relpth.startswith(bindir):
622 relpth = os.path.relpath(relpth, bindir)
623 if relpth in sanity_required_utilities:
624 sanity_required_utilities.remove(relpth)
625 break
626 return ' '.join(sanity_required_utilities)
627
628install_tools() {
629 install -d ${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}
630 scripts="devtool recipetool oe-find-native-sysroot runqemu* wic"
631 for script in $scripts; do
632 for scriptfn in `find ${SDK_OUTPUT}/${SDKPATH}/${scriptrelpath} -maxdepth 1 -executable -name "$script"`; do
633 targetscriptfn="${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/$(basename $scriptfn)"
634 test -e ${targetscriptfn} || ln -rs ${scriptfn} ${targetscriptfn}
635 done
636 done
637 # We can't use the same method as above because files in the sysroot won't exist at this point
638 # (they get populated from sstate on installation)
639 unfsd_path="${SDK_OUTPUT}/${SDKPATHNATIVE}${bindir_nativesdk}/unfsd"
640 if [ "${SDK_INCLUDE_TOOLCHAIN}" = "1" -a ! -e $unfsd_path ] ; then
641 binrelpath=${@os.path.relpath(d.getVar('STAGING_BINDIR_NATIVE'), d.getVar('TMPDIR'))}
642 ln -rs ${SDK_OUTPUT}/${SDKPATH}/tmp/$binrelpath/unfsd $unfsd_path
643 fi
644 touch ${SDK_OUTPUT}/${SDKPATH}/.devtoolbase
645
646 # find latest buildtools-tarball and install it
647 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then
648 install ${SDK_DEPLOY}/${SDK_BUILDTOOLS_INSTALLER} ${SDK_OUTPUT}/${SDKPATH}
649 fi
650
651 install -m 0644 ${COREBASE}/meta/files/ext-sdk-prepare.py ${SDK_OUTPUT}/${SDKPATH}
652}
653do_populate_sdk_ext[file-checksums] += "${COREBASE}/meta/files/ext-sdk-prepare.py:True"
654
655sdk_ext_preinst() {
656 # Since bitbake won't run as root it doesn't make sense to try and install
657 # the extensible sdk as root.
658 if [ "`id -u`" = "0" ]; then
659 echo "ERROR: The extensible sdk cannot be installed as root."
660 exit 1
661 fi
662 if ! command -v locale > /dev/null; then
663 echo "ERROR: The installer requires the locale command, please install it first"
664 exit 1
665 fi
666 # Check setting of LC_ALL set above
667 canonicalised_locale=`echo $LC_ALL | sed 's/UTF-8/utf8/'`
668 if ! locale -a | grep -q $canonicalised_locale ; then
669 echo "ERROR: the installer requires the $LC_ALL locale to be installed (but not selected), please install it first"
670 exit 1
671 fi
672 # The relocation script used by buildtools installer requires python
673 if ! command -v python3 > /dev/null; then
674 echo "ERROR: The installer requires python3, please install it first"
675 exit 1
676 fi
677 missing_utils=""
678 for util in ${SDK_REQUIRED_UTILITIES}; do
679 if ! command -v $util > /dev/null; then
680 missing_utils="$missing_utils $util"
681 fi
682 done
683 if [ -n "$missing_utils" ] ; then
684 echo "ERROR: the SDK requires the following missing utilities, please install them: $missing_utils"
685 exit 1
686 fi
687 SDK_EXTENSIBLE="1"
688 if [ "$publish" = "1" ] && [ "${SDK_EXT_TYPE}" = "minimal" ] ; then
689 EXTRA_TAR_OPTIONS="$EXTRA_TAR_OPTIONS --exclude=sstate-cache"
690 fi
691}
692SDK_PRE_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_preinst}"
693
694# FIXME this preparation should be done as part of the SDK construction
695sdk_ext_postinst() {
696 printf "\nExtracting buildtools...\n"
697 cd $target_sdk_dir
698 env_setup_script="$target_sdk_dir/environment-setup-${REAL_MULTIMACH_TARGET_SYS}"
699 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then
700 printf "buildtools\ny" | ./${SDK_BUILDTOOLS_INSTALLER} > buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
701
702 # Delete the buildtools tar file since it won't be used again
703 rm -f ./${SDK_BUILDTOOLS_INSTALLER}
704 # We don't need the log either since it succeeded
705 rm -f buildtools.log
706
707 # Make sure when the user sets up the environment, they also get
708 # the buildtools-tarball tools in their path.
709 echo "# Save and reset OECORE_NATIVE_SYSROOT as buildtools may change it" >> $env_setup_script
710 echo "SAVED=\"\$OECORE_NATIVE_SYSROOT\"" >> $env_setup_script
711 echo ". $target_sdk_dir/buildtools/environment-setup*" >> $env_setup_script
712 echo "OECORE_NATIVE_SYSROOT=\"\$SAVED\"" >> $env_setup_script
713 fi
714
715 # Allow bitbake environment setup to be ran as part of this sdk.
716 echo "export OE_SKIP_SDK_CHECK=1" >> $env_setup_script
717 # Work around runqemu not knowing how to get this information within the eSDK
718 echo "export DEPLOY_DIR_IMAGE=$target_sdk_dir/tmp/${@os.path.relpath(d.getVar('DEPLOY_DIR_IMAGE'), d.getVar('TMPDIR'))}" >> $env_setup_script
719
720 # A bit of another hack, but we need this in the path only for devtool
721 # so put it at the end of $PATH.
Andrew Geisslerfc113ea2023-03-31 09:59:46 -0500722 echo "export PATH=\"$target_sdk_dir/sysroots/${SDK_SYS}${bindir_nativesdk}:\$PATH\"" >> $env_setup_script
Patrick Williams92b42cb2022-09-03 06:53:57 -0500723
724 echo "printf 'SDK environment now set up; additionally you may now run devtool to perform development tasks.\nRun devtool --help for further details.\n'" >> $env_setup_script
725
726 # Warn if trying to use external bitbake and the ext SDK together
727 echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING: attempting to use the extensible SDK in an environment set up to run bitbake - this may lead to unexpected results. Please source this script in a new shell session instead.') || true" >> $env_setup_script
728
729 if [ "$prepare_buildsystem" != "no" -a -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then
730 printf "Preparing build system...\n"
731 # dash which is /bin/sh on Ubuntu will not preserve the
732 # current working directory when first ran, nor will it set $1 when
733 # sourcing a script. That is why this has to look so ugly.
734 LOGFILE="$target_sdk_dir/preparing_build_system.log"
735 sh -c ". buildtools/environment-setup* > $LOGFILE && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE && python3 $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
736 fi
737 if [ -e $target_sdk_dir/ext-sdk-prepare.py ]; then
738 rm $target_sdk_dir/ext-sdk-prepare.py
739 fi
740 echo done
741}
742
743SDK_POST_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_postinst}"
744
745SDK_POSTPROCESS_COMMAND:prepend:task-populate-sdk-ext = "copy_buildsystem; install_tools; "
746
747SDK_INSTALL_TARGETS = ""
748fakeroot python do_populate_sdk_ext() {
749 # FIXME hopefully we can remove this restriction at some point, but uninative
750 # currently forces this upon us
751 if d.getVar('SDK_ARCH') != d.getVar('BUILD_ARCH'):
752 bb.fatal('The extensible SDK can currently only be built for the same architecture as the machine being built on - SDK_ARCH is set to %s (likely via setting SDKMACHINE) which is different from the architecture of the build machine (%s). Unable to continue.' % (d.getVar('SDK_ARCH'), d.getVar('BUILD_ARCH')))
753
754 # FIXME hopefully we can remove this restriction at some point, but the eSDK
755 # can only be built for the primary (default) multiconfig
756 if d.getVar('BB_CURRENT_MC') != 'default':
757 bb.fatal('The extensible SDK can currently only be built for the default multiconfig. Currently trying to build for %s.' % d.getVar('BB_CURRENT_MC'))
758
759 # eSDK dependencies don't use the traditional variables and things don't work properly if they are set
760 d.setVar("TOOLCHAIN_HOST_TASK", "${TOOLCHAIN_HOST_TASK_ESDK}")
761 d.setVar("TOOLCHAIN_TARGET_TASK", "")
762
763 d.setVar('SDK_INSTALL_TARGETS', get_sdk_install_targets(d))
764 if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1':
765 buildtools_fn = get_current_buildtools(d)
766 else:
767 buildtools_fn = None
768 d.setVar('SDK_REQUIRED_UTILITIES', get_sdk_required_utilities(buildtools_fn, d))
769 d.setVar('SDK_BUILDTOOLS_INSTALLER', buildtools_fn)
770 d.setVar('SDKDEPLOYDIR', '${SDKEXTDEPLOYDIR}')
771 # ESDKs have a libc from the buildtools so ensure we don't ship linguas twice
772 d.delVar('SDKIMAGE_LINGUAS')
773 if d.getVar("SDK_INCLUDE_NATIVESDK") == '1':
774 generate_nativesdk_lockedsigs(d)
775 populate_sdk_common(d)
776}
777
778def generate_nativesdk_lockedsigs(d):
779 import oe.copy_buildsystem
780 sigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc'
781 oe.copy_buildsystem.generate_locked_sigs(sigfile, d)
782
783def get_ext_sdk_depends(d):
784 # Note: the deps varflag is a list not a string, so we need to specify expand=False
785 deps = d.getVarFlag('do_image_complete', 'deps', False)
786 pn = d.getVar('PN')
787 deplist = ['%s:%s' % (pn, dep) for dep in deps]
788 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d)
789 tasklist.append('do_rootfs')
790 for task in tasklist:
791 deplist.extend((d.getVarFlag(task, 'depends') or '').split())
792 return ' '.join(deplist)
793
794python do_sdk_depends() {
795 # We have to do this separately in its own task so we avoid recursing into
796 # dependencies we don't need to (e.g. buildtools-tarball) and bringing those
797 # into the SDK's sstate-cache
798 import oe.copy_buildsystem
799 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc'
800 oe.copy_buildsystem.generate_locked_sigs(sigfile, d)
801}
802addtask sdk_depends
803
804do_sdk_depends[dirs] = "${WORKDIR}"
805do_sdk_depends[depends] = "${@get_ext_sdk_depends(d)} meta-extsdk-toolchain:do_populate_sysroot"
806do_sdk_depends[recrdeptask] = "${@d.getVarFlag('do_populate_sdk', 'recrdeptask', False)}"
807do_sdk_depends[recrdeptask] += "do_populate_lic do_package_qa do_populate_sysroot do_deploy ${SDK_RECRDEP_TASKS}"
808do_sdk_depends[rdepends] = "${@' '.join([x + ':do_package_write_${IMAGE_PKGTYPE} ' + x + ':do_packagedata' for x in d.getVar('TOOLCHAIN_HOST_TASK_ESDK').split()])}"
809
810do_populate_sdk_ext[dirs] = "${@d.getVarFlag('do_populate_sdk', 'dirs', False)}"
811
812do_populate_sdk_ext[depends] = "${@d.getVarFlag('do_populate_sdk', 'depends', False)} \
813 ${@'buildtools-tarball:do_populate_sdk' if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1' else ''} \
814 ${@'meta-world-pkgdata:do_collect_packagedata' if d.getVar('SDK_INCLUDE_PKGDATA') == '1' else ''} \
815 ${@'meta-extsdk-toolchain:do_locked_sigs' if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1' else ''}"
816
817# We must avoid depending on do_build here if rm_work.bbclass is active,
818# because otherwise do_rm_work may run before do_populate_sdk_ext itself.
819# We can't mark do_populate_sdk_ext and do_sdk_depends as having to
820# run before do_rm_work, because then they would also run as part
821# of normal builds.
822do_populate_sdk_ext[rdepends] += "${@' '.join([x + ':' + (d.getVar('RM_WORK_BUILD_WITHOUT') or 'do_build') for x in d.getVar('SDK_TARGETS').split()])}"
823
824# Make sure code changes can result in rebuild
825do_populate_sdk_ext[vardeps] += "copy_buildsystem \
826 sdk_ext_postinst"
827
828# Since any change in the metadata of any layer should cause a rebuild of the
829# sdk(since the layers are put in the sdk) set the task to nostamp so it
830# always runs.
831do_populate_sdk_ext[nostamp] = "1"
832
833SDKEXTDEPLOYDIR = "${WORKDIR}/deploy-${PN}-populate-sdk-ext"
834
835SSTATETASKS += "do_populate_sdk_ext"
836SSTATE_SKIP_CREATION:task-populate-sdk-ext = '1'
837do_populate_sdk_ext[cleandirs] = "${SDKEXTDEPLOYDIR}"
838do_populate_sdk_ext[sstate-inputdirs] = "${SDKEXTDEPLOYDIR}"
839do_populate_sdk_ext[sstate-outputdirs] = "${SDK_DEPLOY}"
840do_populate_sdk_ext[stamp-extra-info] = "${MACHINE_ARCH}"
841
842addtask populate_sdk_ext after do_sdk_depends