blob: 31a84f5b0648c1a8e7b718d33617e4373cc82997 [file] [log] [blame]
Brad Bishopc342db32019-05-15 21:57:59 -04001#
2# SPDX-License-Identifier: GPL-2.0-only
3#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05004# This class should provide easy access to the different aspects of the
5# buildsystem such as layers, bitbake location, etc.
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08006#
7# SDK_LAYERS_EXCLUDE: Layers which will be excluded from SDK layers.
8# SDK_LAYERS_EXCLUDE_PATTERN: The simiar to SDK_LAYERS_EXCLUDE, this supports
9# python regular expression, use space as separator,
10# e.g.: ".*-downloads closed-.*"
11#
12
Patrick Williamsc124f4f2015-09-15 14:41:29 -050013import stat
14import shutil
15
16def _smart_copy(src, dest):
Brad Bishop37a0e4d2017-12-04 01:01:44 -050017 import subprocess
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018 # smart_copy will choose the correct function depending on whether the
19 # source is a file or a directory.
20 mode = os.stat(src).st_mode
21 if stat.S_ISDIR(mode):
Brad Bishop37a0e4d2017-12-04 01:01:44 -050022 bb.utils.mkdirhier(dest)
23 cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \
24 | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)
25 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050026 else:
27 shutil.copyfile(src, dest)
28 shutil.copymode(src, dest)
29
30class BuildSystem(object):
Patrick Williamsf1e5d692016-03-30 15:21:19 -050031 def __init__(self, context, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050032 self.d = d
Patrick Williamsf1e5d692016-03-30 15:21:19 -050033 self.context = context
Brad Bishop6e60e8b2018-02-01 10:27:11 -050034 self.layerdirs = [os.path.abspath(pth) for pth in d.getVar('BBLAYERS').split()]
35 self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE') or "").split()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080036 self.layers_exclude_pattern = d.getVar('SDK_LAYERS_EXCLUDE_PATTERN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050038 def copy_bitbake_and_layers(self, destdir, workspace_name=None):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080039 import re
Patrick Williamsc124f4f2015-09-15 14:41:29 -050040 # Copy in all metadata layers + bitbake (as repositories)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080041 copied_corebase = None
Patrick Williamsc124f4f2015-09-15 14:41:29 -050042 layers_copied = []
43 bb.utils.mkdirhier(destdir)
44 layers = list(self.layerdirs)
45
Brad Bishop6e60e8b2018-02-01 10:27:11 -050046 corebase = os.path.abspath(self.d.getVar('COREBASE'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050047 layers.append(corebase)
Brad Bishop96ff1982019-08-19 13:50:42 -040048 # Get relationship between TOPDIR and COREBASE
49 # Layers should respect it
50 corebase_relative = os.path.dirname(os.path.relpath(os.path.abspath(self.d.getVar('TOPDIR')), corebase))
Brad Bishopd7bf8c12018-02-25 22:55:05 -050051 # The bitbake build system uses the meta-skeleton layer as a layout
52 # for common recipies, e.g: the recipetool script to create kernel recipies
53 # Add the meta-skeleton layer to be included as part of the eSDK installation
54 layers.append(os.path.join(corebase, 'meta-skeleton'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050055
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050056 # Exclude layers
57 for layer_exclude in self.layers_exclude:
58 if layer_exclude in layers:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080059 bb.note('Excluded %s from sdk layers since it is in SDK_LAYERS_EXCLUDE' % layer_exclude)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050060 layers.remove(layer_exclude)
61
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080062 if self.layers_exclude_pattern:
63 layers_cp = layers[:]
64 for pattern in self.layers_exclude_pattern.split():
65 for layer in layers_cp:
66 if re.match(pattern, layer):
67 bb.note('Excluded %s from sdk layers since matched SDK_LAYERS_EXCLUDE_PATTERN' % layer)
68 layers.remove(layer)
69
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050070 workspace_newname = workspace_name
71 if workspace_newname:
72 layernames = [os.path.basename(layer) for layer in layers]
73 extranum = 0
74 while workspace_newname in layernames:
75 extranum += 1
76 workspace_newname = '%s-%d' % (workspace_name, extranum)
77
Brad Bishop6e60e8b2018-02-01 10:27:11 -050078 corebase_files = self.d.getVar('COREBASE_FILES').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050079 corebase_files = [corebase + '/' +x for x in corebase_files]
80 # Make sure bitbake goes in
81 bitbake_dir = bb.__file__.rsplit('/', 3)[0]
82 corebase_files.append(bitbake_dir)
83
84 for layer in layers:
85 layerconf = os.path.join(layer, 'conf', 'layer.conf')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050086 layernewname = os.path.basename(layer)
87 workspace = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -050088 if os.path.exists(layerconf):
89 with open(layerconf, 'r') as f:
90 if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050091 if workspace_newname:
92 layernewname = workspace_newname
93 workspace = True
94 else:
95 bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context))
96 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -050097
98 # If the layer was already under corebase, leave it there
99 # since layers such as meta have issues when moved.
100 layerdestpath = destdir
101 if corebase == os.path.dirname(layer):
102 layerdestpath += '/' + os.path.basename(corebase)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500103 else:
Brad Bishop96ff1982019-08-19 13:50:42 -0400104 layer_relative = os.path.relpath(layer, corebase)
105 if os.path.dirname(layer_relative) == corebase_relative:
106 layer_relative = os.path.dirname(corebase_relative) + '/' + layernewname
107 layer_relative = os.path.basename(corebase) + '/' + layer_relative
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500108 if os.path.dirname(layer_relative) != layernewname:
109 layerdestpath += '/' + os.path.dirname(layer_relative)
110
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500111 layerdestpath += '/' + layernewname
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500112
113 layer_relative = os.path.relpath(layerdestpath,
114 destdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500115 # Treat corebase as special since it typically will contain
116 # build directories or other custom items.
117 if corebase == layer:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800118 copied_corebase = layer_relative
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500119 bb.utils.mkdirhier(layerdestpath)
120 for f in corebase_files:
121 f_basename = os.path.basename(f)
122 destname = os.path.join(layerdestpath, f_basename)
123 _smart_copy(f, destname)
124 else:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800125 layers_copied.append(layer_relative)
126
Brad Bishop316dfdd2018-06-25 12:45:53 -0400127 if os.path.exists(os.path.join(layerdestpath, 'conf/layer.conf')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128 bb.note("Skipping layer %s, already handled" % layer)
129 else:
130 _smart_copy(layer, layerdestpath)
131
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500132 if workspace:
133 # Make some adjustments original workspace layer
134 # Drop sources (recipe tasks will be locked, so we don't need them)
135 srcdir = os.path.join(layerdestpath, 'sources')
136 if os.path.isdir(srcdir):
137 shutil.rmtree(srcdir)
138 # Drop all bbappends except the one for the image the SDK is being built for
139 # (because of externalsrc, the workspace bbappends will interfere with the
140 # locked signatures if present, and we don't need them anyway)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500141 image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE')))[0] + '.bbappend'
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500142 appenddir = os.path.join(layerdestpath, 'appends')
143 if os.path.isdir(appenddir):
144 for fn in os.listdir(appenddir):
145 if fn == image_bbappend:
146 continue
147 else:
148 os.remove(os.path.join(appenddir, fn))
149 # Drop README
150 readme = os.path.join(layerdestpath, 'README')
151 if os.path.exists(readme):
152 os.remove(readme)
153 # Filter out comments in layer.conf and change layer name
154 layerconf = os.path.join(layerdestpath, 'conf', 'layer.conf')
155 with open(layerconf, 'r') as f:
156 origlines = f.readlines()
157 with open(layerconf, 'w') as f:
158 for line in origlines:
159 if line.startswith('#'):
160 continue
161 line = line.replace('workspacelayer', workspace_newname)
162 f.write(line)
163
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500164 # meta-skeleton layer is added as part of the build system
165 # but not as a layer included in the build, therefore it is
166 # not reported to the function caller.
167 for layer in layers_copied:
168 if layer.endswith('/meta-skeleton'):
169 layers_copied.remove(layer)
170 break
171
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800172 return copied_corebase, layers_copied
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500173
174def generate_locked_sigs(sigfile, d):
175 bb.utils.mkdirhier(os.path.dirname(sigfile))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500176 depd = d.getVar('BB_TASKDEPDATA', False)
Brad Bishop08902b02019-08-20 09:16:51 -0400177 tasks = ['%s:%s' % (v[2], v[1]) for v in depd.values()]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500178 bb.parse.siggen.dump_lockedsigs(sigfile, tasks)
179
Brad Bishopa34c0302019-09-23 22:34:48 -0400180def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, onlynative, pruned_output):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500181 with open(lockedsigs, 'r') as infile:
182 bb.utils.mkdirhier(os.path.dirname(pruned_output))
183 with open(pruned_output, 'w') as f:
184 invalue = False
185 for line in infile:
186 if invalue:
187 if line.endswith('\\\n'):
188 splitval = line.strip().split(':')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500189 if not splitval[1] in excluded_tasks and not splitval[0] in excluded_targets:
Brad Bishopa34c0302019-09-23 22:34:48 -0400190 if onlynative:
191 if 'nativesdk' in splitval[0]:
192 f.write(line)
193 else:
194 f.write(line)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500195 else:
196 f.write(line)
197 invalue = False
198 elif line.startswith('SIGGEN_LOCKEDSIGS'):
199 invalue = True
200 f.write(line)
201
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600202def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output=None):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500203 merged = {}
204 arch_order = []
205 with open(lockedsigs_main, 'r') as f:
206 invalue = None
207 for line in f:
208 if invalue:
209 if line.endswith('\\\n'):
210 merged[invalue].append(line)
211 else:
212 invalue = None
213 elif line.startswith('SIGGEN_LOCKEDSIGS_t-'):
214 invalue = line[18:].split('=', 1)[0].rstrip()
215 merged[invalue] = []
216 arch_order.append(invalue)
217
218 with open(lockedsigs_extra, 'r') as f:
219 invalue = None
220 tocopy = {}
221 for line in f:
222 if invalue:
223 if line.endswith('\\\n'):
224 if not line in merged[invalue]:
225 target, task = line.strip().split(':')[:2]
226 if not copy_tasks or task in copy_tasks:
227 tocopy[invalue].append(line)
228 merged[invalue].append(line)
229 else:
230 invalue = None
231 elif line.startswith('SIGGEN_LOCKEDSIGS_t-'):
232 invalue = line[18:].split('=', 1)[0].rstrip()
233 if not invalue in merged:
234 merged[invalue] = []
235 arch_order.append(invalue)
236 tocopy[invalue] = []
237
238 def write_sigs_file(fn, types, sigs):
239 fulltypes = []
240 bb.utils.mkdirhier(os.path.dirname(fn))
241 with open(fn, 'w') as f:
242 for typename in types:
243 lines = sigs[typename]
244 if lines:
245 f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % typename)
246 for line in lines:
247 f.write(line)
248 f.write(' "\n')
249 fulltypes.append(typename)
250 f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes))
251
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600252 if copy_output:
253 write_sigs_file(copy_output, list(tocopy.keys()), tocopy)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500254 if merged_output:
255 write_sigs_file(merged_output, arch_order, merged)
256
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600257def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None):
258 import shutil
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500259 bb.note('Generating sstate-cache...')
260
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500261 nativelsbstring = d.getVar('NATIVELSBSTRING')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600262 bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
263 if fixedlsbstring and nativelsbstring != fixedlsbstring:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500264 nativedir = output_sstate_cache + '/' + nativelsbstring
265 if os.path.isdir(nativedir):
266 destdir = os.path.join(output_sstate_cache, fixedlsbstring)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600267 for root, _, files in os.walk(nativedir):
268 for fn in files:
269 src = os.path.join(root, fn)
270 dest = os.path.join(destdir, os.path.relpath(src, nativedir))
271 if os.path.exists(dest):
272 # Already exists, and it'll be the same file, so just delete it
273 os.unlink(src)
274 else:
275 bb.utils.mkdirhier(os.path.dirname(dest))
276 shutil.move(src, dest)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500277
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600278def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None):
279 import subprocess
280
281 bb.note('Generating sstate task list...')
282
283 if not cwd:
284 cwd = os.getcwd()
285 if logfile:
286 logparam = '-l %s' % logfile
287 else:
288 logparam = ''
289 cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
290 env = dict(d.getVar('BB_ORIGENV', False))
291 env.pop('BUILDDIR', '')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500292 env.pop('BBPATH', '')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600293 pathitems = env['PATH'].split(':')
294 env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')])
295 bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash')