blob: 4b94806c73f28a105ee0caeaeb376b7cd3403734 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# This class should provide easy access to the different aspects of the
2# buildsystem such as layers, bitbake location, etc.
3import stat
4import shutil
5
6def _smart_copy(src, dest):
Brad Bishop37a0e4d2017-12-04 01:01:44 -05007 import subprocess
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008 # smart_copy will choose the correct function depending on whether the
9 # source is a file or a directory.
10 mode = os.stat(src).st_mode
11 if stat.S_ISDIR(mode):
Brad Bishop37a0e4d2017-12-04 01:01:44 -050012 bb.utils.mkdirhier(dest)
13 cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \
14 | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)
15 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050016 else:
17 shutil.copyfile(src, dest)
18 shutil.copymode(src, dest)
19
20class BuildSystem(object):
Patrick Williamsf1e5d692016-03-30 15:21:19 -050021 def __init__(self, context, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050022 self.d = d
Patrick Williamsf1e5d692016-03-30 15:21:19 -050023 self.context = context
Brad Bishop6e60e8b2018-02-01 10:27:11 -050024 self.layerdirs = [os.path.abspath(pth) for pth in d.getVar('BBLAYERS').split()]
25 self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE') or "").split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050026
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050027 def copy_bitbake_and_layers(self, destdir, workspace_name=None):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050028 # Copy in all metadata layers + bitbake (as repositories)
29 layers_copied = []
30 bb.utils.mkdirhier(destdir)
31 layers = list(self.layerdirs)
32
Brad Bishop6e60e8b2018-02-01 10:27:11 -050033 corebase = os.path.abspath(self.d.getVar('COREBASE'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050034 layers.append(corebase)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050035 # The bitbake build system uses the meta-skeleton layer as a layout
36 # for common recipies, e.g: the recipetool script to create kernel recipies
37 # Add the meta-skeleton layer to be included as part of the eSDK installation
38 layers.append(os.path.join(corebase, 'meta-skeleton'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050039
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050040 # Exclude layers
41 for layer_exclude in self.layers_exclude:
42 if layer_exclude in layers:
43 layers.remove(layer_exclude)
44
45 workspace_newname = workspace_name
46 if workspace_newname:
47 layernames = [os.path.basename(layer) for layer in layers]
48 extranum = 0
49 while workspace_newname in layernames:
50 extranum += 1
51 workspace_newname = '%s-%d' % (workspace_name, extranum)
52
Brad Bishop6e60e8b2018-02-01 10:27:11 -050053 corebase_files = self.d.getVar('COREBASE_FILES').split()
Patrick Williamsc124f4f2015-09-15 14:41:29 -050054 corebase_files = [corebase + '/' +x for x in corebase_files]
55 # Make sure bitbake goes in
56 bitbake_dir = bb.__file__.rsplit('/', 3)[0]
57 corebase_files.append(bitbake_dir)
58
59 for layer in layers:
60 layerconf = os.path.join(layer, 'conf', 'layer.conf')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050061 layernewname = os.path.basename(layer)
62 workspace = False
Patrick Williamsc124f4f2015-09-15 14:41:29 -050063 if os.path.exists(layerconf):
64 with open(layerconf, 'r') as f:
65 if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050066 if workspace_newname:
67 layernewname = workspace_newname
68 workspace = True
69 else:
70 bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context))
71 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -050072
73 # If the layer was already under corebase, leave it there
74 # since layers such as meta have issues when moved.
75 layerdestpath = destdir
76 if corebase == os.path.dirname(layer):
77 layerdestpath += '/' + os.path.basename(corebase)
Brad Bishopd7bf8c12018-02-25 22:55:05 -050078 else:
79 layer_relative = os.path.basename(corebase) + '/' + os.path.relpath(layer, corebase)
80 if os.path.dirname(layer_relative) != layernewname:
81 layerdestpath += '/' + os.path.dirname(layer_relative)
82
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050083 layerdestpath += '/' + layernewname
Patrick Williamsc124f4f2015-09-15 14:41:29 -050084
85 layer_relative = os.path.relpath(layerdestpath,
86 destdir)
87 layers_copied.append(layer_relative)
88
89 # Treat corebase as special since it typically will contain
90 # build directories or other custom items.
91 if corebase == layer:
92 bb.utils.mkdirhier(layerdestpath)
93 for f in corebase_files:
94 f_basename = os.path.basename(f)
95 destname = os.path.join(layerdestpath, f_basename)
96 _smart_copy(f, destname)
97 else:
Brad Bishop316dfdd2018-06-25 12:45:53 -040098 if os.path.exists(os.path.join(layerdestpath, 'conf/layer.conf')):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050099 bb.note("Skipping layer %s, already handled" % layer)
100 else:
101 _smart_copy(layer, layerdestpath)
102
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500103 if workspace:
104 # Make some adjustments original workspace layer
105 # Drop sources (recipe tasks will be locked, so we don't need them)
106 srcdir = os.path.join(layerdestpath, 'sources')
107 if os.path.isdir(srcdir):
108 shutil.rmtree(srcdir)
109 # Drop all bbappends except the one for the image the SDK is being built for
110 # (because of externalsrc, the workspace bbappends will interfere with the
111 # locked signatures if present, and we don't need them anyway)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500112 image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE')))[0] + '.bbappend'
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500113 appenddir = os.path.join(layerdestpath, 'appends')
114 if os.path.isdir(appenddir):
115 for fn in os.listdir(appenddir):
116 if fn == image_bbappend:
117 continue
118 else:
119 os.remove(os.path.join(appenddir, fn))
120 # Drop README
121 readme = os.path.join(layerdestpath, 'README')
122 if os.path.exists(readme):
123 os.remove(readme)
124 # Filter out comments in layer.conf and change layer name
125 layerconf = os.path.join(layerdestpath, 'conf', 'layer.conf')
126 with open(layerconf, 'r') as f:
127 origlines = f.readlines()
128 with open(layerconf, 'w') as f:
129 for line in origlines:
130 if line.startswith('#'):
131 continue
132 line = line.replace('workspacelayer', workspace_newname)
133 f.write(line)
134
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500135 # meta-skeleton layer is added as part of the build system
136 # but not as a layer included in the build, therefore it is
137 # not reported to the function caller.
138 for layer in layers_copied:
139 if layer.endswith('/meta-skeleton'):
140 layers_copied.remove(layer)
141 break
142
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500143 return layers_copied
144
145def generate_locked_sigs(sigfile, d):
146 bb.utils.mkdirhier(os.path.dirname(sigfile))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500147 depd = d.getVar('BB_TASKDEPDATA', False)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600148 tasks = ['%s.%s' % (v[2], v[1]) for v in depd.values()]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500149 bb.parse.siggen.dump_lockedsigs(sigfile, tasks)
150
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500151def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, pruned_output):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500152 with open(lockedsigs, 'r') as infile:
153 bb.utils.mkdirhier(os.path.dirname(pruned_output))
154 with open(pruned_output, 'w') as f:
155 invalue = False
156 for line in infile:
157 if invalue:
158 if line.endswith('\\\n'):
159 splitval = line.strip().split(':')
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500160 if not splitval[1] in excluded_tasks and not splitval[0] in excluded_targets:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500161 f.write(line)
162 else:
163 f.write(line)
164 invalue = False
165 elif line.startswith('SIGGEN_LOCKEDSIGS'):
166 invalue = True
167 f.write(line)
168
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600169def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output=None):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500170 merged = {}
171 arch_order = []
172 with open(lockedsigs_main, 'r') as f:
173 invalue = None
174 for line in f:
175 if invalue:
176 if line.endswith('\\\n'):
177 merged[invalue].append(line)
178 else:
179 invalue = None
180 elif line.startswith('SIGGEN_LOCKEDSIGS_t-'):
181 invalue = line[18:].split('=', 1)[0].rstrip()
182 merged[invalue] = []
183 arch_order.append(invalue)
184
185 with open(lockedsigs_extra, 'r') as f:
186 invalue = None
187 tocopy = {}
188 for line in f:
189 if invalue:
190 if line.endswith('\\\n'):
191 if not line in merged[invalue]:
192 target, task = line.strip().split(':')[:2]
193 if not copy_tasks or task in copy_tasks:
194 tocopy[invalue].append(line)
195 merged[invalue].append(line)
196 else:
197 invalue = None
198 elif line.startswith('SIGGEN_LOCKEDSIGS_t-'):
199 invalue = line[18:].split('=', 1)[0].rstrip()
200 if not invalue in merged:
201 merged[invalue] = []
202 arch_order.append(invalue)
203 tocopy[invalue] = []
204
205 def write_sigs_file(fn, types, sigs):
206 fulltypes = []
207 bb.utils.mkdirhier(os.path.dirname(fn))
208 with open(fn, 'w') as f:
209 for typename in types:
210 lines = sigs[typename]
211 if lines:
212 f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % typename)
213 for line in lines:
214 f.write(line)
215 f.write(' "\n')
216 fulltypes.append(typename)
217 f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes))
218
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600219 if copy_output:
220 write_sigs_file(copy_output, list(tocopy.keys()), tocopy)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500221 if merged_output:
222 write_sigs_file(merged_output, arch_order, merged)
223
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600224def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None):
225 import shutil
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500226 bb.note('Generating sstate-cache...')
227
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500228 nativelsbstring = d.getVar('NATIVELSBSTRING')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600229 bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
230 if fixedlsbstring and nativelsbstring != fixedlsbstring:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500231 nativedir = output_sstate_cache + '/' + nativelsbstring
232 if os.path.isdir(nativedir):
233 destdir = os.path.join(output_sstate_cache, fixedlsbstring)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600234 for root, _, files in os.walk(nativedir):
235 for fn in files:
236 src = os.path.join(root, fn)
237 dest = os.path.join(destdir, os.path.relpath(src, nativedir))
238 if os.path.exists(dest):
239 # Already exists, and it'll be the same file, so just delete it
240 os.unlink(src)
241 else:
242 bb.utils.mkdirhier(os.path.dirname(dest))
243 shutil.move(src, dest)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500244
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600245def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None):
246 import subprocess
247
248 bb.note('Generating sstate task list...')
249
250 if not cwd:
251 cwd = os.getcwd()
252 if logfile:
253 logparam = '-l %s' % logfile
254 else:
255 logparam = ''
256 cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
257 env = dict(d.getVar('BB_ORIGENV', False))
258 env.pop('BUILDDIR', '')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500259 env.pop('BBPATH', '')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600260 pathitems = env['PATH'].split(':')
261 env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')])
262 bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash')