blob: 3dab607eebefe4ab9546e067d58eba27dc36e8bc [file] [log] [blame]
Brad Bishopc342db32019-05-15 21:57:59 -04001#
2# SPDX-License-Identifier: MIT
3#
4
Brad Bishopd7bf8c12018-02-25 22:55:05 -05005import os
6import shutil
7import glob
8import subprocess
Brad Bishop316dfdd2018-06-25 12:45:53 -04009import tempfile
Brad Bishopd7bf8c12018-02-25 22:55:05 -050010
11from oeqa.selftest.case import OESelftestTestCase
Brad Bishop316dfdd2018-06-25 12:45:53 -040012from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_test_layer, create_temp_layer
Brad Bishopd7bf8c12018-02-25 22:55:05 -050013from oeqa.selftest.cases.sstate import SStateBase
Brad Bishopd7bf8c12018-02-25 22:55:05 -050014
15import bb.siggen
16
17class SStateTests(SStateBase):
Brad Bishop316dfdd2018-06-25 12:45:53 -040018 def test_autorev_sstate_works(self):
19 # Test that a git repository which changes is correctly handled by SRCREV = ${AUTOREV}
20 # when PV does not contain SRCPV
21
Andrew Geisslerb7d28612020-07-24 16:15:54 -050022 tempdir = tempfile.mkdtemp(prefix='sstate_autorev')
23 tempdldir = tempfile.mkdtemp(prefix='sstate_autorev_dldir')
Brad Bishop316dfdd2018-06-25 12:45:53 -040024 self.track_for_cleanup(tempdir)
Andrew Geisslerb7d28612020-07-24 16:15:54 -050025 self.track_for_cleanup(tempdldir)
Brad Bishop316dfdd2018-06-25 12:45:53 -040026 create_temp_layer(tempdir, 'selftestrecipetool')
27 self.add_command_to_tearDown('bitbake-layers remove-layer %s' % tempdir)
Andrew Geisslerb7d28612020-07-24 16:15:54 -050028 self.append_config("DL_DIR = \"%s\"" % tempdldir)
Brad Bishop316dfdd2018-06-25 12:45:53 -040029 runCmd('bitbake-layers add-layer %s' % tempdir)
30
31 # Use dbus-wait as a local git repo we can add a commit between two builds in
32 pn = 'dbus-wait'
33 srcrev = '6cc6077a36fe2648a5f993fe7c16c9632f946517'
34 url = 'git://git.yoctoproject.org/dbus-wait'
35 result = runCmd('git clone %s noname' % url, cwd=tempdir)
36 srcdir = os.path.join(tempdir, 'noname')
37 result = runCmd('git reset --hard %s' % srcrev, cwd=srcdir)
38 self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory')
39
40 recipefile = os.path.join(tempdir, "recipes-test", "dbus-wait-test", 'dbus-wait-test_git.bb')
41 os.makedirs(os.path.dirname(recipefile))
42 srcuri = 'git://' + srcdir + ';protocol=file'
43 result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri])
44 self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output)
45
46 with open(recipefile, 'a') as f:
47 f.write('SRCREV = "${AUTOREV}"\n')
48 f.write('PV = "1.0"\n')
49
50 bitbake("dbus-wait-test -c fetch")
51 with open(os.path.join(srcdir, "bar.txt"), "w") as f:
52 f.write("foo")
53 result = runCmd('git add bar.txt; git commit -asm "add bar"', cwd=srcdir)
54 bitbake("dbus-wait-test -c unpack")
55
Brad Bishopd7bf8c12018-02-25 22:55:05 -050056
57 # Test sstate files creation and their location
58 def run_test_sstate_creation(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True, should_pass=True):
59 self.config_sstate(temp_sstate_location, [self.sstate_path])
60
61 if self.temp_sstate_location:
62 bitbake(['-cclean'] + targets)
63 else:
64 bitbake(['-ccleansstate'] + targets)
65
66 bitbake(targets)
67 file_tracker = []
68 results = self.search_sstate('|'.join(map(str, targets)), distro_specific, distro_nonspecific)
69 if distro_nonspecific:
70 for r in results:
Andrew Geisslereff27472021-10-29 15:35:00 -050071 if r.endswith(("_populate_lic.tar.zst", "_populate_lic.tar.zst.siginfo", "_fetch.tar.zst.siginfo", "_unpack.tar.zst.siginfo", "_patch.tar.zst.siginfo")):
Brad Bishopd7bf8c12018-02-25 22:55:05 -050072 continue
73 file_tracker.append(r)
74 else:
75 file_tracker = results
76
77 if should_pass:
78 self.assertTrue(file_tracker , msg="Could not find sstate files for: %s" % ', '.join(map(str, targets)))
79 else:
80 self.assertTrue(not file_tracker , msg="Found sstate files in the wrong place for: %s (found %s)" % (', '.join(map(str, targets)), str(file_tracker)))
81
Brad Bishopd7bf8c12018-02-25 22:55:05 -050082 def test_sstate_creation_distro_specific_pass(self):
83 self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True)
84
Brad Bishopd7bf8c12018-02-25 22:55:05 -050085 def test_sstate_creation_distro_specific_fail(self):
86 self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False)
87
Brad Bishopd7bf8c12018-02-25 22:55:05 -050088 def test_sstate_creation_distro_nonspecific_pass(self):
89 self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True)
90
Brad Bishopd7bf8c12018-02-25 22:55:05 -050091 def test_sstate_creation_distro_nonspecific_fail(self):
92 self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False)
93
94 # Test the sstate files deletion part of the do_cleansstate task
95 def run_test_cleansstate_task(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True):
96 self.config_sstate(temp_sstate_location, [self.sstate_path])
97
98 bitbake(['-ccleansstate'] + targets)
99
100 bitbake(targets)
Andrew Geisslereff27472021-10-29 15:35:00 -0500101 archives_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific, distro_nonspecific)
102 self.assertTrue(archives_created, msg="Could not find sstate .tar.zst files for: %s (%s)" % (', '.join(map(str, targets)), str(archives_created)))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500103
Brad Bishop19323692019-04-05 15:28:33 -0400104 siginfo_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.siginfo$' for s in targets])), distro_specific, distro_nonspecific)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500105 self.assertTrue(siginfo_created, msg="Could not find sstate .siginfo files for: %s (%s)" % (', '.join(map(str, targets)), str(siginfo_created)))
106
107 bitbake(['-ccleansstate'] + targets)
Andrew Geisslereff27472021-10-29 15:35:00 -0500108 archives_removed = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific, distro_nonspecific)
109 self.assertTrue(not archives_removed, msg="do_cleansstate didn't remove .tar.zst sstate files for: %s (%s)" % (', '.join(map(str, targets)), str(archives_removed)))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500110
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500111 def test_cleansstate_task_distro_specific_nonspecific(self):
112 targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native']
113 targets.append('linux-libc-headers')
114 self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True)
115
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500116 def test_cleansstate_task_distro_nonspecific(self):
117 self.run_test_cleansstate_task(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True)
118
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500119 def test_cleansstate_task_distro_specific(self):
120 targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native']
121 targets.append('linux-libc-headers')
122 self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=False, temp_sstate_location=True)
123
124
125 # Test rebuilding of distro-specific sstate files
126 def run_test_rebuild_distro_specific_sstate(self, targets, temp_sstate_location=True):
127 self.config_sstate(temp_sstate_location, [self.sstate_path])
128
129 bitbake(['-ccleansstate'] + targets)
130
131 bitbake(targets)
Andrew Geisslereff27472021-10-29 15:35:00 -0500132 results = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=False, distro_nonspecific=True)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500133 filtered_results = []
134 for r in results:
Andrew Geisslereff27472021-10-29 15:35:00 -0500135 if r.endswith(("_populate_lic.tar.zst", "_populate_lic.tar.zst.siginfo")):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500136 continue
137 filtered_results.append(r)
138 self.assertTrue(filtered_results == [], msg="Found distro non-specific sstate for: %s (%s)" % (', '.join(map(str, targets)), str(filtered_results)))
Andrew Geisslereff27472021-10-29 15:35:00 -0500139 file_tracker_1 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=True, distro_nonspecific=False)
140 self.assertTrue(len(file_tracker_1) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets)))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500141
142 self.track_for_cleanup(self.distro_specific_sstate + "_old")
143 shutil.copytree(self.distro_specific_sstate, self.distro_specific_sstate + "_old")
144 shutil.rmtree(self.distro_specific_sstate)
145
146 bitbake(['-cclean'] + targets)
147 bitbake(targets)
Andrew Geisslereff27472021-10-29 15:35:00 -0500148 file_tracker_2 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=True, distro_nonspecific=False)
149 self.assertTrue(len(file_tracker_2) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets)))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500150
151 not_recreated = [x for x in file_tracker_1 if x not in file_tracker_2]
Andrew Geisslereff27472021-10-29 15:35:00 -0500152 self.assertTrue(not_recreated == [], msg="The following sstate files were not recreated: %s" % ', '.join(map(str, not_recreated)))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500153
154 created_once = [x for x in file_tracker_2 if x not in file_tracker_1]
Andrew Geisslereff27472021-10-29 15:35:00 -0500155 self.assertTrue(created_once == [], msg="The following sstate files were created only in the second run: %s" % ', '.join(map(str, created_once)))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500156
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500157 def test_rebuild_distro_specific_sstate_cross_native_targets(self):
158 self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch, 'binutils-native'], temp_sstate_location=True)
159
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500160 def test_rebuild_distro_specific_sstate_cross_target(self):
161 self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch], temp_sstate_location=True)
162
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500163 def test_rebuild_distro_specific_sstate_native_target(self):
164 self.run_test_rebuild_distro_specific_sstate(['binutils-native'], temp_sstate_location=True)
165
166
167 # Test the sstate-cache-management script. Each element in the global_config list is used with the corresponding element in the target_config list
168 # global_config elements are expected to not generate any sstate files that would be removed by sstate-cache-management.sh (such as changing the value of MACHINE)
169 def run_test_sstate_cache_management_script(self, target, global_config=[''], target_config=[''], ignore_patterns=[]):
170 self.assertTrue(global_config)
171 self.assertTrue(target_config)
172 self.assertTrue(len(global_config) == len(target_config), msg='Lists global_config and target_config should have the same number of elements')
173 self.config_sstate(temp_sstate_location=True, add_local_mirrors=[self.sstate_path])
174
175 # If buildhistory is enabled, we need to disable version-going-backwards
176 # QA checks for this test. It may report errors otherwise.
Patrick Williams213cb262021-08-07 19:21:33 -0500177 self.append_config('ERROR_QA:remove = "version-going-backwards"')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500178
179 # For not this only checks if random sstate tasks are handled correctly as a group.
180 # In the future we should add control over what tasks we check for.
181
182 sstate_archs_list = []
183 expected_remaining_sstate = []
184 for idx in range(len(target_config)):
185 self.append_config(global_config[idx])
186 self.append_recipeinc(target, target_config[idx])
187 sstate_arch = get_bb_var('SSTATE_PKGARCH', target)
188 if not sstate_arch in sstate_archs_list:
189 sstate_archs_list.append(sstate_arch)
190 if target_config[idx] == target_config[-1]:
Andrew Geisslereff27472021-10-29 15:35:00 -0500191 target_sstate_before_build = self.search_sstate(target + r'.*?\.tar.zst$')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500192 bitbake("-cclean %s" % target)
193 result = bitbake(target, ignore_status=True)
194 if target_config[idx] == target_config[-1]:
Andrew Geisslereff27472021-10-29 15:35:00 -0500195 target_sstate_after_build = self.search_sstate(target + r'.*?\.tar.zst$')
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500196 expected_remaining_sstate += [x for x in target_sstate_after_build if x not in target_sstate_before_build if not any(pattern in x for pattern in ignore_patterns)]
197 self.remove_config(global_config[idx])
198 self.remove_recipeinc(target, target_config[idx])
199 self.assertEqual(result.status, 0, msg = "build of %s failed with %s" % (target, result.output))
200
201 runCmd("sstate-cache-management.sh -y --cache-dir=%s --remove-duplicated --extra-archs=%s" % (self.sstate_path, ','.join(map(str, sstate_archs_list))))
Andrew Geisslereff27472021-10-29 15:35:00 -0500202 actual_remaining_sstate = [x for x in self.search_sstate(target + r'.*?\.tar.zst$') if not any(pattern in x for pattern in ignore_patterns)]
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500203
204 actual_not_expected = [x for x in actual_remaining_sstate if x not in expected_remaining_sstate]
Andrew Geisslereff27472021-10-29 15:35:00 -0500205 self.assertFalse(actual_not_expected, msg="Files should have been removed but were not: %s" % ', '.join(map(str, actual_not_expected)))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500206 expected_not_actual = [x for x in expected_remaining_sstate if x not in actual_remaining_sstate]
Andrew Geisslereff27472021-10-29 15:35:00 -0500207 self.assertFalse(expected_not_actual, msg="Extra files were removed: %s" ', '.join(map(str, expected_not_actual)))
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500208
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500209 def test_sstate_cache_management_script_using_pr_1(self):
210 global_config = []
211 target_config = []
212 global_config.append('')
213 target_config.append('PR = "0"')
214 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic'])
215
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500216 def test_sstate_cache_management_script_using_pr_2(self):
217 global_config = []
218 target_config = []
219 global_config.append('')
220 target_config.append('PR = "0"')
221 global_config.append('')
222 target_config.append('PR = "1"')
223 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic'])
224
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500225 def test_sstate_cache_management_script_using_pr_3(self):
226 global_config = []
227 target_config = []
228 global_config.append('MACHINE = "qemux86-64"')
229 target_config.append('PR = "0"')
230 global_config.append(global_config[0])
231 target_config.append('PR = "1"')
232 global_config.append('MACHINE = "qemux86"')
233 target_config.append('PR = "1"')
234 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic'])
235
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500236 def test_sstate_cache_management_script_using_machine(self):
237 global_config = []
238 target_config = []
239 global_config.append('MACHINE = "qemux86-64"')
240 target_config.append('')
241 global_config.append('MACHINE = "qemux86"')
242 target_config.append('')
243 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic'])
244
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500245 def test_sstate_32_64_same_hash(self):
246 """
247 The sstate checksums for both native and target should not vary whether
248 they're built on a 32 or 64 bit system. Rather than requiring two different
249 build machines and running a builds, override the variables calling uname()
250 manually and check using bitbake -S.
251 """
252
253 self.write_config("""
254MACHINE = "qemux86"
255TMPDIR = "${TOPDIR}/tmp-sstatesamehash"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800256TCLIBCAPPEND = ""
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500257BUILD_ARCH = "x86_64"
258BUILD_OS = "linux"
259SDKMACHINE = "x86_64"
260PACKAGE_CLASSES = "package_rpm package_ipk package_deb"
Brad Bishop6dbb3162019-11-25 09:41:34 -0500261BB_SIGNATURE_HANDLER = "OEBasicHash"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500262""")
263 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash")
Andrew Geisslerc926e172021-05-07 16:11:35 -0500264 bitbake("core-image-weston -S none")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500265 self.write_config("""
266MACHINE = "qemux86"
267TMPDIR = "${TOPDIR}/tmp-sstatesamehash2"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800268TCLIBCAPPEND = ""
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500269BUILD_ARCH = "i686"
270BUILD_OS = "linux"
271SDKMACHINE = "i686"
272PACKAGE_CLASSES = "package_rpm package_ipk package_deb"
Brad Bishop6dbb3162019-11-25 09:41:34 -0500273BB_SIGNATURE_HANDLER = "OEBasicHash"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500274""")
275 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2")
Andrew Geisslerc926e172021-05-07 16:11:35 -0500276 bitbake("core-image-weston -S none")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500277
278 def get_files(d):
279 f = []
280 for root, dirs, files in os.walk(d):
Andrew Geisslerc926e172021-05-07 16:11:35 -0500281 if "core-image-weston" in root:
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500282 # SDKMACHINE changing will change
283 # do_rootfs/do_testimage/do_build stamps of images which
284 # is safe to ignore.
285 continue
286 f.extend(os.path.join(root, name) for name in files)
287 return f
288 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/")
289 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/")
290 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash").replace("i686-linux", "x86_64-linux").replace("i686" + self.target_vendor + "-linux", "x86_64" + self.target_vendor + "-linux", ) for x in files2]
291 self.maxDiff = None
292 self.assertCountEqual(files1, files2)
293
294
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500295 def test_sstate_nativelsbstring_same_hash(self):
296 """
297 The sstate checksums should be independent of whichever NATIVELSBSTRING is
298 detected. Rather than requiring two different build machines and running
299 builds, override the variables manually and check using bitbake -S.
300 """
301
302 self.write_config("""
303TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800304TCLIBCAPPEND = \"\"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500305NATIVELSBSTRING = \"DistroA\"
Brad Bishop6dbb3162019-11-25 09:41:34 -0500306BB_SIGNATURE_HANDLER = "OEBasicHash"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500307""")
308 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash")
Andrew Geisslerc926e172021-05-07 16:11:35 -0500309 bitbake("core-image-weston -S none")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500310 self.write_config("""
311TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800312TCLIBCAPPEND = \"\"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500313NATIVELSBSTRING = \"DistroB\"
Brad Bishop6dbb3162019-11-25 09:41:34 -0500314BB_SIGNATURE_HANDLER = "OEBasicHash"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500315""")
316 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2")
Andrew Geisslerc926e172021-05-07 16:11:35 -0500317 bitbake("core-image-weston -S none")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500318
319 def get_files(d):
320 f = []
321 for root, dirs, files in os.walk(d):
322 f.extend(os.path.join(root, name) for name in files)
323 return f
324 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/")
325 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/")
326 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2]
327 self.maxDiff = None
328 self.assertCountEqual(files1, files2)
329
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500330 def test_sstate_allarch_samesigs(self):
331 """
332 The sstate checksums of allarch packages should be independent of whichever
333 MACHINE is set. Check this using bitbake -S.
334 Also, rather than duplicate the test, check nativesdk stamps are the same between
335 the two MACHINE values.
336 """
337
338 configA = """
339TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800340TCLIBCAPPEND = \"\"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500341MACHINE = \"qemux86-64\"
Brad Bishop6dbb3162019-11-25 09:41:34 -0500342BB_SIGNATURE_HANDLER = "OEBasicHash"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500343"""
344 configB = """
345TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800346TCLIBCAPPEND = \"\"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500347MACHINE = \"qemuarm\"
Brad Bishop6dbb3162019-11-25 09:41:34 -0500348BB_SIGNATURE_HANDLER = "OEBasicHash"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500349"""
350 self.sstate_allarch_samesigs(configA, configB)
351
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800352 def test_sstate_nativesdk_samesigs_multilib(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500353 """
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800354 check nativesdk stamps are the same between the two MACHINE values.
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500355 """
356
357 configA = """
358TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800359TCLIBCAPPEND = \"\"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500360MACHINE = \"qemux86-64\"
361require conf/multilib.conf
362MULTILIBS = \"multilib:lib32\"
Patrick Williams213cb262021-08-07 19:21:33 -0500363DEFAULTTUNE:virtclass-multilib-lib32 = \"x86\"
Brad Bishop6dbb3162019-11-25 09:41:34 -0500364BB_SIGNATURE_HANDLER = "OEBasicHash"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500365"""
366 configB = """
367TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800368TCLIBCAPPEND = \"\"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500369MACHINE = \"qemuarm\"
370require conf/multilib.conf
371MULTILIBS = \"\"
Brad Bishop6dbb3162019-11-25 09:41:34 -0500372BB_SIGNATURE_HANDLER = "OEBasicHash"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500373"""
374 self.sstate_allarch_samesigs(configA, configB)
375
376 def sstate_allarch_samesigs(self, configA, configB):
377
378 self.write_config(configA)
379 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash")
380 bitbake("world meta-toolchain -S none")
381 self.write_config(configB)
382 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2")
383 bitbake("world meta-toolchain -S none")
384
385 def get_files(d):
386 f = {}
387 for root, dirs, files in os.walk(d):
388 for name in files:
389 if "meta-environment" in root or "cross-canadian" in root:
390 continue
391 if "do_build" not in name:
392 # 1.4.1+gitAUTOINC+302fca9f4c-r0.do_package_write_ipk.sigdata.f3a2a38697da743f0dbed8b56aafcf79
393 (_, task, _, shash) = name.rsplit(".", 3)
394 f[os.path.join(os.path.basename(root), task)] = shash
395 return f
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500396
397 nativesdkdir = os.path.basename(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/*-nativesdk*-linux")[0])
398
399 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/" + nativesdkdir)
400 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/" + nativesdkdir)
401 self.maxDiff = None
402 self.assertEqual(files1, files2)
403
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500404 def test_sstate_sametune_samesigs(self):
405 """
406 The sstate checksums of two identical machines (using the same tune) should be the
407 same, apart from changes within the machine specific stamps directory. We use the
408 qemux86copy machine to test this. Also include multilibs in the test.
409 """
410
411 self.write_config("""
412TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800413TCLIBCAPPEND = \"\"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500414MACHINE = \"qemux86\"
415require conf/multilib.conf
416MULTILIBS = "multilib:lib32"
Patrick Williams213cb262021-08-07 19:21:33 -0500417DEFAULTTUNE:virtclass-multilib-lib32 = "x86"
Brad Bishop6dbb3162019-11-25 09:41:34 -0500418BB_SIGNATURE_HANDLER = "OEBasicHash"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500419""")
420 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash")
421 bitbake("world meta-toolchain -S none")
422 self.write_config("""
423TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800424TCLIBCAPPEND = \"\"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500425MACHINE = \"qemux86copy\"
426require conf/multilib.conf
427MULTILIBS = "multilib:lib32"
Patrick Williams213cb262021-08-07 19:21:33 -0500428DEFAULTTUNE:virtclass-multilib-lib32 = "x86"
Brad Bishop6dbb3162019-11-25 09:41:34 -0500429BB_SIGNATURE_HANDLER = "OEBasicHash"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500430""")
431 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2")
432 bitbake("world meta-toolchain -S none")
433
434 def get_files(d):
435 f = []
436 for root, dirs, files in os.walk(d):
437 for name in files:
438 if "meta-environment" in root or "cross-canadian" in root:
439 continue
440 if "qemux86copy-" in root or "qemux86-" in root:
441 continue
442 if "do_build" not in name and "do_populate_sdk" not in name:
443 f.append(os.path.join(root, name))
444 return f
445 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps")
446 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps")
447 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2]
448 self.maxDiff = None
449 self.assertCountEqual(files1, files2)
450
451
Andrew Geissler82c905d2020-04-13 13:39:40 -0500452 def test_sstate_multilib_or_not_native_samesigs(self):
453 """The sstate checksums of two native recipes (and their dependencies)
454 where the target is using multilib in one but not the other
455 should be the same. We use the qemux86copy machine to test
456 this.
457 """
458
459 self.write_config("""
460TMPDIR = \"${TOPDIR}/tmp-sstatesamehash\"
461TCLIBCAPPEND = \"\"
462MACHINE = \"qemux86\"
463require conf/multilib.conf
464MULTILIBS = "multilib:lib32"
Patrick Williams213cb262021-08-07 19:21:33 -0500465DEFAULTTUNE:virtclass-multilib-lib32 = "x86"
Andrew Geissler82c905d2020-04-13 13:39:40 -0500466BB_SIGNATURE_HANDLER = "OEBasicHash"
467""")
468 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash")
469 bitbake("binutils-native -S none")
470 self.write_config("""
471TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\"
472TCLIBCAPPEND = \"\"
473MACHINE = \"qemux86copy\"
474BB_SIGNATURE_HANDLER = "OEBasicHash"
475""")
476 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2")
477 bitbake("binutils-native -S none")
478
479 def get_files(d):
480 f = []
481 for root, dirs, files in os.walk(d):
482 for name in files:
483 f.append(os.path.join(root, name))
484 return f
485 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps")
486 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps")
487 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2]
488 self.maxDiff = None
489 self.assertCountEqual(files1, files2)
490
491
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500492 def test_sstate_noop_samesigs(self):
493 """
494 The sstate checksums of two builds with these variables changed or
495 classes inherits should be the same.
496 """
497
498 self.write_config("""
499TMPDIR = "${TOPDIR}/tmp-sstatesamehash"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800500TCLIBCAPPEND = ""
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500501BB_NUMBER_THREADS = "${@oe.utils.cpu_count()}"
502PARALLEL_MAKE = "-j 1"
503DL_DIR = "${TOPDIR}/download1"
504TIME = "111111"
505DATE = "20161111"
Patrick Williams213cb262021-08-07 19:21:33 -0500506INHERIT:remove = "buildstats-summary buildhistory uninative"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500507http_proxy = ""
Brad Bishop6dbb3162019-11-25 09:41:34 -0500508BB_SIGNATURE_HANDLER = "OEBasicHash"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500509""")
510 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash")
511 self.track_for_cleanup(self.topdir + "/download1")
512 bitbake("world meta-toolchain -S none")
513 self.write_config("""
514TMPDIR = "${TOPDIR}/tmp-sstatesamehash2"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800515TCLIBCAPPEND = ""
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500516BB_NUMBER_THREADS = "${@oe.utils.cpu_count()+1}"
517PARALLEL_MAKE = "-j 2"
518DL_DIR = "${TOPDIR}/download2"
519TIME = "222222"
520DATE = "20161212"
521# Always remove uninative as we're changing proxies
Patrick Williams213cb262021-08-07 19:21:33 -0500522INHERIT:remove = "uninative"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500523INHERIT += "buildstats-summary buildhistory"
524http_proxy = "http://example.com/"
Brad Bishop6dbb3162019-11-25 09:41:34 -0500525BB_SIGNATURE_HANDLER = "OEBasicHash"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500526""")
527 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2")
528 self.track_for_cleanup(self.topdir + "/download2")
529 bitbake("world meta-toolchain -S none")
530
531 def get_files(d):
532 f = {}
533 for root, dirs, files in os.walk(d):
534 for name in files:
535 name, shash = name.rsplit('.', 1)
536 # Extract just the machine and recipe name
537 base = os.sep.join(root.rsplit(os.sep, 2)[-2:] + [name])
538 f[base] = shash
539 return f
540
541 def compare_sigfiles(files, files1, files2, compare=False):
542 for k in files:
543 if k in files1 and k in files2:
544 print("%s differs:" % k)
545 if compare:
546 sigdatafile1 = self.topdir + "/tmp-sstatesamehash/stamps/" + k + "." + files1[k]
547 sigdatafile2 = self.topdir + "/tmp-sstatesamehash2/stamps/" + k + "." + files2[k]
548 output = bb.siggen.compare_sigfiles(sigdatafile1, sigdatafile2)
549 if output:
550 print('\n'.join(output))
551 elif k in files1 and k not in files2:
552 print("%s in files1" % k)
553 elif k not in files1 and k in files2:
554 print("%s in files2" % k)
555 else:
556 assert "shouldn't reach here"
557
558 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/")
559 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/")
560 # Remove items that are identical in both sets
561 for k,v in files1.items() & files2.items():
562 del files1[k]
563 del files2[k]
564 if not files1 and not files2:
565 # No changes, so we're done
566 return
567
568 files = list(files1.keys() | files2.keys())
569 # this is an expensive computation, thus just compare the first 'max_sigfiles_to_compare' k files
570 max_sigfiles_to_compare = 20
571 first, rest = files[:max_sigfiles_to_compare], files[max_sigfiles_to_compare:]
Brad Bishop316dfdd2018-06-25 12:45:53 -0400572 compare_sigfiles(first, files1, files2, compare=True)
573 compare_sigfiles(rest, files1, files2, compare=False)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500574
575 self.fail("sstate hashes not identical.")