blob: 0af06e46e503ccb4348cad31413548702ccdc56b [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# BitBake Tests for the Fetcher (fetch2/)
3#
4# Copyright (C) 2012 Richard Purdie
5#
Brad Bishopc342db32019-05-15 21:57:59 -04006# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007#
8
9import unittest
Brad Bishop316dfdd2018-06-25 12:45:53 -040010import hashlib
Patrick Williamsc124f4f2015-09-15 14:41:29 -050011import tempfile
Patrick Williamsc0f7c042017-02-23 20:41:17 -060012import collections
Patrick Williamsc124f4f2015-09-15 14:41:29 -050013import os
Patrick Williams92b42cb2022-09-03 06:53:57 -050014import signal
Andrew Geissler9aee5002022-03-30 16:27:02 +000015import tarfile
Patrick Williamsc124f4f2015-09-15 14:41:29 -050016from bb.fetch2 import URI
17from bb.fetch2 import FetchMethod
18import bb
Andrew Geissler82c905d2020-04-13 13:39:40 -050019from bb.tests.support.httpserver import HTTPService
Patrick Williamsc124f4f2015-09-15 14:41:29 -050020
Brad Bishopd7bf8c12018-02-25 22:55:05 -050021def skipIfNoNetwork():
22 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
Andrew Geissler7e0e3c02022-02-25 20:34:39 +000023 return unittest.skip("network test")
Brad Bishopd7bf8c12018-02-25 22:55:05 -050024 return lambda f: f
25
Patrick Williams92b42cb2022-09-03 06:53:57 -050026class TestTimeout(Exception):
27 pass
28
29class Timeout():
30
31 def __init__(self, seconds):
32 self.seconds = seconds
33
34 def handle_timeout(self, signum, frame):
35 raise TestTimeout("Test failed: timeout reached")
36
37 def __enter__(self):
38 signal.signal(signal.SIGALRM, self.handle_timeout)
39 signal.alarm(self.seconds)
40
41 def __exit__(self, exc_type, exc_val, exc_tb):
42 signal.alarm(0)
43
Patrick Williamsc124f4f2015-09-15 14:41:29 -050044class URITest(unittest.TestCase):
45 test_uris = {
46 "http://www.google.com/index.html" : {
47 'uri': 'http://www.google.com/index.html',
48 'scheme': 'http',
49 'hostname': 'www.google.com',
50 'port': None,
51 'hostport': 'www.google.com',
52 'path': '/index.html',
53 'userinfo': '',
54 'username': '',
55 'password': '',
56 'params': {},
57 'query': {},
58 'relative': False
59 },
60 "http://www.google.com/index.html;param1=value1" : {
61 'uri': 'http://www.google.com/index.html;param1=value1',
62 'scheme': 'http',
63 'hostname': 'www.google.com',
64 'port': None,
65 'hostport': 'www.google.com',
66 'path': '/index.html',
67 'userinfo': '',
68 'username': '',
69 'password': '',
70 'params': {
71 'param1': 'value1'
72 },
73 'query': {},
74 'relative': False
75 },
76 "http://www.example.org/index.html?param1=value1" : {
77 'uri': 'http://www.example.org/index.html?param1=value1',
78 'scheme': 'http',
79 'hostname': 'www.example.org',
80 'port': None,
81 'hostport': 'www.example.org',
82 'path': '/index.html',
83 'userinfo': '',
84 'username': '',
85 'password': '',
86 'params': {},
87 'query': {
88 'param1': 'value1'
89 },
90 'relative': False
91 },
92 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : {
93 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2',
94 'scheme': 'http',
95 'hostname': 'www.example.org',
96 'port': None,
97 'hostport': 'www.example.org',
98 'path': '/index.html',
99 'userinfo': '',
100 'username': '',
101 'password': '',
102 'params': {
103 'param2': 'value2'
104 },
105 'query': {
106 'qparam1': 'qvalue1'
107 },
108 'relative': False
109 },
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600110 # Check that trailing semicolons are handled correctly
111 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2;" : {
112 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2',
113 'scheme': 'http',
114 'hostname': 'www.example.org',
115 'port': None,
116 'hostport': 'www.example.org',
117 'path': '/index.html',
118 'userinfo': '',
119 'username': '',
120 'password': '',
121 'params': {
122 'param2': 'value2'
123 },
124 'query': {
125 'qparam1': 'qvalue1'
126 },
127 'relative': False
128 },
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500129 "http://www.example.com:8080/index.html" : {
130 'uri': 'http://www.example.com:8080/index.html',
131 'scheme': 'http',
132 'hostname': 'www.example.com',
133 'port': 8080,
134 'hostport': 'www.example.com:8080',
135 'path': '/index.html',
136 'userinfo': '',
137 'username': '',
138 'password': '',
139 'params': {},
140 'query': {},
141 'relative': False
142 },
143 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : {
144 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg',
145 'scheme': 'cvs',
146 'hostname': 'cvs.handhelds.org',
147 'port': None,
148 'hostport': 'cvs.handhelds.org',
149 'path': '/cvs',
150 'userinfo': 'anoncvs',
151 'username': 'anoncvs',
152 'password': '',
153 'params': {
154 'module': 'familiar/dist/ipkg'
155 },
156 'query': {},
157 'relative': False
158 },
159 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": {
160 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg',
161 'scheme': 'cvs',
162 'hostname': 'cvs.handhelds.org',
163 'port': None,
164 'hostport': 'cvs.handhelds.org',
165 'path': '/cvs',
166 'userinfo': 'anoncvs:anonymous',
167 'username': 'anoncvs',
168 'password': 'anonymous',
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600169 'params': collections.OrderedDict([
170 ('tag', 'V0-99-81'),
171 ('module', 'familiar/dist/ipkg')
172 ]),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500173 'query': {},
174 'relative': False
175 },
176 "file://example.diff": { # NOTE: Not RFC compliant!
177 'uri': 'file:example.diff',
178 'scheme': 'file',
179 'hostname': '',
180 'port': None,
181 'hostport': '',
182 'path': 'example.diff',
183 'userinfo': '',
184 'username': '',
185 'password': '',
186 'params': {},
187 'query': {},
188 'relative': True
189 },
190 "file:example.diff": { # NOTE: RFC compliant version of the former
191 'uri': 'file:example.diff',
192 'scheme': 'file',
193 'hostname': '',
194 'port': None,
195 'hostport': '',
196 'path': 'example.diff',
197 'userinfo': '',
198 'userinfo': '',
199 'username': '',
200 'password': '',
201 'params': {},
202 'query': {},
203 'relative': True
204 },
205 "file:///tmp/example.diff": {
206 'uri': 'file:///tmp/example.diff',
207 'scheme': 'file',
208 'hostname': '',
209 'port': None,
210 'hostport': '',
211 'path': '/tmp/example.diff',
212 'userinfo': '',
213 'userinfo': '',
214 'username': '',
215 'password': '',
216 'params': {},
217 'query': {},
218 'relative': False
219 },
220 "git:///path/example.git": {
221 'uri': 'git:///path/example.git',
222 'scheme': 'git',
223 'hostname': '',
224 'port': None,
225 'hostport': '',
226 'path': '/path/example.git',
227 'userinfo': '',
228 'userinfo': '',
229 'username': '',
230 'password': '',
231 'params': {},
232 'query': {},
233 'relative': False
234 },
235 "git:path/example.git": {
236 'uri': 'git:path/example.git',
237 'scheme': 'git',
238 'hostname': '',
239 'port': None,
240 'hostport': '',
241 'path': 'path/example.git',
242 'userinfo': '',
243 'userinfo': '',
244 'username': '',
245 'password': '',
246 'params': {},
247 'query': {},
248 'relative': True
249 },
250 "git://example.net/path/example.git": {
251 'uri': 'git://example.net/path/example.git',
252 'scheme': 'git',
253 'hostname': 'example.net',
254 'port': None,
255 'hostport': 'example.net',
256 'path': '/path/example.git',
257 'userinfo': '',
258 'userinfo': '',
259 'username': '',
260 'password': '',
261 'params': {},
262 'query': {},
263 'relative': False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500264 },
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500265 "git://tfs-example.org:22/tfs/example%20path/example.git": {
266 'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git',
267 'scheme': 'git',
268 'hostname': 'tfs-example.org',
269 'port': 22,
270 'hostport': 'tfs-example.org:22',
271 'path': '/tfs/example path/example.git',
272 'userinfo': '',
273 'userinfo': '',
274 'username': '',
275 'password': '',
276 'params': {},
277 'query': {},
278 'relative': False
279 },
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500280 "http://somesite.net;someparam=1": {
281 'uri': 'http://somesite.net;someparam=1',
282 'scheme': 'http',
283 'hostname': 'somesite.net',
284 'port': None,
285 'hostport': 'somesite.net',
286 'path': '',
287 'userinfo': '',
288 'userinfo': '',
289 'username': '',
290 'password': '',
291 'params': {"someparam" : "1"},
292 'query': {},
293 'relative': False
294 },
295 "file://somelocation;someparam=1": {
296 'uri': 'file:somelocation;someparam=1',
297 'scheme': 'file',
298 'hostname': '',
299 'port': None,
300 'hostport': '',
301 'path': 'somelocation',
302 'userinfo': '',
303 'userinfo': '',
304 'username': '',
305 'password': '',
306 'params': {"someparam" : "1"},
307 'query': {},
308 'relative': True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500309 }
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500310
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500311 }
312
313 def test_uri(self):
314 for test_uri, ref in self.test_uris.items():
315 uri = URI(test_uri)
316
317 self.assertEqual(str(uri), ref['uri'])
318
319 # expected attributes
320 self.assertEqual(uri.scheme, ref['scheme'])
321
322 self.assertEqual(uri.userinfo, ref['userinfo'])
323 self.assertEqual(uri.username, ref['username'])
324 self.assertEqual(uri.password, ref['password'])
325
326 self.assertEqual(uri.hostname, ref['hostname'])
327 self.assertEqual(uri.port, ref['port'])
328 self.assertEqual(uri.hostport, ref['hostport'])
329
330 self.assertEqual(uri.path, ref['path'])
331 self.assertEqual(uri.params, ref['params'])
332
333 self.assertEqual(uri.relative, ref['relative'])
334
335 def test_dict(self):
336 for test in self.test_uris.values():
337 uri = URI()
338
339 self.assertEqual(uri.scheme, '')
340 self.assertEqual(uri.userinfo, '')
341 self.assertEqual(uri.username, '')
342 self.assertEqual(uri.password, '')
343 self.assertEqual(uri.hostname, '')
344 self.assertEqual(uri.port, None)
345 self.assertEqual(uri.path, '')
346 self.assertEqual(uri.params, {})
347
348
349 uri.scheme = test['scheme']
350 self.assertEqual(uri.scheme, test['scheme'])
351
352 uri.userinfo = test['userinfo']
353 self.assertEqual(uri.userinfo, test['userinfo'])
354 self.assertEqual(uri.username, test['username'])
355 self.assertEqual(uri.password, test['password'])
356
357 # make sure changing the values doesn't do anything unexpected
358 uri.username = 'changeme'
359 self.assertEqual(uri.username, 'changeme')
360 self.assertEqual(uri.password, test['password'])
361 uri.password = 'insecure'
362 self.assertEqual(uri.username, 'changeme')
363 self.assertEqual(uri.password, 'insecure')
364
365 # reset back after our trickery
366 uri.userinfo = test['userinfo']
367 self.assertEqual(uri.userinfo, test['userinfo'])
368 self.assertEqual(uri.username, test['username'])
369 self.assertEqual(uri.password, test['password'])
370
371 uri.hostname = test['hostname']
372 self.assertEqual(uri.hostname, test['hostname'])
373 self.assertEqual(uri.hostport, test['hostname'])
374
375 uri.port = test['port']
376 self.assertEqual(uri.port, test['port'])
377 self.assertEqual(uri.hostport, test['hostport'])
378
379 uri.path = test['path']
380 self.assertEqual(uri.path, test['path'])
381
382 uri.params = test['params']
383 self.assertEqual(uri.params, test['params'])
384
385 uri.query = test['query']
386 self.assertEqual(uri.query, test['query'])
387
388 self.assertEqual(str(uri), test['uri'])
389
390 uri.params = {}
391 self.assertEqual(uri.params, {})
392 self.assertEqual(str(uri), (str(uri).split(";"))[0])
393
394class FetcherTest(unittest.TestCase):
395
396 def setUp(self):
397 self.origdir = os.getcwd()
398 self.d = bb.data.init()
Andrew Geisslereff27472021-10-29 15:35:00 -0500399 self.tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500400 self.dldir = os.path.join(self.tempdir, "download")
401 os.mkdir(self.dldir)
402 self.d.setVar("DL_DIR", self.dldir)
403 self.unpackdir = os.path.join(self.tempdir, "unpacked")
404 os.mkdir(self.unpackdir)
405 persistdir = os.path.join(self.tempdir, "persistdata")
406 self.d.setVar("PERSISTENT_DIR", persistdir)
407
408 def tearDown(self):
409 os.chdir(self.origdir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600410 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
411 print("Not cleaning up %s. Please remove manually." % self.tempdir)
412 else:
Andrew Geisslerc926e172021-05-07 16:11:35 -0500413 bb.process.run('chmod u+rw -R %s' % self.tempdir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600414 bb.utils.prunedir(self.tempdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500415
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000416 def git(self, cmd, cwd=None):
417 if isinstance(cmd, str):
418 cmd = 'git ' + cmd
419 else:
420 cmd = ['git'] + cmd
421 if cwd is None:
422 cwd = self.gitdir
423 return bb.process.run(cmd, cwd=cwd)[0]
424
425 def git_init(self, cwd=None):
426 self.git('init', cwd=cwd)
427 if not self.git(['config', 'user.email'], cwd=cwd):
428 self.git(['config', 'user.email', 'you@example.com'], cwd=cwd)
429 if not self.git(['config', 'user.name'], cwd=cwd):
430 self.git(['config', 'user.name', 'Your Name'], cwd=cwd)
431
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500432class MirrorUriTest(FetcherTest):
433
434 replaceuris = {
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000435 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500436 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000437 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
438 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
439 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
440 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
441 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http")
442 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500443 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake")
444 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890",
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000445 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500446 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000447 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500448 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000449 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500450 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000451 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500452 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
453 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist")
454 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2",
455 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/")
456 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2",
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000457 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
458 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
459 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
460 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
461 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http")
462 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800463 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org")
464 : "http://somewhere2.org/somefile_1.2.3.tar.gz",
465 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/")
466 : "http://somewhere2.org/somefile_1.2.3.tar.gz",
467 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http")
468 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
Andrew Geissler595f6302022-01-24 19:11:47 +0000469 ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http")
470 : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
471 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=git;branch=master", "git://someserver.org/bitbake", "git://someotherserver.org/bitbake;protocol=https")
472 : "git://someotherserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=https;branch=master",
Andrew Geissler595f6302022-01-24 19:11:47 +0000473 ("gitsm://git.qemu.org/git/seabios.git/;protocol=https;name=roms/seabios;subpath=roms/seabios;bareclone=1;nobranch=1;rev=1234567890123456789012345678901234567890", "gitsm://.*/.*", "http://petalinux.xilinx.com/sswreleases/rel-v${XILINX_VER_MAIN}/downloads") : "http://petalinux.xilinx.com/sswreleases/rel-v%24%7BXILINX_VER_MAIN%7D/downloads/git2_git.qemu.org.git.seabios.git..tar.gz",
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000474 ("https://somewhere.org/example/1.0.0/example;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/PATH")
475 : "file:///mirror/example/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz",
476 ("https://somewhere.org/example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/some-example-1.0.0.tgz")
477 : "file:///mirror/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz",
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500478
479 #Renaming files doesn't work
480 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
481 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
482 }
483
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000484 mirrorvar = "http://.*/.* file:///somepath/downloads/ " \
485 "git://someserver.org/bitbake git://git.openembedded.org/bitbake " \
486 "https://.*/.* file:///someotherpath/downloads/ " \
487 "http://.*/.* file:///someotherpath/downloads/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500488
489 def test_urireplace(self):
Andrew Geissler615f2f12022-07-15 14:00:58 -0500490 self.d.setVar("FILESPATH", ".")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500491 for k, v in self.replaceuris.items():
492 ud = bb.fetch.FetchData(k[0], self.d)
493 ud.setup_localpath(self.d)
494 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2]))
495 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d)
496 self.assertEqual([v], newuris)
497
498 def test_urilist1(self):
499 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
500 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
501 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
502 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
503
504 def test_urilist2(self):
505 # Catch https:// -> files:// bug
506 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
507 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
508 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
509 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
510
511 def test_mirror_of_mirror(self):
512 # Test if mirror of a mirror works
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000513 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/"
514 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500515 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
516 mirrors = bb.fetch2.mirror_from_string(mirrorvar)
517 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
518 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz',
519 'file:///someotherpath/downloads/bitbake-1.0.tar.gz',
520 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz',
521 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz'])
522
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000523 recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ " \
524 "https://.*/[^/]* https://BBBB/B/B/B/"
Patrick Williamsd7e96312015-09-22 08:09:05 -0500525
526 def test_recursive(self):
527 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
528 mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar)
529 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
530 self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz',
531 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz',
532 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500533
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800534
535class GitDownloadDirectoryNamingTest(FetcherTest):
536 def setUp(self):
537 super(GitDownloadDirectoryNamingTest, self).setUp()
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000538 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800539 self.recipe_dir = "git.openembedded.org.bitbake"
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000540 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800541 self.mirror_dir = "github.com.openembedded.bitbake.git"
542
543 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
544
545 def setup_mirror_rewrite(self):
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000546 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800547
548 @skipIfNoNetwork()
549 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self):
550 self.setup_mirror_rewrite()
551 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
552
553 fetcher.download()
554
555 dir = os.listdir(self.dldir + "/git2")
556 self.assertIn(self.recipe_dir, dir)
557
558 @skipIfNoNetwork()
559 def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self):
560 self.setup_mirror_rewrite()
561 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
562
563 fetcher.download()
564
565 dir = os.listdir(self.dldir + "/git2")
566 self.assertIn(self.mirror_dir, dir)
567 self.assertIn(self.recipe_dir, dir)
568
569 @skipIfNoNetwork()
570 def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self):
571 self.setup_mirror_rewrite()
572 fetcher = bb.fetch.Fetch([self.mirror_url], self.d)
573 fetcher.download()
574 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
575
576 fetcher.download()
577
578 dir = os.listdir(self.dldir + "/git2")
579 self.assertIn(self.mirror_dir, dir)
580 self.assertIn(self.recipe_dir, dir)
581
582
583class TarballNamingTest(FetcherTest):
584 def setUp(self):
585 super(TarballNamingTest, self).setUp()
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000586 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800587 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000588 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800589 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz"
590
591 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
592 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
593
594 def setup_mirror_rewrite(self):
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000595 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800596
597 @skipIfNoNetwork()
598 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self):
599 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
600
601 fetcher.download()
602
603 dir = os.listdir(self.dldir)
604 self.assertIn(self.recipe_tarball, dir)
605
606 @skipIfNoNetwork()
607 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
608 self.setup_mirror_rewrite()
609 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
610
611 fetcher.download()
612
613 dir = os.listdir(self.dldir)
614 self.assertIn(self.mirror_tarball, dir)
615
616
617class GitShallowTarballNamingTest(FetcherTest):
618 def setUp(self):
619 super(GitShallowTarballNamingTest, self).setUp()
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000620 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800621 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz"
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000622 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800623 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz"
624
625 self.d.setVar('BB_GIT_SHALLOW', '1')
626 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
627 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
628
629 def setup_mirror_rewrite(self):
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000630 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800631
632 @skipIfNoNetwork()
633 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self):
634 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
635
636 fetcher.download()
637
638 dir = os.listdir(self.dldir)
639 self.assertIn(self.recipe_tarball, dir)
640
641 @skipIfNoNetwork()
642 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
643 self.setup_mirror_rewrite()
644 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
645
646 fetcher.download()
647
648 dir = os.listdir(self.dldir)
649 self.assertIn(self.mirror_tarball, dir)
650
651
Andrew Geissler9aee5002022-03-30 16:27:02 +0000652class CleanTarballTest(FetcherTest):
653 def setUp(self):
654 super(CleanTarballTest, self).setUp()
655 self.recipe_url = "git://git.openembedded.org/bitbake"
656 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
657
658 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
659 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
660
661 @skipIfNoNetwork()
662 def test_that_the_tarball_contents_does_not_leak_info(self):
663 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
664
665 fetcher.download()
666
667 fetcher.unpack(self.unpackdir)
668 mtime = bb.process.run('git log --all -1 --format=%ct',
669 cwd=os.path.join(self.unpackdir, 'git'))
670 self.assertEqual(len(mtime), 2)
671 mtime = int(mtime[0])
672
673 archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball))
674 self.assertNotEqual(len(archive.members), 0)
675 for member in archive.members:
Patrick Williams03907ee2022-05-01 06:28:52 -0500676 self.assertEqual(member.uname, 'oe')
677 self.assertEqual(member.uid, 0)
678 self.assertEqual(member.gname, 'oe')
679 self.assertEqual(member.gid, 0)
Andrew Geissler9aee5002022-03-30 16:27:02 +0000680 self.assertEqual(member.mtime, mtime)
681
682
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500683class FetcherLocalTest(FetcherTest):
684 def setUp(self):
685 def touch(fn):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600686 with open(fn, 'a'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500687 os.utime(fn, None)
688
689 super(FetcherLocalTest, self).setUp()
690 self.localsrcdir = os.path.join(self.tempdir, 'localsrc')
691 os.makedirs(self.localsrcdir)
692 touch(os.path.join(self.localsrcdir, 'a'))
693 touch(os.path.join(self.localsrcdir, 'b'))
694 os.makedirs(os.path.join(self.localsrcdir, 'dir'))
695 touch(os.path.join(self.localsrcdir, 'dir', 'c'))
696 touch(os.path.join(self.localsrcdir, 'dir', 'd'))
697 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir'))
698 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e'))
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500699 touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device'))
Andrew Geissler595f6302022-01-24 19:11:47 +0000700 bb.process.run('tar cf archive.tar -C dir .', cwd=self.localsrcdir)
701 bb.process.run('tar czf archive.tar.gz -C dir .', cwd=self.localsrcdir)
702 bb.process.run('tar cjf archive.tar.bz2 -C dir .', cwd=self.localsrcdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500703 self.d.setVar("FILESPATH", self.localsrcdir)
704
705 def fetchUnpack(self, uris):
706 fetcher = bb.fetch.Fetch(uris, self.d)
707 fetcher.download()
708 fetcher.unpack(self.unpackdir)
709 flst = []
710 for root, dirs, files in os.walk(self.unpackdir):
711 for f in files:
712 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir))
713 flst.sort()
714 return flst
715
Andrew Geissler615f2f12022-07-15 14:00:58 -0500716 def test_local_checksum_fails_no_file(self):
717 self.d.setVar("SRC_URI", "file://404")
718 with self.assertRaises(bb.BBHandledException):
719 bb.fetch.get_checksum_file_list(self.d)
720
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500721 def test_local(self):
722 tree = self.fetchUnpack(['file://a', 'file://dir/c'])
723 self.assertEqual(tree, ['a', 'dir/c'])
724
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500725 def test_local_backslash(self):
726 tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device'])
727 self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device'])
728
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500729 def test_local_wildcard(self):
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500730 with self.assertRaises(bb.fetch2.ParameterError):
731 tree = self.fetchUnpack(['file://a', 'file://dir/*'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500732
733 def test_local_dir(self):
734 tree = self.fetchUnpack(['file://a', 'file://dir'])
735 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e'])
736
737 def test_local_subdir(self):
738 tree = self.fetchUnpack(['file://dir/subdir'])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500739 self.assertEqual(tree, ['dir/subdir/e'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500740
741 def test_local_subdir_file(self):
742 tree = self.fetchUnpack(['file://dir/subdir/e'])
743 self.assertEqual(tree, ['dir/subdir/e'])
744
745 def test_local_subdirparam(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500746 tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo'])
747 self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500748
749 def test_local_deepsubdirparam(self):
750 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar'])
751 self.assertEqual(tree, ['bar/dir/subdir/e'])
752
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600753 def test_local_absolutedir(self):
754 # Unpacking to an absolute path that is a subdirectory of the root
755 # should work
756 tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')])
757
758 # Unpacking to an absolute path outside of the root should fail
759 with self.assertRaises(bb.fetch2.UnpackError):
760 self.fetchUnpack(['file://a;subdir=/bin/sh'])
761
Andrew Geissler595f6302022-01-24 19:11:47 +0000762 def test_local_striplevel(self):
763 tree = self.fetchUnpack(['file://archive.tar;subdir=bar;striplevel=1'])
764 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e'])
765
766 def test_local_striplevel_gzip(self):
767 tree = self.fetchUnpack(['file://archive.tar.gz;subdir=bar;striplevel=1'])
768 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e'])
769
770 def test_local_striplevel_bzip2(self):
771 tree = self.fetchUnpack(['file://archive.tar.bz2;subdir=bar;striplevel=1'])
772 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e'])
773
Andrew Geisslerc926e172021-05-07 16:11:35 -0500774 def dummyGitTest(self, suffix):
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600775 # Create dummy local Git repo
776 src_dir = tempfile.mkdtemp(dir=self.tempdir,
777 prefix='gitfetch_localusehead_')
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000778 self.gitdir = os.path.abspath(src_dir)
779 self.git_init()
780 self.git(['commit', '--allow-empty', '-m', 'Dummy commit'])
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600781 # Use other branch than master
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000782 self.git(['checkout', '-b', 'my-devel'])
783 self.git(['commit', '--allow-empty', '-m', 'Dummy commit 2'])
784 orig_rev = self.git(['rev-parse', 'HEAD']).strip()
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600785
786 # Fetch and check revision
787 self.d.setVar("SRCREV", "AUTOINC")
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000788 self.d.setVar("__BBSEENSRCREV", "1")
789 url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600790 fetcher = bb.fetch.Fetch([url], self.d)
791 fetcher.download()
792 fetcher.unpack(self.unpackdir)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000793 unpack_rev = self.git(['rev-parse', 'HEAD'],
794 cwd=os.path.join(self.unpackdir, 'git')).strip()
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600795 self.assertEqual(orig_rev, unpack_rev)
796
Andrew Geisslerc926e172021-05-07 16:11:35 -0500797 def test_local_gitfetch_usehead(self):
798 self.dummyGitTest("usehead=1")
799
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600800 def test_local_gitfetch_usehead_withname(self):
Andrew Geisslerc926e172021-05-07 16:11:35 -0500801 self.dummyGitTest("usehead=1;name=newName")
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600802
Andrew Geisslerc926e172021-05-07 16:11:35 -0500803 def test_local_gitfetch_shared(self):
804 self.dummyGitTest("usehead=1;name=sharedName")
805 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
806 self.assertTrue(os.path.exists(alt))
807
808 def test_local_gitfetch_noshared(self):
809 self.d.setVar('BB_GIT_NOSHARED', '1')
810 self.unpackdir += '_noshared'
811 self.dummyGitTest("usehead=1;name=noSharedName")
812 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
813 self.assertFalse(os.path.exists(alt))
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600814
Brad Bishop316dfdd2018-06-25 12:45:53 -0400815class FetcherNoNetworkTest(FetcherTest):
816 def setUp(self):
817 super().setUp()
818 # all test cases are based on not having network
819 self.d.setVar("BB_NO_NETWORK", "1")
820
821 def test_missing(self):
822 string = "this is a test file\n".encode("utf-8")
823 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
824 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
825
826 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
827 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
828 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
829 with self.assertRaises(bb.fetch2.NetworkAccess):
830 fetcher.download()
831
832 def test_valid_missing_donestamp(self):
833 # create the file in the download directory with correct hash
834 string = "this is a test file\n".encode("utf-8")
835 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f:
836 f.write(string)
837
838 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
839 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
840
841 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
842 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
843 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
844 fetcher.download()
845 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
846
847 def test_invalid_missing_donestamp(self):
848 # create an invalid file in the download directory with incorrect hash
849 string = "this is a test file\n".encode("utf-8")
850 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
851 pass
852
853 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
854 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
855
856 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
857 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
858 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
859 with self.assertRaises(bb.fetch2.NetworkAccess):
860 fetcher.download()
861 # the existing file should not exist or should have be moved to "bad-checksum"
862 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
863
864 def test_nochecksums_missing(self):
865 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
866 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
867 # ssh fetch does not support checksums
868 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
869 # attempts to download with missing donestamp
870 with self.assertRaises(bb.fetch2.NetworkAccess):
871 fetcher.download()
872
873 def test_nochecksums_missing_donestamp(self):
874 # create a file in the download directory
875 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
876 pass
877
878 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
879 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
880 # ssh fetch does not support checksums
881 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
882 # attempts to download with missing donestamp
883 with self.assertRaises(bb.fetch2.NetworkAccess):
884 fetcher.download()
885
886 def test_nochecksums_has_donestamp(self):
887 # create a file in the download directory with the donestamp
888 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
889 pass
890 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
891 pass
892
893 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
894 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
895 # ssh fetch does not support checksums
896 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
897 # should not fetch
898 fetcher.download()
899 # both files should still exist
900 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
901 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
902
903 def test_nochecksums_missing_has_donestamp(self):
904 # create a file in the download directory with the donestamp
905 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
906 pass
907
908 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
909 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
910 # ssh fetch does not support checksums
911 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
912 with self.assertRaises(bb.fetch2.NetworkAccess):
913 fetcher.download()
914 # both files should still exist
915 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
916 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
917
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500918class FetcherNetworkTest(FetcherTest):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500919 @skipIfNoNetwork()
920 def test_fetch(self):
Andrew Geisslereff27472021-10-29 15:35:00 -0500921 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500922 fetcher.download()
923 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
924 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892)
925 self.d.setVar("BB_NO_NETWORK", "1")
Andrew Geisslereff27472021-10-29 15:35:00 -0500926 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500927 fetcher.download()
928 fetcher.unpack(self.unpackdir)
929 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9)
930 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500931
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500932 @skipIfNoNetwork()
933 def test_fetch_mirror(self):
Andrew Geisslereff27472021-10-29 15:35:00 -0500934 self.d.setVar("MIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500935 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
936 fetcher.download()
937 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
938
939 @skipIfNoNetwork()
940 def test_fetch_mirror_of_mirror(self):
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000941 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500942 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
943 fetcher.download()
944 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
945
946 @skipIfNoNetwork()
947 def test_fetch_file_mirror_of_mirror(self):
Andrew Geissler615f2f12022-07-15 14:00:58 -0500948 self.d.setVar("FILESPATH", ".")
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000949 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500950 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
951 os.mkdir(self.dldir + "/some2where")
952 fetcher.download()
953 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
954
955 @skipIfNoNetwork()
956 def test_fetch_premirror(self):
Andrew Geisslereff27472021-10-29 15:35:00 -0500957 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500958 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
959 fetcher.download()
960 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
961
962 @skipIfNoNetwork()
Andrew Geissler5199d832021-09-24 16:47:35 -0500963 def test_fetch_specify_downloadfilename(self):
Andrew Geisslereff27472021-10-29 15:35:00 -0500964 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d)
Andrew Geissler5199d832021-09-24 16:47:35 -0500965 fetcher.download()
966 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749)
967
968 @skipIfNoNetwork()
969 def test_fetch_premirror_specify_downloadfilename_regex_uri(self):
Andrew Geisslereff27472021-10-29 15:35:00 -0500970 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/")
Andrew Geissler595f6302022-01-24 19:11:47 +0000971 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d)
Andrew Geissler5199d832021-09-24 16:47:35 -0500972 fetcher.download()
Andrew Geissler595f6302022-01-24 19:11:47 +0000973 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
Andrew Geissler5199d832021-09-24 16:47:35 -0500974
975 @skipIfNoNetwork()
976 # BZ13039
977 def test_fetch_premirror_specify_downloadfilename_specific_uri(self):
Andrew Geisslereff27472021-10-29 15:35:00 -0500978 self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake")
Andrew Geissler595f6302022-01-24 19:11:47 +0000979 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d)
Andrew Geissler5199d832021-09-24 16:47:35 -0500980 fetcher.download()
Andrew Geissler595f6302022-01-24 19:11:47 +0000981 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
982
983 @skipIfNoNetwork()
984 def test_fetch_premirror_use_downloadfilename_to_fetch(self):
985 # Ensure downloadfilename is used when fetching from premirror.
986 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
987 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d)
988 fetcher.download()
989 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
Andrew Geissler5199d832021-09-24 16:47:35 -0500990
991 @skipIfNoNetwork()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500992 def gitfetcher(self, url1, url2):
993 def checkrevision(self, fetcher):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500994 fetcher.unpack(self.unpackdir)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +0000995 revision = self.git(['rev-parse', 'HEAD'],
996 cwd=os.path.join(self.unpackdir, 'git')).strip()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500997 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500998
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500999 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
1000 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
1001 fetcher = bb.fetch.Fetch([url1], self.d)
1002 fetcher.download()
1003 checkrevision(self, fetcher)
1004 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works
1005 bb.utils.prunedir(self.dldir + "/git2/")
1006 bb.utils.prunedir(self.unpackdir)
1007 self.d.setVar("BB_NO_NETWORK", "1")
1008 fetcher = bb.fetch.Fetch([url2], self.d)
1009 fetcher.download()
1010 checkrevision(self, fetcher)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001011
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001012 @skipIfNoNetwork()
1013 def test_gitfetch(self):
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001014 url1 = url2 = "git://git.openembedded.org/bitbake;branch=master"
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001015 self.gitfetcher(url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001016
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001017 @skipIfNoNetwork()
1018 def test_gitfetch_goodsrcrev(self):
1019 # SRCREV is set but matches rev= parameter
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001020 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master"
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001021 self.gitfetcher(url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001022
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001023 @skipIfNoNetwork()
1024 def test_gitfetch_badsrcrev(self):
1025 # SRCREV is set but does not match rev= parameter
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001026 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master"
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001027 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001028
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001029 @skipIfNoNetwork()
1030 def test_gitfetch_tagandrev(self):
1031 # SRCREV is set but does not match rev= parameter
1032 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
1033 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001034
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001035 @skipIfNoNetwork()
Andrew Geisslerd1e89492021-02-12 15:35:20 -06001036 def test_gitfetch_usehead(self):
1037 # Since self.gitfetcher() sets SRCREV we expect this to override
1038 # `usehead=1' and instead fetch the specified SRCREV. See
1039 # test_local_gitfetch_usehead() for a positive use of the usehead
1040 # feature.
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001041 url = "git://git.openembedded.org/bitbake;usehead=1;branch=master"
Andrew Geisslerd1e89492021-02-12 15:35:20 -06001042 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001043
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001044 @skipIfNoNetwork()
Andrew Geisslerd1e89492021-02-12 15:35:20 -06001045 def test_gitfetch_usehead_withname(self):
1046 # Since self.gitfetcher() sets SRCREV we expect this to override
1047 # `usehead=1' and instead fetch the specified SRCREV. See
1048 # test_local_gitfetch_usehead() for a positive use of the usehead
1049 # feature.
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001050 url = "git://git.openembedded.org/bitbake;usehead=1;name=newName;branch=master"
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001051 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001052
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001053 @skipIfNoNetwork()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001054 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self):
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001055 recipeurl = "git://git.openembedded.org/bitbake;branch=master"
1056 mirrorurl = "git://someserver.org/bitbake;branch=master"
1057 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001058 self.gitfetcher(recipeurl, mirrorurl)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001059
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001060 @skipIfNoNetwork()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001061 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self):
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001062 recipeurl = "git://someserver.org/bitbake;branch=master"
1063 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001064 self.gitfetcher(recipeurl, recipeurl)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001065
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001066 @skipIfNoNetwork()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001067 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001068 realurl = "git://git.openembedded.org/bitbake"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001069 recipeurl = "git://someserver.org/bitbake"
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001070 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git")
1071 os.chdir(self.tempdir)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001072 self.git(['clone', realurl, self.sourcedir], cwd=self.tempdir)
1073 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipeurl, self.sourcedir))
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001074 self.gitfetcher(recipeurl, recipeurl)
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001075
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001076 @skipIfNoNetwork()
1077 def test_git_submodule(self):
Brad Bishopf8caae32019-03-25 13:13:56 -04001078 # URL with ssh submodules
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001079 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master"
Brad Bishopf8caae32019-03-25 13:13:56 -04001080 # Original URL (comment this if you have ssh access to git.yoctoproject.org)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001081 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master"
Brad Bishopf8caae32019-03-25 13:13:56 -04001082 fetcher = bb.fetch.Fetch([url], self.d)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001083 fetcher.download()
1084 # Previous cwd has been deleted
1085 os.chdir(os.path.dirname(self.unpackdir))
1086 fetcher.unpack(self.unpackdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001087
Brad Bishopf8caae32019-03-25 13:13:56 -04001088 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1089 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing')
1090 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing')
1091 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present')
1092
1093 # Only when we're running the extended test with a submodule's submodule, can we check this.
1094 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
1095 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
1096
Brad Bishop96ff1982019-08-19 13:50:42 -04001097 @skipIfNoNetwork()
Brad Bishopf8caae32019-03-25 13:13:56 -04001098 def test_git_submodule_dbus_broker(self):
1099 # The following external repositories have show failures in fetch and unpack operations
1100 # We want to avoid regressions!
Andrew Geissler595f6302022-01-24 19:11:47 +00001101 url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main"
Brad Bishopf8caae32019-03-25 13:13:56 -04001102 fetcher = bb.fetch.Fetch([url], self.d)
1103 fetcher.download()
1104 # Previous cwd has been deleted
1105 os.chdir(os.path.dirname(self.unpackdir))
1106 fetcher.unpack(self.unpackdir)
1107
1108 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1109 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"')
1110 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"')
1111 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"')
1112 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"')
1113 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"')
1114
Brad Bishop96ff1982019-08-19 13:50:42 -04001115 @skipIfNoNetwork()
Brad Bishopf8caae32019-03-25 13:13:56 -04001116 def test_git_submodule_CLI11(self):
Andrew Geissler595f6302022-01-24 19:11:47 +00001117 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main"
Brad Bishopf8caae32019-03-25 13:13:56 -04001118 fetcher = bb.fetch.Fetch([url], self.d)
1119 fetcher.download()
1120 # Previous cwd has been deleted
1121 os.chdir(os.path.dirname(self.unpackdir))
1122 fetcher.unpack(self.unpackdir)
1123
1124 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1125 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
1126 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
1127 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
1128
Brad Bishop96ff1982019-08-19 13:50:42 -04001129 @skipIfNoNetwork()
Brad Bishop19323692019-04-05 15:28:33 -04001130 def test_git_submodule_update_CLI11(self):
1131 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
Andrew Geissler595f6302022-01-24 19:11:47 +00001132 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main"
Brad Bishop19323692019-04-05 15:28:33 -04001133 fetcher = bb.fetch.Fetch([url], self.d)
1134 fetcher.download()
1135
1136 # CLI11 that pulls in a newer nlohmann-json
Andrew Geissler595f6302022-01-24 19:11:47 +00001137 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main"
Brad Bishop19323692019-04-05 15:28:33 -04001138 fetcher = bb.fetch.Fetch([url], self.d)
1139 fetcher.download()
1140 # Previous cwd has been deleted
1141 os.chdir(os.path.dirname(self.unpackdir))
1142 fetcher.unpack(self.unpackdir)
1143
1144 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1145 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
1146 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
1147 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
1148
Brad Bishop96ff1982019-08-19 13:50:42 -04001149 @skipIfNoNetwork()
Brad Bishopf8caae32019-03-25 13:13:56 -04001150 def test_git_submodule_aktualizr(self):
Andrew Geissler595f6302022-01-24 19:11:47 +00001151 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
Brad Bishopf8caae32019-03-25 13:13:56 -04001152 fetcher = bb.fetch.Fetch([url], self.d)
1153 fetcher.download()
1154 # Previous cwd has been deleted
1155 os.chdir(os.path.dirname(self.unpackdir))
1156 fetcher.unpack(self.unpackdir)
1157
1158 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1159 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"')
1160 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"')
1161 self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly")
1162 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"')
1163 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"')
1164 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001165
Brad Bishop96ff1982019-08-19 13:50:42 -04001166 @skipIfNoNetwork()
Brad Bishop393846f2019-05-20 12:24:11 -04001167 def test_git_submodule_iotedge(self):
1168 """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """
1169
1170 # This repository also has submodules where the module (name), path and url do not align
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001171 url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699;branch=main"
Brad Bishop393846f2019-05-20 12:24:11 -04001172 fetcher = bb.fetch.Fetch([url], self.d)
1173 fetcher.download()
1174 # Previous cwd has been deleted
1175 os.chdir(os.path.dirname(self.unpackdir))
1176 fetcher.unpack(self.unpackdir)
1177
1178 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1179
1180 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/README.md')), msg='Missing submodule checkout')
1181 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/ctest/README.md')), msg='Missing submodule checkout')
1182 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/testrunner/readme.md')), msg='Missing submodule checkout')
1183 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/readme.md')), msg='Missing submodule checkout')
1184 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
1185 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
1186 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/README.md')), msg='Missing submodule checkout')
1187 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/README.md')), msg='Missing submodule checkout')
1188 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/ctest/README.md')), msg='Missing submodule checkout')
1189 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/testrunner/readme.md')), msg='Missing submodule checkout')
1190 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/readme.md')), msg='Missing submodule checkout')
1191 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
1192 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
1193
Patrick Williams92b42cb2022-09-03 06:53:57 -05001194 @skipIfNoNetwork()
1195 def test_git_submodule_reference_to_parent(self):
1196 self.recipe_url = "gitsm://github.com/gflags/gflags.git;protocol=https;branch=master"
1197 self.d.setVar("SRCREV", "14e1138441bbbb584160cb1c0a0426ec1bac35f1")
1198 with Timeout(60):
1199 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
1200 with self.assertRaises(bb.fetch2.FetchError):
1201 fetcher.download()
1202
Brad Bishop15ae2502019-06-18 21:44:24 -04001203class SVNTest(FetcherTest):
1204 def skipIfNoSvn():
1205 import shutil
1206 if not shutil.which("svn"):
1207 return unittest.skip("svn not installed, tests being skipped")
1208
1209 if not shutil.which("svnadmin"):
1210 return unittest.skip("svnadmin not installed, tests being skipped")
1211
1212 return lambda f: f
1213
1214 @skipIfNoSvn()
1215 def setUp(self):
1216 """ Create a local repository """
1217
1218 super(SVNTest, self).setUp()
1219
1220 # Create something we can fetch
1221 src_dir = tempfile.mkdtemp(dir=self.tempdir,
1222 prefix='svnfetch_srcdir_')
1223 src_dir = os.path.abspath(src_dir)
1224 bb.process.run("echo readme > README.md", cwd=src_dir)
1225
1226 # Store it in a local SVN repository
1227 repo_dir = tempfile.mkdtemp(dir=self.tempdir,
1228 prefix='svnfetch_localrepo_')
1229 repo_dir = os.path.abspath(repo_dir)
1230 bb.process.run("svnadmin create project", cwd=repo_dir)
1231
1232 self.repo_url = "file://%s/project" % repo_dir
1233 bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url),
1234 cwd=repo_dir)
1235
1236 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
1237 # Github will emulate SVN. Use this to check if we're downloding...
Andrew Geissler595f6302022-01-24 19:11:47 +00001238 bb.process.run("svn propset svn:externals 'bitbake https://github.com/PhilipHazel/pcre2.git' .",
Brad Bishop15ae2502019-06-18 21:44:24 -04001239 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
1240 bb.process.run("svn commit --non-interactive -m 'Add external'",
1241 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
1242
1243 self.src_dir = src_dir
1244 self.repo_dir = repo_dir
1245
1246 @skipIfNoSvn()
1247 def tearDown(self):
1248 os.chdir(self.origdir)
1249 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
1250 print("Not cleaning up %s. Please remove manually." % self.tempdir)
1251 else:
1252 bb.utils.prunedir(self.tempdir)
1253
1254 @skipIfNoSvn()
1255 @skipIfNoNetwork()
1256 def test_noexternal_svn(self):
1257 # Always match the rev count from setUp (currently rev 2)
1258 url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '')
1259 fetcher = bb.fetch.Fetch([url], self.d)
1260 fetcher.download()
1261 os.chdir(os.path.dirname(self.unpackdir))
1262 fetcher.unpack(self.unpackdir)
1263
1264 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
1265 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
1266 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should NOT exist")
1267 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should NOT exit")
1268
1269 @skipIfNoSvn()
1270 def test_external_svn(self):
1271 # Always match the rev count from setUp (currently rev 2)
1272 url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '')
1273 fetcher = bb.fetch.Fetch([url], self.d)
1274 fetcher.download()
1275 os.chdir(os.path.dirname(self.unpackdir))
1276 fetcher.unpack(self.unpackdir)
1277
1278 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
1279 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
1280 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should exist")
1281 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should exit")
1282
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001283class TrustedNetworksTest(FetcherTest):
1284 def test_trusted_network(self):
1285 # Ensure trusted_network returns False when the host IS in the list.
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001286 url = "git://Someserver.org/foo;rev=1;branch=master"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001287 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org")
1288 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001289
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001290 def test_wild_trusted_network(self):
1291 # Ensure trusted_network returns true when the *.host IS in the list.
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001292 url = "git://Someserver.org/foo;rev=1;branch=master"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001293 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1294 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001295
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001296 def test_prefix_wild_trusted_network(self):
1297 # Ensure trusted_network returns true when the prefix matches *.host.
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001298 url = "git://git.Someserver.org/foo;rev=1;branch=master"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001299 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1300 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001301
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001302 def test_two_prefix_wild_trusted_network(self):
1303 # Ensure trusted_network returns true when the prefix matches *.host.
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001304 url = "git://something.git.Someserver.org/foo;rev=1;branch=master"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001305 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1306 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001307
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001308 def test_port_trusted_network(self):
1309 # Ensure trusted_network returns True, even if the url specifies a port.
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001310 url = "git://someserver.org:8080/foo;rev=1;branch=master"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001311 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org")
1312 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001313
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001314 def test_untrusted_network(self):
1315 # Ensure trusted_network returns False when the host is NOT in the list.
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001316 url = "git://someserver.org/foo;rev=1;branch=master"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001317 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
1318 self.assertFalse(bb.fetch.trusted_network(self.d, url))
1319
1320 def test_wild_untrusted_network(self):
1321 # Ensure trusted_network returns False when the host is NOT in the list.
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001322 url = "git://*.someserver.org/foo;rev=1;branch=master"
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001323 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
1324 self.assertFalse(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001325
1326class URLHandle(unittest.TestCase):
1327
1328 datatable = {
1329 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}),
1330 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}),
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001331 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001332 "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'}),
1333 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}),
Patrick Williams2390b1b2022-11-03 13:47:49 -05001334 r'git://s.o-me_ONE:!#$%^&*()-_={}[]\|:?,.<>~`@git.openembedded.org/bitbake;branch=main': ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', r'!#$%^&*()-_={}[]\|:?,.<>~`', {'branch': 'main'}),
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001335 }
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001336 # we require a pathname to encodeurl but users can still pass such urls to
1337 # decodeurl and we need to handle them
1338 decodedata = datatable.copy()
1339 decodedata.update({
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001340 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}),
Patrick Williams2390b1b2022-11-03 13:47:49 -05001341 "npmsw://some.registry.url;package=@pkg;version=latest": ('npmsw', 'some.registry.url', '/', '', '', {'package': '@pkg', 'version': 'latest'}),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001342 })
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001343
1344 def test_decodeurl(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001345 for k, v in self.decodedata.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001346 result = bb.fetch.decodeurl(k)
1347 self.assertEqual(result, v)
1348
1349 def test_encodeurl(self):
1350 for k, v in self.datatable.items():
1351 result = bb.fetch.encodeurl(v)
1352 self.assertEqual(result, k)
1353
1354class FetchLatestVersionTest(FetcherTest):
1355
1356 test_git_uris = {
1357 # version pattern "X.Y.Z"
Andrew Geissler595f6302022-01-24 19:11:47 +00001358 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001359 : "1.99.4",
1360 # version pattern "vX.Y"
Andrew Geisslerd25ed322020-06-27 00:28:28 -05001361 # mirror of git.infradead.org since network issues interfered with testing
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001362 ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001363 : "1.5.0",
1364 # version pattern "pkg_name-X.Y"
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001365 # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001366 ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001367 : "1.0",
1368 # version pattern "pkg_name-vX.Y.Z"
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001369 ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001370 : "1.4.0",
1371 # combination version pattern
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001372 ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001373 : "1.2.0",
1374 ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "")
1375 : "2014.01",
1376 # version pattern "yyyymmdd"
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001377 ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001378 : "20120614",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001379 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001380 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001381 ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001382 : "0.4.3",
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001383 ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001384 : "11.0.0",
Andrew Geissler595f6302022-01-24 19:11:47 +00001385 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001386 : "1.3.59",
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001387 ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001388 : "3.82+dbg0.9",
1389 }
1390
1391 test_wget_uris = {
Andrew Geissler82c905d2020-04-13 13:39:40 -05001392 #
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001393 # packages with versions inside directory name
Andrew Geissler82c905d2020-04-13 13:39:40 -05001394 #
1395 # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2
1396 ("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001397 : "2.24.2",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001398 # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz
1399 ("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001400 : "1.6.0",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001401 # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz
1402 ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001403 : "2.8.12.1",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001404 #
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001405 # packages with versions only in current directory
Andrew Geissler82c905d2020-04-13 13:39:40 -05001406 #
Andrew Geisslereff27472021-10-29 15:35:00 -05001407 # https://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2
Andrew Geissler82c905d2020-04-13 13:39:40 -05001408 ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001409 : "2.19",
Andrew Geisslereff27472021-10-29 15:35:00 -05001410 # https://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2
Andrew Geissler82c905d2020-04-13 13:39:40 -05001411 ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001412 : "20120814",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001413 #
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001414 # packages with "99" in the name of possible version
Andrew Geissler82c905d2020-04-13 13:39:40 -05001415 #
1416 # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz
1417 ("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001418 : "5.0",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001419 # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2
1420 ("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001421 : "1.15.1",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001422 #
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001423 # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX
Andrew Geissler82c905d2020-04-13 13:39:40 -05001424 #
1425 # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2
1426 # https://github.com/apple/cups/releases
Andrew Geissler5199d832021-09-24 16:47:35 -05001427 ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", r"(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001428 : "2.0.0",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001429 # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz
1430 # http://ftp.debian.org/debian/pool/main/d/db5.3/
Andrew Geissler5199d832021-09-24 16:47:35 -05001431 ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", r"(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz")
Brad Bishop79641f22019-09-10 07:20:22 -04001432 : "5.3.10",
Andrew Geissler595f6302022-01-24 19:11:47 +00001433 #
1434 # packages where the tarball compression changed in the new version
1435 #
1436 # http://ftp.debian.org/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz
1437 ("minicom", "/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz", "", "")
1438 : "2.8",
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001439 }
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001440
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001441 @skipIfNoNetwork()
1442 def test_git_latest_versionstring(self):
1443 for k, v in self.test_git_uris.items():
1444 self.d.setVar("PN", k[0])
1445 self.d.setVar("SRCREV", k[2])
1446 self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3])
1447 ud = bb.fetch2.FetchData(k[1], self.d)
1448 pupver= ud.method.latest_versionstring(ud, self.d)
1449 verstring = pupver[0]
Brad Bishop316dfdd2018-06-25 12:45:53 -04001450 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001451 r = bb.utils.vercmp_string(v, verstring)
1452 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
1453
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001454 def test_wget_latest_versionstring(self):
Andrew Geissler82c905d2020-04-13 13:39:40 -05001455 testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata"
1456 server = HTTPService(testdata)
1457 server.start()
1458 port = server.port
1459 try:
1460 for k, v in self.test_wget_uris.items():
1461 self.d.setVar("PN", k[0])
1462 checkuri = ""
1463 if k[2]:
1464 checkuri = "http://localhost:%s/" % port + k[2]
1465 self.d.setVar("UPSTREAM_CHECK_URI", checkuri)
1466 self.d.setVar("UPSTREAM_CHECK_REGEX", k[3])
1467 url = "http://localhost:%s/" % port + k[1]
1468 ud = bb.fetch2.FetchData(url, self.d)
1469 pupver = ud.method.latest_versionstring(ud, self.d)
1470 verstring = pupver[0]
1471 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
1472 r = bb.utils.vercmp_string(v, verstring)
1473 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
1474 finally:
1475 server.stop()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001476
1477
1478class FetchCheckStatusTest(FetcherTest):
Andrew Geisslereff27472021-10-29 15:35:00 -05001479 test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
1480 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
1481 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001482 "https://yoctoproject.org/",
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001483 "https://docs.yoctoproject.org",
Andrew Geisslereff27472021-10-29 15:35:00 -05001484 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
1485 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001486 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001487 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD
1488 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001489 ]
1490
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001491 @skipIfNoNetwork()
1492 def test_wget_checkstatus(self):
1493 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d)
1494 for u in self.test_wget_uris:
1495 with self.subTest(url=u):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001496 ud = fetch.ud[u]
1497 m = ud.method
1498 ret = m.checkstatus(fetch, ud, self.d)
1499 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
1500
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001501 @skipIfNoNetwork()
1502 def test_wget_checkstatus_connection_cache(self):
1503 from bb.fetch2 import FetchConnectionCache
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001504
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001505 connection_cache = FetchConnectionCache()
1506 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d,
1507 connection_cache = connection_cache)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001508
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001509 for u in self.test_wget_uris:
1510 with self.subTest(url=u):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001511 ud = fetch.ud[u]
1512 m = ud.method
1513 ret = m.checkstatus(fetch, ud, self.d)
1514 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
1515
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001516 connection_cache.close_connections()
1517
1518
1519class GitMakeShallowTest(FetcherTest):
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001520 def setUp(self):
1521 FetcherTest.setUp(self)
1522 self.gitdir = os.path.join(self.tempdir, 'gitshallow')
1523 bb.utils.mkdirhier(self.gitdir)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001524 self.git_init()
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001525
1526 def assertRefs(self, expected_refs):
1527 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
1528 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines()
1529 self.assertEqual(sorted(full_expected), sorted(actual_refs))
1530
1531 def assertRevCount(self, expected_count, args=None):
1532 if args is None:
1533 args = ['HEAD']
1534 revs = self.git(['rev-list'] + args)
1535 actual_count = len(revs.splitlines())
1536 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1537
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001538 def make_shallow(self, args=None):
1539 if args is None:
1540 args = ['HEAD']
Brad Bishop316dfdd2018-06-25 12:45:53 -04001541 return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001542
1543 def add_empty_file(self, path, msg=None):
1544 if msg is None:
1545 msg = path
1546 open(os.path.join(self.gitdir, path), 'w').close()
1547 self.git(['add', path])
1548 self.git(['commit', '-m', msg, path])
1549
1550 def test_make_shallow_single_branch_no_merge(self):
1551 self.add_empty_file('a')
1552 self.add_empty_file('b')
1553 self.assertRevCount(2)
1554 self.make_shallow()
1555 self.assertRevCount(1)
1556
1557 def test_make_shallow_single_branch_one_merge(self):
1558 self.add_empty_file('a')
1559 self.add_empty_file('b')
1560 self.git('checkout -b a_branch')
1561 self.add_empty_file('c')
1562 self.git('checkout master')
1563 self.add_empty_file('d')
1564 self.git('merge --no-ff --no-edit a_branch')
1565 self.git('branch -d a_branch')
1566 self.add_empty_file('e')
1567 self.assertRevCount(6)
1568 self.make_shallow(['HEAD~2'])
1569 self.assertRevCount(5)
1570
1571 def test_make_shallow_at_merge(self):
1572 self.add_empty_file('a')
1573 self.git('checkout -b a_branch')
1574 self.add_empty_file('b')
1575 self.git('checkout master')
1576 self.git('merge --no-ff --no-edit a_branch')
1577 self.git('branch -d a_branch')
1578 self.assertRevCount(3)
1579 self.make_shallow()
1580 self.assertRevCount(1)
1581
1582 def test_make_shallow_annotated_tag(self):
1583 self.add_empty_file('a')
1584 self.add_empty_file('b')
1585 self.git('tag -a -m a_tag a_tag')
1586 self.assertRevCount(2)
1587 self.make_shallow(['a_tag'])
1588 self.assertRevCount(1)
1589
1590 def test_make_shallow_multi_ref(self):
1591 self.add_empty_file('a')
1592 self.add_empty_file('b')
1593 self.git('checkout -b a_branch')
1594 self.add_empty_file('c')
1595 self.git('checkout master')
1596 self.add_empty_file('d')
1597 self.git('checkout -b a_branch_2')
1598 self.add_empty_file('a_tag')
1599 self.git('tag a_tag')
1600 self.git('checkout master')
1601 self.git('branch -D a_branch_2')
1602 self.add_empty_file('e')
1603 self.assertRevCount(6, ['--all'])
1604 self.make_shallow()
1605 self.assertRevCount(5, ['--all'])
1606
1607 def test_make_shallow_multi_ref_trim(self):
1608 self.add_empty_file('a')
1609 self.git('checkout -b a_branch')
1610 self.add_empty_file('c')
1611 self.git('checkout master')
1612 self.assertRevCount(1)
1613 self.assertRevCount(2, ['--all'])
1614 self.assertRefs(['master', 'a_branch'])
1615 self.make_shallow(['-r', 'master', 'HEAD'])
1616 self.assertRevCount(1, ['--all'])
1617 self.assertRefs(['master'])
1618
1619 def test_make_shallow_noop(self):
1620 self.add_empty_file('a')
1621 self.assertRevCount(1)
1622 self.make_shallow()
1623 self.assertRevCount(1)
1624
1625 @skipIfNoNetwork()
1626 def test_make_shallow_bitbake(self):
1627 self.git('remote add origin https://github.com/openembedded/bitbake')
1628 self.git('fetch --tags origin')
1629 orig_revs = len(self.git('rev-list --all').splitlines())
1630 self.make_shallow(['refs/tags/1.10.0'])
1631 self.assertRevCount(orig_revs - 1746, ['--all'])
1632
1633class GitShallowTest(FetcherTest):
1634 def setUp(self):
1635 FetcherTest.setUp(self)
1636 self.gitdir = os.path.join(self.tempdir, 'git')
1637 self.srcdir = os.path.join(self.tempdir, 'gitsource')
1638
1639 bb.utils.mkdirhier(self.srcdir)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001640 self.git_init(cwd=self.srcdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001641 self.d.setVar('WORKDIR', self.tempdir)
1642 self.d.setVar('S', self.gitdir)
1643 self.d.delVar('PREMIRRORS')
1644 self.d.delVar('MIRRORS')
1645
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001646 uri = 'git://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001647 self.d.setVar('SRC_URI', uri)
1648 self.d.setVar('SRCREV', '${AUTOREV}')
1649 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
1650
1651 self.d.setVar('BB_GIT_SHALLOW', '1')
1652 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
1653 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001654 self.d.setVar("__BBSEENSRCREV", "1")
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001655
1656 def assertRefs(self, expected_refs, cwd=None):
1657 if cwd is None:
1658 cwd = self.gitdir
1659 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines()
1660 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines()
1661 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs)))
1662
1663 def assertRevCount(self, expected_count, args=None, cwd=None):
1664 if args is None:
1665 args = ['HEAD']
1666 if cwd is None:
1667 cwd = self.gitdir
1668 revs = self.git(['rev-list'] + args, cwd=cwd)
1669 actual_count = len(revs.splitlines())
1670 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1671
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001672 def add_empty_file(self, path, cwd=None, msg=None):
1673 if msg is None:
1674 msg = path
1675 if cwd is None:
1676 cwd = self.srcdir
1677 open(os.path.join(cwd, path), 'w').close()
1678 self.git(['add', path], cwd)
1679 self.git(['commit', '-m', msg, path], cwd)
1680
1681 def fetch(self, uri=None):
1682 if uri is None:
Brad Bishop19323692019-04-05 15:28:33 -04001683 uris = self.d.getVar('SRC_URI').split()
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001684 uri = uris[0]
1685 d = self.d
1686 else:
1687 d = self.d.createCopy()
1688 d.setVar('SRC_URI', uri)
1689 uri = d.expand(uri)
1690 uris = [uri]
1691
1692 fetcher = bb.fetch2.Fetch(uris, d)
1693 fetcher.download()
1694 ud = fetcher.ud[uri]
1695 return fetcher, ud
1696
1697 def fetch_and_unpack(self, uri=None):
1698 fetcher, ud = self.fetch(uri)
1699 fetcher.unpack(self.d.getVar('WORKDIR'))
1700 assert os.path.exists(self.d.getVar('S'))
1701 return fetcher, ud
1702
1703 def fetch_shallow(self, uri=None, disabled=False, keepclone=False):
1704 """Fetch a uri, generating a shallow tarball, then unpack using it"""
1705 fetcher, ud = self.fetch_and_unpack(uri)
1706 assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri)
1707
1708 # Confirm that the unpacked repo is unshallow
1709 if not disabled:
1710 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1711
1712 # fetch and unpack, from the shallow tarball
1713 bb.utils.remove(self.gitdir, recurse=True)
Andrew Geisslerc926e172021-05-07 16:11:35 -05001714 bb.process.run('chmod u+w -R "%s"' % ud.clonedir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001715 bb.utils.remove(ud.clonedir, recurse=True)
Brad Bishopf8caae32019-03-25 13:13:56 -04001716 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001717
1718 # confirm that the unpacked repo is used when no git clone or git
1719 # mirror tarball is available
1720 fetcher, ud = self.fetch_and_unpack(uri)
1721 if not disabled:
1722 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir
1723 else:
1724 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir
1725 return fetcher, ud
1726
1727 def test_shallow_disabled(self):
1728 self.add_empty_file('a')
1729 self.add_empty_file('b')
1730 self.assertRevCount(2, cwd=self.srcdir)
1731
1732 self.d.setVar('BB_GIT_SHALLOW', '0')
1733 self.fetch_shallow(disabled=True)
1734 self.assertRevCount(2)
1735
1736 def test_shallow_nobranch(self):
1737 self.add_empty_file('a')
1738 self.add_empty_file('b')
1739 self.assertRevCount(2, cwd=self.srcdir)
1740
1741 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip()
1742 self.d.setVar('SRCREV', srcrev)
Brad Bishop19323692019-04-05 15:28:33 -04001743 uri = self.d.getVar('SRC_URI').split()[0]
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001744 uri = '%s;nobranch=1;bare=1' % uri
1745
1746 self.fetch_shallow(uri)
1747 self.assertRevCount(1)
1748
1749 # shallow refs are used to ensure the srcrev sticks around when we
1750 # have no other branches referencing it
1751 self.assertRefs(['refs/shallow/default'])
1752
1753 def test_shallow_default_depth_1(self):
1754 # Create initial git repo
1755 self.add_empty_file('a')
1756 self.add_empty_file('b')
1757 self.assertRevCount(2, cwd=self.srcdir)
1758
1759 self.fetch_shallow()
1760 self.assertRevCount(1)
1761
1762 def test_shallow_depth_0_disables(self):
1763 self.add_empty_file('a')
1764 self.add_empty_file('b')
1765 self.assertRevCount(2, cwd=self.srcdir)
1766
1767 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1768 self.fetch_shallow(disabled=True)
1769 self.assertRevCount(2)
1770
1771 def test_shallow_depth_default_override(self):
1772 self.add_empty_file('a')
1773 self.add_empty_file('b')
1774 self.assertRevCount(2, cwd=self.srcdir)
1775
1776 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2')
1777 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1')
1778 self.fetch_shallow()
1779 self.assertRevCount(1)
1780
1781 def test_shallow_depth_default_override_disable(self):
1782 self.add_empty_file('a')
1783 self.add_empty_file('b')
1784 self.add_empty_file('c')
1785 self.assertRevCount(3, cwd=self.srcdir)
1786
1787 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1788 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2')
1789 self.fetch_shallow()
1790 self.assertRevCount(2)
1791
1792 def test_current_shallow_out_of_date_clone(self):
1793 # Create initial git repo
1794 self.add_empty_file('a')
1795 self.add_empty_file('b')
1796 self.add_empty_file('c')
1797 self.assertRevCount(3, cwd=self.srcdir)
1798
1799 # Clone and generate mirror tarball
1800 fetcher, ud = self.fetch()
1801
1802 # Ensure we have a current mirror tarball, but an out of date clone
1803 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir)
1804 self.assertRevCount(2, cwd=ud.clonedir)
1805
1806 # Fetch and unpack, from the current tarball, not the out of date clone
1807 bb.utils.remove(self.gitdir, recurse=True)
1808 fetcher, ud = self.fetch()
1809 fetcher.unpack(self.d.getVar('WORKDIR'))
1810 self.assertRevCount(1)
1811
1812 def test_shallow_single_branch_no_merge(self):
1813 self.add_empty_file('a')
1814 self.add_empty_file('b')
1815 self.assertRevCount(2, cwd=self.srcdir)
1816
1817 self.fetch_shallow()
1818 self.assertRevCount(1)
1819 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1820 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1821
1822 def test_shallow_no_dangling(self):
1823 self.add_empty_file('a')
1824 self.add_empty_file('b')
1825 self.assertRevCount(2, cwd=self.srcdir)
1826
1827 self.fetch_shallow()
1828 self.assertRevCount(1)
1829 assert not self.git('fsck --dangling')
1830
1831 def test_shallow_srcrev_branch_truncation(self):
1832 self.add_empty_file('a')
1833 self.add_empty_file('b')
1834 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip()
1835 self.add_empty_file('c')
1836 self.assertRevCount(3, cwd=self.srcdir)
1837
1838 self.d.setVar('SRCREV', b_commit)
1839 self.fetch_shallow()
1840
1841 # The 'c' commit was removed entirely, and 'a' was removed from history
1842 self.assertRevCount(1, ['--all'])
1843 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit)
1844 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1845 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1846 assert not os.path.exists(os.path.join(self.gitdir, 'c'))
1847
1848 def test_shallow_ref_pruning(self):
1849 self.add_empty_file('a')
1850 self.add_empty_file('b')
1851 self.git('branch a_branch', cwd=self.srcdir)
1852 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
1853 self.assertRevCount(2, cwd=self.srcdir)
1854
1855 self.fetch_shallow()
1856
1857 self.assertRefs(['master', 'origin/master'])
1858 self.assertRevCount(1)
1859
1860 def test_shallow_submodules(self):
1861 self.add_empty_file('a')
1862 self.add_empty_file('b')
1863
1864 smdir = os.path.join(self.tempdir, 'gitsubmodule')
1865 bb.utils.mkdirhier(smdir)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001866 self.git_init(cwd=smdir)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001867 # Make this look like it was cloned from a remote...
1868 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
1869 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001870 self.add_empty_file('asub', cwd=smdir)
Brad Bishopf8caae32019-03-25 13:13:56 -04001871 self.add_empty_file('bsub', cwd=smdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001872
1873 self.git('submodule init', cwd=self.srcdir)
Patrick Williams2390b1b2022-11-03 13:47:49 -05001874 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001875 self.git('submodule update', cwd=self.srcdir)
1876 self.git('commit -m submodule -a', cwd=self.srcdir)
1877
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001878 uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001879 fetcher, ud = self.fetch_shallow(uri)
1880
Brad Bishopf8caae32019-03-25 13:13:56 -04001881 # Verify the main repository is shallow
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001882 self.assertRevCount(1)
Brad Bishopf8caae32019-03-25 13:13:56 -04001883
1884 # Verify the gitsubmodule directory is present
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001885 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
1886
Brad Bishopf8caae32019-03-25 13:13:56 -04001887 # Verify the submodule is also shallow
1888 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
1889
Andrew Geissler82c905d2020-04-13 13:39:40 -05001890 def test_shallow_submodule_mirrors(self):
1891 self.add_empty_file('a')
1892 self.add_empty_file('b')
1893
1894 smdir = os.path.join(self.tempdir, 'gitsubmodule')
1895 bb.utils.mkdirhier(smdir)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001896 self.git_init(cwd=smdir)
Andrew Geissler82c905d2020-04-13 13:39:40 -05001897 # Make this look like it was cloned from a remote...
1898 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
1899 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
1900 self.add_empty_file('asub', cwd=smdir)
1901 self.add_empty_file('bsub', cwd=smdir)
1902
1903 self.git('submodule init', cwd=self.srcdir)
Patrick Williams2390b1b2022-11-03 13:47:49 -05001904 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir)
Andrew Geissler82c905d2020-04-13 13:39:40 -05001905 self.git('submodule update', cwd=self.srcdir)
1906 self.git('commit -m submodule -a', cwd=self.srcdir)
1907
1908 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
1909
1910 # Fetch once to generate the shallow tarball
1911 fetcher, ud = self.fetch(uri)
1912
1913 # Set up the mirror
1914 mirrordir = os.path.join(self.tempdir, 'mirror')
Andrew Geisslerc926e172021-05-07 16:11:35 -05001915 bb.utils.rename(self.dldir, mirrordir)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001916 self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/' % mirrordir)
Andrew Geissler82c905d2020-04-13 13:39:40 -05001917
1918 # Fetch from the mirror
1919 bb.utils.remove(self.dldir, recurse=True)
1920 bb.utils.remove(self.gitdir, recurse=True)
1921 self.fetch_and_unpack(uri)
1922
1923 # Verify the main repository is shallow
1924 self.assertRevCount(1)
1925
1926 # Verify the gitsubmodule directory is present
1927 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
1928
1929 # Verify the submodule is also shallow
1930 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
Brad Bishopf8caae32019-03-25 13:13:56 -04001931
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001932 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')):
1933 def test_shallow_annex(self):
1934 self.add_empty_file('a')
1935 self.add_empty_file('b')
1936 self.git('annex init', cwd=self.srcdir)
1937 open(os.path.join(self.srcdir, 'c'), 'w').close()
1938 self.git('annex add c', cwd=self.srcdir)
Andrew Geisslerc926e172021-05-07 16:11:35 -05001939 self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir)
1940 bb.process.run('chmod u+w -R %s' % self.srcdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001941
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00001942 uri = 'gitannex://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001943 fetcher, ud = self.fetch_shallow(uri)
1944
1945 self.assertRevCount(1)
1946 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0]
1947 assert os.path.exists(os.path.join(self.gitdir, 'c'))
1948
1949 def test_shallow_multi_one_uri(self):
1950 # Create initial git repo
1951 self.add_empty_file('a')
1952 self.add_empty_file('b')
1953 self.git('checkout -b a_branch', cwd=self.srcdir)
1954 self.add_empty_file('c')
1955 self.add_empty_file('d')
1956 self.git('checkout master', cwd=self.srcdir)
1957 self.git('tag v0.0 a_branch', cwd=self.srcdir)
1958 self.add_empty_file('e')
1959 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1960 self.add_empty_file('f')
1961 self.assertRevCount(7, cwd=self.srcdir)
1962
Brad Bishop19323692019-04-05 15:28:33 -04001963 uri = self.d.getVar('SRC_URI').split()[0]
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001964 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
1965
1966 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1967 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1968 self.d.setVar('SRCREV_master', '${AUTOREV}')
1969 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
1970
1971 self.fetch_shallow(uri)
1972
1973 self.assertRevCount(5)
1974 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1975
1976 def test_shallow_multi_one_uri_depths(self):
1977 # Create initial git repo
1978 self.add_empty_file('a')
1979 self.add_empty_file('b')
1980 self.git('checkout -b a_branch', cwd=self.srcdir)
1981 self.add_empty_file('c')
1982 self.add_empty_file('d')
1983 self.git('checkout master', cwd=self.srcdir)
1984 self.add_empty_file('e')
1985 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1986 self.add_empty_file('f')
1987 self.assertRevCount(7, cwd=self.srcdir)
1988
Brad Bishop19323692019-04-05 15:28:33 -04001989 uri = self.d.getVar('SRC_URI').split()[0]
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001990 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
1991
1992 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1993 self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3')
1994 self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1')
1995 self.d.setVar('SRCREV_master', '${AUTOREV}')
1996 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
1997
1998 self.fetch_shallow(uri)
1999
2000 self.assertRevCount(4, ['--all'])
2001 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
2002
2003 def test_shallow_clone_preferred_over_shallow(self):
2004 self.add_empty_file('a')
2005 self.add_empty_file('b')
2006
2007 # Fetch once to generate the shallow tarball
2008 fetcher, ud = self.fetch()
2009 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
2010
2011 # Fetch and unpack with both the clonedir and shallow tarball available
2012 bb.utils.remove(self.gitdir, recurse=True)
2013 fetcher, ud = self.fetch_and_unpack()
2014
2015 # The unpacked tree should *not* be shallow
2016 self.assertRevCount(2)
2017 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow'))
2018
2019 def test_shallow_mirrors(self):
2020 self.add_empty_file('a')
2021 self.add_empty_file('b')
2022
2023 # Fetch once to generate the shallow tarball
2024 fetcher, ud = self.fetch()
2025 mirrortarball = ud.mirrortarballs[0]
2026 assert os.path.exists(os.path.join(self.dldir, mirrortarball))
2027
2028 # Set up the mirror
2029 mirrordir = os.path.join(self.tempdir, 'mirror')
2030 bb.utils.mkdirhier(mirrordir)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002031 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % mirrordir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05002032
Andrew Geisslerc926e172021-05-07 16:11:35 -05002033 bb.utils.rename(os.path.join(self.dldir, mirrortarball),
Brad Bishopd7bf8c12018-02-25 22:55:05 -05002034 os.path.join(mirrordir, mirrortarball))
2035
2036 # Fetch from the mirror
2037 bb.utils.remove(self.dldir, recurse=True)
2038 bb.utils.remove(self.gitdir, recurse=True)
2039 self.fetch_and_unpack()
2040 self.assertRevCount(1)
2041
2042 def test_shallow_invalid_depth(self):
2043 self.add_empty_file('a')
2044 self.add_empty_file('b')
2045
2046 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12')
2047 with self.assertRaises(bb.fetch2.FetchError):
2048 self.fetch()
2049
2050 def test_shallow_invalid_depth_default(self):
2051 self.add_empty_file('a')
2052 self.add_empty_file('b')
2053
2054 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12')
2055 with self.assertRaises(bb.fetch2.FetchError):
2056 self.fetch()
2057
2058 def test_shallow_extra_refs(self):
2059 self.add_empty_file('a')
2060 self.add_empty_file('b')
2061 self.git('branch a_branch', cwd=self.srcdir)
2062 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
2063 self.assertRevCount(2, cwd=self.srcdir)
2064
2065 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch')
2066 self.fetch_shallow()
2067
2068 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
2069 self.assertRevCount(1)
2070
2071 def test_shallow_extra_refs_wildcard(self):
2072 self.add_empty_file('a')
2073 self.add_empty_file('b')
2074 self.git('branch a_branch', cwd=self.srcdir)
2075 self.git('tag v1.0', cwd=self.srcdir)
2076 self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir)
2077 self.assertRevCount(2, cwd=self.srcdir)
2078
2079 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
2080 self.fetch_shallow()
2081
2082 self.assertRefs(['master', 'origin/master', 'v1.0'])
2083 self.assertRevCount(1)
2084
2085 def test_shallow_missing_extra_refs(self):
2086 self.add_empty_file('a')
2087 self.add_empty_file('b')
2088
2089 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo')
2090 with self.assertRaises(bb.fetch2.FetchError):
2091 self.fetch()
2092
2093 def test_shallow_missing_extra_refs_wildcard(self):
2094 self.add_empty_file('a')
2095 self.add_empty_file('b')
2096
2097 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
2098 self.fetch()
2099
2100 def test_shallow_remove_revs(self):
2101 # Create initial git repo
2102 self.add_empty_file('a')
2103 self.add_empty_file('b')
2104 self.git('checkout -b a_branch', cwd=self.srcdir)
2105 self.add_empty_file('c')
2106 self.add_empty_file('d')
2107 self.git('checkout master', cwd=self.srcdir)
2108 self.git('tag v0.0 a_branch', cwd=self.srcdir)
2109 self.add_empty_file('e')
2110 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
2111 self.git('branch -d a_branch', cwd=self.srcdir)
2112 self.add_empty_file('f')
2113 self.assertRevCount(7, cwd=self.srcdir)
2114
2115 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2116 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
2117
2118 self.fetch_shallow()
2119
2120 self.assertRevCount(5)
2121
2122 def test_shallow_invalid_revs(self):
2123 self.add_empty_file('a')
2124 self.add_empty_file('b')
2125
2126 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2127 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
2128
2129 with self.assertRaises(bb.fetch2.FetchError):
2130 self.fetch()
2131
Brad Bishop64c979e2019-11-04 13:55:29 -05002132 def test_shallow_fetch_missing_revs(self):
2133 self.add_empty_file('a')
2134 self.add_empty_file('b')
2135 fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
2136 self.git('tag v0.0 master', cwd=self.srcdir)
2137 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2138 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
2139 self.fetch_shallow()
2140
2141 def test_shallow_fetch_missing_revs_fails(self):
2142 self.add_empty_file('a')
2143 self.add_empty_file('b')
2144 fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
2145 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2146 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
2147
2148 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm:
2149 self.fetch_shallow()
2150 self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0])
2151
Brad Bishopd7bf8c12018-02-25 22:55:05 -05002152 @skipIfNoNetwork()
2153 def test_bitbake(self):
Andrew Geissler595f6302022-01-24 19:11:47 +00002154 self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05002155 self.git('config core.bare true', cwd=self.srcdir)
2156 self.git('fetch', cwd=self.srcdir)
2157
2158 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2159 # Note that the 1.10.0 tag is annotated, so this also tests
2160 # reference of an annotated vs unannotated tag
2161 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0')
2162
2163 self.fetch_shallow()
2164
2165 # Confirm that the history of 1.10.0 was removed
2166 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines())
2167 revs = len(self.git('rev-list master').splitlines())
2168 self.assertNotEqual(orig_revs, revs)
2169 self.assertRefs(['master', 'origin/master'])
2170 self.assertRevCount(orig_revs - 1758)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08002171
2172 def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self):
2173 self.add_empty_file('a')
2174 fetcher, ud = self.fetch()
2175 bb.utils.remove(self.gitdir, recurse=True)
2176 bb.utils.remove(self.dldir, recurse=True)
2177
2178 with self.assertRaises(bb.fetch2.UnpackError) as context:
2179 fetcher.unpack(self.d.getVar('WORKDIR'))
2180
2181 self.assertIn("No up to date source found", context.exception.msg)
2182 self.assertIn("clone directory not available or not up to date", context.exception.msg)
2183
2184 @skipIfNoNetwork()
2185 def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self):
2186 self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0')
2187 self.d.setVar('BB_GIT_SHALLOW', '1')
2188 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002189 fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests;branch=master"], self.d)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08002190 fetcher.download()
2191
2192 bb.utils.remove(self.dldir + "/*.tar.gz")
2193 fetcher.unpack(self.unpackdir)
2194
2195 dir = os.listdir(self.unpackdir + "/git/")
2196 self.assertIn("fstests.doap", dir)
Brad Bishop00e122a2019-10-05 11:10:57 -04002197
2198class GitLfsTest(FetcherTest):
2199 def setUp(self):
2200 FetcherTest.setUp(self)
2201
2202 self.gitdir = os.path.join(self.tempdir, 'git')
2203 self.srcdir = os.path.join(self.tempdir, 'gitsource')
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002204
Brad Bishop00e122a2019-10-05 11:10:57 -04002205 self.d.setVar('WORKDIR', self.tempdir)
2206 self.d.setVar('S', self.gitdir)
2207 self.d.delVar('PREMIRRORS')
2208 self.d.delVar('MIRRORS')
2209
2210 self.d.setVar('SRCREV', '${AUTOREV}')
2211 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002212 self.d.setVar("__BBSEENSRCREV", "1")
Brad Bishop00e122a2019-10-05 11:10:57 -04002213
2214 bb.utils.mkdirhier(self.srcdir)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002215 self.git_init(cwd=self.srcdir)
Brad Bishop00e122a2019-10-05 11:10:57 -04002216 with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs:
2217 attrs.write('*.mp3 filter=lfs -text')
2218 self.git(['add', '.gitattributes'], cwd=self.srcdir)
2219 self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir)
2220
Andrew Geisslerd1e89492021-02-12 15:35:20 -06002221 def fetch(self, uri=None, download=True):
Brad Bishop00e122a2019-10-05 11:10:57 -04002222 uris = self.d.getVar('SRC_URI').split()
2223 uri = uris[0]
2224 d = self.d
2225
2226 fetcher = bb.fetch2.Fetch(uris, d)
Andrew Geisslerd1e89492021-02-12 15:35:20 -06002227 if download:
2228 fetcher.download()
Brad Bishop00e122a2019-10-05 11:10:57 -04002229 ud = fetcher.ud[uri]
2230 return fetcher, ud
2231
2232 def test_lfs_enabled(self):
2233 import shutil
2234
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002235 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir
Brad Bishop00e122a2019-10-05 11:10:57 -04002236 self.d.setVar('SRC_URI', uri)
2237
Andrew Geisslerd1e89492021-02-12 15:35:20 -06002238 # Careful: suppress initial attempt at downloading until
2239 # we know whether git-lfs is installed.
2240 fetcher, ud = self.fetch(uri=None, download=False)
Brad Bishop00e122a2019-10-05 11:10:57 -04002241 self.assertIsNotNone(ud.method._find_git_lfs)
2242
Andrew Geisslerd1e89492021-02-12 15:35:20 -06002243 # If git-lfs can be found, the unpack should be successful. Only
2244 # attempt this with the real live copy of git-lfs installed.
2245 if ud.method._find_git_lfs(self.d):
2246 fetcher.download()
2247 shutil.rmtree(self.gitdir, ignore_errors=True)
2248 fetcher.unpack(self.d.getVar('WORKDIR'))
Brad Bishop00e122a2019-10-05 11:10:57 -04002249
2250 # If git-lfs cannot be found, the unpack should throw an error
2251 with self.assertRaises(bb.fetch2.FetchError):
Andrew Geisslerd1e89492021-02-12 15:35:20 -06002252 fetcher.download()
Brad Bishop00e122a2019-10-05 11:10:57 -04002253 ud.method._find_git_lfs = lambda d: False
2254 shutil.rmtree(self.gitdir, ignore_errors=True)
2255 fetcher.unpack(self.d.getVar('WORKDIR'))
2256
2257 def test_lfs_disabled(self):
2258 import shutil
2259
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002260 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir
Brad Bishop00e122a2019-10-05 11:10:57 -04002261 self.d.setVar('SRC_URI', uri)
2262
Andrew Geisslerd1e89492021-02-12 15:35:20 -06002263 # In contrast to test_lfs_enabled(), allow the implicit download
2264 # done by self.fetch() to occur here. The point of this test case
2265 # is to verify that the fetcher can survive even if the source
2266 # repository has Git LFS usage configured.
Brad Bishop00e122a2019-10-05 11:10:57 -04002267 fetcher, ud = self.fetch()
2268 self.assertIsNotNone(ud.method._find_git_lfs)
2269
Andrew Geisslerd1e89492021-02-12 15:35:20 -06002270 # If git-lfs can be found, the unpack should be successful. A
2271 # live copy of git-lfs is not required for this case, so
2272 # unconditionally forge its presence.
Brad Bishop00e122a2019-10-05 11:10:57 -04002273 ud.method._find_git_lfs = lambda d: True
2274 shutil.rmtree(self.gitdir, ignore_errors=True)
2275 fetcher.unpack(self.d.getVar('WORKDIR'))
2276
2277 # If git-lfs cannot be found, the unpack should be successful
2278 ud.method._find_git_lfs = lambda d: False
2279 shutil.rmtree(self.gitdir, ignore_errors=True)
2280 fetcher.unpack(self.d.getVar('WORKDIR'))
Andrew Geissler82c905d2020-04-13 13:39:40 -05002281
Andrew Geisslerc3d88e42020-10-02 09:45:00 -05002282class GitURLWithSpacesTest(FetcherTest):
2283 test_git_urls = {
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002284 "git://tfs-example.org:22/tfs/example%20path/example.git;branch=master" : {
2285 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git;branch=master',
Andrew Geisslerc3d88e42020-10-02 09:45:00 -05002286 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git',
2287 'path': '/tfs/example path/example.git'
2288 },
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002289 "git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master" : {
2290 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master',
Andrew Geisslerc3d88e42020-10-02 09:45:00 -05002291 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git',
2292 'path': '/tfs/example path/example repo.git'
2293 }
2294 }
2295
2296 def test_urls(self):
2297
2298 # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo
2299 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
2300
2301 for test_git_url, ref in self.test_git_urls.items():
2302
2303 fetcher = bb.fetch.Fetch([test_git_url], self.d)
2304 ud = fetcher.ud[fetcher.urls[0]]
2305
2306 self.assertEqual(ud.url, ref['url'])
2307 self.assertEqual(ud.path, ref['path'])
2308 self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname']))
2309 self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname']))
2310 self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock'))
2311 self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname']))
2312 self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz'))
2313
Andrew Geissler595f6302022-01-24 19:11:47 +00002314class CrateTest(FetcherTest):
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002315 @skipIfNoNetwork()
Andrew Geissler595f6302022-01-24 19:11:47 +00002316 def test_crate_url(self):
2317
2318 uri = "crate://crates.io/glob/0.2.11"
2319 self.d.setVar('SRC_URI', uri)
2320
2321 uris = self.d.getVar('SRC_URI').split()
2322 d = self.d
2323
2324 fetcher = bb.fetch2.Fetch(uris, self.d)
2325 fetcher.download()
2326 fetcher.unpack(self.tempdir)
2327 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked'])
2328 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done'])
2329 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json"))
2330 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs"))
2331
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002332 @skipIfNoNetwork()
Andrew Geissler595f6302022-01-24 19:11:47 +00002333 def test_crate_url_multi(self):
2334
2335 uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35"
2336 self.d.setVar('SRC_URI', uri)
2337
2338 uris = self.d.getVar('SRC_URI').split()
2339 d = self.d
2340
2341 fetcher = bb.fetch2.Fetch(uris, self.d)
2342 fetcher.download()
2343 fetcher.unpack(self.tempdir)
2344 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked'])
2345 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done', 'time-0.1.35.crate', 'time-0.1.35.crate.done'])
2346 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json"))
2347 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs"))
2348 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json"))
2349 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs"))
2350
Andrew Geissler82c905d2020-04-13 13:39:40 -05002351class NPMTest(FetcherTest):
2352 def skipIfNoNpm():
2353 import shutil
2354 if not shutil.which('npm'):
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002355 return unittest.skip('npm not installed')
Andrew Geissler82c905d2020-04-13 13:39:40 -05002356 return lambda f: f
2357
2358 @skipIfNoNpm()
2359 @skipIfNoNetwork()
2360 def test_npm(self):
2361 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2362 fetcher = bb.fetch.Fetch([url], self.d)
2363 ud = fetcher.ud[fetcher.urls[0]]
2364 fetcher.download()
2365 self.assertTrue(os.path.exists(ud.localpath))
2366 self.assertTrue(os.path.exists(ud.localpath + '.done'))
2367 self.assertTrue(os.path.exists(ud.resolvefile))
2368 fetcher.unpack(self.unpackdir)
2369 unpackdir = os.path.join(self.unpackdir, 'npm')
2370 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2371
2372 @skipIfNoNpm()
2373 @skipIfNoNetwork()
2374 def test_npm_bad_checksum(self):
2375 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2376 # Fetch once to get a tarball
2377 fetcher = bb.fetch.Fetch([url], self.d)
2378 ud = fetcher.ud[fetcher.urls[0]]
2379 fetcher.download()
2380 self.assertTrue(os.path.exists(ud.localpath))
2381 # Modify the tarball
2382 bad = b'bad checksum'
2383 with open(ud.localpath, 'wb') as f:
2384 f.write(bad)
2385 # Verify that the tarball is fetched again
2386 fetcher.download()
2387 badsum = hashlib.sha512(bad).hexdigest()
2388 self.assertTrue(os.path.exists(ud.localpath + '_bad-checksum_' + badsum))
2389 self.assertTrue(os.path.exists(ud.localpath))
2390
2391 @skipIfNoNpm()
2392 @skipIfNoNetwork()
2393 def test_npm_premirrors(self):
2394 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2395 # Fetch once to get a tarball
2396 fetcher = bb.fetch.Fetch([url], self.d)
2397 ud = fetcher.ud[fetcher.urls[0]]
2398 fetcher.download()
2399 self.assertTrue(os.path.exists(ud.localpath))
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002400
2401 # Setup the mirror by renaming the download directory
Andrew Geissler82c905d2020-04-13 13:39:40 -05002402 mirrordir = os.path.join(self.tempdir, 'mirror')
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002403 bb.utils.rename(self.dldir, mirrordir)
2404 os.mkdir(self.dldir)
2405
2406 # Configure the premirror to be used
2407 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/npm2' % mirrordir)
Andrew Geissler82c905d2020-04-13 13:39:40 -05002408 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002409
Andrew Geissler82c905d2020-04-13 13:39:40 -05002410 # Fetch again
2411 self.assertFalse(os.path.exists(ud.localpath))
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002412 # The npm fetcher doesn't handle that the .resolved file disappears
2413 # while the fetcher object exists, which it does when we rename the
2414 # download directory to "mirror" above. Thus we need a new fetcher to go
2415 # with the now empty download directory.
2416 fetcher = bb.fetch.Fetch([url], self.d)
2417 ud = fetcher.ud[fetcher.urls[0]]
Andrew Geissler82c905d2020-04-13 13:39:40 -05002418 fetcher.download()
2419 self.assertTrue(os.path.exists(ud.localpath))
2420
2421 @skipIfNoNpm()
2422 @skipIfNoNetwork()
Andrew Geissler5199d832021-09-24 16:47:35 -05002423 def test_npm_premirrors_with_specified_filename(self):
2424 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2425 # Fetch once to get a tarball
2426 fetcher = bb.fetch.Fetch([url], self.d)
2427 ud = fetcher.ud[fetcher.urls[0]]
2428 fetcher.download()
2429 self.assertTrue(os.path.exists(ud.localpath))
2430 # Setup the mirror
2431 mirrordir = os.path.join(self.tempdir, 'mirror')
2432 bb.utils.mkdirhier(mirrordir)
2433 mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath))
2434 os.replace(ud.localpath, mirrorfilename)
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002435 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s' % mirrorfilename)
Andrew Geissler5199d832021-09-24 16:47:35 -05002436 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
2437 # Fetch again
2438 self.assertFalse(os.path.exists(ud.localpath))
2439 fetcher.download()
2440 self.assertTrue(os.path.exists(ud.localpath))
2441
2442 @skipIfNoNpm()
2443 @skipIfNoNetwork()
Andrew Geissler82c905d2020-04-13 13:39:40 -05002444 def test_npm_mirrors(self):
2445 # Fetch once to get a tarball
2446 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2447 fetcher = bb.fetch.Fetch([url], self.d)
2448 ud = fetcher.ud[fetcher.urls[0]]
2449 fetcher.download()
2450 self.assertTrue(os.path.exists(ud.localpath))
2451 # Setup the mirror
2452 mirrordir = os.path.join(self.tempdir, 'mirror')
2453 bb.utils.mkdirhier(mirrordir)
2454 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002455 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir)
Andrew Geissler82c905d2020-04-13 13:39:40 -05002456 # Update the resolved url to an invalid url
2457 with open(ud.resolvefile, 'r') as f:
2458 url = f.read()
2459 uri = URI(url)
2460 uri.path = '/invalid'
2461 with open(ud.resolvefile, 'w') as f:
2462 f.write(str(uri))
2463 # Fetch again
2464 self.assertFalse(os.path.exists(ud.localpath))
2465 fetcher.download()
2466 self.assertTrue(os.path.exists(ud.localpath))
2467
2468 @skipIfNoNpm()
2469 @skipIfNoNetwork()
2470 def test_npm_destsuffix_downloadfilename(self):
2471 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz'
2472 fetcher = bb.fetch.Fetch([url], self.d)
2473 fetcher.download()
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002474 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'foo-bar.tgz')))
Andrew Geissler82c905d2020-04-13 13:39:40 -05002475 fetcher.unpack(self.unpackdir)
2476 unpackdir = os.path.join(self.unpackdir, 'foo', 'bar')
2477 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2478
2479 def test_npm_no_network_no_tarball(self):
2480 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2481 self.d.setVar('BB_NO_NETWORK', '1')
2482 fetcher = bb.fetch.Fetch([url], self.d)
2483 with self.assertRaises(bb.fetch2.NetworkAccess):
2484 fetcher.download()
2485
2486 @skipIfNoNpm()
2487 @skipIfNoNetwork()
2488 def test_npm_no_network_with_tarball(self):
2489 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2490 # Fetch once to get a tarball
2491 fetcher = bb.fetch.Fetch([url], self.d)
2492 fetcher.download()
2493 # Disable network access
2494 self.d.setVar('BB_NO_NETWORK', '1')
2495 # Fetch again
2496 fetcher.download()
2497 fetcher.unpack(self.unpackdir)
2498 unpackdir = os.path.join(self.unpackdir, 'npm')
2499 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2500
2501 @skipIfNoNpm()
2502 @skipIfNoNetwork()
2503 def test_npm_registry_alternate(self):
Andrew Geissler5199d832021-09-24 16:47:35 -05002504 url = 'npm://skimdb.npmjs.com;package=@savoirfairelinux/node-server-example;version=1.0.0'
Andrew Geissler82c905d2020-04-13 13:39:40 -05002505 fetcher = bb.fetch.Fetch([url], self.d)
2506 fetcher.download()
2507 fetcher.unpack(self.unpackdir)
2508 unpackdir = os.path.join(self.unpackdir, 'npm')
2509 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2510
2511 @skipIfNoNpm()
2512 @skipIfNoNetwork()
2513 def test_npm_version_latest(self):
2514 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest'
2515 fetcher = bb.fetch.Fetch([url], self.d)
2516 fetcher.download()
2517 fetcher.unpack(self.unpackdir)
2518 unpackdir = os.path.join(self.unpackdir, 'npm')
2519 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2520
2521 @skipIfNoNpm()
2522 @skipIfNoNetwork()
2523 def test_npm_registry_invalid(self):
2524 url = 'npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2525 fetcher = bb.fetch.Fetch([url], self.d)
2526 with self.assertRaises(bb.fetch2.FetchError):
2527 fetcher.download()
2528
2529 @skipIfNoNpm()
2530 @skipIfNoNetwork()
2531 def test_npm_package_invalid(self):
2532 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0'
2533 fetcher = bb.fetch.Fetch([url], self.d)
2534 with self.assertRaises(bb.fetch2.FetchError):
2535 fetcher.download()
2536
2537 @skipIfNoNpm()
2538 @skipIfNoNetwork()
2539 def test_npm_version_invalid(self):
2540 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid'
2541 with self.assertRaises(bb.fetch2.ParameterError):
2542 fetcher = bb.fetch.Fetch([url], self.d)
2543
2544 @skipIfNoNpm()
2545 @skipIfNoNetwork()
2546 def test_npm_registry_none(self):
2547 url = 'npm://;package=@savoirfairelinux/node-server-example;version=1.0.0'
2548 with self.assertRaises(bb.fetch2.MalformedUrl):
2549 fetcher = bb.fetch.Fetch([url], self.d)
2550
2551 @skipIfNoNpm()
2552 @skipIfNoNetwork()
2553 def test_npm_package_none(self):
2554 url = 'npm://registry.npmjs.org;version=1.0.0'
2555 with self.assertRaises(bb.fetch2.MissingParameterError):
2556 fetcher = bb.fetch.Fetch([url], self.d)
2557
2558 @skipIfNoNpm()
2559 @skipIfNoNetwork()
2560 def test_npm_version_none(self):
2561 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example'
2562 with self.assertRaises(bb.fetch2.MissingParameterError):
2563 fetcher = bb.fetch.Fetch([url], self.d)
2564
2565 def create_shrinkwrap_file(self, data):
2566 import json
2567 datadir = os.path.join(self.tempdir, 'data')
2568 swfile = os.path.join(datadir, 'npm-shrinkwrap.json')
2569 bb.utils.mkdirhier(datadir)
2570 with open(swfile, 'w') as f:
2571 json.dump(data, f)
2572 # Also configure the S directory
2573 self.sdir = os.path.join(self.unpackdir, 'S')
2574 self.d.setVar('S', self.sdir)
2575 return swfile
2576
2577 @skipIfNoNpm()
2578 @skipIfNoNetwork()
2579 def test_npmsw(self):
2580 swfile = self.create_shrinkwrap_file({
2581 'dependencies': {
2582 'array-flatten': {
2583 'version': '1.1.1',
2584 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2585 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=',
2586 'dependencies': {
2587 'content-type': {
2588 'version': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
2589 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
2590 'dependencies': {
2591 'cookie': {
2592 'version': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09',
2593 'from': 'git+https://github.com/jshttp/cookie.git'
2594 }
2595 }
2596 }
2597 }
2598 }
2599 }
2600 })
2601 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2602 fetcher.download()
2603 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
2604 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
2605 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git')))
2606 fetcher.unpack(self.unpackdir)
2607 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'npm-shrinkwrap.json')))
2608 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json')))
2609 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json')))
2610 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json')))
2611
2612 @skipIfNoNpm()
2613 @skipIfNoNetwork()
2614 def test_npmsw_dev(self):
2615 swfile = self.create_shrinkwrap_file({
2616 'dependencies': {
2617 'array-flatten': {
2618 'version': '1.1.1',
2619 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2620 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2621 },
2622 'content-type': {
2623 'version': '1.0.4',
2624 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
2625 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
2626 'dev': True
2627 }
2628 }
2629 })
2630 # Fetch with dev disabled
2631 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2632 fetcher.download()
2633 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
2634 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
2635 # Fetch with dev enabled
2636 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d)
2637 fetcher.download()
2638 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
2639 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
2640
2641 @skipIfNoNpm()
2642 @skipIfNoNetwork()
2643 def test_npmsw_destsuffix(self):
2644 swfile = self.create_shrinkwrap_file({
2645 'dependencies': {
2646 'array-flatten': {
2647 'version': '1.1.1',
2648 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2649 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2650 }
2651 }
2652 })
2653 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d)
2654 fetcher.download()
2655 fetcher.unpack(self.unpackdir)
2656 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json')))
2657
2658 def test_npmsw_no_network_no_tarball(self):
2659 swfile = self.create_shrinkwrap_file({
2660 'dependencies': {
2661 'array-flatten': {
2662 'version': '1.1.1',
2663 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2664 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2665 }
2666 }
2667 })
2668 self.d.setVar('BB_NO_NETWORK', '1')
2669 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2670 with self.assertRaises(bb.fetch2.NetworkAccess):
2671 fetcher.download()
2672
2673 @skipIfNoNpm()
2674 @skipIfNoNetwork()
2675 def test_npmsw_no_network_with_tarball(self):
2676 # Fetch once to get a tarball
2677 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2678 fetcher.download()
2679 # Disable network access
2680 self.d.setVar('BB_NO_NETWORK', '1')
2681 # Fetch again
2682 swfile = self.create_shrinkwrap_file({
2683 'dependencies': {
2684 'array-flatten': {
2685 'version': '1.1.1',
2686 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2687 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2688 }
2689 }
2690 })
2691 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2692 fetcher.download()
2693 fetcher.unpack(self.unpackdir)
2694 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json')))
2695
2696 @skipIfNoNpm()
2697 @skipIfNoNetwork()
2698 def test_npmsw_npm_reusability(self):
2699 # Fetch once with npmsw
2700 swfile = self.create_shrinkwrap_file({
2701 'dependencies': {
2702 'array-flatten': {
2703 'version': '1.1.1',
2704 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2705 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2706 }
2707 }
2708 })
2709 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2710 fetcher.download()
2711 # Disable network access
2712 self.d.setVar('BB_NO_NETWORK', '1')
2713 # Fetch again with npm
2714 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2715 fetcher.download()
2716 fetcher.unpack(self.unpackdir)
2717 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json')))
2718
2719 @skipIfNoNpm()
2720 @skipIfNoNetwork()
2721 def test_npmsw_bad_checksum(self):
2722 # Try to fetch with bad checksum
2723 swfile = self.create_shrinkwrap_file({
2724 'dependencies': {
2725 'array-flatten': {
2726 'version': '1.1.1',
2727 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2728 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg='
2729 }
2730 }
2731 })
2732 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2733 with self.assertRaises(bb.fetch2.FetchError):
2734 fetcher.download()
2735 # Fetch correctly to get a tarball
2736 swfile = self.create_shrinkwrap_file({
2737 'dependencies': {
2738 'array-flatten': {
2739 'version': '1.1.1',
2740 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2741 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2742 }
2743 }
2744 })
2745 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2746 fetcher.download()
2747 localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')
2748 self.assertTrue(os.path.exists(localpath))
2749 # Modify the tarball
2750 bad = b'bad checksum'
2751 with open(localpath, 'wb') as f:
2752 f.write(bad)
2753 # Verify that the tarball is fetched again
2754 fetcher.download()
2755 badsum = hashlib.sha1(bad).hexdigest()
2756 self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum))
2757 self.assertTrue(os.path.exists(localpath))
2758
2759 @skipIfNoNpm()
2760 @skipIfNoNetwork()
2761 def test_npmsw_premirrors(self):
2762 # Fetch once to get a tarball
2763 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2764 ud = fetcher.ud[fetcher.urls[0]]
2765 fetcher.download()
2766 self.assertTrue(os.path.exists(ud.localpath))
2767 # Setup the mirror
2768 mirrordir = os.path.join(self.tempdir, 'mirror')
2769 bb.utils.mkdirhier(mirrordir)
2770 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002771 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir)
Andrew Geissler82c905d2020-04-13 13:39:40 -05002772 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
2773 # Fetch again
2774 self.assertFalse(os.path.exists(ud.localpath))
2775 swfile = self.create_shrinkwrap_file({
2776 'dependencies': {
2777 'array-flatten': {
2778 'version': '1.1.1',
2779 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2780 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2781 }
2782 }
2783 })
2784 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2785 fetcher.download()
2786 self.assertTrue(os.path.exists(ud.localpath))
2787
2788 @skipIfNoNpm()
2789 @skipIfNoNetwork()
2790 def test_npmsw_mirrors(self):
2791 # Fetch once to get a tarball
2792 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2793 ud = fetcher.ud[fetcher.urls[0]]
2794 fetcher.download()
2795 self.assertTrue(os.path.exists(ud.localpath))
2796 # Setup the mirror
2797 mirrordir = os.path.join(self.tempdir, 'mirror')
2798 bb.utils.mkdirhier(mirrordir)
2799 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002800 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir)
Andrew Geissler82c905d2020-04-13 13:39:40 -05002801 # Fetch again with invalid url
2802 self.assertFalse(os.path.exists(ud.localpath))
2803 swfile = self.create_shrinkwrap_file({
2804 'dependencies': {
2805 'array-flatten': {
2806 'version': '1.1.1',
2807 'resolved': 'https://invalid',
2808 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2809 }
2810 }
2811 })
2812 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2813 fetcher.download()
2814 self.assertTrue(os.path.exists(ud.localpath))
Andrew Geisslerc926e172021-05-07 16:11:35 -05002815
2816class GitSharedTest(FetcherTest):
2817 def setUp(self):
2818 super(GitSharedTest, self).setUp()
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002819 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master"
Andrew Geisslerc926e172021-05-07 16:11:35 -05002820 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
Andrew Geissler7e0e3c02022-02-25 20:34:39 +00002821 self.d.setVar("__BBSEENSRCREV", "1")
Andrew Geisslerc926e172021-05-07 16:11:35 -05002822
2823 @skipIfNoNetwork()
2824 def test_shared_unpack(self):
2825 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
2826
2827 fetcher.download()
2828 fetcher.unpack(self.unpackdir)
2829 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
2830 self.assertTrue(os.path.exists(alt))
2831
2832 @skipIfNoNetwork()
2833 def test_noshared_unpack(self):
2834 self.d.setVar('BB_GIT_NOSHARED', '1')
2835 self.unpackdir += '_noshared'
2836 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
2837
2838 fetcher.download()
2839 fetcher.unpack(self.unpackdir)
2840 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
2841 self.assertFalse(os.path.exists(alt))
Andrew Geissler78b72792022-06-14 06:47:25 -05002842
2843
2844class FetchPremirroronlyLocalTest(FetcherTest):
2845
2846 def setUp(self):
2847 super(FetchPremirroronlyLocalTest, self).setUp()
2848 self.mirrordir = os.path.join(self.tempdir, "mirrors")
2849 os.mkdir(self.mirrordir)
2850 self.reponame = "bitbake"
2851 self.gitdir = os.path.join(self.tempdir, "git", self.reponame)
2852 self.recipe_url = "git://git.fake.repo/bitbake"
2853 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
2854 self.d.setVar("BB_NO_NETWORK", "1")
2855 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
2856
2857 def make_git_repo(self):
2858 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz"
2859 recipeurl = "git:/git.fake.repo/bitbake"
2860 os.makedirs(self.gitdir)
2861 self.git("init", self.gitdir)
2862 for i in range(0):
2863 self.git_new_commit()
2864 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir)
2865
2866 def git_new_commit(self):
2867 import random
2868 testfilename = "bibake-fetch.test"
2869 os.unlink(os.path.join(self.mirrordir, self.mirrorname))
2870 with open(os.path.join(self.gitdir, testfilename), "w") as testfile:
2871 testfile.write("Useless random data {}".format(random.random()))
2872 self.git("add {}".format(testfilename), self.gitdir)
2873 self.git("commit -a -m \"This random commit {}. I'm useless.\"".format(random.random()), self.gitdir)
2874 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir)
2875 return self.git("rev-parse HEAD", self.gitdir).strip()
2876
2877 def test_mirror_commit_nonexistent(self):
2878 self.make_git_repo()
2879 self.d.setVar("SRCREV", "0"*40)
2880 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
2881 with self.assertRaises(bb.fetch2.NetworkAccess):
2882 fetcher.download()
2883
2884 def test_mirror_commit_exists(self):
2885 self.make_git_repo()
2886 self.d.setVar("SRCREV", self.git_new_commit())
2887 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
2888 fetcher.download()
2889 fetcher.unpack(self.unpackdir)
2890
2891 def test_mirror_tarball_nonexistent(self):
2892 self.d.setVar("SRCREV", "0"*40)
2893 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
2894 with self.assertRaises(bb.fetch2.NetworkAccess):
2895 fetcher.download()
2896
2897class FetchPremirroronlyNetworkTest(FetcherTest):
2898
2899 def setUp(self):
2900 super(FetchPremirroronlyNetworkTest, self).setUp()
2901 self.mirrordir = os.path.join(self.tempdir, "mirrors")
2902 os.mkdir(self.mirrordir)
2903 self.reponame = "fstests"
2904 self.clonedir = os.path.join(self.tempdir, "git")
2905 self.gitdir = os.path.join(self.tempdir, "git", "{}.git".format(self.reponame))
2906 self.recipe_url = "git://git.yoctoproject.org/fstests"
2907 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
2908 self.d.setVar("BB_NO_NETWORK", "0")
2909 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
2910
2911 def make_git_repo(self):
2912 import shutil
2913 self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz"
2914 os.makedirs(self.clonedir)
2915 self.git("clone --bare --shallow-since=\"01.01.2013\" {}".format(self.recipe_url), self.clonedir)
2916 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir)
2917 shutil.rmtree(self.clonedir)
2918
2919 @skipIfNoNetwork()
2920 def test_mirror_tarball_updated(self):
2921 self.make_git_repo()
2922 ## Upstream commit is in the mirror
2923 self.d.setVar("SRCREV", "49d65d53c2bf558ae6e9185af0f3af7b79d255ec")
2924 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
2925 fetcher.download()
2926
2927 @skipIfNoNetwork()
2928 def test_mirror_tarball_outdated(self):
2929 self.make_git_repo()
2930 ## Upstream commit not in the mirror
2931 self.d.setVar("SRCREV", "15413486df1f5a5b5af699b6f3ba5f0984e52a9f")
2932 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
2933 with self.assertRaises(bb.fetch2.NetworkAccess):
2934 fetcher.download()
Andrew Geissler615f2f12022-07-15 14:00:58 -05002935
2936class FetchPremirroronlyBrokenTarball(FetcherTest):
2937
2938 def setUp(self):
2939 super(FetchPremirroronlyBrokenTarball, self).setUp()
2940 self.mirrordir = os.path.join(self.tempdir, "mirrors")
2941 os.mkdir(self.mirrordir)
2942 self.reponame = "bitbake"
2943 self.gitdir = os.path.join(self.tempdir, "git", self.reponame)
2944 self.recipe_url = "git://git.fake.repo/bitbake"
2945 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
2946 self.d.setVar("BB_NO_NETWORK", "1")
2947 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
2948 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz"
2949 with open(os.path.join(self.mirrordir, self.mirrorname), 'w') as targz:
2950 targz.write("This is not tar.gz file!")
2951
2952 def test_mirror_broken_download(self):
2953 import sys
2954 self.d.setVar("SRCREV", "0"*40)
2955 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
2956 with self.assertRaises(bb.fetch2.FetchError):
2957 fetcher.download()
2958 stdout = sys.stdout.getvalue()
2959 self.assertFalse(" not a git repository (or any parent up to mount point /)" in stdout)