blob: 5a4db9ca4c0b56a287480a73cbf19b0f5c16086d [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# BitBake Tests for the Fetcher (fetch2/)
3#
4# Copyright (C) 2012 Richard Purdie
5#
Brad Bishopc342db32019-05-15 21:57:59 -04006# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007#
8
9import unittest
Brad Bishop316dfdd2018-06-25 12:45:53 -040010import hashlib
Patrick Williamsc124f4f2015-09-15 14:41:29 -050011import tempfile
Patrick Williamsc0f7c042017-02-23 20:41:17 -060012import collections
Patrick Williamsc124f4f2015-09-15 14:41:29 -050013import os
14from bb.fetch2 import URI
15from bb.fetch2 import FetchMethod
16import bb
Andrew Geissler82c905d2020-04-13 13:39:40 -050017from bb.tests.support.httpserver import HTTPService
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018
Brad Bishopd7bf8c12018-02-25 22:55:05 -050019def skipIfNoNetwork():
20 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
21 return unittest.skip("Network tests being skipped")
22 return lambda f: f
23
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024class URITest(unittest.TestCase):
25 test_uris = {
26 "http://www.google.com/index.html" : {
27 'uri': 'http://www.google.com/index.html',
28 'scheme': 'http',
29 'hostname': 'www.google.com',
30 'port': None,
31 'hostport': 'www.google.com',
32 'path': '/index.html',
33 'userinfo': '',
34 'username': '',
35 'password': '',
36 'params': {},
37 'query': {},
38 'relative': False
39 },
40 "http://www.google.com/index.html;param1=value1" : {
41 'uri': 'http://www.google.com/index.html;param1=value1',
42 'scheme': 'http',
43 'hostname': 'www.google.com',
44 'port': None,
45 'hostport': 'www.google.com',
46 'path': '/index.html',
47 'userinfo': '',
48 'username': '',
49 'password': '',
50 'params': {
51 'param1': 'value1'
52 },
53 'query': {},
54 'relative': False
55 },
56 "http://www.example.org/index.html?param1=value1" : {
57 'uri': 'http://www.example.org/index.html?param1=value1',
58 'scheme': 'http',
59 'hostname': 'www.example.org',
60 'port': None,
61 'hostport': 'www.example.org',
62 'path': '/index.html',
63 'userinfo': '',
64 'username': '',
65 'password': '',
66 'params': {},
67 'query': {
68 'param1': 'value1'
69 },
70 'relative': False
71 },
72 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : {
73 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2',
74 'scheme': 'http',
75 'hostname': 'www.example.org',
76 'port': None,
77 'hostport': 'www.example.org',
78 'path': '/index.html',
79 'userinfo': '',
80 'username': '',
81 'password': '',
82 'params': {
83 'param2': 'value2'
84 },
85 'query': {
86 'qparam1': 'qvalue1'
87 },
88 'relative': False
89 },
90 "http://www.example.com:8080/index.html" : {
91 'uri': 'http://www.example.com:8080/index.html',
92 'scheme': 'http',
93 'hostname': 'www.example.com',
94 'port': 8080,
95 'hostport': 'www.example.com:8080',
96 'path': '/index.html',
97 'userinfo': '',
98 'username': '',
99 'password': '',
100 'params': {},
101 'query': {},
102 'relative': False
103 },
104 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : {
105 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg',
106 'scheme': 'cvs',
107 'hostname': 'cvs.handhelds.org',
108 'port': None,
109 'hostport': 'cvs.handhelds.org',
110 'path': '/cvs',
111 'userinfo': 'anoncvs',
112 'username': 'anoncvs',
113 'password': '',
114 'params': {
115 'module': 'familiar/dist/ipkg'
116 },
117 'query': {},
118 'relative': False
119 },
120 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": {
121 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg',
122 'scheme': 'cvs',
123 'hostname': 'cvs.handhelds.org',
124 'port': None,
125 'hostport': 'cvs.handhelds.org',
126 'path': '/cvs',
127 'userinfo': 'anoncvs:anonymous',
128 'username': 'anoncvs',
129 'password': 'anonymous',
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600130 'params': collections.OrderedDict([
131 ('tag', 'V0-99-81'),
132 ('module', 'familiar/dist/ipkg')
133 ]),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500134 'query': {},
135 'relative': False
136 },
137 "file://example.diff": { # NOTE: Not RFC compliant!
138 'uri': 'file:example.diff',
139 'scheme': 'file',
140 'hostname': '',
141 'port': None,
142 'hostport': '',
143 'path': 'example.diff',
144 'userinfo': '',
145 'username': '',
146 'password': '',
147 'params': {},
148 'query': {},
149 'relative': True
150 },
151 "file:example.diff": { # NOTE: RFC compliant version of the former
152 'uri': 'file:example.diff',
153 'scheme': 'file',
154 'hostname': '',
155 'port': None,
156 'hostport': '',
157 'path': 'example.diff',
158 'userinfo': '',
159 'userinfo': '',
160 'username': '',
161 'password': '',
162 'params': {},
163 'query': {},
164 'relative': True
165 },
166 "file:///tmp/example.diff": {
167 'uri': 'file:///tmp/example.diff',
168 'scheme': 'file',
169 'hostname': '',
170 'port': None,
171 'hostport': '',
172 'path': '/tmp/example.diff',
173 'userinfo': '',
174 'userinfo': '',
175 'username': '',
176 'password': '',
177 'params': {},
178 'query': {},
179 'relative': False
180 },
181 "git:///path/example.git": {
182 'uri': 'git:///path/example.git',
183 'scheme': 'git',
184 'hostname': '',
185 'port': None,
186 'hostport': '',
187 'path': '/path/example.git',
188 'userinfo': '',
189 'userinfo': '',
190 'username': '',
191 'password': '',
192 'params': {},
193 'query': {},
194 'relative': False
195 },
196 "git:path/example.git": {
197 'uri': 'git:path/example.git',
198 'scheme': 'git',
199 'hostname': '',
200 'port': None,
201 'hostport': '',
202 'path': 'path/example.git',
203 'userinfo': '',
204 'userinfo': '',
205 'username': '',
206 'password': '',
207 'params': {},
208 'query': {},
209 'relative': True
210 },
211 "git://example.net/path/example.git": {
212 'uri': 'git://example.net/path/example.git',
213 'scheme': 'git',
214 'hostname': 'example.net',
215 'port': None,
216 'hostport': 'example.net',
217 'path': '/path/example.git',
218 'userinfo': '',
219 'userinfo': '',
220 'username': '',
221 'password': '',
222 'params': {},
223 'query': {},
224 'relative': False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500225 },
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500226 "git://tfs-example.org:22/tfs/example%20path/example.git": {
227 'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git',
228 'scheme': 'git',
229 'hostname': 'tfs-example.org',
230 'port': 22,
231 'hostport': 'tfs-example.org:22',
232 'path': '/tfs/example path/example.git',
233 'userinfo': '',
234 'userinfo': '',
235 'username': '',
236 'password': '',
237 'params': {},
238 'query': {},
239 'relative': False
240 },
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500241 "http://somesite.net;someparam=1": {
242 'uri': 'http://somesite.net;someparam=1',
243 'scheme': 'http',
244 'hostname': 'somesite.net',
245 'port': None,
246 'hostport': 'somesite.net',
247 'path': '',
248 'userinfo': '',
249 'userinfo': '',
250 'username': '',
251 'password': '',
252 'params': {"someparam" : "1"},
253 'query': {},
254 'relative': False
255 },
256 "file://somelocation;someparam=1": {
257 'uri': 'file:somelocation;someparam=1',
258 'scheme': 'file',
259 'hostname': '',
260 'port': None,
261 'hostport': '',
262 'path': 'somelocation',
263 'userinfo': '',
264 'userinfo': '',
265 'username': '',
266 'password': '',
267 'params': {"someparam" : "1"},
268 'query': {},
269 'relative': True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500270 }
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500271
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500272 }
273
274 def test_uri(self):
275 for test_uri, ref in self.test_uris.items():
276 uri = URI(test_uri)
277
278 self.assertEqual(str(uri), ref['uri'])
279
280 # expected attributes
281 self.assertEqual(uri.scheme, ref['scheme'])
282
283 self.assertEqual(uri.userinfo, ref['userinfo'])
284 self.assertEqual(uri.username, ref['username'])
285 self.assertEqual(uri.password, ref['password'])
286
287 self.assertEqual(uri.hostname, ref['hostname'])
288 self.assertEqual(uri.port, ref['port'])
289 self.assertEqual(uri.hostport, ref['hostport'])
290
291 self.assertEqual(uri.path, ref['path'])
292 self.assertEqual(uri.params, ref['params'])
293
294 self.assertEqual(uri.relative, ref['relative'])
295
296 def test_dict(self):
297 for test in self.test_uris.values():
298 uri = URI()
299
300 self.assertEqual(uri.scheme, '')
301 self.assertEqual(uri.userinfo, '')
302 self.assertEqual(uri.username, '')
303 self.assertEqual(uri.password, '')
304 self.assertEqual(uri.hostname, '')
305 self.assertEqual(uri.port, None)
306 self.assertEqual(uri.path, '')
307 self.assertEqual(uri.params, {})
308
309
310 uri.scheme = test['scheme']
311 self.assertEqual(uri.scheme, test['scheme'])
312
313 uri.userinfo = test['userinfo']
314 self.assertEqual(uri.userinfo, test['userinfo'])
315 self.assertEqual(uri.username, test['username'])
316 self.assertEqual(uri.password, test['password'])
317
318 # make sure changing the values doesn't do anything unexpected
319 uri.username = 'changeme'
320 self.assertEqual(uri.username, 'changeme')
321 self.assertEqual(uri.password, test['password'])
322 uri.password = 'insecure'
323 self.assertEqual(uri.username, 'changeme')
324 self.assertEqual(uri.password, 'insecure')
325
326 # reset back after our trickery
327 uri.userinfo = test['userinfo']
328 self.assertEqual(uri.userinfo, test['userinfo'])
329 self.assertEqual(uri.username, test['username'])
330 self.assertEqual(uri.password, test['password'])
331
332 uri.hostname = test['hostname']
333 self.assertEqual(uri.hostname, test['hostname'])
334 self.assertEqual(uri.hostport, test['hostname'])
335
336 uri.port = test['port']
337 self.assertEqual(uri.port, test['port'])
338 self.assertEqual(uri.hostport, test['hostport'])
339
340 uri.path = test['path']
341 self.assertEqual(uri.path, test['path'])
342
343 uri.params = test['params']
344 self.assertEqual(uri.params, test['params'])
345
346 uri.query = test['query']
347 self.assertEqual(uri.query, test['query'])
348
349 self.assertEqual(str(uri), test['uri'])
350
351 uri.params = {}
352 self.assertEqual(uri.params, {})
353 self.assertEqual(str(uri), (str(uri).split(";"))[0])
354
355class FetcherTest(unittest.TestCase):
356
357 def setUp(self):
358 self.origdir = os.getcwd()
359 self.d = bb.data.init()
360 self.tempdir = tempfile.mkdtemp()
361 self.dldir = os.path.join(self.tempdir, "download")
362 os.mkdir(self.dldir)
363 self.d.setVar("DL_DIR", self.dldir)
364 self.unpackdir = os.path.join(self.tempdir, "unpacked")
365 os.mkdir(self.unpackdir)
366 persistdir = os.path.join(self.tempdir, "persistdata")
367 self.d.setVar("PERSISTENT_DIR", persistdir)
368
369 def tearDown(self):
370 os.chdir(self.origdir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600371 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
372 print("Not cleaning up %s. Please remove manually." % self.tempdir)
373 else:
374 bb.utils.prunedir(self.tempdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500375
376class MirrorUriTest(FetcherTest):
377
378 replaceuris = {
379 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/")
380 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
381 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
382 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
383 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
384 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
385 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http")
386 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
387 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake")
388 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890",
389 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache")
390 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
391 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/")
392 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
393 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3")
394 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
395 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz")
396 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
397 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist")
398 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2",
399 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/")
400 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2",
401 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
402 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
403 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
404 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
405 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http")
406 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800407 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org")
408 : "http://somewhere2.org/somefile_1.2.3.tar.gz",
409 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/")
410 : "http://somewhere2.org/somefile_1.2.3.tar.gz",
411 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http")
412 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500413
414 #Renaming files doesn't work
415 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
416 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
417 }
418
419 mirrorvar = "http://.*/.* file:///somepath/downloads/ \n" \
420 "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n" \
421 "https://.*/.* file:///someotherpath/downloads/ \n" \
422 "http://.*/.* file:///someotherpath/downloads/ \n"
423
424 def test_urireplace(self):
425 for k, v in self.replaceuris.items():
426 ud = bb.fetch.FetchData(k[0], self.d)
427 ud.setup_localpath(self.d)
428 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2]))
429 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d)
430 self.assertEqual([v], newuris)
431
432 def test_urilist1(self):
433 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
434 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
435 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
436 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
437
438 def test_urilist2(self):
439 # Catch https:// -> files:// bug
440 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
441 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
442 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
443 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
444
445 def test_mirror_of_mirror(self):
446 # Test if mirror of a mirror works
447 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/ \n"
448 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/ \n"
449 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
450 mirrors = bb.fetch2.mirror_from_string(mirrorvar)
451 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
452 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz',
453 'file:///someotherpath/downloads/bitbake-1.0.tar.gz',
454 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz',
455 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz'])
456
Patrick Williamsd7e96312015-09-22 08:09:05 -0500457 recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ \n" \
458 "https://.*/[^/]* https://BBBB/B/B/B/ \n"
459
460 def test_recursive(self):
461 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
462 mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar)
463 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
464 self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz',
465 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz',
466 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500467
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800468
469class GitDownloadDirectoryNamingTest(FetcherTest):
470 def setUp(self):
471 super(GitDownloadDirectoryNamingTest, self).setUp()
472 self.recipe_url = "git://git.openembedded.org/bitbake"
473 self.recipe_dir = "git.openembedded.org.bitbake"
474 self.mirror_url = "git://github.com/openembedded/bitbake.git"
475 self.mirror_dir = "github.com.openembedded.bitbake.git"
476
477 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
478
479 def setup_mirror_rewrite(self):
480 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
481
482 @skipIfNoNetwork()
483 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self):
484 self.setup_mirror_rewrite()
485 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
486
487 fetcher.download()
488
489 dir = os.listdir(self.dldir + "/git2")
490 self.assertIn(self.recipe_dir, dir)
491
492 @skipIfNoNetwork()
493 def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self):
494 self.setup_mirror_rewrite()
495 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
496
497 fetcher.download()
498
499 dir = os.listdir(self.dldir + "/git2")
500 self.assertIn(self.mirror_dir, dir)
501 self.assertIn(self.recipe_dir, dir)
502
503 @skipIfNoNetwork()
504 def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self):
505 self.setup_mirror_rewrite()
506 fetcher = bb.fetch.Fetch([self.mirror_url], self.d)
507 fetcher.download()
508 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
509
510 fetcher.download()
511
512 dir = os.listdir(self.dldir + "/git2")
513 self.assertIn(self.mirror_dir, dir)
514 self.assertIn(self.recipe_dir, dir)
515
516
517class TarballNamingTest(FetcherTest):
518 def setUp(self):
519 super(TarballNamingTest, self).setUp()
520 self.recipe_url = "git://git.openembedded.org/bitbake"
521 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
522 self.mirror_url = "git://github.com/openembedded/bitbake.git"
523 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz"
524
525 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
526 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
527
528 def setup_mirror_rewrite(self):
529 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
530
531 @skipIfNoNetwork()
532 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self):
533 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
534
535 fetcher.download()
536
537 dir = os.listdir(self.dldir)
538 self.assertIn(self.recipe_tarball, dir)
539
540 @skipIfNoNetwork()
541 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
542 self.setup_mirror_rewrite()
543 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
544
545 fetcher.download()
546
547 dir = os.listdir(self.dldir)
548 self.assertIn(self.mirror_tarball, dir)
549
550
551class GitShallowTarballNamingTest(FetcherTest):
552 def setUp(self):
553 super(GitShallowTarballNamingTest, self).setUp()
554 self.recipe_url = "git://git.openembedded.org/bitbake"
555 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz"
556 self.mirror_url = "git://github.com/openembedded/bitbake.git"
557 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz"
558
559 self.d.setVar('BB_GIT_SHALLOW', '1')
560 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
561 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
562
563 def setup_mirror_rewrite(self):
564 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
565
566 @skipIfNoNetwork()
567 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self):
568 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
569
570 fetcher.download()
571
572 dir = os.listdir(self.dldir)
573 self.assertIn(self.recipe_tarball, dir)
574
575 @skipIfNoNetwork()
576 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
577 self.setup_mirror_rewrite()
578 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
579
580 fetcher.download()
581
582 dir = os.listdir(self.dldir)
583 self.assertIn(self.mirror_tarball, dir)
584
585
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500586class FetcherLocalTest(FetcherTest):
587 def setUp(self):
588 def touch(fn):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600589 with open(fn, 'a'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500590 os.utime(fn, None)
591
592 super(FetcherLocalTest, self).setUp()
593 self.localsrcdir = os.path.join(self.tempdir, 'localsrc')
594 os.makedirs(self.localsrcdir)
595 touch(os.path.join(self.localsrcdir, 'a'))
596 touch(os.path.join(self.localsrcdir, 'b'))
597 os.makedirs(os.path.join(self.localsrcdir, 'dir'))
598 touch(os.path.join(self.localsrcdir, 'dir', 'c'))
599 touch(os.path.join(self.localsrcdir, 'dir', 'd'))
600 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir'))
601 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e'))
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500602 touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500603 self.d.setVar("FILESPATH", self.localsrcdir)
604
605 def fetchUnpack(self, uris):
606 fetcher = bb.fetch.Fetch(uris, self.d)
607 fetcher.download()
608 fetcher.unpack(self.unpackdir)
609 flst = []
610 for root, dirs, files in os.walk(self.unpackdir):
611 for f in files:
612 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir))
613 flst.sort()
614 return flst
615
616 def test_local(self):
617 tree = self.fetchUnpack(['file://a', 'file://dir/c'])
618 self.assertEqual(tree, ['a', 'dir/c'])
619
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500620 def test_local_backslash(self):
621 tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device'])
622 self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device'])
623
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500624 def test_local_wildcard(self):
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500625 with self.assertRaises(bb.fetch2.ParameterError):
626 tree = self.fetchUnpack(['file://a', 'file://dir/*'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500627
628 def test_local_dir(self):
629 tree = self.fetchUnpack(['file://a', 'file://dir'])
630 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e'])
631
632 def test_local_subdir(self):
633 tree = self.fetchUnpack(['file://dir/subdir'])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500634 self.assertEqual(tree, ['dir/subdir/e'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500635
636 def test_local_subdir_file(self):
637 tree = self.fetchUnpack(['file://dir/subdir/e'])
638 self.assertEqual(tree, ['dir/subdir/e'])
639
640 def test_local_subdirparam(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500641 tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo'])
642 self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500643
644 def test_local_deepsubdirparam(self):
645 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar'])
646 self.assertEqual(tree, ['bar/dir/subdir/e'])
647
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600648 def test_local_absolutedir(self):
649 # Unpacking to an absolute path that is a subdirectory of the root
650 # should work
651 tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')])
652
653 # Unpacking to an absolute path outside of the root should fail
654 with self.assertRaises(bb.fetch2.UnpackError):
655 self.fetchUnpack(['file://a;subdir=/bin/sh'])
656
Brad Bishop316dfdd2018-06-25 12:45:53 -0400657class FetcherNoNetworkTest(FetcherTest):
658 def setUp(self):
659 super().setUp()
660 # all test cases are based on not having network
661 self.d.setVar("BB_NO_NETWORK", "1")
662
663 def test_missing(self):
664 string = "this is a test file\n".encode("utf-8")
665 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
666 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
667
668 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
669 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
670 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
671 with self.assertRaises(bb.fetch2.NetworkAccess):
672 fetcher.download()
673
674 def test_valid_missing_donestamp(self):
675 # create the file in the download directory with correct hash
676 string = "this is a test file\n".encode("utf-8")
677 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f:
678 f.write(string)
679
680 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
681 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
682
683 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
684 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
685 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
686 fetcher.download()
687 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
688
689 def test_invalid_missing_donestamp(self):
690 # create an invalid file in the download directory with incorrect hash
691 string = "this is a test file\n".encode("utf-8")
692 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
693 pass
694
695 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
696 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
697
698 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
699 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
700 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
701 with self.assertRaises(bb.fetch2.NetworkAccess):
702 fetcher.download()
703 # the existing file should not exist or should have be moved to "bad-checksum"
704 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
705
706 def test_nochecksums_missing(self):
707 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
708 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
709 # ssh fetch does not support checksums
710 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
711 # attempts to download with missing donestamp
712 with self.assertRaises(bb.fetch2.NetworkAccess):
713 fetcher.download()
714
715 def test_nochecksums_missing_donestamp(self):
716 # create a file in the download directory
717 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
718 pass
719
720 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
721 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
722 # ssh fetch does not support checksums
723 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
724 # attempts to download with missing donestamp
725 with self.assertRaises(bb.fetch2.NetworkAccess):
726 fetcher.download()
727
728 def test_nochecksums_has_donestamp(self):
729 # create a file in the download directory with the donestamp
730 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
731 pass
732 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
733 pass
734
735 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
736 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
737 # ssh fetch does not support checksums
738 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
739 # should not fetch
740 fetcher.download()
741 # both files should still exist
742 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
743 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
744
745 def test_nochecksums_missing_has_donestamp(self):
746 # create a file in the download directory with the donestamp
747 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
748 pass
749
750 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
751 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
752 # ssh fetch does not support checksums
753 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
754 with self.assertRaises(bb.fetch2.NetworkAccess):
755 fetcher.download()
756 # both files should still exist
757 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
758 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
759
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500760class FetcherNetworkTest(FetcherTest):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500761 @skipIfNoNetwork()
762 def test_fetch(self):
763 fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
764 fetcher.download()
765 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
766 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892)
767 self.d.setVar("BB_NO_NETWORK", "1")
768 fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
769 fetcher.download()
770 fetcher.unpack(self.unpackdir)
771 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9)
772 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500773
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500774 @skipIfNoNetwork()
775 def test_fetch_mirror(self):
776 self.d.setVar("MIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
777 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
778 fetcher.download()
779 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
780
781 @skipIfNoNetwork()
782 def test_fetch_mirror_of_mirror(self):
783 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ \n http://invalid2.yoctoproject.org/.* http://downloads.yoctoproject.org/releases/bitbake")
784 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
785 fetcher.download()
786 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
787
788 @skipIfNoNetwork()
789 def test_fetch_file_mirror_of_mirror(self):
790 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ \n file:///some1where/.* file://some2where/ \n file://some2where/.* http://downloads.yoctoproject.org/releases/bitbake")
791 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
792 os.mkdir(self.dldir + "/some2where")
793 fetcher.download()
794 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
795
796 @skipIfNoNetwork()
797 def test_fetch_premirror(self):
798 self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
799 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
800 fetcher.download()
801 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
802
803 @skipIfNoNetwork()
804 def gitfetcher(self, url1, url2):
805 def checkrevision(self, fetcher):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500806 fetcher.unpack(self.unpackdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500807 revision = bb.process.run("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git")[0].strip()
808 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500809
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500810 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
811 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
812 fetcher = bb.fetch.Fetch([url1], self.d)
813 fetcher.download()
814 checkrevision(self, fetcher)
815 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works
816 bb.utils.prunedir(self.dldir + "/git2/")
817 bb.utils.prunedir(self.unpackdir)
818 self.d.setVar("BB_NO_NETWORK", "1")
819 fetcher = bb.fetch.Fetch([url2], self.d)
820 fetcher.download()
821 checkrevision(self, fetcher)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500822
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500823 @skipIfNoNetwork()
824 def test_gitfetch(self):
825 url1 = url2 = "git://git.openembedded.org/bitbake"
826 self.gitfetcher(url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500827
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500828 @skipIfNoNetwork()
829 def test_gitfetch_goodsrcrev(self):
830 # SRCREV is set but matches rev= parameter
831 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
832 self.gitfetcher(url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500833
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500834 @skipIfNoNetwork()
835 def test_gitfetch_badsrcrev(self):
836 # SRCREV is set but does not match rev= parameter
837 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5"
838 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500839
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500840 @skipIfNoNetwork()
841 def test_gitfetch_tagandrev(self):
842 # SRCREV is set but does not match rev= parameter
843 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
844 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500845
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500846 @skipIfNoNetwork()
847 def test_gitfetch_localusehead(self):
848 # Create dummy local Git repo
849 src_dir = tempfile.mkdtemp(dir=self.tempdir,
850 prefix='gitfetch_localusehead_')
851 src_dir = os.path.abspath(src_dir)
852 bb.process.run("git init", cwd=src_dir)
853 bb.process.run("git commit --allow-empty -m'Dummy commit'",
854 cwd=src_dir)
855 # Use other branch than master
856 bb.process.run("git checkout -b my-devel", cwd=src_dir)
857 bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
858 cwd=src_dir)
859 stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
860 orig_rev = stdout[0].strip()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500861
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500862 # Fetch and check revision
863 self.d.setVar("SRCREV", "AUTOINC")
864 url = "git://" + src_dir + ";protocol=file;usehead=1"
865 fetcher = bb.fetch.Fetch([url], self.d)
866 fetcher.download()
867 fetcher.unpack(self.unpackdir)
868 stdout = bb.process.run("git rev-parse HEAD",
869 cwd=os.path.join(self.unpackdir, 'git'))
870 unpack_rev = stdout[0].strip()
871 self.assertEqual(orig_rev, unpack_rev)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500872
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500873 @skipIfNoNetwork()
874 def test_gitfetch_remoteusehead(self):
875 url = "git://git.openembedded.org/bitbake;usehead=1"
876 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500877
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500878 @skipIfNoNetwork()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800879 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self):
880 recipeurl = "git://git.openembedded.org/bitbake"
881 mirrorurl = "git://someserver.org/bitbake"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500882 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800883 self.gitfetcher(recipeurl, mirrorurl)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500884
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500885 @skipIfNoNetwork()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800886 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self):
887 recipeurl = "git://someserver.org/bitbake"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500888 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800889 self.gitfetcher(recipeurl, recipeurl)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500890
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500891 @skipIfNoNetwork()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800892 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500893 realurl = "git://git.openembedded.org/bitbake"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800894 recipeurl = "git://someserver.org/bitbake"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500895 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git")
896 os.chdir(self.tempdir)
897 bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800898 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (recipeurl, self.sourcedir))
899 self.gitfetcher(recipeurl, recipeurl)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600900
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500901 @skipIfNoNetwork()
902 def test_git_submodule(self):
Brad Bishopf8caae32019-03-25 13:13:56 -0400903 # URL with ssh submodules
904 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7"
905 # Original URL (comment this if you have ssh access to git.yoctoproject.org)
906 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee"
907 fetcher = bb.fetch.Fetch([url], self.d)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500908 fetcher.download()
909 # Previous cwd has been deleted
910 os.chdir(os.path.dirname(self.unpackdir))
911 fetcher.unpack(self.unpackdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500912
Brad Bishopf8caae32019-03-25 13:13:56 -0400913 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
914 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing')
915 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing')
916 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present')
917
918 # Only when we're running the extended test with a submodule's submodule, can we check this.
919 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
920 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
921
Brad Bishop96ff1982019-08-19 13:50:42 -0400922 @skipIfNoNetwork()
Brad Bishopf8caae32019-03-25 13:13:56 -0400923 def test_git_submodule_dbus_broker(self):
924 # The following external repositories have show failures in fetch and unpack operations
925 # We want to avoid regressions!
926 url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2"
927 fetcher = bb.fetch.Fetch([url], self.d)
928 fetcher.download()
929 # Previous cwd has been deleted
930 os.chdir(os.path.dirname(self.unpackdir))
931 fetcher.unpack(self.unpackdir)
932
933 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
934 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"')
935 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"')
936 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"')
937 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"')
938 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"')
939
Brad Bishop96ff1982019-08-19 13:50:42 -0400940 @skipIfNoNetwork()
Brad Bishopf8caae32019-03-25 13:13:56 -0400941 def test_git_submodule_CLI11(self):
942 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
943 fetcher = bb.fetch.Fetch([url], self.d)
944 fetcher.download()
945 # Previous cwd has been deleted
946 os.chdir(os.path.dirname(self.unpackdir))
947 fetcher.unpack(self.unpackdir)
948
949 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
950 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
951 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
952 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
953
Brad Bishop96ff1982019-08-19 13:50:42 -0400954 @skipIfNoNetwork()
Brad Bishop19323692019-04-05 15:28:33 -0400955 def test_git_submodule_update_CLI11(self):
956 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
957 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714"
958 fetcher = bb.fetch.Fetch([url], self.d)
959 fetcher.download()
960
961 # CLI11 that pulls in a newer nlohmann-json
962 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca"
963 fetcher = bb.fetch.Fetch([url], self.d)
964 fetcher.download()
965 # Previous cwd has been deleted
966 os.chdir(os.path.dirname(self.unpackdir))
967 fetcher.unpack(self.unpackdir)
968
969 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
970 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
971 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
972 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
973
Brad Bishop96ff1982019-08-19 13:50:42 -0400974 @skipIfNoNetwork()
Brad Bishopf8caae32019-03-25 13:13:56 -0400975 def test_git_submodule_aktualizr(self):
976 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
977 fetcher = bb.fetch.Fetch([url], self.d)
978 fetcher.download()
979 # Previous cwd has been deleted
980 os.chdir(os.path.dirname(self.unpackdir))
981 fetcher.unpack(self.unpackdir)
982
983 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
984 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"')
985 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"')
986 self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly")
987 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"')
988 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"')
989 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500990
Brad Bishop96ff1982019-08-19 13:50:42 -0400991 @skipIfNoNetwork()
Brad Bishop393846f2019-05-20 12:24:11 -0400992 def test_git_submodule_iotedge(self):
993 """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """
994
995 # This repository also has submodules where the module (name), path and url do not align
996 url = "gitsm://github.com/azure/iotedge.git;protocol=git;rev=d76e0316c6f324345d77c48a83ce836d09392699"
997 fetcher = bb.fetch.Fetch([url], self.d)
998 fetcher.download()
999 # Previous cwd has been deleted
1000 os.chdir(os.path.dirname(self.unpackdir))
1001 fetcher.unpack(self.unpackdir)
1002
1003 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1004
1005 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/README.md')), msg='Missing submodule checkout')
1006 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/ctest/README.md')), msg='Missing submodule checkout')
1007 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/testrunner/readme.md')), msg='Missing submodule checkout')
1008 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/readme.md')), msg='Missing submodule checkout')
1009 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
1010 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
1011 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/README.md')), msg='Missing submodule checkout')
1012 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/README.md')), msg='Missing submodule checkout')
1013 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/ctest/README.md')), msg='Missing submodule checkout')
1014 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/testrunner/readme.md')), msg='Missing submodule checkout')
1015 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/readme.md')), msg='Missing submodule checkout')
1016 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
1017 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
1018
Brad Bishop15ae2502019-06-18 21:44:24 -04001019class SVNTest(FetcherTest):
1020 def skipIfNoSvn():
1021 import shutil
1022 if not shutil.which("svn"):
1023 return unittest.skip("svn not installed, tests being skipped")
1024
1025 if not shutil.which("svnadmin"):
1026 return unittest.skip("svnadmin not installed, tests being skipped")
1027
1028 return lambda f: f
1029
1030 @skipIfNoSvn()
1031 def setUp(self):
1032 """ Create a local repository """
1033
1034 super(SVNTest, self).setUp()
1035
1036 # Create something we can fetch
1037 src_dir = tempfile.mkdtemp(dir=self.tempdir,
1038 prefix='svnfetch_srcdir_')
1039 src_dir = os.path.abspath(src_dir)
1040 bb.process.run("echo readme > README.md", cwd=src_dir)
1041
1042 # Store it in a local SVN repository
1043 repo_dir = tempfile.mkdtemp(dir=self.tempdir,
1044 prefix='svnfetch_localrepo_')
1045 repo_dir = os.path.abspath(repo_dir)
1046 bb.process.run("svnadmin create project", cwd=repo_dir)
1047
1048 self.repo_url = "file://%s/project" % repo_dir
1049 bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url),
1050 cwd=repo_dir)
1051
1052 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
1053 # Github will emulate SVN. Use this to check if we're downloding...
Andrew Geissler475cb722020-07-10 16:00:51 -05001054 bb.process.run("svn propset svn:externals 'bitbake svn://vcs.pcre.org/pcre2/code' .",
Brad Bishop15ae2502019-06-18 21:44:24 -04001055 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
1056 bb.process.run("svn commit --non-interactive -m 'Add external'",
1057 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
1058
1059 self.src_dir = src_dir
1060 self.repo_dir = repo_dir
1061
1062 @skipIfNoSvn()
1063 def tearDown(self):
1064 os.chdir(self.origdir)
1065 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
1066 print("Not cleaning up %s. Please remove manually." % self.tempdir)
1067 else:
1068 bb.utils.prunedir(self.tempdir)
1069
1070 @skipIfNoSvn()
1071 @skipIfNoNetwork()
1072 def test_noexternal_svn(self):
1073 # Always match the rev count from setUp (currently rev 2)
1074 url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '')
1075 fetcher = bb.fetch.Fetch([url], self.d)
1076 fetcher.download()
1077 os.chdir(os.path.dirname(self.unpackdir))
1078 fetcher.unpack(self.unpackdir)
1079
1080 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
1081 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
1082 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should NOT exist")
1083 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should NOT exit")
1084
1085 @skipIfNoSvn()
1086 def test_external_svn(self):
1087 # Always match the rev count from setUp (currently rev 2)
1088 url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '')
1089 fetcher = bb.fetch.Fetch([url], self.d)
1090 fetcher.download()
1091 os.chdir(os.path.dirname(self.unpackdir))
1092 fetcher.unpack(self.unpackdir)
1093
1094 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
1095 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
1096 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should exist")
1097 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should exit")
1098
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001099class TrustedNetworksTest(FetcherTest):
1100 def test_trusted_network(self):
1101 # Ensure trusted_network returns False when the host IS in the list.
1102 url = "git://Someserver.org/foo;rev=1"
1103 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org")
1104 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001105
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001106 def test_wild_trusted_network(self):
1107 # Ensure trusted_network returns true when the *.host IS in the list.
1108 url = "git://Someserver.org/foo;rev=1"
1109 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1110 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001111
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001112 def test_prefix_wild_trusted_network(self):
1113 # Ensure trusted_network returns true when the prefix matches *.host.
1114 url = "git://git.Someserver.org/foo;rev=1"
1115 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1116 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001117
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001118 def test_two_prefix_wild_trusted_network(self):
1119 # Ensure trusted_network returns true when the prefix matches *.host.
1120 url = "git://something.git.Someserver.org/foo;rev=1"
1121 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1122 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001123
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001124 def test_port_trusted_network(self):
1125 # Ensure trusted_network returns True, even if the url specifies a port.
1126 url = "git://someserver.org:8080/foo;rev=1"
1127 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org")
1128 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001129
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001130 def test_untrusted_network(self):
1131 # Ensure trusted_network returns False when the host is NOT in the list.
1132 url = "git://someserver.org/foo;rev=1"
1133 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
1134 self.assertFalse(bb.fetch.trusted_network(self.d, url))
1135
1136 def test_wild_untrusted_network(self):
1137 # Ensure trusted_network returns False when the host is NOT in the list.
1138 url = "git://*.someserver.org/foo;rev=1"
1139 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
1140 self.assertFalse(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001141
1142class URLHandle(unittest.TestCase):
1143
1144 datatable = {
1145 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}),
1146 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}),
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001147 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001148 "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'}),
1149 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}),
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001150 }
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001151 # we require a pathname to encodeurl but users can still pass such urls to
1152 # decodeurl and we need to handle them
1153 decodedata = datatable.copy()
1154 decodedata.update({
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001155 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001156 })
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001157
1158 def test_decodeurl(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001159 for k, v in self.decodedata.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001160 result = bb.fetch.decodeurl(k)
1161 self.assertEqual(result, v)
1162
1163 def test_encodeurl(self):
1164 for k, v in self.datatable.items():
1165 result = bb.fetch.encodeurl(v)
1166 self.assertEqual(result, k)
1167
1168class FetchLatestVersionTest(FetcherTest):
1169
1170 test_git_uris = {
1171 # version pattern "X.Y.Z"
1172 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
1173 : "1.99.4",
1174 # version pattern "vX.Y"
Andrew Geisslerd25ed322020-06-27 00:28:28 -05001175 # mirror of git.infradead.org since network issues interfered with testing
1176 ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001177 : "1.5.0",
1178 # version pattern "pkg_name-X.Y"
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001179 # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing
1180 ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001181 : "1.0",
1182 # version pattern "pkg_name-vX.Y.Z"
1183 ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
1184 : "1.4.0",
1185 # combination version pattern
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001186 ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001187 : "1.2.0",
1188 ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "")
1189 : "2014.01",
1190 # version pattern "yyyymmdd"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001191 ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001192 : "20120614",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001193 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001194 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing
1195 ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001196 : "0.4.3",
1197 ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", "(?P<pver>(([0-9][\.|_]?)+[0-9]))")
1198 : "11.0.0",
1199 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", "chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
1200 : "1.3.59",
1201 ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", "(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
1202 : "3.82+dbg0.9",
1203 }
1204
1205 test_wget_uris = {
Andrew Geissler82c905d2020-04-13 13:39:40 -05001206 #
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001207 # packages with versions inside directory name
Andrew Geissler82c905d2020-04-13 13:39:40 -05001208 #
1209 # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2
1210 ("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001211 : "2.24.2",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001212 # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz
1213 ("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001214 : "1.6.0",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001215 # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz
1216 ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001217 : "2.8.12.1",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001218 #
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001219 # packages with versions only in current directory
Andrew Geissler82c905d2020-04-13 13:39:40 -05001220 #
1221 # http://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2
1222 ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001223 : "2.19",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001224 # http://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2
1225 ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001226 : "20120814",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001227 #
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001228 # packages with "99" in the name of possible version
Andrew Geissler82c905d2020-04-13 13:39:40 -05001229 #
1230 # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz
1231 ("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001232 : "5.0",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001233 # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2
1234 ("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001235 : "1.15.1",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001236 #
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001237 # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX
Andrew Geissler82c905d2020-04-13 13:39:40 -05001238 #
1239 # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2
1240 # https://github.com/apple/cups/releases
1241 ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", "(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001242 : "2.0.0",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001243 # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz
1244 # http://ftp.debian.org/debian/pool/main/d/db5.3/
1245 ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", "(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz")
Brad Bishop79641f22019-09-10 07:20:22 -04001246 : "5.3.10",
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001247 }
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001248
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001249 @skipIfNoNetwork()
1250 def test_git_latest_versionstring(self):
1251 for k, v in self.test_git_uris.items():
1252 self.d.setVar("PN", k[0])
1253 self.d.setVar("SRCREV", k[2])
1254 self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3])
1255 ud = bb.fetch2.FetchData(k[1], self.d)
1256 pupver= ud.method.latest_versionstring(ud, self.d)
1257 verstring = pupver[0]
Brad Bishop316dfdd2018-06-25 12:45:53 -04001258 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001259 r = bb.utils.vercmp_string(v, verstring)
1260 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
1261
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001262 def test_wget_latest_versionstring(self):
Andrew Geissler82c905d2020-04-13 13:39:40 -05001263 testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata"
1264 server = HTTPService(testdata)
1265 server.start()
1266 port = server.port
1267 try:
1268 for k, v in self.test_wget_uris.items():
1269 self.d.setVar("PN", k[0])
1270 checkuri = ""
1271 if k[2]:
1272 checkuri = "http://localhost:%s/" % port + k[2]
1273 self.d.setVar("UPSTREAM_CHECK_URI", checkuri)
1274 self.d.setVar("UPSTREAM_CHECK_REGEX", k[3])
1275 url = "http://localhost:%s/" % port + k[1]
1276 ud = bb.fetch2.FetchData(url, self.d)
1277 pupver = ud.method.latest_versionstring(ud, self.d)
1278 verstring = pupver[0]
1279 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
1280 r = bb.utils.vercmp_string(v, verstring)
1281 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
1282 finally:
1283 server.stop()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001284
1285
1286class FetchCheckStatusTest(FetcherTest):
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001287 test_wget_uris = ["http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001288 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
1289 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
1290 "https://yoctoproject.org/",
1291 "https://yoctoproject.org/documentation",
1292 "http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
1293 "http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001294 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz",
1295 "http://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz",
1296 "https://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz",
1297 "https://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001298 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD
1299 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001300 ]
1301
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001302 @skipIfNoNetwork()
1303 def test_wget_checkstatus(self):
1304 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d)
1305 for u in self.test_wget_uris:
1306 with self.subTest(url=u):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001307 ud = fetch.ud[u]
1308 m = ud.method
1309 ret = m.checkstatus(fetch, ud, self.d)
1310 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
1311
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001312 @skipIfNoNetwork()
1313 def test_wget_checkstatus_connection_cache(self):
1314 from bb.fetch2 import FetchConnectionCache
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001315
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001316 connection_cache = FetchConnectionCache()
1317 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d,
1318 connection_cache = connection_cache)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001319
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001320 for u in self.test_wget_uris:
1321 with self.subTest(url=u):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001322 ud = fetch.ud[u]
1323 m = ud.method
1324 ret = m.checkstatus(fetch, ud, self.d)
1325 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
1326
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001327 connection_cache.close_connections()
1328
1329
1330class GitMakeShallowTest(FetcherTest):
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001331 def setUp(self):
1332 FetcherTest.setUp(self)
1333 self.gitdir = os.path.join(self.tempdir, 'gitshallow')
1334 bb.utils.mkdirhier(self.gitdir)
1335 bb.process.run('git init', cwd=self.gitdir)
1336
1337 def assertRefs(self, expected_refs):
1338 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
1339 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines()
1340 self.assertEqual(sorted(full_expected), sorted(actual_refs))
1341
1342 def assertRevCount(self, expected_count, args=None):
1343 if args is None:
1344 args = ['HEAD']
1345 revs = self.git(['rev-list'] + args)
1346 actual_count = len(revs.splitlines())
1347 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1348
1349 def git(self, cmd):
1350 if isinstance(cmd, str):
1351 cmd = 'git ' + cmd
1352 else:
1353 cmd = ['git'] + cmd
1354 return bb.process.run(cmd, cwd=self.gitdir)[0]
1355
1356 def make_shallow(self, args=None):
1357 if args is None:
1358 args = ['HEAD']
Brad Bishop316dfdd2018-06-25 12:45:53 -04001359 return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001360
1361 def add_empty_file(self, path, msg=None):
1362 if msg is None:
1363 msg = path
1364 open(os.path.join(self.gitdir, path), 'w').close()
1365 self.git(['add', path])
1366 self.git(['commit', '-m', msg, path])
1367
1368 def test_make_shallow_single_branch_no_merge(self):
1369 self.add_empty_file('a')
1370 self.add_empty_file('b')
1371 self.assertRevCount(2)
1372 self.make_shallow()
1373 self.assertRevCount(1)
1374
1375 def test_make_shallow_single_branch_one_merge(self):
1376 self.add_empty_file('a')
1377 self.add_empty_file('b')
1378 self.git('checkout -b a_branch')
1379 self.add_empty_file('c')
1380 self.git('checkout master')
1381 self.add_empty_file('d')
1382 self.git('merge --no-ff --no-edit a_branch')
1383 self.git('branch -d a_branch')
1384 self.add_empty_file('e')
1385 self.assertRevCount(6)
1386 self.make_shallow(['HEAD~2'])
1387 self.assertRevCount(5)
1388
1389 def test_make_shallow_at_merge(self):
1390 self.add_empty_file('a')
1391 self.git('checkout -b a_branch')
1392 self.add_empty_file('b')
1393 self.git('checkout master')
1394 self.git('merge --no-ff --no-edit a_branch')
1395 self.git('branch -d a_branch')
1396 self.assertRevCount(3)
1397 self.make_shallow()
1398 self.assertRevCount(1)
1399
1400 def test_make_shallow_annotated_tag(self):
1401 self.add_empty_file('a')
1402 self.add_empty_file('b')
1403 self.git('tag -a -m a_tag a_tag')
1404 self.assertRevCount(2)
1405 self.make_shallow(['a_tag'])
1406 self.assertRevCount(1)
1407
1408 def test_make_shallow_multi_ref(self):
1409 self.add_empty_file('a')
1410 self.add_empty_file('b')
1411 self.git('checkout -b a_branch')
1412 self.add_empty_file('c')
1413 self.git('checkout master')
1414 self.add_empty_file('d')
1415 self.git('checkout -b a_branch_2')
1416 self.add_empty_file('a_tag')
1417 self.git('tag a_tag')
1418 self.git('checkout master')
1419 self.git('branch -D a_branch_2')
1420 self.add_empty_file('e')
1421 self.assertRevCount(6, ['--all'])
1422 self.make_shallow()
1423 self.assertRevCount(5, ['--all'])
1424
1425 def test_make_shallow_multi_ref_trim(self):
1426 self.add_empty_file('a')
1427 self.git('checkout -b a_branch')
1428 self.add_empty_file('c')
1429 self.git('checkout master')
1430 self.assertRevCount(1)
1431 self.assertRevCount(2, ['--all'])
1432 self.assertRefs(['master', 'a_branch'])
1433 self.make_shallow(['-r', 'master', 'HEAD'])
1434 self.assertRevCount(1, ['--all'])
1435 self.assertRefs(['master'])
1436
1437 def test_make_shallow_noop(self):
1438 self.add_empty_file('a')
1439 self.assertRevCount(1)
1440 self.make_shallow()
1441 self.assertRevCount(1)
1442
1443 @skipIfNoNetwork()
1444 def test_make_shallow_bitbake(self):
1445 self.git('remote add origin https://github.com/openembedded/bitbake')
1446 self.git('fetch --tags origin')
1447 orig_revs = len(self.git('rev-list --all').splitlines())
1448 self.make_shallow(['refs/tags/1.10.0'])
1449 self.assertRevCount(orig_revs - 1746, ['--all'])
1450
1451class GitShallowTest(FetcherTest):
1452 def setUp(self):
1453 FetcherTest.setUp(self)
1454 self.gitdir = os.path.join(self.tempdir, 'git')
1455 self.srcdir = os.path.join(self.tempdir, 'gitsource')
1456
1457 bb.utils.mkdirhier(self.srcdir)
1458 self.git('init', cwd=self.srcdir)
1459 self.d.setVar('WORKDIR', self.tempdir)
1460 self.d.setVar('S', self.gitdir)
1461 self.d.delVar('PREMIRRORS')
1462 self.d.delVar('MIRRORS')
1463
1464 uri = 'git://%s;protocol=file;subdir=${S}' % self.srcdir
1465 self.d.setVar('SRC_URI', uri)
1466 self.d.setVar('SRCREV', '${AUTOREV}')
1467 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
1468
1469 self.d.setVar('BB_GIT_SHALLOW', '1')
1470 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
1471 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
1472
1473 def assertRefs(self, expected_refs, cwd=None):
1474 if cwd is None:
1475 cwd = self.gitdir
1476 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines()
1477 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines()
1478 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs)))
1479
1480 def assertRevCount(self, expected_count, args=None, cwd=None):
1481 if args is None:
1482 args = ['HEAD']
1483 if cwd is None:
1484 cwd = self.gitdir
1485 revs = self.git(['rev-list'] + args, cwd=cwd)
1486 actual_count = len(revs.splitlines())
1487 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1488
1489 def git(self, cmd, cwd=None):
1490 if isinstance(cmd, str):
1491 cmd = 'git ' + cmd
1492 else:
1493 cmd = ['git'] + cmd
1494 if cwd is None:
1495 cwd = self.gitdir
1496 return bb.process.run(cmd, cwd=cwd)[0]
1497
1498 def add_empty_file(self, path, cwd=None, msg=None):
1499 if msg is None:
1500 msg = path
1501 if cwd is None:
1502 cwd = self.srcdir
1503 open(os.path.join(cwd, path), 'w').close()
1504 self.git(['add', path], cwd)
1505 self.git(['commit', '-m', msg, path], cwd)
1506
1507 def fetch(self, uri=None):
1508 if uri is None:
Brad Bishop19323692019-04-05 15:28:33 -04001509 uris = self.d.getVar('SRC_URI').split()
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001510 uri = uris[0]
1511 d = self.d
1512 else:
1513 d = self.d.createCopy()
1514 d.setVar('SRC_URI', uri)
1515 uri = d.expand(uri)
1516 uris = [uri]
1517
1518 fetcher = bb.fetch2.Fetch(uris, d)
1519 fetcher.download()
1520 ud = fetcher.ud[uri]
1521 return fetcher, ud
1522
1523 def fetch_and_unpack(self, uri=None):
1524 fetcher, ud = self.fetch(uri)
1525 fetcher.unpack(self.d.getVar('WORKDIR'))
1526 assert os.path.exists(self.d.getVar('S'))
1527 return fetcher, ud
1528
1529 def fetch_shallow(self, uri=None, disabled=False, keepclone=False):
1530 """Fetch a uri, generating a shallow tarball, then unpack using it"""
1531 fetcher, ud = self.fetch_and_unpack(uri)
1532 assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri)
1533
1534 # Confirm that the unpacked repo is unshallow
1535 if not disabled:
1536 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1537
1538 # fetch and unpack, from the shallow tarball
1539 bb.utils.remove(self.gitdir, recurse=True)
1540 bb.utils.remove(ud.clonedir, recurse=True)
Brad Bishopf8caae32019-03-25 13:13:56 -04001541 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001542
1543 # confirm that the unpacked repo is used when no git clone or git
1544 # mirror tarball is available
1545 fetcher, ud = self.fetch_and_unpack(uri)
1546 if not disabled:
1547 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir
1548 else:
1549 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir
1550 return fetcher, ud
1551
1552 def test_shallow_disabled(self):
1553 self.add_empty_file('a')
1554 self.add_empty_file('b')
1555 self.assertRevCount(2, cwd=self.srcdir)
1556
1557 self.d.setVar('BB_GIT_SHALLOW', '0')
1558 self.fetch_shallow(disabled=True)
1559 self.assertRevCount(2)
1560
1561 def test_shallow_nobranch(self):
1562 self.add_empty_file('a')
1563 self.add_empty_file('b')
1564 self.assertRevCount(2, cwd=self.srcdir)
1565
1566 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip()
1567 self.d.setVar('SRCREV', srcrev)
Brad Bishop19323692019-04-05 15:28:33 -04001568 uri = self.d.getVar('SRC_URI').split()[0]
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001569 uri = '%s;nobranch=1;bare=1' % uri
1570
1571 self.fetch_shallow(uri)
1572 self.assertRevCount(1)
1573
1574 # shallow refs are used to ensure the srcrev sticks around when we
1575 # have no other branches referencing it
1576 self.assertRefs(['refs/shallow/default'])
1577
1578 def test_shallow_default_depth_1(self):
1579 # Create initial git repo
1580 self.add_empty_file('a')
1581 self.add_empty_file('b')
1582 self.assertRevCount(2, cwd=self.srcdir)
1583
1584 self.fetch_shallow()
1585 self.assertRevCount(1)
1586
1587 def test_shallow_depth_0_disables(self):
1588 self.add_empty_file('a')
1589 self.add_empty_file('b')
1590 self.assertRevCount(2, cwd=self.srcdir)
1591
1592 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1593 self.fetch_shallow(disabled=True)
1594 self.assertRevCount(2)
1595
1596 def test_shallow_depth_default_override(self):
1597 self.add_empty_file('a')
1598 self.add_empty_file('b')
1599 self.assertRevCount(2, cwd=self.srcdir)
1600
1601 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2')
1602 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1')
1603 self.fetch_shallow()
1604 self.assertRevCount(1)
1605
1606 def test_shallow_depth_default_override_disable(self):
1607 self.add_empty_file('a')
1608 self.add_empty_file('b')
1609 self.add_empty_file('c')
1610 self.assertRevCount(3, cwd=self.srcdir)
1611
1612 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1613 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2')
1614 self.fetch_shallow()
1615 self.assertRevCount(2)
1616
1617 def test_current_shallow_out_of_date_clone(self):
1618 # Create initial git repo
1619 self.add_empty_file('a')
1620 self.add_empty_file('b')
1621 self.add_empty_file('c')
1622 self.assertRevCount(3, cwd=self.srcdir)
1623
1624 # Clone and generate mirror tarball
1625 fetcher, ud = self.fetch()
1626
1627 # Ensure we have a current mirror tarball, but an out of date clone
1628 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir)
1629 self.assertRevCount(2, cwd=ud.clonedir)
1630
1631 # Fetch and unpack, from the current tarball, not the out of date clone
1632 bb.utils.remove(self.gitdir, recurse=True)
1633 fetcher, ud = self.fetch()
1634 fetcher.unpack(self.d.getVar('WORKDIR'))
1635 self.assertRevCount(1)
1636
1637 def test_shallow_single_branch_no_merge(self):
1638 self.add_empty_file('a')
1639 self.add_empty_file('b')
1640 self.assertRevCount(2, cwd=self.srcdir)
1641
1642 self.fetch_shallow()
1643 self.assertRevCount(1)
1644 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1645 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1646
1647 def test_shallow_no_dangling(self):
1648 self.add_empty_file('a')
1649 self.add_empty_file('b')
1650 self.assertRevCount(2, cwd=self.srcdir)
1651
1652 self.fetch_shallow()
1653 self.assertRevCount(1)
1654 assert not self.git('fsck --dangling')
1655
1656 def test_shallow_srcrev_branch_truncation(self):
1657 self.add_empty_file('a')
1658 self.add_empty_file('b')
1659 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip()
1660 self.add_empty_file('c')
1661 self.assertRevCount(3, cwd=self.srcdir)
1662
1663 self.d.setVar('SRCREV', b_commit)
1664 self.fetch_shallow()
1665
1666 # The 'c' commit was removed entirely, and 'a' was removed from history
1667 self.assertRevCount(1, ['--all'])
1668 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit)
1669 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1670 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1671 assert not os.path.exists(os.path.join(self.gitdir, 'c'))
1672
1673 def test_shallow_ref_pruning(self):
1674 self.add_empty_file('a')
1675 self.add_empty_file('b')
1676 self.git('branch a_branch', cwd=self.srcdir)
1677 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
1678 self.assertRevCount(2, cwd=self.srcdir)
1679
1680 self.fetch_shallow()
1681
1682 self.assertRefs(['master', 'origin/master'])
1683 self.assertRevCount(1)
1684
1685 def test_shallow_submodules(self):
1686 self.add_empty_file('a')
1687 self.add_empty_file('b')
1688
1689 smdir = os.path.join(self.tempdir, 'gitsubmodule')
1690 bb.utils.mkdirhier(smdir)
1691 self.git('init', cwd=smdir)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001692 # Make this look like it was cloned from a remote...
1693 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
1694 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001695 self.add_empty_file('asub', cwd=smdir)
Brad Bishopf8caae32019-03-25 13:13:56 -04001696 self.add_empty_file('bsub', cwd=smdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001697
1698 self.git('submodule init', cwd=self.srcdir)
1699 self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
1700 self.git('submodule update', cwd=self.srcdir)
1701 self.git('commit -m submodule -a', cwd=self.srcdir)
1702
1703 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
1704 fetcher, ud = self.fetch_shallow(uri)
1705
Brad Bishopf8caae32019-03-25 13:13:56 -04001706 # Verify the main repository is shallow
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001707 self.assertRevCount(1)
Brad Bishopf8caae32019-03-25 13:13:56 -04001708
1709 # Verify the gitsubmodule directory is present
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001710 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
1711
Brad Bishopf8caae32019-03-25 13:13:56 -04001712 # Verify the submodule is also shallow
1713 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
1714
Andrew Geissler82c905d2020-04-13 13:39:40 -05001715 def test_shallow_submodule_mirrors(self):
1716 self.add_empty_file('a')
1717 self.add_empty_file('b')
1718
1719 smdir = os.path.join(self.tempdir, 'gitsubmodule')
1720 bb.utils.mkdirhier(smdir)
1721 self.git('init', cwd=smdir)
1722 # Make this look like it was cloned from a remote...
1723 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
1724 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
1725 self.add_empty_file('asub', cwd=smdir)
1726 self.add_empty_file('bsub', cwd=smdir)
1727
1728 self.git('submodule init', cwd=self.srcdir)
1729 self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
1730 self.git('submodule update', cwd=self.srcdir)
1731 self.git('commit -m submodule -a', cwd=self.srcdir)
1732
1733 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
1734
1735 # Fetch once to generate the shallow tarball
1736 fetcher, ud = self.fetch(uri)
1737
1738 # Set up the mirror
1739 mirrordir = os.path.join(self.tempdir, 'mirror')
1740 os.rename(self.dldir, mirrordir)
1741 self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/\n' % mirrordir)
1742
1743 # Fetch from the mirror
1744 bb.utils.remove(self.dldir, recurse=True)
1745 bb.utils.remove(self.gitdir, recurse=True)
1746 self.fetch_and_unpack(uri)
1747
1748 # Verify the main repository is shallow
1749 self.assertRevCount(1)
1750
1751 # Verify the gitsubmodule directory is present
1752 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
1753
1754 # Verify the submodule is also shallow
1755 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
Brad Bishopf8caae32019-03-25 13:13:56 -04001756
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001757 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')):
1758 def test_shallow_annex(self):
1759 self.add_empty_file('a')
1760 self.add_empty_file('b')
1761 self.git('annex init', cwd=self.srcdir)
1762 open(os.path.join(self.srcdir, 'c'), 'w').close()
1763 self.git('annex add c', cwd=self.srcdir)
1764 self.git('commit -m annex-c -a', cwd=self.srcdir)
1765 bb.process.run('chmod u+w -R %s' % os.path.join(self.srcdir, '.git', 'annex'))
1766
1767 uri = 'gitannex://%s;protocol=file;subdir=${S}' % self.srcdir
1768 fetcher, ud = self.fetch_shallow(uri)
1769
1770 self.assertRevCount(1)
1771 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0]
1772 assert os.path.exists(os.path.join(self.gitdir, 'c'))
1773
1774 def test_shallow_multi_one_uri(self):
1775 # Create initial git repo
1776 self.add_empty_file('a')
1777 self.add_empty_file('b')
1778 self.git('checkout -b a_branch', cwd=self.srcdir)
1779 self.add_empty_file('c')
1780 self.add_empty_file('d')
1781 self.git('checkout master', cwd=self.srcdir)
1782 self.git('tag v0.0 a_branch', cwd=self.srcdir)
1783 self.add_empty_file('e')
1784 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1785 self.add_empty_file('f')
1786 self.assertRevCount(7, cwd=self.srcdir)
1787
Brad Bishop19323692019-04-05 15:28:33 -04001788 uri = self.d.getVar('SRC_URI').split()[0]
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001789 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
1790
1791 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1792 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1793 self.d.setVar('SRCREV_master', '${AUTOREV}')
1794 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
1795
1796 self.fetch_shallow(uri)
1797
1798 self.assertRevCount(5)
1799 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1800
1801 def test_shallow_multi_one_uri_depths(self):
1802 # Create initial git repo
1803 self.add_empty_file('a')
1804 self.add_empty_file('b')
1805 self.git('checkout -b a_branch', cwd=self.srcdir)
1806 self.add_empty_file('c')
1807 self.add_empty_file('d')
1808 self.git('checkout master', cwd=self.srcdir)
1809 self.add_empty_file('e')
1810 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1811 self.add_empty_file('f')
1812 self.assertRevCount(7, cwd=self.srcdir)
1813
Brad Bishop19323692019-04-05 15:28:33 -04001814 uri = self.d.getVar('SRC_URI').split()[0]
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001815 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
1816
1817 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1818 self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3')
1819 self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1')
1820 self.d.setVar('SRCREV_master', '${AUTOREV}')
1821 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
1822
1823 self.fetch_shallow(uri)
1824
1825 self.assertRevCount(4, ['--all'])
1826 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1827
1828 def test_shallow_clone_preferred_over_shallow(self):
1829 self.add_empty_file('a')
1830 self.add_empty_file('b')
1831
1832 # Fetch once to generate the shallow tarball
1833 fetcher, ud = self.fetch()
1834 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1835
1836 # Fetch and unpack with both the clonedir and shallow tarball available
1837 bb.utils.remove(self.gitdir, recurse=True)
1838 fetcher, ud = self.fetch_and_unpack()
1839
1840 # The unpacked tree should *not* be shallow
1841 self.assertRevCount(2)
1842 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow'))
1843
1844 def test_shallow_mirrors(self):
1845 self.add_empty_file('a')
1846 self.add_empty_file('b')
1847
1848 # Fetch once to generate the shallow tarball
1849 fetcher, ud = self.fetch()
1850 mirrortarball = ud.mirrortarballs[0]
1851 assert os.path.exists(os.path.join(self.dldir, mirrortarball))
1852
1853 # Set up the mirror
1854 mirrordir = os.path.join(self.tempdir, 'mirror')
1855 bb.utils.mkdirhier(mirrordir)
1856 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir)
1857
1858 os.rename(os.path.join(self.dldir, mirrortarball),
1859 os.path.join(mirrordir, mirrortarball))
1860
1861 # Fetch from the mirror
1862 bb.utils.remove(self.dldir, recurse=True)
1863 bb.utils.remove(self.gitdir, recurse=True)
1864 self.fetch_and_unpack()
1865 self.assertRevCount(1)
1866
1867 def test_shallow_invalid_depth(self):
1868 self.add_empty_file('a')
1869 self.add_empty_file('b')
1870
1871 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12')
1872 with self.assertRaises(bb.fetch2.FetchError):
1873 self.fetch()
1874
1875 def test_shallow_invalid_depth_default(self):
1876 self.add_empty_file('a')
1877 self.add_empty_file('b')
1878
1879 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12')
1880 with self.assertRaises(bb.fetch2.FetchError):
1881 self.fetch()
1882
1883 def test_shallow_extra_refs(self):
1884 self.add_empty_file('a')
1885 self.add_empty_file('b')
1886 self.git('branch a_branch', cwd=self.srcdir)
1887 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
1888 self.assertRevCount(2, cwd=self.srcdir)
1889
1890 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch')
1891 self.fetch_shallow()
1892
1893 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1894 self.assertRevCount(1)
1895
1896 def test_shallow_extra_refs_wildcard(self):
1897 self.add_empty_file('a')
1898 self.add_empty_file('b')
1899 self.git('branch a_branch', cwd=self.srcdir)
1900 self.git('tag v1.0', cwd=self.srcdir)
1901 self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir)
1902 self.assertRevCount(2, cwd=self.srcdir)
1903
1904 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
1905 self.fetch_shallow()
1906
1907 self.assertRefs(['master', 'origin/master', 'v1.0'])
1908 self.assertRevCount(1)
1909
1910 def test_shallow_missing_extra_refs(self):
1911 self.add_empty_file('a')
1912 self.add_empty_file('b')
1913
1914 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo')
1915 with self.assertRaises(bb.fetch2.FetchError):
1916 self.fetch()
1917
1918 def test_shallow_missing_extra_refs_wildcard(self):
1919 self.add_empty_file('a')
1920 self.add_empty_file('b')
1921
1922 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
1923 self.fetch()
1924
1925 def test_shallow_remove_revs(self):
1926 # Create initial git repo
1927 self.add_empty_file('a')
1928 self.add_empty_file('b')
1929 self.git('checkout -b a_branch', cwd=self.srcdir)
1930 self.add_empty_file('c')
1931 self.add_empty_file('d')
1932 self.git('checkout master', cwd=self.srcdir)
1933 self.git('tag v0.0 a_branch', cwd=self.srcdir)
1934 self.add_empty_file('e')
1935 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1936 self.git('branch -d a_branch', cwd=self.srcdir)
1937 self.add_empty_file('f')
1938 self.assertRevCount(7, cwd=self.srcdir)
1939
1940 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1941 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1942
1943 self.fetch_shallow()
1944
1945 self.assertRevCount(5)
1946
1947 def test_shallow_invalid_revs(self):
1948 self.add_empty_file('a')
1949 self.add_empty_file('b')
1950
1951 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1952 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1953
1954 with self.assertRaises(bb.fetch2.FetchError):
1955 self.fetch()
1956
Brad Bishop64c979e2019-11-04 13:55:29 -05001957 def test_shallow_fetch_missing_revs(self):
1958 self.add_empty_file('a')
1959 self.add_empty_file('b')
1960 fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
1961 self.git('tag v0.0 master', cwd=self.srcdir)
1962 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1963 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1964 self.fetch_shallow()
1965
1966 def test_shallow_fetch_missing_revs_fails(self):
1967 self.add_empty_file('a')
1968 self.add_empty_file('b')
1969 fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
1970 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1971 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1972
1973 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm:
1974 self.fetch_shallow()
1975 self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0])
1976
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001977 @skipIfNoNetwork()
1978 def test_bitbake(self):
1979 self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
1980 self.git('config core.bare true', cwd=self.srcdir)
1981 self.git('fetch', cwd=self.srcdir)
1982
1983 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1984 # Note that the 1.10.0 tag is annotated, so this also tests
1985 # reference of an annotated vs unannotated tag
1986 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0')
1987
1988 self.fetch_shallow()
1989
1990 # Confirm that the history of 1.10.0 was removed
1991 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines())
1992 revs = len(self.git('rev-list master').splitlines())
1993 self.assertNotEqual(orig_revs, revs)
1994 self.assertRefs(['master', 'origin/master'])
1995 self.assertRevCount(orig_revs - 1758)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001996
1997 def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self):
1998 self.add_empty_file('a')
1999 fetcher, ud = self.fetch()
2000 bb.utils.remove(self.gitdir, recurse=True)
2001 bb.utils.remove(self.dldir, recurse=True)
2002
2003 with self.assertRaises(bb.fetch2.UnpackError) as context:
2004 fetcher.unpack(self.d.getVar('WORKDIR'))
2005
2006 self.assertIn("No up to date source found", context.exception.msg)
2007 self.assertIn("clone directory not available or not up to date", context.exception.msg)
2008
2009 @skipIfNoNetwork()
2010 def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self):
2011 self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0')
2012 self.d.setVar('BB_GIT_SHALLOW', '1')
2013 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
2014 fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests"], self.d)
2015 fetcher.download()
2016
2017 bb.utils.remove(self.dldir + "/*.tar.gz")
2018 fetcher.unpack(self.unpackdir)
2019
2020 dir = os.listdir(self.unpackdir + "/git/")
2021 self.assertIn("fstests.doap", dir)
Brad Bishop00e122a2019-10-05 11:10:57 -04002022
2023class GitLfsTest(FetcherTest):
2024 def setUp(self):
2025 FetcherTest.setUp(self)
2026
2027 self.gitdir = os.path.join(self.tempdir, 'git')
2028 self.srcdir = os.path.join(self.tempdir, 'gitsource')
2029
2030 self.d.setVar('WORKDIR', self.tempdir)
2031 self.d.setVar('S', self.gitdir)
2032 self.d.delVar('PREMIRRORS')
2033 self.d.delVar('MIRRORS')
2034
2035 self.d.setVar('SRCREV', '${AUTOREV}')
2036 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
2037
2038 bb.utils.mkdirhier(self.srcdir)
2039 self.git('init', cwd=self.srcdir)
2040 with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs:
2041 attrs.write('*.mp3 filter=lfs -text')
2042 self.git(['add', '.gitattributes'], cwd=self.srcdir)
2043 self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir)
2044
2045 def git(self, cmd, cwd=None):
2046 if isinstance(cmd, str):
2047 cmd = 'git ' + cmd
2048 else:
2049 cmd = ['git'] + cmd
2050 if cwd is None:
2051 cwd = self.gitdir
2052 return bb.process.run(cmd, cwd=cwd)[0]
2053
2054 def fetch(self, uri=None):
2055 uris = self.d.getVar('SRC_URI').split()
2056 uri = uris[0]
2057 d = self.d
2058
2059 fetcher = bb.fetch2.Fetch(uris, d)
2060 fetcher.download()
2061 ud = fetcher.ud[uri]
2062 return fetcher, ud
2063
2064 def test_lfs_enabled(self):
2065 import shutil
2066
2067 uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir
2068 self.d.setVar('SRC_URI', uri)
2069
2070 fetcher, ud = self.fetch()
2071 self.assertIsNotNone(ud.method._find_git_lfs)
2072
2073 # If git-lfs can be found, the unpack should be successful
2074 ud.method._find_git_lfs = lambda d: True
2075 shutil.rmtree(self.gitdir, ignore_errors=True)
2076 fetcher.unpack(self.d.getVar('WORKDIR'))
2077
2078 # If git-lfs cannot be found, the unpack should throw an error
2079 with self.assertRaises(bb.fetch2.FetchError):
2080 ud.method._find_git_lfs = lambda d: False
2081 shutil.rmtree(self.gitdir, ignore_errors=True)
2082 fetcher.unpack(self.d.getVar('WORKDIR'))
2083
2084 def test_lfs_disabled(self):
2085 import shutil
2086
2087 uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir
2088 self.d.setVar('SRC_URI', uri)
2089
2090 fetcher, ud = self.fetch()
2091 self.assertIsNotNone(ud.method._find_git_lfs)
2092
2093 # If git-lfs can be found, the unpack should be successful
2094 ud.method._find_git_lfs = lambda d: True
2095 shutil.rmtree(self.gitdir, ignore_errors=True)
2096 fetcher.unpack(self.d.getVar('WORKDIR'))
2097
2098 # If git-lfs cannot be found, the unpack should be successful
2099 ud.method._find_git_lfs = lambda d: False
2100 shutil.rmtree(self.gitdir, ignore_errors=True)
2101 fetcher.unpack(self.d.getVar('WORKDIR'))
Andrew Geissler82c905d2020-04-13 13:39:40 -05002102
Andrew Geisslerc3d88e42020-10-02 09:45:00 -05002103class GitURLWithSpacesTest(FetcherTest):
2104 test_git_urls = {
2105 "git://tfs-example.org:22/tfs/example%20path/example.git" : {
2106 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git',
2107 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git',
2108 'path': '/tfs/example path/example.git'
2109 },
2110 "git://tfs-example.org:22/tfs/example%20path/example%20repo.git" : {
2111 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git',
2112 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git',
2113 'path': '/tfs/example path/example repo.git'
2114 }
2115 }
2116
2117 def test_urls(self):
2118
2119 # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo
2120 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
2121
2122 for test_git_url, ref in self.test_git_urls.items():
2123
2124 fetcher = bb.fetch.Fetch([test_git_url], self.d)
2125 ud = fetcher.ud[fetcher.urls[0]]
2126
2127 self.assertEqual(ud.url, ref['url'])
2128 self.assertEqual(ud.path, ref['path'])
2129 self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname']))
2130 self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname']))
2131 self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock'))
2132 self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname']))
2133 self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz'))
2134
Andrew Geissler82c905d2020-04-13 13:39:40 -05002135class NPMTest(FetcherTest):
2136 def skipIfNoNpm():
2137 import shutil
2138 if not shutil.which('npm'):
2139 return unittest.skip('npm not installed, tests being skipped')
2140 return lambda f: f
2141
2142 @skipIfNoNpm()
2143 @skipIfNoNetwork()
2144 def test_npm(self):
2145 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2146 fetcher = bb.fetch.Fetch([url], self.d)
2147 ud = fetcher.ud[fetcher.urls[0]]
2148 fetcher.download()
2149 self.assertTrue(os.path.exists(ud.localpath))
2150 self.assertTrue(os.path.exists(ud.localpath + '.done'))
2151 self.assertTrue(os.path.exists(ud.resolvefile))
2152 fetcher.unpack(self.unpackdir)
2153 unpackdir = os.path.join(self.unpackdir, 'npm')
2154 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2155
2156 @skipIfNoNpm()
2157 @skipIfNoNetwork()
2158 def test_npm_bad_checksum(self):
2159 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2160 # Fetch once to get a tarball
2161 fetcher = bb.fetch.Fetch([url], self.d)
2162 ud = fetcher.ud[fetcher.urls[0]]
2163 fetcher.download()
2164 self.assertTrue(os.path.exists(ud.localpath))
2165 # Modify the tarball
2166 bad = b'bad checksum'
2167 with open(ud.localpath, 'wb') as f:
2168 f.write(bad)
2169 # Verify that the tarball is fetched again
2170 fetcher.download()
2171 badsum = hashlib.sha512(bad).hexdigest()
2172 self.assertTrue(os.path.exists(ud.localpath + '_bad-checksum_' + badsum))
2173 self.assertTrue(os.path.exists(ud.localpath))
2174
2175 @skipIfNoNpm()
2176 @skipIfNoNetwork()
2177 def test_npm_premirrors(self):
2178 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2179 # Fetch once to get a tarball
2180 fetcher = bb.fetch.Fetch([url], self.d)
2181 ud = fetcher.ud[fetcher.urls[0]]
2182 fetcher.download()
2183 self.assertTrue(os.path.exists(ud.localpath))
2184 # Setup the mirror
2185 mirrordir = os.path.join(self.tempdir, 'mirror')
2186 bb.utils.mkdirhier(mirrordir)
2187 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
2188 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
2189 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
2190 # Fetch again
2191 self.assertFalse(os.path.exists(ud.localpath))
2192 fetcher.download()
2193 self.assertTrue(os.path.exists(ud.localpath))
2194
2195 @skipIfNoNpm()
2196 @skipIfNoNetwork()
2197 def test_npm_mirrors(self):
2198 # Fetch once to get a tarball
2199 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2200 fetcher = bb.fetch.Fetch([url], self.d)
2201 ud = fetcher.ud[fetcher.urls[0]]
2202 fetcher.download()
2203 self.assertTrue(os.path.exists(ud.localpath))
2204 # Setup the mirror
2205 mirrordir = os.path.join(self.tempdir, 'mirror')
2206 bb.utils.mkdirhier(mirrordir)
2207 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
2208 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
2209 # Update the resolved url to an invalid url
2210 with open(ud.resolvefile, 'r') as f:
2211 url = f.read()
2212 uri = URI(url)
2213 uri.path = '/invalid'
2214 with open(ud.resolvefile, 'w') as f:
2215 f.write(str(uri))
2216 # Fetch again
2217 self.assertFalse(os.path.exists(ud.localpath))
2218 fetcher.download()
2219 self.assertTrue(os.path.exists(ud.localpath))
2220
2221 @skipIfNoNpm()
2222 @skipIfNoNetwork()
2223 def test_npm_destsuffix_downloadfilename(self):
2224 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz'
2225 fetcher = bb.fetch.Fetch([url], self.d)
2226 fetcher.download()
2227 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'foo-bar.tgz')))
2228 fetcher.unpack(self.unpackdir)
2229 unpackdir = os.path.join(self.unpackdir, 'foo', 'bar')
2230 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2231
2232 def test_npm_no_network_no_tarball(self):
2233 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2234 self.d.setVar('BB_NO_NETWORK', '1')
2235 fetcher = bb.fetch.Fetch([url], self.d)
2236 with self.assertRaises(bb.fetch2.NetworkAccess):
2237 fetcher.download()
2238
2239 @skipIfNoNpm()
2240 @skipIfNoNetwork()
2241 def test_npm_no_network_with_tarball(self):
2242 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2243 # Fetch once to get a tarball
2244 fetcher = bb.fetch.Fetch([url], self.d)
2245 fetcher.download()
2246 # Disable network access
2247 self.d.setVar('BB_NO_NETWORK', '1')
2248 # Fetch again
2249 fetcher.download()
2250 fetcher.unpack(self.unpackdir)
2251 unpackdir = os.path.join(self.unpackdir, 'npm')
2252 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2253
2254 @skipIfNoNpm()
2255 @skipIfNoNetwork()
2256 def test_npm_registry_alternate(self):
2257 url = 'npm://registry.freajs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2258 fetcher = bb.fetch.Fetch([url], self.d)
2259 fetcher.download()
2260 fetcher.unpack(self.unpackdir)
2261 unpackdir = os.path.join(self.unpackdir, 'npm')
2262 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2263
2264 @skipIfNoNpm()
2265 @skipIfNoNetwork()
2266 def test_npm_version_latest(self):
2267 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest'
2268 fetcher = bb.fetch.Fetch([url], self.d)
2269 fetcher.download()
2270 fetcher.unpack(self.unpackdir)
2271 unpackdir = os.path.join(self.unpackdir, 'npm')
2272 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2273
2274 @skipIfNoNpm()
2275 @skipIfNoNetwork()
2276 def test_npm_registry_invalid(self):
2277 url = 'npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2278 fetcher = bb.fetch.Fetch([url], self.d)
2279 with self.assertRaises(bb.fetch2.FetchError):
2280 fetcher.download()
2281
2282 @skipIfNoNpm()
2283 @skipIfNoNetwork()
2284 def test_npm_package_invalid(self):
2285 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0'
2286 fetcher = bb.fetch.Fetch([url], self.d)
2287 with self.assertRaises(bb.fetch2.FetchError):
2288 fetcher.download()
2289
2290 @skipIfNoNpm()
2291 @skipIfNoNetwork()
2292 def test_npm_version_invalid(self):
2293 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid'
2294 with self.assertRaises(bb.fetch2.ParameterError):
2295 fetcher = bb.fetch.Fetch([url], self.d)
2296
2297 @skipIfNoNpm()
2298 @skipIfNoNetwork()
2299 def test_npm_registry_none(self):
2300 url = 'npm://;package=@savoirfairelinux/node-server-example;version=1.0.0'
2301 with self.assertRaises(bb.fetch2.MalformedUrl):
2302 fetcher = bb.fetch.Fetch([url], self.d)
2303
2304 @skipIfNoNpm()
2305 @skipIfNoNetwork()
2306 def test_npm_package_none(self):
2307 url = 'npm://registry.npmjs.org;version=1.0.0'
2308 with self.assertRaises(bb.fetch2.MissingParameterError):
2309 fetcher = bb.fetch.Fetch([url], self.d)
2310
2311 @skipIfNoNpm()
2312 @skipIfNoNetwork()
2313 def test_npm_version_none(self):
2314 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example'
2315 with self.assertRaises(bb.fetch2.MissingParameterError):
2316 fetcher = bb.fetch.Fetch([url], self.d)
2317
2318 def create_shrinkwrap_file(self, data):
2319 import json
2320 datadir = os.path.join(self.tempdir, 'data')
2321 swfile = os.path.join(datadir, 'npm-shrinkwrap.json')
2322 bb.utils.mkdirhier(datadir)
2323 with open(swfile, 'w') as f:
2324 json.dump(data, f)
2325 # Also configure the S directory
2326 self.sdir = os.path.join(self.unpackdir, 'S')
2327 self.d.setVar('S', self.sdir)
2328 return swfile
2329
2330 @skipIfNoNpm()
2331 @skipIfNoNetwork()
2332 def test_npmsw(self):
2333 swfile = self.create_shrinkwrap_file({
2334 'dependencies': {
2335 'array-flatten': {
2336 'version': '1.1.1',
2337 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2338 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=',
2339 'dependencies': {
2340 'content-type': {
2341 'version': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
2342 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
2343 'dependencies': {
2344 'cookie': {
2345 'version': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09',
2346 'from': 'git+https://github.com/jshttp/cookie.git'
2347 }
2348 }
2349 }
2350 }
2351 }
2352 }
2353 })
2354 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2355 fetcher.download()
2356 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
2357 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
2358 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git')))
2359 fetcher.unpack(self.unpackdir)
2360 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'npm-shrinkwrap.json')))
2361 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json')))
2362 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json')))
2363 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json')))
2364
2365 @skipIfNoNpm()
2366 @skipIfNoNetwork()
2367 def test_npmsw_dev(self):
2368 swfile = self.create_shrinkwrap_file({
2369 'dependencies': {
2370 'array-flatten': {
2371 'version': '1.1.1',
2372 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2373 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2374 },
2375 'content-type': {
2376 'version': '1.0.4',
2377 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
2378 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
2379 'dev': True
2380 }
2381 }
2382 })
2383 # Fetch with dev disabled
2384 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2385 fetcher.download()
2386 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
2387 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
2388 # Fetch with dev enabled
2389 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d)
2390 fetcher.download()
2391 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
2392 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
2393
2394 @skipIfNoNpm()
2395 @skipIfNoNetwork()
2396 def test_npmsw_destsuffix(self):
2397 swfile = self.create_shrinkwrap_file({
2398 'dependencies': {
2399 'array-flatten': {
2400 'version': '1.1.1',
2401 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2402 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2403 }
2404 }
2405 })
2406 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d)
2407 fetcher.download()
2408 fetcher.unpack(self.unpackdir)
2409 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json')))
2410
2411 def test_npmsw_no_network_no_tarball(self):
2412 swfile = self.create_shrinkwrap_file({
2413 'dependencies': {
2414 'array-flatten': {
2415 'version': '1.1.1',
2416 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2417 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2418 }
2419 }
2420 })
2421 self.d.setVar('BB_NO_NETWORK', '1')
2422 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2423 with self.assertRaises(bb.fetch2.NetworkAccess):
2424 fetcher.download()
2425
2426 @skipIfNoNpm()
2427 @skipIfNoNetwork()
2428 def test_npmsw_no_network_with_tarball(self):
2429 # Fetch once to get a tarball
2430 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2431 fetcher.download()
2432 # Disable network access
2433 self.d.setVar('BB_NO_NETWORK', '1')
2434 # Fetch again
2435 swfile = self.create_shrinkwrap_file({
2436 'dependencies': {
2437 'array-flatten': {
2438 'version': '1.1.1',
2439 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2440 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2441 }
2442 }
2443 })
2444 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2445 fetcher.download()
2446 fetcher.unpack(self.unpackdir)
2447 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json')))
2448
2449 @skipIfNoNpm()
2450 @skipIfNoNetwork()
2451 def test_npmsw_npm_reusability(self):
2452 # Fetch once with npmsw
2453 swfile = self.create_shrinkwrap_file({
2454 'dependencies': {
2455 'array-flatten': {
2456 'version': '1.1.1',
2457 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2458 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2459 }
2460 }
2461 })
2462 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2463 fetcher.download()
2464 # Disable network access
2465 self.d.setVar('BB_NO_NETWORK', '1')
2466 # Fetch again with npm
2467 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2468 fetcher.download()
2469 fetcher.unpack(self.unpackdir)
2470 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json')))
2471
2472 @skipIfNoNpm()
2473 @skipIfNoNetwork()
2474 def test_npmsw_bad_checksum(self):
2475 # Try to fetch with bad checksum
2476 swfile = self.create_shrinkwrap_file({
2477 'dependencies': {
2478 'array-flatten': {
2479 'version': '1.1.1',
2480 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2481 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg='
2482 }
2483 }
2484 })
2485 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2486 with self.assertRaises(bb.fetch2.FetchError):
2487 fetcher.download()
2488 # Fetch correctly to get a tarball
2489 swfile = self.create_shrinkwrap_file({
2490 'dependencies': {
2491 'array-flatten': {
2492 'version': '1.1.1',
2493 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2494 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2495 }
2496 }
2497 })
2498 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2499 fetcher.download()
2500 localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')
2501 self.assertTrue(os.path.exists(localpath))
2502 # Modify the tarball
2503 bad = b'bad checksum'
2504 with open(localpath, 'wb') as f:
2505 f.write(bad)
2506 # Verify that the tarball is fetched again
2507 fetcher.download()
2508 badsum = hashlib.sha1(bad).hexdigest()
2509 self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum))
2510 self.assertTrue(os.path.exists(localpath))
2511
2512 @skipIfNoNpm()
2513 @skipIfNoNetwork()
2514 def test_npmsw_premirrors(self):
2515 # Fetch once to get a tarball
2516 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2517 ud = fetcher.ud[fetcher.urls[0]]
2518 fetcher.download()
2519 self.assertTrue(os.path.exists(ud.localpath))
2520 # Setup the mirror
2521 mirrordir = os.path.join(self.tempdir, 'mirror')
2522 bb.utils.mkdirhier(mirrordir)
2523 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
2524 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
2525 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
2526 # Fetch again
2527 self.assertFalse(os.path.exists(ud.localpath))
2528 swfile = self.create_shrinkwrap_file({
2529 'dependencies': {
2530 'array-flatten': {
2531 'version': '1.1.1',
2532 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2533 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2534 }
2535 }
2536 })
2537 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2538 fetcher.download()
2539 self.assertTrue(os.path.exists(ud.localpath))
2540
2541 @skipIfNoNpm()
2542 @skipIfNoNetwork()
2543 def test_npmsw_mirrors(self):
2544 # Fetch once to get a tarball
2545 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2546 ud = fetcher.ud[fetcher.urls[0]]
2547 fetcher.download()
2548 self.assertTrue(os.path.exists(ud.localpath))
2549 # Setup the mirror
2550 mirrordir = os.path.join(self.tempdir, 'mirror')
2551 bb.utils.mkdirhier(mirrordir)
2552 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
2553 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
2554 # Fetch again with invalid url
2555 self.assertFalse(os.path.exists(ud.localpath))
2556 swfile = self.create_shrinkwrap_file({
2557 'dependencies': {
2558 'array-flatten': {
2559 'version': '1.1.1',
2560 'resolved': 'https://invalid',
2561 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2562 }
2563 }
2564 })
2565 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2566 fetcher.download()
2567 self.assertTrue(os.path.exists(ud.localpath))