blob: 8ad1c859907ccaed85784791cfdab27d5c3e9e74 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# BitBake Tests for the Fetcher (fetch2/)
3#
4# Copyright (C) 2012 Richard Purdie
5#
Brad Bishopc342db32019-05-15 21:57:59 -04006# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007#
8
9import unittest
Brad Bishop316dfdd2018-06-25 12:45:53 -040010import hashlib
Patrick Williamsc124f4f2015-09-15 14:41:29 -050011import tempfile
Patrick Williamsc0f7c042017-02-23 20:41:17 -060012import collections
Patrick Williamsc124f4f2015-09-15 14:41:29 -050013import os
14from bb.fetch2 import URI
15from bb.fetch2 import FetchMethod
16import bb
Andrew Geissler82c905d2020-04-13 13:39:40 -050017from bb.tests.support.httpserver import HTTPService
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018
Brad Bishopd7bf8c12018-02-25 22:55:05 -050019def skipIfNoNetwork():
20 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
21 return unittest.skip("Network tests being skipped")
22 return lambda f: f
23
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024class URITest(unittest.TestCase):
25 test_uris = {
26 "http://www.google.com/index.html" : {
27 'uri': 'http://www.google.com/index.html',
28 'scheme': 'http',
29 'hostname': 'www.google.com',
30 'port': None,
31 'hostport': 'www.google.com',
32 'path': '/index.html',
33 'userinfo': '',
34 'username': '',
35 'password': '',
36 'params': {},
37 'query': {},
38 'relative': False
39 },
40 "http://www.google.com/index.html;param1=value1" : {
41 'uri': 'http://www.google.com/index.html;param1=value1',
42 'scheme': 'http',
43 'hostname': 'www.google.com',
44 'port': None,
45 'hostport': 'www.google.com',
46 'path': '/index.html',
47 'userinfo': '',
48 'username': '',
49 'password': '',
50 'params': {
51 'param1': 'value1'
52 },
53 'query': {},
54 'relative': False
55 },
56 "http://www.example.org/index.html?param1=value1" : {
57 'uri': 'http://www.example.org/index.html?param1=value1',
58 'scheme': 'http',
59 'hostname': 'www.example.org',
60 'port': None,
61 'hostport': 'www.example.org',
62 'path': '/index.html',
63 'userinfo': '',
64 'username': '',
65 'password': '',
66 'params': {},
67 'query': {
68 'param1': 'value1'
69 },
70 'relative': False
71 },
72 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : {
73 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2',
74 'scheme': 'http',
75 'hostname': 'www.example.org',
76 'port': None,
77 'hostport': 'www.example.org',
78 'path': '/index.html',
79 'userinfo': '',
80 'username': '',
81 'password': '',
82 'params': {
83 'param2': 'value2'
84 },
85 'query': {
86 'qparam1': 'qvalue1'
87 },
88 'relative': False
89 },
Andrew Geisslerd1e89492021-02-12 15:35:20 -060090 # Check that trailing semicolons are handled correctly
91 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2;" : {
92 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2',
93 'scheme': 'http',
94 'hostname': 'www.example.org',
95 'port': None,
96 'hostport': 'www.example.org',
97 'path': '/index.html',
98 'userinfo': '',
99 'username': '',
100 'password': '',
101 'params': {
102 'param2': 'value2'
103 },
104 'query': {
105 'qparam1': 'qvalue1'
106 },
107 'relative': False
108 },
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500109 "http://www.example.com:8080/index.html" : {
110 'uri': 'http://www.example.com:8080/index.html',
111 'scheme': 'http',
112 'hostname': 'www.example.com',
113 'port': 8080,
114 'hostport': 'www.example.com:8080',
115 'path': '/index.html',
116 'userinfo': '',
117 'username': '',
118 'password': '',
119 'params': {},
120 'query': {},
121 'relative': False
122 },
123 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : {
124 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg',
125 'scheme': 'cvs',
126 'hostname': 'cvs.handhelds.org',
127 'port': None,
128 'hostport': 'cvs.handhelds.org',
129 'path': '/cvs',
130 'userinfo': 'anoncvs',
131 'username': 'anoncvs',
132 'password': '',
133 'params': {
134 'module': 'familiar/dist/ipkg'
135 },
136 'query': {},
137 'relative': False
138 },
139 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": {
140 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg',
141 'scheme': 'cvs',
142 'hostname': 'cvs.handhelds.org',
143 'port': None,
144 'hostport': 'cvs.handhelds.org',
145 'path': '/cvs',
146 'userinfo': 'anoncvs:anonymous',
147 'username': 'anoncvs',
148 'password': 'anonymous',
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600149 'params': collections.OrderedDict([
150 ('tag', 'V0-99-81'),
151 ('module', 'familiar/dist/ipkg')
152 ]),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500153 'query': {},
154 'relative': False
155 },
156 "file://example.diff": { # NOTE: Not RFC compliant!
157 'uri': 'file:example.diff',
158 'scheme': 'file',
159 'hostname': '',
160 'port': None,
161 'hostport': '',
162 'path': 'example.diff',
163 'userinfo': '',
164 'username': '',
165 'password': '',
166 'params': {},
167 'query': {},
168 'relative': True
169 },
170 "file:example.diff": { # NOTE: RFC compliant version of the former
171 'uri': 'file:example.diff',
172 'scheme': 'file',
173 'hostname': '',
174 'port': None,
175 'hostport': '',
176 'path': 'example.diff',
177 'userinfo': '',
178 'userinfo': '',
179 'username': '',
180 'password': '',
181 'params': {},
182 'query': {},
183 'relative': True
184 },
185 "file:///tmp/example.diff": {
186 'uri': 'file:///tmp/example.diff',
187 'scheme': 'file',
188 'hostname': '',
189 'port': None,
190 'hostport': '',
191 'path': '/tmp/example.diff',
192 'userinfo': '',
193 'userinfo': '',
194 'username': '',
195 'password': '',
196 'params': {},
197 'query': {},
198 'relative': False
199 },
200 "git:///path/example.git": {
201 'uri': 'git:///path/example.git',
202 'scheme': 'git',
203 'hostname': '',
204 'port': None,
205 'hostport': '',
206 'path': '/path/example.git',
207 'userinfo': '',
208 'userinfo': '',
209 'username': '',
210 'password': '',
211 'params': {},
212 'query': {},
213 'relative': False
214 },
215 "git:path/example.git": {
216 'uri': 'git:path/example.git',
217 'scheme': 'git',
218 'hostname': '',
219 'port': None,
220 'hostport': '',
221 'path': 'path/example.git',
222 'userinfo': '',
223 'userinfo': '',
224 'username': '',
225 'password': '',
226 'params': {},
227 'query': {},
228 'relative': True
229 },
230 "git://example.net/path/example.git": {
231 'uri': 'git://example.net/path/example.git',
232 'scheme': 'git',
233 'hostname': 'example.net',
234 'port': None,
235 'hostport': 'example.net',
236 'path': '/path/example.git',
237 'userinfo': '',
238 'userinfo': '',
239 'username': '',
240 'password': '',
241 'params': {},
242 'query': {},
243 'relative': False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500244 },
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500245 "git://tfs-example.org:22/tfs/example%20path/example.git": {
246 'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git',
247 'scheme': 'git',
248 'hostname': 'tfs-example.org',
249 'port': 22,
250 'hostport': 'tfs-example.org:22',
251 'path': '/tfs/example path/example.git',
252 'userinfo': '',
253 'userinfo': '',
254 'username': '',
255 'password': '',
256 'params': {},
257 'query': {},
258 'relative': False
259 },
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500260 "http://somesite.net;someparam=1": {
261 'uri': 'http://somesite.net;someparam=1',
262 'scheme': 'http',
263 'hostname': 'somesite.net',
264 'port': None,
265 'hostport': 'somesite.net',
266 'path': '',
267 'userinfo': '',
268 'userinfo': '',
269 'username': '',
270 'password': '',
271 'params': {"someparam" : "1"},
272 'query': {},
273 'relative': False
274 },
275 "file://somelocation;someparam=1": {
276 'uri': 'file:somelocation;someparam=1',
277 'scheme': 'file',
278 'hostname': '',
279 'port': None,
280 'hostport': '',
281 'path': 'somelocation',
282 'userinfo': '',
283 'userinfo': '',
284 'username': '',
285 'password': '',
286 'params': {"someparam" : "1"},
287 'query': {},
288 'relative': True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500289 }
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500290
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500291 }
292
293 def test_uri(self):
294 for test_uri, ref in self.test_uris.items():
295 uri = URI(test_uri)
296
297 self.assertEqual(str(uri), ref['uri'])
298
299 # expected attributes
300 self.assertEqual(uri.scheme, ref['scheme'])
301
302 self.assertEqual(uri.userinfo, ref['userinfo'])
303 self.assertEqual(uri.username, ref['username'])
304 self.assertEqual(uri.password, ref['password'])
305
306 self.assertEqual(uri.hostname, ref['hostname'])
307 self.assertEqual(uri.port, ref['port'])
308 self.assertEqual(uri.hostport, ref['hostport'])
309
310 self.assertEqual(uri.path, ref['path'])
311 self.assertEqual(uri.params, ref['params'])
312
313 self.assertEqual(uri.relative, ref['relative'])
314
315 def test_dict(self):
316 for test in self.test_uris.values():
317 uri = URI()
318
319 self.assertEqual(uri.scheme, '')
320 self.assertEqual(uri.userinfo, '')
321 self.assertEqual(uri.username, '')
322 self.assertEqual(uri.password, '')
323 self.assertEqual(uri.hostname, '')
324 self.assertEqual(uri.port, None)
325 self.assertEqual(uri.path, '')
326 self.assertEqual(uri.params, {})
327
328
329 uri.scheme = test['scheme']
330 self.assertEqual(uri.scheme, test['scheme'])
331
332 uri.userinfo = test['userinfo']
333 self.assertEqual(uri.userinfo, test['userinfo'])
334 self.assertEqual(uri.username, test['username'])
335 self.assertEqual(uri.password, test['password'])
336
337 # make sure changing the values doesn't do anything unexpected
338 uri.username = 'changeme'
339 self.assertEqual(uri.username, 'changeme')
340 self.assertEqual(uri.password, test['password'])
341 uri.password = 'insecure'
342 self.assertEqual(uri.username, 'changeme')
343 self.assertEqual(uri.password, 'insecure')
344
345 # reset back after our trickery
346 uri.userinfo = test['userinfo']
347 self.assertEqual(uri.userinfo, test['userinfo'])
348 self.assertEqual(uri.username, test['username'])
349 self.assertEqual(uri.password, test['password'])
350
351 uri.hostname = test['hostname']
352 self.assertEqual(uri.hostname, test['hostname'])
353 self.assertEqual(uri.hostport, test['hostname'])
354
355 uri.port = test['port']
356 self.assertEqual(uri.port, test['port'])
357 self.assertEqual(uri.hostport, test['hostport'])
358
359 uri.path = test['path']
360 self.assertEqual(uri.path, test['path'])
361
362 uri.params = test['params']
363 self.assertEqual(uri.params, test['params'])
364
365 uri.query = test['query']
366 self.assertEqual(uri.query, test['query'])
367
368 self.assertEqual(str(uri), test['uri'])
369
370 uri.params = {}
371 self.assertEqual(uri.params, {})
372 self.assertEqual(str(uri), (str(uri).split(";"))[0])
373
374class FetcherTest(unittest.TestCase):
375
376 def setUp(self):
377 self.origdir = os.getcwd()
378 self.d = bb.data.init()
Andrew Geisslereff27472021-10-29 15:35:00 -0500379 self.tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500380 self.dldir = os.path.join(self.tempdir, "download")
381 os.mkdir(self.dldir)
382 self.d.setVar("DL_DIR", self.dldir)
383 self.unpackdir = os.path.join(self.tempdir, "unpacked")
384 os.mkdir(self.unpackdir)
385 persistdir = os.path.join(self.tempdir, "persistdata")
386 self.d.setVar("PERSISTENT_DIR", persistdir)
387
388 def tearDown(self):
389 os.chdir(self.origdir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600390 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
391 print("Not cleaning up %s. Please remove manually." % self.tempdir)
392 else:
Andrew Geisslerc926e172021-05-07 16:11:35 -0500393 bb.process.run('chmod u+rw -R %s' % self.tempdir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600394 bb.utils.prunedir(self.tempdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500395
396class MirrorUriTest(FetcherTest):
397
398 replaceuris = {
399 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/")
400 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
401 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
402 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
403 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
404 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
405 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http")
406 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
407 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake")
408 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890",
409 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache")
410 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
411 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/")
412 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
413 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3")
414 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
415 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz")
416 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
417 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist")
418 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2",
419 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/")
420 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2",
421 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
422 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
423 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
424 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
425 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http")
426 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800427 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org")
428 : "http://somewhere2.org/somefile_1.2.3.tar.gz",
429 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/")
430 : "http://somewhere2.org/somefile_1.2.3.tar.gz",
431 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http")
432 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500433
434 #Renaming files doesn't work
435 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
436 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
437 }
438
439 mirrorvar = "http://.*/.* file:///somepath/downloads/ \n" \
440 "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n" \
441 "https://.*/.* file:///someotherpath/downloads/ \n" \
442 "http://.*/.* file:///someotherpath/downloads/ \n"
443
444 def test_urireplace(self):
445 for k, v in self.replaceuris.items():
446 ud = bb.fetch.FetchData(k[0], self.d)
447 ud.setup_localpath(self.d)
448 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2]))
449 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d)
450 self.assertEqual([v], newuris)
451
452 def test_urilist1(self):
453 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
454 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
455 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
456 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
457
458 def test_urilist2(self):
459 # Catch https:// -> files:// bug
460 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
461 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
462 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
463 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
464
465 def test_mirror_of_mirror(self):
466 # Test if mirror of a mirror works
467 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/ \n"
468 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/ \n"
469 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
470 mirrors = bb.fetch2.mirror_from_string(mirrorvar)
471 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
472 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz',
473 'file:///someotherpath/downloads/bitbake-1.0.tar.gz',
474 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz',
475 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz'])
476
Patrick Williamsd7e96312015-09-22 08:09:05 -0500477 recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ \n" \
478 "https://.*/[^/]* https://BBBB/B/B/B/ \n"
479
480 def test_recursive(self):
481 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
482 mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar)
483 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
484 self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz',
485 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz',
486 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500487
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800488
489class GitDownloadDirectoryNamingTest(FetcherTest):
490 def setUp(self):
491 super(GitDownloadDirectoryNamingTest, self).setUp()
492 self.recipe_url = "git://git.openembedded.org/bitbake"
493 self.recipe_dir = "git.openembedded.org.bitbake"
494 self.mirror_url = "git://github.com/openembedded/bitbake.git"
495 self.mirror_dir = "github.com.openembedded.bitbake.git"
496
497 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
498
499 def setup_mirror_rewrite(self):
500 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
501
502 @skipIfNoNetwork()
503 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self):
504 self.setup_mirror_rewrite()
505 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
506
507 fetcher.download()
508
509 dir = os.listdir(self.dldir + "/git2")
510 self.assertIn(self.recipe_dir, dir)
511
512 @skipIfNoNetwork()
513 def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self):
514 self.setup_mirror_rewrite()
515 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
516
517 fetcher.download()
518
519 dir = os.listdir(self.dldir + "/git2")
520 self.assertIn(self.mirror_dir, dir)
521 self.assertIn(self.recipe_dir, dir)
522
523 @skipIfNoNetwork()
524 def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self):
525 self.setup_mirror_rewrite()
526 fetcher = bb.fetch.Fetch([self.mirror_url], self.d)
527 fetcher.download()
528 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
529
530 fetcher.download()
531
532 dir = os.listdir(self.dldir + "/git2")
533 self.assertIn(self.mirror_dir, dir)
534 self.assertIn(self.recipe_dir, dir)
535
536
537class TarballNamingTest(FetcherTest):
538 def setUp(self):
539 super(TarballNamingTest, self).setUp()
540 self.recipe_url = "git://git.openembedded.org/bitbake"
541 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
542 self.mirror_url = "git://github.com/openembedded/bitbake.git"
543 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz"
544
545 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
546 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
547
548 def setup_mirror_rewrite(self):
549 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
550
551 @skipIfNoNetwork()
552 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self):
553 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
554
555 fetcher.download()
556
557 dir = os.listdir(self.dldir)
558 self.assertIn(self.recipe_tarball, dir)
559
560 @skipIfNoNetwork()
561 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
562 self.setup_mirror_rewrite()
563 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
564
565 fetcher.download()
566
567 dir = os.listdir(self.dldir)
568 self.assertIn(self.mirror_tarball, dir)
569
570
571class GitShallowTarballNamingTest(FetcherTest):
572 def setUp(self):
573 super(GitShallowTarballNamingTest, self).setUp()
574 self.recipe_url = "git://git.openembedded.org/bitbake"
575 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz"
576 self.mirror_url = "git://github.com/openembedded/bitbake.git"
577 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz"
578
579 self.d.setVar('BB_GIT_SHALLOW', '1')
580 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
581 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
582
583 def setup_mirror_rewrite(self):
584 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
585
586 @skipIfNoNetwork()
587 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self):
588 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
589
590 fetcher.download()
591
592 dir = os.listdir(self.dldir)
593 self.assertIn(self.recipe_tarball, dir)
594
595 @skipIfNoNetwork()
596 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
597 self.setup_mirror_rewrite()
598 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
599
600 fetcher.download()
601
602 dir = os.listdir(self.dldir)
603 self.assertIn(self.mirror_tarball, dir)
604
605
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500606class FetcherLocalTest(FetcherTest):
607 def setUp(self):
608 def touch(fn):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600609 with open(fn, 'a'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500610 os.utime(fn, None)
611
612 super(FetcherLocalTest, self).setUp()
613 self.localsrcdir = os.path.join(self.tempdir, 'localsrc')
614 os.makedirs(self.localsrcdir)
615 touch(os.path.join(self.localsrcdir, 'a'))
616 touch(os.path.join(self.localsrcdir, 'b'))
617 os.makedirs(os.path.join(self.localsrcdir, 'dir'))
618 touch(os.path.join(self.localsrcdir, 'dir', 'c'))
619 touch(os.path.join(self.localsrcdir, 'dir', 'd'))
620 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir'))
621 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e'))
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500622 touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device'))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500623 self.d.setVar("FILESPATH", self.localsrcdir)
624
625 def fetchUnpack(self, uris):
626 fetcher = bb.fetch.Fetch(uris, self.d)
627 fetcher.download()
628 fetcher.unpack(self.unpackdir)
629 flst = []
630 for root, dirs, files in os.walk(self.unpackdir):
631 for f in files:
632 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir))
633 flst.sort()
634 return flst
635
636 def test_local(self):
637 tree = self.fetchUnpack(['file://a', 'file://dir/c'])
638 self.assertEqual(tree, ['a', 'dir/c'])
639
Andrew Geisslerc3d88e42020-10-02 09:45:00 -0500640 def test_local_backslash(self):
641 tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device'])
642 self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device'])
643
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500644 def test_local_wildcard(self):
Andrew Geisslerc9f78652020-09-18 14:11:35 -0500645 with self.assertRaises(bb.fetch2.ParameterError):
646 tree = self.fetchUnpack(['file://a', 'file://dir/*'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500647
648 def test_local_dir(self):
649 tree = self.fetchUnpack(['file://a', 'file://dir'])
650 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e'])
651
652 def test_local_subdir(self):
653 tree = self.fetchUnpack(['file://dir/subdir'])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500654 self.assertEqual(tree, ['dir/subdir/e'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500655
656 def test_local_subdir_file(self):
657 tree = self.fetchUnpack(['file://dir/subdir/e'])
658 self.assertEqual(tree, ['dir/subdir/e'])
659
660 def test_local_subdirparam(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500661 tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo'])
662 self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500663
664 def test_local_deepsubdirparam(self):
665 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar'])
666 self.assertEqual(tree, ['bar/dir/subdir/e'])
667
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600668 def test_local_absolutedir(self):
669 # Unpacking to an absolute path that is a subdirectory of the root
670 # should work
671 tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')])
672
673 # Unpacking to an absolute path outside of the root should fail
674 with self.assertRaises(bb.fetch2.UnpackError):
675 self.fetchUnpack(['file://a;subdir=/bin/sh'])
676
Andrew Geisslerc926e172021-05-07 16:11:35 -0500677 def dummyGitTest(self, suffix):
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600678 # Create dummy local Git repo
679 src_dir = tempfile.mkdtemp(dir=self.tempdir,
680 prefix='gitfetch_localusehead_')
681 src_dir = os.path.abspath(src_dir)
682 bb.process.run("git init", cwd=src_dir)
Andrew Geisslerc926e172021-05-07 16:11:35 -0500683 bb.process.run("git config user.email 'you@example.com'", cwd=src_dir)
684 bb.process.run("git config user.name 'Your Name'", cwd=src_dir)
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600685 bb.process.run("git commit --allow-empty -m'Dummy commit'",
686 cwd=src_dir)
687 # Use other branch than master
688 bb.process.run("git checkout -b my-devel", cwd=src_dir)
689 bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
690 cwd=src_dir)
691 stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
692 orig_rev = stdout[0].strip()
693
694 # Fetch and check revision
695 self.d.setVar("SRCREV", "AUTOINC")
Andrew Geisslerc926e172021-05-07 16:11:35 -0500696 url = "git://" + src_dir + ";protocol=file;" + suffix
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600697 fetcher = bb.fetch.Fetch([url], self.d)
698 fetcher.download()
699 fetcher.unpack(self.unpackdir)
700 stdout = bb.process.run("git rev-parse HEAD",
701 cwd=os.path.join(self.unpackdir, 'git'))
702 unpack_rev = stdout[0].strip()
703 self.assertEqual(orig_rev, unpack_rev)
704
Andrew Geisslerc926e172021-05-07 16:11:35 -0500705 def test_local_gitfetch_usehead(self):
706 self.dummyGitTest("usehead=1")
707
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600708 def test_local_gitfetch_usehead_withname(self):
Andrew Geisslerc926e172021-05-07 16:11:35 -0500709 self.dummyGitTest("usehead=1;name=newName")
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600710
Andrew Geisslerc926e172021-05-07 16:11:35 -0500711 def test_local_gitfetch_shared(self):
712 self.dummyGitTest("usehead=1;name=sharedName")
713 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
714 self.assertTrue(os.path.exists(alt))
715
716 def test_local_gitfetch_noshared(self):
717 self.d.setVar('BB_GIT_NOSHARED', '1')
718 self.unpackdir += '_noshared'
719 self.dummyGitTest("usehead=1;name=noSharedName")
720 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
721 self.assertFalse(os.path.exists(alt))
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600722
Brad Bishop316dfdd2018-06-25 12:45:53 -0400723class FetcherNoNetworkTest(FetcherTest):
724 def setUp(self):
725 super().setUp()
726 # all test cases are based on not having network
727 self.d.setVar("BB_NO_NETWORK", "1")
728
729 def test_missing(self):
730 string = "this is a test file\n".encode("utf-8")
731 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
732 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
733
734 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
735 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
736 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
737 with self.assertRaises(bb.fetch2.NetworkAccess):
738 fetcher.download()
739
740 def test_valid_missing_donestamp(self):
741 # create the file in the download directory with correct hash
742 string = "this is a test file\n".encode("utf-8")
743 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f:
744 f.write(string)
745
746 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
747 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
748
749 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
750 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
751 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
752 fetcher.download()
753 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
754
755 def test_invalid_missing_donestamp(self):
756 # create an invalid file in the download directory with incorrect hash
757 string = "this is a test file\n".encode("utf-8")
758 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
759 pass
760
761 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
762 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
763
764 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
765 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
766 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
767 with self.assertRaises(bb.fetch2.NetworkAccess):
768 fetcher.download()
769 # the existing file should not exist or should have be moved to "bad-checksum"
770 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
771
772 def test_nochecksums_missing(self):
773 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
774 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
775 # ssh fetch does not support checksums
776 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
777 # attempts to download with missing donestamp
778 with self.assertRaises(bb.fetch2.NetworkAccess):
779 fetcher.download()
780
781 def test_nochecksums_missing_donestamp(self):
782 # create a file in the download directory
783 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
784 pass
785
786 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
787 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
788 # ssh fetch does not support checksums
789 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
790 # attempts to download with missing donestamp
791 with self.assertRaises(bb.fetch2.NetworkAccess):
792 fetcher.download()
793
794 def test_nochecksums_has_donestamp(self):
795 # create a file in the download directory with the donestamp
796 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
797 pass
798 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
799 pass
800
801 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
802 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
803 # ssh fetch does not support checksums
804 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
805 # should not fetch
806 fetcher.download()
807 # both files should still exist
808 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
809 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
810
811 def test_nochecksums_missing_has_donestamp(self):
812 # create a file in the download directory with the donestamp
813 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
814 pass
815
816 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
817 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
818 # ssh fetch does not support checksums
819 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
820 with self.assertRaises(bb.fetch2.NetworkAccess):
821 fetcher.download()
822 # both files should still exist
823 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
824 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
825
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500826class FetcherNetworkTest(FetcherTest):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500827 @skipIfNoNetwork()
828 def test_fetch(self):
Andrew Geisslereff27472021-10-29 15:35:00 -0500829 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500830 fetcher.download()
831 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
832 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892)
833 self.d.setVar("BB_NO_NETWORK", "1")
Andrew Geisslereff27472021-10-29 15:35:00 -0500834 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500835 fetcher.download()
836 fetcher.unpack(self.unpackdir)
837 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9)
838 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500839
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500840 @skipIfNoNetwork()
841 def test_fetch_mirror(self):
Andrew Geisslereff27472021-10-29 15:35:00 -0500842 self.d.setVar("MIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500843 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
844 fetcher.download()
845 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
846
847 @skipIfNoNetwork()
848 def test_fetch_mirror_of_mirror(self):
Andrew Geisslereff27472021-10-29 15:35:00 -0500849 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ \n http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500850 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
851 fetcher.download()
852 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
853
854 @skipIfNoNetwork()
855 def test_fetch_file_mirror_of_mirror(self):
Andrew Geisslereff27472021-10-29 15:35:00 -0500856 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ \n file:///some1where/.* file://some2where/ \n file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500857 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
858 os.mkdir(self.dldir + "/some2where")
859 fetcher.download()
860 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
861
862 @skipIfNoNetwork()
863 def test_fetch_premirror(self):
Andrew Geisslereff27472021-10-29 15:35:00 -0500864 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500865 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
866 fetcher.download()
867 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
868
869 @skipIfNoNetwork()
Andrew Geissler5199d832021-09-24 16:47:35 -0500870 def test_fetch_specify_downloadfilename(self):
Andrew Geisslereff27472021-10-29 15:35:00 -0500871 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d)
Andrew Geissler5199d832021-09-24 16:47:35 -0500872 fetcher.download()
873 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749)
874
875 @skipIfNoNetwork()
876 def test_fetch_premirror_specify_downloadfilename_regex_uri(self):
Andrew Geisslereff27472021-10-29 15:35:00 -0500877 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/")
Andrew Geissler5199d832021-09-24 16:47:35 -0500878 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d)
879 fetcher.download()
880 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749)
881
882 @skipIfNoNetwork()
883 # BZ13039
884 def test_fetch_premirror_specify_downloadfilename_specific_uri(self):
Andrew Geisslereff27472021-10-29 15:35:00 -0500885 self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake")
Andrew Geissler5199d832021-09-24 16:47:35 -0500886 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d)
887 fetcher.download()
888 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749)
889
890 @skipIfNoNetwork()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500891 def gitfetcher(self, url1, url2):
892 def checkrevision(self, fetcher):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500893 fetcher.unpack(self.unpackdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500894 revision = bb.process.run("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git")[0].strip()
895 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500896
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500897 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
898 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
899 fetcher = bb.fetch.Fetch([url1], self.d)
900 fetcher.download()
901 checkrevision(self, fetcher)
902 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works
903 bb.utils.prunedir(self.dldir + "/git2/")
904 bb.utils.prunedir(self.unpackdir)
905 self.d.setVar("BB_NO_NETWORK", "1")
906 fetcher = bb.fetch.Fetch([url2], self.d)
907 fetcher.download()
908 checkrevision(self, fetcher)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500909
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500910 @skipIfNoNetwork()
911 def test_gitfetch(self):
912 url1 = url2 = "git://git.openembedded.org/bitbake"
913 self.gitfetcher(url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500914
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500915 @skipIfNoNetwork()
916 def test_gitfetch_goodsrcrev(self):
917 # SRCREV is set but matches rev= parameter
918 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
919 self.gitfetcher(url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500920
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500921 @skipIfNoNetwork()
922 def test_gitfetch_badsrcrev(self):
923 # SRCREV is set but does not match rev= parameter
924 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5"
925 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500926
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500927 @skipIfNoNetwork()
928 def test_gitfetch_tagandrev(self):
929 # SRCREV is set but does not match rev= parameter
930 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
931 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500932
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500933 @skipIfNoNetwork()
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600934 def test_gitfetch_usehead(self):
935 # Since self.gitfetcher() sets SRCREV we expect this to override
936 # `usehead=1' and instead fetch the specified SRCREV. See
937 # test_local_gitfetch_usehead() for a positive use of the usehead
938 # feature.
939 url = "git://git.openembedded.org/bitbake;usehead=1"
940 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500941
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500942 @skipIfNoNetwork()
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600943 def test_gitfetch_usehead_withname(self):
944 # Since self.gitfetcher() sets SRCREV we expect this to override
945 # `usehead=1' and instead fetch the specified SRCREV. See
946 # test_local_gitfetch_usehead() for a positive use of the usehead
947 # feature.
948 url = "git://git.openembedded.org/bitbake;usehead=1;name=newName"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500949 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500950
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500951 @skipIfNoNetwork()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800952 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self):
953 recipeurl = "git://git.openembedded.org/bitbake"
954 mirrorurl = "git://someserver.org/bitbake"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500955 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800956 self.gitfetcher(recipeurl, mirrorurl)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500957
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500958 @skipIfNoNetwork()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800959 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self):
960 recipeurl = "git://someserver.org/bitbake"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500961 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800962 self.gitfetcher(recipeurl, recipeurl)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500963
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500964 @skipIfNoNetwork()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800965 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500966 realurl = "git://git.openembedded.org/bitbake"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800967 recipeurl = "git://someserver.org/bitbake"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500968 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git")
969 os.chdir(self.tempdir)
970 bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800971 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (recipeurl, self.sourcedir))
972 self.gitfetcher(recipeurl, recipeurl)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600973
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500974 @skipIfNoNetwork()
975 def test_git_submodule(self):
Brad Bishopf8caae32019-03-25 13:13:56 -0400976 # URL with ssh submodules
977 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7"
978 # Original URL (comment this if you have ssh access to git.yoctoproject.org)
979 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee"
980 fetcher = bb.fetch.Fetch([url], self.d)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500981 fetcher.download()
982 # Previous cwd has been deleted
983 os.chdir(os.path.dirname(self.unpackdir))
984 fetcher.unpack(self.unpackdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500985
Brad Bishopf8caae32019-03-25 13:13:56 -0400986 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
987 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing')
988 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing')
989 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present')
990
991 # Only when we're running the extended test with a submodule's submodule, can we check this.
992 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
993 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
994
Brad Bishop96ff1982019-08-19 13:50:42 -0400995 @skipIfNoNetwork()
Brad Bishopf8caae32019-03-25 13:13:56 -0400996 def test_git_submodule_dbus_broker(self):
997 # The following external repositories have show failures in fetch and unpack operations
998 # We want to avoid regressions!
Andrew Geissler4c19ea12020-10-27 13:52:24 -0500999 url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main"
Brad Bishopf8caae32019-03-25 13:13:56 -04001000 fetcher = bb.fetch.Fetch([url], self.d)
1001 fetcher.download()
1002 # Previous cwd has been deleted
1003 os.chdir(os.path.dirname(self.unpackdir))
1004 fetcher.unpack(self.unpackdir)
1005
1006 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1007 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"')
1008 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"')
1009 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"')
1010 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"')
1011 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"')
1012
Brad Bishop96ff1982019-08-19 13:50:42 -04001013 @skipIfNoNetwork()
Brad Bishopf8caae32019-03-25 13:13:56 -04001014 def test_git_submodule_CLI11(self):
Andrew Geisslereff27472021-10-29 15:35:00 -05001015 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main"
Brad Bishopf8caae32019-03-25 13:13:56 -04001016 fetcher = bb.fetch.Fetch([url], self.d)
1017 fetcher.download()
1018 # Previous cwd has been deleted
1019 os.chdir(os.path.dirname(self.unpackdir))
1020 fetcher.unpack(self.unpackdir)
1021
1022 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1023 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
1024 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
1025 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
1026
Brad Bishop96ff1982019-08-19 13:50:42 -04001027 @skipIfNoNetwork()
Brad Bishop19323692019-04-05 15:28:33 -04001028 def test_git_submodule_update_CLI11(self):
1029 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
Andrew Geisslereff27472021-10-29 15:35:00 -05001030 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main"
Brad Bishop19323692019-04-05 15:28:33 -04001031 fetcher = bb.fetch.Fetch([url], self.d)
1032 fetcher.download()
1033
1034 # CLI11 that pulls in a newer nlohmann-json
Andrew Geisslereff27472021-10-29 15:35:00 -05001035 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main"
Brad Bishop19323692019-04-05 15:28:33 -04001036 fetcher = bb.fetch.Fetch([url], self.d)
1037 fetcher.download()
1038 # Previous cwd has been deleted
1039 os.chdir(os.path.dirname(self.unpackdir))
1040 fetcher.unpack(self.unpackdir)
1041
1042 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1043 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
1044 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
1045 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
1046
Brad Bishop96ff1982019-08-19 13:50:42 -04001047 @skipIfNoNetwork()
Brad Bishopf8caae32019-03-25 13:13:56 -04001048 def test_git_submodule_aktualizr(self):
1049 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
1050 fetcher = bb.fetch.Fetch([url], self.d)
1051 fetcher.download()
1052 # Previous cwd has been deleted
1053 os.chdir(os.path.dirname(self.unpackdir))
1054 fetcher.unpack(self.unpackdir)
1055
1056 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1057 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"')
1058 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"')
1059 self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly")
1060 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"')
1061 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"')
1062 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"')
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001063
Brad Bishop96ff1982019-08-19 13:50:42 -04001064 @skipIfNoNetwork()
Brad Bishop393846f2019-05-20 12:24:11 -04001065 def test_git_submodule_iotedge(self):
1066 """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """
1067
1068 # This repository also has submodules where the module (name), path and url do not align
1069 url = "gitsm://github.com/azure/iotedge.git;protocol=git;rev=d76e0316c6f324345d77c48a83ce836d09392699"
1070 fetcher = bb.fetch.Fetch([url], self.d)
1071 fetcher.download()
1072 # Previous cwd has been deleted
1073 os.chdir(os.path.dirname(self.unpackdir))
1074 fetcher.unpack(self.unpackdir)
1075
1076 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
1077
1078 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/README.md')), msg='Missing submodule checkout')
1079 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/ctest/README.md')), msg='Missing submodule checkout')
1080 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/testrunner/readme.md')), msg='Missing submodule checkout')
1081 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/readme.md')), msg='Missing submodule checkout')
1082 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
1083 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
1084 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/README.md')), msg='Missing submodule checkout')
1085 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/README.md')), msg='Missing submodule checkout')
1086 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/ctest/README.md')), msg='Missing submodule checkout')
1087 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/testrunner/readme.md')), msg='Missing submodule checkout')
1088 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/readme.md')), msg='Missing submodule checkout')
1089 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
1090 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
1091
Brad Bishop15ae2502019-06-18 21:44:24 -04001092class SVNTest(FetcherTest):
1093 def skipIfNoSvn():
1094 import shutil
1095 if not shutil.which("svn"):
1096 return unittest.skip("svn not installed, tests being skipped")
1097
1098 if not shutil.which("svnadmin"):
1099 return unittest.skip("svnadmin not installed, tests being skipped")
1100
1101 return lambda f: f
1102
1103 @skipIfNoSvn()
1104 def setUp(self):
1105 """ Create a local repository """
1106
1107 super(SVNTest, self).setUp()
1108
1109 # Create something we can fetch
1110 src_dir = tempfile.mkdtemp(dir=self.tempdir,
1111 prefix='svnfetch_srcdir_')
1112 src_dir = os.path.abspath(src_dir)
1113 bb.process.run("echo readme > README.md", cwd=src_dir)
1114
1115 # Store it in a local SVN repository
1116 repo_dir = tempfile.mkdtemp(dir=self.tempdir,
1117 prefix='svnfetch_localrepo_')
1118 repo_dir = os.path.abspath(repo_dir)
1119 bb.process.run("svnadmin create project", cwd=repo_dir)
1120
1121 self.repo_url = "file://%s/project" % repo_dir
1122 bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url),
1123 cwd=repo_dir)
1124
1125 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
1126 # Github will emulate SVN. Use this to check if we're downloding...
Andrew Geissler475cb722020-07-10 16:00:51 -05001127 bb.process.run("svn propset svn:externals 'bitbake svn://vcs.pcre.org/pcre2/code' .",
Brad Bishop15ae2502019-06-18 21:44:24 -04001128 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
1129 bb.process.run("svn commit --non-interactive -m 'Add external'",
1130 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
1131
1132 self.src_dir = src_dir
1133 self.repo_dir = repo_dir
1134
1135 @skipIfNoSvn()
1136 def tearDown(self):
1137 os.chdir(self.origdir)
1138 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
1139 print("Not cleaning up %s. Please remove manually." % self.tempdir)
1140 else:
1141 bb.utils.prunedir(self.tempdir)
1142
1143 @skipIfNoSvn()
1144 @skipIfNoNetwork()
1145 def test_noexternal_svn(self):
1146 # Always match the rev count from setUp (currently rev 2)
1147 url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '')
1148 fetcher = bb.fetch.Fetch([url], self.d)
1149 fetcher.download()
1150 os.chdir(os.path.dirname(self.unpackdir))
1151 fetcher.unpack(self.unpackdir)
1152
1153 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
1154 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
1155 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should NOT exist")
1156 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should NOT exit")
1157
1158 @skipIfNoSvn()
1159 def test_external_svn(self):
1160 # Always match the rev count from setUp (currently rev 2)
1161 url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '')
1162 fetcher = bb.fetch.Fetch([url], self.d)
1163 fetcher.download()
1164 os.chdir(os.path.dirname(self.unpackdir))
1165 fetcher.unpack(self.unpackdir)
1166
1167 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
1168 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
1169 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should exist")
1170 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should exit")
1171
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001172class TrustedNetworksTest(FetcherTest):
1173 def test_trusted_network(self):
1174 # Ensure trusted_network returns False when the host IS in the list.
1175 url = "git://Someserver.org/foo;rev=1"
1176 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org")
1177 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001178
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001179 def test_wild_trusted_network(self):
1180 # Ensure trusted_network returns true when the *.host IS in the list.
1181 url = "git://Someserver.org/foo;rev=1"
1182 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1183 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001184
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001185 def test_prefix_wild_trusted_network(self):
1186 # Ensure trusted_network returns true when the prefix matches *.host.
1187 url = "git://git.Someserver.org/foo;rev=1"
1188 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1189 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001190
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001191 def test_two_prefix_wild_trusted_network(self):
1192 # Ensure trusted_network returns true when the prefix matches *.host.
1193 url = "git://something.git.Someserver.org/foo;rev=1"
1194 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1195 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001196
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001197 def test_port_trusted_network(self):
1198 # Ensure trusted_network returns True, even if the url specifies a port.
1199 url = "git://someserver.org:8080/foo;rev=1"
1200 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org")
1201 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001202
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001203 def test_untrusted_network(self):
1204 # Ensure trusted_network returns False when the host is NOT in the list.
1205 url = "git://someserver.org/foo;rev=1"
1206 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
1207 self.assertFalse(bb.fetch.trusted_network(self.d, url))
1208
1209 def test_wild_untrusted_network(self):
1210 # Ensure trusted_network returns False when the host is NOT in the list.
1211 url = "git://*.someserver.org/foo;rev=1"
1212 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
1213 self.assertFalse(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001214
1215class URLHandle(unittest.TestCase):
1216
1217 datatable = {
1218 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}),
1219 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}),
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001220 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001221 "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'}),
1222 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}),
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001223 }
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001224 # we require a pathname to encodeurl but users can still pass such urls to
1225 # decodeurl and we need to handle them
1226 decodedata = datatable.copy()
1227 decodedata.update({
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001228 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001229 })
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001230
1231 def test_decodeurl(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001232 for k, v in self.decodedata.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001233 result = bb.fetch.decodeurl(k)
1234 self.assertEqual(result, v)
1235
1236 def test_encodeurl(self):
1237 for k, v in self.datatable.items():
1238 result = bb.fetch.encodeurl(v)
1239 self.assertEqual(result, k)
1240
1241class FetchLatestVersionTest(FetcherTest):
1242
1243 test_git_uris = {
1244 # version pattern "X.Y.Z"
1245 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
1246 : "1.99.4",
1247 # version pattern "vX.Y"
Andrew Geisslerd25ed322020-06-27 00:28:28 -05001248 # mirror of git.infradead.org since network issues interfered with testing
1249 ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001250 : "1.5.0",
1251 # version pattern "pkg_name-X.Y"
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001252 # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing
1253 ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001254 : "1.0",
1255 # version pattern "pkg_name-vX.Y.Z"
Andrew Geissler5199d832021-09-24 16:47:35 -05001256 ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001257 : "1.4.0",
1258 # combination version pattern
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001259 ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001260 : "1.2.0",
1261 ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "")
1262 : "2014.01",
1263 # version pattern "yyyymmdd"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001264 ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001265 : "20120614",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001266 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
Andrew Geisslerc9f78652020-09-18 14:11:35 -05001267 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing
Andrew Geissler5199d832021-09-24 16:47:35 -05001268 ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001269 : "0.4.3",
Andrew Geissler5199d832021-09-24 16:47:35 -05001270 ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001271 : "11.0.0",
Andrew Geissler5199d832021-09-24 16:47:35 -05001272 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001273 : "1.3.59",
Andrew Geissler5199d832021-09-24 16:47:35 -05001274 ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001275 : "3.82+dbg0.9",
1276 }
1277
1278 test_wget_uris = {
Andrew Geissler82c905d2020-04-13 13:39:40 -05001279 #
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001280 # packages with versions inside directory name
Andrew Geissler82c905d2020-04-13 13:39:40 -05001281 #
1282 # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2
1283 ("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001284 : "2.24.2",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001285 # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz
1286 ("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001287 : "1.6.0",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001288 # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz
1289 ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001290 : "2.8.12.1",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001291 #
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001292 # packages with versions only in current directory
Andrew Geissler82c905d2020-04-13 13:39:40 -05001293 #
Andrew Geisslereff27472021-10-29 15:35:00 -05001294 # https://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2
Andrew Geissler82c905d2020-04-13 13:39:40 -05001295 ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001296 : "2.19",
Andrew Geisslereff27472021-10-29 15:35:00 -05001297 # https://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2
Andrew Geissler82c905d2020-04-13 13:39:40 -05001298 ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001299 : "20120814",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001300 #
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001301 # packages with "99" in the name of possible version
Andrew Geissler82c905d2020-04-13 13:39:40 -05001302 #
1303 # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz
1304 ("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001305 : "5.0",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001306 # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2
1307 ("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001308 : "1.15.1",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001309 #
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001310 # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX
Andrew Geissler82c905d2020-04-13 13:39:40 -05001311 #
1312 # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2
1313 # https://github.com/apple/cups/releases
Andrew Geissler5199d832021-09-24 16:47:35 -05001314 ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", r"(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001315 : "2.0.0",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001316 # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz
1317 # http://ftp.debian.org/debian/pool/main/d/db5.3/
Andrew Geissler5199d832021-09-24 16:47:35 -05001318 ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", r"(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz")
Brad Bishop79641f22019-09-10 07:20:22 -04001319 : "5.3.10",
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001320 }
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001321
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001322 @skipIfNoNetwork()
1323 def test_git_latest_versionstring(self):
1324 for k, v in self.test_git_uris.items():
1325 self.d.setVar("PN", k[0])
1326 self.d.setVar("SRCREV", k[2])
1327 self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3])
1328 ud = bb.fetch2.FetchData(k[1], self.d)
1329 pupver= ud.method.latest_versionstring(ud, self.d)
1330 verstring = pupver[0]
Brad Bishop316dfdd2018-06-25 12:45:53 -04001331 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001332 r = bb.utils.vercmp_string(v, verstring)
1333 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
1334
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001335 def test_wget_latest_versionstring(self):
Andrew Geissler82c905d2020-04-13 13:39:40 -05001336 testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata"
1337 server = HTTPService(testdata)
1338 server.start()
1339 port = server.port
1340 try:
1341 for k, v in self.test_wget_uris.items():
1342 self.d.setVar("PN", k[0])
1343 checkuri = ""
1344 if k[2]:
1345 checkuri = "http://localhost:%s/" % port + k[2]
1346 self.d.setVar("UPSTREAM_CHECK_URI", checkuri)
1347 self.d.setVar("UPSTREAM_CHECK_REGEX", k[3])
1348 url = "http://localhost:%s/" % port + k[1]
1349 ud = bb.fetch2.FetchData(url, self.d)
1350 pupver = ud.method.latest_versionstring(ud, self.d)
1351 verstring = pupver[0]
1352 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
1353 r = bb.utils.vercmp_string(v, verstring)
1354 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
1355 finally:
1356 server.stop()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001357
1358
1359class FetchCheckStatusTest(FetcherTest):
Andrew Geisslereff27472021-10-29 15:35:00 -05001360 test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
1361 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
1362 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001363 "https://yoctoproject.org/",
Andrew Geissler95ac1b82021-03-31 14:34:31 -05001364 "https://docs.yoctoproject.org",
Andrew Geisslereff27472021-10-29 15:35:00 -05001365 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
1366 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001367 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz",
1368 "http://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz",
1369 "https://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz",
1370 "https://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001371 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD
1372 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001373 ]
1374
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001375 @skipIfNoNetwork()
1376 def test_wget_checkstatus(self):
1377 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d)
1378 for u in self.test_wget_uris:
1379 with self.subTest(url=u):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001380 ud = fetch.ud[u]
1381 m = ud.method
1382 ret = m.checkstatus(fetch, ud, self.d)
1383 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
1384
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001385 @skipIfNoNetwork()
1386 def test_wget_checkstatus_connection_cache(self):
1387 from bb.fetch2 import FetchConnectionCache
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001388
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001389 connection_cache = FetchConnectionCache()
1390 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d,
1391 connection_cache = connection_cache)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001392
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001393 for u in self.test_wget_uris:
1394 with self.subTest(url=u):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001395 ud = fetch.ud[u]
1396 m = ud.method
1397 ret = m.checkstatus(fetch, ud, self.d)
1398 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
1399
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001400 connection_cache.close_connections()
1401
1402
1403class GitMakeShallowTest(FetcherTest):
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001404 def setUp(self):
1405 FetcherTest.setUp(self)
1406 self.gitdir = os.path.join(self.tempdir, 'gitshallow')
1407 bb.utils.mkdirhier(self.gitdir)
1408 bb.process.run('git init', cwd=self.gitdir)
Andrew Geisslerc926e172021-05-07 16:11:35 -05001409 bb.process.run('git config user.email "you@example.com"', cwd=self.gitdir)
1410 bb.process.run('git config user.name "Your Name"', cwd=self.gitdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001411
1412 def assertRefs(self, expected_refs):
1413 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
1414 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines()
1415 self.assertEqual(sorted(full_expected), sorted(actual_refs))
1416
1417 def assertRevCount(self, expected_count, args=None):
1418 if args is None:
1419 args = ['HEAD']
1420 revs = self.git(['rev-list'] + args)
1421 actual_count = len(revs.splitlines())
1422 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1423
1424 def git(self, cmd):
1425 if isinstance(cmd, str):
1426 cmd = 'git ' + cmd
1427 else:
1428 cmd = ['git'] + cmd
1429 return bb.process.run(cmd, cwd=self.gitdir)[0]
1430
1431 def make_shallow(self, args=None):
1432 if args is None:
1433 args = ['HEAD']
Brad Bishop316dfdd2018-06-25 12:45:53 -04001434 return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001435
1436 def add_empty_file(self, path, msg=None):
1437 if msg is None:
1438 msg = path
1439 open(os.path.join(self.gitdir, path), 'w').close()
1440 self.git(['add', path])
1441 self.git(['commit', '-m', msg, path])
1442
1443 def test_make_shallow_single_branch_no_merge(self):
1444 self.add_empty_file('a')
1445 self.add_empty_file('b')
1446 self.assertRevCount(2)
1447 self.make_shallow()
1448 self.assertRevCount(1)
1449
1450 def test_make_shallow_single_branch_one_merge(self):
1451 self.add_empty_file('a')
1452 self.add_empty_file('b')
1453 self.git('checkout -b a_branch')
1454 self.add_empty_file('c')
1455 self.git('checkout master')
1456 self.add_empty_file('d')
1457 self.git('merge --no-ff --no-edit a_branch')
1458 self.git('branch -d a_branch')
1459 self.add_empty_file('e')
1460 self.assertRevCount(6)
1461 self.make_shallow(['HEAD~2'])
1462 self.assertRevCount(5)
1463
1464 def test_make_shallow_at_merge(self):
1465 self.add_empty_file('a')
1466 self.git('checkout -b a_branch')
1467 self.add_empty_file('b')
1468 self.git('checkout master')
1469 self.git('merge --no-ff --no-edit a_branch')
1470 self.git('branch -d a_branch')
1471 self.assertRevCount(3)
1472 self.make_shallow()
1473 self.assertRevCount(1)
1474
1475 def test_make_shallow_annotated_tag(self):
1476 self.add_empty_file('a')
1477 self.add_empty_file('b')
1478 self.git('tag -a -m a_tag a_tag')
1479 self.assertRevCount(2)
1480 self.make_shallow(['a_tag'])
1481 self.assertRevCount(1)
1482
1483 def test_make_shallow_multi_ref(self):
1484 self.add_empty_file('a')
1485 self.add_empty_file('b')
1486 self.git('checkout -b a_branch')
1487 self.add_empty_file('c')
1488 self.git('checkout master')
1489 self.add_empty_file('d')
1490 self.git('checkout -b a_branch_2')
1491 self.add_empty_file('a_tag')
1492 self.git('tag a_tag')
1493 self.git('checkout master')
1494 self.git('branch -D a_branch_2')
1495 self.add_empty_file('e')
1496 self.assertRevCount(6, ['--all'])
1497 self.make_shallow()
1498 self.assertRevCount(5, ['--all'])
1499
1500 def test_make_shallow_multi_ref_trim(self):
1501 self.add_empty_file('a')
1502 self.git('checkout -b a_branch')
1503 self.add_empty_file('c')
1504 self.git('checkout master')
1505 self.assertRevCount(1)
1506 self.assertRevCount(2, ['--all'])
1507 self.assertRefs(['master', 'a_branch'])
1508 self.make_shallow(['-r', 'master', 'HEAD'])
1509 self.assertRevCount(1, ['--all'])
1510 self.assertRefs(['master'])
1511
1512 def test_make_shallow_noop(self):
1513 self.add_empty_file('a')
1514 self.assertRevCount(1)
1515 self.make_shallow()
1516 self.assertRevCount(1)
1517
1518 @skipIfNoNetwork()
1519 def test_make_shallow_bitbake(self):
1520 self.git('remote add origin https://github.com/openembedded/bitbake')
1521 self.git('fetch --tags origin')
1522 orig_revs = len(self.git('rev-list --all').splitlines())
1523 self.make_shallow(['refs/tags/1.10.0'])
1524 self.assertRevCount(orig_revs - 1746, ['--all'])
1525
1526class GitShallowTest(FetcherTest):
1527 def setUp(self):
1528 FetcherTest.setUp(self)
1529 self.gitdir = os.path.join(self.tempdir, 'git')
1530 self.srcdir = os.path.join(self.tempdir, 'gitsource')
1531
1532 bb.utils.mkdirhier(self.srcdir)
1533 self.git('init', cwd=self.srcdir)
Andrew Geisslerc926e172021-05-07 16:11:35 -05001534 self.git('config user.email "you@example.com"', cwd=self.srcdir)
1535 self.git('config user.name "Your Name"', cwd=self.srcdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001536 self.d.setVar('WORKDIR', self.tempdir)
1537 self.d.setVar('S', self.gitdir)
1538 self.d.delVar('PREMIRRORS')
1539 self.d.delVar('MIRRORS')
1540
1541 uri = 'git://%s;protocol=file;subdir=${S}' % self.srcdir
1542 self.d.setVar('SRC_URI', uri)
1543 self.d.setVar('SRCREV', '${AUTOREV}')
1544 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
1545
1546 self.d.setVar('BB_GIT_SHALLOW', '1')
1547 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
1548 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
1549
1550 def assertRefs(self, expected_refs, cwd=None):
1551 if cwd is None:
1552 cwd = self.gitdir
1553 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines()
1554 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines()
1555 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs)))
1556
1557 def assertRevCount(self, expected_count, args=None, cwd=None):
1558 if args is None:
1559 args = ['HEAD']
1560 if cwd is None:
1561 cwd = self.gitdir
1562 revs = self.git(['rev-list'] + args, cwd=cwd)
1563 actual_count = len(revs.splitlines())
1564 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1565
1566 def git(self, cmd, cwd=None):
1567 if isinstance(cmd, str):
1568 cmd = 'git ' + cmd
1569 else:
1570 cmd = ['git'] + cmd
1571 if cwd is None:
1572 cwd = self.gitdir
1573 return bb.process.run(cmd, cwd=cwd)[0]
1574
1575 def add_empty_file(self, path, cwd=None, msg=None):
1576 if msg is None:
1577 msg = path
1578 if cwd is None:
1579 cwd = self.srcdir
1580 open(os.path.join(cwd, path), 'w').close()
1581 self.git(['add', path], cwd)
1582 self.git(['commit', '-m', msg, path], cwd)
1583
1584 def fetch(self, uri=None):
1585 if uri is None:
Brad Bishop19323692019-04-05 15:28:33 -04001586 uris = self.d.getVar('SRC_URI').split()
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001587 uri = uris[0]
1588 d = self.d
1589 else:
1590 d = self.d.createCopy()
1591 d.setVar('SRC_URI', uri)
1592 uri = d.expand(uri)
1593 uris = [uri]
1594
1595 fetcher = bb.fetch2.Fetch(uris, d)
1596 fetcher.download()
1597 ud = fetcher.ud[uri]
1598 return fetcher, ud
1599
1600 def fetch_and_unpack(self, uri=None):
1601 fetcher, ud = self.fetch(uri)
1602 fetcher.unpack(self.d.getVar('WORKDIR'))
1603 assert os.path.exists(self.d.getVar('S'))
1604 return fetcher, ud
1605
1606 def fetch_shallow(self, uri=None, disabled=False, keepclone=False):
1607 """Fetch a uri, generating a shallow tarball, then unpack using it"""
1608 fetcher, ud = self.fetch_and_unpack(uri)
1609 assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri)
1610
1611 # Confirm that the unpacked repo is unshallow
1612 if not disabled:
1613 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1614
1615 # fetch and unpack, from the shallow tarball
1616 bb.utils.remove(self.gitdir, recurse=True)
Andrew Geisslerc926e172021-05-07 16:11:35 -05001617 bb.process.run('chmod u+w -R "%s"' % ud.clonedir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001618 bb.utils.remove(ud.clonedir, recurse=True)
Brad Bishopf8caae32019-03-25 13:13:56 -04001619 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001620
1621 # confirm that the unpacked repo is used when no git clone or git
1622 # mirror tarball is available
1623 fetcher, ud = self.fetch_and_unpack(uri)
1624 if not disabled:
1625 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir
1626 else:
1627 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir
1628 return fetcher, ud
1629
1630 def test_shallow_disabled(self):
1631 self.add_empty_file('a')
1632 self.add_empty_file('b')
1633 self.assertRevCount(2, cwd=self.srcdir)
1634
1635 self.d.setVar('BB_GIT_SHALLOW', '0')
1636 self.fetch_shallow(disabled=True)
1637 self.assertRevCount(2)
1638
1639 def test_shallow_nobranch(self):
1640 self.add_empty_file('a')
1641 self.add_empty_file('b')
1642 self.assertRevCount(2, cwd=self.srcdir)
1643
1644 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip()
1645 self.d.setVar('SRCREV', srcrev)
Brad Bishop19323692019-04-05 15:28:33 -04001646 uri = self.d.getVar('SRC_URI').split()[0]
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001647 uri = '%s;nobranch=1;bare=1' % uri
1648
1649 self.fetch_shallow(uri)
1650 self.assertRevCount(1)
1651
1652 # shallow refs are used to ensure the srcrev sticks around when we
1653 # have no other branches referencing it
1654 self.assertRefs(['refs/shallow/default'])
1655
1656 def test_shallow_default_depth_1(self):
1657 # Create initial git repo
1658 self.add_empty_file('a')
1659 self.add_empty_file('b')
1660 self.assertRevCount(2, cwd=self.srcdir)
1661
1662 self.fetch_shallow()
1663 self.assertRevCount(1)
1664
1665 def test_shallow_depth_0_disables(self):
1666 self.add_empty_file('a')
1667 self.add_empty_file('b')
1668 self.assertRevCount(2, cwd=self.srcdir)
1669
1670 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1671 self.fetch_shallow(disabled=True)
1672 self.assertRevCount(2)
1673
1674 def test_shallow_depth_default_override(self):
1675 self.add_empty_file('a')
1676 self.add_empty_file('b')
1677 self.assertRevCount(2, cwd=self.srcdir)
1678
1679 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2')
1680 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1')
1681 self.fetch_shallow()
1682 self.assertRevCount(1)
1683
1684 def test_shallow_depth_default_override_disable(self):
1685 self.add_empty_file('a')
1686 self.add_empty_file('b')
1687 self.add_empty_file('c')
1688 self.assertRevCount(3, cwd=self.srcdir)
1689
1690 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1691 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2')
1692 self.fetch_shallow()
1693 self.assertRevCount(2)
1694
1695 def test_current_shallow_out_of_date_clone(self):
1696 # Create initial git repo
1697 self.add_empty_file('a')
1698 self.add_empty_file('b')
1699 self.add_empty_file('c')
1700 self.assertRevCount(3, cwd=self.srcdir)
1701
1702 # Clone and generate mirror tarball
1703 fetcher, ud = self.fetch()
1704
1705 # Ensure we have a current mirror tarball, but an out of date clone
1706 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir)
1707 self.assertRevCount(2, cwd=ud.clonedir)
1708
1709 # Fetch and unpack, from the current tarball, not the out of date clone
1710 bb.utils.remove(self.gitdir, recurse=True)
1711 fetcher, ud = self.fetch()
1712 fetcher.unpack(self.d.getVar('WORKDIR'))
1713 self.assertRevCount(1)
1714
1715 def test_shallow_single_branch_no_merge(self):
1716 self.add_empty_file('a')
1717 self.add_empty_file('b')
1718 self.assertRevCount(2, cwd=self.srcdir)
1719
1720 self.fetch_shallow()
1721 self.assertRevCount(1)
1722 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1723 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1724
1725 def test_shallow_no_dangling(self):
1726 self.add_empty_file('a')
1727 self.add_empty_file('b')
1728 self.assertRevCount(2, cwd=self.srcdir)
1729
1730 self.fetch_shallow()
1731 self.assertRevCount(1)
1732 assert not self.git('fsck --dangling')
1733
1734 def test_shallow_srcrev_branch_truncation(self):
1735 self.add_empty_file('a')
1736 self.add_empty_file('b')
1737 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip()
1738 self.add_empty_file('c')
1739 self.assertRevCount(3, cwd=self.srcdir)
1740
1741 self.d.setVar('SRCREV', b_commit)
1742 self.fetch_shallow()
1743
1744 # The 'c' commit was removed entirely, and 'a' was removed from history
1745 self.assertRevCount(1, ['--all'])
1746 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit)
1747 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1748 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1749 assert not os.path.exists(os.path.join(self.gitdir, 'c'))
1750
1751 def test_shallow_ref_pruning(self):
1752 self.add_empty_file('a')
1753 self.add_empty_file('b')
1754 self.git('branch a_branch', cwd=self.srcdir)
1755 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
1756 self.assertRevCount(2, cwd=self.srcdir)
1757
1758 self.fetch_shallow()
1759
1760 self.assertRefs(['master', 'origin/master'])
1761 self.assertRevCount(1)
1762
1763 def test_shallow_submodules(self):
1764 self.add_empty_file('a')
1765 self.add_empty_file('b')
1766
1767 smdir = os.path.join(self.tempdir, 'gitsubmodule')
1768 bb.utils.mkdirhier(smdir)
1769 self.git('init', cwd=smdir)
Andrew Geisslerc926e172021-05-07 16:11:35 -05001770 self.git('config user.email "you@example.com"', cwd=smdir)
1771 self.git('config user.name "Your Name"', cwd=smdir)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001772 # Make this look like it was cloned from a remote...
1773 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
1774 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001775 self.add_empty_file('asub', cwd=smdir)
Brad Bishopf8caae32019-03-25 13:13:56 -04001776 self.add_empty_file('bsub', cwd=smdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001777
1778 self.git('submodule init', cwd=self.srcdir)
1779 self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
1780 self.git('submodule update', cwd=self.srcdir)
1781 self.git('commit -m submodule -a', cwd=self.srcdir)
1782
1783 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
1784 fetcher, ud = self.fetch_shallow(uri)
1785
Brad Bishopf8caae32019-03-25 13:13:56 -04001786 # Verify the main repository is shallow
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001787 self.assertRevCount(1)
Brad Bishopf8caae32019-03-25 13:13:56 -04001788
1789 # Verify the gitsubmodule directory is present
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001790 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
1791
Brad Bishopf8caae32019-03-25 13:13:56 -04001792 # Verify the submodule is also shallow
1793 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
1794
Andrew Geissler82c905d2020-04-13 13:39:40 -05001795 def test_shallow_submodule_mirrors(self):
1796 self.add_empty_file('a')
1797 self.add_empty_file('b')
1798
1799 smdir = os.path.join(self.tempdir, 'gitsubmodule')
1800 bb.utils.mkdirhier(smdir)
1801 self.git('init', cwd=smdir)
Andrew Geisslerc926e172021-05-07 16:11:35 -05001802 self.git('config user.email "you@example.com"', cwd=smdir)
1803 self.git('config user.name "Your Name"', cwd=smdir)
Andrew Geissler82c905d2020-04-13 13:39:40 -05001804 # Make this look like it was cloned from a remote...
1805 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
1806 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
1807 self.add_empty_file('asub', cwd=smdir)
1808 self.add_empty_file('bsub', cwd=smdir)
1809
1810 self.git('submodule init', cwd=self.srcdir)
1811 self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
1812 self.git('submodule update', cwd=self.srcdir)
1813 self.git('commit -m submodule -a', cwd=self.srcdir)
1814
1815 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
1816
1817 # Fetch once to generate the shallow tarball
1818 fetcher, ud = self.fetch(uri)
1819
1820 # Set up the mirror
1821 mirrordir = os.path.join(self.tempdir, 'mirror')
Andrew Geisslerc926e172021-05-07 16:11:35 -05001822 bb.utils.rename(self.dldir, mirrordir)
Andrew Geissler82c905d2020-04-13 13:39:40 -05001823 self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/\n' % mirrordir)
1824
1825 # Fetch from the mirror
1826 bb.utils.remove(self.dldir, recurse=True)
1827 bb.utils.remove(self.gitdir, recurse=True)
1828 self.fetch_and_unpack(uri)
1829
1830 # Verify the main repository is shallow
1831 self.assertRevCount(1)
1832
1833 # Verify the gitsubmodule directory is present
1834 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
1835
1836 # Verify the submodule is also shallow
1837 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
Brad Bishopf8caae32019-03-25 13:13:56 -04001838
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001839 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')):
1840 def test_shallow_annex(self):
1841 self.add_empty_file('a')
1842 self.add_empty_file('b')
1843 self.git('annex init', cwd=self.srcdir)
1844 open(os.path.join(self.srcdir, 'c'), 'w').close()
1845 self.git('annex add c', cwd=self.srcdir)
Andrew Geisslerc926e172021-05-07 16:11:35 -05001846 self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir)
1847 bb.process.run('chmod u+w -R %s' % self.srcdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001848
1849 uri = 'gitannex://%s;protocol=file;subdir=${S}' % self.srcdir
1850 fetcher, ud = self.fetch_shallow(uri)
1851
1852 self.assertRevCount(1)
1853 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0]
1854 assert os.path.exists(os.path.join(self.gitdir, 'c'))
1855
1856 def test_shallow_multi_one_uri(self):
1857 # Create initial git repo
1858 self.add_empty_file('a')
1859 self.add_empty_file('b')
1860 self.git('checkout -b a_branch', cwd=self.srcdir)
1861 self.add_empty_file('c')
1862 self.add_empty_file('d')
1863 self.git('checkout master', cwd=self.srcdir)
1864 self.git('tag v0.0 a_branch', cwd=self.srcdir)
1865 self.add_empty_file('e')
1866 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1867 self.add_empty_file('f')
1868 self.assertRevCount(7, cwd=self.srcdir)
1869
Brad Bishop19323692019-04-05 15:28:33 -04001870 uri = self.d.getVar('SRC_URI').split()[0]
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001871 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
1872
1873 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1874 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1875 self.d.setVar('SRCREV_master', '${AUTOREV}')
1876 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
1877
1878 self.fetch_shallow(uri)
1879
1880 self.assertRevCount(5)
1881 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1882
1883 def test_shallow_multi_one_uri_depths(self):
1884 # Create initial git repo
1885 self.add_empty_file('a')
1886 self.add_empty_file('b')
1887 self.git('checkout -b a_branch', cwd=self.srcdir)
1888 self.add_empty_file('c')
1889 self.add_empty_file('d')
1890 self.git('checkout master', cwd=self.srcdir)
1891 self.add_empty_file('e')
1892 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1893 self.add_empty_file('f')
1894 self.assertRevCount(7, cwd=self.srcdir)
1895
Brad Bishop19323692019-04-05 15:28:33 -04001896 uri = self.d.getVar('SRC_URI').split()[0]
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001897 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
1898
1899 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1900 self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3')
1901 self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1')
1902 self.d.setVar('SRCREV_master', '${AUTOREV}')
1903 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
1904
1905 self.fetch_shallow(uri)
1906
1907 self.assertRevCount(4, ['--all'])
1908 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1909
1910 def test_shallow_clone_preferred_over_shallow(self):
1911 self.add_empty_file('a')
1912 self.add_empty_file('b')
1913
1914 # Fetch once to generate the shallow tarball
1915 fetcher, ud = self.fetch()
1916 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1917
1918 # Fetch and unpack with both the clonedir and shallow tarball available
1919 bb.utils.remove(self.gitdir, recurse=True)
1920 fetcher, ud = self.fetch_and_unpack()
1921
1922 # The unpacked tree should *not* be shallow
1923 self.assertRevCount(2)
1924 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow'))
1925
1926 def test_shallow_mirrors(self):
1927 self.add_empty_file('a')
1928 self.add_empty_file('b')
1929
1930 # Fetch once to generate the shallow tarball
1931 fetcher, ud = self.fetch()
1932 mirrortarball = ud.mirrortarballs[0]
1933 assert os.path.exists(os.path.join(self.dldir, mirrortarball))
1934
1935 # Set up the mirror
1936 mirrordir = os.path.join(self.tempdir, 'mirror')
1937 bb.utils.mkdirhier(mirrordir)
1938 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir)
1939
Andrew Geisslerc926e172021-05-07 16:11:35 -05001940 bb.utils.rename(os.path.join(self.dldir, mirrortarball),
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001941 os.path.join(mirrordir, mirrortarball))
1942
1943 # Fetch from the mirror
1944 bb.utils.remove(self.dldir, recurse=True)
1945 bb.utils.remove(self.gitdir, recurse=True)
1946 self.fetch_and_unpack()
1947 self.assertRevCount(1)
1948
1949 def test_shallow_invalid_depth(self):
1950 self.add_empty_file('a')
1951 self.add_empty_file('b')
1952
1953 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12')
1954 with self.assertRaises(bb.fetch2.FetchError):
1955 self.fetch()
1956
1957 def test_shallow_invalid_depth_default(self):
1958 self.add_empty_file('a')
1959 self.add_empty_file('b')
1960
1961 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12')
1962 with self.assertRaises(bb.fetch2.FetchError):
1963 self.fetch()
1964
1965 def test_shallow_extra_refs(self):
1966 self.add_empty_file('a')
1967 self.add_empty_file('b')
1968 self.git('branch a_branch', cwd=self.srcdir)
1969 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
1970 self.assertRevCount(2, cwd=self.srcdir)
1971
1972 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch')
1973 self.fetch_shallow()
1974
1975 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1976 self.assertRevCount(1)
1977
1978 def test_shallow_extra_refs_wildcard(self):
1979 self.add_empty_file('a')
1980 self.add_empty_file('b')
1981 self.git('branch a_branch', cwd=self.srcdir)
1982 self.git('tag v1.0', cwd=self.srcdir)
1983 self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir)
1984 self.assertRevCount(2, cwd=self.srcdir)
1985
1986 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
1987 self.fetch_shallow()
1988
1989 self.assertRefs(['master', 'origin/master', 'v1.0'])
1990 self.assertRevCount(1)
1991
1992 def test_shallow_missing_extra_refs(self):
1993 self.add_empty_file('a')
1994 self.add_empty_file('b')
1995
1996 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo')
1997 with self.assertRaises(bb.fetch2.FetchError):
1998 self.fetch()
1999
2000 def test_shallow_missing_extra_refs_wildcard(self):
2001 self.add_empty_file('a')
2002 self.add_empty_file('b')
2003
2004 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
2005 self.fetch()
2006
2007 def test_shallow_remove_revs(self):
2008 # Create initial git repo
2009 self.add_empty_file('a')
2010 self.add_empty_file('b')
2011 self.git('checkout -b a_branch', cwd=self.srcdir)
2012 self.add_empty_file('c')
2013 self.add_empty_file('d')
2014 self.git('checkout master', cwd=self.srcdir)
2015 self.git('tag v0.0 a_branch', cwd=self.srcdir)
2016 self.add_empty_file('e')
2017 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
2018 self.git('branch -d a_branch', cwd=self.srcdir)
2019 self.add_empty_file('f')
2020 self.assertRevCount(7, cwd=self.srcdir)
2021
2022 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2023 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
2024
2025 self.fetch_shallow()
2026
2027 self.assertRevCount(5)
2028
2029 def test_shallow_invalid_revs(self):
2030 self.add_empty_file('a')
2031 self.add_empty_file('b')
2032
2033 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2034 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
2035
2036 with self.assertRaises(bb.fetch2.FetchError):
2037 self.fetch()
2038
Brad Bishop64c979e2019-11-04 13:55:29 -05002039 def test_shallow_fetch_missing_revs(self):
2040 self.add_empty_file('a')
2041 self.add_empty_file('b')
2042 fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
2043 self.git('tag v0.0 master', cwd=self.srcdir)
2044 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2045 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
2046 self.fetch_shallow()
2047
2048 def test_shallow_fetch_missing_revs_fails(self):
2049 self.add_empty_file('a')
2050 self.add_empty_file('b')
2051 fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
2052 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2053 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
2054
2055 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm:
2056 self.fetch_shallow()
2057 self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0])
2058
Brad Bishopd7bf8c12018-02-25 22:55:05 -05002059 @skipIfNoNetwork()
2060 def test_bitbake(self):
2061 self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
2062 self.git('config core.bare true', cwd=self.srcdir)
2063 self.git('fetch', cwd=self.srcdir)
2064
2065 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2066 # Note that the 1.10.0 tag is annotated, so this also tests
2067 # reference of an annotated vs unannotated tag
2068 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0')
2069
2070 self.fetch_shallow()
2071
2072 # Confirm that the history of 1.10.0 was removed
2073 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines())
2074 revs = len(self.git('rev-list master').splitlines())
2075 self.assertNotEqual(orig_revs, revs)
2076 self.assertRefs(['master', 'origin/master'])
2077 self.assertRevCount(orig_revs - 1758)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08002078
2079 def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self):
2080 self.add_empty_file('a')
2081 fetcher, ud = self.fetch()
2082 bb.utils.remove(self.gitdir, recurse=True)
2083 bb.utils.remove(self.dldir, recurse=True)
2084
2085 with self.assertRaises(bb.fetch2.UnpackError) as context:
2086 fetcher.unpack(self.d.getVar('WORKDIR'))
2087
2088 self.assertIn("No up to date source found", context.exception.msg)
2089 self.assertIn("clone directory not available or not up to date", context.exception.msg)
2090
2091 @skipIfNoNetwork()
2092 def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self):
2093 self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0')
2094 self.d.setVar('BB_GIT_SHALLOW', '1')
2095 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
2096 fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests"], self.d)
2097 fetcher.download()
2098
2099 bb.utils.remove(self.dldir + "/*.tar.gz")
2100 fetcher.unpack(self.unpackdir)
2101
2102 dir = os.listdir(self.unpackdir + "/git/")
2103 self.assertIn("fstests.doap", dir)
Brad Bishop00e122a2019-10-05 11:10:57 -04002104
2105class GitLfsTest(FetcherTest):
2106 def setUp(self):
2107 FetcherTest.setUp(self)
2108
2109 self.gitdir = os.path.join(self.tempdir, 'git')
2110 self.srcdir = os.path.join(self.tempdir, 'gitsource')
2111
2112 self.d.setVar('WORKDIR', self.tempdir)
2113 self.d.setVar('S', self.gitdir)
2114 self.d.delVar('PREMIRRORS')
2115 self.d.delVar('MIRRORS')
2116
2117 self.d.setVar('SRCREV', '${AUTOREV}')
2118 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
2119
2120 bb.utils.mkdirhier(self.srcdir)
2121 self.git('init', cwd=self.srcdir)
Andrew Geisslerc926e172021-05-07 16:11:35 -05002122 self.git('config user.email "you@example.com"', cwd=self.srcdir)
2123 self.git('config user.name "Your Name"', cwd=self.srcdir)
Brad Bishop00e122a2019-10-05 11:10:57 -04002124 with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs:
2125 attrs.write('*.mp3 filter=lfs -text')
2126 self.git(['add', '.gitattributes'], cwd=self.srcdir)
2127 self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir)
2128
2129 def git(self, cmd, cwd=None):
2130 if isinstance(cmd, str):
2131 cmd = 'git ' + cmd
2132 else:
2133 cmd = ['git'] + cmd
2134 if cwd is None:
2135 cwd = self.gitdir
2136 return bb.process.run(cmd, cwd=cwd)[0]
2137
Andrew Geisslerd1e89492021-02-12 15:35:20 -06002138 def fetch(self, uri=None, download=True):
Brad Bishop00e122a2019-10-05 11:10:57 -04002139 uris = self.d.getVar('SRC_URI').split()
2140 uri = uris[0]
2141 d = self.d
2142
2143 fetcher = bb.fetch2.Fetch(uris, d)
Andrew Geisslerd1e89492021-02-12 15:35:20 -06002144 if download:
2145 fetcher.download()
Brad Bishop00e122a2019-10-05 11:10:57 -04002146 ud = fetcher.ud[uri]
2147 return fetcher, ud
2148
2149 def test_lfs_enabled(self):
2150 import shutil
2151
2152 uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir
2153 self.d.setVar('SRC_URI', uri)
2154
Andrew Geisslerd1e89492021-02-12 15:35:20 -06002155 # Careful: suppress initial attempt at downloading until
2156 # we know whether git-lfs is installed.
2157 fetcher, ud = self.fetch(uri=None, download=False)
Brad Bishop00e122a2019-10-05 11:10:57 -04002158 self.assertIsNotNone(ud.method._find_git_lfs)
2159
Andrew Geisslerd1e89492021-02-12 15:35:20 -06002160 # If git-lfs can be found, the unpack should be successful. Only
2161 # attempt this with the real live copy of git-lfs installed.
2162 if ud.method._find_git_lfs(self.d):
2163 fetcher.download()
2164 shutil.rmtree(self.gitdir, ignore_errors=True)
2165 fetcher.unpack(self.d.getVar('WORKDIR'))
Brad Bishop00e122a2019-10-05 11:10:57 -04002166
2167 # If git-lfs cannot be found, the unpack should throw an error
2168 with self.assertRaises(bb.fetch2.FetchError):
Andrew Geisslerd1e89492021-02-12 15:35:20 -06002169 fetcher.download()
Brad Bishop00e122a2019-10-05 11:10:57 -04002170 ud.method._find_git_lfs = lambda d: False
2171 shutil.rmtree(self.gitdir, ignore_errors=True)
2172 fetcher.unpack(self.d.getVar('WORKDIR'))
2173
2174 def test_lfs_disabled(self):
2175 import shutil
2176
2177 uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir
2178 self.d.setVar('SRC_URI', uri)
2179
Andrew Geisslerd1e89492021-02-12 15:35:20 -06002180 # In contrast to test_lfs_enabled(), allow the implicit download
2181 # done by self.fetch() to occur here. The point of this test case
2182 # is to verify that the fetcher can survive even if the source
2183 # repository has Git LFS usage configured.
Brad Bishop00e122a2019-10-05 11:10:57 -04002184 fetcher, ud = self.fetch()
2185 self.assertIsNotNone(ud.method._find_git_lfs)
2186
Andrew Geisslerd1e89492021-02-12 15:35:20 -06002187 # If git-lfs can be found, the unpack should be successful. A
2188 # live copy of git-lfs is not required for this case, so
2189 # unconditionally forge its presence.
Brad Bishop00e122a2019-10-05 11:10:57 -04002190 ud.method._find_git_lfs = lambda d: True
2191 shutil.rmtree(self.gitdir, ignore_errors=True)
2192 fetcher.unpack(self.d.getVar('WORKDIR'))
2193
2194 # If git-lfs cannot be found, the unpack should be successful
2195 ud.method._find_git_lfs = lambda d: False
2196 shutil.rmtree(self.gitdir, ignore_errors=True)
2197 fetcher.unpack(self.d.getVar('WORKDIR'))
Andrew Geissler82c905d2020-04-13 13:39:40 -05002198
Andrew Geisslerc3d88e42020-10-02 09:45:00 -05002199class GitURLWithSpacesTest(FetcherTest):
2200 test_git_urls = {
2201 "git://tfs-example.org:22/tfs/example%20path/example.git" : {
2202 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git',
2203 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git',
2204 'path': '/tfs/example path/example.git'
2205 },
2206 "git://tfs-example.org:22/tfs/example%20path/example%20repo.git" : {
2207 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git',
2208 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git',
2209 'path': '/tfs/example path/example repo.git'
2210 }
2211 }
2212
2213 def test_urls(self):
2214
2215 # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo
2216 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
2217
2218 for test_git_url, ref in self.test_git_urls.items():
2219
2220 fetcher = bb.fetch.Fetch([test_git_url], self.d)
2221 ud = fetcher.ud[fetcher.urls[0]]
2222
2223 self.assertEqual(ud.url, ref['url'])
2224 self.assertEqual(ud.path, ref['path'])
2225 self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname']))
2226 self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname']))
2227 self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock'))
2228 self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname']))
2229 self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz'))
2230
Andrew Geissler82c905d2020-04-13 13:39:40 -05002231class NPMTest(FetcherTest):
2232 def skipIfNoNpm():
2233 import shutil
2234 if not shutil.which('npm'):
2235 return unittest.skip('npm not installed, tests being skipped')
2236 return lambda f: f
2237
2238 @skipIfNoNpm()
2239 @skipIfNoNetwork()
2240 def test_npm(self):
2241 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2242 fetcher = bb.fetch.Fetch([url], self.d)
2243 ud = fetcher.ud[fetcher.urls[0]]
2244 fetcher.download()
2245 self.assertTrue(os.path.exists(ud.localpath))
2246 self.assertTrue(os.path.exists(ud.localpath + '.done'))
2247 self.assertTrue(os.path.exists(ud.resolvefile))
2248 fetcher.unpack(self.unpackdir)
2249 unpackdir = os.path.join(self.unpackdir, 'npm')
2250 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2251
2252 @skipIfNoNpm()
2253 @skipIfNoNetwork()
2254 def test_npm_bad_checksum(self):
2255 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2256 # Fetch once to get a tarball
2257 fetcher = bb.fetch.Fetch([url], self.d)
2258 ud = fetcher.ud[fetcher.urls[0]]
2259 fetcher.download()
2260 self.assertTrue(os.path.exists(ud.localpath))
2261 # Modify the tarball
2262 bad = b'bad checksum'
2263 with open(ud.localpath, 'wb') as f:
2264 f.write(bad)
2265 # Verify that the tarball is fetched again
2266 fetcher.download()
2267 badsum = hashlib.sha512(bad).hexdigest()
2268 self.assertTrue(os.path.exists(ud.localpath + '_bad-checksum_' + badsum))
2269 self.assertTrue(os.path.exists(ud.localpath))
2270
2271 @skipIfNoNpm()
2272 @skipIfNoNetwork()
2273 def test_npm_premirrors(self):
2274 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2275 # Fetch once to get a tarball
2276 fetcher = bb.fetch.Fetch([url], self.d)
2277 ud = fetcher.ud[fetcher.urls[0]]
2278 fetcher.download()
2279 self.assertTrue(os.path.exists(ud.localpath))
2280 # Setup the mirror
Andrew Geissler5199d832021-09-24 16:47:35 -05002281 pkgname = os.path.basename(ud.proxy.urls[0].split(';')[0])
Andrew Geissler82c905d2020-04-13 13:39:40 -05002282 mirrordir = os.path.join(self.tempdir, 'mirror')
2283 bb.utils.mkdirhier(mirrordir)
Andrew Geissler5199d832021-09-24 16:47:35 -05002284 os.replace(ud.localpath, os.path.join(mirrordir, pkgname))
Andrew Geissler82c905d2020-04-13 13:39:40 -05002285 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
2286 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
2287 # Fetch again
2288 self.assertFalse(os.path.exists(ud.localpath))
2289 fetcher.download()
2290 self.assertTrue(os.path.exists(ud.localpath))
2291
2292 @skipIfNoNpm()
2293 @skipIfNoNetwork()
Andrew Geissler5199d832021-09-24 16:47:35 -05002294 def test_npm_premirrors_with_specified_filename(self):
2295 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2296 # Fetch once to get a tarball
2297 fetcher = bb.fetch.Fetch([url], self.d)
2298 ud = fetcher.ud[fetcher.urls[0]]
2299 fetcher.download()
2300 self.assertTrue(os.path.exists(ud.localpath))
2301 # Setup the mirror
2302 mirrordir = os.path.join(self.tempdir, 'mirror')
2303 bb.utils.mkdirhier(mirrordir)
2304 mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath))
2305 os.replace(ud.localpath, mirrorfilename)
2306 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s\n' % mirrorfilename)
2307 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
2308 # Fetch again
2309 self.assertFalse(os.path.exists(ud.localpath))
2310 fetcher.download()
2311 self.assertTrue(os.path.exists(ud.localpath))
2312
2313 @skipIfNoNpm()
2314 @skipIfNoNetwork()
Andrew Geissler82c905d2020-04-13 13:39:40 -05002315 def test_npm_mirrors(self):
2316 # Fetch once to get a tarball
2317 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2318 fetcher = bb.fetch.Fetch([url], self.d)
2319 ud = fetcher.ud[fetcher.urls[0]]
2320 fetcher.download()
2321 self.assertTrue(os.path.exists(ud.localpath))
2322 # Setup the mirror
2323 mirrordir = os.path.join(self.tempdir, 'mirror')
2324 bb.utils.mkdirhier(mirrordir)
2325 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
2326 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
2327 # Update the resolved url to an invalid url
2328 with open(ud.resolvefile, 'r') as f:
2329 url = f.read()
2330 uri = URI(url)
2331 uri.path = '/invalid'
2332 with open(ud.resolvefile, 'w') as f:
2333 f.write(str(uri))
2334 # Fetch again
2335 self.assertFalse(os.path.exists(ud.localpath))
2336 fetcher.download()
2337 self.assertTrue(os.path.exists(ud.localpath))
2338
2339 @skipIfNoNpm()
2340 @skipIfNoNetwork()
2341 def test_npm_destsuffix_downloadfilename(self):
2342 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz'
2343 fetcher = bb.fetch.Fetch([url], self.d)
2344 fetcher.download()
2345 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'foo-bar.tgz')))
2346 fetcher.unpack(self.unpackdir)
2347 unpackdir = os.path.join(self.unpackdir, 'foo', 'bar')
2348 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2349
2350 def test_npm_no_network_no_tarball(self):
2351 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2352 self.d.setVar('BB_NO_NETWORK', '1')
2353 fetcher = bb.fetch.Fetch([url], self.d)
2354 with self.assertRaises(bb.fetch2.NetworkAccess):
2355 fetcher.download()
2356
2357 @skipIfNoNpm()
2358 @skipIfNoNetwork()
2359 def test_npm_no_network_with_tarball(self):
2360 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2361 # Fetch once to get a tarball
2362 fetcher = bb.fetch.Fetch([url], self.d)
2363 fetcher.download()
2364 # Disable network access
2365 self.d.setVar('BB_NO_NETWORK', '1')
2366 # Fetch again
2367 fetcher.download()
2368 fetcher.unpack(self.unpackdir)
2369 unpackdir = os.path.join(self.unpackdir, 'npm')
2370 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2371
2372 @skipIfNoNpm()
2373 @skipIfNoNetwork()
2374 def test_npm_registry_alternate(self):
Andrew Geissler5199d832021-09-24 16:47:35 -05002375 url = 'npm://skimdb.npmjs.com;package=@savoirfairelinux/node-server-example;version=1.0.0'
Andrew Geissler82c905d2020-04-13 13:39:40 -05002376 fetcher = bb.fetch.Fetch([url], self.d)
2377 fetcher.download()
2378 fetcher.unpack(self.unpackdir)
2379 unpackdir = os.path.join(self.unpackdir, 'npm')
2380 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2381
2382 @skipIfNoNpm()
2383 @skipIfNoNetwork()
2384 def test_npm_version_latest(self):
2385 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest'
2386 fetcher = bb.fetch.Fetch([url], self.d)
2387 fetcher.download()
2388 fetcher.unpack(self.unpackdir)
2389 unpackdir = os.path.join(self.unpackdir, 'npm')
2390 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2391
2392 @skipIfNoNpm()
2393 @skipIfNoNetwork()
2394 def test_npm_registry_invalid(self):
2395 url = 'npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2396 fetcher = bb.fetch.Fetch([url], self.d)
2397 with self.assertRaises(bb.fetch2.FetchError):
2398 fetcher.download()
2399
2400 @skipIfNoNpm()
2401 @skipIfNoNetwork()
2402 def test_npm_package_invalid(self):
2403 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0'
2404 fetcher = bb.fetch.Fetch([url], self.d)
2405 with self.assertRaises(bb.fetch2.FetchError):
2406 fetcher.download()
2407
2408 @skipIfNoNpm()
2409 @skipIfNoNetwork()
2410 def test_npm_version_invalid(self):
2411 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid'
2412 with self.assertRaises(bb.fetch2.ParameterError):
2413 fetcher = bb.fetch.Fetch([url], self.d)
2414
2415 @skipIfNoNpm()
2416 @skipIfNoNetwork()
2417 def test_npm_registry_none(self):
2418 url = 'npm://;package=@savoirfairelinux/node-server-example;version=1.0.0'
2419 with self.assertRaises(bb.fetch2.MalformedUrl):
2420 fetcher = bb.fetch.Fetch([url], self.d)
2421
2422 @skipIfNoNpm()
2423 @skipIfNoNetwork()
2424 def test_npm_package_none(self):
2425 url = 'npm://registry.npmjs.org;version=1.0.0'
2426 with self.assertRaises(bb.fetch2.MissingParameterError):
2427 fetcher = bb.fetch.Fetch([url], self.d)
2428
2429 @skipIfNoNpm()
2430 @skipIfNoNetwork()
2431 def test_npm_version_none(self):
2432 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example'
2433 with self.assertRaises(bb.fetch2.MissingParameterError):
2434 fetcher = bb.fetch.Fetch([url], self.d)
2435
2436 def create_shrinkwrap_file(self, data):
2437 import json
2438 datadir = os.path.join(self.tempdir, 'data')
2439 swfile = os.path.join(datadir, 'npm-shrinkwrap.json')
2440 bb.utils.mkdirhier(datadir)
2441 with open(swfile, 'w') as f:
2442 json.dump(data, f)
2443 # Also configure the S directory
2444 self.sdir = os.path.join(self.unpackdir, 'S')
2445 self.d.setVar('S', self.sdir)
2446 return swfile
2447
2448 @skipIfNoNpm()
2449 @skipIfNoNetwork()
2450 def test_npmsw(self):
2451 swfile = self.create_shrinkwrap_file({
2452 'dependencies': {
2453 'array-flatten': {
2454 'version': '1.1.1',
2455 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2456 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=',
2457 'dependencies': {
2458 'content-type': {
2459 'version': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
2460 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
2461 'dependencies': {
2462 'cookie': {
2463 'version': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09',
2464 'from': 'git+https://github.com/jshttp/cookie.git'
2465 }
2466 }
2467 }
2468 }
2469 }
2470 }
2471 })
2472 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2473 fetcher.download()
2474 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
2475 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
2476 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git')))
2477 fetcher.unpack(self.unpackdir)
2478 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'npm-shrinkwrap.json')))
2479 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json')))
2480 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json')))
2481 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json')))
2482
2483 @skipIfNoNpm()
2484 @skipIfNoNetwork()
2485 def test_npmsw_dev(self):
2486 swfile = self.create_shrinkwrap_file({
2487 'dependencies': {
2488 'array-flatten': {
2489 'version': '1.1.1',
2490 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2491 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2492 },
2493 'content-type': {
2494 'version': '1.0.4',
2495 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
2496 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
2497 'dev': True
2498 }
2499 }
2500 })
2501 # Fetch with dev disabled
2502 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2503 fetcher.download()
2504 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
2505 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
2506 # Fetch with dev enabled
2507 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d)
2508 fetcher.download()
2509 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
2510 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
2511
2512 @skipIfNoNpm()
2513 @skipIfNoNetwork()
2514 def test_npmsw_destsuffix(self):
2515 swfile = self.create_shrinkwrap_file({
2516 'dependencies': {
2517 'array-flatten': {
2518 'version': '1.1.1',
2519 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2520 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2521 }
2522 }
2523 })
2524 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d)
2525 fetcher.download()
2526 fetcher.unpack(self.unpackdir)
2527 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json')))
2528
2529 def test_npmsw_no_network_no_tarball(self):
2530 swfile = self.create_shrinkwrap_file({
2531 'dependencies': {
2532 'array-flatten': {
2533 'version': '1.1.1',
2534 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2535 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2536 }
2537 }
2538 })
2539 self.d.setVar('BB_NO_NETWORK', '1')
2540 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2541 with self.assertRaises(bb.fetch2.NetworkAccess):
2542 fetcher.download()
2543
2544 @skipIfNoNpm()
2545 @skipIfNoNetwork()
2546 def test_npmsw_no_network_with_tarball(self):
2547 # Fetch once to get a tarball
2548 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2549 fetcher.download()
2550 # Disable network access
2551 self.d.setVar('BB_NO_NETWORK', '1')
2552 # Fetch again
2553 swfile = self.create_shrinkwrap_file({
2554 'dependencies': {
2555 'array-flatten': {
2556 'version': '1.1.1',
2557 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2558 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2559 }
2560 }
2561 })
2562 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2563 fetcher.download()
2564 fetcher.unpack(self.unpackdir)
2565 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json')))
2566
2567 @skipIfNoNpm()
2568 @skipIfNoNetwork()
2569 def test_npmsw_npm_reusability(self):
2570 # Fetch once with npmsw
2571 swfile = self.create_shrinkwrap_file({
2572 'dependencies': {
2573 'array-flatten': {
2574 'version': '1.1.1',
2575 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2576 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2577 }
2578 }
2579 })
2580 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2581 fetcher.download()
2582 # Disable network access
2583 self.d.setVar('BB_NO_NETWORK', '1')
2584 # Fetch again with npm
2585 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2586 fetcher.download()
2587 fetcher.unpack(self.unpackdir)
2588 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json')))
2589
2590 @skipIfNoNpm()
2591 @skipIfNoNetwork()
2592 def test_npmsw_bad_checksum(self):
2593 # Try to fetch with bad checksum
2594 swfile = self.create_shrinkwrap_file({
2595 'dependencies': {
2596 'array-flatten': {
2597 'version': '1.1.1',
2598 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2599 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg='
2600 }
2601 }
2602 })
2603 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2604 with self.assertRaises(bb.fetch2.FetchError):
2605 fetcher.download()
2606 # Fetch correctly to get a tarball
2607 swfile = self.create_shrinkwrap_file({
2608 'dependencies': {
2609 'array-flatten': {
2610 'version': '1.1.1',
2611 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2612 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2613 }
2614 }
2615 })
2616 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2617 fetcher.download()
2618 localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')
2619 self.assertTrue(os.path.exists(localpath))
2620 # Modify the tarball
2621 bad = b'bad checksum'
2622 with open(localpath, 'wb') as f:
2623 f.write(bad)
2624 # Verify that the tarball is fetched again
2625 fetcher.download()
2626 badsum = hashlib.sha1(bad).hexdigest()
2627 self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum))
2628 self.assertTrue(os.path.exists(localpath))
2629
2630 @skipIfNoNpm()
2631 @skipIfNoNetwork()
2632 def test_npmsw_premirrors(self):
2633 # Fetch once to get a tarball
2634 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2635 ud = fetcher.ud[fetcher.urls[0]]
2636 fetcher.download()
2637 self.assertTrue(os.path.exists(ud.localpath))
2638 # Setup the mirror
2639 mirrordir = os.path.join(self.tempdir, 'mirror')
2640 bb.utils.mkdirhier(mirrordir)
2641 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
2642 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
2643 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
2644 # Fetch again
2645 self.assertFalse(os.path.exists(ud.localpath))
2646 swfile = self.create_shrinkwrap_file({
2647 'dependencies': {
2648 'array-flatten': {
2649 'version': '1.1.1',
2650 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2651 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2652 }
2653 }
2654 })
2655 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2656 fetcher.download()
2657 self.assertTrue(os.path.exists(ud.localpath))
2658
2659 @skipIfNoNpm()
2660 @skipIfNoNetwork()
2661 def test_npmsw_mirrors(self):
2662 # Fetch once to get a tarball
2663 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2664 ud = fetcher.ud[fetcher.urls[0]]
2665 fetcher.download()
2666 self.assertTrue(os.path.exists(ud.localpath))
2667 # Setup the mirror
2668 mirrordir = os.path.join(self.tempdir, 'mirror')
2669 bb.utils.mkdirhier(mirrordir)
2670 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
2671 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
2672 # Fetch again with invalid url
2673 self.assertFalse(os.path.exists(ud.localpath))
2674 swfile = self.create_shrinkwrap_file({
2675 'dependencies': {
2676 'array-flatten': {
2677 'version': '1.1.1',
2678 'resolved': 'https://invalid',
2679 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2680 }
2681 }
2682 })
2683 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2684 fetcher.download()
2685 self.assertTrue(os.path.exists(ud.localpath))
Andrew Geisslerc926e172021-05-07 16:11:35 -05002686
2687class GitSharedTest(FetcherTest):
2688 def setUp(self):
2689 super(GitSharedTest, self).setUp()
2690 self.recipe_url = "git://git.openembedded.org/bitbake"
2691 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
2692
2693 @skipIfNoNetwork()
2694 def test_shared_unpack(self):
2695 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
2696
2697 fetcher.download()
2698 fetcher.unpack(self.unpackdir)
2699 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
2700 self.assertTrue(os.path.exists(alt))
2701
2702 @skipIfNoNetwork()
2703 def test_noshared_unpack(self):
2704 self.d.setVar('BB_GIT_NOSHARED', '1')
2705 self.unpackdir += '_noshared'
2706 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
2707
2708 fetcher.download()
2709 fetcher.unpack(self.unpackdir)
2710 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
2711 self.assertFalse(os.path.exists(alt))