blob: 4697ef59b5b5a735a606a65e9adfb4ed4abee8a8 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# BitBake Tests for the Fetcher (fetch2/)
3#
4# Copyright (C) 2012 Richard Purdie
5#
Brad Bishopc342db32019-05-15 21:57:59 -04006# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007#
8
9import unittest
Brad Bishop316dfdd2018-06-25 12:45:53 -040010import hashlib
Patrick Williamsc124f4f2015-09-15 14:41:29 -050011import tempfile
Patrick Williamsc0f7c042017-02-23 20:41:17 -060012import collections
Patrick Williamsc124f4f2015-09-15 14:41:29 -050013import os
14from bb.fetch2 import URI
15from bb.fetch2 import FetchMethod
16import bb
Andrew Geissler82c905d2020-04-13 13:39:40 -050017from bb.tests.support.httpserver import HTTPService
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018
Brad Bishopd7bf8c12018-02-25 22:55:05 -050019def skipIfNoNetwork():
20 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
21 return unittest.skip("Network tests being skipped")
22 return lambda f: f
23
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024class URITest(unittest.TestCase):
25 test_uris = {
26 "http://www.google.com/index.html" : {
27 'uri': 'http://www.google.com/index.html',
28 'scheme': 'http',
29 'hostname': 'www.google.com',
30 'port': None,
31 'hostport': 'www.google.com',
32 'path': '/index.html',
33 'userinfo': '',
34 'username': '',
35 'password': '',
36 'params': {},
37 'query': {},
38 'relative': False
39 },
40 "http://www.google.com/index.html;param1=value1" : {
41 'uri': 'http://www.google.com/index.html;param1=value1',
42 'scheme': 'http',
43 'hostname': 'www.google.com',
44 'port': None,
45 'hostport': 'www.google.com',
46 'path': '/index.html',
47 'userinfo': '',
48 'username': '',
49 'password': '',
50 'params': {
51 'param1': 'value1'
52 },
53 'query': {},
54 'relative': False
55 },
56 "http://www.example.org/index.html?param1=value1" : {
57 'uri': 'http://www.example.org/index.html?param1=value1',
58 'scheme': 'http',
59 'hostname': 'www.example.org',
60 'port': None,
61 'hostport': 'www.example.org',
62 'path': '/index.html',
63 'userinfo': '',
64 'username': '',
65 'password': '',
66 'params': {},
67 'query': {
68 'param1': 'value1'
69 },
70 'relative': False
71 },
72 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : {
73 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2',
74 'scheme': 'http',
75 'hostname': 'www.example.org',
76 'port': None,
77 'hostport': 'www.example.org',
78 'path': '/index.html',
79 'userinfo': '',
80 'username': '',
81 'password': '',
82 'params': {
83 'param2': 'value2'
84 },
85 'query': {
86 'qparam1': 'qvalue1'
87 },
88 'relative': False
89 },
90 "http://www.example.com:8080/index.html" : {
91 'uri': 'http://www.example.com:8080/index.html',
92 'scheme': 'http',
93 'hostname': 'www.example.com',
94 'port': 8080,
95 'hostport': 'www.example.com:8080',
96 'path': '/index.html',
97 'userinfo': '',
98 'username': '',
99 'password': '',
100 'params': {},
101 'query': {},
102 'relative': False
103 },
104 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : {
105 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg',
106 'scheme': 'cvs',
107 'hostname': 'cvs.handhelds.org',
108 'port': None,
109 'hostport': 'cvs.handhelds.org',
110 'path': '/cvs',
111 'userinfo': 'anoncvs',
112 'username': 'anoncvs',
113 'password': '',
114 'params': {
115 'module': 'familiar/dist/ipkg'
116 },
117 'query': {},
118 'relative': False
119 },
120 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": {
121 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg',
122 'scheme': 'cvs',
123 'hostname': 'cvs.handhelds.org',
124 'port': None,
125 'hostport': 'cvs.handhelds.org',
126 'path': '/cvs',
127 'userinfo': 'anoncvs:anonymous',
128 'username': 'anoncvs',
129 'password': 'anonymous',
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600130 'params': collections.OrderedDict([
131 ('tag', 'V0-99-81'),
132 ('module', 'familiar/dist/ipkg')
133 ]),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500134 'query': {},
135 'relative': False
136 },
137 "file://example.diff": { # NOTE: Not RFC compliant!
138 'uri': 'file:example.diff',
139 'scheme': 'file',
140 'hostname': '',
141 'port': None,
142 'hostport': '',
143 'path': 'example.diff',
144 'userinfo': '',
145 'username': '',
146 'password': '',
147 'params': {},
148 'query': {},
149 'relative': True
150 },
151 "file:example.diff": { # NOTE: RFC compliant version of the former
152 'uri': 'file:example.diff',
153 'scheme': 'file',
154 'hostname': '',
155 'port': None,
156 'hostport': '',
157 'path': 'example.diff',
158 'userinfo': '',
159 'userinfo': '',
160 'username': '',
161 'password': '',
162 'params': {},
163 'query': {},
164 'relative': True
165 },
166 "file:///tmp/example.diff": {
167 'uri': 'file:///tmp/example.diff',
168 'scheme': 'file',
169 'hostname': '',
170 'port': None,
171 'hostport': '',
172 'path': '/tmp/example.diff',
173 'userinfo': '',
174 'userinfo': '',
175 'username': '',
176 'password': '',
177 'params': {},
178 'query': {},
179 'relative': False
180 },
181 "git:///path/example.git": {
182 'uri': 'git:///path/example.git',
183 'scheme': 'git',
184 'hostname': '',
185 'port': None,
186 'hostport': '',
187 'path': '/path/example.git',
188 'userinfo': '',
189 'userinfo': '',
190 'username': '',
191 'password': '',
192 'params': {},
193 'query': {},
194 'relative': False
195 },
196 "git:path/example.git": {
197 'uri': 'git:path/example.git',
198 'scheme': 'git',
199 'hostname': '',
200 'port': None,
201 'hostport': '',
202 'path': 'path/example.git',
203 'userinfo': '',
204 'userinfo': '',
205 'username': '',
206 'password': '',
207 'params': {},
208 'query': {},
209 'relative': True
210 },
211 "git://example.net/path/example.git": {
212 'uri': 'git://example.net/path/example.git',
213 'scheme': 'git',
214 'hostname': 'example.net',
215 'port': None,
216 'hostport': 'example.net',
217 'path': '/path/example.git',
218 'userinfo': '',
219 'userinfo': '',
220 'username': '',
221 'password': '',
222 'params': {},
223 'query': {},
224 'relative': False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500225 },
226 "http://somesite.net;someparam=1": {
227 'uri': 'http://somesite.net;someparam=1',
228 'scheme': 'http',
229 'hostname': 'somesite.net',
230 'port': None,
231 'hostport': 'somesite.net',
232 'path': '',
233 'userinfo': '',
234 'userinfo': '',
235 'username': '',
236 'password': '',
237 'params': {"someparam" : "1"},
238 'query': {},
239 'relative': False
240 },
241 "file://somelocation;someparam=1": {
242 'uri': 'file:somelocation;someparam=1',
243 'scheme': 'file',
244 'hostname': '',
245 'port': None,
246 'hostport': '',
247 'path': 'somelocation',
248 'userinfo': '',
249 'userinfo': '',
250 'username': '',
251 'password': '',
252 'params': {"someparam" : "1"},
253 'query': {},
254 'relative': True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500255 }
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500256
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500257 }
258
259 def test_uri(self):
260 for test_uri, ref in self.test_uris.items():
261 uri = URI(test_uri)
262
263 self.assertEqual(str(uri), ref['uri'])
264
265 # expected attributes
266 self.assertEqual(uri.scheme, ref['scheme'])
267
268 self.assertEqual(uri.userinfo, ref['userinfo'])
269 self.assertEqual(uri.username, ref['username'])
270 self.assertEqual(uri.password, ref['password'])
271
272 self.assertEqual(uri.hostname, ref['hostname'])
273 self.assertEqual(uri.port, ref['port'])
274 self.assertEqual(uri.hostport, ref['hostport'])
275
276 self.assertEqual(uri.path, ref['path'])
277 self.assertEqual(uri.params, ref['params'])
278
279 self.assertEqual(uri.relative, ref['relative'])
280
281 def test_dict(self):
282 for test in self.test_uris.values():
283 uri = URI()
284
285 self.assertEqual(uri.scheme, '')
286 self.assertEqual(uri.userinfo, '')
287 self.assertEqual(uri.username, '')
288 self.assertEqual(uri.password, '')
289 self.assertEqual(uri.hostname, '')
290 self.assertEqual(uri.port, None)
291 self.assertEqual(uri.path, '')
292 self.assertEqual(uri.params, {})
293
294
295 uri.scheme = test['scheme']
296 self.assertEqual(uri.scheme, test['scheme'])
297
298 uri.userinfo = test['userinfo']
299 self.assertEqual(uri.userinfo, test['userinfo'])
300 self.assertEqual(uri.username, test['username'])
301 self.assertEqual(uri.password, test['password'])
302
303 # make sure changing the values doesn't do anything unexpected
304 uri.username = 'changeme'
305 self.assertEqual(uri.username, 'changeme')
306 self.assertEqual(uri.password, test['password'])
307 uri.password = 'insecure'
308 self.assertEqual(uri.username, 'changeme')
309 self.assertEqual(uri.password, 'insecure')
310
311 # reset back after our trickery
312 uri.userinfo = test['userinfo']
313 self.assertEqual(uri.userinfo, test['userinfo'])
314 self.assertEqual(uri.username, test['username'])
315 self.assertEqual(uri.password, test['password'])
316
317 uri.hostname = test['hostname']
318 self.assertEqual(uri.hostname, test['hostname'])
319 self.assertEqual(uri.hostport, test['hostname'])
320
321 uri.port = test['port']
322 self.assertEqual(uri.port, test['port'])
323 self.assertEqual(uri.hostport, test['hostport'])
324
325 uri.path = test['path']
326 self.assertEqual(uri.path, test['path'])
327
328 uri.params = test['params']
329 self.assertEqual(uri.params, test['params'])
330
331 uri.query = test['query']
332 self.assertEqual(uri.query, test['query'])
333
334 self.assertEqual(str(uri), test['uri'])
335
336 uri.params = {}
337 self.assertEqual(uri.params, {})
338 self.assertEqual(str(uri), (str(uri).split(";"))[0])
339
340class FetcherTest(unittest.TestCase):
341
342 def setUp(self):
343 self.origdir = os.getcwd()
344 self.d = bb.data.init()
345 self.tempdir = tempfile.mkdtemp()
346 self.dldir = os.path.join(self.tempdir, "download")
347 os.mkdir(self.dldir)
348 self.d.setVar("DL_DIR", self.dldir)
349 self.unpackdir = os.path.join(self.tempdir, "unpacked")
350 os.mkdir(self.unpackdir)
351 persistdir = os.path.join(self.tempdir, "persistdata")
352 self.d.setVar("PERSISTENT_DIR", persistdir)
353
354 def tearDown(self):
355 os.chdir(self.origdir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600356 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
357 print("Not cleaning up %s. Please remove manually." % self.tempdir)
358 else:
359 bb.utils.prunedir(self.tempdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500360
361class MirrorUriTest(FetcherTest):
362
363 replaceuris = {
364 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/")
365 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
366 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
367 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
368 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
369 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
370 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http")
371 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
372 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake")
373 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890",
374 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache")
375 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
376 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/")
377 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
378 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3")
379 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
380 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz")
381 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
382 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist")
383 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2",
384 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/")
385 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2",
386 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
387 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
388 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
389 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
390 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http")
391 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800392 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org")
393 : "http://somewhere2.org/somefile_1.2.3.tar.gz",
394 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/")
395 : "http://somewhere2.org/somefile_1.2.3.tar.gz",
396 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http")
397 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500398
399 #Renaming files doesn't work
400 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
401 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
402 }
403
404 mirrorvar = "http://.*/.* file:///somepath/downloads/ \n" \
405 "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n" \
406 "https://.*/.* file:///someotherpath/downloads/ \n" \
407 "http://.*/.* file:///someotherpath/downloads/ \n"
408
409 def test_urireplace(self):
410 for k, v in self.replaceuris.items():
411 ud = bb.fetch.FetchData(k[0], self.d)
412 ud.setup_localpath(self.d)
413 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2]))
414 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d)
415 self.assertEqual([v], newuris)
416
417 def test_urilist1(self):
418 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
419 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
420 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
421 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
422
423 def test_urilist2(self):
424 # Catch https:// -> files:// bug
425 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
426 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
427 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
428 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
429
430 def test_mirror_of_mirror(self):
431 # Test if mirror of a mirror works
432 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/ \n"
433 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/ \n"
434 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
435 mirrors = bb.fetch2.mirror_from_string(mirrorvar)
436 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
437 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz',
438 'file:///someotherpath/downloads/bitbake-1.0.tar.gz',
439 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz',
440 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz'])
441
Patrick Williamsd7e96312015-09-22 08:09:05 -0500442 recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ \n" \
443 "https://.*/[^/]* https://BBBB/B/B/B/ \n"
444
445 def test_recursive(self):
446 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
447 mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar)
448 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
449 self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz',
450 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz',
451 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500452
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800453
454class GitDownloadDirectoryNamingTest(FetcherTest):
455 def setUp(self):
456 super(GitDownloadDirectoryNamingTest, self).setUp()
457 self.recipe_url = "git://git.openembedded.org/bitbake"
458 self.recipe_dir = "git.openembedded.org.bitbake"
459 self.mirror_url = "git://github.com/openembedded/bitbake.git"
460 self.mirror_dir = "github.com.openembedded.bitbake.git"
461
462 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
463
464 def setup_mirror_rewrite(self):
465 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
466
467 @skipIfNoNetwork()
468 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self):
469 self.setup_mirror_rewrite()
470 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
471
472 fetcher.download()
473
474 dir = os.listdir(self.dldir + "/git2")
475 self.assertIn(self.recipe_dir, dir)
476
477 @skipIfNoNetwork()
478 def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self):
479 self.setup_mirror_rewrite()
480 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
481
482 fetcher.download()
483
484 dir = os.listdir(self.dldir + "/git2")
485 self.assertIn(self.mirror_dir, dir)
486 self.assertIn(self.recipe_dir, dir)
487
488 @skipIfNoNetwork()
489 def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self):
490 self.setup_mirror_rewrite()
491 fetcher = bb.fetch.Fetch([self.mirror_url], self.d)
492 fetcher.download()
493 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
494
495 fetcher.download()
496
497 dir = os.listdir(self.dldir + "/git2")
498 self.assertIn(self.mirror_dir, dir)
499 self.assertIn(self.recipe_dir, dir)
500
501
502class TarballNamingTest(FetcherTest):
503 def setUp(self):
504 super(TarballNamingTest, self).setUp()
505 self.recipe_url = "git://git.openembedded.org/bitbake"
506 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
507 self.mirror_url = "git://github.com/openembedded/bitbake.git"
508 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz"
509
510 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
511 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
512
513 def setup_mirror_rewrite(self):
514 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
515
516 @skipIfNoNetwork()
517 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self):
518 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
519
520 fetcher.download()
521
522 dir = os.listdir(self.dldir)
523 self.assertIn(self.recipe_tarball, dir)
524
525 @skipIfNoNetwork()
526 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
527 self.setup_mirror_rewrite()
528 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
529
530 fetcher.download()
531
532 dir = os.listdir(self.dldir)
533 self.assertIn(self.mirror_tarball, dir)
534
535
536class GitShallowTarballNamingTest(FetcherTest):
537 def setUp(self):
538 super(GitShallowTarballNamingTest, self).setUp()
539 self.recipe_url = "git://git.openembedded.org/bitbake"
540 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz"
541 self.mirror_url = "git://github.com/openembedded/bitbake.git"
542 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz"
543
544 self.d.setVar('BB_GIT_SHALLOW', '1')
545 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
546 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
547
548 def setup_mirror_rewrite(self):
549 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
550
551 @skipIfNoNetwork()
552 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self):
553 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
554
555 fetcher.download()
556
557 dir = os.listdir(self.dldir)
558 self.assertIn(self.recipe_tarball, dir)
559
560 @skipIfNoNetwork()
561 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
562 self.setup_mirror_rewrite()
563 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
564
565 fetcher.download()
566
567 dir = os.listdir(self.dldir)
568 self.assertIn(self.mirror_tarball, dir)
569
570
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500571class FetcherLocalTest(FetcherTest):
572 def setUp(self):
573 def touch(fn):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600574 with open(fn, 'a'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500575 os.utime(fn, None)
576
577 super(FetcherLocalTest, self).setUp()
578 self.localsrcdir = os.path.join(self.tempdir, 'localsrc')
579 os.makedirs(self.localsrcdir)
580 touch(os.path.join(self.localsrcdir, 'a'))
581 touch(os.path.join(self.localsrcdir, 'b'))
582 os.makedirs(os.path.join(self.localsrcdir, 'dir'))
583 touch(os.path.join(self.localsrcdir, 'dir', 'c'))
584 touch(os.path.join(self.localsrcdir, 'dir', 'd'))
585 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir'))
586 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e'))
587 self.d.setVar("FILESPATH", self.localsrcdir)
588
589 def fetchUnpack(self, uris):
590 fetcher = bb.fetch.Fetch(uris, self.d)
591 fetcher.download()
592 fetcher.unpack(self.unpackdir)
593 flst = []
594 for root, dirs, files in os.walk(self.unpackdir):
595 for f in files:
596 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir))
597 flst.sort()
598 return flst
599
600 def test_local(self):
601 tree = self.fetchUnpack(['file://a', 'file://dir/c'])
602 self.assertEqual(tree, ['a', 'dir/c'])
603
604 def test_local_wildcard(self):
605 tree = self.fetchUnpack(['file://a', 'file://dir/*'])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500606 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500607
608 def test_local_dir(self):
609 tree = self.fetchUnpack(['file://a', 'file://dir'])
610 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e'])
611
612 def test_local_subdir(self):
613 tree = self.fetchUnpack(['file://dir/subdir'])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500614 self.assertEqual(tree, ['dir/subdir/e'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500615
616 def test_local_subdir_file(self):
617 tree = self.fetchUnpack(['file://dir/subdir/e'])
618 self.assertEqual(tree, ['dir/subdir/e'])
619
620 def test_local_subdirparam(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500621 tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo'])
622 self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500623
624 def test_local_deepsubdirparam(self):
625 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar'])
626 self.assertEqual(tree, ['bar/dir/subdir/e'])
627
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600628 def test_local_absolutedir(self):
629 # Unpacking to an absolute path that is a subdirectory of the root
630 # should work
631 tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')])
632
633 # Unpacking to an absolute path outside of the root should fail
634 with self.assertRaises(bb.fetch2.UnpackError):
635 self.fetchUnpack(['file://a;subdir=/bin/sh'])
636
Brad Bishop316dfdd2018-06-25 12:45:53 -0400637class FetcherNoNetworkTest(FetcherTest):
638 def setUp(self):
639 super().setUp()
640 # all test cases are based on not having network
641 self.d.setVar("BB_NO_NETWORK", "1")
642
643 def test_missing(self):
644 string = "this is a test file\n".encode("utf-8")
645 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
646 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
647
648 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
649 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
650 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
651 with self.assertRaises(bb.fetch2.NetworkAccess):
652 fetcher.download()
653
654 def test_valid_missing_donestamp(self):
655 # create the file in the download directory with correct hash
656 string = "this is a test file\n".encode("utf-8")
657 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f:
658 f.write(string)
659
660 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
661 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
662
663 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
664 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
665 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
666 fetcher.download()
667 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
668
669 def test_invalid_missing_donestamp(self):
670 # create an invalid file in the download directory with incorrect hash
671 string = "this is a test file\n".encode("utf-8")
672 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
673 pass
674
675 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
676 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
677
678 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
679 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
680 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
681 with self.assertRaises(bb.fetch2.NetworkAccess):
682 fetcher.download()
683 # the existing file should not exist or should have be moved to "bad-checksum"
684 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
685
686 def test_nochecksums_missing(self):
687 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
688 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
689 # ssh fetch does not support checksums
690 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
691 # attempts to download with missing donestamp
692 with self.assertRaises(bb.fetch2.NetworkAccess):
693 fetcher.download()
694
695 def test_nochecksums_missing_donestamp(self):
696 # create a file in the download directory
697 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
698 pass
699
700 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
701 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
702 # ssh fetch does not support checksums
703 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
704 # attempts to download with missing donestamp
705 with self.assertRaises(bb.fetch2.NetworkAccess):
706 fetcher.download()
707
708 def test_nochecksums_has_donestamp(self):
709 # create a file in the download directory with the donestamp
710 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
711 pass
712 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
713 pass
714
715 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
716 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
717 # ssh fetch does not support checksums
718 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
719 # should not fetch
720 fetcher.download()
721 # both files should still exist
722 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
723 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
724
725 def test_nochecksums_missing_has_donestamp(self):
726 # create a file in the download directory with the donestamp
727 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
728 pass
729
730 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
731 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
732 # ssh fetch does not support checksums
733 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
734 with self.assertRaises(bb.fetch2.NetworkAccess):
735 fetcher.download()
736 # both files should still exist
737 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
738 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
739
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500740class FetcherNetworkTest(FetcherTest):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500741 @skipIfNoNetwork()
742 def test_fetch(self):
743 fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
744 fetcher.download()
745 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
746 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892)
747 self.d.setVar("BB_NO_NETWORK", "1")
748 fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
749 fetcher.download()
750 fetcher.unpack(self.unpackdir)
751 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9)
752 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500753
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500754 @skipIfNoNetwork()
755 def test_fetch_mirror(self):
756 self.d.setVar("MIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
757 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
758 fetcher.download()
759 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
760
761 @skipIfNoNetwork()
762 def test_fetch_mirror_of_mirror(self):
763 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ \n http://invalid2.yoctoproject.org/.* http://downloads.yoctoproject.org/releases/bitbake")
764 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
765 fetcher.download()
766 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
767
768 @skipIfNoNetwork()
769 def test_fetch_file_mirror_of_mirror(self):
770 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ \n file:///some1where/.* file://some2where/ \n file://some2where/.* http://downloads.yoctoproject.org/releases/bitbake")
771 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
772 os.mkdir(self.dldir + "/some2where")
773 fetcher.download()
774 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
775
776 @skipIfNoNetwork()
777 def test_fetch_premirror(self):
778 self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
779 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
780 fetcher.download()
781 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
782
783 @skipIfNoNetwork()
784 def gitfetcher(self, url1, url2):
785 def checkrevision(self, fetcher):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500786 fetcher.unpack(self.unpackdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500787 revision = bb.process.run("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git")[0].strip()
788 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500789
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500790 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
791 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
792 fetcher = bb.fetch.Fetch([url1], self.d)
793 fetcher.download()
794 checkrevision(self, fetcher)
795 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works
796 bb.utils.prunedir(self.dldir + "/git2/")
797 bb.utils.prunedir(self.unpackdir)
798 self.d.setVar("BB_NO_NETWORK", "1")
799 fetcher = bb.fetch.Fetch([url2], self.d)
800 fetcher.download()
801 checkrevision(self, fetcher)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500802
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500803 @skipIfNoNetwork()
804 def test_gitfetch(self):
805 url1 = url2 = "git://git.openembedded.org/bitbake"
806 self.gitfetcher(url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500807
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500808 @skipIfNoNetwork()
809 def test_gitfetch_goodsrcrev(self):
810 # SRCREV is set but matches rev= parameter
811 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
812 self.gitfetcher(url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500813
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500814 @skipIfNoNetwork()
815 def test_gitfetch_badsrcrev(self):
816 # SRCREV is set but does not match rev= parameter
817 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5"
818 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500819
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500820 @skipIfNoNetwork()
821 def test_gitfetch_tagandrev(self):
822 # SRCREV is set but does not match rev= parameter
823 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
824 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500825
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500826 @skipIfNoNetwork()
827 def test_gitfetch_localusehead(self):
828 # Create dummy local Git repo
829 src_dir = tempfile.mkdtemp(dir=self.tempdir,
830 prefix='gitfetch_localusehead_')
831 src_dir = os.path.abspath(src_dir)
832 bb.process.run("git init", cwd=src_dir)
833 bb.process.run("git commit --allow-empty -m'Dummy commit'",
834 cwd=src_dir)
835 # Use other branch than master
836 bb.process.run("git checkout -b my-devel", cwd=src_dir)
837 bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
838 cwd=src_dir)
839 stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
840 orig_rev = stdout[0].strip()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500841
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500842 # Fetch and check revision
843 self.d.setVar("SRCREV", "AUTOINC")
844 url = "git://" + src_dir + ";protocol=file;usehead=1"
845 fetcher = bb.fetch.Fetch([url], self.d)
846 fetcher.download()
847 fetcher.unpack(self.unpackdir)
848 stdout = bb.process.run("git rev-parse HEAD",
849 cwd=os.path.join(self.unpackdir, 'git'))
850 unpack_rev = stdout[0].strip()
851 self.assertEqual(orig_rev, unpack_rev)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500852
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500853 @skipIfNoNetwork()
854 def test_gitfetch_remoteusehead(self):
855 url = "git://git.openembedded.org/bitbake;usehead=1"
856 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500857
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500858 @skipIfNoNetwork()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800859 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self):
860 recipeurl = "git://git.openembedded.org/bitbake"
861 mirrorurl = "git://someserver.org/bitbake"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500862 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800863 self.gitfetcher(recipeurl, mirrorurl)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500864
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500865 @skipIfNoNetwork()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800866 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self):
867 recipeurl = "git://someserver.org/bitbake"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500868 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800869 self.gitfetcher(recipeurl, recipeurl)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500870
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500871 @skipIfNoNetwork()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800872 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500873 realurl = "git://git.openembedded.org/bitbake"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800874 recipeurl = "git://someserver.org/bitbake"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500875 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git")
876 os.chdir(self.tempdir)
877 bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800878 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (recipeurl, self.sourcedir))
879 self.gitfetcher(recipeurl, recipeurl)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600880
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500881 @skipIfNoNetwork()
882 def test_git_submodule(self):
Brad Bishopf8caae32019-03-25 13:13:56 -0400883 # URL with ssh submodules
884 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7"
885 # Original URL (comment this if you have ssh access to git.yoctoproject.org)
886 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee"
887 fetcher = bb.fetch.Fetch([url], self.d)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500888 fetcher.download()
889 # Previous cwd has been deleted
890 os.chdir(os.path.dirname(self.unpackdir))
891 fetcher.unpack(self.unpackdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500892
Brad Bishopf8caae32019-03-25 13:13:56 -0400893 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
894 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing')
895 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing')
896 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present')
897
898 # Only when we're running the extended test with a submodule's submodule, can we check this.
899 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
900 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
901
Brad Bishop96ff1982019-08-19 13:50:42 -0400902 @skipIfNoNetwork()
Brad Bishopf8caae32019-03-25 13:13:56 -0400903 def test_git_submodule_dbus_broker(self):
904 # The following external repositories have show failures in fetch and unpack operations
905 # We want to avoid regressions!
906 url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2"
907 fetcher = bb.fetch.Fetch([url], self.d)
908 fetcher.download()
909 # Previous cwd has been deleted
910 os.chdir(os.path.dirname(self.unpackdir))
911 fetcher.unpack(self.unpackdir)
912
913 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
914 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"')
915 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"')
916 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"')
917 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"')
918 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"')
919
Brad Bishop96ff1982019-08-19 13:50:42 -0400920 @skipIfNoNetwork()
Brad Bishopf8caae32019-03-25 13:13:56 -0400921 def test_git_submodule_CLI11(self):
922 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
923 fetcher = bb.fetch.Fetch([url], self.d)
924 fetcher.download()
925 # Previous cwd has been deleted
926 os.chdir(os.path.dirname(self.unpackdir))
927 fetcher.unpack(self.unpackdir)
928
929 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
930 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
931 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
932 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
933
Brad Bishop96ff1982019-08-19 13:50:42 -0400934 @skipIfNoNetwork()
Brad Bishop19323692019-04-05 15:28:33 -0400935 def test_git_submodule_update_CLI11(self):
936 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
937 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714"
938 fetcher = bb.fetch.Fetch([url], self.d)
939 fetcher.download()
940
941 # CLI11 that pulls in a newer nlohmann-json
942 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca"
943 fetcher = bb.fetch.Fetch([url], self.d)
944 fetcher.download()
945 # Previous cwd has been deleted
946 os.chdir(os.path.dirname(self.unpackdir))
947 fetcher.unpack(self.unpackdir)
948
949 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
950 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
951 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
952 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
953
Brad Bishop96ff1982019-08-19 13:50:42 -0400954 @skipIfNoNetwork()
Brad Bishopf8caae32019-03-25 13:13:56 -0400955 def test_git_submodule_aktualizr(self):
956 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
957 fetcher = bb.fetch.Fetch([url], self.d)
958 fetcher.download()
959 # Previous cwd has been deleted
960 os.chdir(os.path.dirname(self.unpackdir))
961 fetcher.unpack(self.unpackdir)
962
963 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
964 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"')
965 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"')
966 self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly")
967 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"')
968 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"')
969 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500970
Brad Bishop96ff1982019-08-19 13:50:42 -0400971 @skipIfNoNetwork()
Brad Bishop393846f2019-05-20 12:24:11 -0400972 def test_git_submodule_iotedge(self):
973 """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """
974
975 # This repository also has submodules where the module (name), path and url do not align
976 url = "gitsm://github.com/azure/iotedge.git;protocol=git;rev=d76e0316c6f324345d77c48a83ce836d09392699"
977 fetcher = bb.fetch.Fetch([url], self.d)
978 fetcher.download()
979 # Previous cwd has been deleted
980 os.chdir(os.path.dirname(self.unpackdir))
981 fetcher.unpack(self.unpackdir)
982
983 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
984
985 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/README.md')), msg='Missing submodule checkout')
986 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/ctest/README.md')), msg='Missing submodule checkout')
987 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/testrunner/readme.md')), msg='Missing submodule checkout')
988 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/readme.md')), msg='Missing submodule checkout')
989 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
990 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
991 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/README.md')), msg='Missing submodule checkout')
992 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/README.md')), msg='Missing submodule checkout')
993 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/ctest/README.md')), msg='Missing submodule checkout')
994 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/testrunner/readme.md')), msg='Missing submodule checkout')
995 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/readme.md')), msg='Missing submodule checkout')
996 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
997 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
998
Brad Bishop15ae2502019-06-18 21:44:24 -0400999class SVNTest(FetcherTest):
1000 def skipIfNoSvn():
1001 import shutil
1002 if not shutil.which("svn"):
1003 return unittest.skip("svn not installed, tests being skipped")
1004
1005 if not shutil.which("svnadmin"):
1006 return unittest.skip("svnadmin not installed, tests being skipped")
1007
1008 return lambda f: f
1009
1010 @skipIfNoSvn()
1011 def setUp(self):
1012 """ Create a local repository """
1013
1014 super(SVNTest, self).setUp()
1015
1016 # Create something we can fetch
1017 src_dir = tempfile.mkdtemp(dir=self.tempdir,
1018 prefix='svnfetch_srcdir_')
1019 src_dir = os.path.abspath(src_dir)
1020 bb.process.run("echo readme > README.md", cwd=src_dir)
1021
1022 # Store it in a local SVN repository
1023 repo_dir = tempfile.mkdtemp(dir=self.tempdir,
1024 prefix='svnfetch_localrepo_')
1025 repo_dir = os.path.abspath(repo_dir)
1026 bb.process.run("svnadmin create project", cwd=repo_dir)
1027
1028 self.repo_url = "file://%s/project" % repo_dir
1029 bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url),
1030 cwd=repo_dir)
1031
1032 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
1033 # Github will emulate SVN. Use this to check if we're downloding...
1034 bb.process.run("svn propset svn:externals 'bitbake http://github.com/openembedded/bitbake' .",
1035 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
1036 bb.process.run("svn commit --non-interactive -m 'Add external'",
1037 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
1038
1039 self.src_dir = src_dir
1040 self.repo_dir = repo_dir
1041
1042 @skipIfNoSvn()
1043 def tearDown(self):
1044 os.chdir(self.origdir)
1045 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
1046 print("Not cleaning up %s. Please remove manually." % self.tempdir)
1047 else:
1048 bb.utils.prunedir(self.tempdir)
1049
1050 @skipIfNoSvn()
1051 @skipIfNoNetwork()
1052 def test_noexternal_svn(self):
1053 # Always match the rev count from setUp (currently rev 2)
1054 url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '')
1055 fetcher = bb.fetch.Fetch([url], self.d)
1056 fetcher.download()
1057 os.chdir(os.path.dirname(self.unpackdir))
1058 fetcher.unpack(self.unpackdir)
1059
1060 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
1061 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
1062 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should NOT exist")
1063 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should NOT exit")
1064
1065 @skipIfNoSvn()
1066 def test_external_svn(self):
1067 # Always match the rev count from setUp (currently rev 2)
1068 url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '')
1069 fetcher = bb.fetch.Fetch([url], self.d)
1070 fetcher.download()
1071 os.chdir(os.path.dirname(self.unpackdir))
1072 fetcher.unpack(self.unpackdir)
1073
1074 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
1075 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
1076 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should exist")
1077 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should exit")
1078
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001079class TrustedNetworksTest(FetcherTest):
1080 def test_trusted_network(self):
1081 # Ensure trusted_network returns False when the host IS in the list.
1082 url = "git://Someserver.org/foo;rev=1"
1083 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org")
1084 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001085
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001086 def test_wild_trusted_network(self):
1087 # Ensure trusted_network returns true when the *.host IS in the list.
1088 url = "git://Someserver.org/foo;rev=1"
1089 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1090 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001091
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001092 def test_prefix_wild_trusted_network(self):
1093 # Ensure trusted_network returns true when the prefix matches *.host.
1094 url = "git://git.Someserver.org/foo;rev=1"
1095 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1096 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001097
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001098 def test_two_prefix_wild_trusted_network(self):
1099 # Ensure trusted_network returns true when the prefix matches *.host.
1100 url = "git://something.git.Someserver.org/foo;rev=1"
1101 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1102 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001103
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001104 def test_port_trusted_network(self):
1105 # Ensure trusted_network returns True, even if the url specifies a port.
1106 url = "git://someserver.org:8080/foo;rev=1"
1107 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org")
1108 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001109
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001110 def test_untrusted_network(self):
1111 # Ensure trusted_network returns False when the host is NOT in the list.
1112 url = "git://someserver.org/foo;rev=1"
1113 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
1114 self.assertFalse(bb.fetch.trusted_network(self.d, url))
1115
1116 def test_wild_untrusted_network(self):
1117 # Ensure trusted_network returns False when the host is NOT in the list.
1118 url = "git://*.someserver.org/foo;rev=1"
1119 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
1120 self.assertFalse(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001121
1122class URLHandle(unittest.TestCase):
1123
1124 datatable = {
1125 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}),
1126 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}),
Patrick Williamsc0f7c042017-02-23 20:41:17 -06001127 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001128 "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'}),
1129 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}),
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001130 }
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001131 # we require a pathname to encodeurl but users can still pass such urls to
1132 # decodeurl and we need to handle them
1133 decodedata = datatable.copy()
1134 decodedata.update({
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001135 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001136 })
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001137
1138 def test_decodeurl(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001139 for k, v in self.decodedata.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001140 result = bb.fetch.decodeurl(k)
1141 self.assertEqual(result, v)
1142
1143 def test_encodeurl(self):
1144 for k, v in self.datatable.items():
1145 result = bb.fetch.encodeurl(v)
1146 self.assertEqual(result, k)
1147
1148class FetchLatestVersionTest(FetcherTest):
1149
1150 test_git_uris = {
1151 # version pattern "X.Y.Z"
1152 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
1153 : "1.99.4",
1154 # version pattern "vX.Y"
Andrew Geisslerd25ed322020-06-27 00:28:28 -05001155 # mirror of git.infradead.org since network issues interfered with testing
1156 ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001157 : "1.5.0",
1158 # version pattern "pkg_name-X.Y"
1159 ("presentproto", "git://anongit.freedesktop.org/git/xorg/proto/presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")
1160 : "1.0",
1161 # version pattern "pkg_name-vX.Y.Z"
1162 ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
1163 : "1.4.0",
1164 # combination version pattern
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001165 ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001166 : "1.2.0",
1167 ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "")
1168 : "2014.01",
1169 # version pattern "yyyymmdd"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001170 ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001171 : "20120614",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001172 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001173 ("xf86-video-omap", "git://anongit.freedesktop.org/xorg/driver/xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))")
1174 : "0.4.3",
1175 ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", "(?P<pver>(([0-9][\.|_]?)+[0-9]))")
1176 : "11.0.0",
1177 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", "chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
1178 : "1.3.59",
1179 ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", "(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
1180 : "3.82+dbg0.9",
1181 }
1182
1183 test_wget_uris = {
Andrew Geissler82c905d2020-04-13 13:39:40 -05001184 #
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001185 # packages with versions inside directory name
Andrew Geissler82c905d2020-04-13 13:39:40 -05001186 #
1187 # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2
1188 ("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001189 : "2.24.2",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001190 # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz
1191 ("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001192 : "1.6.0",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001193 # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz
1194 ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001195 : "2.8.12.1",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001196 #
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001197 # packages with versions only in current directory
Andrew Geissler82c905d2020-04-13 13:39:40 -05001198 #
1199 # http://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2
1200 ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001201 : "2.19",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001202 # http://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2
1203 ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001204 : "20120814",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001205 #
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001206 # packages with "99" in the name of possible version
Andrew Geissler82c905d2020-04-13 13:39:40 -05001207 #
1208 # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz
1209 ("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001210 : "5.0",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001211 # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2
1212 ("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001213 : "1.15.1",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001214 #
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001215 # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX
Andrew Geissler82c905d2020-04-13 13:39:40 -05001216 #
1217 # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2
1218 # https://github.com/apple/cups/releases
1219 ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", "(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz")
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001220 : "2.0.0",
Andrew Geissler82c905d2020-04-13 13:39:40 -05001221 # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz
1222 # http://ftp.debian.org/debian/pool/main/d/db5.3/
1223 ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", "(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz")
Brad Bishop79641f22019-09-10 07:20:22 -04001224 : "5.3.10",
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001225 }
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001226
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001227 @skipIfNoNetwork()
1228 def test_git_latest_versionstring(self):
1229 for k, v in self.test_git_uris.items():
1230 self.d.setVar("PN", k[0])
1231 self.d.setVar("SRCREV", k[2])
1232 self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3])
1233 ud = bb.fetch2.FetchData(k[1], self.d)
1234 pupver= ud.method.latest_versionstring(ud, self.d)
1235 verstring = pupver[0]
Brad Bishop316dfdd2018-06-25 12:45:53 -04001236 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001237 r = bb.utils.vercmp_string(v, verstring)
1238 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
1239
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001240 def test_wget_latest_versionstring(self):
Andrew Geissler82c905d2020-04-13 13:39:40 -05001241 testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata"
1242 server = HTTPService(testdata)
1243 server.start()
1244 port = server.port
1245 try:
1246 for k, v in self.test_wget_uris.items():
1247 self.d.setVar("PN", k[0])
1248 checkuri = ""
1249 if k[2]:
1250 checkuri = "http://localhost:%s/" % port + k[2]
1251 self.d.setVar("UPSTREAM_CHECK_URI", checkuri)
1252 self.d.setVar("UPSTREAM_CHECK_REGEX", k[3])
1253 url = "http://localhost:%s/" % port + k[1]
1254 ud = bb.fetch2.FetchData(url, self.d)
1255 pupver = ud.method.latest_versionstring(ud, self.d)
1256 verstring = pupver[0]
1257 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
1258 r = bb.utils.vercmp_string(v, verstring)
1259 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
1260 finally:
1261 server.stop()
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001262
1263
1264class FetchCheckStatusTest(FetcherTest):
1265 test_wget_uris = ["http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2",
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001266 "http://www.cups.org/",
1267 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
1268 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
1269 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
1270 "https://yoctoproject.org/",
1271 "https://yoctoproject.org/documentation",
1272 "http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
1273 "http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001274 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz",
1275 "http://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz",
1276 "https://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz",
1277 "https://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -05001278 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD
1279 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz"
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001280 ]
1281
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001282 @skipIfNoNetwork()
1283 def test_wget_checkstatus(self):
1284 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d)
1285 for u in self.test_wget_uris:
1286 with self.subTest(url=u):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001287 ud = fetch.ud[u]
1288 m = ud.method
1289 ret = m.checkstatus(fetch, ud, self.d)
1290 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
1291
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001292 @skipIfNoNetwork()
1293 def test_wget_checkstatus_connection_cache(self):
1294 from bb.fetch2 import FetchConnectionCache
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001295
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001296 connection_cache = FetchConnectionCache()
1297 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d,
1298 connection_cache = connection_cache)
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001299
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001300 for u in self.test_wget_uris:
1301 with self.subTest(url=u):
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001302 ud = fetch.ud[u]
1303 m = ud.method
1304 ret = m.checkstatus(fetch, ud, self.d)
1305 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
1306
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001307 connection_cache.close_connections()
1308
1309
1310class GitMakeShallowTest(FetcherTest):
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001311 def setUp(self):
1312 FetcherTest.setUp(self)
1313 self.gitdir = os.path.join(self.tempdir, 'gitshallow')
1314 bb.utils.mkdirhier(self.gitdir)
1315 bb.process.run('git init', cwd=self.gitdir)
1316
1317 def assertRefs(self, expected_refs):
1318 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
1319 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines()
1320 self.assertEqual(sorted(full_expected), sorted(actual_refs))
1321
1322 def assertRevCount(self, expected_count, args=None):
1323 if args is None:
1324 args = ['HEAD']
1325 revs = self.git(['rev-list'] + args)
1326 actual_count = len(revs.splitlines())
1327 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1328
1329 def git(self, cmd):
1330 if isinstance(cmd, str):
1331 cmd = 'git ' + cmd
1332 else:
1333 cmd = ['git'] + cmd
1334 return bb.process.run(cmd, cwd=self.gitdir)[0]
1335
1336 def make_shallow(self, args=None):
1337 if args is None:
1338 args = ['HEAD']
Brad Bishop316dfdd2018-06-25 12:45:53 -04001339 return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001340
1341 def add_empty_file(self, path, msg=None):
1342 if msg is None:
1343 msg = path
1344 open(os.path.join(self.gitdir, path), 'w').close()
1345 self.git(['add', path])
1346 self.git(['commit', '-m', msg, path])
1347
1348 def test_make_shallow_single_branch_no_merge(self):
1349 self.add_empty_file('a')
1350 self.add_empty_file('b')
1351 self.assertRevCount(2)
1352 self.make_shallow()
1353 self.assertRevCount(1)
1354
1355 def test_make_shallow_single_branch_one_merge(self):
1356 self.add_empty_file('a')
1357 self.add_empty_file('b')
1358 self.git('checkout -b a_branch')
1359 self.add_empty_file('c')
1360 self.git('checkout master')
1361 self.add_empty_file('d')
1362 self.git('merge --no-ff --no-edit a_branch')
1363 self.git('branch -d a_branch')
1364 self.add_empty_file('e')
1365 self.assertRevCount(6)
1366 self.make_shallow(['HEAD~2'])
1367 self.assertRevCount(5)
1368
1369 def test_make_shallow_at_merge(self):
1370 self.add_empty_file('a')
1371 self.git('checkout -b a_branch')
1372 self.add_empty_file('b')
1373 self.git('checkout master')
1374 self.git('merge --no-ff --no-edit a_branch')
1375 self.git('branch -d a_branch')
1376 self.assertRevCount(3)
1377 self.make_shallow()
1378 self.assertRevCount(1)
1379
1380 def test_make_shallow_annotated_tag(self):
1381 self.add_empty_file('a')
1382 self.add_empty_file('b')
1383 self.git('tag -a -m a_tag a_tag')
1384 self.assertRevCount(2)
1385 self.make_shallow(['a_tag'])
1386 self.assertRevCount(1)
1387
1388 def test_make_shallow_multi_ref(self):
1389 self.add_empty_file('a')
1390 self.add_empty_file('b')
1391 self.git('checkout -b a_branch')
1392 self.add_empty_file('c')
1393 self.git('checkout master')
1394 self.add_empty_file('d')
1395 self.git('checkout -b a_branch_2')
1396 self.add_empty_file('a_tag')
1397 self.git('tag a_tag')
1398 self.git('checkout master')
1399 self.git('branch -D a_branch_2')
1400 self.add_empty_file('e')
1401 self.assertRevCount(6, ['--all'])
1402 self.make_shallow()
1403 self.assertRevCount(5, ['--all'])
1404
1405 def test_make_shallow_multi_ref_trim(self):
1406 self.add_empty_file('a')
1407 self.git('checkout -b a_branch')
1408 self.add_empty_file('c')
1409 self.git('checkout master')
1410 self.assertRevCount(1)
1411 self.assertRevCount(2, ['--all'])
1412 self.assertRefs(['master', 'a_branch'])
1413 self.make_shallow(['-r', 'master', 'HEAD'])
1414 self.assertRevCount(1, ['--all'])
1415 self.assertRefs(['master'])
1416
1417 def test_make_shallow_noop(self):
1418 self.add_empty_file('a')
1419 self.assertRevCount(1)
1420 self.make_shallow()
1421 self.assertRevCount(1)
1422
1423 @skipIfNoNetwork()
1424 def test_make_shallow_bitbake(self):
1425 self.git('remote add origin https://github.com/openembedded/bitbake')
1426 self.git('fetch --tags origin')
1427 orig_revs = len(self.git('rev-list --all').splitlines())
1428 self.make_shallow(['refs/tags/1.10.0'])
1429 self.assertRevCount(orig_revs - 1746, ['--all'])
1430
1431class GitShallowTest(FetcherTest):
1432 def setUp(self):
1433 FetcherTest.setUp(self)
1434 self.gitdir = os.path.join(self.tempdir, 'git')
1435 self.srcdir = os.path.join(self.tempdir, 'gitsource')
1436
1437 bb.utils.mkdirhier(self.srcdir)
1438 self.git('init', cwd=self.srcdir)
1439 self.d.setVar('WORKDIR', self.tempdir)
1440 self.d.setVar('S', self.gitdir)
1441 self.d.delVar('PREMIRRORS')
1442 self.d.delVar('MIRRORS')
1443
1444 uri = 'git://%s;protocol=file;subdir=${S}' % self.srcdir
1445 self.d.setVar('SRC_URI', uri)
1446 self.d.setVar('SRCREV', '${AUTOREV}')
1447 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
1448
1449 self.d.setVar('BB_GIT_SHALLOW', '1')
1450 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
1451 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
1452
1453 def assertRefs(self, expected_refs, cwd=None):
1454 if cwd is None:
1455 cwd = self.gitdir
1456 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines()
1457 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines()
1458 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs)))
1459
1460 def assertRevCount(self, expected_count, args=None, cwd=None):
1461 if args is None:
1462 args = ['HEAD']
1463 if cwd is None:
1464 cwd = self.gitdir
1465 revs = self.git(['rev-list'] + args, cwd=cwd)
1466 actual_count = len(revs.splitlines())
1467 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1468
1469 def git(self, cmd, cwd=None):
1470 if isinstance(cmd, str):
1471 cmd = 'git ' + cmd
1472 else:
1473 cmd = ['git'] + cmd
1474 if cwd is None:
1475 cwd = self.gitdir
1476 return bb.process.run(cmd, cwd=cwd)[0]
1477
1478 def add_empty_file(self, path, cwd=None, msg=None):
1479 if msg is None:
1480 msg = path
1481 if cwd is None:
1482 cwd = self.srcdir
1483 open(os.path.join(cwd, path), 'w').close()
1484 self.git(['add', path], cwd)
1485 self.git(['commit', '-m', msg, path], cwd)
1486
1487 def fetch(self, uri=None):
1488 if uri is None:
Brad Bishop19323692019-04-05 15:28:33 -04001489 uris = self.d.getVar('SRC_URI').split()
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001490 uri = uris[0]
1491 d = self.d
1492 else:
1493 d = self.d.createCopy()
1494 d.setVar('SRC_URI', uri)
1495 uri = d.expand(uri)
1496 uris = [uri]
1497
1498 fetcher = bb.fetch2.Fetch(uris, d)
1499 fetcher.download()
1500 ud = fetcher.ud[uri]
1501 return fetcher, ud
1502
1503 def fetch_and_unpack(self, uri=None):
1504 fetcher, ud = self.fetch(uri)
1505 fetcher.unpack(self.d.getVar('WORKDIR'))
1506 assert os.path.exists(self.d.getVar('S'))
1507 return fetcher, ud
1508
1509 def fetch_shallow(self, uri=None, disabled=False, keepclone=False):
1510 """Fetch a uri, generating a shallow tarball, then unpack using it"""
1511 fetcher, ud = self.fetch_and_unpack(uri)
1512 assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri)
1513
1514 # Confirm that the unpacked repo is unshallow
1515 if not disabled:
1516 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1517
1518 # fetch and unpack, from the shallow tarball
1519 bb.utils.remove(self.gitdir, recurse=True)
1520 bb.utils.remove(ud.clonedir, recurse=True)
Brad Bishopf8caae32019-03-25 13:13:56 -04001521 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001522
1523 # confirm that the unpacked repo is used when no git clone or git
1524 # mirror tarball is available
1525 fetcher, ud = self.fetch_and_unpack(uri)
1526 if not disabled:
1527 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir
1528 else:
1529 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir
1530 return fetcher, ud
1531
1532 def test_shallow_disabled(self):
1533 self.add_empty_file('a')
1534 self.add_empty_file('b')
1535 self.assertRevCount(2, cwd=self.srcdir)
1536
1537 self.d.setVar('BB_GIT_SHALLOW', '0')
1538 self.fetch_shallow(disabled=True)
1539 self.assertRevCount(2)
1540
1541 def test_shallow_nobranch(self):
1542 self.add_empty_file('a')
1543 self.add_empty_file('b')
1544 self.assertRevCount(2, cwd=self.srcdir)
1545
1546 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip()
1547 self.d.setVar('SRCREV', srcrev)
Brad Bishop19323692019-04-05 15:28:33 -04001548 uri = self.d.getVar('SRC_URI').split()[0]
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001549 uri = '%s;nobranch=1;bare=1' % uri
1550
1551 self.fetch_shallow(uri)
1552 self.assertRevCount(1)
1553
1554 # shallow refs are used to ensure the srcrev sticks around when we
1555 # have no other branches referencing it
1556 self.assertRefs(['refs/shallow/default'])
1557
1558 def test_shallow_default_depth_1(self):
1559 # Create initial git repo
1560 self.add_empty_file('a')
1561 self.add_empty_file('b')
1562 self.assertRevCount(2, cwd=self.srcdir)
1563
1564 self.fetch_shallow()
1565 self.assertRevCount(1)
1566
1567 def test_shallow_depth_0_disables(self):
1568 self.add_empty_file('a')
1569 self.add_empty_file('b')
1570 self.assertRevCount(2, cwd=self.srcdir)
1571
1572 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1573 self.fetch_shallow(disabled=True)
1574 self.assertRevCount(2)
1575
1576 def test_shallow_depth_default_override(self):
1577 self.add_empty_file('a')
1578 self.add_empty_file('b')
1579 self.assertRevCount(2, cwd=self.srcdir)
1580
1581 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2')
1582 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1')
1583 self.fetch_shallow()
1584 self.assertRevCount(1)
1585
1586 def test_shallow_depth_default_override_disable(self):
1587 self.add_empty_file('a')
1588 self.add_empty_file('b')
1589 self.add_empty_file('c')
1590 self.assertRevCount(3, cwd=self.srcdir)
1591
1592 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1593 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2')
1594 self.fetch_shallow()
1595 self.assertRevCount(2)
1596
1597 def test_current_shallow_out_of_date_clone(self):
1598 # Create initial git repo
1599 self.add_empty_file('a')
1600 self.add_empty_file('b')
1601 self.add_empty_file('c')
1602 self.assertRevCount(3, cwd=self.srcdir)
1603
1604 # Clone and generate mirror tarball
1605 fetcher, ud = self.fetch()
1606
1607 # Ensure we have a current mirror tarball, but an out of date clone
1608 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir)
1609 self.assertRevCount(2, cwd=ud.clonedir)
1610
1611 # Fetch and unpack, from the current tarball, not the out of date clone
1612 bb.utils.remove(self.gitdir, recurse=True)
1613 fetcher, ud = self.fetch()
1614 fetcher.unpack(self.d.getVar('WORKDIR'))
1615 self.assertRevCount(1)
1616
1617 def test_shallow_single_branch_no_merge(self):
1618 self.add_empty_file('a')
1619 self.add_empty_file('b')
1620 self.assertRevCount(2, cwd=self.srcdir)
1621
1622 self.fetch_shallow()
1623 self.assertRevCount(1)
1624 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1625 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1626
1627 def test_shallow_no_dangling(self):
1628 self.add_empty_file('a')
1629 self.add_empty_file('b')
1630 self.assertRevCount(2, cwd=self.srcdir)
1631
1632 self.fetch_shallow()
1633 self.assertRevCount(1)
1634 assert not self.git('fsck --dangling')
1635
1636 def test_shallow_srcrev_branch_truncation(self):
1637 self.add_empty_file('a')
1638 self.add_empty_file('b')
1639 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip()
1640 self.add_empty_file('c')
1641 self.assertRevCount(3, cwd=self.srcdir)
1642
1643 self.d.setVar('SRCREV', b_commit)
1644 self.fetch_shallow()
1645
1646 # The 'c' commit was removed entirely, and 'a' was removed from history
1647 self.assertRevCount(1, ['--all'])
1648 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit)
1649 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1650 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1651 assert not os.path.exists(os.path.join(self.gitdir, 'c'))
1652
1653 def test_shallow_ref_pruning(self):
1654 self.add_empty_file('a')
1655 self.add_empty_file('b')
1656 self.git('branch a_branch', cwd=self.srcdir)
1657 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
1658 self.assertRevCount(2, cwd=self.srcdir)
1659
1660 self.fetch_shallow()
1661
1662 self.assertRefs(['master', 'origin/master'])
1663 self.assertRevCount(1)
1664
1665 def test_shallow_submodules(self):
1666 self.add_empty_file('a')
1667 self.add_empty_file('b')
1668
1669 smdir = os.path.join(self.tempdir, 'gitsubmodule')
1670 bb.utils.mkdirhier(smdir)
1671 self.git('init', cwd=smdir)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001672 # Make this look like it was cloned from a remote...
1673 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
1674 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001675 self.add_empty_file('asub', cwd=smdir)
Brad Bishopf8caae32019-03-25 13:13:56 -04001676 self.add_empty_file('bsub', cwd=smdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001677
1678 self.git('submodule init', cwd=self.srcdir)
1679 self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
1680 self.git('submodule update', cwd=self.srcdir)
1681 self.git('commit -m submodule -a', cwd=self.srcdir)
1682
1683 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
1684 fetcher, ud = self.fetch_shallow(uri)
1685
Brad Bishopf8caae32019-03-25 13:13:56 -04001686 # Verify the main repository is shallow
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001687 self.assertRevCount(1)
Brad Bishopf8caae32019-03-25 13:13:56 -04001688
1689 # Verify the gitsubmodule directory is present
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001690 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
1691
Brad Bishopf8caae32019-03-25 13:13:56 -04001692 # Verify the submodule is also shallow
1693 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
1694
Andrew Geissler82c905d2020-04-13 13:39:40 -05001695 def test_shallow_submodule_mirrors(self):
1696 self.add_empty_file('a')
1697 self.add_empty_file('b')
1698
1699 smdir = os.path.join(self.tempdir, 'gitsubmodule')
1700 bb.utils.mkdirhier(smdir)
1701 self.git('init', cwd=smdir)
1702 # Make this look like it was cloned from a remote...
1703 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
1704 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
1705 self.add_empty_file('asub', cwd=smdir)
1706 self.add_empty_file('bsub', cwd=smdir)
1707
1708 self.git('submodule init', cwd=self.srcdir)
1709 self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
1710 self.git('submodule update', cwd=self.srcdir)
1711 self.git('commit -m submodule -a', cwd=self.srcdir)
1712
1713 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
1714
1715 # Fetch once to generate the shallow tarball
1716 fetcher, ud = self.fetch(uri)
1717
1718 # Set up the mirror
1719 mirrordir = os.path.join(self.tempdir, 'mirror')
1720 os.rename(self.dldir, mirrordir)
1721 self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/\n' % mirrordir)
1722
1723 # Fetch from the mirror
1724 bb.utils.remove(self.dldir, recurse=True)
1725 bb.utils.remove(self.gitdir, recurse=True)
1726 self.fetch_and_unpack(uri)
1727
1728 # Verify the main repository is shallow
1729 self.assertRevCount(1)
1730
1731 # Verify the gitsubmodule directory is present
1732 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
1733
1734 # Verify the submodule is also shallow
1735 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
Brad Bishopf8caae32019-03-25 13:13:56 -04001736
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001737 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')):
1738 def test_shallow_annex(self):
1739 self.add_empty_file('a')
1740 self.add_empty_file('b')
1741 self.git('annex init', cwd=self.srcdir)
1742 open(os.path.join(self.srcdir, 'c'), 'w').close()
1743 self.git('annex add c', cwd=self.srcdir)
1744 self.git('commit -m annex-c -a', cwd=self.srcdir)
1745 bb.process.run('chmod u+w -R %s' % os.path.join(self.srcdir, '.git', 'annex'))
1746
1747 uri = 'gitannex://%s;protocol=file;subdir=${S}' % self.srcdir
1748 fetcher, ud = self.fetch_shallow(uri)
1749
1750 self.assertRevCount(1)
1751 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0]
1752 assert os.path.exists(os.path.join(self.gitdir, 'c'))
1753
1754 def test_shallow_multi_one_uri(self):
1755 # Create initial git repo
1756 self.add_empty_file('a')
1757 self.add_empty_file('b')
1758 self.git('checkout -b a_branch', cwd=self.srcdir)
1759 self.add_empty_file('c')
1760 self.add_empty_file('d')
1761 self.git('checkout master', cwd=self.srcdir)
1762 self.git('tag v0.0 a_branch', cwd=self.srcdir)
1763 self.add_empty_file('e')
1764 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1765 self.add_empty_file('f')
1766 self.assertRevCount(7, cwd=self.srcdir)
1767
Brad Bishop19323692019-04-05 15:28:33 -04001768 uri = self.d.getVar('SRC_URI').split()[0]
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001769 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
1770
1771 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1772 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1773 self.d.setVar('SRCREV_master', '${AUTOREV}')
1774 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
1775
1776 self.fetch_shallow(uri)
1777
1778 self.assertRevCount(5)
1779 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1780
1781 def test_shallow_multi_one_uri_depths(self):
1782 # Create initial git repo
1783 self.add_empty_file('a')
1784 self.add_empty_file('b')
1785 self.git('checkout -b a_branch', cwd=self.srcdir)
1786 self.add_empty_file('c')
1787 self.add_empty_file('d')
1788 self.git('checkout master', cwd=self.srcdir)
1789 self.add_empty_file('e')
1790 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1791 self.add_empty_file('f')
1792 self.assertRevCount(7, cwd=self.srcdir)
1793
Brad Bishop19323692019-04-05 15:28:33 -04001794 uri = self.d.getVar('SRC_URI').split()[0]
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001795 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
1796
1797 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1798 self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3')
1799 self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1')
1800 self.d.setVar('SRCREV_master', '${AUTOREV}')
1801 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
1802
1803 self.fetch_shallow(uri)
1804
1805 self.assertRevCount(4, ['--all'])
1806 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1807
1808 def test_shallow_clone_preferred_over_shallow(self):
1809 self.add_empty_file('a')
1810 self.add_empty_file('b')
1811
1812 # Fetch once to generate the shallow tarball
1813 fetcher, ud = self.fetch()
1814 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1815
1816 # Fetch and unpack with both the clonedir and shallow tarball available
1817 bb.utils.remove(self.gitdir, recurse=True)
1818 fetcher, ud = self.fetch_and_unpack()
1819
1820 # The unpacked tree should *not* be shallow
1821 self.assertRevCount(2)
1822 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow'))
1823
1824 def test_shallow_mirrors(self):
1825 self.add_empty_file('a')
1826 self.add_empty_file('b')
1827
1828 # Fetch once to generate the shallow tarball
1829 fetcher, ud = self.fetch()
1830 mirrortarball = ud.mirrortarballs[0]
1831 assert os.path.exists(os.path.join(self.dldir, mirrortarball))
1832
1833 # Set up the mirror
1834 mirrordir = os.path.join(self.tempdir, 'mirror')
1835 bb.utils.mkdirhier(mirrordir)
1836 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir)
1837
1838 os.rename(os.path.join(self.dldir, mirrortarball),
1839 os.path.join(mirrordir, mirrortarball))
1840
1841 # Fetch from the mirror
1842 bb.utils.remove(self.dldir, recurse=True)
1843 bb.utils.remove(self.gitdir, recurse=True)
1844 self.fetch_and_unpack()
1845 self.assertRevCount(1)
1846
1847 def test_shallow_invalid_depth(self):
1848 self.add_empty_file('a')
1849 self.add_empty_file('b')
1850
1851 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12')
1852 with self.assertRaises(bb.fetch2.FetchError):
1853 self.fetch()
1854
1855 def test_shallow_invalid_depth_default(self):
1856 self.add_empty_file('a')
1857 self.add_empty_file('b')
1858
1859 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12')
1860 with self.assertRaises(bb.fetch2.FetchError):
1861 self.fetch()
1862
1863 def test_shallow_extra_refs(self):
1864 self.add_empty_file('a')
1865 self.add_empty_file('b')
1866 self.git('branch a_branch', cwd=self.srcdir)
1867 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
1868 self.assertRevCount(2, cwd=self.srcdir)
1869
1870 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch')
1871 self.fetch_shallow()
1872
1873 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1874 self.assertRevCount(1)
1875
1876 def test_shallow_extra_refs_wildcard(self):
1877 self.add_empty_file('a')
1878 self.add_empty_file('b')
1879 self.git('branch a_branch', cwd=self.srcdir)
1880 self.git('tag v1.0', cwd=self.srcdir)
1881 self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir)
1882 self.assertRevCount(2, cwd=self.srcdir)
1883
1884 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
1885 self.fetch_shallow()
1886
1887 self.assertRefs(['master', 'origin/master', 'v1.0'])
1888 self.assertRevCount(1)
1889
1890 def test_shallow_missing_extra_refs(self):
1891 self.add_empty_file('a')
1892 self.add_empty_file('b')
1893
1894 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo')
1895 with self.assertRaises(bb.fetch2.FetchError):
1896 self.fetch()
1897
1898 def test_shallow_missing_extra_refs_wildcard(self):
1899 self.add_empty_file('a')
1900 self.add_empty_file('b')
1901
1902 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
1903 self.fetch()
1904
1905 def test_shallow_remove_revs(self):
1906 # Create initial git repo
1907 self.add_empty_file('a')
1908 self.add_empty_file('b')
1909 self.git('checkout -b a_branch', cwd=self.srcdir)
1910 self.add_empty_file('c')
1911 self.add_empty_file('d')
1912 self.git('checkout master', cwd=self.srcdir)
1913 self.git('tag v0.0 a_branch', cwd=self.srcdir)
1914 self.add_empty_file('e')
1915 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1916 self.git('branch -d a_branch', cwd=self.srcdir)
1917 self.add_empty_file('f')
1918 self.assertRevCount(7, cwd=self.srcdir)
1919
1920 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1921 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1922
1923 self.fetch_shallow()
1924
1925 self.assertRevCount(5)
1926
1927 def test_shallow_invalid_revs(self):
1928 self.add_empty_file('a')
1929 self.add_empty_file('b')
1930
1931 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1932 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1933
1934 with self.assertRaises(bb.fetch2.FetchError):
1935 self.fetch()
1936
Brad Bishop64c979e2019-11-04 13:55:29 -05001937 def test_shallow_fetch_missing_revs(self):
1938 self.add_empty_file('a')
1939 self.add_empty_file('b')
1940 fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
1941 self.git('tag v0.0 master', cwd=self.srcdir)
1942 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1943 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1944 self.fetch_shallow()
1945
1946 def test_shallow_fetch_missing_revs_fails(self):
1947 self.add_empty_file('a')
1948 self.add_empty_file('b')
1949 fetcher, ud = self.fetch(self.d.getVar('SRC_URI'))
1950 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1951 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1952
1953 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm:
1954 self.fetch_shallow()
1955 self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0])
1956
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001957 @skipIfNoNetwork()
1958 def test_bitbake(self):
1959 self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
1960 self.git('config core.bare true', cwd=self.srcdir)
1961 self.git('fetch', cwd=self.srcdir)
1962
1963 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1964 # Note that the 1.10.0 tag is annotated, so this also tests
1965 # reference of an annotated vs unannotated tag
1966 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0')
1967
1968 self.fetch_shallow()
1969
1970 # Confirm that the history of 1.10.0 was removed
1971 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines())
1972 revs = len(self.git('rev-list master').splitlines())
1973 self.assertNotEqual(orig_revs, revs)
1974 self.assertRefs(['master', 'origin/master'])
1975 self.assertRevCount(orig_revs - 1758)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08001976
1977 def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self):
1978 self.add_empty_file('a')
1979 fetcher, ud = self.fetch()
1980 bb.utils.remove(self.gitdir, recurse=True)
1981 bb.utils.remove(self.dldir, recurse=True)
1982
1983 with self.assertRaises(bb.fetch2.UnpackError) as context:
1984 fetcher.unpack(self.d.getVar('WORKDIR'))
1985
1986 self.assertIn("No up to date source found", context.exception.msg)
1987 self.assertIn("clone directory not available or not up to date", context.exception.msg)
1988
1989 @skipIfNoNetwork()
1990 def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self):
1991 self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0')
1992 self.d.setVar('BB_GIT_SHALLOW', '1')
1993 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
1994 fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests"], self.d)
1995 fetcher.download()
1996
1997 bb.utils.remove(self.dldir + "/*.tar.gz")
1998 fetcher.unpack(self.unpackdir)
1999
2000 dir = os.listdir(self.unpackdir + "/git/")
2001 self.assertIn("fstests.doap", dir)
Brad Bishop00e122a2019-10-05 11:10:57 -04002002
2003class GitLfsTest(FetcherTest):
2004 def setUp(self):
2005 FetcherTest.setUp(self)
2006
2007 self.gitdir = os.path.join(self.tempdir, 'git')
2008 self.srcdir = os.path.join(self.tempdir, 'gitsource')
2009
2010 self.d.setVar('WORKDIR', self.tempdir)
2011 self.d.setVar('S', self.gitdir)
2012 self.d.delVar('PREMIRRORS')
2013 self.d.delVar('MIRRORS')
2014
2015 self.d.setVar('SRCREV', '${AUTOREV}')
2016 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
2017
2018 bb.utils.mkdirhier(self.srcdir)
2019 self.git('init', cwd=self.srcdir)
2020 with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs:
2021 attrs.write('*.mp3 filter=lfs -text')
2022 self.git(['add', '.gitattributes'], cwd=self.srcdir)
2023 self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir)
2024
2025 def git(self, cmd, cwd=None):
2026 if isinstance(cmd, str):
2027 cmd = 'git ' + cmd
2028 else:
2029 cmd = ['git'] + cmd
2030 if cwd is None:
2031 cwd = self.gitdir
2032 return bb.process.run(cmd, cwd=cwd)[0]
2033
2034 def fetch(self, uri=None):
2035 uris = self.d.getVar('SRC_URI').split()
2036 uri = uris[0]
2037 d = self.d
2038
2039 fetcher = bb.fetch2.Fetch(uris, d)
2040 fetcher.download()
2041 ud = fetcher.ud[uri]
2042 return fetcher, ud
2043
2044 def test_lfs_enabled(self):
2045 import shutil
2046
2047 uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir
2048 self.d.setVar('SRC_URI', uri)
2049
2050 fetcher, ud = self.fetch()
2051 self.assertIsNotNone(ud.method._find_git_lfs)
2052
2053 # If git-lfs can be found, the unpack should be successful
2054 ud.method._find_git_lfs = lambda d: True
2055 shutil.rmtree(self.gitdir, ignore_errors=True)
2056 fetcher.unpack(self.d.getVar('WORKDIR'))
2057
2058 # If git-lfs cannot be found, the unpack should throw an error
2059 with self.assertRaises(bb.fetch2.FetchError):
2060 ud.method._find_git_lfs = lambda d: False
2061 shutil.rmtree(self.gitdir, ignore_errors=True)
2062 fetcher.unpack(self.d.getVar('WORKDIR'))
2063
2064 def test_lfs_disabled(self):
2065 import shutil
2066
2067 uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir
2068 self.d.setVar('SRC_URI', uri)
2069
2070 fetcher, ud = self.fetch()
2071 self.assertIsNotNone(ud.method._find_git_lfs)
2072
2073 # If git-lfs can be found, the unpack should be successful
2074 ud.method._find_git_lfs = lambda d: True
2075 shutil.rmtree(self.gitdir, ignore_errors=True)
2076 fetcher.unpack(self.d.getVar('WORKDIR'))
2077
2078 # If git-lfs cannot be found, the unpack should be successful
2079 ud.method._find_git_lfs = lambda d: False
2080 shutil.rmtree(self.gitdir, ignore_errors=True)
2081 fetcher.unpack(self.d.getVar('WORKDIR'))
Andrew Geissler82c905d2020-04-13 13:39:40 -05002082
2083class NPMTest(FetcherTest):
2084 def skipIfNoNpm():
2085 import shutil
2086 if not shutil.which('npm'):
2087 return unittest.skip('npm not installed, tests being skipped')
2088 return lambda f: f
2089
2090 @skipIfNoNpm()
2091 @skipIfNoNetwork()
2092 def test_npm(self):
2093 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2094 fetcher = bb.fetch.Fetch([url], self.d)
2095 ud = fetcher.ud[fetcher.urls[0]]
2096 fetcher.download()
2097 self.assertTrue(os.path.exists(ud.localpath))
2098 self.assertTrue(os.path.exists(ud.localpath + '.done'))
2099 self.assertTrue(os.path.exists(ud.resolvefile))
2100 fetcher.unpack(self.unpackdir)
2101 unpackdir = os.path.join(self.unpackdir, 'npm')
2102 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2103
2104 @skipIfNoNpm()
2105 @skipIfNoNetwork()
2106 def test_npm_bad_checksum(self):
2107 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2108 # Fetch once to get a tarball
2109 fetcher = bb.fetch.Fetch([url], self.d)
2110 ud = fetcher.ud[fetcher.urls[0]]
2111 fetcher.download()
2112 self.assertTrue(os.path.exists(ud.localpath))
2113 # Modify the tarball
2114 bad = b'bad checksum'
2115 with open(ud.localpath, 'wb') as f:
2116 f.write(bad)
2117 # Verify that the tarball is fetched again
2118 fetcher.download()
2119 badsum = hashlib.sha512(bad).hexdigest()
2120 self.assertTrue(os.path.exists(ud.localpath + '_bad-checksum_' + badsum))
2121 self.assertTrue(os.path.exists(ud.localpath))
2122
2123 @skipIfNoNpm()
2124 @skipIfNoNetwork()
2125 def test_npm_premirrors(self):
2126 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2127 # Fetch once to get a tarball
2128 fetcher = bb.fetch.Fetch([url], self.d)
2129 ud = fetcher.ud[fetcher.urls[0]]
2130 fetcher.download()
2131 self.assertTrue(os.path.exists(ud.localpath))
2132 # Setup the mirror
2133 mirrordir = os.path.join(self.tempdir, 'mirror')
2134 bb.utils.mkdirhier(mirrordir)
2135 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
2136 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
2137 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
2138 # Fetch again
2139 self.assertFalse(os.path.exists(ud.localpath))
2140 fetcher.download()
2141 self.assertTrue(os.path.exists(ud.localpath))
2142
2143 @skipIfNoNpm()
2144 @skipIfNoNetwork()
2145 def test_npm_mirrors(self):
2146 # Fetch once to get a tarball
2147 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2148 fetcher = bb.fetch.Fetch([url], self.d)
2149 ud = fetcher.ud[fetcher.urls[0]]
2150 fetcher.download()
2151 self.assertTrue(os.path.exists(ud.localpath))
2152 # Setup the mirror
2153 mirrordir = os.path.join(self.tempdir, 'mirror')
2154 bb.utils.mkdirhier(mirrordir)
2155 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
2156 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
2157 # Update the resolved url to an invalid url
2158 with open(ud.resolvefile, 'r') as f:
2159 url = f.read()
2160 uri = URI(url)
2161 uri.path = '/invalid'
2162 with open(ud.resolvefile, 'w') as f:
2163 f.write(str(uri))
2164 # Fetch again
2165 self.assertFalse(os.path.exists(ud.localpath))
2166 fetcher.download()
2167 self.assertTrue(os.path.exists(ud.localpath))
2168
2169 @skipIfNoNpm()
2170 @skipIfNoNetwork()
2171 def test_npm_destsuffix_downloadfilename(self):
2172 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz'
2173 fetcher = bb.fetch.Fetch([url], self.d)
2174 fetcher.download()
2175 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'foo-bar.tgz')))
2176 fetcher.unpack(self.unpackdir)
2177 unpackdir = os.path.join(self.unpackdir, 'foo', 'bar')
2178 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2179
2180 def test_npm_no_network_no_tarball(self):
2181 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2182 self.d.setVar('BB_NO_NETWORK', '1')
2183 fetcher = bb.fetch.Fetch([url], self.d)
2184 with self.assertRaises(bb.fetch2.NetworkAccess):
2185 fetcher.download()
2186
2187 @skipIfNoNpm()
2188 @skipIfNoNetwork()
2189 def test_npm_no_network_with_tarball(self):
2190 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2191 # Fetch once to get a tarball
2192 fetcher = bb.fetch.Fetch([url], self.d)
2193 fetcher.download()
2194 # Disable network access
2195 self.d.setVar('BB_NO_NETWORK', '1')
2196 # Fetch again
2197 fetcher.download()
2198 fetcher.unpack(self.unpackdir)
2199 unpackdir = os.path.join(self.unpackdir, 'npm')
2200 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2201
2202 @skipIfNoNpm()
2203 @skipIfNoNetwork()
2204 def test_npm_registry_alternate(self):
2205 url = 'npm://registry.freajs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2206 fetcher = bb.fetch.Fetch([url], self.d)
2207 fetcher.download()
2208 fetcher.unpack(self.unpackdir)
2209 unpackdir = os.path.join(self.unpackdir, 'npm')
2210 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2211
2212 @skipIfNoNpm()
2213 @skipIfNoNetwork()
2214 def test_npm_version_latest(self):
2215 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest'
2216 fetcher = bb.fetch.Fetch([url], self.d)
2217 fetcher.download()
2218 fetcher.unpack(self.unpackdir)
2219 unpackdir = os.path.join(self.unpackdir, 'npm')
2220 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
2221
2222 @skipIfNoNpm()
2223 @skipIfNoNetwork()
2224 def test_npm_registry_invalid(self):
2225 url = 'npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
2226 fetcher = bb.fetch.Fetch([url], self.d)
2227 with self.assertRaises(bb.fetch2.FetchError):
2228 fetcher.download()
2229
2230 @skipIfNoNpm()
2231 @skipIfNoNetwork()
2232 def test_npm_package_invalid(self):
2233 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0'
2234 fetcher = bb.fetch.Fetch([url], self.d)
2235 with self.assertRaises(bb.fetch2.FetchError):
2236 fetcher.download()
2237
2238 @skipIfNoNpm()
2239 @skipIfNoNetwork()
2240 def test_npm_version_invalid(self):
2241 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid'
2242 with self.assertRaises(bb.fetch2.ParameterError):
2243 fetcher = bb.fetch.Fetch([url], self.d)
2244
2245 @skipIfNoNpm()
2246 @skipIfNoNetwork()
2247 def test_npm_registry_none(self):
2248 url = 'npm://;package=@savoirfairelinux/node-server-example;version=1.0.0'
2249 with self.assertRaises(bb.fetch2.MalformedUrl):
2250 fetcher = bb.fetch.Fetch([url], self.d)
2251
2252 @skipIfNoNpm()
2253 @skipIfNoNetwork()
2254 def test_npm_package_none(self):
2255 url = 'npm://registry.npmjs.org;version=1.0.0'
2256 with self.assertRaises(bb.fetch2.MissingParameterError):
2257 fetcher = bb.fetch.Fetch([url], self.d)
2258
2259 @skipIfNoNpm()
2260 @skipIfNoNetwork()
2261 def test_npm_version_none(self):
2262 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example'
2263 with self.assertRaises(bb.fetch2.MissingParameterError):
2264 fetcher = bb.fetch.Fetch([url], self.d)
2265
2266 def create_shrinkwrap_file(self, data):
2267 import json
2268 datadir = os.path.join(self.tempdir, 'data')
2269 swfile = os.path.join(datadir, 'npm-shrinkwrap.json')
2270 bb.utils.mkdirhier(datadir)
2271 with open(swfile, 'w') as f:
2272 json.dump(data, f)
2273 # Also configure the S directory
2274 self.sdir = os.path.join(self.unpackdir, 'S')
2275 self.d.setVar('S', self.sdir)
2276 return swfile
2277
2278 @skipIfNoNpm()
2279 @skipIfNoNetwork()
2280 def test_npmsw(self):
2281 swfile = self.create_shrinkwrap_file({
2282 'dependencies': {
2283 'array-flatten': {
2284 'version': '1.1.1',
2285 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2286 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=',
2287 'dependencies': {
2288 'content-type': {
2289 'version': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
2290 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
2291 'dependencies': {
2292 'cookie': {
2293 'version': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09',
2294 'from': 'git+https://github.com/jshttp/cookie.git'
2295 }
2296 }
2297 }
2298 }
2299 }
2300 }
2301 })
2302 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2303 fetcher.download()
2304 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
2305 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
2306 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git')))
2307 fetcher.unpack(self.unpackdir)
2308 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'npm-shrinkwrap.json')))
2309 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json')))
2310 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json')))
2311 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json')))
2312
2313 @skipIfNoNpm()
2314 @skipIfNoNetwork()
2315 def test_npmsw_dev(self):
2316 swfile = self.create_shrinkwrap_file({
2317 'dependencies': {
2318 'array-flatten': {
2319 'version': '1.1.1',
2320 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2321 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2322 },
2323 'content-type': {
2324 'version': '1.0.4',
2325 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
2326 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
2327 'dev': True
2328 }
2329 }
2330 })
2331 # Fetch with dev disabled
2332 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2333 fetcher.download()
2334 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
2335 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
2336 # Fetch with dev enabled
2337 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d)
2338 fetcher.download()
2339 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
2340 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
2341
2342 @skipIfNoNpm()
2343 @skipIfNoNetwork()
2344 def test_npmsw_destsuffix(self):
2345 swfile = self.create_shrinkwrap_file({
2346 'dependencies': {
2347 'array-flatten': {
2348 'version': '1.1.1',
2349 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2350 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2351 }
2352 }
2353 })
2354 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d)
2355 fetcher.download()
2356 fetcher.unpack(self.unpackdir)
2357 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json')))
2358
2359 def test_npmsw_no_network_no_tarball(self):
2360 swfile = self.create_shrinkwrap_file({
2361 'dependencies': {
2362 'array-flatten': {
2363 'version': '1.1.1',
2364 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2365 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2366 }
2367 }
2368 })
2369 self.d.setVar('BB_NO_NETWORK', '1')
2370 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2371 with self.assertRaises(bb.fetch2.NetworkAccess):
2372 fetcher.download()
2373
2374 @skipIfNoNpm()
2375 @skipIfNoNetwork()
2376 def test_npmsw_no_network_with_tarball(self):
2377 # Fetch once to get a tarball
2378 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2379 fetcher.download()
2380 # Disable network access
2381 self.d.setVar('BB_NO_NETWORK', '1')
2382 # Fetch again
2383 swfile = self.create_shrinkwrap_file({
2384 'dependencies': {
2385 'array-flatten': {
2386 'version': '1.1.1',
2387 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2388 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2389 }
2390 }
2391 })
2392 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2393 fetcher.download()
2394 fetcher.unpack(self.unpackdir)
2395 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json')))
2396
2397 @skipIfNoNpm()
2398 @skipIfNoNetwork()
2399 def test_npmsw_npm_reusability(self):
2400 # Fetch once with npmsw
2401 swfile = self.create_shrinkwrap_file({
2402 'dependencies': {
2403 'array-flatten': {
2404 'version': '1.1.1',
2405 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2406 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2407 }
2408 }
2409 })
2410 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2411 fetcher.download()
2412 # Disable network access
2413 self.d.setVar('BB_NO_NETWORK', '1')
2414 # Fetch again with npm
2415 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2416 fetcher.download()
2417 fetcher.unpack(self.unpackdir)
2418 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json')))
2419
2420 @skipIfNoNpm()
2421 @skipIfNoNetwork()
2422 def test_npmsw_bad_checksum(self):
2423 # Try to fetch with bad checksum
2424 swfile = self.create_shrinkwrap_file({
2425 'dependencies': {
2426 'array-flatten': {
2427 'version': '1.1.1',
2428 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2429 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg='
2430 }
2431 }
2432 })
2433 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2434 with self.assertRaises(bb.fetch2.FetchError):
2435 fetcher.download()
2436 # Fetch correctly to get a tarball
2437 swfile = self.create_shrinkwrap_file({
2438 'dependencies': {
2439 'array-flatten': {
2440 'version': '1.1.1',
2441 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2442 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2443 }
2444 }
2445 })
2446 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2447 fetcher.download()
2448 localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')
2449 self.assertTrue(os.path.exists(localpath))
2450 # Modify the tarball
2451 bad = b'bad checksum'
2452 with open(localpath, 'wb') as f:
2453 f.write(bad)
2454 # Verify that the tarball is fetched again
2455 fetcher.download()
2456 badsum = hashlib.sha1(bad).hexdigest()
2457 self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum))
2458 self.assertTrue(os.path.exists(localpath))
2459
2460 @skipIfNoNpm()
2461 @skipIfNoNetwork()
2462 def test_npmsw_premirrors(self):
2463 # Fetch once to get a tarball
2464 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2465 ud = fetcher.ud[fetcher.urls[0]]
2466 fetcher.download()
2467 self.assertTrue(os.path.exists(ud.localpath))
2468 # Setup the mirror
2469 mirrordir = os.path.join(self.tempdir, 'mirror')
2470 bb.utils.mkdirhier(mirrordir)
2471 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
2472 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
2473 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
2474 # Fetch again
2475 self.assertFalse(os.path.exists(ud.localpath))
2476 swfile = self.create_shrinkwrap_file({
2477 'dependencies': {
2478 'array-flatten': {
2479 'version': '1.1.1',
2480 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
2481 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2482 }
2483 }
2484 })
2485 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2486 fetcher.download()
2487 self.assertTrue(os.path.exists(ud.localpath))
2488
2489 @skipIfNoNpm()
2490 @skipIfNoNetwork()
2491 def test_npmsw_mirrors(self):
2492 # Fetch once to get a tarball
2493 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d)
2494 ud = fetcher.ud[fetcher.urls[0]]
2495 fetcher.download()
2496 self.assertTrue(os.path.exists(ud.localpath))
2497 # Setup the mirror
2498 mirrordir = os.path.join(self.tempdir, 'mirror')
2499 bb.utils.mkdirhier(mirrordir)
2500 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
2501 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
2502 # Fetch again with invalid url
2503 self.assertFalse(os.path.exists(ud.localpath))
2504 swfile = self.create_shrinkwrap_file({
2505 'dependencies': {
2506 'array-flatten': {
2507 'version': '1.1.1',
2508 'resolved': 'https://invalid',
2509 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
2510 }
2511 }
2512 })
2513 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
2514 fetcher.download()
2515 self.assertTrue(os.path.exists(ud.localpath))