Gitiles
Code Review Sign In
gerrit.openbmc.org / stefanberger / openbmc / 0dd04f33864280128a3d2869833d56fddad804d2 / . / poky / bitbake / lib / bb / tests / fetch.py
blob: 429998b34fb2ec341dd5f40cf570e491fa1909ec [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3#
4# BitBake Tests for the Fetcher (fetch2/)
5#
6# Copyright (C) 2012 Richard Purdie
7#
8# This program is free software; you can redistribute it and/or modify
9# it under the terms of the GNU General Public License version 2 as
10# published by the Free Software Foundation.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License along
18# with this program; if not, write to the Free Software Foundation, Inc.,
19# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20#
21
22import unittest
Brad Bishop316dfdd2018-06-25 12:45:53 -0400[diff] [blame]23import hashlib
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]24import tempfile
25import subprocess
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]26import collections
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]27import os
28from bb.fetch2 import URI
29from bb.fetch2 import FetchMethod
30import bb
31
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]32def skipIfNoNetwork():
33 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
34 return unittest.skip("Network tests being skipped")
35 return lambda f: f
36
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]37class URITest(unittest.TestCase):
38 test_uris = {
39 "http://www.google.com/index.html" : {
40 'uri': 'http://www.google.com/index.html',
41 'scheme': 'http',
42 'hostname': 'www.google.com',
43 'port': None,
44 'hostport': 'www.google.com',
45 'path': '/index.html',
46 'userinfo': '',
47 'username': '',
48 'password': '',
49 'params': {},
50 'query': {},
51 'relative': False
52 },
53 "http://www.google.com/index.html;param1=value1" : {
54 'uri': 'http://www.google.com/index.html;param1=value1',
55 'scheme': 'http',
56 'hostname': 'www.google.com',
57 'port': None,
58 'hostport': 'www.google.com',
59 'path': '/index.html',
60 'userinfo': '',
61 'username': '',
62 'password': '',
63 'params': {
64 'param1': 'value1'
65 },
66 'query': {},
67 'relative': False
68 },
69 "http://www.example.org/index.html?param1=value1" : {
70 'uri': 'http://www.example.org/index.html?param1=value1',
71 'scheme': 'http',
72 'hostname': 'www.example.org',
73 'port': None,
74 'hostport': 'www.example.org',
75 'path': '/index.html',
76 'userinfo': '',
77 'username': '',
78 'password': '',
79 'params': {},
80 'query': {
81 'param1': 'value1'
82 },
83 'relative': False
84 },
85 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : {
86 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2',
87 'scheme': 'http',
88 'hostname': 'www.example.org',
89 'port': None,
90 'hostport': 'www.example.org',
91 'path': '/index.html',
92 'userinfo': '',
93 'username': '',
94 'password': '',
95 'params': {
96 'param2': 'value2'
97 },
98 'query': {
99 'qparam1': 'qvalue1'
100 },
101 'relative': False
102 },
103 "http://www.example.com:8080/index.html" : {
104 'uri': 'http://www.example.com:8080/index.html',
105 'scheme': 'http',
106 'hostname': 'www.example.com',
107 'port': 8080,
108 'hostport': 'www.example.com:8080',
109 'path': '/index.html',
110 'userinfo': '',
111 'username': '',
112 'password': '',
113 'params': {},
114 'query': {},
115 'relative': False
116 },
117 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : {
118 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg',
119 'scheme': 'cvs',
120 'hostname': 'cvs.handhelds.org',
121 'port': None,
122 'hostport': 'cvs.handhelds.org',
123 'path': '/cvs',
124 'userinfo': 'anoncvs',
125 'username': 'anoncvs',
126 'password': '',
127 'params': {
128 'module': 'familiar/dist/ipkg'
129 },
130 'query': {},
131 'relative': False
132 },
133 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": {
134 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg',
135 'scheme': 'cvs',
136 'hostname': 'cvs.handhelds.org',
137 'port': None,
138 'hostport': 'cvs.handhelds.org',
139 'path': '/cvs',
140 'userinfo': 'anoncvs:anonymous',
141 'username': 'anoncvs',
142 'password': 'anonymous',
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]143 'params': collections.OrderedDict([
144 ('tag', 'V0-99-81'),
145 ('module', 'familiar/dist/ipkg')
146 ]),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]147 'query': {},
148 'relative': False
149 },
150 "file://example.diff": { # NOTE: Not RFC compliant!
151 'uri': 'file:example.diff',
152 'scheme': 'file',
153 'hostname': '',
154 'port': None,
155 'hostport': '',
156 'path': 'example.diff',
157 'userinfo': '',
158 'username': '',
159 'password': '',
160 'params': {},
161 'query': {},
162 'relative': True
163 },
164 "file:example.diff": { # NOTE: RFC compliant version of the former
165 'uri': 'file:example.diff',
166 'scheme': 'file',
167 'hostname': '',
168 'port': None,
169 'hostport': '',
170 'path': 'example.diff',
171 'userinfo': '',
172 'userinfo': '',
173 'username': '',
174 'password': '',
175 'params': {},
176 'query': {},
177 'relative': True
178 },
179 "file:///tmp/example.diff": {
180 'uri': 'file:///tmp/example.diff',
181 'scheme': 'file',
182 'hostname': '',
183 'port': None,
184 'hostport': '',
185 'path': '/tmp/example.diff',
186 'userinfo': '',
187 'userinfo': '',
188 'username': '',
189 'password': '',
190 'params': {},
191 'query': {},
192 'relative': False
193 },
194 "git:///path/example.git": {
195 'uri': 'git:///path/example.git',
196 'scheme': 'git',
197 'hostname': '',
198 'port': None,
199 'hostport': '',
200 'path': '/path/example.git',
201 'userinfo': '',
202 'userinfo': '',
203 'username': '',
204 'password': '',
205 'params': {},
206 'query': {},
207 'relative': False
208 },
209 "git:path/example.git": {
210 'uri': 'git:path/example.git',
211 'scheme': 'git',
212 'hostname': '',
213 'port': None,
214 'hostport': '',
215 'path': 'path/example.git',
216 'userinfo': '',
217 'userinfo': '',
218 'username': '',
219 'password': '',
220 'params': {},
221 'query': {},
222 'relative': True
223 },
224 "git://example.net/path/example.git": {
225 'uri': 'git://example.net/path/example.git',
226 'scheme': 'git',
227 'hostname': 'example.net',
228 'port': None,
229 'hostport': 'example.net',
230 'path': '/path/example.git',
231 'userinfo': '',
232 'userinfo': '',
233 'username': '',
234 'password': '',
235 'params': {},
236 'query': {},
237 'relative': False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]238 },
239 "http://somesite.net;someparam=1": {
240 'uri': 'http://somesite.net;someparam=1',
241 'scheme': 'http',
242 'hostname': 'somesite.net',
243 'port': None,
244 'hostport': 'somesite.net',
245 'path': '',
246 'userinfo': '',
247 'userinfo': '',
248 'username': '',
249 'password': '',
250 'params': {"someparam" : "1"},
251 'query': {},
252 'relative': False
253 },
254 "file://somelocation;someparam=1": {
255 'uri': 'file:somelocation;someparam=1',
256 'scheme': 'file',
257 'hostname': '',
258 'port': None,
259 'hostport': '',
260 'path': 'somelocation',
261 'userinfo': '',
262 'userinfo': '',
263 'username': '',
264 'password': '',
265 'params': {"someparam" : "1"},
266 'query': {},
267 'relative': True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]268 }
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]269
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]270 }
271
272 def test_uri(self):
273 for test_uri, ref in self.test_uris.items():
274 uri = URI(test_uri)
275
276 self.assertEqual(str(uri), ref['uri'])
277
278 # expected attributes
279 self.assertEqual(uri.scheme, ref['scheme'])
280
281 self.assertEqual(uri.userinfo, ref['userinfo'])
282 self.assertEqual(uri.username, ref['username'])
283 self.assertEqual(uri.password, ref['password'])
284
285 self.assertEqual(uri.hostname, ref['hostname'])
286 self.assertEqual(uri.port, ref['port'])
287 self.assertEqual(uri.hostport, ref['hostport'])
288
289 self.assertEqual(uri.path, ref['path'])
290 self.assertEqual(uri.params, ref['params'])
291
292 self.assertEqual(uri.relative, ref['relative'])
293
294 def test_dict(self):
295 for test in self.test_uris.values():
296 uri = URI()
297
298 self.assertEqual(uri.scheme, '')
299 self.assertEqual(uri.userinfo, '')
300 self.assertEqual(uri.username, '')
301 self.assertEqual(uri.password, '')
302 self.assertEqual(uri.hostname, '')
303 self.assertEqual(uri.port, None)
304 self.assertEqual(uri.path, '')
305 self.assertEqual(uri.params, {})
306
307
308 uri.scheme = test['scheme']
309 self.assertEqual(uri.scheme, test['scheme'])
310
311 uri.userinfo = test['userinfo']
312 self.assertEqual(uri.userinfo, test['userinfo'])
313 self.assertEqual(uri.username, test['username'])
314 self.assertEqual(uri.password, test['password'])
315
316 # make sure changing the values doesn't do anything unexpected
317 uri.username = 'changeme'
318 self.assertEqual(uri.username, 'changeme')
319 self.assertEqual(uri.password, test['password'])
320 uri.password = 'insecure'
321 self.assertEqual(uri.username, 'changeme')
322 self.assertEqual(uri.password, 'insecure')
323
324 # reset back after our trickery
325 uri.userinfo = test['userinfo']
326 self.assertEqual(uri.userinfo, test['userinfo'])
327 self.assertEqual(uri.username, test['username'])
328 self.assertEqual(uri.password, test['password'])
329
330 uri.hostname = test['hostname']
331 self.assertEqual(uri.hostname, test['hostname'])
332 self.assertEqual(uri.hostport, test['hostname'])
333
334 uri.port = test['port']
335 self.assertEqual(uri.port, test['port'])
336 self.assertEqual(uri.hostport, test['hostport'])
337
338 uri.path = test['path']
339 self.assertEqual(uri.path, test['path'])
340
341 uri.params = test['params']
342 self.assertEqual(uri.params, test['params'])
343
344 uri.query = test['query']
345 self.assertEqual(uri.query, test['query'])
346
347 self.assertEqual(str(uri), test['uri'])
348
349 uri.params = {}
350 self.assertEqual(uri.params, {})
351 self.assertEqual(str(uri), (str(uri).split(";"))[0])
352
353class FetcherTest(unittest.TestCase):
354
355 def setUp(self):
356 self.origdir = os.getcwd()
357 self.d = bb.data.init()
358 self.tempdir = tempfile.mkdtemp()
359 self.dldir = os.path.join(self.tempdir, "download")
360 os.mkdir(self.dldir)
361 self.d.setVar("DL_DIR", self.dldir)
362 self.unpackdir = os.path.join(self.tempdir, "unpacked")
363 os.mkdir(self.unpackdir)
364 persistdir = os.path.join(self.tempdir, "persistdata")
365 self.d.setVar("PERSISTENT_DIR", persistdir)
366
367 def tearDown(self):
368 os.chdir(self.origdir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]369 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
370 print("Not cleaning up %s. Please remove manually." % self.tempdir)
371 else:
372 bb.utils.prunedir(self.tempdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]373
374class MirrorUriTest(FetcherTest):
375
376 replaceuris = {
377 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/")
378 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
379 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
380 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
381 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
382 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
383 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http")
384 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
385 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake")
386 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890",
387 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache")
388 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
389 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/")
390 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
391 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3")
392 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
393 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz")
394 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
395 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist")
396 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2",
397 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/")
398 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2",
399 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
400 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
401 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
402 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
403 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http")
404 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800[diff] [blame]405 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org")
406 : "http://somewhere2.org/somefile_1.2.3.tar.gz",
407 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/")
408 : "http://somewhere2.org/somefile_1.2.3.tar.gz",
409 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http")
410 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]411
412 #Renaming files doesn't work
413 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
414 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
415 }
416
417 mirrorvar = "http://.*/.* file:///somepath/downloads/ \n" \
418 "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n" \
419 "https://.*/.* file:///someotherpath/downloads/ \n" \
420 "http://.*/.* file:///someotherpath/downloads/ \n"
421
422 def test_urireplace(self):
423 for k, v in self.replaceuris.items():
424 ud = bb.fetch.FetchData(k[0], self.d)
425 ud.setup_localpath(self.d)
426 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2]))
427 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d)
428 self.assertEqual([v], newuris)
429
430 def test_urilist1(self):
431 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
432 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
433 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
434 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
435
436 def test_urilist2(self):
437 # Catch https:// -> files:// bug
438 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
439 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
440 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
441 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
442
443 def test_mirror_of_mirror(self):
444 # Test if mirror of a mirror works
445 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/ \n"
446 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/ \n"
447 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
448 mirrors = bb.fetch2.mirror_from_string(mirrorvar)
449 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
450 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz',
451 'file:///someotherpath/downloads/bitbake-1.0.tar.gz',
452 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz',
453 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz'])
454
Patrick Williamsd7e96312015-09-22 08:09:05 -0500[diff] [blame]455 recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ \n" \
456 "https://.*/[^/]* https://BBBB/B/B/B/ \n"
457
458 def test_recursive(self):
459 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
460 mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar)
461 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
462 self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz',
463 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz',
464 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]465
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800[diff] [blame]466
467class GitDownloadDirectoryNamingTest(FetcherTest):
468 def setUp(self):
469 super(GitDownloadDirectoryNamingTest, self).setUp()
470 self.recipe_url = "git://git.openembedded.org/bitbake"
471 self.recipe_dir = "git.openembedded.org.bitbake"
472 self.mirror_url = "git://github.com/openembedded/bitbake.git"
473 self.mirror_dir = "github.com.openembedded.bitbake.git"
474
475 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
476
477 def setup_mirror_rewrite(self):
478 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
479
480 @skipIfNoNetwork()
481 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self):
482 self.setup_mirror_rewrite()
483 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
484
485 fetcher.download()
486
487 dir = os.listdir(self.dldir + "/git2")
488 self.assertIn(self.recipe_dir, dir)
489
490 @skipIfNoNetwork()
491 def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self):
492 self.setup_mirror_rewrite()
493 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
494
495 fetcher.download()
496
497 dir = os.listdir(self.dldir + "/git2")
498 self.assertIn(self.mirror_dir, dir)
499 self.assertIn(self.recipe_dir, dir)
500
501 @skipIfNoNetwork()
502 def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self):
503 self.setup_mirror_rewrite()
504 fetcher = bb.fetch.Fetch([self.mirror_url], self.d)
505 fetcher.download()
506 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
507
508 fetcher.download()
509
510 dir = os.listdir(self.dldir + "/git2")
511 self.assertIn(self.mirror_dir, dir)
512 self.assertIn(self.recipe_dir, dir)
513
514
515class TarballNamingTest(FetcherTest):
516 def setUp(self):
517 super(TarballNamingTest, self).setUp()
518 self.recipe_url = "git://git.openembedded.org/bitbake"
519 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
520 self.mirror_url = "git://github.com/openembedded/bitbake.git"
521 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz"
522
523 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
524 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
525
526 def setup_mirror_rewrite(self):
527 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
528
529 @skipIfNoNetwork()
530 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self):
531 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
532
533 fetcher.download()
534
535 dir = os.listdir(self.dldir)
536 self.assertIn(self.recipe_tarball, dir)
537
538 @skipIfNoNetwork()
539 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
540 self.setup_mirror_rewrite()
541 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
542
543 fetcher.download()
544
545 dir = os.listdir(self.dldir)
546 self.assertIn(self.mirror_tarball, dir)
547
548
549class GitShallowTarballNamingTest(FetcherTest):
550 def setUp(self):
551 super(GitShallowTarballNamingTest, self).setUp()
552 self.recipe_url = "git://git.openembedded.org/bitbake"
553 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz"
554 self.mirror_url = "git://github.com/openembedded/bitbake.git"
555 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz"
556
557 self.d.setVar('BB_GIT_SHALLOW', '1')
558 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
559 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
560
561 def setup_mirror_rewrite(self):
562 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
563
564 @skipIfNoNetwork()
565 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self):
566 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
567
568 fetcher.download()
569
570 dir = os.listdir(self.dldir)
571 self.assertIn(self.recipe_tarball, dir)
572
573 @skipIfNoNetwork()
574 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self):
575 self.setup_mirror_rewrite()
576 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
577
578 fetcher.download()
579
580 dir = os.listdir(self.dldir)
581 self.assertIn(self.mirror_tarball, dir)
582
583
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]584class FetcherLocalTest(FetcherTest):
585 def setUp(self):
586 def touch(fn):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]587 with open(fn, 'a'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]588 os.utime(fn, None)
589
590 super(FetcherLocalTest, self).setUp()
591 self.localsrcdir = os.path.join(self.tempdir, 'localsrc')
592 os.makedirs(self.localsrcdir)
593 touch(os.path.join(self.localsrcdir, 'a'))
594 touch(os.path.join(self.localsrcdir, 'b'))
595 os.makedirs(os.path.join(self.localsrcdir, 'dir'))
596 touch(os.path.join(self.localsrcdir, 'dir', 'c'))
597 touch(os.path.join(self.localsrcdir, 'dir', 'd'))
598 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir'))
599 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e'))
600 self.d.setVar("FILESPATH", self.localsrcdir)
601
602 def fetchUnpack(self, uris):
603 fetcher = bb.fetch.Fetch(uris, self.d)
604 fetcher.download()
605 fetcher.unpack(self.unpackdir)
606 flst = []
607 for root, dirs, files in os.walk(self.unpackdir):
608 for f in files:
609 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir))
610 flst.sort()
611 return flst
612
613 def test_local(self):
614 tree = self.fetchUnpack(['file://a', 'file://dir/c'])
615 self.assertEqual(tree, ['a', 'dir/c'])
616
617 def test_local_wildcard(self):
618 tree = self.fetchUnpack(['file://a', 'file://dir/*'])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]619 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]620
621 def test_local_dir(self):
622 tree = self.fetchUnpack(['file://a', 'file://dir'])
623 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e'])
624
625 def test_local_subdir(self):
626 tree = self.fetchUnpack(['file://dir/subdir'])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]627 self.assertEqual(tree, ['dir/subdir/e'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]628
629 def test_local_subdir_file(self):
630 tree = self.fetchUnpack(['file://dir/subdir/e'])
631 self.assertEqual(tree, ['dir/subdir/e'])
632
633 def test_local_subdirparam(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]634 tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo'])
635 self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]636
637 def test_local_deepsubdirparam(self):
638 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar'])
639 self.assertEqual(tree, ['bar/dir/subdir/e'])
640
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]641 def test_local_absolutedir(self):
642 # Unpacking to an absolute path that is a subdirectory of the root
643 # should work
644 tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')])
645
646 # Unpacking to an absolute path outside of the root should fail
647 with self.assertRaises(bb.fetch2.UnpackError):
648 self.fetchUnpack(['file://a;subdir=/bin/sh'])
649
Brad Bishop316dfdd2018-06-25 12:45:53 -0400[diff] [blame]650class FetcherNoNetworkTest(FetcherTest):
651 def setUp(self):
652 super().setUp()
653 # all test cases are based on not having network
654 self.d.setVar("BB_NO_NETWORK", "1")
655
656 def test_missing(self):
657 string = "this is a test file\n".encode("utf-8")
658 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
659 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
660
661 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
662 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
663 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
664 with self.assertRaises(bb.fetch2.NetworkAccess):
665 fetcher.download()
666
667 def test_valid_missing_donestamp(self):
668 # create the file in the download directory with correct hash
669 string = "this is a test file\n".encode("utf-8")
670 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f:
671 f.write(string)
672
673 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
674 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
675
676 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
677 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
678 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
679 fetcher.download()
680 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
681
682 def test_invalid_missing_donestamp(self):
683 # create an invalid file in the download directory with incorrect hash
684 string = "this is a test file\n".encode("utf-8")
685 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
686 pass
687
688 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
689 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
690
691 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
692 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
693 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
694 with self.assertRaises(bb.fetch2.NetworkAccess):
695 fetcher.download()
696 # the existing file should not exist or should have be moved to "bad-checksum"
697 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
698
699 def test_nochecksums_missing(self):
700 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
701 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
702 # ssh fetch does not support checksums
703 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
704 # attempts to download with missing donestamp
705 with self.assertRaises(bb.fetch2.NetworkAccess):
706 fetcher.download()
707
708 def test_nochecksums_missing_donestamp(self):
709 # create a file in the download directory
710 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
711 pass
712
713 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
714 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
715 # ssh fetch does not support checksums
716 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
717 # attempts to download with missing donestamp
718 with self.assertRaises(bb.fetch2.NetworkAccess):
719 fetcher.download()
720
721 def test_nochecksums_has_donestamp(self):
722 # create a file in the download directory with the donestamp
723 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
724 pass
725 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
726 pass
727
728 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
729 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
730 # ssh fetch does not support checksums
731 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
732 # should not fetch
733 fetcher.download()
734 # both files should still exist
735 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
736 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
737
738 def test_nochecksums_missing_has_donestamp(self):
739 # create a file in the download directory with the donestamp
740 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
741 pass
742
743 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
744 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
745 # ssh fetch does not support checksums
746 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
747 with self.assertRaises(bb.fetch2.NetworkAccess):
748 fetcher.download()
749 # both files should still exist
750 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
751 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
752
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]753class FetcherNetworkTest(FetcherTest):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]754 @skipIfNoNetwork()
755 def test_fetch(self):
756 fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
757 fetcher.download()
758 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
759 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892)
760 self.d.setVar("BB_NO_NETWORK", "1")
761 fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
762 fetcher.download()
763 fetcher.unpack(self.unpackdir)
764 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9)
765 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]766
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]767 @skipIfNoNetwork()
768 def test_fetch_mirror(self):
769 self.d.setVar("MIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
770 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
771 fetcher.download()
772 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
773
774 @skipIfNoNetwork()
775 def test_fetch_mirror_of_mirror(self):
776 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ \n http://invalid2.yoctoproject.org/.* http://downloads.yoctoproject.org/releases/bitbake")
777 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
778 fetcher.download()
779 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
780
781 @skipIfNoNetwork()
782 def test_fetch_file_mirror_of_mirror(self):
783 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ \n file:///some1where/.* file://some2where/ \n file://some2where/.* http://downloads.yoctoproject.org/releases/bitbake")
784 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
785 os.mkdir(self.dldir + "/some2where")
786 fetcher.download()
787 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
788
789 @skipIfNoNetwork()
790 def test_fetch_premirror(self):
791 self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
792 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
793 fetcher.download()
794 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
795
796 @skipIfNoNetwork()
797 def gitfetcher(self, url1, url2):
798 def checkrevision(self, fetcher):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]799 fetcher.unpack(self.unpackdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]800 revision = bb.process.run("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git")[0].strip()
801 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]802
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]803 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
804 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
805 fetcher = bb.fetch.Fetch([url1], self.d)
806 fetcher.download()
807 checkrevision(self, fetcher)
808 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works
809 bb.utils.prunedir(self.dldir + "/git2/")
810 bb.utils.prunedir(self.unpackdir)
811 self.d.setVar("BB_NO_NETWORK", "1")
812 fetcher = bb.fetch.Fetch([url2], self.d)
813 fetcher.download()
814 checkrevision(self, fetcher)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]815
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]816 @skipIfNoNetwork()
817 def test_gitfetch(self):
818 url1 = url2 = "git://git.openembedded.org/bitbake"
819 self.gitfetcher(url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]820
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]821 @skipIfNoNetwork()
822 def test_gitfetch_goodsrcrev(self):
823 # SRCREV is set but matches rev= parameter
824 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
825 self.gitfetcher(url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]826
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]827 @skipIfNoNetwork()
828 def test_gitfetch_badsrcrev(self):
829 # SRCREV is set but does not match rev= parameter
830 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5"
831 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]832
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]833 @skipIfNoNetwork()
834 def test_gitfetch_tagandrev(self):
835 # SRCREV is set but does not match rev= parameter
836 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
837 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]838
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]839 @skipIfNoNetwork()
840 def test_gitfetch_localusehead(self):
841 # Create dummy local Git repo
842 src_dir = tempfile.mkdtemp(dir=self.tempdir,
843 prefix='gitfetch_localusehead_')
844 src_dir = os.path.abspath(src_dir)
845 bb.process.run("git init", cwd=src_dir)
846 bb.process.run("git commit --allow-empty -m'Dummy commit'",
847 cwd=src_dir)
848 # Use other branch than master
849 bb.process.run("git checkout -b my-devel", cwd=src_dir)
850 bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
851 cwd=src_dir)
852 stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
853 orig_rev = stdout[0].strip()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]854
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]855 # Fetch and check revision
856 self.d.setVar("SRCREV", "AUTOINC")
857 url = "git://" + src_dir + ";protocol=file;usehead=1"
858 fetcher = bb.fetch.Fetch([url], self.d)
859 fetcher.download()
860 fetcher.unpack(self.unpackdir)
861 stdout = bb.process.run("git rev-parse HEAD",
862 cwd=os.path.join(self.unpackdir, 'git'))
863 unpack_rev = stdout[0].strip()
864 self.assertEqual(orig_rev, unpack_rev)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]865
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]866 @skipIfNoNetwork()
867 def test_gitfetch_remoteusehead(self):
868 url = "git://git.openembedded.org/bitbake;usehead=1"
869 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]870
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]871 @skipIfNoNetwork()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800[diff] [blame]872 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self):
873 recipeurl = "git://git.openembedded.org/bitbake"
874 mirrorurl = "git://someserver.org/bitbake"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]875 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800[diff] [blame]876 self.gitfetcher(recipeurl, mirrorurl)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]877
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]878 @skipIfNoNetwork()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800[diff] [blame]879 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self):
880 recipeurl = "git://someserver.org/bitbake"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]881 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800[diff] [blame]882 self.gitfetcher(recipeurl, recipeurl)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]883
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]884 @skipIfNoNetwork()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800[diff] [blame]885 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]886 realurl = "git://git.openembedded.org/bitbake"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800[diff] [blame]887 recipeurl = "git://someserver.org/bitbake"
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]888 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git")
889 os.chdir(self.tempdir)
890 bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800[diff] [blame]891 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (recipeurl, self.sourcedir))
892 self.gitfetcher(recipeurl, recipeurl)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]893
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]894 @skipIfNoNetwork()
895 def test_git_submodule(self):
Brad Bishopf8caae32019-03-25 13:13:56 -0400[diff] [blame]896 # URL with ssh submodules
897 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7"
898 # Original URL (comment this if you have ssh access to git.yoctoproject.org)
899 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee"
900 fetcher = bb.fetch.Fetch([url], self.d)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]901 fetcher.download()
902 # Previous cwd has been deleted
903 os.chdir(os.path.dirname(self.unpackdir))
904 fetcher.unpack(self.unpackdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]905
Brad Bishopf8caae32019-03-25 13:13:56 -0400[diff] [blame]906 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
907 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing')
908 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing')
909 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present')
910
911 # Only when we're running the extended test with a submodule's submodule, can we check this.
912 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
913 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
914
915 def test_git_submodule_dbus_broker(self):
916 # The following external repositories have show failures in fetch and unpack operations
917 # We want to avoid regressions!
918 url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2"
919 fetcher = bb.fetch.Fetch([url], self.d)
920 fetcher.download()
921 # Previous cwd has been deleted
922 os.chdir(os.path.dirname(self.unpackdir))
923 fetcher.unpack(self.unpackdir)
924
925 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
926 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"')
927 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"')
928 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"')
929 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"')
930 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"')
931
932 def test_git_submodule_CLI11(self):
933 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
934 fetcher = bb.fetch.Fetch([url], self.d)
935 fetcher.download()
936 # Previous cwd has been deleted
937 os.chdir(os.path.dirname(self.unpackdir))
938 fetcher.unpack(self.unpackdir)
939
940 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
941 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
942 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
943 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
944
Brad Bishop19323692019-04-05 15:28:33 -0400[diff] [blame]945 def test_git_submodule_update_CLI11(self):
946 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
947 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714"
948 fetcher = bb.fetch.Fetch([url], self.d)
949 fetcher.download()
950
951 # CLI11 that pulls in a newer nlohmann-json
952 url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca"
953 fetcher = bb.fetch.Fetch([url], self.d)
954 fetcher.download()
955 # Previous cwd has been deleted
956 os.chdir(os.path.dirname(self.unpackdir))
957 fetcher.unpack(self.unpackdir)
958
959 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
960 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"')
961 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"')
962 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"')
963
Brad Bishopf8caae32019-03-25 13:13:56 -0400[diff] [blame]964 def test_git_submodule_aktualizr(self):
965 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
966 fetcher = bb.fetch.Fetch([url], self.d)
967 fetcher.download()
968 # Previous cwd has been deleted
969 os.chdir(os.path.dirname(self.unpackdir))
970 fetcher.unpack(self.unpackdir)
971
972 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
973 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"')
974 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"')
975 self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly")
976 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"')
977 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"')
978 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]979
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]980class TrustedNetworksTest(FetcherTest):
981 def test_trusted_network(self):
982 # Ensure trusted_network returns False when the host IS in the list.
983 url = "git://Someserver.org/foo;rev=1"
984 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org")
985 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]986
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]987 def test_wild_trusted_network(self):
988 # Ensure trusted_network returns true when the *.host IS in the list.
989 url = "git://Someserver.org/foo;rev=1"
990 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
991 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]992
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]993 def test_prefix_wild_trusted_network(self):
994 # Ensure trusted_network returns true when the prefix matches *.host.
995 url = "git://git.Someserver.org/foo;rev=1"
996 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
997 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]998
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]999 def test_two_prefix_wild_trusted_network(self):
1000 # Ensure trusted_network returns true when the prefix matches *.host.
1001 url = "git://something.git.Someserver.org/foo;rev=1"
1002 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
1003 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1004
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]1005 def test_port_trusted_network(self):
1006 # Ensure trusted_network returns True, even if the url specifies a port.
1007 url = "git://someserver.org:8080/foo;rev=1"
1008 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org")
1009 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1010
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]1011 def test_untrusted_network(self):
1012 # Ensure trusted_network returns False when the host is NOT in the list.
1013 url = "git://someserver.org/foo;rev=1"
1014 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
1015 self.assertFalse(bb.fetch.trusted_network(self.d, url))
1016
1017 def test_wild_untrusted_network(self):
1018 # Ensure trusted_network returns False when the host is NOT in the list.
1019 url = "git://*.someserver.org/foo;rev=1"
1020 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
1021 self.assertFalse(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1022
1023class URLHandle(unittest.TestCase):
1024
1025 datatable = {
1026 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}),
1027 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}),
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600[diff] [blame]1028 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]1029 "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'}),
1030 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1031 }
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]1032 # we require a pathname to encodeurl but users can still pass such urls to
1033 # decodeurl and we need to handle them
1034 decodedata = datatable.copy()
1035 decodedata.update({
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800[diff] [blame]1036 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]1037 })
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1038
1039 def test_decodeurl(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]1040 for k, v in self.decodedata.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1041 result = bb.fetch.decodeurl(k)
1042 self.assertEqual(result, v)
1043
1044 def test_encodeurl(self):
1045 for k, v in self.datatable.items():
1046 result = bb.fetch.encodeurl(v)
1047 self.assertEqual(result, k)
1048
1049class FetchLatestVersionTest(FetcherTest):
1050
1051 test_git_uris = {
1052 # version pattern "X.Y.Z"
1053 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
1054 : "1.99.4",
1055 # version pattern "vX.Y"
1056 ("mtd-utils", "git://git.infradead.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "")
1057 : "1.5.0",
1058 # version pattern "pkg_name-X.Y"
1059 ("presentproto", "git://anongit.freedesktop.org/git/xorg/proto/presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")
1060 : "1.0",
1061 # version pattern "pkg_name-vX.Y.Z"
1062 ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
1063 : "1.4.0",
1064 # combination version pattern
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800[diff] [blame]1065 ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1066 : "1.2.0",
1067 ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "")
1068 : "2014.01",
1069 # version pattern "yyyymmdd"
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800[diff] [blame]1070 ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1071 : "20120614",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]1072 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1073 ("xf86-video-omap", "git://anongit.freedesktop.org/xorg/driver/xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))")
1074 : "0.4.3",
1075 ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", "(?P<pver>(([0-9][\.|_]?)+[0-9]))")
1076 : "11.0.0",
1077 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", "chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
1078 : "1.3.59",
1079 ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", "(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
1080 : "3.82+dbg0.9",
1081 }
1082
1083 test_wget_uris = {
1084 # packages with versions inside directory name
1085 ("util-linux", "http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "")
1086 : "2.24.2",
1087 ("enchant", "http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "")
1088 : "1.6.0",
1089 ("cmake", "http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz", "", "")
1090 : "2.8.12.1",
1091 # packages with versions only in current directory
1092 ("eglic", "http://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "")
1093 : "2.19",
1094 ("gnu-config", "http://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "")
1095 : "20120814",
1096 # packages with "99" in the name of possible version
1097 ("pulseaudio", "http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "")
1098 : "5.0",
1099 ("xserver-xorg", "http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "")
1100 : "1.15.1",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]1101 # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX
1102 ("cups", "http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2", "https://github.com/apple/cups/releases", "(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1103 : "2.0.0",
1104 ("db", "http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz", "http://www.oracle.com/technetwork/products/berkeleydb/downloads/index-082944.html", "http://download.oracle.com/otn/berkeley-db/(?P<name>db-)(?P<pver>((\d+[\.\-_]*)+))\.tar\.gz")
1105 : "6.1.19",
1106 }
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1107
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1108 @skipIfNoNetwork()
1109 def test_git_latest_versionstring(self):
1110 for k, v in self.test_git_uris.items():
1111 self.d.setVar("PN", k[0])
1112 self.d.setVar("SRCREV", k[2])
1113 self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3])
1114 ud = bb.fetch2.FetchData(k[1], self.d)
1115 pupver= ud.method.latest_versionstring(ud, self.d)
1116 verstring = pupver[0]
Brad Bishop316dfdd2018-06-25 12:45:53 -0400[diff] [blame]1117 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1118 r = bb.utils.vercmp_string(v, verstring)
1119 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
1120
1121 @skipIfNoNetwork()
1122 def test_wget_latest_versionstring(self):
1123 for k, v in self.test_wget_uris.items():
1124 self.d.setVar("PN", k[0])
1125 self.d.setVar("UPSTREAM_CHECK_URI", k[2])
1126 self.d.setVar("UPSTREAM_CHECK_REGEX", k[3])
1127 ud = bb.fetch2.FetchData(k[1], self.d)
1128 pupver = ud.method.latest_versionstring(ud, self.d)
1129 verstring = pupver[0]
Brad Bishop316dfdd2018-06-25 12:45:53 -0400[diff] [blame]1130 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1131 r = bb.utils.vercmp_string(v, verstring)
1132 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1133
1134
1135class FetchCheckStatusTest(FetcherTest):
1136 test_wget_uris = ["http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2",
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1137 "http://www.cups.org/",
1138 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
1139 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
1140 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
1141 "https://yoctoproject.org/",
1142 "https://yoctoproject.org/documentation",
1143 "http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
1144 "http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1145 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz",
1146 "http://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz",
1147 "https://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz",
1148 "https://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500[diff] [blame]1149 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD
1150 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1151 ]
1152
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1153 @skipIfNoNetwork()
1154 def test_wget_checkstatus(self):
1155 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d)
1156 for u in self.test_wget_uris:
1157 with self.subTest(url=u):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1158 ud = fetch.ud[u]
1159 m = ud.method
1160 ret = m.checkstatus(fetch, ud, self.d)
1161 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
1162
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1163 @skipIfNoNetwork()
1164 def test_wget_checkstatus_connection_cache(self):
1165 from bb.fetch2 import FetchConnectionCache
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1166
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1167 connection_cache = FetchConnectionCache()
1168 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d,
1169 connection_cache = connection_cache)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1170
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1171 for u in self.test_wget_uris:
1172 with self.subTest(url=u):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame]1173 ud = fetch.ud[u]
1174 m = ud.method
1175 ret = m.checkstatus(fetch, ud, self.d)
1176 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
1177
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1178 connection_cache.close_connections()
1179
1180
1181class GitMakeShallowTest(FetcherTest):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1182 def setUp(self):
1183 FetcherTest.setUp(self)
1184 self.gitdir = os.path.join(self.tempdir, 'gitshallow')
1185 bb.utils.mkdirhier(self.gitdir)
1186 bb.process.run('git init', cwd=self.gitdir)
1187
1188 def assertRefs(self, expected_refs):
1189 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
1190 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines()
1191 self.assertEqual(sorted(full_expected), sorted(actual_refs))
1192
1193 def assertRevCount(self, expected_count, args=None):
1194 if args is None:
1195 args = ['HEAD']
1196 revs = self.git(['rev-list'] + args)
1197 actual_count = len(revs.splitlines())
1198 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1199
1200 def git(self, cmd):
1201 if isinstance(cmd, str):
1202 cmd = 'git ' + cmd
1203 else:
1204 cmd = ['git'] + cmd
1205 return bb.process.run(cmd, cwd=self.gitdir)[0]
1206
1207 def make_shallow(self, args=None):
1208 if args is None:
1209 args = ['HEAD']
Brad Bishop316dfdd2018-06-25 12:45:53 -0400[diff] [blame]1210 return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1211
1212 def add_empty_file(self, path, msg=None):
1213 if msg is None:
1214 msg = path
1215 open(os.path.join(self.gitdir, path), 'w').close()
1216 self.git(['add', path])
1217 self.git(['commit', '-m', msg, path])
1218
1219 def test_make_shallow_single_branch_no_merge(self):
1220 self.add_empty_file('a')
1221 self.add_empty_file('b')
1222 self.assertRevCount(2)
1223 self.make_shallow()
1224 self.assertRevCount(1)
1225
1226 def test_make_shallow_single_branch_one_merge(self):
1227 self.add_empty_file('a')
1228 self.add_empty_file('b')
1229 self.git('checkout -b a_branch')
1230 self.add_empty_file('c')
1231 self.git('checkout master')
1232 self.add_empty_file('d')
1233 self.git('merge --no-ff --no-edit a_branch')
1234 self.git('branch -d a_branch')
1235 self.add_empty_file('e')
1236 self.assertRevCount(6)
1237 self.make_shallow(['HEAD~2'])
1238 self.assertRevCount(5)
1239
1240 def test_make_shallow_at_merge(self):
1241 self.add_empty_file('a')
1242 self.git('checkout -b a_branch')
1243 self.add_empty_file('b')
1244 self.git('checkout master')
1245 self.git('merge --no-ff --no-edit a_branch')
1246 self.git('branch -d a_branch')
1247 self.assertRevCount(3)
1248 self.make_shallow()
1249 self.assertRevCount(1)
1250
1251 def test_make_shallow_annotated_tag(self):
1252 self.add_empty_file('a')
1253 self.add_empty_file('b')
1254 self.git('tag -a -m a_tag a_tag')
1255 self.assertRevCount(2)
1256 self.make_shallow(['a_tag'])
1257 self.assertRevCount(1)
1258
1259 def test_make_shallow_multi_ref(self):
1260 self.add_empty_file('a')
1261 self.add_empty_file('b')
1262 self.git('checkout -b a_branch')
1263 self.add_empty_file('c')
1264 self.git('checkout master')
1265 self.add_empty_file('d')
1266 self.git('checkout -b a_branch_2')
1267 self.add_empty_file('a_tag')
1268 self.git('tag a_tag')
1269 self.git('checkout master')
1270 self.git('branch -D a_branch_2')
1271 self.add_empty_file('e')
1272 self.assertRevCount(6, ['--all'])
1273 self.make_shallow()
1274 self.assertRevCount(5, ['--all'])
1275
1276 def test_make_shallow_multi_ref_trim(self):
1277 self.add_empty_file('a')
1278 self.git('checkout -b a_branch')
1279 self.add_empty_file('c')
1280 self.git('checkout master')
1281 self.assertRevCount(1)
1282 self.assertRevCount(2, ['--all'])
1283 self.assertRefs(['master', 'a_branch'])
1284 self.make_shallow(['-r', 'master', 'HEAD'])
1285 self.assertRevCount(1, ['--all'])
1286 self.assertRefs(['master'])
1287
1288 def test_make_shallow_noop(self):
1289 self.add_empty_file('a')
1290 self.assertRevCount(1)
1291 self.make_shallow()
1292 self.assertRevCount(1)
1293
1294 @skipIfNoNetwork()
1295 def test_make_shallow_bitbake(self):
1296 self.git('remote add origin https://github.com/openembedded/bitbake')
1297 self.git('fetch --tags origin')
1298 orig_revs = len(self.git('rev-list --all').splitlines())
1299 self.make_shallow(['refs/tags/1.10.0'])
1300 self.assertRevCount(orig_revs - 1746, ['--all'])
1301
1302class GitShallowTest(FetcherTest):
1303 def setUp(self):
1304 FetcherTest.setUp(self)
1305 self.gitdir = os.path.join(self.tempdir, 'git')
1306 self.srcdir = os.path.join(self.tempdir, 'gitsource')
1307
1308 bb.utils.mkdirhier(self.srcdir)
1309 self.git('init', cwd=self.srcdir)
1310 self.d.setVar('WORKDIR', self.tempdir)
1311 self.d.setVar('S', self.gitdir)
1312 self.d.delVar('PREMIRRORS')
1313 self.d.delVar('MIRRORS')
1314
1315 uri = 'git://%s;protocol=file;subdir=${S}' % self.srcdir
1316 self.d.setVar('SRC_URI', uri)
1317 self.d.setVar('SRCREV', '${AUTOREV}')
1318 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
1319
1320 self.d.setVar('BB_GIT_SHALLOW', '1')
1321 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
1322 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
1323
1324 def assertRefs(self, expected_refs, cwd=None):
1325 if cwd is None:
1326 cwd = self.gitdir
1327 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines()
1328 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines()
1329 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs)))
1330
1331 def assertRevCount(self, expected_count, args=None, cwd=None):
1332 if args is None:
1333 args = ['HEAD']
1334 if cwd is None:
1335 cwd = self.gitdir
1336 revs = self.git(['rev-list'] + args, cwd=cwd)
1337 actual_count = len(revs.splitlines())
1338 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1339
1340 def git(self, cmd, cwd=None):
1341 if isinstance(cmd, str):
1342 cmd = 'git ' + cmd
1343 else:
1344 cmd = ['git'] + cmd
1345 if cwd is None:
1346 cwd = self.gitdir
1347 return bb.process.run(cmd, cwd=cwd)[0]
1348
1349 def add_empty_file(self, path, cwd=None, msg=None):
1350 if msg is None:
1351 msg = path
1352 if cwd is None:
1353 cwd = self.srcdir
1354 open(os.path.join(cwd, path), 'w').close()
1355 self.git(['add', path], cwd)
1356 self.git(['commit', '-m', msg, path], cwd)
1357
1358 def fetch(self, uri=None):
1359 if uri is None:
Brad Bishop19323692019-04-05 15:28:33 -0400[diff] [blame]1360 uris = self.d.getVar('SRC_URI').split()
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1361 uri = uris[0]
1362 d = self.d
1363 else:
1364 d = self.d.createCopy()
1365 d.setVar('SRC_URI', uri)
1366 uri = d.expand(uri)
1367 uris = [uri]
1368
1369 fetcher = bb.fetch2.Fetch(uris, d)
1370 fetcher.download()
1371 ud = fetcher.ud[uri]
1372 return fetcher, ud
1373
1374 def fetch_and_unpack(self, uri=None):
1375 fetcher, ud = self.fetch(uri)
1376 fetcher.unpack(self.d.getVar('WORKDIR'))
1377 assert os.path.exists(self.d.getVar('S'))
1378 return fetcher, ud
1379
1380 def fetch_shallow(self, uri=None, disabled=False, keepclone=False):
1381 """Fetch a uri, generating a shallow tarball, then unpack using it"""
1382 fetcher, ud = self.fetch_and_unpack(uri)
1383 assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri)
1384
1385 # Confirm that the unpacked repo is unshallow
1386 if not disabled:
1387 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1388
1389 # fetch and unpack, from the shallow tarball
1390 bb.utils.remove(self.gitdir, recurse=True)
1391 bb.utils.remove(ud.clonedir, recurse=True)
Brad Bishopf8caae32019-03-25 13:13:56 -0400[diff] [blame]1392 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1393
1394 # confirm that the unpacked repo is used when no git clone or git
1395 # mirror tarball is available
1396 fetcher, ud = self.fetch_and_unpack(uri)
1397 if not disabled:
1398 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir
1399 else:
1400 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir
1401 return fetcher, ud
1402
1403 def test_shallow_disabled(self):
1404 self.add_empty_file('a')
1405 self.add_empty_file('b')
1406 self.assertRevCount(2, cwd=self.srcdir)
1407
1408 self.d.setVar('BB_GIT_SHALLOW', '0')
1409 self.fetch_shallow(disabled=True)
1410 self.assertRevCount(2)
1411
1412 def test_shallow_nobranch(self):
1413 self.add_empty_file('a')
1414 self.add_empty_file('b')
1415 self.assertRevCount(2, cwd=self.srcdir)
1416
1417 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip()
1418 self.d.setVar('SRCREV', srcrev)
Brad Bishop19323692019-04-05 15:28:33 -0400[diff] [blame]1419 uri = self.d.getVar('SRC_URI').split()[0]
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1420 uri = '%s;nobranch=1;bare=1' % uri
1421
1422 self.fetch_shallow(uri)
1423 self.assertRevCount(1)
1424
1425 # shallow refs are used to ensure the srcrev sticks around when we
1426 # have no other branches referencing it
1427 self.assertRefs(['refs/shallow/default'])
1428
1429 def test_shallow_default_depth_1(self):
1430 # Create initial git repo
1431 self.add_empty_file('a')
1432 self.add_empty_file('b')
1433 self.assertRevCount(2, cwd=self.srcdir)
1434
1435 self.fetch_shallow()
1436 self.assertRevCount(1)
1437
1438 def test_shallow_depth_0_disables(self):
1439 self.add_empty_file('a')
1440 self.add_empty_file('b')
1441 self.assertRevCount(2, cwd=self.srcdir)
1442
1443 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1444 self.fetch_shallow(disabled=True)
1445 self.assertRevCount(2)
1446
1447 def test_shallow_depth_default_override(self):
1448 self.add_empty_file('a')
1449 self.add_empty_file('b')
1450 self.assertRevCount(2, cwd=self.srcdir)
1451
1452 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2')
1453 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1')
1454 self.fetch_shallow()
1455 self.assertRevCount(1)
1456
1457 def test_shallow_depth_default_override_disable(self):
1458 self.add_empty_file('a')
1459 self.add_empty_file('b')
1460 self.add_empty_file('c')
1461 self.assertRevCount(3, cwd=self.srcdir)
1462
1463 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1464 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2')
1465 self.fetch_shallow()
1466 self.assertRevCount(2)
1467
1468 def test_current_shallow_out_of_date_clone(self):
1469 # Create initial git repo
1470 self.add_empty_file('a')
1471 self.add_empty_file('b')
1472 self.add_empty_file('c')
1473 self.assertRevCount(3, cwd=self.srcdir)
1474
1475 # Clone and generate mirror tarball
1476 fetcher, ud = self.fetch()
1477
1478 # Ensure we have a current mirror tarball, but an out of date clone
1479 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir)
1480 self.assertRevCount(2, cwd=ud.clonedir)
1481
1482 # Fetch and unpack, from the current tarball, not the out of date clone
1483 bb.utils.remove(self.gitdir, recurse=True)
1484 fetcher, ud = self.fetch()
1485 fetcher.unpack(self.d.getVar('WORKDIR'))
1486 self.assertRevCount(1)
1487
1488 def test_shallow_single_branch_no_merge(self):
1489 self.add_empty_file('a')
1490 self.add_empty_file('b')
1491 self.assertRevCount(2, cwd=self.srcdir)
1492
1493 self.fetch_shallow()
1494 self.assertRevCount(1)
1495 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1496 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1497
1498 def test_shallow_no_dangling(self):
1499 self.add_empty_file('a')
1500 self.add_empty_file('b')
1501 self.assertRevCount(2, cwd=self.srcdir)
1502
1503 self.fetch_shallow()
1504 self.assertRevCount(1)
1505 assert not self.git('fsck --dangling')
1506
1507 def test_shallow_srcrev_branch_truncation(self):
1508 self.add_empty_file('a')
1509 self.add_empty_file('b')
1510 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip()
1511 self.add_empty_file('c')
1512 self.assertRevCount(3, cwd=self.srcdir)
1513
1514 self.d.setVar('SRCREV', b_commit)
1515 self.fetch_shallow()
1516
1517 # The 'c' commit was removed entirely, and 'a' was removed from history
1518 self.assertRevCount(1, ['--all'])
1519 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit)
1520 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1521 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1522 assert not os.path.exists(os.path.join(self.gitdir, 'c'))
1523
1524 def test_shallow_ref_pruning(self):
1525 self.add_empty_file('a')
1526 self.add_empty_file('b')
1527 self.git('branch a_branch', cwd=self.srcdir)
1528 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
1529 self.assertRevCount(2, cwd=self.srcdir)
1530
1531 self.fetch_shallow()
1532
1533 self.assertRefs(['master', 'origin/master'])
1534 self.assertRevCount(1)
1535
1536 def test_shallow_submodules(self):
1537 self.add_empty_file('a')
1538 self.add_empty_file('b')
1539
1540 smdir = os.path.join(self.tempdir, 'gitsubmodule')
1541 bb.utils.mkdirhier(smdir)
1542 self.git('init', cwd=smdir)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800[diff] [blame]1543 # Make this look like it was cloned from a remote...
1544 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
1545 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1546 self.add_empty_file('asub', cwd=smdir)
Brad Bishopf8caae32019-03-25 13:13:56 -0400[diff] [blame]1547 self.add_empty_file('bsub', cwd=smdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1548
1549 self.git('submodule init', cwd=self.srcdir)
1550 self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
1551 self.git('submodule update', cwd=self.srcdir)
1552 self.git('commit -m submodule -a', cwd=self.srcdir)
1553
1554 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
1555 fetcher, ud = self.fetch_shallow(uri)
1556
Brad Bishopf8caae32019-03-25 13:13:56 -0400[diff] [blame]1557 # Verify the main repository is shallow
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1558 self.assertRevCount(1)
Brad Bishopf8caae32019-03-25 13:13:56 -0400[diff] [blame]1559
1560 # Verify the gitsubmodule directory is present
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1561 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
1562
Brad Bishopf8caae32019-03-25 13:13:56 -0400[diff] [blame]1563 # Verify the submodule is also shallow
1564 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
1565
1566
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1567 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')):
1568 def test_shallow_annex(self):
1569 self.add_empty_file('a')
1570 self.add_empty_file('b')
1571 self.git('annex init', cwd=self.srcdir)
1572 open(os.path.join(self.srcdir, 'c'), 'w').close()
1573 self.git('annex add c', cwd=self.srcdir)
1574 self.git('commit -m annex-c -a', cwd=self.srcdir)
1575 bb.process.run('chmod u+w -R %s' % os.path.join(self.srcdir, '.git', 'annex'))
1576
1577 uri = 'gitannex://%s;protocol=file;subdir=${S}' % self.srcdir
1578 fetcher, ud = self.fetch_shallow(uri)
1579
1580 self.assertRevCount(1)
1581 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0]
1582 assert os.path.exists(os.path.join(self.gitdir, 'c'))
1583
1584 def test_shallow_multi_one_uri(self):
1585 # Create initial git repo
1586 self.add_empty_file('a')
1587 self.add_empty_file('b')
1588 self.git('checkout -b a_branch', cwd=self.srcdir)
1589 self.add_empty_file('c')
1590 self.add_empty_file('d')
1591 self.git('checkout master', cwd=self.srcdir)
1592 self.git('tag v0.0 a_branch', cwd=self.srcdir)
1593 self.add_empty_file('e')
1594 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1595 self.add_empty_file('f')
1596 self.assertRevCount(7, cwd=self.srcdir)
1597
Brad Bishop19323692019-04-05 15:28:33 -0400[diff] [blame]1598 uri = self.d.getVar('SRC_URI').split()[0]
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1599 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
1600
1601 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1602 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1603 self.d.setVar('SRCREV_master', '${AUTOREV}')
1604 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
1605
1606 self.fetch_shallow(uri)
1607
1608 self.assertRevCount(5)
1609 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1610
1611 def test_shallow_multi_one_uri_depths(self):
1612 # Create initial git repo
1613 self.add_empty_file('a')
1614 self.add_empty_file('b')
1615 self.git('checkout -b a_branch', cwd=self.srcdir)
1616 self.add_empty_file('c')
1617 self.add_empty_file('d')
1618 self.git('checkout master', cwd=self.srcdir)
1619 self.add_empty_file('e')
1620 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1621 self.add_empty_file('f')
1622 self.assertRevCount(7, cwd=self.srcdir)
1623
Brad Bishop19323692019-04-05 15:28:33 -0400[diff] [blame]1624 uri = self.d.getVar('SRC_URI').split()[0]
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500[diff] [blame]1625 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
1626
1627 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1628 self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3')
1629 self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1')
1630 self.d.setVar('SRCREV_master', '${AUTOREV}')
1631 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
1632
1633 self.fetch_shallow(uri)
1634
1635 self.assertRevCount(4, ['--all'])
1636 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1637
1638 def test_shallow_clone_preferred_over_shallow(self):
1639 self.add_empty_file('a')
1640 self.add_empty_file('b')
1641
1642 # Fetch once to generate the shallow tarball
1643 fetcher, ud = self.fetch()
1644 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1645
1646 # Fetch and unpack with both the clonedir and shallow tarball available
1647 bb.utils.remove(self.gitdir, recurse=True)
1648 fetcher, ud = self.fetch_and_unpack()
1649
1650 # The unpacked tree should *not* be shallow
1651 self.assertRevCount(2)
1652 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow'))
1653
1654 def test_shallow_mirrors(self):
1655 self.add_empty_file('a')
1656 self.add_empty_file('b')
1657
1658 # Fetch once to generate the shallow tarball
1659 fetcher, ud = self.fetch()
1660 mirrortarball = ud.mirrortarballs[0]
1661 assert os.path.exists(os.path.join(self.dldir, mirrortarball))
1662
1663 # Set up the mirror
1664 mirrordir = os.path.join(self.tempdir, 'mirror')
1665 bb.utils.mkdirhier(mirrordir)
1666 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir)
1667
1668 os.rename(os.path.join(self.dldir, mirrortarball),
1669 os.path.join(mirrordir, mirrortarball))
1670
1671 # Fetch from the mirror
1672 bb.utils.remove(self.dldir, recurse=True)
1673 bb.utils.remove(self.gitdir, recurse=True)
1674 self.fetch_and_unpack()
1675 self.assertRevCount(1)
1676
1677 def test_shallow_invalid_depth(self):
1678 self.add_empty_file('a')
1679 self.add_empty_file('b')
1680
1681 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12')
1682 with self.assertRaises(bb.fetch2.FetchError):
1683 self.fetch()
1684
1685 def test_shallow_invalid_depth_default(self):
1686 self.add_empty_file('a')
1687 self.add_empty_file('b')
1688
1689 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12')
1690 with self.assertRaises(bb.fetch2.FetchError):
1691 self.fetch()
1692
1693 def test_shallow_extra_refs(self):
1694 self.add_empty_file('a')
1695 self.add_empty_file('b')
1696 self.git('branch a_branch', cwd=self.srcdir)
1697 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
1698 self.assertRevCount(2, cwd=self.srcdir)
1699
1700 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch')
1701 self.fetch_shallow()
1702
1703 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1704 self.assertRevCount(1)
1705
1706 def test_shallow_extra_refs_wildcard(self):
1707 self.add_empty_file('a')
1708 self.add_empty_file('b')
1709 self.git('branch a_branch', cwd=self.srcdir)
1710 self.git('tag v1.0', cwd=self.srcdir)
1711 self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir)
1712 self.assertRevCount(2, cwd=self.srcdir)
1713
1714 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
1715 self.fetch_shallow()
1716
1717 self.assertRefs(['master', 'origin/master', 'v1.0'])
1718 self.assertRevCount(1)
1719
1720 def test_shallow_missing_extra_refs(self):
1721 self.add_empty_file('a')
1722 self.add_empty_file('b')
1723
1724 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo')
1725 with self.assertRaises(bb.fetch2.FetchError):
1726 self.fetch()
1727
1728 def test_shallow_missing_extra_refs_wildcard(self):
1729 self.add_empty_file('a')
1730 self.add_empty_file('b')
1731
1732 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
1733 self.fetch()
1734
1735 def test_shallow_remove_revs(self):
1736 # Create initial git repo
1737 self.add_empty_file('a')
1738 self.add_empty_file('b')
1739 self.git('checkout -b a_branch', cwd=self.srcdir)
1740 self.add_empty_file('c')
1741 self.add_empty_file('d')
1742 self.git('checkout master', cwd=self.srcdir)
1743 self.git('tag v0.0 a_branch', cwd=self.srcdir)
1744 self.add_empty_file('e')
1745 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1746 self.git('branch -d a_branch', cwd=self.srcdir)
1747 self.add_empty_file('f')
1748 self.assertRevCount(7, cwd=self.srcdir)
1749
1750 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1751 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1752
1753 self.fetch_shallow()
1754
1755 self.assertRevCount(5)
1756
1757 def test_shallow_invalid_revs(self):
1758 self.add_empty_file('a')
1759 self.add_empty_file('b')
1760
1761 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1762 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1763
1764 with self.assertRaises(bb.fetch2.FetchError):
1765 self.fetch()
1766
1767 @skipIfNoNetwork()
1768 def test_bitbake(self):
1769 self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
1770 self.git('config core.bare true', cwd=self.srcdir)
1771 self.git('fetch', cwd=self.srcdir)
1772
1773 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1774 # Note that the 1.10.0 tag is annotated, so this also tests
1775 # reference of an annotated vs unannotated tag
1776 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0')
1777
1778 self.fetch_shallow()
1779
1780 # Confirm that the history of 1.10.0 was removed
1781 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines())
1782 revs = len(self.git('rev-list master').splitlines())
1783 self.assertNotEqual(orig_revs, revs)
1784 self.assertRefs(['master', 'origin/master'])
1785 self.assertRevCount(orig_revs - 1758)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800[diff] [blame]1786
1787 def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self):
1788 self.add_empty_file('a')
1789 fetcher, ud = self.fetch()
1790 bb.utils.remove(self.gitdir, recurse=True)
1791 bb.utils.remove(self.dldir, recurse=True)
1792
1793 with self.assertRaises(bb.fetch2.UnpackError) as context:
1794 fetcher.unpack(self.d.getVar('WORKDIR'))
1795
1796 self.assertIn("No up to date source found", context.exception.msg)
1797 self.assertIn("clone directory not available or not up to date", context.exception.msg)
1798
1799 @skipIfNoNetwork()
1800 def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self):
1801 self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0')
1802 self.d.setVar('BB_GIT_SHALLOW', '1')
1803 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
1804 fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests"], self.d)
1805 fetcher.download()
1806
1807 bb.utils.remove(self.dldir + "/*.tar.gz")
1808 fetcher.unpack(self.unpackdir)
1809
1810 dir = os.listdir(self.unpackdir + "/git/")
1811 self.assertIn("fstests.doap", dir)