blob: 74859f9d302e4c63f27dd502b326b6a6e421a74d [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3#
4# BitBake Tests for the Fetcher (fetch2/)
5#
6# Copyright (C) 2012 Richard Purdie
7#
8# This program is free software; you can redistribute it and/or modify
9# it under the terms of the GNU General Public License version 2 as
10# published by the Free Software Foundation.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License along
18# with this program; if not, write to the Free Software Foundation, Inc.,
19# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20#
21
22import unittest
Brad Bishop316dfdd2018-06-25 12:45:53 -040023import hashlib
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024import tempfile
25import subprocess
Patrick Williamsc0f7c042017-02-23 20:41:17 -060026import collections
Patrick Williamsc124f4f2015-09-15 14:41:29 -050027import os
28from bb.fetch2 import URI
29from bb.fetch2 import FetchMethod
30import bb
31
Brad Bishopd7bf8c12018-02-25 22:55:05 -050032def skipIfNoNetwork():
33 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
34 return unittest.skip("Network tests being skipped")
35 return lambda f: f
36
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037class URITest(unittest.TestCase):
38 test_uris = {
39 "http://www.google.com/index.html" : {
40 'uri': 'http://www.google.com/index.html',
41 'scheme': 'http',
42 'hostname': 'www.google.com',
43 'port': None,
44 'hostport': 'www.google.com',
45 'path': '/index.html',
46 'userinfo': '',
47 'username': '',
48 'password': '',
49 'params': {},
50 'query': {},
51 'relative': False
52 },
53 "http://www.google.com/index.html;param1=value1" : {
54 'uri': 'http://www.google.com/index.html;param1=value1',
55 'scheme': 'http',
56 'hostname': 'www.google.com',
57 'port': None,
58 'hostport': 'www.google.com',
59 'path': '/index.html',
60 'userinfo': '',
61 'username': '',
62 'password': '',
63 'params': {
64 'param1': 'value1'
65 },
66 'query': {},
67 'relative': False
68 },
69 "http://www.example.org/index.html?param1=value1" : {
70 'uri': 'http://www.example.org/index.html?param1=value1',
71 'scheme': 'http',
72 'hostname': 'www.example.org',
73 'port': None,
74 'hostport': 'www.example.org',
75 'path': '/index.html',
76 'userinfo': '',
77 'username': '',
78 'password': '',
79 'params': {},
80 'query': {
81 'param1': 'value1'
82 },
83 'relative': False
84 },
85 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : {
86 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2',
87 'scheme': 'http',
88 'hostname': 'www.example.org',
89 'port': None,
90 'hostport': 'www.example.org',
91 'path': '/index.html',
92 'userinfo': '',
93 'username': '',
94 'password': '',
95 'params': {
96 'param2': 'value2'
97 },
98 'query': {
99 'qparam1': 'qvalue1'
100 },
101 'relative': False
102 },
103 "http://www.example.com:8080/index.html" : {
104 'uri': 'http://www.example.com:8080/index.html',
105 'scheme': 'http',
106 'hostname': 'www.example.com',
107 'port': 8080,
108 'hostport': 'www.example.com:8080',
109 'path': '/index.html',
110 'userinfo': '',
111 'username': '',
112 'password': '',
113 'params': {},
114 'query': {},
115 'relative': False
116 },
117 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : {
118 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg',
119 'scheme': 'cvs',
120 'hostname': 'cvs.handhelds.org',
121 'port': None,
122 'hostport': 'cvs.handhelds.org',
123 'path': '/cvs',
124 'userinfo': 'anoncvs',
125 'username': 'anoncvs',
126 'password': '',
127 'params': {
128 'module': 'familiar/dist/ipkg'
129 },
130 'query': {},
131 'relative': False
132 },
133 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": {
134 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg',
135 'scheme': 'cvs',
136 'hostname': 'cvs.handhelds.org',
137 'port': None,
138 'hostport': 'cvs.handhelds.org',
139 'path': '/cvs',
140 'userinfo': 'anoncvs:anonymous',
141 'username': 'anoncvs',
142 'password': 'anonymous',
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600143 'params': collections.OrderedDict([
144 ('tag', 'V0-99-81'),
145 ('module', 'familiar/dist/ipkg')
146 ]),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500147 'query': {},
148 'relative': False
149 },
150 "file://example.diff": { # NOTE: Not RFC compliant!
151 'uri': 'file:example.diff',
152 'scheme': 'file',
153 'hostname': '',
154 'port': None,
155 'hostport': '',
156 'path': 'example.diff',
157 'userinfo': '',
158 'username': '',
159 'password': '',
160 'params': {},
161 'query': {},
162 'relative': True
163 },
164 "file:example.diff": { # NOTE: RFC compliant version of the former
165 'uri': 'file:example.diff',
166 'scheme': 'file',
167 'hostname': '',
168 'port': None,
169 'hostport': '',
170 'path': 'example.diff',
171 'userinfo': '',
172 'userinfo': '',
173 'username': '',
174 'password': '',
175 'params': {},
176 'query': {},
177 'relative': True
178 },
179 "file:///tmp/example.diff": {
180 'uri': 'file:///tmp/example.diff',
181 'scheme': 'file',
182 'hostname': '',
183 'port': None,
184 'hostport': '',
185 'path': '/tmp/example.diff',
186 'userinfo': '',
187 'userinfo': '',
188 'username': '',
189 'password': '',
190 'params': {},
191 'query': {},
192 'relative': False
193 },
194 "git:///path/example.git": {
195 'uri': 'git:///path/example.git',
196 'scheme': 'git',
197 'hostname': '',
198 'port': None,
199 'hostport': '',
200 'path': '/path/example.git',
201 'userinfo': '',
202 'userinfo': '',
203 'username': '',
204 'password': '',
205 'params': {},
206 'query': {},
207 'relative': False
208 },
209 "git:path/example.git": {
210 'uri': 'git:path/example.git',
211 'scheme': 'git',
212 'hostname': '',
213 'port': None,
214 'hostport': '',
215 'path': 'path/example.git',
216 'userinfo': '',
217 'userinfo': '',
218 'username': '',
219 'password': '',
220 'params': {},
221 'query': {},
222 'relative': True
223 },
224 "git://example.net/path/example.git": {
225 'uri': 'git://example.net/path/example.git',
226 'scheme': 'git',
227 'hostname': 'example.net',
228 'port': None,
229 'hostport': 'example.net',
230 'path': '/path/example.git',
231 'userinfo': '',
232 'userinfo': '',
233 'username': '',
234 'password': '',
235 'params': {},
236 'query': {},
237 'relative': False
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500238 },
239 "http://somesite.net;someparam=1": {
240 'uri': 'http://somesite.net;someparam=1',
241 'scheme': 'http',
242 'hostname': 'somesite.net',
243 'port': None,
244 'hostport': 'somesite.net',
245 'path': '',
246 'userinfo': '',
247 'userinfo': '',
248 'username': '',
249 'password': '',
250 'params': {"someparam" : "1"},
251 'query': {},
252 'relative': False
253 },
254 "file://somelocation;someparam=1": {
255 'uri': 'file:somelocation;someparam=1',
256 'scheme': 'file',
257 'hostname': '',
258 'port': None,
259 'hostport': '',
260 'path': 'somelocation',
261 'userinfo': '',
262 'userinfo': '',
263 'username': '',
264 'password': '',
265 'params': {"someparam" : "1"},
266 'query': {},
267 'relative': True
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500268 }
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500269
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500270 }
271
272 def test_uri(self):
273 for test_uri, ref in self.test_uris.items():
274 uri = URI(test_uri)
275
276 self.assertEqual(str(uri), ref['uri'])
277
278 # expected attributes
279 self.assertEqual(uri.scheme, ref['scheme'])
280
281 self.assertEqual(uri.userinfo, ref['userinfo'])
282 self.assertEqual(uri.username, ref['username'])
283 self.assertEqual(uri.password, ref['password'])
284
285 self.assertEqual(uri.hostname, ref['hostname'])
286 self.assertEqual(uri.port, ref['port'])
287 self.assertEqual(uri.hostport, ref['hostport'])
288
289 self.assertEqual(uri.path, ref['path'])
290 self.assertEqual(uri.params, ref['params'])
291
292 self.assertEqual(uri.relative, ref['relative'])
293
294 def test_dict(self):
295 for test in self.test_uris.values():
296 uri = URI()
297
298 self.assertEqual(uri.scheme, '')
299 self.assertEqual(uri.userinfo, '')
300 self.assertEqual(uri.username, '')
301 self.assertEqual(uri.password, '')
302 self.assertEqual(uri.hostname, '')
303 self.assertEqual(uri.port, None)
304 self.assertEqual(uri.path, '')
305 self.assertEqual(uri.params, {})
306
307
308 uri.scheme = test['scheme']
309 self.assertEqual(uri.scheme, test['scheme'])
310
311 uri.userinfo = test['userinfo']
312 self.assertEqual(uri.userinfo, test['userinfo'])
313 self.assertEqual(uri.username, test['username'])
314 self.assertEqual(uri.password, test['password'])
315
316 # make sure changing the values doesn't do anything unexpected
317 uri.username = 'changeme'
318 self.assertEqual(uri.username, 'changeme')
319 self.assertEqual(uri.password, test['password'])
320 uri.password = 'insecure'
321 self.assertEqual(uri.username, 'changeme')
322 self.assertEqual(uri.password, 'insecure')
323
324 # reset back after our trickery
325 uri.userinfo = test['userinfo']
326 self.assertEqual(uri.userinfo, test['userinfo'])
327 self.assertEqual(uri.username, test['username'])
328 self.assertEqual(uri.password, test['password'])
329
330 uri.hostname = test['hostname']
331 self.assertEqual(uri.hostname, test['hostname'])
332 self.assertEqual(uri.hostport, test['hostname'])
333
334 uri.port = test['port']
335 self.assertEqual(uri.port, test['port'])
336 self.assertEqual(uri.hostport, test['hostport'])
337
338 uri.path = test['path']
339 self.assertEqual(uri.path, test['path'])
340
341 uri.params = test['params']
342 self.assertEqual(uri.params, test['params'])
343
344 uri.query = test['query']
345 self.assertEqual(uri.query, test['query'])
346
347 self.assertEqual(str(uri), test['uri'])
348
349 uri.params = {}
350 self.assertEqual(uri.params, {})
351 self.assertEqual(str(uri), (str(uri).split(";"))[0])
352
353class FetcherTest(unittest.TestCase):
354
355 def setUp(self):
356 self.origdir = os.getcwd()
357 self.d = bb.data.init()
358 self.tempdir = tempfile.mkdtemp()
359 self.dldir = os.path.join(self.tempdir, "download")
360 os.mkdir(self.dldir)
361 self.d.setVar("DL_DIR", self.dldir)
362 self.unpackdir = os.path.join(self.tempdir, "unpacked")
363 os.mkdir(self.unpackdir)
364 persistdir = os.path.join(self.tempdir, "persistdata")
365 self.d.setVar("PERSISTENT_DIR", persistdir)
366
367 def tearDown(self):
368 os.chdir(self.origdir)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600369 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
370 print("Not cleaning up %s. Please remove manually." % self.tempdir)
371 else:
372 bb.utils.prunedir(self.tempdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500373
374class MirrorUriTest(FetcherTest):
375
376 replaceuris = {
377 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/")
378 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
379 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
380 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
381 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
382 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
383 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http")
384 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
385 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake")
386 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890",
387 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache")
388 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
389 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/")
390 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
391 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3")
392 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
393 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz")
394 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
395 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist")
396 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2",
397 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/")
398 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2",
399 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
400 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
401 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
402 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
403 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http")
404 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
405
406 #Renaming files doesn't work
407 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
408 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
409 }
410
411 mirrorvar = "http://.*/.* file:///somepath/downloads/ \n" \
412 "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n" \
413 "https://.*/.* file:///someotherpath/downloads/ \n" \
414 "http://.*/.* file:///someotherpath/downloads/ \n"
415
416 def test_urireplace(self):
417 for k, v in self.replaceuris.items():
418 ud = bb.fetch.FetchData(k[0], self.d)
419 ud.setup_localpath(self.d)
420 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2]))
421 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d)
422 self.assertEqual([v], newuris)
423
424 def test_urilist1(self):
425 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
426 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
427 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
428 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
429
430 def test_urilist2(self):
431 # Catch https:// -> files:// bug
432 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
433 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
434 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
435 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
436
437 def test_mirror_of_mirror(self):
438 # Test if mirror of a mirror works
439 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/ \n"
440 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/ \n"
441 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
442 mirrors = bb.fetch2.mirror_from_string(mirrorvar)
443 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
444 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz',
445 'file:///someotherpath/downloads/bitbake-1.0.tar.gz',
446 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz',
447 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz'])
448
Patrick Williamsd7e96312015-09-22 08:09:05 -0500449 recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ \n" \
450 "https://.*/[^/]* https://BBBB/B/B/B/ \n"
451
452 def test_recursive(self):
453 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
454 mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar)
455 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
456 self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz',
457 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz',
458 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500459
460class FetcherLocalTest(FetcherTest):
461 def setUp(self):
462 def touch(fn):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600463 with open(fn, 'a'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500464 os.utime(fn, None)
465
466 super(FetcherLocalTest, self).setUp()
467 self.localsrcdir = os.path.join(self.tempdir, 'localsrc')
468 os.makedirs(self.localsrcdir)
469 touch(os.path.join(self.localsrcdir, 'a'))
470 touch(os.path.join(self.localsrcdir, 'b'))
471 os.makedirs(os.path.join(self.localsrcdir, 'dir'))
472 touch(os.path.join(self.localsrcdir, 'dir', 'c'))
473 touch(os.path.join(self.localsrcdir, 'dir', 'd'))
474 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir'))
475 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e'))
476 self.d.setVar("FILESPATH", self.localsrcdir)
477
478 def fetchUnpack(self, uris):
479 fetcher = bb.fetch.Fetch(uris, self.d)
480 fetcher.download()
481 fetcher.unpack(self.unpackdir)
482 flst = []
483 for root, dirs, files in os.walk(self.unpackdir):
484 for f in files:
485 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir))
486 flst.sort()
487 return flst
488
489 def test_local(self):
490 tree = self.fetchUnpack(['file://a', 'file://dir/c'])
491 self.assertEqual(tree, ['a', 'dir/c'])
492
493 def test_local_wildcard(self):
494 tree = self.fetchUnpack(['file://a', 'file://dir/*'])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500495 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500496
497 def test_local_dir(self):
498 tree = self.fetchUnpack(['file://a', 'file://dir'])
499 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e'])
500
501 def test_local_subdir(self):
502 tree = self.fetchUnpack(['file://dir/subdir'])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500503 self.assertEqual(tree, ['dir/subdir/e'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500504
505 def test_local_subdir_file(self):
506 tree = self.fetchUnpack(['file://dir/subdir/e'])
507 self.assertEqual(tree, ['dir/subdir/e'])
508
509 def test_local_subdirparam(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500510 tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo'])
511 self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e'])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500512
513 def test_local_deepsubdirparam(self):
514 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar'])
515 self.assertEqual(tree, ['bar/dir/subdir/e'])
516
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600517 def test_local_absolutedir(self):
518 # Unpacking to an absolute path that is a subdirectory of the root
519 # should work
520 tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')])
521
522 # Unpacking to an absolute path outside of the root should fail
523 with self.assertRaises(bb.fetch2.UnpackError):
524 self.fetchUnpack(['file://a;subdir=/bin/sh'])
525
Brad Bishop316dfdd2018-06-25 12:45:53 -0400526class FetcherNoNetworkTest(FetcherTest):
527 def setUp(self):
528 super().setUp()
529 # all test cases are based on not having network
530 self.d.setVar("BB_NO_NETWORK", "1")
531
532 def test_missing(self):
533 string = "this is a test file\n".encode("utf-8")
534 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
535 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
536
537 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
538 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
539 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
540 with self.assertRaises(bb.fetch2.NetworkAccess):
541 fetcher.download()
542
543 def test_valid_missing_donestamp(self):
544 # create the file in the download directory with correct hash
545 string = "this is a test file\n".encode("utf-8")
546 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f:
547 f.write(string)
548
549 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
550 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
551
552 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
553 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
554 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
555 fetcher.download()
556 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
557
558 def test_invalid_missing_donestamp(self):
559 # create an invalid file in the download directory with incorrect hash
560 string = "this is a test file\n".encode("utf-8")
561 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
562 pass
563
564 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest())
565 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest())
566
567 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
568 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
569 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d)
570 with self.assertRaises(bb.fetch2.NetworkAccess):
571 fetcher.download()
572 # the existing file should not exist or should have be moved to "bad-checksum"
573 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
574
575 def test_nochecksums_missing(self):
576 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
577 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
578 # ssh fetch does not support checksums
579 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
580 # attempts to download with missing donestamp
581 with self.assertRaises(bb.fetch2.NetworkAccess):
582 fetcher.download()
583
584 def test_nochecksums_missing_donestamp(self):
585 # create a file in the download directory
586 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
587 pass
588
589 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
590 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
591 # ssh fetch does not support checksums
592 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
593 # attempts to download with missing donestamp
594 with self.assertRaises(bb.fetch2.NetworkAccess):
595 fetcher.download()
596
597 def test_nochecksums_has_donestamp(self):
598 # create a file in the download directory with the donestamp
599 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"):
600 pass
601 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
602 pass
603
604 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
605 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
606 # ssh fetch does not support checksums
607 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
608 # should not fetch
609 fetcher.download()
610 # both files should still exist
611 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
612 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
613
614 def test_nochecksums_missing_has_donestamp(self):
615 # create a file in the download directory with the donestamp
616 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"):
617 pass
618
619 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
620 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
621 # ssh fetch does not support checksums
622 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d)
623 with self.assertRaises(bb.fetch2.NetworkAccess):
624 fetcher.download()
625 # both files should still exist
626 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz")))
627 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done")))
628
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500629class FetcherNetworkTest(FetcherTest):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500630 @skipIfNoNetwork()
631 def test_fetch(self):
632 fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
633 fetcher.download()
634 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
635 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892)
636 self.d.setVar("BB_NO_NETWORK", "1")
637 fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
638 fetcher.download()
639 fetcher.unpack(self.unpackdir)
640 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9)
641 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500642
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500643 @skipIfNoNetwork()
644 def test_fetch_mirror(self):
645 self.d.setVar("MIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
646 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
647 fetcher.download()
648 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
649
650 @skipIfNoNetwork()
651 def test_fetch_mirror_of_mirror(self):
652 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ \n http://invalid2.yoctoproject.org/.* http://downloads.yoctoproject.org/releases/bitbake")
653 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
654 fetcher.download()
655 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
656
657 @skipIfNoNetwork()
658 def test_fetch_file_mirror_of_mirror(self):
659 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ \n file:///some1where/.* file://some2where/ \n file://some2where/.* http://downloads.yoctoproject.org/releases/bitbake")
660 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
661 os.mkdir(self.dldir + "/some2where")
662 fetcher.download()
663 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
664
665 @skipIfNoNetwork()
666 def test_fetch_premirror(self):
667 self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
668 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
669 fetcher.download()
670 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
671
672 @skipIfNoNetwork()
673 def gitfetcher(self, url1, url2):
674 def checkrevision(self, fetcher):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500675 fetcher.unpack(self.unpackdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500676 revision = bb.process.run("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git")[0].strip()
677 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500678
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500679 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
680 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
681 fetcher = bb.fetch.Fetch([url1], self.d)
682 fetcher.download()
683 checkrevision(self, fetcher)
684 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works
685 bb.utils.prunedir(self.dldir + "/git2/")
686 bb.utils.prunedir(self.unpackdir)
687 self.d.setVar("BB_NO_NETWORK", "1")
688 fetcher = bb.fetch.Fetch([url2], self.d)
689 fetcher.download()
690 checkrevision(self, fetcher)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500691
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500692 @skipIfNoNetwork()
693 def test_gitfetch(self):
694 url1 = url2 = "git://git.openembedded.org/bitbake"
695 self.gitfetcher(url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500696
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500697 @skipIfNoNetwork()
698 def test_gitfetch_goodsrcrev(self):
699 # SRCREV is set but matches rev= parameter
700 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
701 self.gitfetcher(url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500702
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500703 @skipIfNoNetwork()
704 def test_gitfetch_badsrcrev(self):
705 # SRCREV is set but does not match rev= parameter
706 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5"
707 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500708
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500709 @skipIfNoNetwork()
710 def test_gitfetch_tagandrev(self):
711 # SRCREV is set but does not match rev= parameter
712 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
713 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500714
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500715 @skipIfNoNetwork()
716 def test_gitfetch_localusehead(self):
717 # Create dummy local Git repo
718 src_dir = tempfile.mkdtemp(dir=self.tempdir,
719 prefix='gitfetch_localusehead_')
720 src_dir = os.path.abspath(src_dir)
721 bb.process.run("git init", cwd=src_dir)
722 bb.process.run("git commit --allow-empty -m'Dummy commit'",
723 cwd=src_dir)
724 # Use other branch than master
725 bb.process.run("git checkout -b my-devel", cwd=src_dir)
726 bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
727 cwd=src_dir)
728 stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
729 orig_rev = stdout[0].strip()
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500730
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500731 # Fetch and check revision
732 self.d.setVar("SRCREV", "AUTOINC")
733 url = "git://" + src_dir + ";protocol=file;usehead=1"
734 fetcher = bb.fetch.Fetch([url], self.d)
735 fetcher.download()
736 fetcher.unpack(self.unpackdir)
737 stdout = bb.process.run("git rev-parse HEAD",
738 cwd=os.path.join(self.unpackdir, 'git'))
739 unpack_rev = stdout[0].strip()
740 self.assertEqual(orig_rev, unpack_rev)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500741
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500742 @skipIfNoNetwork()
743 def test_gitfetch_remoteusehead(self):
744 url = "git://git.openembedded.org/bitbake;usehead=1"
745 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500746
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500747 @skipIfNoNetwork()
748 def test_gitfetch_premirror(self):
749 url1 = "git://git.openembedded.org/bitbake"
750 url2 = "git://someserver.org/bitbake"
751 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
752 self.gitfetcher(url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500753
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500754 @skipIfNoNetwork()
755 def test_gitfetch_premirror2(self):
756 url1 = url2 = "git://someserver.org/bitbake"
757 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
758 self.gitfetcher(url1, url2)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500759
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500760 @skipIfNoNetwork()
761 def test_gitfetch_premirror3(self):
762 realurl = "git://git.openembedded.org/bitbake"
763 dummyurl = "git://someserver.org/bitbake"
764 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git")
765 os.chdir(self.tempdir)
766 bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True)
767 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (dummyurl, self.sourcedir))
768 self.gitfetcher(dummyurl, dummyurl)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600769
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500770 @skipIfNoNetwork()
771 def test_git_submodule(self):
772 fetcher = bb.fetch.Fetch(["gitsm://git.yoctoproject.org/git-submodule-test;rev=f12e57f2edf0aa534cf1616fa983d165a92b0842"], self.d)
773 fetcher.download()
774 # Previous cwd has been deleted
775 os.chdir(os.path.dirname(self.unpackdir))
776 fetcher.unpack(self.unpackdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500777
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500778
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500779class TrustedNetworksTest(FetcherTest):
780 def test_trusted_network(self):
781 # Ensure trusted_network returns False when the host IS in the list.
782 url = "git://Someserver.org/foo;rev=1"
783 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org")
784 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500785
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500786 def test_wild_trusted_network(self):
787 # Ensure trusted_network returns true when the *.host IS in the list.
788 url = "git://Someserver.org/foo;rev=1"
789 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
790 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500791
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500792 def test_prefix_wild_trusted_network(self):
793 # Ensure trusted_network returns true when the prefix matches *.host.
794 url = "git://git.Someserver.org/foo;rev=1"
795 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
796 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500797
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500798 def test_two_prefix_wild_trusted_network(self):
799 # Ensure trusted_network returns true when the prefix matches *.host.
800 url = "git://something.git.Someserver.org/foo;rev=1"
801 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
802 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500803
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500804 def test_port_trusted_network(self):
805 # Ensure trusted_network returns True, even if the url specifies a port.
806 url = "git://someserver.org:8080/foo;rev=1"
807 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org")
808 self.assertTrue(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500809
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500810 def test_untrusted_network(self):
811 # Ensure trusted_network returns False when the host is NOT in the list.
812 url = "git://someserver.org/foo;rev=1"
813 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
814 self.assertFalse(bb.fetch.trusted_network(self.d, url))
815
816 def test_wild_untrusted_network(self):
817 # Ensure trusted_network returns False when the host is NOT in the list.
818 url = "git://*.someserver.org/foo;rev=1"
819 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
820 self.assertFalse(bb.fetch.trusted_network(self.d, url))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500821
822class URLHandle(unittest.TestCase):
823
824 datatable = {
825 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}),
826 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}),
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600827 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])),
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500828 "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'}),
829 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}),
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500830 }
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500831 # we require a pathname to encodeurl but users can still pass such urls to
832 # decodeurl and we need to handle them
833 decodedata = datatable.copy()
834 decodedata.update({
835 "http://somesite.net;someparam=1": ('http', 'somesite.net', '', '', '', {'someparam': '1'}),
836 })
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500837
838 def test_decodeurl(self):
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500839 for k, v in self.decodedata.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500840 result = bb.fetch.decodeurl(k)
841 self.assertEqual(result, v)
842
843 def test_encodeurl(self):
844 for k, v in self.datatable.items():
845 result = bb.fetch.encodeurl(v)
846 self.assertEqual(result, k)
847
848class FetchLatestVersionTest(FetcherTest):
849
850 test_git_uris = {
851 # version pattern "X.Y.Z"
852 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
853 : "1.99.4",
854 # version pattern "vX.Y"
855 ("mtd-utils", "git://git.infradead.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "")
856 : "1.5.0",
857 # version pattern "pkg_name-X.Y"
858 ("presentproto", "git://anongit.freedesktop.org/git/xorg/proto/presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")
859 : "1.0",
860 # version pattern "pkg_name-vX.Y.Z"
861 ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
862 : "1.4.0",
863 # combination version pattern
864 ("sysprof", "git://git.gnome.org/sysprof", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
865 : "1.2.0",
866 ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "")
867 : "2014.01",
868 # version pattern "yyyymmdd"
869 ("mobile-broadband-provider-info", "git://git.gnome.org/mobile-broadband-provider-info", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
870 : "20120614",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500871 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500872 ("xf86-video-omap", "git://anongit.freedesktop.org/xorg/driver/xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))")
873 : "0.4.3",
874 ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", "(?P<pver>(([0-9][\.|_]?)+[0-9]))")
875 : "11.0.0",
876 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", "chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
877 : "1.3.59",
878 ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", "(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
879 : "3.82+dbg0.9",
880 }
881
882 test_wget_uris = {
883 # packages with versions inside directory name
884 ("util-linux", "http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "")
885 : "2.24.2",
886 ("enchant", "http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "")
887 : "1.6.0",
888 ("cmake", "http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz", "", "")
889 : "2.8.12.1",
890 # packages with versions only in current directory
891 ("eglic", "http://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "")
892 : "2.19",
893 ("gnu-config", "http://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "")
894 : "20120814",
895 # packages with "99" in the name of possible version
896 ("pulseaudio", "http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "")
897 : "5.0",
898 ("xserver-xorg", "http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "")
899 : "1.15.1",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500900 # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX
901 ("cups", "http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2", "https://github.com/apple/cups/releases", "(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500902 : "2.0.0",
903 ("db", "http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz", "http://www.oracle.com/technetwork/products/berkeleydb/downloads/index-082944.html", "http://download.oracle.com/otn/berkeley-db/(?P<name>db-)(?P<pver>((\d+[\.\-_]*)+))\.tar\.gz")
904 : "6.1.19",
905 }
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500906
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500907 @skipIfNoNetwork()
908 def test_git_latest_versionstring(self):
909 for k, v in self.test_git_uris.items():
910 self.d.setVar("PN", k[0])
911 self.d.setVar("SRCREV", k[2])
912 self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3])
913 ud = bb.fetch2.FetchData(k[1], self.d)
914 pupver= ud.method.latest_versionstring(ud, self.d)
915 verstring = pupver[0]
Brad Bishop316dfdd2018-06-25 12:45:53 -0400916 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500917 r = bb.utils.vercmp_string(v, verstring)
918 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
919
920 @skipIfNoNetwork()
921 def test_wget_latest_versionstring(self):
922 for k, v in self.test_wget_uris.items():
923 self.d.setVar("PN", k[0])
924 self.d.setVar("UPSTREAM_CHECK_URI", k[2])
925 self.d.setVar("UPSTREAM_CHECK_REGEX", k[3])
926 ud = bb.fetch2.FetchData(k[1], self.d)
927 pupver = ud.method.latest_versionstring(ud, self.d)
928 verstring = pupver[0]
Brad Bishop316dfdd2018-06-25 12:45:53 -0400929 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500930 r = bb.utils.vercmp_string(v, verstring)
931 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500932
933
934class FetchCheckStatusTest(FetcherTest):
935 test_wget_uris = ["http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2",
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500936 "http://www.cups.org/",
937 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
938 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
939 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
940 "https://yoctoproject.org/",
941 "https://yoctoproject.org/documentation",
942 "http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
943 "http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500944 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz",
945 "http://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz",
946 "https://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz",
947 "https://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz",
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500948 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD
949 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500950 ]
951
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500952 @skipIfNoNetwork()
953 def test_wget_checkstatus(self):
954 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d)
955 for u in self.test_wget_uris:
956 with self.subTest(url=u):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500957 ud = fetch.ud[u]
958 m = ud.method
959 ret = m.checkstatus(fetch, ud, self.d)
960 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
961
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500962 @skipIfNoNetwork()
963 def test_wget_checkstatus_connection_cache(self):
964 from bb.fetch2 import FetchConnectionCache
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500965
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500966 connection_cache = FetchConnectionCache()
967 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d,
968 connection_cache = connection_cache)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500969
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500970 for u in self.test_wget_uris:
971 with self.subTest(url=u):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500972 ud = fetch.ud[u]
973 m = ud.method
974 ret = m.checkstatus(fetch, ud, self.d)
975 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
976
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500977 connection_cache.close_connections()
978
979
980class GitMakeShallowTest(FetcherTest):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500981 def setUp(self):
982 FetcherTest.setUp(self)
983 self.gitdir = os.path.join(self.tempdir, 'gitshallow')
984 bb.utils.mkdirhier(self.gitdir)
985 bb.process.run('git init', cwd=self.gitdir)
986
987 def assertRefs(self, expected_refs):
988 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
989 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines()
990 self.assertEqual(sorted(full_expected), sorted(actual_refs))
991
992 def assertRevCount(self, expected_count, args=None):
993 if args is None:
994 args = ['HEAD']
995 revs = self.git(['rev-list'] + args)
996 actual_count = len(revs.splitlines())
997 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
998
999 def git(self, cmd):
1000 if isinstance(cmd, str):
1001 cmd = 'git ' + cmd
1002 else:
1003 cmd = ['git'] + cmd
1004 return bb.process.run(cmd, cwd=self.gitdir)[0]
1005
1006 def make_shallow(self, args=None):
1007 if args is None:
1008 args = ['HEAD']
Brad Bishop316dfdd2018-06-25 12:45:53 -04001009 return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir)
Brad Bishopd7bf8c12018-02-25 22:55:05 -05001010
1011 def add_empty_file(self, path, msg=None):
1012 if msg is None:
1013 msg = path
1014 open(os.path.join(self.gitdir, path), 'w').close()
1015 self.git(['add', path])
1016 self.git(['commit', '-m', msg, path])
1017
1018 def test_make_shallow_single_branch_no_merge(self):
1019 self.add_empty_file('a')
1020 self.add_empty_file('b')
1021 self.assertRevCount(2)
1022 self.make_shallow()
1023 self.assertRevCount(1)
1024
1025 def test_make_shallow_single_branch_one_merge(self):
1026 self.add_empty_file('a')
1027 self.add_empty_file('b')
1028 self.git('checkout -b a_branch')
1029 self.add_empty_file('c')
1030 self.git('checkout master')
1031 self.add_empty_file('d')
1032 self.git('merge --no-ff --no-edit a_branch')
1033 self.git('branch -d a_branch')
1034 self.add_empty_file('e')
1035 self.assertRevCount(6)
1036 self.make_shallow(['HEAD~2'])
1037 self.assertRevCount(5)
1038
1039 def test_make_shallow_at_merge(self):
1040 self.add_empty_file('a')
1041 self.git('checkout -b a_branch')
1042 self.add_empty_file('b')
1043 self.git('checkout master')
1044 self.git('merge --no-ff --no-edit a_branch')
1045 self.git('branch -d a_branch')
1046 self.assertRevCount(3)
1047 self.make_shallow()
1048 self.assertRevCount(1)
1049
1050 def test_make_shallow_annotated_tag(self):
1051 self.add_empty_file('a')
1052 self.add_empty_file('b')
1053 self.git('tag -a -m a_tag a_tag')
1054 self.assertRevCount(2)
1055 self.make_shallow(['a_tag'])
1056 self.assertRevCount(1)
1057
1058 def test_make_shallow_multi_ref(self):
1059 self.add_empty_file('a')
1060 self.add_empty_file('b')
1061 self.git('checkout -b a_branch')
1062 self.add_empty_file('c')
1063 self.git('checkout master')
1064 self.add_empty_file('d')
1065 self.git('checkout -b a_branch_2')
1066 self.add_empty_file('a_tag')
1067 self.git('tag a_tag')
1068 self.git('checkout master')
1069 self.git('branch -D a_branch_2')
1070 self.add_empty_file('e')
1071 self.assertRevCount(6, ['--all'])
1072 self.make_shallow()
1073 self.assertRevCount(5, ['--all'])
1074
1075 def test_make_shallow_multi_ref_trim(self):
1076 self.add_empty_file('a')
1077 self.git('checkout -b a_branch')
1078 self.add_empty_file('c')
1079 self.git('checkout master')
1080 self.assertRevCount(1)
1081 self.assertRevCount(2, ['--all'])
1082 self.assertRefs(['master', 'a_branch'])
1083 self.make_shallow(['-r', 'master', 'HEAD'])
1084 self.assertRevCount(1, ['--all'])
1085 self.assertRefs(['master'])
1086
1087 def test_make_shallow_noop(self):
1088 self.add_empty_file('a')
1089 self.assertRevCount(1)
1090 self.make_shallow()
1091 self.assertRevCount(1)
1092
1093 @skipIfNoNetwork()
1094 def test_make_shallow_bitbake(self):
1095 self.git('remote add origin https://github.com/openembedded/bitbake')
1096 self.git('fetch --tags origin')
1097 orig_revs = len(self.git('rev-list --all').splitlines())
1098 self.make_shallow(['refs/tags/1.10.0'])
1099 self.assertRevCount(orig_revs - 1746, ['--all'])
1100
1101class GitShallowTest(FetcherTest):
1102 def setUp(self):
1103 FetcherTest.setUp(self)
1104 self.gitdir = os.path.join(self.tempdir, 'git')
1105 self.srcdir = os.path.join(self.tempdir, 'gitsource')
1106
1107 bb.utils.mkdirhier(self.srcdir)
1108 self.git('init', cwd=self.srcdir)
1109 self.d.setVar('WORKDIR', self.tempdir)
1110 self.d.setVar('S', self.gitdir)
1111 self.d.delVar('PREMIRRORS')
1112 self.d.delVar('MIRRORS')
1113
1114 uri = 'git://%s;protocol=file;subdir=${S}' % self.srcdir
1115 self.d.setVar('SRC_URI', uri)
1116 self.d.setVar('SRCREV', '${AUTOREV}')
1117 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
1118
1119 self.d.setVar('BB_GIT_SHALLOW', '1')
1120 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
1121 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
1122
1123 def assertRefs(self, expected_refs, cwd=None):
1124 if cwd is None:
1125 cwd = self.gitdir
1126 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines()
1127 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines()
1128 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs)))
1129
1130 def assertRevCount(self, expected_count, args=None, cwd=None):
1131 if args is None:
1132 args = ['HEAD']
1133 if cwd is None:
1134 cwd = self.gitdir
1135 revs = self.git(['rev-list'] + args, cwd=cwd)
1136 actual_count = len(revs.splitlines())
1137 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1138
1139 def git(self, cmd, cwd=None):
1140 if isinstance(cmd, str):
1141 cmd = 'git ' + cmd
1142 else:
1143 cmd = ['git'] + cmd
1144 if cwd is None:
1145 cwd = self.gitdir
1146 return bb.process.run(cmd, cwd=cwd)[0]
1147
1148 def add_empty_file(self, path, cwd=None, msg=None):
1149 if msg is None:
1150 msg = path
1151 if cwd is None:
1152 cwd = self.srcdir
1153 open(os.path.join(cwd, path), 'w').close()
1154 self.git(['add', path], cwd)
1155 self.git(['commit', '-m', msg, path], cwd)
1156
1157 def fetch(self, uri=None):
1158 if uri is None:
1159 uris = self.d.getVar('SRC_URI', True).split()
1160 uri = uris[0]
1161 d = self.d
1162 else:
1163 d = self.d.createCopy()
1164 d.setVar('SRC_URI', uri)
1165 uri = d.expand(uri)
1166 uris = [uri]
1167
1168 fetcher = bb.fetch2.Fetch(uris, d)
1169 fetcher.download()
1170 ud = fetcher.ud[uri]
1171 return fetcher, ud
1172
1173 def fetch_and_unpack(self, uri=None):
1174 fetcher, ud = self.fetch(uri)
1175 fetcher.unpack(self.d.getVar('WORKDIR'))
1176 assert os.path.exists(self.d.getVar('S'))
1177 return fetcher, ud
1178
1179 def fetch_shallow(self, uri=None, disabled=False, keepclone=False):
1180 """Fetch a uri, generating a shallow tarball, then unpack using it"""
1181 fetcher, ud = self.fetch_and_unpack(uri)
1182 assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri)
1183
1184 # Confirm that the unpacked repo is unshallow
1185 if not disabled:
1186 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1187
1188 # fetch and unpack, from the shallow tarball
1189 bb.utils.remove(self.gitdir, recurse=True)
1190 bb.utils.remove(ud.clonedir, recurse=True)
1191
1192 # confirm that the unpacked repo is used when no git clone or git
1193 # mirror tarball is available
1194 fetcher, ud = self.fetch_and_unpack(uri)
1195 if not disabled:
1196 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir
1197 else:
1198 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir
1199 return fetcher, ud
1200
1201 def test_shallow_disabled(self):
1202 self.add_empty_file('a')
1203 self.add_empty_file('b')
1204 self.assertRevCount(2, cwd=self.srcdir)
1205
1206 self.d.setVar('BB_GIT_SHALLOW', '0')
1207 self.fetch_shallow(disabled=True)
1208 self.assertRevCount(2)
1209
1210 def test_shallow_nobranch(self):
1211 self.add_empty_file('a')
1212 self.add_empty_file('b')
1213 self.assertRevCount(2, cwd=self.srcdir)
1214
1215 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip()
1216 self.d.setVar('SRCREV', srcrev)
1217 uri = self.d.getVar('SRC_URI', True).split()[0]
1218 uri = '%s;nobranch=1;bare=1' % uri
1219
1220 self.fetch_shallow(uri)
1221 self.assertRevCount(1)
1222
1223 # shallow refs are used to ensure the srcrev sticks around when we
1224 # have no other branches referencing it
1225 self.assertRefs(['refs/shallow/default'])
1226
1227 def test_shallow_default_depth_1(self):
1228 # Create initial git repo
1229 self.add_empty_file('a')
1230 self.add_empty_file('b')
1231 self.assertRevCount(2, cwd=self.srcdir)
1232
1233 self.fetch_shallow()
1234 self.assertRevCount(1)
1235
1236 def test_shallow_depth_0_disables(self):
1237 self.add_empty_file('a')
1238 self.add_empty_file('b')
1239 self.assertRevCount(2, cwd=self.srcdir)
1240
1241 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1242 self.fetch_shallow(disabled=True)
1243 self.assertRevCount(2)
1244
1245 def test_shallow_depth_default_override(self):
1246 self.add_empty_file('a')
1247 self.add_empty_file('b')
1248 self.assertRevCount(2, cwd=self.srcdir)
1249
1250 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2')
1251 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1')
1252 self.fetch_shallow()
1253 self.assertRevCount(1)
1254
1255 def test_shallow_depth_default_override_disable(self):
1256 self.add_empty_file('a')
1257 self.add_empty_file('b')
1258 self.add_empty_file('c')
1259 self.assertRevCount(3, cwd=self.srcdir)
1260
1261 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1262 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2')
1263 self.fetch_shallow()
1264 self.assertRevCount(2)
1265
1266 def test_current_shallow_out_of_date_clone(self):
1267 # Create initial git repo
1268 self.add_empty_file('a')
1269 self.add_empty_file('b')
1270 self.add_empty_file('c')
1271 self.assertRevCount(3, cwd=self.srcdir)
1272
1273 # Clone and generate mirror tarball
1274 fetcher, ud = self.fetch()
1275
1276 # Ensure we have a current mirror tarball, but an out of date clone
1277 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir)
1278 self.assertRevCount(2, cwd=ud.clonedir)
1279
1280 # Fetch and unpack, from the current tarball, not the out of date clone
1281 bb.utils.remove(self.gitdir, recurse=True)
1282 fetcher, ud = self.fetch()
1283 fetcher.unpack(self.d.getVar('WORKDIR'))
1284 self.assertRevCount(1)
1285
1286 def test_shallow_single_branch_no_merge(self):
1287 self.add_empty_file('a')
1288 self.add_empty_file('b')
1289 self.assertRevCount(2, cwd=self.srcdir)
1290
1291 self.fetch_shallow()
1292 self.assertRevCount(1)
1293 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1294 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1295
1296 def test_shallow_no_dangling(self):
1297 self.add_empty_file('a')
1298 self.add_empty_file('b')
1299 self.assertRevCount(2, cwd=self.srcdir)
1300
1301 self.fetch_shallow()
1302 self.assertRevCount(1)
1303 assert not self.git('fsck --dangling')
1304
1305 def test_shallow_srcrev_branch_truncation(self):
1306 self.add_empty_file('a')
1307 self.add_empty_file('b')
1308 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip()
1309 self.add_empty_file('c')
1310 self.assertRevCount(3, cwd=self.srcdir)
1311
1312 self.d.setVar('SRCREV', b_commit)
1313 self.fetch_shallow()
1314
1315 # The 'c' commit was removed entirely, and 'a' was removed from history
1316 self.assertRevCount(1, ['--all'])
1317 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit)
1318 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1319 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1320 assert not os.path.exists(os.path.join(self.gitdir, 'c'))
1321
1322 def test_shallow_ref_pruning(self):
1323 self.add_empty_file('a')
1324 self.add_empty_file('b')
1325 self.git('branch a_branch', cwd=self.srcdir)
1326 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
1327 self.assertRevCount(2, cwd=self.srcdir)
1328
1329 self.fetch_shallow()
1330
1331 self.assertRefs(['master', 'origin/master'])
1332 self.assertRevCount(1)
1333
1334 def test_shallow_submodules(self):
1335 self.add_empty_file('a')
1336 self.add_empty_file('b')
1337
1338 smdir = os.path.join(self.tempdir, 'gitsubmodule')
1339 bb.utils.mkdirhier(smdir)
1340 self.git('init', cwd=smdir)
1341 self.add_empty_file('asub', cwd=smdir)
1342
1343 self.git('submodule init', cwd=self.srcdir)
1344 self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
1345 self.git('submodule update', cwd=self.srcdir)
1346 self.git('commit -m submodule -a', cwd=self.srcdir)
1347
1348 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
1349 fetcher, ud = self.fetch_shallow(uri)
1350
1351 self.assertRevCount(1)
1352 assert './.git/modules/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0]
1353 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
1354
1355 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')):
1356 def test_shallow_annex(self):
1357 self.add_empty_file('a')
1358 self.add_empty_file('b')
1359 self.git('annex init', cwd=self.srcdir)
1360 open(os.path.join(self.srcdir, 'c'), 'w').close()
1361 self.git('annex add c', cwd=self.srcdir)
1362 self.git('commit -m annex-c -a', cwd=self.srcdir)
1363 bb.process.run('chmod u+w -R %s' % os.path.join(self.srcdir, '.git', 'annex'))
1364
1365 uri = 'gitannex://%s;protocol=file;subdir=${S}' % self.srcdir
1366 fetcher, ud = self.fetch_shallow(uri)
1367
1368 self.assertRevCount(1)
1369 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0]
1370 assert os.path.exists(os.path.join(self.gitdir, 'c'))
1371
1372 def test_shallow_multi_one_uri(self):
1373 # Create initial git repo
1374 self.add_empty_file('a')
1375 self.add_empty_file('b')
1376 self.git('checkout -b a_branch', cwd=self.srcdir)
1377 self.add_empty_file('c')
1378 self.add_empty_file('d')
1379 self.git('checkout master', cwd=self.srcdir)
1380 self.git('tag v0.0 a_branch', cwd=self.srcdir)
1381 self.add_empty_file('e')
1382 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1383 self.add_empty_file('f')
1384 self.assertRevCount(7, cwd=self.srcdir)
1385
1386 uri = self.d.getVar('SRC_URI', True).split()[0]
1387 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
1388
1389 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1390 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1391 self.d.setVar('SRCREV_master', '${AUTOREV}')
1392 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
1393
1394 self.fetch_shallow(uri)
1395
1396 self.assertRevCount(5)
1397 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1398
1399 def test_shallow_multi_one_uri_depths(self):
1400 # Create initial git repo
1401 self.add_empty_file('a')
1402 self.add_empty_file('b')
1403 self.git('checkout -b a_branch', cwd=self.srcdir)
1404 self.add_empty_file('c')
1405 self.add_empty_file('d')
1406 self.git('checkout master', cwd=self.srcdir)
1407 self.add_empty_file('e')
1408 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1409 self.add_empty_file('f')
1410 self.assertRevCount(7, cwd=self.srcdir)
1411
1412 uri = self.d.getVar('SRC_URI', True).split()[0]
1413 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
1414
1415 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1416 self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3')
1417 self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1')
1418 self.d.setVar('SRCREV_master', '${AUTOREV}')
1419 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
1420
1421 self.fetch_shallow(uri)
1422
1423 self.assertRevCount(4, ['--all'])
1424 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1425
1426 def test_shallow_clone_preferred_over_shallow(self):
1427 self.add_empty_file('a')
1428 self.add_empty_file('b')
1429
1430 # Fetch once to generate the shallow tarball
1431 fetcher, ud = self.fetch()
1432 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1433
1434 # Fetch and unpack with both the clonedir and shallow tarball available
1435 bb.utils.remove(self.gitdir, recurse=True)
1436 fetcher, ud = self.fetch_and_unpack()
1437
1438 # The unpacked tree should *not* be shallow
1439 self.assertRevCount(2)
1440 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow'))
1441
1442 def test_shallow_mirrors(self):
1443 self.add_empty_file('a')
1444 self.add_empty_file('b')
1445
1446 # Fetch once to generate the shallow tarball
1447 fetcher, ud = self.fetch()
1448 mirrortarball = ud.mirrortarballs[0]
1449 assert os.path.exists(os.path.join(self.dldir, mirrortarball))
1450
1451 # Set up the mirror
1452 mirrordir = os.path.join(self.tempdir, 'mirror')
1453 bb.utils.mkdirhier(mirrordir)
1454 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir)
1455
1456 os.rename(os.path.join(self.dldir, mirrortarball),
1457 os.path.join(mirrordir, mirrortarball))
1458
1459 # Fetch from the mirror
1460 bb.utils.remove(self.dldir, recurse=True)
1461 bb.utils.remove(self.gitdir, recurse=True)
1462 self.fetch_and_unpack()
1463 self.assertRevCount(1)
1464
1465 def test_shallow_invalid_depth(self):
1466 self.add_empty_file('a')
1467 self.add_empty_file('b')
1468
1469 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12')
1470 with self.assertRaises(bb.fetch2.FetchError):
1471 self.fetch()
1472
1473 def test_shallow_invalid_depth_default(self):
1474 self.add_empty_file('a')
1475 self.add_empty_file('b')
1476
1477 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12')
1478 with self.assertRaises(bb.fetch2.FetchError):
1479 self.fetch()
1480
1481 def test_shallow_extra_refs(self):
1482 self.add_empty_file('a')
1483 self.add_empty_file('b')
1484 self.git('branch a_branch', cwd=self.srcdir)
1485 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
1486 self.assertRevCount(2, cwd=self.srcdir)
1487
1488 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch')
1489 self.fetch_shallow()
1490
1491 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1492 self.assertRevCount(1)
1493
1494 def test_shallow_extra_refs_wildcard(self):
1495 self.add_empty_file('a')
1496 self.add_empty_file('b')
1497 self.git('branch a_branch', cwd=self.srcdir)
1498 self.git('tag v1.0', cwd=self.srcdir)
1499 self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir)
1500 self.assertRevCount(2, cwd=self.srcdir)
1501
1502 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
1503 self.fetch_shallow()
1504
1505 self.assertRefs(['master', 'origin/master', 'v1.0'])
1506 self.assertRevCount(1)
1507
1508 def test_shallow_missing_extra_refs(self):
1509 self.add_empty_file('a')
1510 self.add_empty_file('b')
1511
1512 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo')
1513 with self.assertRaises(bb.fetch2.FetchError):
1514 self.fetch()
1515
1516 def test_shallow_missing_extra_refs_wildcard(self):
1517 self.add_empty_file('a')
1518 self.add_empty_file('b')
1519
1520 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
1521 self.fetch()
1522
1523 def test_shallow_remove_revs(self):
1524 # Create initial git repo
1525 self.add_empty_file('a')
1526 self.add_empty_file('b')
1527 self.git('checkout -b a_branch', cwd=self.srcdir)
1528 self.add_empty_file('c')
1529 self.add_empty_file('d')
1530 self.git('checkout master', cwd=self.srcdir)
1531 self.git('tag v0.0 a_branch', cwd=self.srcdir)
1532 self.add_empty_file('e')
1533 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1534 self.git('branch -d a_branch', cwd=self.srcdir)
1535 self.add_empty_file('f')
1536 self.assertRevCount(7, cwd=self.srcdir)
1537
1538 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1539 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1540
1541 self.fetch_shallow()
1542
1543 self.assertRevCount(5)
1544
1545 def test_shallow_invalid_revs(self):
1546 self.add_empty_file('a')
1547 self.add_empty_file('b')
1548
1549 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1550 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1551
1552 with self.assertRaises(bb.fetch2.FetchError):
1553 self.fetch()
1554
1555 @skipIfNoNetwork()
1556 def test_bitbake(self):
1557 self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
1558 self.git('config core.bare true', cwd=self.srcdir)
1559 self.git('fetch', cwd=self.srcdir)
1560
1561 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1562 # Note that the 1.10.0 tag is annotated, so this also tests
1563 # reference of an annotated vs unannotated tag
1564 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0')
1565
1566 self.fetch_shallow()
1567
1568 # Confirm that the history of 1.10.0 was removed
1569 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines())
1570 revs = len(self.git('rev-list master').splitlines())
1571 self.assertNotEqual(orig_revs, revs)
1572 self.assertRefs(['master', 'origin/master'])
1573 self.assertRevCount(orig_revs - 1758)