Gitiles
Code Review Sign In
gerrit.openbmc.org / openbmc / openbmc / c124f4f2e04dca16a428a76c89677328bc7bf908 / . / bitbake / lib / bb / tests / fetch.py
blob: 1e61f3a1168da9c6503fee586c6e595ddec851f5 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500[diff] [blame^]1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3#
4# BitBake Tests for the Fetcher (fetch2/)
5#
6# Copyright (C) 2012 Richard Purdie
7#
8# This program is free software; you can redistribute it and/or modify
9# it under the terms of the GNU General Public License version 2 as
10# published by the Free Software Foundation.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License along
18# with this program; if not, write to the Free Software Foundation, Inc.,
19# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20#
21
22import unittest
23import tempfile
24import subprocess
25import os
26from bb.fetch2 import URI
27from bb.fetch2 import FetchMethod
28import bb
29
30class URITest(unittest.TestCase):
31 test_uris = {
32 "http://www.google.com/index.html" : {
33 'uri': 'http://www.google.com/index.html',
34 'scheme': 'http',
35 'hostname': 'www.google.com',
36 'port': None,
37 'hostport': 'www.google.com',
38 'path': '/index.html',
39 'userinfo': '',
40 'username': '',
41 'password': '',
42 'params': {},
43 'query': {},
44 'relative': False
45 },
46 "http://www.google.com/index.html;param1=value1" : {
47 'uri': 'http://www.google.com/index.html;param1=value1',
48 'scheme': 'http',
49 'hostname': 'www.google.com',
50 'port': None,
51 'hostport': 'www.google.com',
52 'path': '/index.html',
53 'userinfo': '',
54 'username': '',
55 'password': '',
56 'params': {
57 'param1': 'value1'
58 },
59 'query': {},
60 'relative': False
61 },
62 "http://www.example.org/index.html?param1=value1" : {
63 'uri': 'http://www.example.org/index.html?param1=value1',
64 'scheme': 'http',
65 'hostname': 'www.example.org',
66 'port': None,
67 'hostport': 'www.example.org',
68 'path': '/index.html',
69 'userinfo': '',
70 'username': '',
71 'password': '',
72 'params': {},
73 'query': {
74 'param1': 'value1'
75 },
76 'relative': False
77 },
78 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : {
79 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2',
80 'scheme': 'http',
81 'hostname': 'www.example.org',
82 'port': None,
83 'hostport': 'www.example.org',
84 'path': '/index.html',
85 'userinfo': '',
86 'username': '',
87 'password': '',
88 'params': {
89 'param2': 'value2'
90 },
91 'query': {
92 'qparam1': 'qvalue1'
93 },
94 'relative': False
95 },
96 "http://www.example.com:8080/index.html" : {
97 'uri': 'http://www.example.com:8080/index.html',
98 'scheme': 'http',
99 'hostname': 'www.example.com',
100 'port': 8080,
101 'hostport': 'www.example.com:8080',
102 'path': '/index.html',
103 'userinfo': '',
104 'username': '',
105 'password': '',
106 'params': {},
107 'query': {},
108 'relative': False
109 },
110 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : {
111 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg',
112 'scheme': 'cvs',
113 'hostname': 'cvs.handhelds.org',
114 'port': None,
115 'hostport': 'cvs.handhelds.org',
116 'path': '/cvs',
117 'userinfo': 'anoncvs',
118 'username': 'anoncvs',
119 'password': '',
120 'params': {
121 'module': 'familiar/dist/ipkg'
122 },
123 'query': {},
124 'relative': False
125 },
126 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": {
127 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg',
128 'scheme': 'cvs',
129 'hostname': 'cvs.handhelds.org',
130 'port': None,
131 'hostport': 'cvs.handhelds.org',
132 'path': '/cvs',
133 'userinfo': 'anoncvs:anonymous',
134 'username': 'anoncvs',
135 'password': 'anonymous',
136 'params': {
137 'tag': 'V0-99-81',
138 'module': 'familiar/dist/ipkg'
139 },
140 'query': {},
141 'relative': False
142 },
143 "file://example.diff": { # NOTE: Not RFC compliant!
144 'uri': 'file:example.diff',
145 'scheme': 'file',
146 'hostname': '',
147 'port': None,
148 'hostport': '',
149 'path': 'example.diff',
150 'userinfo': '',
151 'username': '',
152 'password': '',
153 'params': {},
154 'query': {},
155 'relative': True
156 },
157 "file:example.diff": { # NOTE: RFC compliant version of the former
158 'uri': 'file:example.diff',
159 'scheme': 'file',
160 'hostname': '',
161 'port': None,
162 'hostport': '',
163 'path': 'example.diff',
164 'userinfo': '',
165 'userinfo': '',
166 'username': '',
167 'password': '',
168 'params': {},
169 'query': {},
170 'relative': True
171 },
172 "file:///tmp/example.diff": {
173 'uri': 'file:///tmp/example.diff',
174 'scheme': 'file',
175 'hostname': '',
176 'port': None,
177 'hostport': '',
178 'path': '/tmp/example.diff',
179 'userinfo': '',
180 'userinfo': '',
181 'username': '',
182 'password': '',
183 'params': {},
184 'query': {},
185 'relative': False
186 },
187 "git:///path/example.git": {
188 'uri': 'git:///path/example.git',
189 'scheme': 'git',
190 'hostname': '',
191 'port': None,
192 'hostport': '',
193 'path': '/path/example.git',
194 'userinfo': '',
195 'userinfo': '',
196 'username': '',
197 'password': '',
198 'params': {},
199 'query': {},
200 'relative': False
201 },
202 "git:path/example.git": {
203 'uri': 'git:path/example.git',
204 'scheme': 'git',
205 'hostname': '',
206 'port': None,
207 'hostport': '',
208 'path': 'path/example.git',
209 'userinfo': '',
210 'userinfo': '',
211 'username': '',
212 'password': '',
213 'params': {},
214 'query': {},
215 'relative': True
216 },
217 "git://example.net/path/example.git": {
218 'uri': 'git://example.net/path/example.git',
219 'scheme': 'git',
220 'hostname': 'example.net',
221 'port': None,
222 'hostport': 'example.net',
223 'path': '/path/example.git',
224 'userinfo': '',
225 'userinfo': '',
226 'username': '',
227 'password': '',
228 'params': {},
229 'query': {},
230 'relative': False
231 }
232 }
233
234 def test_uri(self):
235 for test_uri, ref in self.test_uris.items():
236 uri = URI(test_uri)
237
238 self.assertEqual(str(uri), ref['uri'])
239
240 # expected attributes
241 self.assertEqual(uri.scheme, ref['scheme'])
242
243 self.assertEqual(uri.userinfo, ref['userinfo'])
244 self.assertEqual(uri.username, ref['username'])
245 self.assertEqual(uri.password, ref['password'])
246
247 self.assertEqual(uri.hostname, ref['hostname'])
248 self.assertEqual(uri.port, ref['port'])
249 self.assertEqual(uri.hostport, ref['hostport'])
250
251 self.assertEqual(uri.path, ref['path'])
252 self.assertEqual(uri.params, ref['params'])
253
254 self.assertEqual(uri.relative, ref['relative'])
255
256 def test_dict(self):
257 for test in self.test_uris.values():
258 uri = URI()
259
260 self.assertEqual(uri.scheme, '')
261 self.assertEqual(uri.userinfo, '')
262 self.assertEqual(uri.username, '')
263 self.assertEqual(uri.password, '')
264 self.assertEqual(uri.hostname, '')
265 self.assertEqual(uri.port, None)
266 self.assertEqual(uri.path, '')
267 self.assertEqual(uri.params, {})
268
269
270 uri.scheme = test['scheme']
271 self.assertEqual(uri.scheme, test['scheme'])
272
273 uri.userinfo = test['userinfo']
274 self.assertEqual(uri.userinfo, test['userinfo'])
275 self.assertEqual(uri.username, test['username'])
276 self.assertEqual(uri.password, test['password'])
277
278 # make sure changing the values doesn't do anything unexpected
279 uri.username = 'changeme'
280 self.assertEqual(uri.username, 'changeme')
281 self.assertEqual(uri.password, test['password'])
282 uri.password = 'insecure'
283 self.assertEqual(uri.username, 'changeme')
284 self.assertEqual(uri.password, 'insecure')
285
286 # reset back after our trickery
287 uri.userinfo = test['userinfo']
288 self.assertEqual(uri.userinfo, test['userinfo'])
289 self.assertEqual(uri.username, test['username'])
290 self.assertEqual(uri.password, test['password'])
291
292 uri.hostname = test['hostname']
293 self.assertEqual(uri.hostname, test['hostname'])
294 self.assertEqual(uri.hostport, test['hostname'])
295
296 uri.port = test['port']
297 self.assertEqual(uri.port, test['port'])
298 self.assertEqual(uri.hostport, test['hostport'])
299
300 uri.path = test['path']
301 self.assertEqual(uri.path, test['path'])
302
303 uri.params = test['params']
304 self.assertEqual(uri.params, test['params'])
305
306 uri.query = test['query']
307 self.assertEqual(uri.query, test['query'])
308
309 self.assertEqual(str(uri), test['uri'])
310
311 uri.params = {}
312 self.assertEqual(uri.params, {})
313 self.assertEqual(str(uri), (str(uri).split(";"))[0])
314
315class FetcherTest(unittest.TestCase):
316
317 def setUp(self):
318 self.origdir = os.getcwd()
319 self.d = bb.data.init()
320 self.tempdir = tempfile.mkdtemp()
321 self.dldir = os.path.join(self.tempdir, "download")
322 os.mkdir(self.dldir)
323 self.d.setVar("DL_DIR", self.dldir)
324 self.unpackdir = os.path.join(self.tempdir, "unpacked")
325 os.mkdir(self.unpackdir)
326 persistdir = os.path.join(self.tempdir, "persistdata")
327 self.d.setVar("PERSISTENT_DIR", persistdir)
328
329 def tearDown(self):
330 os.chdir(self.origdir)
331 bb.utils.prunedir(self.tempdir)
332
333class MirrorUriTest(FetcherTest):
334
335 replaceuris = {
336 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/")
337 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
338 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
339 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
340 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
341 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
342 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http")
343 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
344 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake")
345 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890",
346 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache")
347 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
348 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/")
349 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
350 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3")
351 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
352 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz")
353 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
354 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist")
355 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2",
356 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/")
357 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2",
358 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
359 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
360 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
361 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
362 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http")
363 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http",
364
365 #Renaming files doesn't work
366 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
367 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
368 }
369
370 mirrorvar = "http://.*/.* file:///somepath/downloads/ \n" \
371 "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n" \
372 "https://.*/.* file:///someotherpath/downloads/ \n" \
373 "http://.*/.* file:///someotherpath/downloads/ \n"
374
375 def test_urireplace(self):
376 for k, v in self.replaceuris.items():
377 ud = bb.fetch.FetchData(k[0], self.d)
378 ud.setup_localpath(self.d)
379 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2]))
380 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d)
381 self.assertEqual([v], newuris)
382
383 def test_urilist1(self):
384 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
385 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
386 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
387 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
388
389 def test_urilist2(self):
390 # Catch https:// -> files:// bug
391 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
392 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
393 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
394 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
395
396 def test_mirror_of_mirror(self):
397 # Test if mirror of a mirror works
398 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/ \n"
399 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/ \n"
400 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
401 mirrors = bb.fetch2.mirror_from_string(mirrorvar)
402 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
403 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz',
404 'file:///someotherpath/downloads/bitbake-1.0.tar.gz',
405 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz',
406 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz'])
407
408
409class FetcherLocalTest(FetcherTest):
410 def setUp(self):
411 def touch(fn):
412 with file(fn, 'a'):
413 os.utime(fn, None)
414
415 super(FetcherLocalTest, self).setUp()
416 self.localsrcdir = os.path.join(self.tempdir, 'localsrc')
417 os.makedirs(self.localsrcdir)
418 touch(os.path.join(self.localsrcdir, 'a'))
419 touch(os.path.join(self.localsrcdir, 'b'))
420 os.makedirs(os.path.join(self.localsrcdir, 'dir'))
421 touch(os.path.join(self.localsrcdir, 'dir', 'c'))
422 touch(os.path.join(self.localsrcdir, 'dir', 'd'))
423 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir'))
424 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e'))
425 self.d.setVar("FILESPATH", self.localsrcdir)
426
427 def fetchUnpack(self, uris):
428 fetcher = bb.fetch.Fetch(uris, self.d)
429 fetcher.download()
430 fetcher.unpack(self.unpackdir)
431 flst = []
432 for root, dirs, files in os.walk(self.unpackdir):
433 for f in files:
434 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir))
435 flst.sort()
436 return flst
437
438 def test_local(self):
439 tree = self.fetchUnpack(['file://a', 'file://dir/c'])
440 self.assertEqual(tree, ['a', 'dir/c'])
441
442 def test_local_wildcard(self):
443 tree = self.fetchUnpack(['file://a', 'file://dir/*'])
444 # FIXME: this is broken - it should return ['a', 'dir/c', 'dir/d', 'dir/subdir/e']
445 # see https://bugzilla.yoctoproject.org/show_bug.cgi?id=6128
446 self.assertEqual(tree, ['a', 'b', 'dir/c', 'dir/d', 'dir/subdir/e'])
447
448 def test_local_dir(self):
449 tree = self.fetchUnpack(['file://a', 'file://dir'])
450 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e'])
451
452 def test_local_subdir(self):
453 tree = self.fetchUnpack(['file://dir/subdir'])
454 # FIXME: this is broken - it should return ['dir/subdir/e']
455 # see https://bugzilla.yoctoproject.org/show_bug.cgi?id=6129
456 self.assertEqual(tree, ['subdir/e'])
457
458 def test_local_subdir_file(self):
459 tree = self.fetchUnpack(['file://dir/subdir/e'])
460 self.assertEqual(tree, ['dir/subdir/e'])
461
462 def test_local_subdirparam(self):
463 tree = self.fetchUnpack(['file://a;subdir=bar'])
464 self.assertEqual(tree, ['bar/a'])
465
466 def test_local_deepsubdirparam(self):
467 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar'])
468 self.assertEqual(tree, ['bar/dir/subdir/e'])
469
470class FetcherNetworkTest(FetcherTest):
471
472 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
473 print("Unset BB_SKIP_NETTESTS to run network tests")
474 else:
475 def test_fetch(self):
476 fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
477 fetcher.download()
478 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
479 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892)
480 self.d.setVar("BB_NO_NETWORK", "1")
481 fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
482 fetcher.download()
483 fetcher.unpack(self.unpackdir)
484 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9)
485 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9)
486
487 def test_fetch_mirror(self):
488 self.d.setVar("MIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
489 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
490 fetcher.download()
491 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
492
493 def test_fetch_mirror_of_mirror(self):
494 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ \n http://invalid2.yoctoproject.org/.* http://downloads.yoctoproject.org/releases/bitbake")
495 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
496 fetcher.download()
497 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
498
499 def test_fetch_file_mirror_of_mirror(self):
500 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ \n file:///some1where/.* file://some2where/ \n file://some2where/.* http://downloads.yoctoproject.org/releases/bitbake")
501 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
502 os.mkdir(self.dldir + "/some2where")
503 fetcher.download()
504 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
505
506 def test_fetch_premirror(self):
507 self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
508 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
509 fetcher.download()
510 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
511
512 def gitfetcher(self, url1, url2):
513 def checkrevision(self, fetcher):
514 fetcher.unpack(self.unpackdir)
515 revision = bb.process.run("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git")[0].strip()
516 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
517
518 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
519 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
520 fetcher = bb.fetch.Fetch([url1], self.d)
521 fetcher.download()
522 checkrevision(self, fetcher)
523 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works
524 bb.utils.prunedir(self.dldir + "/git2/")
525 bb.utils.prunedir(self.unpackdir)
526 self.d.setVar("BB_NO_NETWORK", "1")
527 fetcher = bb.fetch.Fetch([url2], self.d)
528 fetcher.download()
529 checkrevision(self, fetcher)
530
531 def test_gitfetch(self):
532 url1 = url2 = "git://git.openembedded.org/bitbake"
533 self.gitfetcher(url1, url2)
534
535 def test_gitfetch_goodsrcrev(self):
536 # SRCREV is set but matches rev= parameter
537 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
538 self.gitfetcher(url1, url2)
539
540 def test_gitfetch_badsrcrev(self):
541 # SRCREV is set but does not match rev= parameter
542 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5"
543 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
544
545 def test_gitfetch_tagandrev(self):
546 # SRCREV is set but does not match rev= parameter
547 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
548 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
549
550 def test_gitfetch_premirror(self):
551 url1 = "git://git.openembedded.org/bitbake"
552 url2 = "git://someserver.org/bitbake"
553 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
554 self.gitfetcher(url1, url2)
555
556 def test_gitfetch_premirror2(self):
557 url1 = url2 = "git://someserver.org/bitbake"
558 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
559 self.gitfetcher(url1, url2)
560
561 def test_gitfetch_premirror3(self):
562 realurl = "git://git.openembedded.org/bitbake"
563 dummyurl = "git://someserver.org/bitbake"
564 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git")
565 os.chdir(self.tempdir)
566 bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True)
567 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (dummyurl, self.sourcedir))
568 self.gitfetcher(dummyurl, dummyurl)
569
570 def test_git_submodule(self):
571 fetcher = bb.fetch.Fetch(["gitsm://git.yoctoproject.org/git-submodule-test;rev=f12e57f2edf0aa534cf1616fa983d165a92b0842"], self.d)
572 fetcher.download()
573 # Previous cwd has been deleted
574 os.chdir(os.path.dirname(self.unpackdir))
575 fetcher.unpack(self.unpackdir)
576
577 def test_trusted_network(self):
578 # Ensure trusted_network returns False when the host IS in the list.
579 url = "git://Someserver.org/foo;rev=1"
580 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org")
581 self.assertTrue(bb.fetch.trusted_network(self.d, url))
582
583 def test_wild_trusted_network(self):
584 # Ensure trusted_network returns true when the *.host IS in the list.
585 url = "git://Someserver.org/foo;rev=1"
586 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
587 self.assertTrue(bb.fetch.trusted_network(self.d, url))
588
589 def test_prefix_wild_trusted_network(self):
590 # Ensure trusted_network returns true when the prefix matches *.host.
591 url = "git://git.Someserver.org/foo;rev=1"
592 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
593 self.assertTrue(bb.fetch.trusted_network(self.d, url))
594
595 def test_two_prefix_wild_trusted_network(self):
596 # Ensure trusted_network returns true when the prefix matches *.host.
597 url = "git://something.git.Someserver.org/foo;rev=1"
598 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
599 self.assertTrue(bb.fetch.trusted_network(self.d, url))
600
601 def test_untrusted_network(self):
602 # Ensure trusted_network returns False when the host is NOT in the list.
603 url = "git://someserver.org/foo;rev=1"
604 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
605 self.assertFalse(bb.fetch.trusted_network(self.d, url))
606
607 def test_wild_untrusted_network(self):
608 # Ensure trusted_network returns False when the host is NOT in the list.
609 url = "git://*.someserver.org/foo;rev=1"
610 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
611 self.assertFalse(bb.fetch.trusted_network(self.d, url))
612
613
614class URLHandle(unittest.TestCase):
615
616 datatable = {
617 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}),
618 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}),
619 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', {'tag': 'V0-99-81', 'module': 'familiar/dist/ipkg'}),
620 "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'})
621 }
622
623 def test_decodeurl(self):
624 for k, v in self.datatable.items():
625 result = bb.fetch.decodeurl(k)
626 self.assertEqual(result, v)
627
628 def test_encodeurl(self):
629 for k, v in self.datatable.items():
630 result = bb.fetch.encodeurl(v)
631 self.assertEqual(result, k)
632
633class FetchLatestVersionTest(FetcherTest):
634
635 test_git_uris = {
636 # version pattern "X.Y.Z"
637 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
638 : "1.99.4",
639 # version pattern "vX.Y"
640 ("mtd-utils", "git://git.infradead.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "")
641 : "1.5.0",
642 # version pattern "pkg_name-X.Y"
643 ("presentproto", "git://anongit.freedesktop.org/git/xorg/proto/presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")
644 : "1.0",
645 # version pattern "pkg_name-vX.Y.Z"
646 ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
647 : "1.4.0",
648 # combination version pattern
649 ("sysprof", "git://git.gnome.org/sysprof", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
650 : "1.2.0",
651 ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "")
652 : "2014.01",
653 # version pattern "yyyymmdd"
654 ("mobile-broadband-provider-info", "git://git.gnome.org/mobile-broadband-provider-info", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
655 : "20120614",
656 # packages with a valid GITTAGREGEX
657 ("xf86-video-omap", "git://anongit.freedesktop.org/xorg/driver/xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))")
658 : "0.4.3",
659 ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", "(?P<pver>(([0-9][\.|_]?)+[0-9]))")
660 : "11.0.0",
661 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", "chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
662 : "1.3.59",
663 ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", "(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
664 : "3.82+dbg0.9",
665 }
666
667 test_wget_uris = {
668 # packages with versions inside directory name
669 ("util-linux", "http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "")
670 : "2.24.2",
671 ("enchant", "http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "")
672 : "1.6.0",
673 ("cmake", "http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz", "", "")
674 : "2.8.12.1",
675 # packages with versions only in current directory
676 ("eglic", "http://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "")
677 : "2.19",
678 ("gnu-config", "http://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "")
679 : "20120814",
680 # packages with "99" in the name of possible version
681 ("pulseaudio", "http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "")
682 : "5.0",
683 ("xserver-xorg", "http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "")
684 : "1.15.1",
685 # packages with valid REGEX_URI and REGEX
686 ("cups", "http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2", "http://www.cups.org/software.php", "(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz")
687 : "2.0.0",
688 ("db", "http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz", "http://www.oracle.com/technetwork/products/berkeleydb/downloads/index-082944.html", "http://download.oracle.com/otn/berkeley-db/(?P<name>db-)(?P<pver>((\d+[\.\-_]*)+))\.tar\.gz")
689 : "6.1.19",
690 }
691 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
692 print("Unset BB_SKIP_NETTESTS to run network tests")
693 else:
694 def test_git_latest_versionstring(self):
695 for k, v in self.test_git_uris.items():
696 self.d.setVar("PN", k[0])
697 self.d.setVar("SRCREV", k[2])
698 self.d.setVar("GITTAGREGEX", k[3])
699 ud = bb.fetch2.FetchData(k[1], self.d)
700 pupver= ud.method.latest_versionstring(ud, self.d)
701 verstring = pupver[0]
702 r = bb.utils.vercmp_string(v, verstring)
703 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
704
705 def test_wget_latest_versionstring(self):
706 for k, v in self.test_wget_uris.items():
707 self.d.setVar("PN", k[0])
708 self.d.setVar("REGEX_URI", k[2])
709 self.d.setVar("REGEX", k[3])
710 ud = bb.fetch2.FetchData(k[1], self.d)
711 pupver = ud.method.latest_versionstring(ud, self.d)
712 verstring = pupver[0]
713 r = bb.utils.vercmp_string(v, verstring)
714 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
715
716
717class FetchCheckStatusTest(FetcherTest):
718 test_wget_uris = ["http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2",
719 "http://www.cups.org/software/ipptool/ipptool-20130731-linux-ubuntu-i686.tar.gz",
720 "http://www.cups.org/",
721 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
722 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
723 "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
724 "https://yoctoproject.org/",
725 "https://yoctoproject.org/documentation",
726 "http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
727 "http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
728 "ftp://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz",
729 "ftp://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz",
730 "ftp://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz",
731 ]
732
733 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
734 print("Unset BB_SKIP_NETTESTS to run network tests")
735 else:
736
737 def test_wget_checkstatus(self):
738 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d)
739 for u in self.test_wget_uris:
740 ud = fetch.ud[u]
741 m = ud.method
742 ret = m.checkstatus(fetch, ud, self.d)
743 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
744
745
746 def test_wget_checkstatus_connection_cache(self):
747 from bb.fetch2 import FetchConnectionCache
748
749 connection_cache = FetchConnectionCache()
750 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d,
751 connection_cache = connection_cache)
752
753 for u in self.test_wget_uris:
754 ud = fetch.ud[u]
755 m = ud.method
756 ret = m.checkstatus(fetch, ud, self.d)
757 self.assertTrue(ret, msg="URI %s, can't check status" % (u))
758
759 connection_cache.close_connections()