blob: 71ed382ab821972637841b29ab92d10730ddb500 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# BitBake Test for codeparser.py
3#
4# Copyright (C) 2010 Chris Larson
5# Copyright (C) 2012 Richard Purdie
6#
Brad Bishopc342db32019-05-15 21:57:59 -04007# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008#
9
10import unittest
11import logging
12import bb
13
14logger = logging.getLogger('BitBake.TestCodeParser')
15
16# bb.data references bb.parse but can't directly import due to circular dependencies.
17# Hack around it for now :(
18import bb.parse
19import bb.data
20
21class ReferenceTest(unittest.TestCase):
22 def setUp(self):
23 self.d = bb.data.init()
24
25 def setEmptyVars(self, varlist):
26 for k in varlist:
27 self.d.setVar(k, "")
28
29 def setValues(self, values):
30 for k, v in values.items():
31 self.d.setVar(k, v)
32
33 def assertReferences(self, refs):
34 self.assertEqual(self.references, refs)
35
36 def assertExecs(self, execs):
37 self.assertEqual(self.execs, execs)
38
Brad Bishop6e60e8b2018-02-01 10:27:11 -050039 def assertContains(self, contains):
40 self.assertEqual(self.contains, contains)
41
Patrick Williamsc124f4f2015-09-15 14:41:29 -050042class VariableReferenceTest(ReferenceTest):
43
44 def parseExpression(self, exp):
45 parsedvar = self.d.expandWithRefs(exp, None)
46 self.references = parsedvar.references
47
48 def test_simple_reference(self):
49 self.setEmptyVars(["FOO"])
50 self.parseExpression("${FOO}")
51 self.assertReferences(set(["FOO"]))
52
53 def test_nested_reference(self):
54 self.setEmptyVars(["BAR"])
55 self.d.setVar("FOO", "BAR")
56 self.parseExpression("${${FOO}}")
57 self.assertReferences(set(["FOO", "BAR"]))
58
59 def test_python_reference(self):
60 self.setEmptyVars(["BAR"])
Brad Bishop6e60e8b2018-02-01 10:27:11 -050061 self.parseExpression("${@d.getVar('BAR') + 'foo'}")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050062 self.assertReferences(set(["BAR"]))
63
64class ShellReferenceTest(ReferenceTest):
65
66 def parseExpression(self, exp):
67 parsedvar = self.d.expandWithRefs(exp, None)
68 parser = bb.codeparser.ShellParser("ParserTest", logger)
69 parser.parse_shell(parsedvar.value)
70
71 self.references = parsedvar.references
72 self.execs = parser.execs
73
74 def test_quotes_inside_assign(self):
75 self.parseExpression('foo=foo"bar"baz')
76 self.assertReferences(set([]))
77
78 def test_quotes_inside_arg(self):
79 self.parseExpression('sed s#"bar baz"#"alpha beta"#g')
80 self.assertExecs(set(["sed"]))
81
82 def test_arg_continuation(self):
83 self.parseExpression("sed -i -e s,foo,bar,g \\\n *.pc")
84 self.assertExecs(set(["sed"]))
85
86 def test_dollar_in_quoted(self):
87 self.parseExpression('sed -i -e "foo$" *.pc')
88 self.assertExecs(set(["sed"]))
89
90 def test_quotes_inside_arg_continuation(self):
91 self.setEmptyVars(["bindir", "D", "libdir"])
92 self.parseExpression("""
93sed -i -e s#"moc_location=.*$"#"moc_location=${bindir}/moc4"# \\
94-e s#"uic_location=.*$"#"uic_location=${bindir}/uic4"# \\
95${D}${libdir}/pkgconfig/*.pc
96""")
97 self.assertReferences(set(["bindir", "D", "libdir"]))
98
99 def test_assign_subshell_expansion(self):
100 self.parseExpression("foo=$(echo bar)")
101 self.assertExecs(set(["echo"]))
102
103 def test_shell_unexpanded(self):
104 self.setEmptyVars(["QT_BASE_NAME"])
105 self.parseExpression('echo "${QT_BASE_NAME}"')
106 self.assertExecs(set(["echo"]))
107 self.assertReferences(set(["QT_BASE_NAME"]))
108
109 def test_incomplete_varexp_single_quotes(self):
110 self.parseExpression("sed -i -e 's:IP{:I${:g' $pc")
111 self.assertExecs(set(["sed"]))
112
Brad Bishop19323692019-04-05 15:28:33 -0400113 def test_parameter_expansion_modifiers(self):
Patrick Williams213cb262021-08-07 19:21:33 -0500114 # -,+ and : are also valid modifiers for parameter expansion, but are
Brad Bishop19323692019-04-05 15:28:33 -0400115 # valid characters in bitbake variable names, so are not included here
Patrick Williams213cb262021-08-07 19:21:33 -0500116 for i in ('=', '?', '#', '%', '##', '%%'):
Brad Bishop19323692019-04-05 15:28:33 -0400117 name = "foo%sbar" % i
118 self.parseExpression("${%s}" % name)
119 self.assertNotIn(name, self.references)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500120
121 def test_until(self):
122 self.parseExpression("until false; do echo true; done")
123 self.assertExecs(set(["false", "echo"]))
124 self.assertReferences(set())
125
126 def test_case(self):
127 self.parseExpression("""
128case $foo in
129*)
130bar
131;;
132esac
133""")
134 self.assertExecs(set(["bar"]))
135 self.assertReferences(set())
136
137 def test_assign_exec(self):
138 self.parseExpression("a=b c='foo bar' alpha 1 2 3")
139 self.assertExecs(set(["alpha"]))
140
141 def test_redirect_to_file(self):
142 self.setEmptyVars(["foo"])
143 self.parseExpression("echo foo >${foo}/bar")
144 self.assertExecs(set(["echo"]))
145 self.assertReferences(set(["foo"]))
146
147 def test_heredoc(self):
148 self.setEmptyVars(["theta"])
149 self.parseExpression("""
150cat <<END
151alpha
152beta
153${theta}
154END
155""")
156 self.assertReferences(set(["theta"]))
157
158 def test_redirect_from_heredoc(self):
159 v = ["B", "SHADOW_MAILDIR", "SHADOW_MAILFILE", "SHADOW_UTMPDIR", "SHADOW_LOGDIR", "bindir"]
160 self.setEmptyVars(v)
161 self.parseExpression("""
162cat <<END >${B}/cachedpaths
163shadow_cv_maildir=${SHADOW_MAILDIR}
164shadow_cv_mailfile=${SHADOW_MAILFILE}
165shadow_cv_utmpdir=${SHADOW_UTMPDIR}
166shadow_cv_logdir=${SHADOW_LOGDIR}
167shadow_cv_passwd_dir=${bindir}
168END
169""")
170 self.assertReferences(set(v))
171 self.assertExecs(set(["cat"]))
172
173# def test_incomplete_command_expansion(self):
174# self.assertRaises(reftracker.ShellSyntaxError, reftracker.execs,
175# bbvalue.shparse("cp foo`", self.d), self.d)
176
177# def test_rogue_dollarsign(self):
178# self.setValues({"D" : "/tmp"})
179# self.parseExpression("install -d ${D}$")
180# self.assertReferences(set(["D"]))
181# self.assertExecs(set(["install"]))
182
183
184class PythonReferenceTest(ReferenceTest):
185
186 def setUp(self):
187 self.d = bb.data.init()
188 if hasattr(bb.utils, "_context"):
189 self.context = bb.utils._context
190 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600191 import builtins
192 self.context = builtins.__dict__
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500193
194 def parseExpression(self, exp):
195 parsedvar = self.d.expandWithRefs(exp, None)
196 parser = bb.codeparser.PythonParser("ParserTest", logger)
197 parser.parse_python(parsedvar.value)
198
199 self.references = parsedvar.references | parser.references
200 self.execs = parser.execs
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500201 self.contains = parser.contains
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500202
203 @staticmethod
204 def indent(value):
205 """Python Snippets have to be indented, python values don't have to
206be. These unit tests are testing snippets."""
207 return " " + value
208
209 def test_getvar_reference(self):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500210 self.parseExpression("d.getVar('foo')")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500211 self.assertReferences(set(["foo"]))
212 self.assertExecs(set())
213
214 def test_getvar_computed_reference(self):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500215 self.parseExpression("d.getVar('f' + 'o' + 'o')")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500216 self.assertReferences(set())
217 self.assertExecs(set())
218
219 def test_getvar_exec_reference(self):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500220 self.parseExpression("eval('d.getVar(\"foo\")')")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500221 self.assertReferences(set())
222 self.assertExecs(set(["eval"]))
223
224 def test_var_reference(self):
225 self.context["foo"] = lambda x: x
226 self.setEmptyVars(["FOO"])
227 self.parseExpression("foo('${FOO}')")
228 self.assertReferences(set(["FOO"]))
229 self.assertExecs(set(["foo"]))
230 del self.context["foo"]
231
232 def test_var_exec(self):
233 for etype in ("func", "task"):
234 self.d.setVar("do_something", "echo 'hi mom! ${FOO}'")
235 self.d.setVarFlag("do_something", etype, True)
236 self.parseExpression("bb.build.exec_func('do_something', d)")
237 self.assertReferences(set([]))
238 self.assertExecs(set(["do_something"]))
239
240 def test_function_reference(self):
241 self.context["testfunc"] = lambda msg: bb.msg.note(1, None, msg)
242 self.d.setVar("FOO", "Hello, World!")
243 self.parseExpression("testfunc('${FOO}')")
244 self.assertReferences(set(["FOO"]))
245 self.assertExecs(set(["testfunc"]))
246 del self.context["testfunc"]
247
248 def test_qualified_function_reference(self):
249 self.parseExpression("time.time()")
250 self.assertExecs(set(["time.time"]))
251
252 def test_qualified_function_reference_2(self):
253 self.parseExpression("os.path.dirname('/foo/bar')")
254 self.assertExecs(set(["os.path.dirname"]))
255
256 def test_qualified_function_reference_nested(self):
257 self.parseExpression("time.strftime('%Y%m%d',time.gmtime())")
258 self.assertExecs(set(["time.strftime", "time.gmtime"]))
259
260 def test_function_reference_chained(self):
261 self.context["testget"] = lambda: "\tstrip me "
262 self.parseExpression("testget().strip()")
263 self.assertExecs(set(["testget"]))
264 del self.context["testget"]
265
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500266 def test_contains(self):
267 self.parseExpression('bb.utils.contains("TESTVAR", "one", "true", "false", d)')
268 self.assertContains({'TESTVAR': {'one'}})
269
270 def test_contains_multi(self):
271 self.parseExpression('bb.utils.contains("TESTVAR", "one two", "true", "false", d)')
272 self.assertContains({'TESTVAR': {'one two'}})
273
274 def test_contains_any(self):
275 self.parseExpression('bb.utils.contains_any("TESTVAR", "hello", "true", "false", d)')
276 self.assertContains({'TESTVAR': {'hello'}})
277
278 def test_contains_any_multi(self):
279 self.parseExpression('bb.utils.contains_any("TESTVAR", "one two three", "true", "false", d)')
280 self.assertContains({'TESTVAR': {'one', 'two', 'three'}})
281
282 def test_contains_filter(self):
283 self.parseExpression('bb.utils.filter("TESTVAR", "hello there world", d)')
284 self.assertContains({'TESTVAR': {'hello', 'there', 'world'}})
285
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500286
287class DependencyReferenceTest(ReferenceTest):
288
289 pydata = """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500290d.getVar('somevar')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500291def test(d):
292 foo = 'bar %s' % 'foo'
293def test2(d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500294 d.getVar(foo)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500295 d.getVar('bar', False)
296 test2(d)
297
298def a():
299 \"\"\"some
300 stuff
301 \"\"\"
302 return "heh"
303
304test(d)
305
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500306d.expand(d.getVar("something", False))
307d.expand("${inexpand} somethingelse")
308d.getVar(a(), False)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500309"""
310
311 def test_python(self):
312 self.d.setVar("FOO", self.pydata)
313 self.setEmptyVars(["inexpand", "a", "test2", "test"])
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500314 self.d.setVarFlags("FOO", {
315 "func": True,
316 "python": True,
317 "lineno": 1,
318 "filename": "example.bb",
319 })
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320
Patrick Williamsde0582f2022-04-08 10:23:27 -0500321 deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500322
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600323 self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500324
325
326 shelldata = """
327foo () {
328bar
329}
330{
331echo baz
332$(heh)
333eval `moo`
334}
335a=b
336c=d
337(
338true && false
339test -f foo
340testval=something
341$testval
342) || aiee
343! inverted
344echo ${somevar}
345
346case foo in
347bar)
348echo bar
349;;
350baz)
351echo baz
352;;
353foo*)
354echo foo
355;;
356esac
357"""
358
359 def test_shell(self):
360 execs = ["bar", "echo", "heh", "moo", "true", "aiee"]
361 self.d.setVar("somevar", "heh")
362 self.d.setVar("inverted", "echo inverted...")
363 self.d.setVarFlag("inverted", "func", True)
364 self.d.setVar("FOO", self.shelldata)
365 self.d.setVarFlags("FOO", {"func": True})
366 self.setEmptyVars(execs)
367
Patrick Williamsde0582f2022-04-08 10:23:27 -0500368 deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500369
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600370 self.assertEqual(deps, set(["somevar", "inverted"] + execs))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500371
372
373 def test_vardeps(self):
374 self.d.setVar("oe_libinstall", "echo test")
375 self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
376 self.d.setVarFlag("FOO", "vardeps", "oe_libinstall")
377
Patrick Williamsde0582f2022-04-08 10:23:27 -0500378 deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500379
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600380 self.assertEqual(deps, set(["oe_libinstall"]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500381
382 def test_vardeps_expand(self):
383 self.d.setVar("oe_libinstall", "echo test")
384 self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
385 self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}")
386
Patrick Williamsde0582f2022-04-08 10:23:27 -0500387 deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500388
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600389 self.assertEqual(deps, set(["oe_libinstall"]))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500390
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500391 def test_contains_vardeps(self):
392 expr = '${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)} \
393 ${@bb.utils.contains("TESTVAR", "testval testval2", "yetanothervalue", "", d)} \
394 ${@bb.utils.contains("TESTVAR", "testval2 testval3", "blah", "", d)} \
395 ${@bb.utils.contains_any("TESTVAR", "testval2 testval3", "lastone", "", d)}'
396 parsedvar = self.d.expandWithRefs(expr, None)
397 # Check contains
398 self.assertEqual(parsedvar.contains, {'TESTVAR': {'testval2 testval3', 'anothervalue', 'somevalue', 'testval testval2', 'testval2', 'testval3'}})
399 # Check dependencies
400 self.d.setVar('ANOTHERVAR', expr)
401 self.d.setVar('TESTVAR', 'anothervalue testval testval2')
Patrick Williamsde0582f2022-04-08 10:23:27 -0500402 deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), self.d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500403 self.assertEqual(sorted(values.splitlines()),
404 sorted([expr,
405 'TESTVAR{anothervalue} = Set',
406 'TESTVAR{somevalue} = Unset',
407 'TESTVAR{testval testval2} = Set',
408 'TESTVAR{testval2 testval3} = Unset',
409 'TESTVAR{testval2} = Set',
410 'TESTVAR{testval3} = Unset'
411 ]))
412 # Check final value
413 self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['anothervalue', 'yetanothervalue', 'lastone'])
414
Patrick Williamsde0582f2022-04-08 10:23:27 -0500415 def test_contains_vardeps_excluded(self):
416 # Check the ignored_vars option to build_dependencies is handled by contains functionality
417 varval = '${TESTVAR2} ${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)}'
418 self.d.setVar('ANOTHERVAR', varval)
419 self.d.setVar('TESTVAR', 'anothervalue testval testval2')
420 self.d.setVar('TESTVAR2', 'testval3')
421 deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(["TESTVAR"]), self.d)
422 self.assertEqual(sorted(values.splitlines()), sorted([varval]))
423 self.assertEqual(deps, set(["TESTVAR2"]))
424 self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue'])
425
426 # Check the vardepsexclude flag is handled by contains functionality
427 self.d.setVarFlag('ANOTHERVAR', 'vardepsexclude', 'TESTVAR')
428 deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), self.d)
429 self.assertEqual(sorted(values.splitlines()), sorted([varval]))
430 self.assertEqual(deps, set(["TESTVAR2"]))
431 self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue'])
432
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500433 #Currently no wildcard support
434 #def test_vardeps_wildcards(self):
435 # self.d.setVar("oe_libinstall", "echo test")
436 # self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
437 # self.d.setVarFlag("FOO", "vardeps", "oe_*")
438 # self.assertEquals(deps, set(["oe_libinstall"]))
439
440