blob: aaa61dcb4e5114f7a247e2328997c63436dab701 [file] [log] [blame]
Andrew Jeffery89b64b62020-03-13 12:15:48 +10301#!/usr/bin/env python3
Matthew Barthccb7f852016-11-23 17:43:02 -06002
3"""
4This script determines the given package's openbmc dependencies from its
5configure.ac file where it downloads, configures, builds, and installs each of
6these dependencies. Then the given package is configured, built, and installed
7prior to executing its unit tests.
8"""
9
Matthew Barthd1810372016-12-19 16:57:21 -060010from git import Repo
William A. Kennington III3992d102021-05-17 01:41:04 -070011# interpreter is not used directly but this resolves dependency ordering
12# that would be broken if we didn't include it.
13from mesonbuild import interpreter
William A. Kennington IIIfcd70772020-06-04 00:50:23 -070014from mesonbuild import coredata, optinterpreter
Patrick Williamse95626d2021-04-13 11:13:53 -050015from mesonbuild.mesonlib import OptionKey
Patrick Williams95095f12021-04-14 14:42:35 -050016from mesonbuild.mesonlib import version_compare as meson_version_compare
Andrew Jeffery89b64b62020-03-13 12:15:48 +103017from urllib.parse import urljoin
Andrew Jefferya4e31c62018-03-08 13:45:28 +103018from subprocess import check_call, call, CalledProcessError
Matthew Barthccb7f852016-11-23 17:43:02 -060019import os
20import sys
Matthew Barth33df8792016-12-19 14:30:17 -060021import argparse
William A. Kennington IIIa2156732018-06-30 18:38:09 -070022import multiprocessing
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -050023import re
William A. Kennington IIIe67f5fc2018-12-06 17:40:30 -080024import subprocess
William A. Kennington III3f1d1202018-12-06 18:02:07 -080025import shutil
William A. Kennington III4e1d0a12018-07-16 12:04:03 -070026import platform
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -050027
28
29class DepTree():
30 """
31 Represents package dependency tree, where each node is a DepTree with a
32 name and DepTree children.
33 """
34
35 def __init__(self, name):
36 """
37 Create new DepTree.
38
39 Parameter descriptions:
40 name Name of new tree node.
41 """
42 self.name = name
43 self.children = list()
44
45 def AddChild(self, name):
46 """
47 Add new child node to current node.
48
49 Parameter descriptions:
50 name Name of new child
51 """
52 new_child = DepTree(name)
53 self.children.append(new_child)
54 return new_child
55
56 def AddChildNode(self, node):
57 """
58 Add existing child node to current node.
59
60 Parameter descriptions:
61 node Tree node to add
62 """
63 self.children.append(node)
64
65 def RemoveChild(self, name):
66 """
67 Remove child node.
68
69 Parameter descriptions:
70 name Name of child to remove
71 """
72 for child in self.children:
73 if child.name == name:
74 self.children.remove(child)
75 return
76
77 def GetNode(self, name):
78 """
79 Return node with matching name. Return None if not found.
80
81 Parameter descriptions:
82 name Name of node to return
83 """
84 if self.name == name:
85 return self
86 for child in self.children:
87 node = child.GetNode(name)
88 if node:
89 return node
90 return None
91
92 def GetParentNode(self, name, parent_node=None):
93 """
94 Return parent of node with matching name. Return none if not found.
95
96 Parameter descriptions:
97 name Name of node to get parent of
98 parent_node Parent of current node
99 """
100 if self.name == name:
101 return parent_node
102 for child in self.children:
103 found_node = child.GetParentNode(name, self)
104 if found_node:
105 return found_node
106 return None
107
108 def GetPath(self, name, path=None):
109 """
110 Return list of node names from head to matching name.
111 Return None if not found.
112
113 Parameter descriptions:
114 name Name of node
115 path List of node names from head to current node
116 """
117 if not path:
118 path = []
119 if self.name == name:
120 path.append(self.name)
121 return path
122 for child in self.children:
123 match = child.GetPath(name, path + [self.name])
124 if match:
125 return match
126 return None
127
128 def GetPathRegex(self, name, regex_str, path=None):
129 """
130 Return list of node paths that end in name, or match regex_str.
131 Return empty list if not found.
132
133 Parameter descriptions:
134 name Name of node to search for
135 regex_str Regex string to match node names
136 path Path of node names from head to current node
137 """
138 new_paths = []
139 if not path:
140 path = []
141 match = re.match(regex_str, self.name)
142 if (self.name == name) or (match):
143 new_paths.append(path + [self.name])
144 for child in self.children:
145 return_paths = None
146 full_path = path + [self.name]
147 return_paths = child.GetPathRegex(name, regex_str, full_path)
148 for i in return_paths:
149 new_paths.append(i)
150 return new_paths
151
152 def MoveNode(self, from_name, to_name):
153 """
154 Mode existing from_name node to become child of to_name node.
155
156 Parameter descriptions:
157 from_name Name of node to make a child of to_name
158 to_name Name of node to make parent of from_name
159 """
160 parent_from_node = self.GetParentNode(from_name)
161 from_node = self.GetNode(from_name)
162 parent_from_node.RemoveChild(from_name)
163 to_node = self.GetNode(to_name)
164 to_node.AddChildNode(from_node)
165
166 def ReorderDeps(self, name, regex_str):
167 """
168 Reorder dependency tree. If tree contains nodes with names that
169 match 'name' and 'regex_str', move 'regex_str' nodes that are
170 to the right of 'name' node, so that they become children of the
171 'name' node.
172
173 Parameter descriptions:
174 name Name of node to look for
175 regex_str Regex string to match names to
176 """
177 name_path = self.GetPath(name)
178 if not name_path:
179 return
180 paths = self.GetPathRegex(name, regex_str)
181 is_name_in_paths = False
182 name_index = 0
183 for i in range(len(paths)):
184 path = paths[i]
185 if path[-1] == name:
186 is_name_in_paths = True
187 name_index = i
188 break
189 if not is_name_in_paths:
190 return
191 for i in range(name_index + 1, len(paths)):
192 path = paths[i]
193 if name in path:
194 continue
195 from_name = path[-1]
196 self.MoveNode(from_name, name)
197
198 def GetInstallList(self):
199 """
200 Return post-order list of node names.
201
202 Parameter descriptions:
203 """
204 install_list = []
205 for child in self.children:
206 child_install_list = child.GetInstallList()
207 install_list.extend(child_install_list)
208 install_list.append(self.name)
209 return install_list
210
211 def PrintTree(self, level=0):
212 """
213 Print pre-order node names with indentation denoting node depth level.
214
215 Parameter descriptions:
216 level Current depth level
217 """
218 INDENT_PER_LEVEL = 4
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030219 print(' ' * (level * INDENT_PER_LEVEL) + self.name)
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500220 for child in self.children:
221 child.PrintTree(level + 1)
Matthew Barth33df8792016-12-19 14:30:17 -0600222
223
William A. Kennington III1fddb972019-02-06 18:03:53 -0800224def check_call_cmd(*cmd):
Matthew Barth33df8792016-12-19 14:30:17 -0600225 """
226 Verbose prints the directory location the given command is called from and
227 the command, then executes the command using check_call.
228
229 Parameter descriptions:
230 dir Directory location command is to be called from
231 cmd List of parameters constructing the complete command
232 """
William A. Kennington III1fddb972019-02-06 18:03:53 -0800233 printline(os.getcwd(), ">", " ".join(cmd))
Matthew Barth33df8792016-12-19 14:30:17 -0600234 check_call(cmd)
Matthew Barthccb7f852016-11-23 17:43:02 -0600235
236
Andrew Geisslera61acb52019-01-03 16:32:44 -0600237def clone_pkg(pkg, branch):
Matthew Barth33df8792016-12-19 14:30:17 -0600238 """
239 Clone the given openbmc package's git repository from gerrit into
240 the WORKSPACE location
241
242 Parameter descriptions:
243 pkg Name of the package to clone
Andrew Geisslera61acb52019-01-03 16:32:44 -0600244 branch Branch to clone from pkg
Matthew Barth33df8792016-12-19 14:30:17 -0600245 """
Andrew Jeffery7be94ca2018-03-08 13:15:33 +1030246 pkg_dir = os.path.join(WORKSPACE, pkg)
247 if os.path.exists(os.path.join(pkg_dir, '.git')):
248 return pkg_dir
Andrew Geissler37052412022-05-20 15:28:26 -0400249 pkg_repo = urljoin('https://gerrit.openbmc.org/openbmc/', pkg)
Andrew Jeffery7be94ca2018-03-08 13:15:33 +1030250 os.mkdir(pkg_dir)
Andrew Geisslera61acb52019-01-03 16:32:44 -0600251 printline(pkg_dir, "> git clone", pkg_repo, branch, "./")
252 try:
253 # first try the branch
Andrew Jeffery7d4a26f2020-03-13 11:42:34 +1030254 clone = Repo.clone_from(pkg_repo, pkg_dir, branch=branch)
255 repo_inst = clone.working_dir
Andrew Geisslera61acb52019-01-03 16:32:44 -0600256 except:
257 printline("Input branch not found, default to master")
Andrew Jeffery7d4a26f2020-03-13 11:42:34 +1030258 clone = Repo.clone_from(pkg_repo, pkg_dir, branch="master")
259 repo_inst = clone.working_dir
Andrew Geisslera61acb52019-01-03 16:32:44 -0600260 return repo_inst
Matthew Barth33df8792016-12-19 14:30:17 -0600261
262
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030263def make_target_exists(target):
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800264 """
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030265 Runs a check against the makefile in the current directory to determine
266 if the target exists so that it can be built.
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800267
268 Parameter descriptions:
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030269 target The make target we are checking
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800270 """
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030271 try:
Andrew Jeffery881041f2020-03-13 11:46:30 +1030272 cmd = ['make', '-n', target]
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030273 with open(os.devnull, 'w') as devnull:
274 check_call(cmd, stdout=devnull, stderr=devnull)
275 return True
276 except CalledProcessError:
277 return False
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800278
William A. Kennington III3f1d1202018-12-06 18:02:07 -0800279
William A. Kennington IIIa2156732018-06-30 18:38:09 -0700280make_parallel = [
281 'make',
282 # Run enough jobs to saturate all the cpus
283 '-j', str(multiprocessing.cpu_count()),
284 # Don't start more jobs if the load avg is too high
285 '-l', str(multiprocessing.cpu_count()),
286 # Synchronize the output so logs aren't intermixed in stdout / stderr
287 '-O',
288]
289
William A. Kennington III3f1d1202018-12-06 18:02:07 -0800290
Andrew Jefferyff5c5d52020-03-13 10:12:14 +1030291def build_and_install(name, build_for_testing=False):
William A. Kennington III780ec092018-12-06 14:46:50 -0800292 """
293 Builds and installs the package in the environment. Optionally
294 builds the examples and test cases for package.
295
296 Parameter description:
Andrew Jefferyff5c5d52020-03-13 10:12:14 +1030297 name The name of the package we are building
William A. Kennington IIIa0454912018-12-06 14:47:16 -0800298 build_for_testing Enable options related to testing on the package?
William A. Kennington III780ec092018-12-06 14:46:50 -0800299 """
Andrew Jefferyff5c5d52020-03-13 10:12:14 +1030300 os.chdir(os.path.join(WORKSPACE, name))
William A. Kennington III54d4faf2018-12-06 17:46:24 -0800301
302 # Refresh dynamic linker run time bindings for dependencies
William A. Kennington III1fddb972019-02-06 18:03:53 -0800303 check_call_cmd('sudo', '-n', '--', 'ldconfig')
William A. Kennington III54d4faf2018-12-06 17:46:24 -0800304
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030305 pkg = Package()
306 if build_for_testing:
307 pkg.test()
William A. Kennington III3f1d1202018-12-06 18:02:07 -0800308 else:
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030309 pkg.install()
310
William A. Kennington III780ec092018-12-06 14:46:50 -0800311
Andrew Jefferyccf85d62020-03-13 10:25:42 +1030312def build_dep_tree(name, pkgdir, dep_added, head, branch, dep_tree=None):
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500313 """
Andrew Jefferyccf85d62020-03-13 10:25:42 +1030314 For each package (name), starting with the package to be unit tested,
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030315 extract its dependencies. For each package dependency defined, recursively
316 apply the same strategy
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500317
318 Parameter descriptions:
Andrew Jefferyccf85d62020-03-13 10:25:42 +1030319 name Name of the package
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500320 pkgdir Directory where package source is located
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800321 dep_added Current dict of dependencies and added status
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500322 head Head node of the dependency tree
Andrew Geisslera61acb52019-01-03 16:32:44 -0600323 branch Branch to clone from pkg
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500324 dep_tree Current dependency tree node
325 """
326 if not dep_tree:
327 dep_tree = head
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800328
William A. Kennington IIIbe6aab22018-12-06 15:01:54 -0800329 with open("/tmp/depcache", "r") as depcache:
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800330 cache = depcache.readline()
331
332 # Read out pkg dependencies
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030333 pkg = Package(name, pkgdir)
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800334
Patrick Williams38c6b7d2021-02-18 07:26:14 -0600335 build = pkg.build_system()
336 if build == None:
337 raise Exception(f"Unable to find build system for {name}.")
338
339 for dep in set(build.dependencies()):
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800340 if dep in cache:
341 continue
342 # Dependency package not already known
343 if dep_added.get(dep) is None:
Patrick Williams38c6b7d2021-02-18 07:26:14 -0600344 print(f"Adding {dep} dependency to {name}.")
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800345 # Dependency package not added
346 new_child = dep_tree.AddChild(dep)
347 dep_added[dep] = False
Andrew Jeffery3b92fdd2020-03-13 11:49:18 +1030348 dep_pkgdir = clone_pkg(dep, branch)
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800349 # Determine this dependency package's
350 # dependencies and add them before
351 # returning to add this package
352 dep_added = build_dep_tree(dep,
353 dep_pkgdir,
354 dep_added,
355 head,
Andrew Geisslera61acb52019-01-03 16:32:44 -0600356 branch,
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800357 new_child)
358 else:
359 # Dependency package known and added
360 if dep_added[dep]:
Andrew Jeffery2cb0c7a2018-03-08 13:19:08 +1030361 continue
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500362 else:
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800363 # Cyclic dependency failure
Andrew Jefferyccf85d62020-03-13 10:25:42 +1030364 raise Exception("Cyclic dependencies found in "+name)
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500365
Andrew Jefferyccf85d62020-03-13 10:25:42 +1030366 if not dep_added[name]:
367 dep_added[name] = True
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500368
369 return dep_added
Matthew Barthccb7f852016-11-23 17:43:02 -0600370
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700371
William A. Kennington III90b106a2019-02-06 18:08:24 -0800372def run_cppcheck():
Ed Tanousc7198552022-07-01 08:15:50 -0700373 if not os.path.exists(os.path.join("build", "compile_commands.json")):
Brad Bishop48424d42020-01-07 13:01:31 -0500374 return None
375
Ed Tanousc7198552022-07-01 08:15:50 -0700376 try:
377 os.mkdir("cppcheck-temp")
378 except FileExistsError as e:
379 pass
380
Brad Bishop48424d42020-01-07 13:01:31 -0500381 # http://cppcheck.sourceforge.net/manual.pdf
Ed Tanousc7198552022-07-01 08:15:50 -0700382 try:
383 check_call_cmd(
384 'cppcheck',
385 '-j', str(multiprocessing.cpu_count()),
386 '--enable=style,performance,portability,missingInclude',
387 '--suppress=useStlAlgorithm',
388 '--suppress=unusedStructMember',
389 '--suppress=postfixOperator',
390 '--suppress=unreadVariable',
391 '--suppress=knownConditionTrueFalse',
392 '--library=googletest',
393 '--project=build/compile_commands.json',
394 '--cppcheck-build-dir=cppcheck-temp',
395 )
396 except subprocess.CalledProcessError:
Lei YUdbd7cd62022-07-19 19:24:01 +0800397 print("cppcheck found errors")
398
Andrew Jefferye5fffa02020-03-13 11:51:57 +1030399
William A. Kennington III37a89a22018-12-13 14:32:02 -0800400def is_valgrind_safe():
401 """
402 Returns whether it is safe to run valgrind on our platform
403 """
William A. Kennington III0326ded2019-02-07 00:33:28 -0800404 src = 'unit-test-vg.c'
405 exe = './unit-test-vg'
406 with open(src, 'w') as h:
William A. Kennington IIIafb0f982019-04-26 17:30:28 -0700407 h.write('#include <errno.h>\n')
408 h.write('#include <stdio.h>\n')
William A. Kennington III0326ded2019-02-07 00:33:28 -0800409 h.write('#include <stdlib.h>\n')
410 h.write('#include <string.h>\n')
411 h.write('int main() {\n')
412 h.write('char *heap_str = malloc(16);\n')
413 h.write('strcpy(heap_str, "RandString");\n')
414 h.write('int res = strcmp("RandString", heap_str);\n')
415 h.write('free(heap_str);\n')
William A. Kennington IIIafb0f982019-04-26 17:30:28 -0700416 h.write('char errstr[64];\n')
417 h.write('strerror_r(EINVAL, errstr, sizeof(errstr));\n')
418 h.write('printf("%s\\n", errstr);\n')
William A. Kennington III0326ded2019-02-07 00:33:28 -0800419 h.write('return res;\n')
420 h.write('}\n')
421 try:
422 with open(os.devnull, 'w') as devnull:
423 check_call(['gcc', '-O2', '-o', exe, src],
424 stdout=devnull, stderr=devnull)
425 check_call(['valgrind', '--error-exitcode=99', exe],
426 stdout=devnull, stderr=devnull)
427 return True
428 except:
429 sys.stderr.write("###### Platform is not valgrind safe ######\n")
430 return False
431 finally:
432 os.remove(src)
433 os.remove(exe)
William A. Kennington III37a89a22018-12-13 14:32:02 -0800434
Andrew Jefferye5fffa02020-03-13 11:51:57 +1030435
William A. Kennington III282e3302019-02-04 16:55:05 -0800436def is_sanitize_safe():
437 """
438 Returns whether it is safe to run sanitizers on our platform
439 """
William A. Kennington III0b7fb2b2019-02-07 00:33:42 -0800440 src = 'unit-test-sanitize.c'
441 exe = './unit-test-sanitize'
442 with open(src, 'w') as h:
443 h.write('int main() { return 0; }\n')
444 try:
445 with open(os.devnull, 'w') as devnull:
446 check_call(['gcc', '-O2', '-fsanitize=address',
447 '-fsanitize=undefined', '-o', exe, src],
448 stdout=devnull, stderr=devnull)
449 check_call([exe], stdout=devnull, stderr=devnull)
Andrew Geisslercd9578b2021-05-04 10:18:01 -0500450
451 # TODO - Sanitizer not working on ppc64le
452 # https://github.com/openbmc/openbmc-build-scripts/issues/31
453 if (platform.processor() == 'ppc64le'):
454 sys.stderr.write("###### ppc64le is not sanitize safe ######\n")
455 return False
456 else:
457 return True
William A. Kennington III0b7fb2b2019-02-07 00:33:42 -0800458 except:
459 sys.stderr.write("###### Platform is not sanitize safe ######\n")
460 return False
461 finally:
462 os.remove(src)
463 os.remove(exe)
William A. Kennington III282e3302019-02-04 16:55:05 -0800464
William A. Kennington III49d4e592019-02-06 17:59:27 -0800465
William A. Kennington IIIeaff24a2019-02-06 16:57:42 -0800466def maybe_make_valgrind():
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700467 """
468 Potentially runs the unit tests through valgrind for the package
469 via `make check-valgrind`. If the package does not have valgrind testing
470 then it just skips over this.
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700471 """
William A. Kennington III4e1d0a12018-07-16 12:04:03 -0700472 # Valgrind testing is currently broken by an aggressive strcmp optimization
473 # that is inlined into optimized code for POWER by gcc 7+. Until we find
474 # a workaround, just don't run valgrind tests on POWER.
475 # https://github.com/openbmc/openbmc/issues/3315
William A. Kennington III37a89a22018-12-13 14:32:02 -0800476 if not is_valgrind_safe():
William A. Kennington III75130192019-02-07 00:34:14 -0800477 sys.stderr.write("###### Skipping valgrind ######\n")
William A. Kennington III4e1d0a12018-07-16 12:04:03 -0700478 return
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700479 if not make_target_exists('check-valgrind'):
480 return
481
482 try:
Andrew Jeffery881041f2020-03-13 11:46:30 +1030483 cmd = make_parallel + ['check-valgrind']
William A. Kennington III1fddb972019-02-06 18:03:53 -0800484 check_call_cmd(*cmd)
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700485 except CalledProcessError:
William A. Kennington III90b106a2019-02-06 18:08:24 -0800486 for root, _, files in os.walk(os.getcwd()):
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700487 for f in files:
488 if re.search('test-suite-[a-z]+.log', f) is None:
489 continue
William A. Kennington III1fddb972019-02-06 18:03:53 -0800490 check_call_cmd('cat', os.path.join(root, f))
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700491 raise Exception('Valgrind tests failed')
492
Andrew Jefferye5fffa02020-03-13 11:51:57 +1030493
William A. Kennington IIIeaff24a2019-02-06 16:57:42 -0800494def maybe_make_coverage():
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700495 """
496 Potentially runs the unit tests through code coverage for the package
497 via `make check-code-coverage`. If the package does not have code coverage
498 testing then it just skips over this.
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700499 """
500 if not make_target_exists('check-code-coverage'):
501 return
502
503 # Actually run code coverage
504 try:
Andrew Jeffery881041f2020-03-13 11:46:30 +1030505 cmd = make_parallel + ['check-code-coverage']
William A. Kennington III1fddb972019-02-06 18:03:53 -0800506 check_call_cmd(*cmd)
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700507 except CalledProcessError:
508 raise Exception('Code coverage failed')
Matthew Barthccb7f852016-11-23 17:43:02 -0600509
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030510
511class BuildSystem(object):
512 """
513 Build systems generally provide the means to configure, build, install and
514 test software. The BuildSystem class defines a set of interfaces on top of
515 which Autotools, Meson, CMake and possibly other build system drivers can
516 be implemented, separating out the phases to control whether a package
517 should merely be installed or also tested and analyzed.
518 """
Lei YU08d2b922022-04-25 11:21:36 +0800519
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030520 def __init__(self, package, path):
521 """Initialise the driver with properties independent of the build system
522
523 Keyword arguments:
524 package: The name of the package. Derived from the path if None
525 path: The path to the package. Set to the working directory if None
526 """
527 self.path = "." if not path else path
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030528 realpath = os.path.realpath(self.path)
529 self.package = package if package else os.path.basename(realpath)
Andrew Jeffery3f8e5a72020-03-13 11:47:56 +1030530 self.build_for_testing = False
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030531
532 def probe(self):
533 """Test if the build system driver can be applied to the package
534
535 Return True if the driver can drive the package's build system,
536 otherwise False.
537
538 Generally probe() is implemented by testing for the presence of the
539 build system's configuration file(s).
540 """
541 raise NotImplemented
542
543 def dependencies(self):
544 """Provide the package's dependencies
545
546 Returns a list of dependencies. If no dependencies are required then an
547 empty list must be returned.
548
549 Generally dependencies() is implemented by analysing and extracting the
550 data from the build system configuration.
551 """
552 raise NotImplemented
553
554 def configure(self, build_for_testing):
555 """Configure the source ready for building
556
557 Should raise an exception if configuration failed.
558
559 Keyword arguments:
560 build_for_testing: Mark the package as being built for testing rather
561 than for installation as a dependency for the
562 package under test. Setting to True generally
563 implies that the package will be configured to build
564 with debug information, at a low level of
565 optimisation and possibly with sanitizers enabled.
566
567 Generally configure() is implemented by invoking the build system
568 tooling to generate Makefiles or equivalent.
569 """
570 raise NotImplemented
571
572 def build(self):
573 """Build the software ready for installation and/or testing
574
575 Should raise an exception if the build fails
576
577 Generally build() is implemented by invoking `make` or `ninja`.
578 """
579 raise NotImplemented
580
581 def install(self):
582 """Install the software ready for use
583
584 Should raise an exception if installation fails
585
586 Like build(), install() is generally implemented by invoking `make` or
587 `ninja`.
588 """
589 raise NotImplemented
590
591 def test(self):
592 """Build and run the test suite associated with the package
593
594 Should raise an exception if the build or testing fails.
595
596 Like install(), test() is generally implemented by invoking `make` or
597 `ninja`.
598 """
599 raise NotImplemented
600
601 def analyze(self):
602 """Run any supported analysis tools over the codebase
603
604 Should raise an exception if analysis fails.
605
606 Some analysis tools such as scan-build need injection into the build
607 system. analyze() provides the necessary hook to implement such
608 behaviour. Analyzers independent of the build system can also be
609 specified here but at the cost of possible duplication of code between
610 the build system driver implementations.
611 """
612 raise NotImplemented
613
614
615class Autotools(BuildSystem):
616 def __init__(self, package=None, path=None):
617 super(Autotools, self).__init__(package, path)
618
619 def probe(self):
620 return os.path.isfile(os.path.join(self.path, 'configure.ac'))
621
622 def dependencies(self):
623 configure_ac = os.path.join(self.path, 'configure.ac')
624
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030625 contents = ''
Andrew Jeffery7d4a26f2020-03-13 11:42:34 +1030626 # Prepend some special function overrides so we can parse out
627 # dependencies
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030628 for macro in DEPENDENCIES.keys():
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030629 contents += ('m4_define([' + macro + '], [' + macro + '_START$' +
630 str(DEPENDENCIES_OFFSET[macro] + 1) +
631 macro + '_END])\n')
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030632 with open(configure_ac, "rt") as f:
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030633 contents += f.read()
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030634
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030635 autoconf_cmdline = ['autoconf', '-Wno-undefined', '-']
636 autoconf_process = subprocess.Popen(autoconf_cmdline,
Andrew Jeffery7d4a26f2020-03-13 11:42:34 +1030637 stdin=subprocess.PIPE,
638 stdout=subprocess.PIPE,
639 stderr=subprocess.PIPE)
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030640 document = contents.encode('utf-8')
641 (stdout, stderr) = autoconf_process.communicate(input=document)
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030642 if not stdout:
643 print(stderr)
644 raise Exception("Failed to run autoconf for parsing dependencies")
645
646 # Parse out all of the dependency text
647 matches = []
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030648 for macro in DEPENDENCIES.keys():
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030649 pattern = '(' + macro + ')_START(.*?)' + macro + '_END'
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030650 for match in re.compile(pattern).finditer(stdout.decode('utf-8')):
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030651 matches.append((match.group(1), match.group(2)))
652
653 # Look up dependencies from the text
654 found_deps = []
655 for macro, deptext in matches:
656 for potential_dep in deptext.split(' '):
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030657 for known_dep in DEPENDENCIES[macro].keys():
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030658 if potential_dep.startswith(known_dep):
659 found_deps.append(DEPENDENCIES[macro][known_dep])
660
661 return found_deps
662
663 def _configure_feature(self, flag, enabled):
664 """
665 Returns an configure flag as a string
666
667 Parameters:
668 flag The name of the flag
669 enabled Whether the flag is enabled or disabled
670 """
671 return '--' + ('enable' if enabled else 'disable') + '-' + flag
672
673 def configure(self, build_for_testing):
674 self.build_for_testing = build_for_testing
675 conf_flags = [
676 self._configure_feature('silent-rules', False),
677 self._configure_feature('examples', build_for_testing),
678 self._configure_feature('tests', build_for_testing),
Ramin Izadpanah298d56b2020-11-17 16:55:28 +0000679 self._configure_feature('itests', INTEGRATION_TEST),
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030680 ]
Andrew Jefferyd0809692021-05-14 16:23:57 +0930681 conf_flags.extend([
682 self._configure_feature('code-coverage', build_for_testing),
683 self._configure_feature('valgrind', build_for_testing),
684 ])
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030685 # Add any necessary configure flags for package
686 if CONFIGURE_FLAGS.get(self.package) is not None:
687 conf_flags.extend(CONFIGURE_FLAGS.get(self.package))
688 for bootstrap in ['bootstrap.sh', 'bootstrap', 'autogen.sh']:
689 if os.path.exists(bootstrap):
690 check_call_cmd('./' + bootstrap)
691 break
692 check_call_cmd('./configure', *conf_flags)
693
694 def build(self):
695 check_call_cmd(*make_parallel)
696
697 def install(self):
Andrew Jeffery881041f2020-03-13 11:46:30 +1030698 check_call_cmd('sudo', '-n', '--', *(make_parallel + ['install']))
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030699
700 def test(self):
701 try:
Andrew Jeffery881041f2020-03-13 11:46:30 +1030702 cmd = make_parallel + ['check']
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030703 for i in range(0, args.repeat):
704 check_call_cmd(*cmd)
Andrew Jefferyd0809692021-05-14 16:23:57 +0930705
706 maybe_make_valgrind()
707 maybe_make_coverage()
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030708 except CalledProcessError:
709 for root, _, files in os.walk(os.getcwd()):
710 if 'test-suite.log' not in files:
711 continue
712 check_call_cmd('cat', os.path.join(root, 'test-suite.log'))
713 raise Exception('Unit tests failed')
714
715 def analyze(self):
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030716 run_cppcheck()
717
718
719class CMake(BuildSystem):
720 def __init__(self, package=None, path=None):
721 super(CMake, self).__init__(package, path)
722
723 def probe(self):
724 return os.path.isfile(os.path.join(self.path, 'CMakeLists.txt'))
725
726 def dependencies(self):
727 return []
728
729 def configure(self, build_for_testing):
730 self.build_for_testing = build_for_testing
Ramin Izadpanah298d56b2020-11-17 16:55:28 +0000731 if INTEGRATION_TEST:
732 check_call_cmd('cmake', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
733 '-DITESTS=ON', '.')
734 else:
735 check_call_cmd('cmake', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', '.')
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030736
737 def build(self):
738 check_call_cmd('cmake', '--build', '.', '--', '-j',
739 str(multiprocessing.cpu_count()))
740
741 def install(self):
742 pass
743
744 def test(self):
745 if make_target_exists('test'):
746 check_call_cmd('ctest', '.')
747
748 def analyze(self):
749 if os.path.isfile('.clang-tidy'):
Ed Tanousbc5c1c42020-10-01 17:49:20 -0700750 try:
751 os.mkdir("tidy-build")
752 except FileExistsError as e:
753 pass
754 # clang-tidy needs to run on a clang-specific build
755 check_call_cmd('cmake', '-DCMAKE_C_COMPILER=clang',
756 '-DCMAKE_CXX_COMPILER=clang++',
757 '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
758 '-H.',
759 '-Btidy-build')
760 # we need to cd here because otherwise clang-tidy doesn't find the
761 # .clang-tidy file in the roots of repos. Its arguably a "bug"
762 # with run-clang-tidy at a minimum it's "weird" that it requires
763 # the .clang-tidy to be up a dir
Ed Tanous662890f2020-09-28 17:35:20 -0700764 os.chdir("tidy-build")
765 try:
Patrick Williamsf89c8502021-10-01 11:50:09 -0500766 check_call_cmd('run-clang-tidy', "-header-filter=.*", '-p',
Ed Tanousbc5c1c42020-10-01 17:49:20 -0700767 '.')
Ed Tanous662890f2020-09-28 17:35:20 -0700768 finally:
769 os.chdir("..")
770
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030771 maybe_make_valgrind()
772 maybe_make_coverage()
773 run_cppcheck()
774
775
776class Meson(BuildSystem):
777 def __init__(self, package=None, path=None):
778 super(Meson, self).__init__(package, path)
779
780 def probe(self):
781 return os.path.isfile(os.path.join(self.path, 'meson.build'))
782
783 def dependencies(self):
784 meson_build = os.path.join(self.path, 'meson.build')
785 if not os.path.exists(meson_build):
786 return []
787
788 found_deps = []
789 for root, dirs, files in os.walk(self.path):
790 if 'meson.build' not in files:
791 continue
792 with open(os.path.join(root, 'meson.build'), 'rt') as f:
793 build_contents = f.read()
Nan Zhouef13d532020-07-07 09:52:02 -0700794 pattern = r"dependency\('([^']*)'.*?\),?\n"
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030795 for match in re.finditer(pattern, build_contents):
796 group = match.group(1)
797 maybe_dep = DEPENDENCIES['PKG_CHECK_MODULES'].get(group)
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030798 if maybe_dep is not None:
799 found_deps.append(maybe_dep)
800
801 return found_deps
802
803 def _parse_options(self, options_file):
804 """
805 Returns a set of options defined in the provides meson_options.txt file
806
807 Parameters:
808 options_file The file containing options
809 """
William A. Kennington III0ecd0bc2020-06-04 00:44:22 -0700810 oi = optinterpreter.OptionInterpreter('')
811 oi.process(options_file)
812 return oi.options
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030813
William A. Kennington IIIfcd70772020-06-04 00:50:23 -0700814 def _configure_boolean(self, val):
815 """
816 Returns the meson flag which signifies the value
817
818 True is true which requires the boolean.
819 False is false which disables the boolean.
820
821 Parameters:
822 val The value being converted
823 """
824 if val is True:
825 return 'true'
826 elif val is False:
827 return 'false'
828 else:
829 raise Exception("Bad meson boolean value")
830
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030831 def _configure_feature(self, val):
832 """
833 Returns the meson flag which signifies the value
834
835 True is enabled which requires the feature.
836 False is disabled which disables the feature.
837 None is auto which autodetects the feature.
838
839 Parameters:
840 val The value being converted
841 """
842 if val is True:
843 return "enabled"
844 elif val is False:
845 return "disabled"
846 elif val is None:
847 return "auto"
848 else:
849 raise Exception("Bad meson feature value")
850
William A. Kennington IIIfcd70772020-06-04 00:50:23 -0700851 def _configure_option(self, opts, key, val):
852 """
853 Returns the meson flag which signifies the value
854 based on the type of the opt
855
856 Parameters:
857 opt The meson option which we are setting
858 val The value being converted
859 """
860 if isinstance(opts[key], coredata.UserBooleanOption):
861 str_val = self._configure_boolean(val)
862 elif isinstance(opts[key], coredata.UserFeatureOption):
863 str_val = self._configure_feature(val)
864 else:
865 raise Exception('Unknown meson option type')
866 return "-D{}={}".format(key, str_val)
867
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030868 def configure(self, build_for_testing):
869 self.build_for_testing = build_for_testing
William A. Kennington III0ecd0bc2020-06-04 00:44:22 -0700870 meson_options = {}
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030871 if os.path.exists("meson_options.txt"):
872 meson_options = self._parse_options("meson_options.txt")
873 meson_flags = [
874 '-Db_colorout=never',
875 '-Dwerror=true',
876 '-Dwarning_level=3',
877 ]
878 if build_for_testing:
879 meson_flags.append('--buildtype=debug')
880 else:
881 meson_flags.append('--buildtype=debugoptimized')
Patrick Williamse95626d2021-04-13 11:13:53 -0500882 if OptionKey('tests') in meson_options:
Lei YU08d2b922022-04-25 11:21:36 +0800883 meson_flags.append(self._configure_option(
884 meson_options, OptionKey('tests'), build_for_testing))
Patrick Williamse95626d2021-04-13 11:13:53 -0500885 if OptionKey('examples') in meson_options:
Lei YU08d2b922022-04-25 11:21:36 +0800886 meson_flags.append(self._configure_option(
887 meson_options, OptionKey('examples'), build_for_testing))
Patrick Williamse95626d2021-04-13 11:13:53 -0500888 if OptionKey('itests') in meson_options:
Lei YU08d2b922022-04-25 11:21:36 +0800889 meson_flags.append(self._configure_option(
890 meson_options, OptionKey('itests'), INTEGRATION_TEST))
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030891 if MESON_FLAGS.get(self.package) is not None:
892 meson_flags.extend(MESON_FLAGS.get(self.package))
893 try:
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030894 check_call_cmd('meson', 'setup', '--reconfigure', 'build',
895 *meson_flags)
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030896 except:
897 shutil.rmtree('build')
898 check_call_cmd('meson', 'setup', 'build', *meson_flags)
899
900 def build(self):
901 check_call_cmd('ninja', '-C', 'build')
902
903 def install(self):
904 check_call_cmd('sudo', '-n', '--', 'ninja', '-C', 'build', 'install')
905
906 def test(self):
Patrick Williams95095f12021-04-14 14:42:35 -0500907 # It is useful to check various settings of the meson.build file
908 # for compatibility, such as meson_version checks. We shouldn't
909 # do this in the configure path though because it affects subprojects
910 # and dependencies as well, but we only want this applied to the
911 # project-under-test (otherwise an upstream dependency could fail
912 # this check without our control).
913 self._extra_meson_checks()
914
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030915 try:
Brad Bishop7b8cef22020-08-26 15:58:09 -0400916 test_args = ('--repeat', str(args.repeat), '-C', 'build')
William A. Kennington IIId9062c42022-05-04 23:24:44 -0700917 check_call_cmd('meson', 'test', '--print-errorlogs', *test_args)
Brad Bishop7b8cef22020-08-26 15:58:09 -0400918
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030919 except CalledProcessError:
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030920 raise Exception('Unit tests failed')
921
922 def _setup_exists(self, setup):
923 """
924 Returns whether the meson build supports the named test setup.
925
926 Parameter descriptions:
927 setup The setup target to check
928 """
929 try:
930 with open(os.devnull, 'w') as devnull:
931 output = subprocess.check_output(
Lei YU08d2b922022-04-25 11:21:36 +0800932 ['meson', 'test', '-C', 'build',
933 '--setup', setup, '-t', '0'],
934 stderr=subprocess.STDOUT)
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030935 except CalledProcessError as e:
936 output = e.output
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030937 output = output.decode('utf-8')
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030938 return not re.search('Test setup .* not found from project', output)
939
940 def _maybe_valgrind(self):
941 """
942 Potentially runs the unit tests through valgrind for the package
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030943 via `meson test`. The package can specify custom valgrind
944 configurations by utilizing add_test_setup() in a meson.build
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030945 """
946 if not is_valgrind_safe():
947 sys.stderr.write("###### Skipping valgrind ######\n")
948 return
949 try:
950 if self._setup_exists('valgrind'):
Lei YU08d2b922022-04-25 11:21:36 +0800951 check_call_cmd('meson', 'test', '-t', '10', '-C', 'build',
William A. Kennington IIId9062c42022-05-04 23:24:44 -0700952 '--print-errorlogs', '--setup', 'valgrind')
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030953 else:
Lei YU08d2b922022-04-25 11:21:36 +0800954 check_call_cmd('meson', 'test', '-t', '10', '-C', 'build',
William A. Kennington IIId9062c42022-05-04 23:24:44 -0700955 '--print-errorlogs', '--wrapper', 'valgrind')
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030956 except CalledProcessError:
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030957 raise Exception('Valgrind tests failed')
958
959 def analyze(self):
960 self._maybe_valgrind()
961
962 # Run clang-tidy only if the project has a configuration
963 if os.path.isfile('.clang-tidy'):
Manojkiran Eda1aa91992020-10-02 14:11:53 +0530964 os.environ["CXX"] = "clang++"
965 check_call_cmd('meson', 'setup', 'build-clang')
Ed Tanous88db4c22022-01-25 15:15:59 -0800966 os.chdir("build-clang")
967 try:
968 check_call_cmd('run-clang-tidy', '-fix', '-format', '-p', '.')
969 except subprocess.CalledProcessError:
970 check_call_cmd("git", "-C", CODE_SCAN_DIR,
Lei YU08d2b922022-04-25 11:21:36 +0800971 "--no-pager", "diff")
Ed Tanous88db4c22022-01-25 15:15:59 -0800972 raise
973 finally:
974 os.chdir("..")
975
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030976 # Run the basic clang static analyzer otherwise
977 else:
978 check_call_cmd('ninja', '-C', 'build',
979 'scan-build')
980
981 # Run tests through sanitizers
982 # b_lundef is needed if clang++ is CXX since it resolves the
983 # asan symbols at runtime only. We don't want to set it earlier
984 # in the build process to ensure we don't have undefined
985 # runtime code.
986 if is_sanitize_safe():
987 check_call_cmd('meson', 'configure', 'build',
988 '-Db_sanitize=address,undefined',
989 '-Db_lundef=false')
William A. Kennington IIId9062c42022-05-04 23:24:44 -0700990 check_call_cmd('meson', 'test', '-C', 'build', '--print-errorlogs',
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030991 '--logbase', 'testlog-ubasan')
992 # TODO: Fix memory sanitizer
Andrew Jeffery6015ca12020-03-13 11:50:10 +1030993 # check_call_cmd('meson', 'configure', 'build',
994 # '-Db_sanitize=memory')
995 # check_call_cmd('meson', 'test', '-C', 'build'
996 # '--logbase', 'testlog-msan')
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030997 check_call_cmd('meson', 'configure', 'build',
Lei YUf684ad62020-09-21 17:02:26 +0800998 '-Db_sanitize=none')
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030999 else:
1000 sys.stderr.write("###### Skipping sanitizers ######\n")
1001
1002 # Run coverage checks
1003 check_call_cmd('meson', 'configure', 'build',
1004 '-Db_coverage=true')
1005 self.test()
1006 # Only build coverage HTML if coverage files were produced
1007 for root, dirs, files in os.walk('build'):
1008 if any([f.endswith('.gcda') for f in files]):
1009 check_call_cmd('ninja', '-C', 'build',
1010 'coverage-html')
1011 break
1012 check_call_cmd('meson', 'configure', 'build',
1013 '-Db_coverage=false')
1014 run_cppcheck()
1015
Patrick Williams95095f12021-04-14 14:42:35 -05001016 def _extra_meson_checks(self):
1017 with open(os.path.join(self.path, 'meson.build'), 'rt') as f:
1018 build_contents = f.read()
1019
1020 # Find project's specified meson_version.
1021 meson_version = None
1022 pattern = r"meson_version:[^']*'([^']*)'"
1023 for match in re.finditer(pattern, build_contents):
1024 group = match.group(1)
1025 meson_version = group
1026
1027 # C++20 requires at least Meson 0.57 but Meson itself doesn't
1028 # identify this. Add to our unit-test checks so that we don't
1029 # get a meson.build missing this.
1030 pattern = r"'cpp_std=c\+\+20'"
1031 for match in re.finditer(pattern, build_contents):
1032 if not meson_version or \
1033 not meson_version_compare(meson_version, ">=0.57"):
1034 raise Exception(
1035 "C++20 support requires specifying in meson.build: "
1036 + "meson_version: '>=0.57'"
1037 )
1038
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301039
1040class Package(object):
1041 def __init__(self, name=None, path=None):
Manojkiran Eda29d28252020-06-15 18:58:09 +05301042 self.supported = [Meson, Autotools, CMake]
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301043 self.name = name
1044 self.path = path
1045 self.test_only = False
1046
1047 def build_systems(self):
Andrew Jeffery881041f2020-03-13 11:46:30 +10301048 instances = (system(self.name, self.path) for system in self.supported)
1049 return (instance for instance in instances if instance.probe())
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301050
1051 def build_system(self, preferred=None):
Andrew Geissler8cb74fc2020-03-19 14:48:05 -05001052 systems = list(self.build_systems())
1053
1054 if not systems:
1055 return None
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301056
1057 if preferred:
Andrew Jeffery881041f2020-03-13 11:46:30 +10301058 return {type(system): system for system in systems}[preferred]
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301059
1060 return next(iter(systems))
1061
1062 def install(self, system=None):
1063 if not system:
1064 system = self.build_system()
1065
1066 system.configure(False)
1067 system.build()
1068 system.install()
1069
Andrew Jeffery19d75672020-03-13 10:42:08 +10301070 def _test_one(self, system):
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301071 system.configure(True)
1072 system.build()
1073 system.install()
1074 system.test()
Andrew Jefferyd0809692021-05-14 16:23:57 +09301075 if not TEST_ONLY:
1076 system.analyze()
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301077
Andrew Jeffery19d75672020-03-13 10:42:08 +10301078 def test(self):
1079 for system in self.build_systems():
1080 self._test_one(system)
1081
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301082
Matt Spinler9bfaaad2019-10-25 09:51:50 -05001083def find_file(filename, basedir):
1084 """
Patrick Williams55448ad2020-12-14 14:28:28 -06001085 Finds all occurrences of a file (or list of files) in the base
1086 directory and passes them back with their relative paths.
Matt Spinler9bfaaad2019-10-25 09:51:50 -05001087
1088 Parameter descriptions:
Patrick Williams55448ad2020-12-14 14:28:28 -06001089 filename The name of the file (or list of files) to
1090 find
Matt Spinler9bfaaad2019-10-25 09:51:50 -05001091 basedir The base directory search in
1092 """
1093
Patrick Williams55448ad2020-12-14 14:28:28 -06001094 if not isinstance(filename, list):
Lei YU08d2b922022-04-25 11:21:36 +08001095 filename = [filename]
Patrick Williams55448ad2020-12-14 14:28:28 -06001096
Matt Spinler9bfaaad2019-10-25 09:51:50 -05001097 filepaths = []
1098 for root, dirs, files in os.walk(basedir):
Brad Bishopeb667262021-08-06 13:54:58 -04001099 if os.path.split(root)[-1] == 'subprojects':
1100 for f in files:
1101 subproject = '.'.join(f.split('.')[0:-1])
1102 if f.endswith('.wrap') and subproject in dirs:
1103 # don't find files in meson subprojects with wraps
1104 dirs.remove(subproject)
Patrick Williams55448ad2020-12-14 14:28:28 -06001105 for f in filename:
1106 if f in files:
1107 filepaths.append(os.path.join(root, f))
Matt Spinler9bfaaad2019-10-25 09:51:50 -05001108 return filepaths
1109
Andrew Jeffery9cc97d82020-03-13 11:57:44 +10301110
Matthew Barthccb7f852016-11-23 17:43:02 -06001111if __name__ == '__main__':
1112 # CONFIGURE_FLAGS = [GIT REPO]:[CONFIGURE FLAGS]
1113 CONFIGURE_FLAGS = {
Matthew Barth1d1c6732017-03-24 10:00:28 -05001114 'phosphor-logging':
Matt Spinler0744bb82020-01-16 08:23:35 -06001115 ['--enable-metadata-processing', '--enable-openpower-pel-extension',
Deepak Kodihalli3a4e1b42017-06-08 09:52:35 -05001116 'YAML_DIR=/usr/local/share/phosphor-dbus-yaml/yaml']
Matthew Barthccb7f852016-11-23 17:43:02 -06001117 }
1118
William A. Kennington III3f1d1202018-12-06 18:02:07 -08001119 # MESON_FLAGS = [GIT REPO]:[MESON FLAGS]
1120 MESON_FLAGS = {
Matt Spinler8448cfd2020-09-10 17:01:48 -05001121 'phosphor-dbus-interfaces':
Patrick Williams13af98a2021-04-21 15:17:00 -05001122 ['-Ddata_com_ibm=true', '-Ddata_org_open_power=true'],
1123 'phosphor-logging':
1124 ['-Dopenpower-pel-extension=enabled']
William A. Kennington III3f1d1202018-12-06 18:02:07 -08001125 }
1126
Matthew Barthccb7f852016-11-23 17:43:02 -06001127 # DEPENDENCIES = [MACRO]:[library/header]:[GIT REPO]
1128 DEPENDENCIES = {
1129 'AC_CHECK_LIB': {'mapper': 'phosphor-objmgr'},
Matthew Barth710f3f02017-01-18 15:20:19 -06001130 'AC_CHECK_HEADER': {
1131 'host-ipmid': 'phosphor-host-ipmid',
Patrick Ventureb41a4462018-10-03 17:27:38 -07001132 'blobs-ipmid': 'phosphor-ipmi-blobs',
Matthew Barth710f3f02017-01-18 15:20:19 -06001133 'sdbusplus': 'sdbusplus',
William A. Kennington IIIb4f730a2018-09-12 11:21:20 -07001134 'sdeventplus': 'sdeventplus',
William A. Kennington III23705242019-01-15 18:17:25 -08001135 'stdplus': 'stdplus',
Patrick Venture22329962018-09-14 10:23:04 -07001136 'gpioplus': 'gpioplus',
Saqib Khan66145052017-02-14 12:02:07 -06001137 'phosphor-logging/log.hpp': 'phosphor-logging',
Patrick Williamseab8a372017-01-30 11:21:32 -06001138 },
Brad Bishopebb49112017-02-13 11:07:26 -05001139 'AC_PATH_PROG': {'sdbus++': 'sdbusplus'},
Patrick Williamseab8a372017-01-30 11:21:32 -06001140 'PKG_CHECK_MODULES': {
Matthew Barth19e261e2017-02-01 12:55:22 -06001141 'phosphor-dbus-interfaces': 'phosphor-dbus-interfaces',
William A. Kennington III993ab332019-02-07 02:12:31 -08001142 'libipmid': 'phosphor-host-ipmid',
1143 'libipmid-host': 'phosphor-host-ipmid',
Brad Bishopebb49112017-02-13 11:07:26 -05001144 'sdbusplus': 'sdbusplus',
William A. Kennington IIIb4f730a2018-09-12 11:21:20 -07001145 'sdeventplus': 'sdeventplus',
William A. Kennington III23705242019-01-15 18:17:25 -08001146 'stdplus': 'stdplus',
Patrick Venture22329962018-09-14 10:23:04 -07001147 'gpioplus': 'gpioplus',
Brad Bishopebb49112017-02-13 11:07:26 -05001148 'phosphor-logging': 'phosphor-logging',
Marri Devender Raoa3eee8a2018-08-13 05:34:27 -05001149 'phosphor-snmp': 'phosphor-snmp',
Patrick Venturee584c3b2019-03-07 09:44:16 -08001150 'ipmiblob': 'ipmi-blob-tool',
Ben Tyner300ed3c2020-05-14 15:30:37 -05001151 'hei': 'openpower-libhei',
Sui Chen40cf5212020-11-24 12:12:23 -08001152 'phosphor-ipmi-blobs': 'phosphor-ipmi-blobs',
Willy Tu01534ec2021-10-14 10:16:45 -07001153 'libcr51sign': 'google-misc',
Brad Bishopebb49112017-02-13 11:07:26 -05001154 },
Matthew Barthccb7f852016-11-23 17:43:02 -06001155 }
1156
William A. Kennington IIIe67f5fc2018-12-06 17:40:30 -08001157 # Offset into array of macro parameters MACRO(0, 1, ...N)
1158 DEPENDENCIES_OFFSET = {
1159 'AC_CHECK_LIB': 0,
1160 'AC_CHECK_HEADER': 0,
1161 'AC_PATH_PROG': 1,
1162 'PKG_CHECK_MODULES': 1,
1163 }
1164
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -05001165 # DEPENDENCIES_REGEX = [GIT REPO]:[REGEX STRING]
1166 DEPENDENCIES_REGEX = {
Patrick Ventured37b8052018-10-16 16:03:03 -07001167 'phosphor-logging': r'\S+-dbus-interfaces$'
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -05001168 }
1169
Matthew Barth33df8792016-12-19 14:30:17 -06001170 # Set command line arguments
1171 parser = argparse.ArgumentParser()
1172 parser.add_argument("-w", "--workspace", dest="WORKSPACE", required=True,
1173 help="Workspace directory location(i.e. /home)")
1174 parser.add_argument("-p", "--package", dest="PACKAGE", required=True,
1175 help="OpenBMC package to be unit tested")
William A. Kennington III65b37fa2019-01-31 15:15:17 -08001176 parser.add_argument("-t", "--test-only", dest="TEST_ONLY",
1177 action="store_true", required=False, default=False,
1178 help="Only run test cases, no other validation")
Ramin Izadpanah298d56b2020-11-17 16:55:28 +00001179 arg_inttests = parser.add_mutually_exclusive_group()
1180 arg_inttests.add_argument("--integration-tests", dest="INTEGRATION_TEST",
Lei YU08d2b922022-04-25 11:21:36 +08001181 action="store_true", required=False, default=True,
1182 help="Enable integration tests [default].")
Ramin Izadpanah298d56b2020-11-17 16:55:28 +00001183 arg_inttests.add_argument("--no-integration-tests", dest="INTEGRATION_TEST",
Lei YU08d2b922022-04-25 11:21:36 +08001184 action="store_false", required=False,
1185 help="Disable integration tests.")
Matthew Barth33df8792016-12-19 14:30:17 -06001186 parser.add_argument("-v", "--verbose", action="store_true",
1187 help="Print additional package status messages")
Andrew Jeffery468309d2018-03-08 13:46:33 +10301188 parser.add_argument("-r", "--repeat", help="Repeat tests N times",
1189 type=int, default=1)
Andrew Geisslera61acb52019-01-03 16:32:44 -06001190 parser.add_argument("-b", "--branch", dest="BRANCH", required=False,
1191 help="Branch to target for dependent repositories",
1192 default="master")
Lei YU7ef93302019-11-06 13:53:21 +08001193 parser.add_argument("-n", "--noformat", dest="FORMAT",
1194 action="store_false", required=False,
1195 help="Whether or not to run format code")
Matthew Barth33df8792016-12-19 14:30:17 -06001196 args = parser.parse_args(sys.argv[1:])
1197 WORKSPACE = args.WORKSPACE
1198 UNIT_TEST_PKG = args.PACKAGE
William A. Kennington III65b37fa2019-01-31 15:15:17 -08001199 TEST_ONLY = args.TEST_ONLY
Ramin Izadpanah298d56b2020-11-17 16:55:28 +00001200 INTEGRATION_TEST = args.INTEGRATION_TEST
Andrew Geisslera61acb52019-01-03 16:32:44 -06001201 BRANCH = args.BRANCH
Lei YU7ef93302019-11-06 13:53:21 +08001202 FORMAT_CODE = args.FORMAT
Matthew Barth33df8792016-12-19 14:30:17 -06001203 if args.verbose:
1204 def printline(*line):
1205 for arg in line:
Andrew Jeffery89b64b62020-03-13 12:15:48 +10301206 print(arg, end=' ')
1207 print()
Matthew Barth33df8792016-12-19 14:30:17 -06001208 else:
Andrew Jefferyfce557f2020-03-13 12:12:19 +10301209 def printline(*line):
1210 pass
Matthew Barthccb7f852016-11-23 17:43:02 -06001211
Patrick Williamsb6535952020-12-15 06:40:10 -06001212 CODE_SCAN_DIR = os.path.join(WORKSPACE, UNIT_TEST_PKG)
Lei YU7ef93302019-11-06 13:53:21 +08001213
James Feist878df5c2018-07-26 14:54:28 -07001214 # First validate code formatting if repo has style formatting files.
Adriana Kobylakbcee22b2018-01-10 16:58:27 -06001215 # The format-code.sh checks for these files.
Lei YU7ef93302019-11-06 13:53:21 +08001216 if FORMAT_CODE:
Patrick Williams5b4a1e82020-12-15 06:46:31 -06001217 format_scripts = find_file(['format-code.sh', 'format-code'],
1218 CODE_SCAN_DIR)
Andrew Geissler31502dd2021-01-07 14:04:02 -06001219
1220 # use default format-code.sh if no other found
1221 if not format_scripts:
1222 format_scripts.append(os.path.join(WORKSPACE, "format-code.sh"))
1223
Patrick Williams5b4a1e82020-12-15 06:46:31 -06001224 for f in format_scripts:
1225 check_call_cmd(f, CODE_SCAN_DIR)
Andrew Geisslera28286d2018-01-10 11:00:00 -08001226
Ed Tanous32768b82022-01-05 14:14:06 -08001227 # Check to see if any files changed
1228 check_call_cmd("git", "-C", CODE_SCAN_DIR,
1229 "--no-pager", "diff", "--exit-code")
1230
Andrew Geissler8cb74fc2020-03-19 14:48:05 -05001231 # Check if this repo has a supported make infrastructure
Patrick Williamsb6535952020-12-15 06:40:10 -06001232 pkg = Package(UNIT_TEST_PKG, CODE_SCAN_DIR)
Andrew Geissler8cb74fc2020-03-19 14:48:05 -05001233 if not pkg.build_system():
1234 print("No valid build system, exit")
1235 sys.exit(0)
1236
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301237 prev_umask = os.umask(000)
James Feist878df5c2018-07-26 14:54:28 -07001238
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301239 # Determine dependencies and add them
1240 dep_added = dict()
1241 dep_added[UNIT_TEST_PKG] = False
William A. Kennington III40d5c7c2018-12-13 14:37:59 -08001242
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301243 # Create dependency tree
1244 dep_tree = DepTree(UNIT_TEST_PKG)
Patrick Williamsb6535952020-12-15 06:40:10 -06001245 build_dep_tree(UNIT_TEST_PKG, CODE_SCAN_DIR, dep_added, dep_tree, BRANCH)
William A. Kennington III65b37fa2019-01-31 15:15:17 -08001246
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301247 # Reorder Dependency Tree
Andrew Jeffery89b64b62020-03-13 12:15:48 +10301248 for pkg_name, regex_str in DEPENDENCIES_REGEX.items():
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301249 dep_tree.ReorderDeps(pkg_name, regex_str)
1250 if args.verbose:
1251 dep_tree.PrintTree()
William A. Kennington III65b37fa2019-01-31 15:15:17 -08001252
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301253 install_list = dep_tree.GetInstallList()
Andrew Geissler9ced4ed2019-11-18 14:33:58 -06001254
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301255 # We don't want to treat our package as a dependency
1256 install_list.remove(UNIT_TEST_PKG)
James Feist878df5c2018-07-26 14:54:28 -07001257
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301258 # Install reordered dependencies
1259 for dep in install_list:
1260 build_and_install(dep, False)
James Feist878df5c2018-07-26 14:54:28 -07001261
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301262 # Run package unit tests
1263 build_and_install(UNIT_TEST_PKG, True)
James Feist878df5c2018-07-26 14:54:28 -07001264
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301265 os.umask(prev_umask)
Matt Spinler9bfaaad2019-10-25 09:51:50 -05001266
1267 # Run any custom CI scripts the repo has, of which there can be
1268 # multiple of and anywhere in the repository.
Patrick Williamsb6535952020-12-15 06:40:10 -06001269 ci_scripts = find_file(['run-ci.sh', 'run-ci'], CODE_SCAN_DIR)
Matt Spinler9bfaaad2019-10-25 09:51:50 -05001270 if ci_scripts:
Patrick Williamsb6535952020-12-15 06:40:10 -06001271 os.chdir(CODE_SCAN_DIR)
Matt Spinler9bfaaad2019-10-25 09:51:50 -05001272 for ci_script in ci_scripts:
Patrick Williams55448ad2020-12-14 14:28:28 -06001273 check_call_cmd(ci_script)