blob: 8583ae9466b75c133fd2a6b619845d8f815d12d4 [file] [log] [blame]
Andrew Jeffery89b64b62020-03-13 12:15:48 +10301#!/usr/bin/env python3
Matthew Barthccb7f852016-11-23 17:43:02 -06002
3"""
4This script determines the given package's openbmc dependencies from its
5configure.ac file where it downloads, configures, builds, and installs each of
6these dependencies. Then the given package is configured, built, and installed
7prior to executing its unit tests.
8"""
9
Matthew Barthd1810372016-12-19 16:57:21 -060010from git import Repo
William A. Kennington IIIfcd70772020-06-04 00:50:23 -070011from mesonbuild import coredata, optinterpreter
Andrew Jeffery89b64b62020-03-13 12:15:48 +103012from urllib.parse import urljoin
Andrew Jefferya4e31c62018-03-08 13:45:28 +103013from subprocess import check_call, call, CalledProcessError
Matthew Barthccb7f852016-11-23 17:43:02 -060014import os
15import sys
Matthew Barth33df8792016-12-19 14:30:17 -060016import argparse
William A. Kennington IIIa2156732018-06-30 18:38:09 -070017import multiprocessing
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -050018import re
William A. Kennington IIIe67f5fc2018-12-06 17:40:30 -080019import subprocess
William A. Kennington III3f1d1202018-12-06 18:02:07 -080020import shutil
William A. Kennington III4e1d0a12018-07-16 12:04:03 -070021import platform
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -050022
23
24class DepTree():
25 """
26 Represents package dependency tree, where each node is a DepTree with a
27 name and DepTree children.
28 """
29
30 def __init__(self, name):
31 """
32 Create new DepTree.
33
34 Parameter descriptions:
35 name Name of new tree node.
36 """
37 self.name = name
38 self.children = list()
39
40 def AddChild(self, name):
41 """
42 Add new child node to current node.
43
44 Parameter descriptions:
45 name Name of new child
46 """
47 new_child = DepTree(name)
48 self.children.append(new_child)
49 return new_child
50
51 def AddChildNode(self, node):
52 """
53 Add existing child node to current node.
54
55 Parameter descriptions:
56 node Tree node to add
57 """
58 self.children.append(node)
59
60 def RemoveChild(self, name):
61 """
62 Remove child node.
63
64 Parameter descriptions:
65 name Name of child to remove
66 """
67 for child in self.children:
68 if child.name == name:
69 self.children.remove(child)
70 return
71
72 def GetNode(self, name):
73 """
74 Return node with matching name. Return None if not found.
75
76 Parameter descriptions:
77 name Name of node to return
78 """
79 if self.name == name:
80 return self
81 for child in self.children:
82 node = child.GetNode(name)
83 if node:
84 return node
85 return None
86
87 def GetParentNode(self, name, parent_node=None):
88 """
89 Return parent of node with matching name. Return none if not found.
90
91 Parameter descriptions:
92 name Name of node to get parent of
93 parent_node Parent of current node
94 """
95 if self.name == name:
96 return parent_node
97 for child in self.children:
98 found_node = child.GetParentNode(name, self)
99 if found_node:
100 return found_node
101 return None
102
103 def GetPath(self, name, path=None):
104 """
105 Return list of node names from head to matching name.
106 Return None if not found.
107
108 Parameter descriptions:
109 name Name of node
110 path List of node names from head to current node
111 """
112 if not path:
113 path = []
114 if self.name == name:
115 path.append(self.name)
116 return path
117 for child in self.children:
118 match = child.GetPath(name, path + [self.name])
119 if match:
120 return match
121 return None
122
123 def GetPathRegex(self, name, regex_str, path=None):
124 """
125 Return list of node paths that end in name, or match regex_str.
126 Return empty list if not found.
127
128 Parameter descriptions:
129 name Name of node to search for
130 regex_str Regex string to match node names
131 path Path of node names from head to current node
132 """
133 new_paths = []
134 if not path:
135 path = []
136 match = re.match(regex_str, self.name)
137 if (self.name == name) or (match):
138 new_paths.append(path + [self.name])
139 for child in self.children:
140 return_paths = None
141 full_path = path + [self.name]
142 return_paths = child.GetPathRegex(name, regex_str, full_path)
143 for i in return_paths:
144 new_paths.append(i)
145 return new_paths
146
147 def MoveNode(self, from_name, to_name):
148 """
149 Mode existing from_name node to become child of to_name node.
150
151 Parameter descriptions:
152 from_name Name of node to make a child of to_name
153 to_name Name of node to make parent of from_name
154 """
155 parent_from_node = self.GetParentNode(from_name)
156 from_node = self.GetNode(from_name)
157 parent_from_node.RemoveChild(from_name)
158 to_node = self.GetNode(to_name)
159 to_node.AddChildNode(from_node)
160
161 def ReorderDeps(self, name, regex_str):
162 """
163 Reorder dependency tree. If tree contains nodes with names that
164 match 'name' and 'regex_str', move 'regex_str' nodes that are
165 to the right of 'name' node, so that they become children of the
166 'name' node.
167
168 Parameter descriptions:
169 name Name of node to look for
170 regex_str Regex string to match names to
171 """
172 name_path = self.GetPath(name)
173 if not name_path:
174 return
175 paths = self.GetPathRegex(name, regex_str)
176 is_name_in_paths = False
177 name_index = 0
178 for i in range(len(paths)):
179 path = paths[i]
180 if path[-1] == name:
181 is_name_in_paths = True
182 name_index = i
183 break
184 if not is_name_in_paths:
185 return
186 for i in range(name_index + 1, len(paths)):
187 path = paths[i]
188 if name in path:
189 continue
190 from_name = path[-1]
191 self.MoveNode(from_name, name)
192
193 def GetInstallList(self):
194 """
195 Return post-order list of node names.
196
197 Parameter descriptions:
198 """
199 install_list = []
200 for child in self.children:
201 child_install_list = child.GetInstallList()
202 install_list.extend(child_install_list)
203 install_list.append(self.name)
204 return install_list
205
206 def PrintTree(self, level=0):
207 """
208 Print pre-order node names with indentation denoting node depth level.
209
210 Parameter descriptions:
211 level Current depth level
212 """
213 INDENT_PER_LEVEL = 4
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030214 print(' ' * (level * INDENT_PER_LEVEL) + self.name)
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500215 for child in self.children:
216 child.PrintTree(level + 1)
Matthew Barth33df8792016-12-19 14:30:17 -0600217
218
William A. Kennington III1fddb972019-02-06 18:03:53 -0800219def check_call_cmd(*cmd):
Matthew Barth33df8792016-12-19 14:30:17 -0600220 """
221 Verbose prints the directory location the given command is called from and
222 the command, then executes the command using check_call.
223
224 Parameter descriptions:
225 dir Directory location command is to be called from
226 cmd List of parameters constructing the complete command
227 """
William A. Kennington III1fddb972019-02-06 18:03:53 -0800228 printline(os.getcwd(), ">", " ".join(cmd))
Matthew Barth33df8792016-12-19 14:30:17 -0600229 check_call(cmd)
Matthew Barthccb7f852016-11-23 17:43:02 -0600230
231
Andrew Geisslera61acb52019-01-03 16:32:44 -0600232def clone_pkg(pkg, branch):
Matthew Barth33df8792016-12-19 14:30:17 -0600233 """
234 Clone the given openbmc package's git repository from gerrit into
235 the WORKSPACE location
236
237 Parameter descriptions:
238 pkg Name of the package to clone
Andrew Geisslera61acb52019-01-03 16:32:44 -0600239 branch Branch to clone from pkg
Matthew Barth33df8792016-12-19 14:30:17 -0600240 """
Andrew Jeffery7be94ca2018-03-08 13:15:33 +1030241 pkg_dir = os.path.join(WORKSPACE, pkg)
242 if os.path.exists(os.path.join(pkg_dir, '.git')):
243 return pkg_dir
Matthew Barthccb7f852016-11-23 17:43:02 -0600244 pkg_repo = urljoin('https://gerrit.openbmc-project.xyz/openbmc/', pkg)
Andrew Jeffery7be94ca2018-03-08 13:15:33 +1030245 os.mkdir(pkg_dir)
Andrew Geisslera61acb52019-01-03 16:32:44 -0600246 printline(pkg_dir, "> git clone", pkg_repo, branch, "./")
247 try:
248 # first try the branch
Andrew Jeffery7d4a26f2020-03-13 11:42:34 +1030249 clone = Repo.clone_from(pkg_repo, pkg_dir, branch=branch)
250 repo_inst = clone.working_dir
Andrew Geisslera61acb52019-01-03 16:32:44 -0600251 except:
252 printline("Input branch not found, default to master")
Andrew Jeffery7d4a26f2020-03-13 11:42:34 +1030253 clone = Repo.clone_from(pkg_repo, pkg_dir, branch="master")
254 repo_inst = clone.working_dir
Andrew Geisslera61acb52019-01-03 16:32:44 -0600255 return repo_inst
Matthew Barth33df8792016-12-19 14:30:17 -0600256
257
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030258def make_target_exists(target):
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800259 """
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030260 Runs a check against the makefile in the current directory to determine
261 if the target exists so that it can be built.
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800262
263 Parameter descriptions:
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030264 target The make target we are checking
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800265 """
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030266 try:
Andrew Jeffery881041f2020-03-13 11:46:30 +1030267 cmd = ['make', '-n', target]
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030268 with open(os.devnull, 'w') as devnull:
269 check_call(cmd, stdout=devnull, stderr=devnull)
270 return True
271 except CalledProcessError:
272 return False
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800273
William A. Kennington III3f1d1202018-12-06 18:02:07 -0800274
William A. Kennington IIIa2156732018-06-30 18:38:09 -0700275make_parallel = [
276 'make',
277 # Run enough jobs to saturate all the cpus
278 '-j', str(multiprocessing.cpu_count()),
279 # Don't start more jobs if the load avg is too high
280 '-l', str(multiprocessing.cpu_count()),
281 # Synchronize the output so logs aren't intermixed in stdout / stderr
282 '-O',
283]
284
William A. Kennington III3f1d1202018-12-06 18:02:07 -0800285
Andrew Jefferyff5c5d52020-03-13 10:12:14 +1030286def build_and_install(name, build_for_testing=False):
William A. Kennington III780ec092018-12-06 14:46:50 -0800287 """
288 Builds and installs the package in the environment. Optionally
289 builds the examples and test cases for package.
290
291 Parameter description:
Andrew Jefferyff5c5d52020-03-13 10:12:14 +1030292 name The name of the package we are building
William A. Kennington IIIa0454912018-12-06 14:47:16 -0800293 build_for_testing Enable options related to testing on the package?
William A. Kennington III780ec092018-12-06 14:46:50 -0800294 """
Andrew Jefferyff5c5d52020-03-13 10:12:14 +1030295 os.chdir(os.path.join(WORKSPACE, name))
William A. Kennington III54d4faf2018-12-06 17:46:24 -0800296
297 # Refresh dynamic linker run time bindings for dependencies
William A. Kennington III1fddb972019-02-06 18:03:53 -0800298 check_call_cmd('sudo', '-n', '--', 'ldconfig')
William A. Kennington III54d4faf2018-12-06 17:46:24 -0800299
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030300 pkg = Package()
301 if build_for_testing:
302 pkg.test()
William A. Kennington III3f1d1202018-12-06 18:02:07 -0800303 else:
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030304 pkg.install()
305
William A. Kennington III780ec092018-12-06 14:46:50 -0800306
Andrew Jefferyccf85d62020-03-13 10:25:42 +1030307def build_dep_tree(name, pkgdir, dep_added, head, branch, dep_tree=None):
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500308 """
Andrew Jefferyccf85d62020-03-13 10:25:42 +1030309 For each package (name), starting with the package to be unit tested,
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030310 extract its dependencies. For each package dependency defined, recursively
311 apply the same strategy
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500312
313 Parameter descriptions:
Andrew Jefferyccf85d62020-03-13 10:25:42 +1030314 name Name of the package
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500315 pkgdir Directory where package source is located
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800316 dep_added Current dict of dependencies and added status
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500317 head Head node of the dependency tree
Andrew Geisslera61acb52019-01-03 16:32:44 -0600318 branch Branch to clone from pkg
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500319 dep_tree Current dependency tree node
320 """
321 if not dep_tree:
322 dep_tree = head
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800323
William A. Kennington IIIbe6aab22018-12-06 15:01:54 -0800324 with open("/tmp/depcache", "r") as depcache:
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800325 cache = depcache.readline()
326
327 # Read out pkg dependencies
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030328 pkg = Package(name, pkgdir)
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800329
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030330 for dep in set(pkg.build_system().dependencies()):
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800331 if dep in cache:
332 continue
333 # Dependency package not already known
334 if dep_added.get(dep) is None:
335 # Dependency package not added
336 new_child = dep_tree.AddChild(dep)
337 dep_added[dep] = False
Andrew Jeffery3b92fdd2020-03-13 11:49:18 +1030338 dep_pkgdir = clone_pkg(dep, branch)
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800339 # Determine this dependency package's
340 # dependencies and add them before
341 # returning to add this package
342 dep_added = build_dep_tree(dep,
343 dep_pkgdir,
344 dep_added,
345 head,
Andrew Geisslera61acb52019-01-03 16:32:44 -0600346 branch,
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800347 new_child)
348 else:
349 # Dependency package known and added
350 if dep_added[dep]:
Andrew Jeffery2cb0c7a2018-03-08 13:19:08 +1030351 continue
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500352 else:
William A. Kennington IIIc048cc02018-12-06 15:39:18 -0800353 # Cyclic dependency failure
Andrew Jefferyccf85d62020-03-13 10:25:42 +1030354 raise Exception("Cyclic dependencies found in "+name)
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500355
Andrew Jefferyccf85d62020-03-13 10:25:42 +1030356 if not dep_added[name]:
357 dep_added[name] = True
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -0500358
359 return dep_added
Matthew Barthccb7f852016-11-23 17:43:02 -0600360
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700361
William A. Kennington III90b106a2019-02-06 18:08:24 -0800362def run_cppcheck():
Andrew Jefferybf735102020-03-13 12:13:21 +1030363 match_re = re.compile(r'((?!\.mako\.).)*\.[ch](?:pp)?$', re.I)
Brad Bishop48424d42020-01-07 13:01:31 -0500364 cppcheck_files = []
365 stdout = subprocess.check_output(['git', 'ls-files'])
Patrick Venturead4354e2018-10-12 16:59:54 -0700366
Brad Bishop48424d42020-01-07 13:01:31 -0500367 for f in stdout.decode('utf-8').split():
368 if match_re.match(f):
369 cppcheck_files.append(f)
370
371 if not cppcheck_files:
372 # skip cppcheck if there arent' any c or cpp sources.
373 print("no files")
374 return None
375
376 # http://cppcheck.sourceforge.net/manual.pdf
377 params = ['cppcheck', '-j', str(multiprocessing.cpu_count()),
Manojkiran Eda89a46cc2020-08-22 17:08:11 +0530378 '--enable=all', '--library=googletest', '--file-list=-']
Brad Bishop48424d42020-01-07 13:01:31 -0500379
380 cppcheck_process = subprocess.Popen(
381 params,
382 stdout=subprocess.PIPE,
383 stderr=subprocess.PIPE,
384 stdin=subprocess.PIPE)
385 (stdout, stderr) = cppcheck_process.communicate(
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030386 input='\n'.join(cppcheck_files).encode('utf-8'))
Brad Bishop48424d42020-01-07 13:01:31 -0500387
388 if cppcheck_process.wait():
Patrick Venturead4354e2018-10-12 16:59:54 -0700389 raise Exception('Cppcheck failed')
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030390 print(stdout.decode('utf-8'))
391 print(stderr.decode('utf-8'))
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700392
Andrew Jefferye5fffa02020-03-13 11:51:57 +1030393
William A. Kennington III37a89a22018-12-13 14:32:02 -0800394def is_valgrind_safe():
395 """
396 Returns whether it is safe to run valgrind on our platform
397 """
William A. Kennington III0326ded2019-02-07 00:33:28 -0800398 src = 'unit-test-vg.c'
399 exe = './unit-test-vg'
400 with open(src, 'w') as h:
William A. Kennington IIIafb0f982019-04-26 17:30:28 -0700401 h.write('#include <errno.h>\n')
402 h.write('#include <stdio.h>\n')
William A. Kennington III0326ded2019-02-07 00:33:28 -0800403 h.write('#include <stdlib.h>\n')
404 h.write('#include <string.h>\n')
405 h.write('int main() {\n')
406 h.write('char *heap_str = malloc(16);\n')
407 h.write('strcpy(heap_str, "RandString");\n')
408 h.write('int res = strcmp("RandString", heap_str);\n')
409 h.write('free(heap_str);\n')
William A. Kennington IIIafb0f982019-04-26 17:30:28 -0700410 h.write('char errstr[64];\n')
411 h.write('strerror_r(EINVAL, errstr, sizeof(errstr));\n')
412 h.write('printf("%s\\n", errstr);\n')
William A. Kennington III0326ded2019-02-07 00:33:28 -0800413 h.write('return res;\n')
414 h.write('}\n')
415 try:
416 with open(os.devnull, 'w') as devnull:
417 check_call(['gcc', '-O2', '-o', exe, src],
418 stdout=devnull, stderr=devnull)
419 check_call(['valgrind', '--error-exitcode=99', exe],
420 stdout=devnull, stderr=devnull)
421 return True
422 except:
423 sys.stderr.write("###### Platform is not valgrind safe ######\n")
424 return False
425 finally:
426 os.remove(src)
427 os.remove(exe)
William A. Kennington III37a89a22018-12-13 14:32:02 -0800428
Andrew Jefferye5fffa02020-03-13 11:51:57 +1030429
William A. Kennington III282e3302019-02-04 16:55:05 -0800430def is_sanitize_safe():
431 """
432 Returns whether it is safe to run sanitizers on our platform
433 """
William A. Kennington III0b7fb2b2019-02-07 00:33:42 -0800434 src = 'unit-test-sanitize.c'
435 exe = './unit-test-sanitize'
436 with open(src, 'w') as h:
437 h.write('int main() { return 0; }\n')
438 try:
439 with open(os.devnull, 'w') as devnull:
440 check_call(['gcc', '-O2', '-fsanitize=address',
441 '-fsanitize=undefined', '-o', exe, src],
442 stdout=devnull, stderr=devnull)
443 check_call([exe], stdout=devnull, stderr=devnull)
444 return True
445 except:
446 sys.stderr.write("###### Platform is not sanitize safe ######\n")
447 return False
448 finally:
449 os.remove(src)
450 os.remove(exe)
William A. Kennington III282e3302019-02-04 16:55:05 -0800451
William A. Kennington III49d4e592019-02-06 17:59:27 -0800452
William A. Kennington IIIeaff24a2019-02-06 16:57:42 -0800453def maybe_make_valgrind():
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700454 """
455 Potentially runs the unit tests through valgrind for the package
456 via `make check-valgrind`. If the package does not have valgrind testing
457 then it just skips over this.
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700458 """
William A. Kennington III4e1d0a12018-07-16 12:04:03 -0700459 # Valgrind testing is currently broken by an aggressive strcmp optimization
460 # that is inlined into optimized code for POWER by gcc 7+. Until we find
461 # a workaround, just don't run valgrind tests on POWER.
462 # https://github.com/openbmc/openbmc/issues/3315
William A. Kennington III37a89a22018-12-13 14:32:02 -0800463 if not is_valgrind_safe():
William A. Kennington III75130192019-02-07 00:34:14 -0800464 sys.stderr.write("###### Skipping valgrind ######\n")
William A. Kennington III4e1d0a12018-07-16 12:04:03 -0700465 return
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700466 if not make_target_exists('check-valgrind'):
467 return
468
469 try:
Andrew Jeffery881041f2020-03-13 11:46:30 +1030470 cmd = make_parallel + ['check-valgrind']
William A. Kennington III1fddb972019-02-06 18:03:53 -0800471 check_call_cmd(*cmd)
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700472 except CalledProcessError:
William A. Kennington III90b106a2019-02-06 18:08:24 -0800473 for root, _, files in os.walk(os.getcwd()):
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700474 for f in files:
475 if re.search('test-suite-[a-z]+.log', f) is None:
476 continue
William A. Kennington III1fddb972019-02-06 18:03:53 -0800477 check_call_cmd('cat', os.path.join(root, f))
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700478 raise Exception('Valgrind tests failed')
479
Andrew Jefferye5fffa02020-03-13 11:51:57 +1030480
William A. Kennington IIIeaff24a2019-02-06 16:57:42 -0800481def maybe_make_coverage():
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700482 """
483 Potentially runs the unit tests through code coverage for the package
484 via `make check-code-coverage`. If the package does not have code coverage
485 testing then it just skips over this.
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700486 """
487 if not make_target_exists('check-code-coverage'):
488 return
489
490 # Actually run code coverage
491 try:
Andrew Jeffery881041f2020-03-13 11:46:30 +1030492 cmd = make_parallel + ['check-code-coverage']
William A. Kennington III1fddb972019-02-06 18:03:53 -0800493 check_call_cmd(*cmd)
William A. Kennington III0f0a6802018-07-16 11:52:33 -0700494 except CalledProcessError:
495 raise Exception('Code coverage failed')
Matthew Barthccb7f852016-11-23 17:43:02 -0600496
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030497
498class BuildSystem(object):
499 """
500 Build systems generally provide the means to configure, build, install and
501 test software. The BuildSystem class defines a set of interfaces on top of
502 which Autotools, Meson, CMake and possibly other build system drivers can
503 be implemented, separating out the phases to control whether a package
504 should merely be installed or also tested and analyzed.
505 """
506 def __init__(self, package, path):
507 """Initialise the driver with properties independent of the build system
508
509 Keyword arguments:
510 package: The name of the package. Derived from the path if None
511 path: The path to the package. Set to the working directory if None
512 """
513 self.path = "." if not path else path
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030514 realpath = os.path.realpath(self.path)
515 self.package = package if package else os.path.basename(realpath)
Andrew Jeffery3f8e5a72020-03-13 11:47:56 +1030516 self.build_for_testing = False
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030517
518 def probe(self):
519 """Test if the build system driver can be applied to the package
520
521 Return True if the driver can drive the package's build system,
522 otherwise False.
523
524 Generally probe() is implemented by testing for the presence of the
525 build system's configuration file(s).
526 """
527 raise NotImplemented
528
529 def dependencies(self):
530 """Provide the package's dependencies
531
532 Returns a list of dependencies. If no dependencies are required then an
533 empty list must be returned.
534
535 Generally dependencies() is implemented by analysing and extracting the
536 data from the build system configuration.
537 """
538 raise NotImplemented
539
540 def configure(self, build_for_testing):
541 """Configure the source ready for building
542
543 Should raise an exception if configuration failed.
544
545 Keyword arguments:
546 build_for_testing: Mark the package as being built for testing rather
547 than for installation as a dependency for the
548 package under test. Setting to True generally
549 implies that the package will be configured to build
550 with debug information, at a low level of
551 optimisation and possibly with sanitizers enabled.
552
553 Generally configure() is implemented by invoking the build system
554 tooling to generate Makefiles or equivalent.
555 """
556 raise NotImplemented
557
558 def build(self):
559 """Build the software ready for installation and/or testing
560
561 Should raise an exception if the build fails
562
563 Generally build() is implemented by invoking `make` or `ninja`.
564 """
565 raise NotImplemented
566
567 def install(self):
568 """Install the software ready for use
569
570 Should raise an exception if installation fails
571
572 Like build(), install() is generally implemented by invoking `make` or
573 `ninja`.
574 """
575 raise NotImplemented
576
577 def test(self):
578 """Build and run the test suite associated with the package
579
580 Should raise an exception if the build or testing fails.
581
582 Like install(), test() is generally implemented by invoking `make` or
583 `ninja`.
584 """
585 raise NotImplemented
586
587 def analyze(self):
588 """Run any supported analysis tools over the codebase
589
590 Should raise an exception if analysis fails.
591
592 Some analysis tools such as scan-build need injection into the build
593 system. analyze() provides the necessary hook to implement such
594 behaviour. Analyzers independent of the build system can also be
595 specified here but at the cost of possible duplication of code between
596 the build system driver implementations.
597 """
598 raise NotImplemented
599
600
601class Autotools(BuildSystem):
602 def __init__(self, package=None, path=None):
603 super(Autotools, self).__init__(package, path)
604
605 def probe(self):
606 return os.path.isfile(os.path.join(self.path, 'configure.ac'))
607
608 def dependencies(self):
609 configure_ac = os.path.join(self.path, 'configure.ac')
610
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030611 contents = ''
Andrew Jeffery7d4a26f2020-03-13 11:42:34 +1030612 # Prepend some special function overrides so we can parse out
613 # dependencies
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030614 for macro in DEPENDENCIES.keys():
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030615 contents += ('m4_define([' + macro + '], [' + macro + '_START$' +
616 str(DEPENDENCIES_OFFSET[macro] + 1) +
617 macro + '_END])\n')
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030618 with open(configure_ac, "rt") as f:
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030619 contents += f.read()
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030620
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030621 autoconf_cmdline = ['autoconf', '-Wno-undefined', '-']
622 autoconf_process = subprocess.Popen(autoconf_cmdline,
Andrew Jeffery7d4a26f2020-03-13 11:42:34 +1030623 stdin=subprocess.PIPE,
624 stdout=subprocess.PIPE,
625 stderr=subprocess.PIPE)
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030626 document = contents.encode('utf-8')
627 (stdout, stderr) = autoconf_process.communicate(input=document)
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030628 if not stdout:
629 print(stderr)
630 raise Exception("Failed to run autoconf for parsing dependencies")
631
632 # Parse out all of the dependency text
633 matches = []
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030634 for macro in DEPENDENCIES.keys():
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030635 pattern = '(' + macro + ')_START(.*?)' + macro + '_END'
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030636 for match in re.compile(pattern).finditer(stdout.decode('utf-8')):
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030637 matches.append((match.group(1), match.group(2)))
638
639 # Look up dependencies from the text
640 found_deps = []
641 for macro, deptext in matches:
642 for potential_dep in deptext.split(' '):
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030643 for known_dep in DEPENDENCIES[macro].keys():
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030644 if potential_dep.startswith(known_dep):
645 found_deps.append(DEPENDENCIES[macro][known_dep])
646
647 return found_deps
648
649 def _configure_feature(self, flag, enabled):
650 """
651 Returns an configure flag as a string
652
653 Parameters:
654 flag The name of the flag
655 enabled Whether the flag is enabled or disabled
656 """
657 return '--' + ('enable' if enabled else 'disable') + '-' + flag
658
659 def configure(self, build_for_testing):
660 self.build_for_testing = build_for_testing
661 conf_flags = [
662 self._configure_feature('silent-rules', False),
663 self._configure_feature('examples', build_for_testing),
664 self._configure_feature('tests', build_for_testing),
665 ]
666 if not TEST_ONLY:
667 conf_flags.extend([
668 self._configure_feature('code-coverage', build_for_testing),
669 self._configure_feature('valgrind', build_for_testing),
670 ])
671 # Add any necessary configure flags for package
672 if CONFIGURE_FLAGS.get(self.package) is not None:
673 conf_flags.extend(CONFIGURE_FLAGS.get(self.package))
674 for bootstrap in ['bootstrap.sh', 'bootstrap', 'autogen.sh']:
675 if os.path.exists(bootstrap):
676 check_call_cmd('./' + bootstrap)
677 break
678 check_call_cmd('./configure', *conf_flags)
679
680 def build(self):
681 check_call_cmd(*make_parallel)
682
683 def install(self):
Andrew Jeffery881041f2020-03-13 11:46:30 +1030684 check_call_cmd('sudo', '-n', '--', *(make_parallel + ['install']))
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030685
686 def test(self):
687 try:
Andrew Jeffery881041f2020-03-13 11:46:30 +1030688 cmd = make_parallel + ['check']
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030689 for i in range(0, args.repeat):
690 check_call_cmd(*cmd)
691 except CalledProcessError:
692 for root, _, files in os.walk(os.getcwd()):
693 if 'test-suite.log' not in files:
694 continue
695 check_call_cmd('cat', os.path.join(root, 'test-suite.log'))
696 raise Exception('Unit tests failed')
697
698 def analyze(self):
699 maybe_make_valgrind()
700 maybe_make_coverage()
701 run_cppcheck()
702
703
704class CMake(BuildSystem):
705 def __init__(self, package=None, path=None):
706 super(CMake, self).__init__(package, path)
707
708 def probe(self):
709 return os.path.isfile(os.path.join(self.path, 'CMakeLists.txt'))
710
711 def dependencies(self):
712 return []
713
714 def configure(self, build_for_testing):
715 self.build_for_testing = build_for_testing
716 check_call_cmd('cmake', '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', '.')
717
718 def build(self):
719 check_call_cmd('cmake', '--build', '.', '--', '-j',
720 str(multiprocessing.cpu_count()))
721
722 def install(self):
723 pass
724
725 def test(self):
726 if make_target_exists('test'):
727 check_call_cmd('ctest', '.')
728
729 def analyze(self):
Andrew Geissler38356642020-03-19 15:12:44 -0500730 if TEST_ONLY:
731 return
732
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030733 if os.path.isfile('.clang-tidy'):
Patrick Williams80d07282020-05-20 10:56:22 -0500734 check_call_cmd('run-clang-tidy-10.py', '-p', '.')
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030735 maybe_make_valgrind()
736 maybe_make_coverage()
737 run_cppcheck()
738
739
740class Meson(BuildSystem):
741 def __init__(self, package=None, path=None):
742 super(Meson, self).__init__(package, path)
743
744 def probe(self):
745 return os.path.isfile(os.path.join(self.path, 'meson.build'))
746
747 def dependencies(self):
748 meson_build = os.path.join(self.path, 'meson.build')
749 if not os.path.exists(meson_build):
750 return []
751
752 found_deps = []
753 for root, dirs, files in os.walk(self.path):
754 if 'meson.build' not in files:
755 continue
756 with open(os.path.join(root, 'meson.build'), 'rt') as f:
757 build_contents = f.read()
Nan Zhouef13d532020-07-07 09:52:02 -0700758 pattern = r"dependency\('([^']*)'.*?\),?\n"
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030759 for match in re.finditer(pattern, build_contents):
760 group = match.group(1)
761 maybe_dep = DEPENDENCIES['PKG_CHECK_MODULES'].get(group)
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030762 if maybe_dep is not None:
763 found_deps.append(maybe_dep)
764
765 return found_deps
766
767 def _parse_options(self, options_file):
768 """
769 Returns a set of options defined in the provides meson_options.txt file
770
771 Parameters:
772 options_file The file containing options
773 """
William A. Kennington III0ecd0bc2020-06-04 00:44:22 -0700774 oi = optinterpreter.OptionInterpreter('')
775 oi.process(options_file)
776 return oi.options
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030777
William A. Kennington IIIfcd70772020-06-04 00:50:23 -0700778 def _configure_boolean(self, val):
779 """
780 Returns the meson flag which signifies the value
781
782 True is true which requires the boolean.
783 False is false which disables the boolean.
784
785 Parameters:
786 val The value being converted
787 """
788 if val is True:
789 return 'true'
790 elif val is False:
791 return 'false'
792 else:
793 raise Exception("Bad meson boolean value")
794
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030795 def _configure_feature(self, val):
796 """
797 Returns the meson flag which signifies the value
798
799 True is enabled which requires the feature.
800 False is disabled which disables the feature.
801 None is auto which autodetects the feature.
802
803 Parameters:
804 val The value being converted
805 """
806 if val is True:
807 return "enabled"
808 elif val is False:
809 return "disabled"
810 elif val is None:
811 return "auto"
812 else:
813 raise Exception("Bad meson feature value")
814
William A. Kennington IIIfcd70772020-06-04 00:50:23 -0700815 def _configure_option(self, opts, key, val):
816 """
817 Returns the meson flag which signifies the value
818 based on the type of the opt
819
820 Parameters:
821 opt The meson option which we are setting
822 val The value being converted
823 """
824 if isinstance(opts[key], coredata.UserBooleanOption):
825 str_val = self._configure_boolean(val)
826 elif isinstance(opts[key], coredata.UserFeatureOption):
827 str_val = self._configure_feature(val)
828 else:
829 raise Exception('Unknown meson option type')
830 return "-D{}={}".format(key, str_val)
831
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030832 def configure(self, build_for_testing):
833 self.build_for_testing = build_for_testing
William A. Kennington III0ecd0bc2020-06-04 00:44:22 -0700834 meson_options = {}
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030835 if os.path.exists("meson_options.txt"):
836 meson_options = self._parse_options("meson_options.txt")
837 meson_flags = [
838 '-Db_colorout=never',
839 '-Dwerror=true',
840 '-Dwarning_level=3',
841 ]
842 if build_for_testing:
843 meson_flags.append('--buildtype=debug')
844 else:
845 meson_flags.append('--buildtype=debugoptimized')
846 if 'tests' in meson_options:
William A. Kennington IIIfcd70772020-06-04 00:50:23 -0700847 meson_flags.append(self._configure_option(meson_options, 'tests', build_for_testing))
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030848 if 'examples' in meson_options:
William A. Kennington IIIfcd70772020-06-04 00:50:23 -0700849 meson_flags.append(self._configure_option(meson_options, 'examples', build_for_testing))
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030850 if MESON_FLAGS.get(self.package) is not None:
851 meson_flags.extend(MESON_FLAGS.get(self.package))
852 try:
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030853 check_call_cmd('meson', 'setup', '--reconfigure', 'build',
854 *meson_flags)
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030855 except:
856 shutil.rmtree('build')
857 check_call_cmd('meson', 'setup', 'build', *meson_flags)
858
859 def build(self):
860 check_call_cmd('ninja', '-C', 'build')
861
862 def install(self):
863 check_call_cmd('sudo', '-n', '--', 'ninja', '-C', 'build', 'install')
864
865 def test(self):
866 try:
Brad Bishop7b8cef22020-08-26 15:58:09 -0400867 test_args = ('--repeat', str(args.repeat), '-C', 'build')
868 check_call_cmd('meson', 'test', *test_args)
869
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030870 except CalledProcessError:
871 for root, _, files in os.walk(os.getcwd()):
872 if 'testlog.txt' not in files:
873 continue
874 check_call_cmd('cat', os.path.join(root, 'testlog.txt'))
875 raise Exception('Unit tests failed')
876
877 def _setup_exists(self, setup):
878 """
879 Returns whether the meson build supports the named test setup.
880
881 Parameter descriptions:
882 setup The setup target to check
883 """
884 try:
885 with open(os.devnull, 'w') as devnull:
886 output = subprocess.check_output(
887 ['meson', 'test', '-C', 'build',
888 '--setup', setup, '-t', '0'],
889 stderr=subprocess.STDOUT)
890 except CalledProcessError as e:
891 output = e.output
Andrew Jeffery89b64b62020-03-13 12:15:48 +1030892 output = output.decode('utf-8')
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030893 return not re.search('Test setup .* not found from project', output)
894
895 def _maybe_valgrind(self):
896 """
897 Potentially runs the unit tests through valgrind for the package
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030898 via `meson test`. The package can specify custom valgrind
899 configurations by utilizing add_test_setup() in a meson.build
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030900 """
901 if not is_valgrind_safe():
902 sys.stderr.write("###### Skipping valgrind ######\n")
903 return
904 try:
905 if self._setup_exists('valgrind'):
906 check_call_cmd('meson', 'test', '-C', 'build',
907 '--setup', 'valgrind')
908 else:
909 check_call_cmd('meson', 'test', '-C', 'build',
910 '--wrapper', 'valgrind')
911 except CalledProcessError:
912 for root, _, files in os.walk(os.getcwd()):
913 if 'testlog-valgrind.txt' not in files:
914 continue
Andrew Jeffery47fbfa52020-03-13 12:05:09 +1030915 cat_args = os.path.join(root, 'testlog-valgrind.txt')
916 check_call_cmd('cat', cat_args)
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030917 raise Exception('Valgrind tests failed')
918
919 def analyze(self):
Andrew Geissler38356642020-03-19 15:12:44 -0500920 if TEST_ONLY:
921 return
922
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030923 self._maybe_valgrind()
924
925 # Run clang-tidy only if the project has a configuration
926 if os.path.isfile('.clang-tidy'):
Patrick Williams80d07282020-05-20 10:56:22 -0500927 check_call_cmd('run-clang-tidy-10.py', '-p',
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030928 'build')
929 # Run the basic clang static analyzer otherwise
930 else:
931 check_call_cmd('ninja', '-C', 'build',
932 'scan-build')
933
934 # Run tests through sanitizers
935 # b_lundef is needed if clang++ is CXX since it resolves the
936 # asan symbols at runtime only. We don't want to set it earlier
937 # in the build process to ensure we don't have undefined
938 # runtime code.
939 if is_sanitize_safe():
940 check_call_cmd('meson', 'configure', 'build',
941 '-Db_sanitize=address,undefined',
942 '-Db_lundef=false')
943 check_call_cmd('meson', 'test', '-C', 'build',
944 '--logbase', 'testlog-ubasan')
945 # TODO: Fix memory sanitizer
Andrew Jeffery6015ca12020-03-13 11:50:10 +1030946 # check_call_cmd('meson', 'configure', 'build',
947 # '-Db_sanitize=memory')
948 # check_call_cmd('meson', 'test', '-C', 'build'
949 # '--logbase', 'testlog-msan')
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030950 check_call_cmd('meson', 'configure', 'build',
Lei YUf684ad62020-09-21 17:02:26 +0800951 '-Db_sanitize=none')
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030952 else:
953 sys.stderr.write("###### Skipping sanitizers ######\n")
954
955 # Run coverage checks
956 check_call_cmd('meson', 'configure', 'build',
957 '-Db_coverage=true')
958 self.test()
959 # Only build coverage HTML if coverage files were produced
960 for root, dirs, files in os.walk('build'):
961 if any([f.endswith('.gcda') for f in files]):
962 check_call_cmd('ninja', '-C', 'build',
963 'coverage-html')
964 break
965 check_call_cmd('meson', 'configure', 'build',
966 '-Db_coverage=false')
967 run_cppcheck()
968
969
970class Package(object):
971 def __init__(self, name=None, path=None):
Manojkiran Eda29d28252020-06-15 18:58:09 +0530972 self.supported = [Meson, Autotools, CMake]
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030973 self.name = name
974 self.path = path
975 self.test_only = False
976
977 def build_systems(self):
Andrew Jeffery881041f2020-03-13 11:46:30 +1030978 instances = (system(self.name, self.path) for system in self.supported)
979 return (instance for instance in instances if instance.probe())
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030980
981 def build_system(self, preferred=None):
Andrew Geissler8cb74fc2020-03-19 14:48:05 -0500982 systems = list(self.build_systems())
983
984 if not systems:
985 return None
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030986
987 if preferred:
Andrew Jeffery881041f2020-03-13 11:46:30 +1030988 return {type(system): system for system in systems}[preferred]
Andrew Jeffery15e423e2020-03-11 16:51:28 +1030989
990 return next(iter(systems))
991
992 def install(self, system=None):
993 if not system:
994 system = self.build_system()
995
996 system.configure(False)
997 system.build()
998 system.install()
999
Andrew Jeffery19d75672020-03-13 10:42:08 +10301000 def _test_one(self, system):
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301001 system.configure(True)
1002 system.build()
1003 system.install()
1004 system.test()
1005 system.analyze()
1006
Andrew Jeffery19d75672020-03-13 10:42:08 +10301007 def test(self):
1008 for system in self.build_systems():
1009 self._test_one(system)
1010
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301011
Matt Spinler9bfaaad2019-10-25 09:51:50 -05001012def find_file(filename, basedir):
1013 """
1014 Finds all occurrences of a file in the base directory
1015 and passes them back with their relative paths.
1016
1017 Parameter descriptions:
1018 filename The name of the file to find
1019 basedir The base directory search in
1020 """
1021
1022 filepaths = []
1023 for root, dirs, files in os.walk(basedir):
1024 if filename in files:
Andrew Jeffery2ba45fb2020-03-13 11:25:01 +10301025 filepaths.append(os.path.join(root, filename))
Matt Spinler9bfaaad2019-10-25 09:51:50 -05001026 return filepaths
1027
Andrew Jeffery9cc97d82020-03-13 11:57:44 +10301028
Matthew Barthccb7f852016-11-23 17:43:02 -06001029if __name__ == '__main__':
1030 # CONFIGURE_FLAGS = [GIT REPO]:[CONFIGURE FLAGS]
1031 CONFIGURE_FLAGS = {
Matthew Barth1d1c6732017-03-24 10:00:28 -05001032 'phosphor-logging':
Matt Spinler0744bb82020-01-16 08:23:35 -06001033 ['--enable-metadata-processing', '--enable-openpower-pel-extension',
Deepak Kodihalli3a4e1b42017-06-08 09:52:35 -05001034 'YAML_DIR=/usr/local/share/phosphor-dbus-yaml/yaml']
Matthew Barthccb7f852016-11-23 17:43:02 -06001035 }
1036
William A. Kennington III3f1d1202018-12-06 18:02:07 -08001037 # MESON_FLAGS = [GIT REPO]:[MESON FLAGS]
1038 MESON_FLAGS = {
Matt Spinler8448cfd2020-09-10 17:01:48 -05001039 'phosphor-dbus-interfaces':
1040 ['-Ddata_com_ibm=true', '-Ddata_org_open_power=true']
William A. Kennington III3f1d1202018-12-06 18:02:07 -08001041 }
1042
Matthew Barthccb7f852016-11-23 17:43:02 -06001043 # DEPENDENCIES = [MACRO]:[library/header]:[GIT REPO]
1044 DEPENDENCIES = {
1045 'AC_CHECK_LIB': {'mapper': 'phosphor-objmgr'},
Matthew Barth710f3f02017-01-18 15:20:19 -06001046 'AC_CHECK_HEADER': {
1047 'host-ipmid': 'phosphor-host-ipmid',
Patrick Ventureb41a4462018-10-03 17:27:38 -07001048 'blobs-ipmid': 'phosphor-ipmi-blobs',
Matthew Barth710f3f02017-01-18 15:20:19 -06001049 'sdbusplus': 'sdbusplus',
William A. Kennington IIIb4f730a2018-09-12 11:21:20 -07001050 'sdeventplus': 'sdeventplus',
William A. Kennington III23705242019-01-15 18:17:25 -08001051 'stdplus': 'stdplus',
Patrick Venture22329962018-09-14 10:23:04 -07001052 'gpioplus': 'gpioplus',
Saqib Khan66145052017-02-14 12:02:07 -06001053 'phosphor-logging/log.hpp': 'phosphor-logging',
Patrick Williamseab8a372017-01-30 11:21:32 -06001054 },
Brad Bishopebb49112017-02-13 11:07:26 -05001055 'AC_PATH_PROG': {'sdbus++': 'sdbusplus'},
Patrick Williamseab8a372017-01-30 11:21:32 -06001056 'PKG_CHECK_MODULES': {
Matthew Barth19e261e2017-02-01 12:55:22 -06001057 'phosphor-dbus-interfaces': 'phosphor-dbus-interfaces',
William A. Kennington III993ab332019-02-07 02:12:31 -08001058 'libipmid': 'phosphor-host-ipmid',
1059 'libipmid-host': 'phosphor-host-ipmid',
Brad Bishopebb49112017-02-13 11:07:26 -05001060 'sdbusplus': 'sdbusplus',
William A. Kennington IIIb4f730a2018-09-12 11:21:20 -07001061 'sdeventplus': 'sdeventplus',
William A. Kennington III23705242019-01-15 18:17:25 -08001062 'stdplus': 'stdplus',
Patrick Venture22329962018-09-14 10:23:04 -07001063 'gpioplus': 'gpioplus',
Brad Bishopebb49112017-02-13 11:07:26 -05001064 'phosphor-logging': 'phosphor-logging',
Marri Devender Raoa3eee8a2018-08-13 05:34:27 -05001065 'phosphor-snmp': 'phosphor-snmp',
Patrick Venturee584c3b2019-03-07 09:44:16 -08001066 'ipmiblob': 'ipmi-blob-tool',
Ben Tyner300ed3c2020-05-14 15:30:37 -05001067 'hei': 'openpower-libhei',
Brad Bishopebb49112017-02-13 11:07:26 -05001068 },
Matthew Barthccb7f852016-11-23 17:43:02 -06001069 }
1070
William A. Kennington IIIe67f5fc2018-12-06 17:40:30 -08001071 # Offset into array of macro parameters MACRO(0, 1, ...N)
1072 DEPENDENCIES_OFFSET = {
1073 'AC_CHECK_LIB': 0,
1074 'AC_CHECK_HEADER': 0,
1075 'AC_PATH_PROG': 1,
1076 'PKG_CHECK_MODULES': 1,
1077 }
1078
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -05001079 # DEPENDENCIES_REGEX = [GIT REPO]:[REGEX STRING]
1080 DEPENDENCIES_REGEX = {
Patrick Ventured37b8052018-10-16 16:03:03 -07001081 'phosphor-logging': r'\S+-dbus-interfaces$'
Leonel Gonzaleza62a1a12017-03-24 11:03:47 -05001082 }
1083
Matthew Barth33df8792016-12-19 14:30:17 -06001084 # Set command line arguments
1085 parser = argparse.ArgumentParser()
1086 parser.add_argument("-w", "--workspace", dest="WORKSPACE", required=True,
1087 help="Workspace directory location(i.e. /home)")
1088 parser.add_argument("-p", "--package", dest="PACKAGE", required=True,
1089 help="OpenBMC package to be unit tested")
William A. Kennington III65b37fa2019-01-31 15:15:17 -08001090 parser.add_argument("-t", "--test-only", dest="TEST_ONLY",
1091 action="store_true", required=False, default=False,
1092 help="Only run test cases, no other validation")
Matthew Barth33df8792016-12-19 14:30:17 -06001093 parser.add_argument("-v", "--verbose", action="store_true",
1094 help="Print additional package status messages")
Andrew Jeffery468309d2018-03-08 13:46:33 +10301095 parser.add_argument("-r", "--repeat", help="Repeat tests N times",
1096 type=int, default=1)
Andrew Geisslera61acb52019-01-03 16:32:44 -06001097 parser.add_argument("-b", "--branch", dest="BRANCH", required=False,
1098 help="Branch to target for dependent repositories",
1099 default="master")
Lei YU7ef93302019-11-06 13:53:21 +08001100 parser.add_argument("-n", "--noformat", dest="FORMAT",
1101 action="store_false", required=False,
1102 help="Whether or not to run format code")
Matthew Barth33df8792016-12-19 14:30:17 -06001103 args = parser.parse_args(sys.argv[1:])
1104 WORKSPACE = args.WORKSPACE
1105 UNIT_TEST_PKG = args.PACKAGE
William A. Kennington III65b37fa2019-01-31 15:15:17 -08001106 TEST_ONLY = args.TEST_ONLY
Andrew Geisslera61acb52019-01-03 16:32:44 -06001107 BRANCH = args.BRANCH
Lei YU7ef93302019-11-06 13:53:21 +08001108 FORMAT_CODE = args.FORMAT
Matthew Barth33df8792016-12-19 14:30:17 -06001109 if args.verbose:
1110 def printline(*line):
1111 for arg in line:
Andrew Jeffery89b64b62020-03-13 12:15:48 +10301112 print(arg, end=' ')
1113 print()
Matthew Barth33df8792016-12-19 14:30:17 -06001114 else:
Andrew Jefferyfce557f2020-03-13 12:12:19 +10301115 def printline(*line):
1116 pass
Matthew Barthccb7f852016-11-23 17:43:02 -06001117
Lei YU7ef93302019-11-06 13:53:21 +08001118 CODE_SCAN_DIR = WORKSPACE + "/" + UNIT_TEST_PKG
1119
James Feist878df5c2018-07-26 14:54:28 -07001120 # First validate code formatting if repo has style formatting files.
Adriana Kobylakbcee22b2018-01-10 16:58:27 -06001121 # The format-code.sh checks for these files.
Lei YU7ef93302019-11-06 13:53:21 +08001122 if FORMAT_CODE:
1123 check_call_cmd("./format-code.sh", CODE_SCAN_DIR)
Andrew Geisslera28286d2018-01-10 11:00:00 -08001124
Andrew Geissler8cb74fc2020-03-19 14:48:05 -05001125 # Check if this repo has a supported make infrastructure
1126 pkg = Package(UNIT_TEST_PKG, os.path.join(WORKSPACE, UNIT_TEST_PKG))
1127 if not pkg.build_system():
1128 print("No valid build system, exit")
1129 sys.exit(0)
1130
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301131 prev_umask = os.umask(000)
James Feist878df5c2018-07-26 14:54:28 -07001132
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301133 # Determine dependencies and add them
1134 dep_added = dict()
1135 dep_added[UNIT_TEST_PKG] = False
William A. Kennington III40d5c7c2018-12-13 14:37:59 -08001136
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301137 # Create dependency tree
1138 dep_tree = DepTree(UNIT_TEST_PKG)
1139 build_dep_tree(UNIT_TEST_PKG,
1140 os.path.join(WORKSPACE, UNIT_TEST_PKG),
1141 dep_added,
1142 dep_tree,
1143 BRANCH)
William A. Kennington III65b37fa2019-01-31 15:15:17 -08001144
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301145 # Reorder Dependency Tree
Andrew Jeffery89b64b62020-03-13 12:15:48 +10301146 for pkg_name, regex_str in DEPENDENCIES_REGEX.items():
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301147 dep_tree.ReorderDeps(pkg_name, regex_str)
1148 if args.verbose:
1149 dep_tree.PrintTree()
William A. Kennington III65b37fa2019-01-31 15:15:17 -08001150
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301151 install_list = dep_tree.GetInstallList()
Andrew Geissler9ced4ed2019-11-18 14:33:58 -06001152
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301153 # We don't want to treat our package as a dependency
1154 install_list.remove(UNIT_TEST_PKG)
James Feist878df5c2018-07-26 14:54:28 -07001155
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301156 # Install reordered dependencies
1157 for dep in install_list:
1158 build_and_install(dep, False)
James Feist878df5c2018-07-26 14:54:28 -07001159
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301160 # Run package unit tests
1161 build_and_install(UNIT_TEST_PKG, True)
James Feist878df5c2018-07-26 14:54:28 -07001162
Andrew Jeffery15e423e2020-03-11 16:51:28 +10301163 os.umask(prev_umask)
Matt Spinler9bfaaad2019-10-25 09:51:50 -05001164
1165 # Run any custom CI scripts the repo has, of which there can be
1166 # multiple of and anywhere in the repository.
1167 ci_scripts = find_file('run-ci.sh', os.path.join(WORKSPACE, UNIT_TEST_PKG))
1168 if ci_scripts:
1169 os.chdir(os.path.join(WORKSPACE, UNIT_TEST_PKG))
1170 for ci_script in ci_scripts:
1171 check_call_cmd('sh', ci_script)