blob: 5686a62d3f36a39bb9d72fb4df321299db2b498f [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Recipe creation tool - create build system handler for python
2#
3# Copyright (C) 2015 Mentor Graphics Corporation
4#
Brad Bishopc342db32019-05-15 21:57:59 -04005# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05007
8import ast
9import codecs
10import collections
Andrew Geissler595f6302022-01-24 19:11:47 +000011import setuptools.command.build_py
Patrick Williamsc124f4f2015-09-15 14:41:29 -050012import email
13import imp
14import glob
15import itertools
16import logging
17import os
18import re
19import sys
20import subprocess
21from recipetool.create import RecipeHandler
22
23logger = logging.getLogger('recipetool')
24
25tinfoil = None
26
27
28def tinfoil_init(instance):
29 global tinfoil
30 tinfoil = instance
31
32
33class PythonRecipeHandler(RecipeHandler):
Brad Bishop15ae2502019-06-18 21:44:24 -040034 base_pkgdeps = ['python3-core']
35 excluded_pkgdeps = ['python3-dbg']
36 # os.path is provided by python3-core
Patrick Williamsc124f4f2015-09-15 14:41:29 -050037 assume_provided = ['builtins', 'os.path']
Brad Bishop15ae2502019-06-18 21:44:24 -040038 # Assumes that the host python3 builtin_module_names is sane for target too
Patrick Williamsc124f4f2015-09-15 14:41:29 -050039 assume_provided = assume_provided + list(sys.builtin_module_names)
40
41 bbvar_map = {
42 'Name': 'PN',
43 'Version': 'PV',
44 'Home-page': 'HOMEPAGE',
45 'Summary': 'SUMMARY',
46 'Description': 'DESCRIPTION',
47 'License': 'LICENSE',
Patrick Williams213cb262021-08-07 19:21:33 -050048 'Requires': 'RDEPENDS:${PN}',
49 'Provides': 'RPROVIDES:${PN}',
50 'Obsoletes': 'RREPLACES:${PN}',
Patrick Williamsc124f4f2015-09-15 14:41:29 -050051 }
52 # PN/PV are already set by recipetool core & desc can be extremely long
53 excluded_fields = [
Patrick Williamsc124f4f2015-09-15 14:41:29 -050054 'Description',
55 ]
56 setup_parse_map = {
57 'Url': 'Home-page',
58 'Classifiers': 'Classifier',
59 'Description': 'Summary',
60 }
61 setuparg_map = {
62 'Home-page': 'url',
63 'Classifier': 'classifiers',
64 'Summary': 'description',
65 'Description': 'long-description',
66 }
67 # Values which are lists, used by the setup.py argument based metadata
68 # extraction method, to determine how to process the setup.py output.
69 setuparg_list_fields = [
70 'Classifier',
71 'Requires',
72 'Provides',
73 'Obsoletes',
74 'Platform',
75 'Supported-Platform',
76 ]
77 setuparg_multi_line_values = ['Description']
78 replacements = [
Patrick Williamsc0f7c042017-02-23 20:41:17 -060079 ('License', r' +$', ''),
80 ('License', r'^ +', ''),
Patrick Williamsc124f4f2015-09-15 14:41:29 -050081 ('License', r' ', '-'),
Patrick Williamsc0f7c042017-02-23 20:41:17 -060082 ('License', r'^GNU-', ''),
83 ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''),
Patrick Williamsc124f4f2015-09-15 14:41:29 -050084 ('License', r'^UNKNOWN$', ''),
85
86 # Remove currently unhandled version numbers from these variables
87 ('Requires', r' *\([^)]*\)', ''),
88 ('Provides', r' *\([^)]*\)', ''),
89 ('Obsoletes', r' *\([^)]*\)', ''),
90 ('Install-requires', r'^([^><= ]+).*', r'\1'),
91 ('Extras-require', r'^([^><= ]+).*', r'\1'),
92 ('Tests-require', r'^([^><= ]+).*', r'\1'),
93
94 # Remove unhandled dependency on particular features (e.g. foo[PDF])
95 ('Install-requires', r'\[[^\]]+\]$', ''),
96 ]
97
98 classifier_license_map = {
99 'License :: OSI Approved :: Academic Free License (AFL)': 'AFL',
100 'License :: OSI Approved :: Apache Software License': 'Apache',
101 'License :: OSI Approved :: Apple Public Source License': 'APSL',
102 'License :: OSI Approved :: Artistic License': 'Artistic',
103 'License :: OSI Approved :: Attribution Assurance License': 'AAL',
Andrew Geissler5199d832021-09-24 16:47:35 -0500104 'License :: OSI Approved :: BSD License': 'BSD-3-Clause',
Andrew Geissler9aee5002022-03-30 16:27:02 +0000105 'License :: OSI Approved :: Boost Software License 1.0 (BSL-1.0)': 'BSL-1.0',
106 'License :: OSI Approved :: CEA CNRS Inria Logiciel Libre License, version 2.1 (CeCILL-2.1)': 'CECILL-2.1',
107 'License :: OSI Approved :: Common Development and Distribution License 1.0 (CDDL-1.0)': 'CDDL-1.0',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500108 'License :: OSI Approved :: Common Public License': 'CPL',
Andrew Geissler9aee5002022-03-30 16:27:02 +0000109 'License :: OSI Approved :: Eclipse Public License 1.0 (EPL-1.0)': 'EPL-1.0',
110 'License :: OSI Approved :: Eclipse Public License 2.0 (EPL-2.0)': 'EPL-2.0',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500111 'License :: OSI Approved :: Eiffel Forum License': 'EFL',
112 'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0',
113 'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1',
Andrew Geissler9aee5002022-03-30 16:27:02 +0000114 'License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2)': 'EUPL-1.2',
115 'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0-only',
116 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0-or-later',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500117 'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL',
118 'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL',
Andrew Geissler9aee5002022-03-30 16:27:02 +0000119 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0-only',
120 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0-or-later',
121 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0-only',
122 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0-or-later',
123 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0-only',
124 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0-or-later',
125 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0-only',
126 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0-or-later',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500127 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL',
Andrew Geissler9aee5002022-03-30 16:27:02 +0000128 'License :: OSI Approved :: Historical Permission Notice and Disclaimer (HPND)': 'HPND',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500129 'License :: OSI Approved :: IBM Public License': 'IPL',
130 'License :: OSI Approved :: ISC License (ISCL)': 'ISC',
131 'License :: OSI Approved :: Intel Open Source License': 'Intel',
132 'License :: OSI Approved :: Jabber Open Source License': 'Jabber',
133 'License :: OSI Approved :: MIT License': 'MIT',
Andrew Geissler9aee5002022-03-30 16:27:02 +0000134 'License :: OSI Approved :: MIT No Attribution License (MIT-0)': 'MIT-0',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500135 'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL',
Andrew Geissler9aee5002022-03-30 16:27:02 +0000136 'License :: OSI Approved :: MirOS License (MirOS)': 'MirOS',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500137 'License :: OSI Approved :: Motosoto License': 'Motosoto',
138 'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0',
139 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1',
140 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0',
141 'License :: OSI Approved :: Nethack General Public License': 'NGPL',
142 'License :: OSI Approved :: Nokia Open Source License': 'Nokia',
143 'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL',
Andrew Geissler9aee5002022-03-30 16:27:02 +0000144 'License :: OSI Approved :: Open Software License 3.0 (OSL-3.0)': 'OSL-3.0',
145 'License :: OSI Approved :: PostgreSQL License': 'PostgreSQL',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500146 'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python',
Andrew Geissler9aee5002022-03-30 16:27:02 +0000147 'License :: OSI Approved :: Python Software Foundation License': 'PSF-2.0',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500148 'License :: OSI Approved :: Qt Public License (QPL)': 'QPL',
149 'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL',
Andrew Geissler9aee5002022-03-30 16:27:02 +0000150 'License :: OSI Approved :: SIL Open Font License 1.1 (OFL-1.1)': 'OFL-1.1',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500151 'License :: OSI Approved :: Sleepycat License': 'Sleepycat',
Andrew Geissler9aee5002022-03-30 16:27:02 +0000152 'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': 'SISSL',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500153 'License :: OSI Approved :: Sun Public License': 'SPL',
Andrew Geissler9aee5002022-03-30 16:27:02 +0000154 'License :: OSI Approved :: The Unlicense (Unlicense)': 'Unlicense',
155 'License :: OSI Approved :: Universal Permissive License (UPL)': 'UPL-1.0',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500156 'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA',
157 'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0',
158 'License :: OSI Approved :: W3C License': 'W3C',
159 'License :: OSI Approved :: X.Net License': 'Xnet',
160 'License :: OSI Approved :: Zope Public License': 'ZPL',
161 'License :: OSI Approved :: zlib/libpng License': 'Zlib',
Andrew Geissler9aee5002022-03-30 16:27:02 +0000162 'License :: Other/Proprietary License': 'Proprietary',
163 'License :: Public Domain': 'PD',
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500164 }
165
166 def __init__(self):
167 pass
168
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500169 def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500170 if 'buildsystem' in handled:
171 return False
172
Brad Bishop96ff1982019-08-19 13:50:42 -0400173 # Check for non-zero size setup.py files
174 setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py'])
175 for fn in setupfiles:
176 if os.path.getsize(fn):
177 break
178 else:
179 return False
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500180
181 # setup.py is always parsed to get at certain required information, such as
182 # distutils vs setuptools
183 #
184 # If egg info is available, we use it for both its PKG-INFO metadata
185 # and for its requires.txt for install_requires.
186 # If PKG-INFO is available but no egg info is, we use that for metadata in preference to
187 # the parsed setup.py, but use the install_requires info from the
188 # parsed setup.py.
189
190 setupscript = os.path.join(srctree, 'setup.py')
191 try:
192 setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript)
193 except Exception:
194 logger.exception("Failed to parse setup.py")
195 setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], []
196
197 egginfo = glob.glob(os.path.join(srctree, '*.egg-info'))
198 if egginfo:
199 info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO'))
200 requires_txt = os.path.join(egginfo[0], 'requires.txt')
201 if os.path.exists(requires_txt):
202 with codecs.open(requires_txt) as f:
203 inst_req = []
204 extras_req = collections.defaultdict(list)
205 current_feature = None
206 for line in f.readlines():
207 line = line.rstrip()
208 if not line:
209 continue
210
211 if line.startswith('['):
Andrew Geissler615f2f12022-07-15 14:00:58 -0500212 # PACKAGECONFIG must not contain expressions or whitespace
213 line = line.replace(" ", "")
214 line = line.replace(':', "")
215 line = line.replace('.', "-dot-")
216 line = line.replace('"', "")
217 line = line.replace('<', "-smaller-")
218 line = line.replace('>', "-bigger-")
219 line = line.replace('_', "-")
220 line = line.replace('(', "")
221 line = line.replace(')', "")
222 line = line.replace('!', "-not-")
223 line = line.replace('=', "-equals-")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500224 current_feature = line[1:-1]
225 elif current_feature:
226 extras_req[current_feature].append(line)
227 else:
228 inst_req.append(line)
229 info['Install-requires'] = inst_req
230 info['Extras-require'] = extras_req
231 elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']):
232 info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO'))
233
234 if setup_info:
235 if 'Install-requires' in setup_info:
236 info['Install-requires'] = setup_info['Install-requires']
237 if 'Extras-require' in setup_info:
238 info['Extras-require'] = setup_info['Extras-require']
239 else:
240 if setup_info:
241 info = setup_info
242 else:
243 info = self.get_setup_args_info(setupscript)
244
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600245 # Grab the license value before applying replacements
246 license_str = info.get('License', '').strip()
247
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500248 self.apply_info_replacements(info)
249
250 if uses_setuptools:
Brad Bishop15ae2502019-06-18 21:44:24 -0400251 classes.append('setuptools3')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500252 else:
Brad Bishop15ae2502019-06-18 21:44:24 -0400253 classes.append('distutils3')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500254
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600255 if license_str:
256 for i, line in enumerate(lines_before):
257 if line.startswith('LICENSE = '):
258 lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str)
259 break
260
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500261 if 'Classifier' in info:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600262 existing_licenses = info.get('License', '')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500263 licenses = []
264 for classifier in info['Classifier']:
265 if classifier in self.classifier_license_map:
266 license = self.classifier_license_map[classifier]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600267 if license == 'Apache' and 'Apache-2.0' in existing_licenses:
268 license = 'Apache-2.0'
269 elif license == 'GPL':
270 if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
271 license = 'GPL-2.0'
272 elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
273 license = 'GPL-3.0'
274 elif license == 'LGPL':
275 if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
276 license = 'LGPL-2.1'
277 elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
278 license = 'LGPL-2.0'
279 elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
280 license = 'LGPL-3.0'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500281 licenses.append(license)
282
283 if licenses:
284 info['License'] = ' & '.join(licenses)
285
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500286 # Map PKG-INFO & setup.py fields to bitbake variables
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600287 for field, values in info.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500288 if field in self.excluded_fields:
289 continue
290
291 if field not in self.bbvar_map:
292 continue
293
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600294 if isinstance(values, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500295 value = values
296 else:
297 value = ' '.join(str(v) for v in values if v)
298
299 bbvar = self.bbvar_map[field]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600300 if bbvar not in extravalues and value:
301 extravalues[bbvar] = value
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500302
303 mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
304
305 extras_req = set()
306 if 'Extras-require' in info:
307 extras_req = info['Extras-require']
308 if extras_req:
309 lines_after.append('# The following configs & dependencies are from setuptools extras_require.')
310 lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.')
311 lines_after.append('# The upstream names may not correspond exactly to bitbake package names.')
Andrew Geissler615f2f12022-07-15 14:00:58 -0500312 lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500313 lines_after.append('#')
314 lines_after.append('# Uncomment this line to enable all the optional features.')
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600315 lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req)))
316 for feature, feature_reqs in extras_req.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500317 unmapped_deps.difference_update(feature_reqs)
318
Brad Bishop15ae2502019-06-18 21:44:24 -0400319 feature_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(feature_reqs))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500320 lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps)))
321
322 inst_reqs = set()
323 if 'Install-requires' in info:
324 if extras_req:
325 lines_after.append('')
326 inst_reqs = info['Install-requires']
327 if inst_reqs:
328 unmapped_deps.difference_update(inst_reqs)
329
Brad Bishop15ae2502019-06-18 21:44:24 -0400330 inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500331 lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These')
332 lines_after.append('# upstream names may not correspond exactly to bitbake package names.')
Patrick Williams213cb262021-08-07 19:21:33 -0500333 lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps)))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500334
335 if mapped_deps:
336 name = info.get('Name')
337 if name and name[0] in mapped_deps:
338 # Attempt to avoid self-reference
339 mapped_deps.remove(name[0])
340 mapped_deps -= set(self.excluded_pkgdeps)
341 if inst_reqs or extras_req:
342 lines_after.append('')
343 lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the')
344 lines_after.append('# python sources, and might not be 100% accurate.')
Patrick Williams213cb262021-08-07 19:21:33 -0500345 lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps))))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500346
347 unmapped_deps -= set(extensions)
348 unmapped_deps -= set(self.assume_provided)
349 if unmapped_deps:
350 if mapped_deps:
351 lines_after.append('')
352 lines_after.append('# WARNING: We were unable to map the following python package/module')
353 lines_after.append('# dependencies to the bitbake packages which include them:')
354 lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps))
355
356 handled.append('buildsystem')
357
358 def get_pkginfo(self, pkginfo_fn):
359 msg = email.message_from_file(open(pkginfo_fn, 'r'))
360 msginfo = {}
361 for field in msg.keys():
362 values = msg.get_all(field)
363 if len(values) == 1:
364 msginfo[field] = values[0]
365 else:
366 msginfo[field] = values
367 return msginfo
368
369 def parse_setup_py(self, setupscript='./setup.py'):
370 with codecs.open(setupscript) as f:
371 info, imported_modules, non_literals, extensions = gather_setup_info(f)
372
373 def _map(key):
374 key = key.replace('_', '-')
375 key = key[0].upper() + key[1:]
376 if key in self.setup_parse_map:
377 key = self.setup_parse_map[key]
378 return key
379
380 # Naive mapping of setup() arguments to PKG-INFO field names
381 for d in [info, non_literals]:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600382 for key, value in list(d.items()):
Brad Bishopd7bf8c12018-02-25 22:55:05 -0500383 if key is None:
384 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500385 new_key = _map(key)
386 if new_key != key:
387 del d[key]
388 d[new_key] = value
389
390 return info, 'setuptools' in imported_modules, non_literals, extensions
391
392 def get_setup_args_info(self, setupscript='./setup.py'):
Brad Bishop15ae2502019-06-18 21:44:24 -0400393 cmd = ['python3', setupscript]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500394 info = {}
395 keys = set(self.bbvar_map.keys())
396 keys |= set(self.setuparg_list_fields)
397 keys |= set(self.setuparg_multi_line_values)
398 grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values))
399 for index, keys in grouped_keys:
400 if index == (True, False):
401 # Splitlines output for each arg as a list value
402 for key in keys:
403 arg = self.setuparg_map.get(key, key.lower())
404 try:
405 arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
406 except (OSError, subprocess.CalledProcessError):
407 pass
408 else:
409 info[key] = [l.rstrip() for l in arg_info.splitlines()]
410 elif index == (False, True):
411 # Entire output for each arg
412 for key in keys:
413 arg = self.setuparg_map.get(key, key.lower())
414 try:
415 arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
416 except (OSError, subprocess.CalledProcessError):
417 pass
418 else:
419 info[key] = arg_info
420 else:
421 info.update(self.get_setup_byline(list(keys), setupscript))
422 return info
423
424 def get_setup_byline(self, fields, setupscript='./setup.py'):
425 info = {}
426
Brad Bishop15ae2502019-06-18 21:44:24 -0400427 cmd = ['python3', setupscript]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500428 cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields)
429 try:
430 info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines()
431 except (OSError, subprocess.CalledProcessError):
432 pass
433 else:
434 if len(fields) != len(info_lines):
435 logger.error('Mismatch between setup.py output lines and number of fields')
436 sys.exit(1)
437
438 for lineno, line in enumerate(info_lines):
439 line = line.rstrip()
440 info[fields[lineno]] = line
441 return info
442
443 def apply_info_replacements(self, info):
444 for variable, search, replace in self.replacements:
445 if variable not in info:
446 continue
447
448 def replace_value(search, replace, value):
449 if replace is None:
450 if re.search(search, value):
451 return None
452 else:
453 new_value = re.sub(search, replace, value)
454 if value != new_value:
455 return new_value
456 return value
457
458 value = info[variable]
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600459 if isinstance(value, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500460 new_value = replace_value(search, replace, value)
461 if new_value is None:
462 del info[variable]
463 elif new_value != value:
464 info[variable] = new_value
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600465 elif hasattr(value, 'items'):
466 for dkey, dvalue in list(value.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500467 new_list = []
468 for pos, a_value in enumerate(dvalue):
469 new_value = replace_value(search, replace, a_value)
470 if new_value is not None and new_value != value:
471 new_list.append(new_value)
472
473 if value != new_list:
474 value[dkey] = new_list
475 else:
476 new_list = []
477 for pos, a_value in enumerate(value):
478 new_value = replace_value(search, replace, a_value)
479 if new_value is not None and new_value != value:
480 new_list.append(new_value)
481
482 if value != new_list:
483 info[variable] = new_list
484
485 def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
486 if 'Package-dir' in setup_info:
487 package_dir = setup_info['Package-dir']
488 else:
489 package_dir = {}
490
Andrew Geissler595f6302022-01-24 19:11:47 +0000491 dist = setuptools.Distribution()
492
493 class PackageDir(setuptools.command.build_py.build_py):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500494 def __init__(self, package_dir):
495 self.package_dir = package_dir
Andrew Geissler595f6302022-01-24 19:11:47 +0000496 self.dist = dist
497 super().__init__(self.dist)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500498
499 pd = PackageDir(package_dir)
500 to_scan = []
501 if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']):
502 if 'Py-modules' in setup_info:
503 for module in setup_info['Py-modules']:
504 try:
505 package, module = module.rsplit('.', 1)
506 except ValueError:
507 package, module = '.', module
508 module_path = os.path.join(pd.get_package_dir(package), module + '.py')
509 to_scan.append(module_path)
510
511 if 'Packages' in setup_info:
512 for package in setup_info['Packages']:
513 to_scan.append(pd.get_package_dir(package))
514
515 if 'Scripts' in setup_info:
516 to_scan.extend(setup_info['Scripts'])
517 else:
518 logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.")
519
520 if not to_scan:
521 to_scan = ['.']
522
523 logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan))
524
525 provided_packages = self.parse_pkgdata_for_python_packages()
526 scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan])
527 mapped_deps, unmapped_deps = set(self.base_pkgdeps), set()
528 for dep in scanned_deps:
529 mapped = provided_packages.get(dep)
530 if mapped:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600531 logger.debug('Mapped %s to %s' % (dep, mapped))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500532 mapped_deps.add(mapped)
533 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600534 logger.debug('Could not map %s' % dep)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500535 unmapped_deps.add(dep)
536 return mapped_deps, unmapped_deps
537
538 def scan_python_dependencies(self, paths):
539 deps = set()
540 try:
541 dep_output = self.run_command(['pythondeps', '-d'] + paths)
542 except (OSError, subprocess.CalledProcessError):
543 pass
544 else:
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500545 for line in dep_output.splitlines():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500546 line = line.rstrip()
547 dep, filename = line.split('\t', 1)
548 if filename.endswith('/setup.py'):
549 continue
550 deps.add(dep)
551
552 try:
553 provides_output = self.run_command(['pythondeps', '-p'] + paths)
554 except (OSError, subprocess.CalledProcessError):
555 pass
556 else:
557 provides_lines = (l.rstrip() for l in provides_output.splitlines())
558 provides = set(l for l in provides_lines if l and l != 'setup')
559 deps -= provides
560
561 return deps
562
563 def parse_pkgdata_for_python_packages(self):
564 suffixes = [t[0] for t in imp.get_suffixes()]
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500565 pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500566
567 ldata = tinfoil.config_data.createCopy()
Brad Bishop15ae2502019-06-18 21:44:24 -0400568 bb.parse.handle('classes/python3-dir.bbclass', ldata, True)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500569 python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500570
571 dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
572 python_dirs = [python_sitedir + os.sep,
573 os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep,
574 os.path.dirname(python_sitedir) + os.sep]
575 packages = {}
576 for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)):
577 files_info = None
578 with open(pkgdatafile, 'r') as f:
579 for line in f.readlines():
580 field, value = line.split(': ', 1)
Andrew Geisslerd159c7f2021-09-02 21:05:58 -0500581 if field.startswith('FILES_INFO'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500582 files_info = ast.literal_eval(value)
583 break
584 else:
585 continue
586
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600587 for fn in files_info:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500588 for suffix in suffixes:
589 if fn.endswith(suffix):
590 break
591 else:
592 continue
593
594 if fn.startswith(dynload_dir + os.sep):
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600595 if '/.debug/' in fn:
596 continue
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500597 base = os.path.basename(fn)
598 provided = base.split('.', 1)[0]
599 packages[provided] = os.path.basename(pkgdatafile)
600 continue
601
602 for python_dir in python_dirs:
603 if fn.startswith(python_dir):
604 relpath = fn[len(python_dir):]
605 relstart, _, relremaining = relpath.partition(os.sep)
606 if relstart.endswith('.egg'):
607 relpath = relremaining
608 base, _ = os.path.splitext(relpath)
609
610 if '/.debug/' in base:
611 continue
612 if os.path.basename(base) == '__init__':
613 base = os.path.dirname(base)
614 base = base.replace(os.sep + os.sep, os.sep)
615 provided = base.replace(os.sep, '.')
616 packages[provided] = os.path.basename(pkgdatafile)
617 return packages
618
619 @classmethod
620 def run_command(cls, cmd, **popenargs):
621 if 'stderr' not in popenargs:
622 popenargs['stderr'] = subprocess.STDOUT
623 try:
Brad Bishop37a0e4d2017-12-04 01:01:44 -0500624 return subprocess.check_output(cmd, **popenargs).decode('utf-8')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500625 except OSError as exc:
626 logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc)
627 raise
628 except subprocess.CalledProcessError as exc:
629 logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output)
630 raise
631
632
633def gather_setup_info(fileobj):
634 parsed = ast.parse(fileobj.read(), fileobj.name)
635 visitor = SetupScriptVisitor()
636 visitor.visit(parsed)
637
638 non_literals, extensions = {}, []
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600639 for key, value in list(visitor.keywords.items()):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500640 if key == 'ext_modules':
641 if isinstance(value, list):
642 for ext in value:
643 if (isinstance(ext, ast.Call) and
644 isinstance(ext.func, ast.Name) and
645 ext.func.id == 'Extension' and
646 not has_non_literals(ext.args)):
647 extensions.append(ext.args[0])
648 elif has_non_literals(value):
649 non_literals[key] = value
650 del visitor.keywords[key]
651
652 return visitor.keywords, visitor.imported_modules, non_literals, extensions
653
654
655class SetupScriptVisitor(ast.NodeVisitor):
656 def __init__(self):
657 ast.NodeVisitor.__init__(self)
658 self.keywords = {}
659 self.non_literals = []
660 self.imported_modules = set()
661
662 def visit_Expr(self, node):
663 if isinstance(node.value, ast.Call) and \
664 isinstance(node.value.func, ast.Name) and \
665 node.value.func.id == 'setup':
666 self.visit_setup(node.value)
667
668 def visit_setup(self, node):
669 call = LiteralAstTransform().visit(node)
670 self.keywords = call.keywords
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600671 for k, v in self.keywords.items():
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500672 if has_non_literals(v):
673 self.non_literals.append(k)
674
675 def visit_Import(self, node):
676 for alias in node.names:
677 self.imported_modules.add(alias.name)
678
679 def visit_ImportFrom(self, node):
680 self.imported_modules.add(node.module)
681
682
683class LiteralAstTransform(ast.NodeTransformer):
684 """Simplify the ast through evaluation of literals."""
685 excluded_fields = ['ctx']
686
687 def visit(self, node):
688 if not isinstance(node, ast.AST):
689 return node
690 else:
691 return ast.NodeTransformer.visit(self, node)
692
693 def generic_visit(self, node):
694 try:
695 return ast.literal_eval(node)
696 except ValueError:
697 for field, value in ast.iter_fields(node):
698 if field in self.excluded_fields:
699 delattr(node, field)
700 if value is None:
701 continue
702
703 if isinstance(value, list):
704 if field in ('keywords', 'kwargs'):
705 new_value = dict((kw.arg, self.visit(kw.value)) for kw in value)
706 else:
707 new_value = [self.visit(i) for i in value]
708 else:
709 new_value = self.visit(value)
710 setattr(node, field, new_value)
711 return node
712
713 def visit_Name(self, node):
714 if hasattr('__builtins__', node.id):
715 return getattr(__builtins__, node.id)
716 else:
717 return self.generic_visit(node)
718
719 def visit_Tuple(self, node):
720 return tuple(self.visit(v) for v in node.elts)
721
722 def visit_List(self, node):
723 return [self.visit(v) for v in node.elts]
724
725 def visit_Set(self, node):
726 return set(self.visit(v) for v in node.elts)
727
728 def visit_Dict(self, node):
729 keys = (self.visit(k) for k in node.keys)
730 values = (self.visit(v) for v in node.values)
731 return dict(zip(keys, values))
732
733
734def has_non_literals(value):
735 if isinstance(value, ast.AST):
736 return True
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600737 elif isinstance(value, str):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500738 return False
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600739 elif hasattr(value, 'values'):
740 return any(has_non_literals(v) for v in value.values())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500741 elif hasattr(value, '__iter__'):
742 return any(has_non_literals(v) for v in value)
743
744
745def register_recipe_handlers(handlers):
746 # We need to make sure this is ahead of the makefile fallback handler
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500747 handlers.append((PythonRecipeHandler(), 70))