blob: 2e7cd25297ed11cc3faa4388ded1a0fc99a4eef5 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# This class knows how to package up [e]glibc. Its shared since prebuild binary toolchains
3# may need packaging and its pointless to duplicate this code.
4#
5# Caller should set GLIBC_INTERNAL_USE_BINARY_LOCALE to one of:
6# "compile" - Use QEMU to generate the binary locale files
7# "precompiled" - The binary locale files are pregenerated and already present
8# "ondevice" - The device will build the locale files upon first boot through the postinst
9
10GLIBC_INTERNAL_USE_BINARY_LOCALE ?= "ondevice"
11
Brad Bishop6e60e8b2018-02-01 10:27:11 -050012GLIBC_SPLIT_LC_PACKAGES ?= "0"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050013
Brad Bishop6e60e8b2018-02-01 10:27:11 -050014python __anonymous () {
15 enabled = d.getVar("ENABLE_BINARY_LOCALE_GENERATION")
16
17 pn = d.getVar("PN")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018 if pn.endswith("-initial"):
19 enabled = False
20
21 if enabled and int(enabled):
22 import re
23
Brad Bishop6e60e8b2018-02-01 10:27:11 -050024 target_arch = d.getVar("TARGET_ARCH")
25 binary_arches = d.getVar("BINARY_LOCALE_ARCHES") or ""
26 use_cross_localedef = d.getVar("LOCALE_GENERATION_WITH_CROSS-LOCALEDEF") or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -050027
28 for regexp in binary_arches.split(" "):
29 r = re.compile(regexp)
30
31 if r.match(target_arch):
Brad Bishop6e60e8b2018-02-01 10:27:11 -050032 depends = d.getVar("DEPENDS")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050033 if use_cross_localedef == "1" :
34 depends = "%s cross-localedef-native" % depends
35 else:
36 depends = "%s qemu-native" % depends
37 d.setVar("DEPENDS", depends)
38 d.setVar("GLIBC_INTERNAL_USE_BINARY_LOCALE", "compile")
39 break
40
41 # try to fix disable charsets/locales/locale-code compile fail
42 if bb.utils.contains('DISTRO_FEATURES', 'libc-charsets', True, False, d) and \
43 bb.utils.contains('DISTRO_FEATURES', 'libc-locales', True, False, d) and \
44 bb.utils.contains('DISTRO_FEATURES', 'libc-locale-code', True, False, d):
45 d.setVar('PACKAGE_NO_GCONV', '0')
46 else:
47 d.setVar('PACKAGE_NO_GCONV', '1')
48}
49
50OVERRIDES_append = ":${TARGET_ARCH}-${TARGET_OS}"
51
Patrick Williamsc124f4f2015-09-15 14:41:29 -050052locale_base_postinst() {
53#!/bin/sh
54
55if [ "x$D" != "x" ]; then
56 exit 1
57fi
58
Patrick Williamsc0f7c042017-02-23 20:41:17 -060059localedef --inputfile=${datadir}/i18n/locales/%s --charmap=%s %s
Patrick Williamsc124f4f2015-09-15 14:41:29 -050060}
61
Patrick Williamsc124f4f2015-09-15 14:41:29 -050062locale_base_postrm() {
63#!/bin/sh
Patrick Williamsc0f7c042017-02-23 20:41:17 -060064localedef --delete-from-archive --inputfile=${datadir}/locales/%s --charmap=%s %s
Patrick Williamsc124f4f2015-09-15 14:41:29 -050065}
66
Patrick Williamsc124f4f2015-09-15 14:41:29 -050067LOCALETREESRC ?= "${PKGD}"
68
69do_prep_locale_tree() {
70 treedir=${WORKDIR}/locale-tree
71 rm -rf $treedir
72 mkdir -p $treedir/${base_bindir} $treedir/${base_libdir} $treedir/${datadir} $treedir/${localedir}
73 tar -cf - -C ${LOCALETREESRC}${datadir} -p i18n | tar -xf - -C $treedir/${datadir}
74 # unzip to avoid parsing errors
75 for i in $treedir/${datadir}/i18n/charmaps/*gz; do
76 gunzip $i
77 done
78 tar -cf - -C ${LOCALETREESRC}${base_libdir} -p . | tar -xf - -C $treedir/${base_libdir}
79 if [ -f ${STAGING_DIR_NATIVE}${prefix_native}/lib/libgcc_s.* ]; then
80 tar -cf - -C ${STAGING_DIR_NATIVE}/${prefix_native}/${base_libdir} -p libgcc_s.* | tar -xf - -C $treedir/${base_libdir}
81 fi
82 install -m 0755 ${LOCALETREESRC}${bindir}/localedef $treedir/${base_bindir}
83}
84
85do_collect_bins_from_locale_tree() {
86 treedir=${WORKDIR}/locale-tree
87
88 parent=$(dirname ${localedir})
89 mkdir -p ${PKGD}/$parent
90 tar -cf - -C $treedir/$parent -p $(basename ${localedir}) | tar -xf - -C ${PKGD}$parent
91}
92
93inherit qemu
94
95python package_do_split_gconvs () {
96 import re
Brad Bishop6e60e8b2018-02-01 10:27:11 -050097 if (d.getVar('PACKAGE_NO_GCONV') == '1'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050098 bb.note("package requested not splitting gconvs")
99 return
100
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500101 if not d.getVar('PACKAGES'):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500102 return
103
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500104 mlprefix = d.getVar("MLPREFIX") or ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500105
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500106 bpn = d.getVar('BPN')
107 libdir = d.getVar('libdir')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500108 if not libdir:
109 bb.error("libdir not defined")
110 return
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500111 datadir = d.getVar('datadir')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500112 if not datadir:
113 bb.error("datadir not defined")
114 return
115
Brad Bishop316dfdd2018-06-25 12:45:53 -0400116 gconv_libdir = oe.path.join(libdir, "gconv")
117 charmap_dir = oe.path.join(datadir, "i18n", "charmaps")
118 locales_dir = oe.path.join(datadir, "i18n", "locales")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500119 binary_locales_dir = d.getVar('localedir')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500120
121 def calc_gconv_deps(fn, pkg, file_regex, output_pattern, group):
122 deps = []
123 f = open(fn, "rb")
124 c_re = re.compile('^copy "(.*)"')
125 i_re = re.compile('^include "(\w+)".*')
126 for l in f.readlines():
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600127 l = l.decode("latin-1")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500128 m = c_re.match(l) or i_re.match(l)
129 if m:
130 dp = legitimize_package_name('%s%s-gconv-%s' % (mlprefix, bpn, m.group(1)))
131 if not dp in deps:
132 deps.append(dp)
133 f.close()
134 if deps != []:
135 d.setVar('RDEPENDS_%s' % pkg, " ".join(deps))
136 if bpn != 'glibc':
137 d.setVar('RPROVIDES_%s' % pkg, pkg.replace(bpn, 'glibc'))
138
139 do_split_packages(d, gconv_libdir, file_regex='^(.*)\.so$', output_pattern=bpn+'-gconv-%s', \
140 description='gconv module for character set %s', hook=calc_gconv_deps, \
141 extra_depends=bpn+'-gconv')
142
143 def calc_charmap_deps(fn, pkg, file_regex, output_pattern, group):
144 deps = []
145 f = open(fn, "rb")
146 c_re = re.compile('^copy "(.*)"')
147 i_re = re.compile('^include "(\w+)".*')
148 for l in f.readlines():
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600149 l = l.decode("latin-1")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500150 m = c_re.match(l) or i_re.match(l)
151 if m:
152 dp = legitimize_package_name('%s%s-charmap-%s' % (mlprefix, bpn, m.group(1)))
153 if not dp in deps:
154 deps.append(dp)
155 f.close()
156 if deps != []:
157 d.setVar('RDEPENDS_%s' % pkg, " ".join(deps))
158 if bpn != 'glibc':
159 d.setVar('RPROVIDES_%s' % pkg, pkg.replace(bpn, 'glibc'))
160
161 do_split_packages(d, charmap_dir, file_regex='^(.*)\.gz$', output_pattern=bpn+'-charmap-%s', \
162 description='character map for %s encoding', hook=calc_charmap_deps, extra_depends='')
163
164 def calc_locale_deps(fn, pkg, file_regex, output_pattern, group):
165 deps = []
166 f = open(fn, "rb")
167 c_re = re.compile('^copy "(.*)"')
168 i_re = re.compile('^include "(\w+)".*')
169 for l in f.readlines():
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600170 l = l.decode("latin-1")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500171 m = c_re.match(l) or i_re.match(l)
172 if m:
173 dp = legitimize_package_name(mlprefix+bpn+'-localedata-%s' % m.group(1))
174 if not dp in deps:
175 deps.append(dp)
176 f.close()
177 if deps != []:
178 d.setVar('RDEPENDS_%s' % pkg, " ".join(deps))
179 if bpn != 'glibc':
180 d.setVar('RPROVIDES_%s' % pkg, pkg.replace(bpn, 'glibc'))
181
182 do_split_packages(d, locales_dir, file_regex='(.*)', output_pattern=bpn+'-localedata-%s', \
183 description='locale definition for %s', hook=calc_locale_deps, extra_depends='')
184 d.setVar('PACKAGES', d.getVar('PACKAGES', False) + ' ' + d.getVar('MLPREFIX', False) + bpn + '-gconv')
185
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500186 use_bin = d.getVar("GLIBC_INTERNAL_USE_BINARY_LOCALE")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500187
188 dot_re = re.compile("(.*)\.(.*)")
189
190 # Read in supported locales and associated encodings
191 supported = {}
Brad Bishop316dfdd2018-06-25 12:45:53 -0400192 with open(oe.path.join(d.getVar('WORKDIR'), "SUPPORTED")) as f:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500193 for line in f.readlines():
194 try:
195 locale, charset = line.rstrip().split()
196 except ValueError:
197 continue
198 supported[locale] = charset
199
200 # GLIBC_GENERATE_LOCALES var specifies which locales to be generated. empty or "all" means all locales
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500201 to_generate = d.getVar('GLIBC_GENERATE_LOCALES')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500202 if not to_generate or to_generate == 'all':
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600203 to_generate = sorted(supported.keys())
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500204 else:
205 to_generate = to_generate.split()
206 for locale in to_generate:
207 if locale not in supported:
208 if '.' in locale:
209 charset = locale.split('.')[1]
210 else:
211 charset = 'UTF-8'
212 bb.warn("Unsupported locale '%s', assuming encoding '%s'" % (locale, charset))
213 supported[locale] = charset
214
215 def output_locale_source(name, pkgname, locale, encoding):
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500216 d.setVar('RDEPENDS_%s' % pkgname, '%slocaledef %s-localedata-%s %s-charmap-%s' % \
217 (mlprefix, mlprefix+bpn, legitimize_package_name(locale), mlprefix+bpn, legitimize_package_name(encoding)))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500218 d.setVar('pkg_postinst_%s' % pkgname, d.getVar('locale_base_postinst') \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500219 % (locale, encoding, locale))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500220 d.setVar('pkg_postrm_%s' % pkgname, d.getVar('locale_base_postrm') % \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500221 (locale, encoding, locale))
222
223 def output_locale_binary_rdepends(name, pkgname, locale, encoding):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500224 dep = legitimize_package_name('%s-binary-localedata-%s' % (bpn, name))
225 lcsplit = d.getVar('GLIBC_SPLIT_LC_PACKAGES')
226 if lcsplit and int(lcsplit):
227 d.appendVar('PACKAGES', ' ' + dep)
228 d.setVar('ALLOW_EMPTY_%s' % dep, '1')
229 d.setVar('RDEPENDS_%s' % pkgname, mlprefix + dep)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500230
231 commands = {}
232
233 def output_locale_binary(name, pkgname, locale, encoding):
Brad Bishop316dfdd2018-06-25 12:45:53 -0400234 treedir = oe.path.join(d.getVar("WORKDIR"), "locale-tree")
235 ldlibdir = oe.path.join(treedir, d.getVar("base_libdir"))
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500236 path = d.getVar("PATH")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400237 i18npath = oe.path.join(treedir, datadir, "i18n")
238 gconvpath = oe.path.join(treedir, "iconvdata")
239 outputpath = oe.path.join(treedir, binary_locales_dir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500240
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500241 use_cross_localedef = d.getVar("LOCALE_GENERATION_WITH_CROSS-LOCALEDEF") or "0"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500242 if use_cross_localedef == "1":
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500243 target_arch = d.getVar('TARGET_ARCH')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500244 locale_arch_options = { \
245 "arm": " --uint32-align=4 --little-endian ", \
246 "armeb": " --uint32-align=4 --big-endian ", \
247 "aarch64": " --uint32-align=4 --little-endian ", \
248 "aarch64_be": " --uint32-align=4 --big-endian ", \
249 "sh4": " --uint32-align=4 --big-endian ", \
250 "powerpc": " --uint32-align=4 --big-endian ", \
251 "powerpc64": " --uint32-align=4 --big-endian ", \
252 "mips": " --uint32-align=4 --big-endian ", \
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600253 "mipsisa32r6": " --uint32-align=4 --big-endian ", \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500254 "mips64": " --uint32-align=4 --big-endian ", \
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600255 "mipsisa64r6": " --uint32-align=4 --big-endian ", \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500256 "mipsel": " --uint32-align=4 --little-endian ", \
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600257 "mipsisa32r6el": " --uint32-align=4 --little-endian ", \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500258 "mips64el":" --uint32-align=4 --little-endian ", \
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600259 "mipsisa64r6el":" --uint32-align=4 --little-endian ", \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500260 "i586": " --uint32-align=4 --little-endian ", \
261 "i686": " --uint32-align=4 --little-endian ", \
262 "x86_64": " --uint32-align=4 --little-endian " }
263
264 if target_arch in locale_arch_options:
265 localedef_opts = locale_arch_options[target_arch]
266 else:
267 bb.error("locale_arch_options not found for target_arch=" + target_arch)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600268 bb.fatal("unknown arch:" + target_arch + " for locale_arch_options")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500269
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600270 localedef_opts += " --force --no-archive --prefix=%s \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500271 --inputfile=%s/%s/i18n/locales/%s --charmap=%s %s/%s" \
272 % (treedir, treedir, datadir, locale, encoding, outputpath, name)
273
274 cmd = "PATH=\"%s\" I18NPATH=\"%s\" GCONV_PATH=\"%s\" cross-localedef %s" % \
275 (path, i18npath, gconvpath, localedef_opts)
276 else: # earlier slower qemu way
277 qemu = qemu_target_binary(d)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600278 localedef_opts = "--force --no-archive --prefix=%s \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500279 --inputfile=%s/i18n/locales/%s --charmap=%s %s" \
280 % (treedir, datadir, locale, encoding, name)
281
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500282 qemu_options = d.getVar('QEMU_OPTIONS')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500283
284 cmd = "PSEUDO_RELOADED=YES PATH=\"%s\" I18NPATH=\"%s\" %s -L %s \
285 -E LD_LIBRARY_PATH=%s %s %s/bin/localedef %s" % \
286 (path, i18npath, qemu, treedir, ldlibdir, qemu_options, treedir, localedef_opts)
287
288 commands["%s/%s" % (outputpath, name)] = cmd
289
290 bb.note("generating locale %s (%s)" % (locale, encoding))
291
292 def output_locale(name, locale, encoding):
293 pkgname = d.getVar('MLPREFIX', False) + 'locale-base-' + legitimize_package_name(name)
294 d.setVar('ALLOW_EMPTY_%s' % pkgname, '1')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500295 d.setVar('PACKAGES', '%s %s' % (pkgname, d.getVar('PACKAGES')))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500296 rprovides = ' %svirtual-locale-%s' % (mlprefix, legitimize_package_name(name))
297 m = re.match("(.*)_(.*)", name)
298 if m:
299 rprovides += ' %svirtual-locale-%s' % (mlprefix, m.group(1))
300 d.setVar('RPROVIDES_%s' % pkgname, rprovides)
301
302 if use_bin == "compile":
303 output_locale_binary_rdepends(name, pkgname, locale, encoding)
304 output_locale_binary(name, pkgname, locale, encoding)
305 elif use_bin == "precompiled":
306 output_locale_binary_rdepends(name, pkgname, locale, encoding)
307 else:
308 output_locale_source(name, pkgname, locale, encoding)
309
310 if use_bin == "compile":
311 bb.note("preparing tree for binary locale generation")
312 bb.build.exec_func("do_prep_locale_tree", d)
313
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500314 utf8_only = int(d.getVar('LOCALE_UTF8_ONLY') or 0)
315 utf8_is_default = int(d.getVar('LOCALE_UTF8_IS_DEFAULT') or 0)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500316
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500317 encodings = {}
318 for locale in to_generate:
319 charset = supported[locale]
320 if utf8_only and charset != 'UTF-8':
321 continue
322
323 m = dot_re.match(locale)
324 if m:
325 base = m.group(1)
326 else:
327 base = locale
328
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500329 # Non-precompiled locales may be renamed so that the default
330 # (non-suffixed) encoding is always UTF-8, i.e., instead of en_US and
331 # en_US.UTF-8, we have en_US and en_US.ISO-8859-1. This implicitly
332 # contradicts SUPPORTED.
333 if use_bin == "precompiled" or not utf8_is_default:
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500334 output_locale(locale, base, charset)
335 else:
336 if charset == 'UTF-8':
337 output_locale(base, base, charset)
338 else:
339 output_locale('%s.%s' % (base, charset), base, charset)
340
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500341 def metapkg_hook(file, pkg, pattern, format, basename):
342 name = basename.split('/', 1)[0]
343 metapkg = legitimize_package_name('%s-binary-localedata-%s' % (mlprefix+bpn, name))
344 d.appendVar('RDEPENDS_%s' % metapkg, ' ' + pkg)
345
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500346 if use_bin == "compile":
Brad Bishop316dfdd2018-06-25 12:45:53 -0400347 makefile = oe.path.join(d.getVar("WORKDIR"), "locale-tree", "Makefile")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500348 m = open(makefile, "w")
349 m.write("all: %s\n\n" % " ".join(commands.keys()))
350 for cmd in commands:
351 m.write(cmd + ":\n")
352 m.write("\t" + commands[cmd] + "\n\n")
353 m.close()
354 d.setVar("EXTRA_OEMAKE", "-C %s ${PARALLEL_MAKE}" % (os.path.dirname(makefile)))
355 bb.note("Executing binary locale generation makefile")
356 bb.build.exec_func("oe_runmake", d)
357 bb.note("collecting binary locales from locale tree")
358 bb.build.exec_func("do_collect_bins_from_locale_tree", d)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500359
360 if use_bin in ('compile', 'precompiled'):
361 lcsplit = d.getVar('GLIBC_SPLIT_LC_PACKAGES')
362 if lcsplit and int(lcsplit):
363 do_split_packages(d, binary_locales_dir, file_regex='^(.*/LC_\w+)', \
364 output_pattern=bpn+'-binary-localedata-%s', \
365 description='binary locale definition for %s', recursive=True,
366 hook=metapkg_hook, extra_depends='', allow_dirs=True, match_path=True)
367 else:
368 do_split_packages(d, binary_locales_dir, file_regex='(.*)', \
369 output_pattern=bpn+'-binary-localedata-%s', \
370 description='binary locale definition for %s', extra_depends='', allow_dirs=True)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500371 else:
372 bb.note("generation of binary locales disabled. this may break i18n!")
373
374}
375
376# We want to do this indirection so that we can safely 'return'
377# from the called function even though we're prepending
378python populate_packages_prepend () {
379 bb.build.exec_func('package_do_split_gconvs', d)
380}