poky: subtree update:1203d1f24d..2dcd1f2a21
Alejandro Enedino Hernandez Samaniego (2):
python3: Improve logging, syntax and update deprecated modules to create_manifest
python3: Upgrade 3.9.2 -> 3.9.4
Alexander Kanavin (22):
scripts/oe-debuginfod: correct several issues
libmicrohttpd: add a recipe from meta-oe
maintainers.inc: add libmicrohttpd entry
xwayland: add a standalone recipe
weston: use standalone xwayland instead of outdated xserver-xorg version
elfutils: correct debuginfod builds on x32
elfutils: adjust ptests for correct debuginfod testing
default-distrovars.inc: add debuginfod to default DISTRO_FEATURES
oeqa: tear down oeqa decorators if one of them raises an exception in setup
meta/lib/oeqa/core/tests/cases/timeout.py: add a testcase for the previous fix
core-image-weston: add sdk/ptest images
oeqa/core/tests/test_data.py: use weston image instead of sato
oeqa/selftest: transition to weston images
core-image-multilib-example: base on weston, and not sato
dev-manual/common-tasks.rst: correct the documentation for debuginfod
diffoscope: add native libraries to LD_LIBRARY_PATH
Revert "oeqa: Set LD_LIBRARY_PATH when executing native commands"
boost: correct upstream version check
vte: use tarballs again
gdk-pixbuf: update 2.40.0 -> 2.42.6
glib-2.0: update 2.68.0 -> 2.68.1
gnu-config: update to latest revision
Anatol Belski (1):
cross-canadian: Whitelist "mingw32" as TARGET_OS
Anders Wallin (3):
lttng-tools: Fix missing legacy test files
lttng-tools: Fix path for test_python_looging
scripts/contrib/image-manifest: add new script
Andreas Müller (1):
xwayland: remove protocol.txt - it clashes with xserver-xorg
Anthony Bagwell (1):
systemd: upgrade 247.4 -> 247.6
Anuj Mittal (2):
Revert "qemu: fix CVE-2021-3392"
qemu: fix CVE-2021-3392
Armin Kuster (6):
binutils: rename BRANCH var
libseccomp: move recipe from meta-security to core
gnutls: Enable seccomp if FEATURE is set
systemd: Enable seccomp if FEATURE is set
qemu: Enable seccomp if FEATURE is set
default-distrovars.inc: Add seccomp to DISTRO_FEATURES_DEFAULT
Bastian Krause (1):
ccache: add packageconfig docs option
Bruce Ashfield (20):
kern-tools: add dropped options to audit output
linux-yocto/5.4: update to v5.4.109
linux-yocto/5.10: update to v5.10.27
linux-yocto/5.10: BSP configuration fixes
linux-yocto/5.10: update to v5.10.29
linux-yocto/5.4: update to v5.4.111
linux-yocto/5.10: update to v5.10.30
linux-yocto-rt/5.10: update to -rt34
linux-yocto/5.4: update to v5.4.112
linux-yocto/5.4: fix arm defconfig warnings
linux-yocto/5.10: fix arm defconfig warnings
linux-yocto/5.10: aufs fixes
linux-yocto/5.10: qemuriscv32.cfg: RV32 only supports 1G physical memory
linux-yocto/5.10: update to v5.10.32
perf: fix python-audit RDEPENDS
linux-yocto/5.4: update to v5.4.114
linux-yocto/5.10: update to v5.10.34
linux-yocto/5.4: update to v5.4.116
linux-yocto/5.10: qemuppc32: reduce serial shutdown issues
yocto-check-layer: Only note a layer without a conf/layer.conf (versus error)
Changqing Li (2):
libpam: make volatile files created successfully
gcr: fix one parallel build failure
Chen Qi (3):
busybox: fix CVE-2021-28831
weston: fix build failure due to race condition
rsync: fix CVE-2020-14387
Christophe Chapuis (1):
rootfs.py: find .ko.gz and .ko.xz kernel modules as well
Daniel Ammann (1):
archiver: Fix typos
Devendra Tewari (2):
bitbake: lib/bb: Add bb.utils.rename() helper function and use for renaming
classes/lib/scripts: Use bb.utils.rename() instead of os.rename()
Diego Sueiro (3):
oeqa/selftest/bblayers: Add test case for bitbake-layers layerindex-show-depends
bitbake: layerindex: Fix bitbake-layers layerindex-show-depends command
bitbake: layerindex: Add --fetchdir parameter to layerindex-fetch
Douglas Royds (2):
Revert "externalsrc: Detect code changes in submodules"
externalsrc: Detect code changes in submodules
Gavin Li (1):
kmod: do not symlink config.guess/config.sub during autoreconf
Harald Brinkmann (1):
bitbake: fetch/svn: Fix parsing revision of SVN repos with redirects
He Zhe (1):
linux-yocto-dev: add features/scsi/scsi-debug.scc features/gpio/mockup.scc to KERNEL_FEATURES
Henning Schild (3):
bitbake: fetch/git: add support for disabling shared clones on unpack
bitbake: tests/fetch: deduplicate local git testing code
bitbake: tests/fetch: add tests for local and remote "noshared" git fetching
Jon Mason (1):
oeqa/runtime: space needed
Jonas Höppner (1):
ltp: fix empty ltp-dev package
Jose Quaresma (4):
gstreamer1.0: update patch upstream status
ptest-runner: libgcc must be installed for pthread_cancel to work
gstreamer1.0: rename patches
gstreamer1.0: update ptest patch
Joshua Watt (2):
bitbake: knotty: Re-enable command line logging levels
classes/image: Use xargs to set file timestamps
Kai Kang (2):
cmake.bbclass: remove ${B} before cmake_do_configure
kernel-yocto.bbclass: chdir to ${WORKDIR} for do_kernel_checkout
Kevin Hao (3):
modutils-initscripts: Bail out when no module is installed
sysvinit-inittab/start_getty: Check /sys for the tty device existence
Revert "inittab: Add getty launch on hvc0 for qemuppc64"
Khairul Rohaizzat Jamaluddin (1):
qemu: Fix CVE-2020-35517
Khem Raj (54):
gcc: Upgrade to 10.3.0 bug-fix release
glibc: Rename glibc src package
gcc-runtime: Make DEBUG_PREFIX_MAP relative to S
valgrind: Delete trailing whitespaces
valgrind: Add glibc-src to ptest rdeps
valgrind: Add libstdc++ debug symbols for ptest
vte: Upgrade to 0.64.0 release
systemd: Fix build on mips/musl
epiphany: Add missing dependency on gnutls
cups: Turn gnutls into a packageconfig knob
wpa-supplicant: Enable openssl
curl: Use openssl backend
libpsl: Add config knobs for runtime/builtin conversion choices
glib-networking: Prefer openssl backend instead of gnutls
gstreamer1.0-plugins-bad: Add packageconfigs for hls crypto backends
ca-certificates: Fix openssl runtime cert dependencies
weston: Drop loading xwayland.so module
elfutils: Make 64bit time_t fix generic
binutils: Fix linking failures when using dwarf-5
go: Use dl.google.com for SRC_URI
musl: Update to latest master
llvm: Upgrade to LLVM 12 release
python3-docutils: Upgrade to 0.17.1
python3-markupsafe: Enable ptests
python3-jinja2: Enable ptests
python3-pyyaml: Add recipe
apt: Fix build on musl when seccomp is enabled
default-distrovars.inc: Remove seccomp for riscv32
gcc-target: Create a LTO plugin symlink in bfd-plugins directory
bitbake.conf: Use gcc-nm as default NM
gcc-cross: Install linker LTO plugin for binutils tools
gcc-cross-canadian: Install LTO linker plugin to BFD searchable location
gnutls: Point to staging area for finding seccomp libs and includes
libjpeg-turbo: Use --reproducible option for nasm
libid3tag: Filter -ffile-prefix-map too
openssl: Filter out -ffile-prefix-map as well
ltp: Filter out -ffile-prefix-map
gcc-runtime: Fix __FILE__ related reproducablity issues
reproducible_build.bbclass: Enable -Wdate-time
pkgconfig: Fix nativesdk builds for mingw sdk hosts
m4: Do not use SIGSTKSZ
bluez: Fix shadowing of pause function from libc
valgrind: Disable leak_cpp_interior test
findutils: Do not use SIGSTKSZ
bash: Include files needed for run-heredoc ptest
libpam: Provide needed env for tst-pam_start_confdir ptest
cml1.bbclass: Return sorted list of cfg files
busybox: Enable long options for enabled applets
webkitgtk: Fix reproducibility in minibrowser
webkitgtk: Update patch status
libgcc-initial: Do not build fp128 to decimal ppc functions
gcc: Upgrade to GCC 11
busybox: Fix reproducibility
strace: Upgrade to 5.12
Konrad Weihmann (2):
cpan-base: set default UPSTREAM_CHECK_REGEX
cve-update-db-native: skip on empty cpe23Uri
Marek Vasut (1):
linux-firmware: Package RSI 911x WiFi firmware
Martin Jansa (2):
xwayland: add opengl to REQUIRED_DISTRO_FEATURES
ofono: prevent using bundled ell headers and fix build with ell-0.39
Michael Halstead (1):
releases: update to include 3.3
Michael Opdenacker (7):
dev-manual: fix code insertion
manuals: simplify code insertion
manuals: code insertion simplification over two lines
bitbake: doc: bitbake-user-manual: simplify colon usage
bitbake: doc: bitbake-user-manual: code insertion simplification over two lines
dev-manual: update references to Docker installation instructions
sanity.bbclass: mention CONNECTIVITY_CHECK_URIS in network failure message
Mikko Rapeli (4):
bitbake: bitbake: tests/fetch: fix test execution without .gitconfig
bitbake: bitbake: tests/fetch: remove write protected files too
lz4: use CFLAGS from bitbake
unzip: use optimization from bitbake
Mingli Yu (6):
libxshmfence: Build fixes for riscv32
packagegroup-core-tools-profile: Remove valgrind for riscv32
packagegroup-core-tools-testapps.bb: Remove kexec for riscv32
libtool: make sure autoheader run before automake
groff: not ship /usr/bin/grap2graph
rpm: Upgrade to 4.16.1.3
Minjae Kim (1):
qemu: fix CVE-2021-3392
Nicolas Dechesne (1):
bitbake: doc: bitbake-user-manual: fix typo left over from Sphinx migration
Niels Avonds (1):
bitbake: fetch/gitsm: Fix crash when using git LFS and submodules
Oleksandr Kravchuk (2):
python3-setuptools: update to 56.0.0
autoconf-archive: update to 2021.02.19
Otavio Salvador (2):
gstreamer1.0-plugins-base: Add 'viv-fb' OpenGL Window System option
gstreamer1.0-plugins-base: Use bb.utils.filter to reduce code
Paul Barker (10):
bitbake: hashserv: Use generic ConnectionError
bitbake: asyncrpc: Common implementation of RPC using json & asyncio
bitbake: hashserv: Refactor to use asyncrpc
bitbake: prserv: Drop obsolete python version check
bitbake: prserv: Drop unused dump_db method
bitbake: prserv: Add connect function
prservice: Use new connect API
bitbake: prserv: Use multiprocessing to auto start prserver
bitbake: prserv: Extract daemonization from PRServer class
bitbake: prserv: Handle requests in main thread
Paulo Cesar Zaneti (1):
perl: fix startperl configuration option for perl-native
Peter Budny (1):
lib/oe/terminal: Fix tmux new-session on older tmux versions (<1.9)
Petr Vorel (1):
ltp: Replace musl patches with do_patch[postfuncs]
Przemyslaw Gorszkowski (2):
bitbake: progress: LineFilterProgressHandler - Handle parsing line which ends with CR only
bitbake: fetch/s3: Add progress handler for S3 cp command
Randy MacLeod (2):
sqlite3: upgrade 3.35.0 -> 3.35.3
oe-time-dd-test.sh: increase timeout to 15 sec
Reto Schneider (2):
license_image.bbclass: Detect broken symlinks
license_image.bbclass: Fix symlink to generic license files
Richard Purdie (32):
oeqa/selftest: Hardcode test assumptions about heartbeat event timings
pseudo: Upgrade to add trailing slashes ignore path fix
oeqa/selftest: Ensure packages classes are set correctly for maintainers test
layer.conf: Update to add post 3.3 release honister series
sanity: Add error check for '%' in build path
bitbake: runqueue: Fix deferred task issues
bitbake: tinfoil/data_smart: Allow variable history emit() to function remotely
sanity: Further improve directory sanity tests
bitbake: bitbake-server: Remove now unneeded code
bitbake: doc/user-manual-fetching: Remove basepath unpack parameter docs
poky.conf: Post release version bump
runqemu: Ensure we cleanup snapshot files after image run
patchelf: Backport fix from upstream for note section overlap error
pyyaml: Add missing HOMEPAGE
yocto-check-layer: Avoid bug when iterating and autoadding dependencies
libseccomp: Add MAINTAINERS entry and HOMEPAGE
libseccomp: Fix reproducibility issue
apt: Disable libseccomp
libxcrypt: Update to 4.4.19 release and fix symbol version issues
patchelf: Fix note section alignment issues
bitbake: runqueue: Fix multiconfig deferred task sstate validity caching issue
bitbake: runqueue: Handle deferred task rehashing in multiconfig builds
patchelf: Fix alignment patch
pybootchart/draw: Avoid divide by zero error
yocto-uninative: Update to 3.1 which includes a patchelf fix
Revert "perl: fix startperl configuration option for perl-native"
bitbake: bin/bitbake-getvar: Add a new command to query a variable value (with history)
bitbake: bitbake: Switch to post release version number 1.51.0
sanity.conf: Require bitbake 1.51.0
oeqa/qemurunner: Improve logging thread exit handling for qemu shutdown test
oeqa/qemurunner: Handle path length issues for qmp socket
lib/package_manager: Use shutil.copy instead of bb.utils.copyfile for intercepts
Robert Joslyn (3):
btrfs-tools: Update to 5.11.1
btrfs-tools: Add PACKAGECONFIG options
btrfs-tools: Try to follow style guide
Robert P. J. Day (3):
sdk-manual: "beablebone" -> "beaglebone"
sdk-manual: fix broken formatting of sample command
bitbake.conf: sort MIRROR list, add missing SAMBA_MIRROR
Ross Burton (4):
glslang: strip whitespace in pkgconfig file
insane: clean up some more warning messages
bitbake: bitbake-server: ensure server timeout is a float
oe-buildenv-internal: add BitBake's library to PYTHONPATH
Sakib Sajal (12):
oe-time-dd-test.sh: make executable
oe-time-dd-test.sh: provide more information from "top"
qemu: fix CVE-2021-20181
qemu: fix CVE-2020-29443
qemu: fix CVE-2021-20221
qemu: fix CVE-2021-3409
qemu: fix CVE-2021-3416
qemu: fix CVE-2021-20257
oe-time-dd-test.sh: collect cooker log when timeout is exceeded
buildstats.bbclass: collect data in the same file.
qemu: fix CVE-2020-27821
qemu: fix CVE-2021-20263
Samuli Piippo (1):
assimp: BBCLASSEXTEND to native and nativesdk
Saul Wold (4):
pango: re-enable ptest
qemu-system-native: install qmp python module
qemurunner: Add support for qmp commands
qemurunner: change warning to info
Stefan Ghinea (3):
wpa-supplicant: fix CVE-2021-30004
libssh2: fix build failure with option no-ecdsa
xserver-xorg: fix CVE-2021-3472
Stefano Babic (1):
libubootenv: upgrade 0.3.1 -> 0.3.2
Teoh Jay Shen (6):
oeqa/manual/bsp-hw.json : remove boot_from_runlevel_3 and boot_from_runlevel_5 manual test
oeqa/manual/bsp-hw.json : remove ethernet_static_ip_set_in_connman and ethernet_get_IP_in_connman_via_DHCP manual test
oeqa/manual/bsp-hw.json : remove standby and Test_if_LAN_device_works_well_after_resume_from_suspend_state manual test
oeqa/manual/bsp-hw.json : remove click_terminal_icon_on_X_desktop manual test
oeqa/manual/bsp-hw.json :remove Check_if_RTC_(Real_Time_Clock)_can_work_correctly manual test
oeqa/manual/bsp-hw.json : remove Test_if_usb_hid_device_works_well_after_resume_from_suspend_state manual test
Trevor Gamblin (2):
nettle: upgrade 3.7.1 -> 3.7.2
ref-manual/variables.rst: Add incompatibility warning for SERIAL_CONSOLES_CHECK
Ulrich Ölmann (1):
arch-armv6m.inc: fix access rights
Vinay Kumar (1):
binutils: Fix CVE-2021-20197
Vineela Tummalapalli (1):
Adding dunfell 3.1.7 to the switcher and release list.
Wang Mingyu (6):
at-spi2-core: upgrade 2.38.0 -> 2.40.0
babeltrace2: upgrade 2.0.3 -> 2.0.4
boost-build-native: upgrade 4.3.0 -> 4.4.1
libassuan: upgrade 2.5.4 -> 2.5.5
webkitgtk: upgrade 2.30.5 -> 2.30.6
vte: upgrade 0.62.2 -> 0.62.3
Wes Lindauer (1):
oeqa/runtime/cases: Only disable/enable for current boot
Yanfei Xu (1):
parselogs: ignore floppy error on qemu-system-x86 at boot stage
Yi Fan Yu (7):
valgrind: update 3.16.1 -> 3.17.0
valgrind: Disable ptest swapcontext.vgtest
valgrind: Fix ptest swapcontext.vgtest
Revert "glib-2.0: add workaround to fix codegen.py.test failing"
re2c: Upgrade 2.0.3 -> 2.1.1
valgrind: Enable drd/tests/bar_bad* ptest
libevent: Increase ptest timing tolerance 50 ms -> 100 ms
Zqiang (1):
rt-tests: Update rt-tests
hongxu (1):
deb: apply postinstall on sdk
wangmy (34):
ell: upgrade 0.38 -> 0.39
dbus-glib: upgrade 0.110 -> 0.112
ccache: upgrade 4.2 -> 4.2.1
gcr: upgrade 3.38.1 -> 3.40.0
ghostscript: upgrade 9.53.3 -> 9.54.0
libsolv: upgrade 0.7.17 -> 0.7.18
glib-2.0: upgrade 2.66.7 -> 2.68.0
file: upgrade 5.39 -> 5.40
curl: upgrade 7.75.0 -> 7.76.0
acpica: upgrade 20210105 -> 20210331
help2man: upgrade 1.48.2 -> 1.48.3
libportal: upgrade 0.3 -> 0.4
libksba: upgrade 1.5.0 -> 1.5.1
go: upgrade 1.16.2 -> 1.16.3
libcap: upgrade 2.48 -> 2.49
libcomps: upgrade 0.1.15 -> 0.1.16
icu: upgrade 68.2 -> 69.1
mpg123: upgrade 1.26.4 -> 1.26.5
man-pages: upgrade 5.10 -> 5.11
go: update SRC_URI to use https protocol
mesa: upgrade 21.0.1 -> 21.0.2
openssh: upgrade 8.5p1 -> 8.6p1
mtools: upgrade 4.0.26 -> 4.0.27
python3-cython: upgrade 0.29.22 -> 0.29.23
tiff: upgrade 4.2.0 -> 4.3.0
boost: upgrade 1.75.0 -> 1.76.0
wpebackend-fdo: upgrade 1.8.2 -> 1.8.3
mesa: upgrade 21.0.2 -> 21.0.3
gdb: upgrade 10.1 -> 10.2
glib-networking: upgrade 2.66.0 -> 2.68.1
glslang: upgrade 11.2.0 -> 11.4.0
hdparm: upgrade 9.60 -> 9.61
libhandy: upgrade 1.2.1 -> 1.2.2
libjitterentropy: upgrade 3.0.1 -> 3.0.2
zangrc (1):
maintainers.inc: Modify email address
zhengruoqin (19):
epiphany: upgrade 3.38.2 -> 3.38.3
wpebackend-fdo: upgrade 1.8.0 -> 1.8.2
netbase: upgrade 6.2 -> 6.3
python3-dbusmock: upgrade 0.22.0 -> 0.23.0
python3-gitdb: upgrade 4.0.5 -> 4.0.7
libva: upgrade 2.10.0 -> 2.11.0
ruby: upgrade 3.0.0 -> 3.0.1
libva-utils: upgrade 2.10.0 -> 2.11.1
libdazzle: upgrade 3.38.0 -> 3.40.0
librepo: upgrade 1.13.0 -> 1.14.0
libdrm: upgrade 2.4.104 -> 2.4.105
python3-pygobject: upgrade 3.38.0 -> 3.40.1
libedit: upgrade 20210216-3.1 -> 20210419-3.1
libhandy: upgrade 1.2.0 -> 1.2.1
libical: upgrade 3.0.9 -> 3.0.10
libsolv: upgrade 0.7.18 -> 0.7.19
libmicrohttpd: upgrade 0.9.72 -> 0.9.73
python3-numpy: upgrade 1.20.1 -> 1.20.2
wireless-regdb: upgrade 2020.11.20 -> 2021.04.21
Signed-off-by: Andrew Geissler <geissonator@yahoo.com>
Change-Id: Ibdaea694cae40b0749d472bf08b53002a45b31d7
diff --git a/poky/bitbake/bin/bitbake b/poky/bitbake/bin/bitbake
index 7127550..f0ff240 100755
--- a/poky/bitbake/bin/bitbake
+++ b/poky/bitbake/bin/bitbake
@@ -26,7 +26,7 @@
if sys.getfilesystemencoding() != "utf-8":
sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
-__version__ = "1.50.0"
+__version__ = "1.51.0"
if __name__ == "__main__":
if __version__ != bb.__version__:
diff --git a/poky/bitbake/bin/bitbake-getvar b/poky/bitbake/bin/bitbake-getvar
new file mode 100755
index 0000000..9423219
--- /dev/null
+++ b/poky/bitbake/bin/bitbake-getvar
@@ -0,0 +1,48 @@
+#! /usr/bin/env python3
+#
+# Copyright (C) 2021 Richard Purdie
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import argparse
+import io
+import os
+import sys
+
+bindir = os.path.dirname(__file__)
+topdir = os.path.dirname(bindir)
+sys.path[0:0] = [os.path.join(topdir, 'lib')]
+
+import bb.tinfoil
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Bitbake Query Variable")
+ parser.add_argument("variable", help="variable name to query")
+ parser.add_argument("-r", "--recipe", help="Recipe name to query", default=None, required=False)
+ parser.add_argument('-u', '--unexpand', help='Do not expand the value (with --value)', action="store_true")
+ parser.add_argument('-f', '--flag', help='Specify a variable flag to query (with --value)', default=None)
+ parser.add_argument('--value', help='Only report the value, no history and no variable name', action="store_true")
+ args = parser.parse_args()
+
+ if args.unexpand and not args.value:
+ print("--unexpand only makes sense with --value")
+ sys.exit(1)
+
+ if args.flag and not args.value:
+ print("--flag only makes sense with --value")
+ sys.exit(1)
+
+ with bb.tinfoil.Tinfoil(tracking=True) as tinfoil:
+ if args.recipe:
+ tinfoil.prepare(quiet=2)
+ d = tinfoil.parse_recipe(args.recipe)
+ else:
+ tinfoil.prepare(quiet=2, config_only=True)
+ d = tinfoil.config_data
+ if args.flag:
+ print(str(d.getVarFlag(args.variable, args.flag, expand=(not args.unexpand))))
+ elif args.value:
+ print(str(d.getVar(args.variable, expand=(not args.unexpand))))
+ else:
+ bb.data.emit_var(args.variable, d=d, all=True)
diff --git a/poky/bitbake/bin/bitbake-server b/poky/bitbake/bin/bitbake-server
index ffbc789..8e53947 100755
--- a/poky/bitbake/bin/bitbake-server
+++ b/poky/bitbake/bin/bitbake-server
@@ -26,12 +26,10 @@
logfile = sys.argv[4]
lockname = sys.argv[5]
sockname = sys.argv[6]
-timeout = sys.argv[7]
+timeout = float(sys.argv[7])
xmlrpcinterface = (sys.argv[8], int(sys.argv[9]))
if xmlrpcinterface[0] == "None":
xmlrpcinterface = (None, xmlrpcinterface[1])
-if timeout == "None":
- timeout = None
# Replace standard fds with our own
with open('/dev/null', 'r') as si:
diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst
index 56abf77..14c342a 100644
--- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst
+++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst
@@ -16,7 +16,7 @@
This chapter describes BitBake's execution process from start to finish
when you use it to create an image. The execution process is launched
-using the following command form: ::
+using the following command form::
$ bitbake target
@@ -32,7 +32,7 @@
your project's ``local.conf`` configuration file.
A common method to determine this value for your build host is to run
- the following: ::
+ the following::
$ grep processor /proc/cpuinfo
@@ -139,7 +139,7 @@
A good way to get an idea of the configuration files and the class files
used in your execution environment is to run the following BitBake
-command: ::
+command::
$ bitbake -e > mybb.log
@@ -155,7 +155,7 @@
pair of curly braces in a shell function, the closing curly brace
must not be located at the start of the line without leading spaces.
- Here is an example that causes BitBake to produce a parsing error: ::
+ Here is an example that causes BitBake to produce a parsing error::
fakeroot create_shar() {
cat << "EOF" > ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh
@@ -185,7 +185,7 @@
:term:`BBFILES`. BitBake now uses it to construct a
list of recipes to parse, along with any append files (``.bbappend``) to
apply. ``BBFILES`` is a space-separated list of available files and
-supports wildcards. An example would be: ::
+supports wildcards. An example would be::
BBFILES = "/path/to/bbfiles/*.bb /path/to/appends/*.bbappend"
@@ -206,7 +206,7 @@
One common convention is to use the recipe filename to define pieces of
metadata. For example, in ``bitbake.conf`` the recipe name and version
are used to set the variables :term:`PN` and
-:term:`PV`: ::
+:term:`PV`::
PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
@@ -238,7 +238,7 @@
repositories of ``.bb`` files that contain the same exact package. For
example, one could easily use them to make one's own local copy of an
upstream repository, but with custom modifications that one does not
-want upstream. Here is an example: ::
+want upstream. Here is an example::
BBFILES = "/stuff/openembedded/*/*.bb /stuff/openembedded.modified/*/*.bb"
BBFILE_COLLECTIONS = "upstream local"
@@ -270,7 +270,7 @@
When a recipe uses ``PROVIDES``, that recipe's functionality can be
found under an alternative name or names other than the implicit ``PN``
name. As an example, suppose a recipe named ``keyboard_1.0.bb``
-contained the following: ::
+contained the following::
PROVIDES += "fullkeyboard"
@@ -291,7 +291,7 @@
A common example in which a target has multiple providers is
"virtual/kernel", which is on the ``PROVIDES`` list for each kernel
recipe. Each machine often selects the best kernel provider by using a
-line similar to the following in the machine configuration file: ::
+line similar to the following in the machine configuration file::
PREFERRED_PROVIDER_virtual/kernel = "linux-yocto"
@@ -331,7 +331,7 @@
Thus, if a recipe named ``a_1.2.bb`` exists, BitBake will choose 1.2 by
default. However, if you define the following variable in a ``.conf``
-file that BitBake parses, you can change that preference: ::
+file that BitBake parses, you can change that preference::
PREFERRED_VERSION_a = "1.1"
@@ -498,7 +498,7 @@
Like the working directory case, situations exist where dependencies
should be ignored. For these cases, you can instruct the build process
-to ignore a dependency by using a line like the following: ::
+to ignore a dependency by using a line like the following::
PACKAGE_ARCHS[vardepsexclude] = "MACHINE"
@@ -508,7 +508,7 @@
Equally, there are cases where we need to add dependencies BitBake is
not able to find. You can accomplish this by using a line like the
-following: ::
+following::
PACKAGE_ARCHS[vardeps] = "MACHINE"
@@ -536,7 +536,7 @@
it construct the basehash. The following statement effectively results
in a list of global variable dependency excludes - variables never
included in any checksum. This example uses variables from OpenEmbedded
-to help illustrate the concept: ::
+to help illustrate the concept::
BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \
SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL \
@@ -557,7 +557,7 @@
is a dummy "noop" signature handler enabled in BitBake. This means that
behavior is unchanged from previous versions. ``OE-Core`` uses the
"OEBasicHash" signature handler by default through this setting in the
-``bitbake.conf`` file: ::
+``bitbake.conf`` file::
BB_SIGNATURE_HANDLER ?= "OEBasicHash"
diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst
index bd6cc0e..bd1fb4f 100644
--- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst
+++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst
@@ -27,7 +27,7 @@
Patching, however, is not covered by this module.
The code to execute the first part of this process, a fetch, looks
-something like the following: ::
+something like the following::
src_uri = (d.getVar('SRC_URI') or "").split()
fetcher = bb.fetch2.Fetch(src_uri, d)
@@ -37,7 +37,7 @@
space-separated list of URLs from the :term:`SRC_URI`
variable and then calls the ``download`` method to download the files.
-The instantiation of the fetch class is usually followed by: ::
+The instantiation of the fetch class is usually followed by::
rootdir = l.getVar('WORKDIR')
fetcher.unpack(rootdir)
@@ -72,7 +72,7 @@
For each URL passed to the fetcher, the fetcher calls the submodule that
handles that particular URL type. This behavior can be the source of
some confusion when you are providing URLs for the ``SRC_URI`` variable.
-Consider the following two URLs: ::
+Consider the following two URLs::
http://git.yoctoproject.org/git/poky;protocol=git
git://git.yoctoproject.org/git/poky;protocol=http
@@ -81,7 +81,7 @@
understand "git". Therefore, the latter case is the correct form since the Git
fetcher does know how to use HTTP as a transport.
-Here are some examples that show commonly used mirror definitions: ::
+Here are some examples that show commonly used mirror definitions::
PREMIRRORS ?= "\
bzr://.*/.\* http://somemirror.org/sources/ \\n \
@@ -111,19 +111,19 @@
non-local archive downloads, the fetcher code can verify SHA-256 and MD5
checksums to ensure the archives have been downloaded correctly. You can
specify these checksums by using the ``SRC_URI`` variable with the
-appropriate varflags as follows: ::
+appropriate varflags as follows::
SRC_URI[md5sum] = "value"
SRC_URI[sha256sum] = "value"
You can also specify the checksums as
-parameters on the ``SRC_URI`` as shown below: ::
+parameters on the ``SRC_URI`` as shown below::
SRC_URI = "http://example.com/foobar.tar.bz2;md5sum=4a8e0f237e961fd7785d19d07fdb994d"
If multiple URIs exist, you can specify the checksums either directly as
in the previous example, or you can name the URLs. The following syntax
-shows how you name the URIs: ::
+shows how you name the URIs::
SRC_URI = "http://example.com/foobar.tar.bz2;name=foo"
SRC_URI[foo.md5sum] = 4a8e0f237e961fd7785d19d07fdb994d
@@ -163,9 +163,6 @@
- *dos:* Applies to ``.zip`` and ``.jar`` files and specifies whether
to use DOS line ending conversion on text files.
-- *basepath:* Instructs the unpack stage to strip the specified
- directories from the source path when unpacking.
-
- *subdir:* Unpacks the specific URL to the specified subdirectory
within the root directory.
@@ -204,7 +201,7 @@
If you specify a directory, the entire directory is unpacked.
Here are a couple of example URLs, the first relative and the second
-absolute: ::
+absolute::
SRC_URI = "file://relativefile.patch"
SRC_URI = "file:///Users/ich/very_important_software"
@@ -225,7 +222,7 @@
:term:`DL_DIR` when dealing with multiple files that
have the same name.
-Some example URLs are as follows: ::
+Some example URLs are as follows::
SRC_URI = "http://oe.handhelds.org/not_there.aac"
SRC_URI = "ftp://oe.handhelds.org/not_there_as_well.aac"
@@ -235,15 +232,13 @@
Because URL parameters are delimited by semi-colons, this can
introduce ambiguity when parsing URLs that also contain semi-colons,
- for example:
- ::
+ for example::
SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git;a=snapshot;h=a5dd47"
Such URLs should should be modified by replacing semi-colons with '&'
- characters:
- ::
+ characters::
SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&a=snapshot&h=a5dd47"
@@ -251,8 +246,7 @@
In most cases this should work. Treating semi-colons and '&' in
queries identically is recommended by the World Wide Web Consortium
(W3C). Note that due to the nature of the URL, you may have to
- specify the name of the downloaded file as well:
- ::
+ specify the name of the downloaded file as well::
SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&a=snapshot&h=a5dd47;downloadfilename=myfile.bz2"
@@ -321,7 +315,7 @@
- *"port":* The port to which the CVS server connects.
-Some example URLs are as follows: ::
+Some example URLs are as follows::
SRC_URI = "cvs://CVSROOT;module=mymodule;tag=some-version;method=ext"
SRC_URI = "cvs://CVSROOT;module=mymodule;date=20060126;localdir=usethat"
@@ -363,7 +357,7 @@
username is different than the username used in the main URL, which
is passed to the subversion command.
-Following are three examples using svn: ::
+Following are three examples using svn::
SRC_URI = "svn://myrepos/proj1;module=vip;protocol=http;rev=667"
SRC_URI = "svn://myrepos/proj1;module=opie;protocol=svn+ssh"
@@ -436,7 +430,7 @@
parameter implies no branch and only works when the transfer protocol
is ``file://``.
-Here are some example URLs: ::
+Here are some example URLs::
SRC_URI = "git://git.oe.handhelds.org/git/vip.git;tag=version-1"
SRC_URI = "git://git.oe.handhelds.org/git/vip.git;protocol=http"
@@ -484,7 +478,7 @@
To use this fetcher, make sure your recipe has proper
:term:`SRC_URI`, :term:`SRCREV`, and
-:term:`PV` settings. Here is an example: ::
+:term:`PV` settings. Here is an example::
SRC_URI = "ccrc://cc.example.org/ccrc;vob=/example_vob;module=/example_module"
SRCREV = "EXAMPLE_CLEARCASE_TAG"
@@ -506,7 +500,7 @@
The module and vob options are combined to create the load rule in the
view config spec. As an example, consider the vob and module values from
the SRC_URI statement at the start of this section. Combining those values
- results in the following: ::
+ results in the following::
load /example_vob/example_module
@@ -558,7 +552,7 @@
password directly in your recipe within ``SRC_URI``.
Here is an example that relies on ``P4CONFIG`` to specify the server URL
-and port, username, and password, and fetches the Head Revision: ::
+and port, username, and password, and fetches the Head Revision::
SRC_URI = "p4://example-depot/main/source/..."
SRCREV = "${AUTOREV}"
@@ -566,7 +560,7 @@
S = "${WORKDIR}/p4"
Here is an example that specifies the server URL and port, username, and
-password, and fetches a Revision based on a Label: ::
+password, and fetches a Revision based on a Label::
P4PORT = "tcp:p4server.example.net:1666"
SRC_URI = "p4://user:passwd@example-depot/main/source/..."
@@ -592,7 +586,7 @@
paths locally for the specified location, even in combination with the
``module`` parameter.
-Here is an example use of the the ``module`` parameter: ::
+Here is an example use of the the ``module`` parameter::
SRC_URI = "p4://user:passwd@example-depot/main;module=source/..."
@@ -600,7 +594,7 @@
to ``${P4DIR}``, including the directory itself. The top-level directory will
be accesible at ``${P4DIR}/source/``.
-Here is an example use of the the ``remotepath`` parameter: ::
+Here is an example use of the the ``remotepath`` parameter::
SRC_URI = "p4://user:passwd@example-depot/main;module=source/...;remotepath=keep"
@@ -628,7 +622,7 @@
- *"manifest":* Name of the manifest file (default: ``default.xml``).
-Here are some example URLs: ::
+Here are some example URLs::
SRC_URI = "repo://REPOROOT;protocol=git;branch=some_branch;manifest=my_manifest.xml"
SRC_URI = "repo://REPOROOT;protocol=file;branch=some_branch;manifest=my_manifest.xml"
@@ -651,11 +645,11 @@
delegate access to resources, if this variable is set, the Az Fetcher will
use it when fetching artifacts from the cloud.
-You can specify the AZ_SAS variable as shown below: ::
+You can specify the AZ_SAS variable as shown below::
AZ_SAS = "se=2021-01-01&sp=r&sv=2018-11-09&sr=c&skoid=<skoid>&sig=<signature>"
-Here is an example URL: ::
+Here is an example URL::
SRC_URI = "az://<azure-storage-account>.blob.core.windows.net/<foo_container>/<bar_file>"
diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst
index e3fd321..a9c3370 100644
--- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst
+++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst
@@ -20,7 +20,7 @@
See the :ref:`bitbake-user-manual/bitbake-user-manual-hello:obtaining bitbake` section for
information on how to obtain BitBake. Once you have the source code on
-your machine, the BitBake directory appears as follows: ::
+your machine, the BitBake directory appears as follows::
$ ls -al
total 100
@@ -49,7 +49,7 @@
First, you need to be sure that you can run BitBake. Set your working
directory to where your local BitBake files are and run the following
-command: ::
+command::
$ ./bin/bitbake --version
BitBake Build Tool Core version 1.23.0, bitbake version 1.23.0
@@ -61,14 +61,14 @@
choice. To be able to run BitBake from any directory, you need to add
the executable binary to your binary to your shell's environment
``PATH`` variable. First, look at your current ``PATH`` variable by
-entering the following: ::
+entering the following::
$ echo $PATH
Next, add the directory location
for the BitBake binary to the ``PATH``. Here is an example that adds the
``/home/scott-lenovo/bitbake/bin`` directory to the front of the
-``PATH`` variable: ::
+``PATH`` variable::
$ export PATH=/home/scott-lenovo/bitbake/bin:$PATH
@@ -116,7 +116,7 @@
#. **Create a Project Directory:** First, set up a directory for the
"Hello World" project. Here is how you can do so in your home
- directory: ::
+ directory::
$ mkdir ~/hello
$ cd ~/hello
@@ -127,7 +127,7 @@
directory is a good way to isolate your project.
#. **Run BitBake:** At this point, you have nothing but a project
- directory. Run the ``bitbake`` command and see what it does: ::
+ directory. Run the ``bitbake`` command and see what it does::
$ bitbake
The BBPATH variable is not set and bitbake did not
@@ -161,7 +161,7 @@
``BBPATH`` variable up in a configuration file for each project.
From your shell, enter the following commands to set and export the
- ``BBPATH`` variable: ::
+ ``BBPATH`` variable::
$ BBPATH="projectdirectory"
$ export BBPATH
@@ -176,7 +176,7 @@
shell would.
#. **Run BitBake:** Now that you have ``BBPATH`` defined, run the
- ``bitbake`` command again: ::
+ ``bitbake`` command again::
$ bitbake
ERROR: Traceback (most recent call last):
@@ -208,13 +208,13 @@
http://git.openembedded.org/bitbake/tree/conf/bitbake.conf.
Use the following commands to create the ``conf`` directory in the
- project directory: ::
+ project directory::
$ mkdir conf
From within the ``conf`` directory,
use some editor to create the ``bitbake.conf`` so that it contains
- the following: ::
+ the following::
PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
@@ -251,7 +251,7 @@
glossary.
#. **Run BitBake:** After making sure that the ``conf/bitbake.conf`` file
- exists, you can run the ``bitbake`` command again: ::
+ exists, you can run the ``bitbake`` command again::
$ bitbake
ERROR: Traceback (most recent call last):
@@ -278,7 +278,7 @@
in the ``classes`` directory of the project (i.e ``hello/classes``
in this example).
- Create the ``classes`` directory as follows: ::
+ Create the ``classes`` directory as follows::
$ cd $HOME/hello
$ mkdir classes
@@ -291,7 +291,7 @@
environments BitBake is supporting.
#. **Run BitBake:** After making sure that the ``classes/base.bbclass``
- file exists, you can run the ``bitbake`` command again: ::
+ file exists, you can run the ``bitbake`` command again::
$ bitbake
Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.
@@ -314,7 +314,7 @@
Minimally, you need a recipe file and a layer configuration file in
your layer. The configuration file needs to be in the ``conf``
directory inside the layer. Use these commands to set up the layer
- and the ``conf`` directory: ::
+ and the ``conf`` directory::
$ cd $HOME
$ mkdir mylayer
@@ -322,12 +322,12 @@
$ mkdir conf
Move to the ``conf`` directory and create a ``layer.conf`` file that has the
- following: ::
+ following::
BBPATH .= ":${LAYERDIR}"
- BBFILES += "${LAYERDIR}/\*.bb"
+ BBFILES += "${LAYERDIR}/*.bb"
BBFILE_COLLECTIONS += "mylayer"
- `BBFILE_PATTERN_mylayer := "^${LAYERDIR_RE}/"
+ BBFILE_PATTERN_mylayer := "^${LAYERDIR_RE}/"
For information on these variables, click on :term:`BBFILES`,
:term:`LAYERDIR`, :term:`BBFILE_COLLECTIONS` or :term:`BBFILE_PATTERN_mylayer <BBFILE_PATTERN>`
@@ -335,7 +335,7 @@
You need to create the recipe file next. Inside your layer at the
top-level, use an editor and create a recipe file named
- ``printhello.bb`` that has the following: ::
+ ``printhello.bb`` that has the following::
DESCRIPTION = "Prints Hello World"
PN = 'printhello'
@@ -356,7 +356,7 @@
follow the links to the glossary.
#. **Run BitBake With a Target:** Now that a BitBake target exists, run
- the command and provide that target: ::
+ the command and provide that target::
$ cd $HOME/hello
$ bitbake printhello
@@ -376,7 +376,7 @@
``hello/conf`` for this example).
Set your working directory to the ``hello/conf`` directory and then
- create the ``bblayers.conf`` file so that it contains the following: ::
+ create the ``bblayers.conf`` file so that it contains the following::
BBLAYERS ?= " \
/home/<you>/mylayer \
@@ -386,7 +386,7 @@
#. **Run BitBake With a Target:** Now that you have supplied the
``bblayers.conf`` file, run the ``bitbake`` command and provide the
- target: ::
+ target::
$ bitbake printhello
Parsing recipes: 100% |##################################################################################|
diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.rst
index 6f9d392..b3cea61 100644
--- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.rst
+++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.rst
@@ -248,13 +248,13 @@
When you name an append file, you can use the "``%``" wildcard character
to allow for matching recipe names. For example, suppose you have an
-append file named as follows: ::
+append file named as follows::
busybox_1.21.%.bbappend
That append file
would match any ``busybox_1.21.``\ x\ ``.bb`` version of the recipe. So,
-the append file would match the following recipe names: ::
+the append file would match the following recipe names::
busybox_1.21.1.bb
busybox_1.21.2.bb
@@ -290,7 +290,7 @@
are using. The metadata is generally backwards compatible but not
forward compatible.
- Here is an example that clones the BitBake repository: ::
+ Here is an example that clones the BitBake repository::
$ git clone git://git.openembedded.org/bitbake
@@ -298,7 +298,7 @@
Git repository into a directory called ``bitbake``. Alternatively,
you can designate a directory after the ``git clone`` command if you
want to call the new directory something other than ``bitbake``. Here
- is an example that names the directory ``bbdev``: ::
+ is an example that names the directory ``bbdev``::
$ git clone git://git.openembedded.org/bitbake bbdev
@@ -317,7 +317,7 @@
method for getting BitBake. Cloning the repository makes it easier
to update as patches are added to the stable branches.
- The following example downloads a snapshot of BitBake version 1.17.0: ::
+ The following example downloads a snapshot of BitBake version 1.17.0::
$ wget http://git.openembedded.org/bitbake/snapshot/bitbake-1.17.0.tar.gz
$ tar zxpvf bitbake-1.17.0.tar.gz
@@ -347,7 +347,7 @@
Usage and syntax
----------------
-Following is the usage and syntax for BitBake: ::
+Following is the usage and syntax for BitBake::
$ bitbake -h
Usage: bitbake [options] [recipename/target recipe:do_task ...]
@@ -469,11 +469,11 @@
when doing so.
The following command runs the build task, which is the default task, on
-the ``foo_1.0.bb`` recipe file: ::
+the ``foo_1.0.bb`` recipe file::
$ bitbake -b foo_1.0.bb
-The following command runs the clean task on the ``foo.bb`` recipe file: ::
+The following command runs the clean task on the ``foo.bb`` recipe file::
$ bitbake -b foo.bb -c clean
@@ -497,13 +497,13 @@
The ``bitbake`` command, when not using "--buildfile" or "-b" only
accepts a "PROVIDES". You cannot provide anything else. By default, a
recipe file generally "PROVIDES" its "packagename" as shown in the
-following example: ::
+following example::
$ bitbake foo
This next example "PROVIDES" the
package name and also uses the "-c" option to tell BitBake to just
-execute the ``do_clean`` task: ::
+execute the ``do_clean`` task::
$ bitbake -c clean foo
@@ -514,7 +514,7 @@
individual targets when you specify multiple targets. For example,
suppose you had two targets (or recipes) ``myfirstrecipe`` and
``mysecondrecipe`` and you needed BitBake to run ``taskA`` for the first
-recipe and ``taskB`` for the second recipe: ::
+recipe and ``taskB`` for the second recipe::
$ bitbake myfirstrecipe:do_taskA mysecondrecipe:do_taskB
@@ -540,7 +540,7 @@
``DEPENDS`` from inherited classes such as ``base.bbclass``.
Here are two examples that create dependency graphs. The second example
-omits depends common in OpenEmbedded from the graph: ::
+omits depends common in OpenEmbedded from the graph::
$ bitbake -g foo
@@ -582,17 +582,17 @@
configuration files for ``target1`` and ``target2`` defined in the build
directory. The following statement in the ``local.conf`` file both
enables BitBake to perform multiple configuration builds and specifies
-the two extra multiconfigs: ::
+the two extra multiconfigs::
BBMULTICONFIG = "target1 target2"
Once the target configuration files are in place and BitBake has been
enabled to perform multiple configuration builds, use the following
-command form to start the builds: ::
+command form to start the builds::
$ bitbake [mc:multiconfigname:]target [[[mc:multiconfigname:]target] ... ]
-Here is an example for two extra multiconfigs: ``target1`` and ``target2``: ::
+Here is an example for two extra multiconfigs: ``target1`` and ``target2``::
$ bitbake mc::target mc:target1:target mc:target2:target
@@ -613,12 +613,12 @@
To enable dependencies in a multiple configuration build, you must
declare the dependencies in the recipe using the following statement
-form: ::
+form::
task_or_package[mcdepends] = "mc:from_multiconfig:to_multiconfig:recipe_name:task_on_which_to_depend"
To better show how to use this statement, consider an example with two
-multiconfigs: ``target1`` and ``target2``: ::
+multiconfigs: ``target1`` and ``target2``::
image_task[mcdepends] = "mc:target1:target2:image2:rootfs_task"
@@ -629,7 +629,7 @@
associated with the "target2" multiconfig.
Once you set up this dependency, you can build the "target1" multiconfig
-using a BitBake command as follows: ::
+using a BitBake command as follows::
$ bitbake mc:target1:image1
@@ -639,7 +639,7 @@
Having a recipe depend on the root filesystem of another build might not
seem that useful. Consider this change to the statement in the image1
-recipe: ::
+recipe::
image_task[mcdepends] = "mc:target1:target2:image2:image_task"
diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst
index d4190c2..615c9f9 100644
--- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst
+++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst
@@ -26,7 +26,7 @@
VARIABLE = "value"
As expected, if you include leading or
-trailing spaces as part of an assignment, the spaces are retained: ::
+trailing spaces as part of an assignment, the spaces are retained::
VARIABLE = " value"
VARIABLE = "value "
@@ -40,7 +40,7 @@
You can use single quotes instead of double quotes when setting a
variable's value. Doing so allows you to use values that contain the
-double quote character: ::
+double quote character::
VARIABLE = 'I have a " in my value'
@@ -77,7 +77,7 @@
variable. You can make these checks for both configuration and recipe
level changes:
-- For configuration changes, use the following: ::
+- For configuration changes, use the following::
$ bitbake -e
@@ -91,7 +91,7 @@
Variables that are exported to the environment are preceded by the
string "export" in the command's output.
-- For recipe changes, use the following: ::
+- For recipe changes, use the following::
$ bitbake recipe -e \| grep VARIABLE="
@@ -105,7 +105,7 @@
BitBake joins any line ending in
a backslash character ("\") with the following line before parsing
statements. The most common use for the "\" character is to split
-variable assignments over multiple lines, as in the following example: ::
+variable assignments over multiple lines, as in the following example::
FOO = "bar \
baz \
@@ -116,7 +116,7 @@
newline characters end up in the value of ``FOO``.
Consider this additional example where the two assignments both assign
-"barbaz" to ``FOO``: ::
+"barbaz" to ``FOO``::
FOO = "barbaz"
FOO = "bar\
@@ -149,7 +149,7 @@
right-hand side. Instead, expansion is deferred until the variable
assigned to is actually used. The result depends on the current values
of the referenced variables. The following example should clarify this
-behavior: ::
+behavior::
A = "${B} baz"
B = "${C} bar"
@@ -177,7 +177,7 @@
You can use the "?=" operator to achieve a "softer" assignment for a
variable. This type of assignment allows you to define a variable if it
is undefined when the statement is parsed, but to leave the value alone
-if the variable has a value. Here is an example: ::
+if the variable has a value. Here is an example::
A ?= "aval"
@@ -199,7 +199,7 @@
except that the assignment is made at the end of the parsing process
rather than immediately. Consequently, when multiple "??=" assignments
exist, the last one is used. Also, any "=" or "?=" assignment will
-override the value set with "??=". Here is an example: ::
+override the value set with "??=". Here is an example::
A ??= "somevalue"
A ??= "someothervalue"
@@ -215,7 +215,7 @@
---------------------------------
The ":=" operator results in a variable's contents being expanded
-immediately, rather than when the variable is actually used: ::
+immediately, rather than when the variable is actually used::
T = "123"
A := "test ${T}"
@@ -241,7 +241,7 @@
current value and prepended or appended value.
These operators take immediate effect during parsing. Here are some
-examples: ::
+examples::
B = "bval"
B += "additionaldata"
@@ -260,7 +260,7 @@
the ".=" and "=." operators.
These operators take immediate effect during parsing. Here are some
-examples: ::
+examples::
B = "bval"
B .= "additionaldata"
@@ -278,7 +278,7 @@
These operators differ from the ":=", ".=", "=.", "+=", and "=+"
operators in that their effects are applied at variable expansion time
-rather than being immediately applied. Here are some examples: ::
+rather than being immediately applied. Here are some examples::
B = "bval"
B_append = " additional data"
@@ -309,7 +309,7 @@
value to be removed from the variable.
When you use this syntax, BitBake expects one or more strings.
-Surrounding spaces and spacing are preserved. Here is an example: ::
+Surrounding spaces and spacing are preserved. Here is an example::
FOO = "123 456 789 123456 123 456 123 456"
FOO_remove = "123"
@@ -334,27 +334,27 @@
"_remove" as compared to the "+=" and "=+" operators is that the
override style operators provide guaranteed operations. For example,
consider a class ``foo.bbclass`` that needs to add the value "val" to
-the variable ``FOO``, and a recipe that uses ``foo.bbclass`` as follows: ::
+the variable ``FOO``, and a recipe that uses ``foo.bbclass`` as follows::
inherit foo
FOO = "initial"
If ``foo.bbclass`` uses the "+=" operator,
as follows, then the final value of ``FOO`` will be "initial", which is
-not what is desired: ::
+not what is desired::
FOO += "val"
If, on the other hand, ``foo.bbclass``
uses the "_append" operator, then the final value of ``FOO`` will be
-"initial val", as intended: ::
+"initial val", as intended::
FOO_append = " val"
.. note::
It is never necessary to use "+=" together with "_append". The following
- sequence of assignments appends "barbaz" to FOO: ::
+ sequence of assignments appends "barbaz" to FOO::
FOO_append = "bar"
FOO_append = "baz"
@@ -381,7 +381,7 @@
except for override style syntax (i.e. "_prepend", "_append", and
"_remove").
-Here are some examples showing how to set variable flags: ::
+Here are some examples showing how to set variable flags::
FOO[a] = "abc"
FOO[b] = "123"
@@ -393,7 +393,7 @@
No need exists to pre-define variable flags. You can simply start using
them. One extremely common application is to attach some brief
-documentation to a BitBake variable as follows: ::
+documentation to a BitBake variable as follows::
CACHE[doc] = "The directory holding the cache of the metadata."
@@ -401,7 +401,7 @@
--------------------------------
You can use inline Python variable expansion to set variables. Here is
-an example: ::
+an example::
DATE = "${@time.strftime('%Y%m%d',time.gmtime())}"
@@ -410,7 +410,7 @@
Probably the most common use of this feature is to extract the value of
variables from BitBake's internal data dictionary, ``d``. The following
lines select the values of a package name and its version number,
-respectively: ::
+respectively::
PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
@@ -419,12 +419,12 @@
Inline Python expressions work just like variable expansions insofar as the
"=" and ":=" operators are concerned. Given the following assignment, foo()
- is called each time FOO is expanded: ::
+ is called each time FOO is expanded::
FOO = "${@foo()}"
Contrast this with the following immediate assignment, where foo() is only
- called once, while the assignment is parsed: ::
+ called once, while the assignment is parsed::
FOO := "${@foo()}"
@@ -437,7 +437,7 @@
It is possible to completely remove a variable or a variable flag from
BitBake's internal data dictionary by using the "unset" keyword. Here is
-an example: ::
+an example::
unset DATE
unset do_fetch[noexec]
@@ -452,7 +452,7 @@
cause BitBake to not recognize the path since BitBake does not expand
this character in the same way a shell would.
-Instead, provide a fuller path as the following example illustrates: ::
+Instead, provide a fuller path as the following example illustrates::
BBLAYERS ?= " \
/home/scott-lenovo/LayerA \
@@ -463,7 +463,7 @@
You can export variables to the environment of running tasks by using
the ``export`` keyword. For example, in the following example, the
-``do_foo`` task prints "value from the environment" when run: ::
+``do_foo`` task prints "value from the environment" when run::
export ENV_VARIABLE
ENV_VARIABLE = "value from the environment"
@@ -481,7 +481,7 @@
after assignments to ``ENV_VARIABLE``.
It is also possible to combine ``export`` with setting a value for the
-variable. Here is an example: ::
+variable. Here is an example::
export ENV_VARIABLE = "variable-value"
@@ -518,7 +518,7 @@
to satisfy conditions. Thus, if you have a variable that is
conditional on "arm", and "arm" is in ``OVERRIDES``, then the
"arm"-specific version of the variable is used rather than the
- non-conditional version. Here is an example: ::
+ non-conditional version. Here is an example::
OVERRIDES = "architecture:os:machine"
TEST = "default"
@@ -535,7 +535,7 @@
an OpenEmbedded metadata-based Linux kernel recipe file. The
following lines from the recipe file first set the kernel branch
variable ``KBRANCH`` to a default value, then conditionally override
- that value based on the architecture of the build: ::
+ that value based on the architecture of the build::
KBRANCH = "standard/base"
KBRANCH_qemuarm = "standard/arm-versatile-926ejs"
@@ -547,7 +547,7 @@
- *Appending and Prepending:* BitBake also supports append and prepend
operations to variable values based on whether a specific item is
- listed in ``OVERRIDES``. Here is an example: ::
+ listed in ``OVERRIDES``. Here is an example::
DEPENDS = "glibc ncurses"
OVERRIDES = "machine:local"
@@ -557,14 +557,14 @@
Again, using an OpenEmbedded metadata-based kernel recipe file as an
example, the following lines will conditionally append to the
- ``KERNEL_FEATURES`` variable based on the architecture: ::
+ ``KERNEL_FEATURES`` variable based on the architecture::
KERNEL_FEATURES_append = " ${KERNEL_EXTRA_FEATURES}"
KERNEL_FEATURES_append_qemux86=" cfg/sound.scc cfg/paravirt_kvm.scc"
KERNEL_FEATURES_append_qemux86-64=" cfg/sound.scc cfg/paravirt_kvm.scc"
- *Setting a Variable for a Single Task:* BitBake supports setting a
- variable just for the duration of a single task. Here is an example: ::
+ variable just for the duration of a single task. Here is an example::
FOO_task-configure = "val 1"
FOO_task-compile = "val 2"
@@ -580,7 +580,7 @@
``do_compile`` task.
You can also use this syntax with other combinations (e.g.
- "``_prepend``") as shown in the following example: ::
+ "``_prepend``") as shown in the following example::
EXTRA_OEMAKE_prepend_task-compile = "${PARALLEL_MAKE} "
@@ -588,7 +588,7 @@
-------------
Key expansion happens when the BitBake datastore is finalized. To better
-understand this, consider the following example: ::
+understand this, consider the following example::
A${B} = "X"
B = "2"
@@ -614,7 +614,7 @@
various "append" operators take effect. Recall that an append or prepend
operation using "_append" and "_prepend" does not result in an immediate
assignment as would "+=", ".=", "=+", or "=.". Consider the following
-example: ::
+example::
OVERRIDES = "foo"
A = "Z"
@@ -631,7 +631,7 @@
version, which is equal to "X". So effectively, ``A_foo`` replaces
``A``.
-This next example changes the order of the override and the append: ::
+This next example changes the order of the override and the append::
OVERRIDES = "foo"
A = "Z"
@@ -644,7 +644,7 @@
are not appended.
This next example has the order of the appends and overrides reversed
-back as in the first example: ::
+back as in the first example::
OVERRIDES = "foo"
A = "Y"
@@ -658,7 +658,7 @@
"foo" results in the conditional variable ``A`` becoming "ZX" (i.e.
``A`` is replaced with ``A_foo``).
-This final example mixes in some varying operators: ::
+This final example mixes in some varying operators::
A = "1"
A_append = "2"
@@ -720,7 +720,7 @@
As an example, your recipes could use the following directive to inherit
an ``autotools.bbclass`` file. The class file would contain common
-functionality for using Autotools that could be shared across recipes: ::
+functionality for using Autotools that could be shared across recipes::
inherit autotools
@@ -734,7 +734,7 @@
If you want to use the directive to inherit multiple classes, separate
them with spaces. The following example shows how to inherit both the
-``buildhistory`` and ``rm_work`` classes: ::
+``buildhistory`` and ``rm_work`` classes::
inherit buildhistory rm_work
@@ -742,19 +742,19 @@
:ref:`include <bitbake-user-manual/bitbake-user-manual-metadata:\`\`include\`\` directive>` and :ref:`require <bitbake-user-manual/bitbake-user-manual-metadata:\`\`require\`\` directive>`
directives is that you can inherit class files conditionally. You can
accomplish this by using a variable expression after the ``inherit``
-statement. Here is an example: ::
+statement. Here is an example::
inherit ${VARNAME}
If ``VARNAME`` is
going to be set, it needs to be set before the ``inherit`` statement is
parsed. One way to achieve a conditional inherit in this case is to use
-overrides: ::
+overrides::
VARIABLE = ""
VARIABLE_someoverride = "myclass"
-Another method is by using anonymous Python. Here is an example: ::
+Another method is by using anonymous Python. Here is an example::
python () {
if condition == value:
@@ -764,7 +764,7 @@
}
Alternatively, you could use an in-line Python expression in the
-following form: ::
+following form::
inherit ${@'classname' if condition else ''}
inherit ${@functionname(params)}
@@ -790,7 +790,7 @@
``.bbclass`` file.
As an example, suppose you needed a recipe to include some self-test
-definitions: ::
+definitions::
include test_defs.inc
@@ -831,7 +831,7 @@
needed to build "foo". You need to be sure ``foo.inc`` is located in the
same directory as your two recipe files as well. Once these conditions
are set up, you can share the functionality using a ``require``
-directive from within each recipe: ::
+directive from within each recipe::
require foo.inc
@@ -844,7 +844,7 @@
configuration file.
As an example, suppose you needed to inherit a class file called
-``abc.bbclass`` from a configuration file as follows: ::
+``abc.bbclass`` from a configuration file as follows::
INHERIT += "abc"
@@ -862,7 +862,7 @@
If you want to use the directive to inherit multiple classes, you can
provide them on the same line in the ``local.conf`` file. Use spaces to
separate the classes. The following example shows how to inherit both
-the ``autotools`` and ``pkgconfig`` classes: ::
+the ``autotools`` and ``pkgconfig`` classes::
INHERIT += "autotools pkgconfig"
@@ -895,7 +895,7 @@
Functions written in shell script and executed either directly as
functions, tasks, or both. They can also be called by other shell
-functions. Here is an example shell function definition: ::
+functions. Here is an example shell function definition::
some_function () {
echo "Hello World"
@@ -912,7 +912,7 @@
would be used in a ``.bbappend`` file to modify functions in the main
recipe. It can also be used to modify functions inherited from classes.
-As an example, consider the following: ::
+As an example, consider the following::
do_foo() {
bbplain first
@@ -931,7 +931,7 @@
bbplain fourth
}
-Running ``do_foo`` prints the following: ::
+Running ``do_foo`` prints the following::
recipename do_foo: first
recipename do_foo: second
@@ -952,7 +952,7 @@
These functions are written in Python and executed by BitBake or other
Python functions using ``bb.build.exec_func()``.
-An example BitBake function is: ::
+An example BitBake function is::
python some_python_function () {
d.setVar("TEXT", "Hello World")
@@ -975,7 +975,7 @@
Similar to shell functions, you can also apply overrides and
override-style operators to BitBake-style Python functions.
-As an example, consider the following: ::
+As an example, consider the following::
python do_foo_prepend() {
bb.plain("first")
@@ -989,7 +989,7 @@
bb.plain("third")
}
-Running ``do_foo`` prints the following: ::
+Running ``do_foo`` prints the following::
recipename do_foo: first
recipename do_foo: second
@@ -1004,7 +1004,7 @@
These functions are written in Python and are executed by other Python
code. Examples of Python functions are utility functions that you intend
to call from in-line Python or from within other Python functions. Here
-is an example: ::
+is an example::
def get_depends(d):
if d.getVar('SOMECONDITION'):
@@ -1056,7 +1056,7 @@
- Regular Python functions are called with the usual Python syntax.
BitBake-style Python functions are usually tasks and are called
directly by BitBake, but can also be called manually from Python code
- by using the ``bb.build.exec_func()`` function. Here is an example: ::
+ by using the ``bb.build.exec_func()`` function. Here is an example::
bb.build.exec_func("my_bitbake_style_function", d)
@@ -1094,7 +1094,7 @@
programmatically during parsing. To do this, you can define special
Python functions, called anonymous Python functions, that run at the end
of parsing. For example, the following conditionally sets a variable
-based on the value of another variable: ::
+based on the value of another variable::
python () {
if d.getVar('SOMEVAR') == 'value':
@@ -1107,7 +1107,7 @@
Anonymous Python functions always run at the end of parsing, regardless
of where they are defined. If a recipe contains many anonymous
functions, they run in the same order as they are defined within the
-recipe. As an example, consider the following snippet: ::
+recipe. As an example, consider the following snippet::
python () {
d.setVar('FOO', 'foo 2')
@@ -1122,7 +1122,7 @@
BAR = "bar 1"
The previous example is conceptually
-equivalent to the following snippet: ::
+equivalent to the following snippet::
FOO = "foo 1"
BAR = "bar 1"
@@ -1136,7 +1136,7 @@
Overrides and override-style operators such as "``_append``" are applied
before anonymous functions run. In the following example, ``FOO`` ends
-up with the value "foo from anonymous": ::
+up with the value "foo from anonymous"::
FOO = "foo"
FOO_append = " from outside"
@@ -1173,24 +1173,24 @@
To make use of this technique, you need the following things in place:
-- The class needs to define the function as follows: ::
+- The class needs to define the function as follows::
classname_functionname
For example, if you have a class file
``bar.bbclass`` and a function named ``do_foo``, the class must
- define the function as follows: ::
+ define the function as follows::
bar_do_foo
- The class needs to contain the ``EXPORT_FUNCTIONS`` statement as
- follows: ::
+ follows::
EXPORT_FUNCTIONS functionname
For example, continuing with
the same example, the statement in the ``bar.bbclass`` would be as
- follows: ::
+ follows::
EXPORT_FUNCTIONS do_foo
@@ -1199,7 +1199,7 @@
class version of the function, it should call ``bar_do_foo``.
Assuming ``do_foo`` was a shell function and ``EXPORT_FUNCTIONS`` was
used as above, the recipe's function could conditionally call the
- class version of the function as follows: ::
+ class version of the function as follows::
do_foo() {
if [ somecondition ] ; then
@@ -1233,7 +1233,7 @@
that have been promoted to tasks by using the ``addtask`` command. The
``addtask`` command can also optionally describe dependencies between
the task and other tasks. Here is an example that shows how to define a
-task and declare some dependencies: ::
+task and declare some dependencies::
python do_printdate () {
import time
@@ -1264,12 +1264,12 @@
rerun for experimentation purposes, you can make BitBake always
consider the task "out-of-date" by using the
:ref:`[nostamp] <bitbake-user-manual/bitbake-user-manual-metadata:Variable Flags>`
- variable flag, as follows: ::
+ variable flag, as follows::
do_printdate[nostamp] = "1"
You can also explicitly run the task and provide the
- -f option as follows: ::
+ -f option as follows::
$ bitbake recipe -c printdate -f
@@ -1278,7 +1278,7 @@
name.
You might wonder about the practical effects of using ``addtask``
-without specifying any dependencies as is done in the following example: ::
+without specifying any dependencies as is done in the following example::
addtask printdate
@@ -1286,7 +1286,7 @@
added through some other means, the only way to run the task is by
explicitly selecting it with ``bitbake`` recipe ``-c printdate``. You
can use the ``do_listtasks`` task to list all tasks defined in a recipe
-as shown in the following example: ::
+as shown in the following example::
$ bitbake recipe -c listtasks
@@ -1312,7 +1312,7 @@
As well as being able to add tasks, you can delete them. Simply use the
``deltask`` command to delete a task. For example, to delete the example
-task used in the previous sections, you would use: ::
+task used in the previous sections, you would use::
deltask printdate
@@ -1328,7 +1328,7 @@
If you want dependencies such as these to remain intact, use the
``[noexec]`` varflag to disable the task instead of using the
-``deltask`` command to delete it: ::
+``deltask`` command to delete it::
do_b[noexec] = "1"
@@ -1356,7 +1356,7 @@
example, assume you want to prevent the build system from accessing
your ``$HOME/.ccache`` directory. The following command "whitelists"
the environment variable ``CCACHE_DIR`` causing BitBake to allow that
- variable into the datastore: ::
+ variable into the datastore::
export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE CCACHE_DIR"
@@ -1366,7 +1366,7 @@
available in the datastore. To export it to the task environment of
every running task, use a command similar to the following in your
local configuration file ``local.conf`` or your distribution
- configuration file: ::
+ configuration file::
export CCACHE_DIR
@@ -1385,7 +1385,7 @@
The ``BB_ORIGENV`` variable returns a datastore object that can be
queried using the standard datastore operators such as
``getVar(, False)``. The datastore object is useful, for example, to
-find the original ``DISPLAY`` variable. Here is an example: ::
+find the original ``DISPLAY`` variable. Here is an example::
origenv = d.getVar("BB_ORIGENV", False)
bar = origenv.getVar("BAR", False)
@@ -1398,7 +1398,7 @@
Variable flags (varflags) help control a task's functionality and
dependencies. BitBake reads and writes varflags to the datastore using
-the following command forms: ::
+the following command forms::
variable = d.getVarFlags("variable")
self.d.setVarFlags("FOO", {"func": True})
@@ -1537,7 +1537,7 @@
failures.
Following is an example event handler that prints the name of the event
-and the content of the ``FILE`` variable: ::
+and the content of the ``FILE`` variable::
addhandler myclass_eventhandler
python myclass_eventhandler() {
@@ -1676,7 +1676,7 @@
also specify conditional metadata (using the
:term:`OVERRIDES` mechanism) for a single
version, or an optionally named range of versions. Here is an
- example: ::
+ example::
BBVERSIONS = "1.0 2.0 git"
SRC_URI_git = "git://someurl/somepath.git"
@@ -1719,7 +1719,7 @@
BitBake uses the ``addtask`` directive to manage dependencies that are
internal to a given recipe file. You can use the ``addtask`` directive
to indicate when a task is dependent on other tasks or when other tasks
-depend on that recipe. Here is an example: ::
+depend on that recipe. Here is an example::
addtask printdate after do_fetch before do_build
@@ -1743,7 +1743,7 @@
- The directive ``addtask mytask after do_configure`` by itself
never causes ``do_mytask`` to run. ``do_mytask`` can still be run
- manually as follows: ::
+ manually as follows::
$ bitbake recipe -c mytask
@@ -1757,7 +1757,7 @@
BitBake uses the :term:`DEPENDS` variable to manage
build time dependencies. The ``[deptask]`` varflag for tasks signifies
the task of each item listed in ``DEPENDS`` that must complete before
-that task can be executed. Here is an example: ::
+that task can be executed. Here is an example::
do_configure[deptask] = "do_populate_sysroot"
@@ -1799,7 +1799,7 @@
The ``[recrdeptask]`` flag is most commonly used in high-level recipes
that need to wait for some task to finish "globally". For example,
-``image.bbclass`` has the following: ::
+``image.bbclass`` has the following::
do_rootfs[recrdeptask] += "do_packagedata"
@@ -1808,7 +1808,7 @@
from the image recipe must run before the ``do_rootfs`` task can run.
BitBake allows a task to recursively depend on itself by
-referencing itself in the task list: ::
+referencing itself in the task list::
do_a[recrdeptask] = "do_a do_b"
@@ -1825,7 +1825,7 @@
BitBake uses the ``[depends]`` flag in a more generic form to manage
inter-task dependencies. This more generic form allows for
inter-dependency checks for specific tasks rather than checks for the
-data in ``DEPENDS``. Here is an example: ::
+data in ``DEPENDS``. Here is an example::
do_patch[depends] = "quilt-native:do_populate_sysroot"
@@ -1920,7 +1920,7 @@
OpenEmbedded metadata-based example.
These checksums are stored in :term:`STAMP`. You can
-examine the checksums using the following BitBake command: ::
+examine the checksums using the following BitBake command::
$ bitbake-dumpsigs
diff --git a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst
index 489fa15..e1b640e 100644
--- a/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst
+++ b/poky/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst
@@ -130,7 +130,7 @@
you to control the build based on these parameters.
Disk space monitoring is disabled by default. When setting this
- variable, use the following form: ::
+ variable, use the following form::
BB_DISKMON_DIRS = "<action>,<dir>,<threshold> [...]"
@@ -166,7 +166,7 @@
not specify G, M, or K, Kbytes is assumed by
default. Do not use GB, MB, or KB.
- Here are some examples: ::
+ Here are some examples::
BB_DISKMON_DIRS = "ABORT,${TMPDIR},1G,100K WARN,${SSTATE_DIR},1G,100K"
BB_DISKMON_DIRS = "STOPTASKS,${TMPDIR},1G"
@@ -207,7 +207,7 @@
BB_DISKMON_WARNINTERVAL = "50M,5K"
When specifying the variable in your configuration file, use the
- following form: ::
+ following form::
BB_DISKMON_WARNINTERVAL = "<disk_space_interval>,<disk_inode_interval>"
@@ -223,7 +223,7 @@
G, M, or K for Gbytes, Mbytes, or Kbytes,
respectively. You cannot use GB, MB, or KB.
- Here is an example: ::
+ Here is an example::
BB_DISKMON_DIRS = "WARN,${SSTATE_DIR},1G,100K"
BB_DISKMON_WARNINTERVAL = "50M,5K"
@@ -329,7 +329,7 @@
Specifies the name of the log files saved into
``${``\ :term:`T`\ ``}``. By default, the ``BB_LOGFMT``
variable is undefined and the log file names get created using the
- following form: ::
+ following form::
log.{task}.{pid}
@@ -383,7 +383,7 @@
Specifies the name of the executable script files (i.e. run files)
saved into ``${``\ :term:`T`\ ``}``. By default, the
``BB_RUNFMT`` variable is undefined and the run file names get
- created using the following form: ::
+ created using the following form::
run.{task}.{pid}
@@ -511,7 +511,7 @@
This variable works similarly to the :term:`BB_TASK_NICE_LEVEL`
variable except with a task's I/O priorities.
- Set the variable as follows: ::
+ Set the variable as follows::
BB_TASK_IONICE_LEVEL = "class.prio"
@@ -529,7 +529,7 @@
In order for your I/O priority settings to take effect, you need the
Completely Fair Queuing (CFQ) Scheduler selected for the backing block
device. To select the scheduler, use the following command form where
- device is the device (e.g. sda, sdb, and so forth): ::
+ device is the device (e.g. sda, sdb, and so forth)::
$ sudo sh -c "echo cfq > /sys/block/device/queu/scheduler"
@@ -570,7 +570,7 @@
To build a different variant of the recipe with a minimal amount of
code, it usually is as simple as adding the variable to your recipe.
Here are two examples. The "native" variants are from the
- OpenEmbedded-Core metadata: ::
+ OpenEmbedded-Core metadata::
BBCLASSEXTEND =+ "native nativesdk"
BBCLASSEXTEND =+ "multilib:multilib_name"
@@ -658,12 +658,12 @@
``.bb`` files in case a layer is not present. Use this avoid hard
dependency on those other layers.
- Use the following form for ``BBFILES_DYNAMIC``: ::
+ Use the following form for ``BBFILES_DYNAMIC``::
collection_name:filename_pattern
The following example identifies two collection names and two filename
- patterns: ::
+ patterns::
BBFILES_DYNAMIC += "\
clang-layer:${LAYERDIR}/bbappends/meta-clang/*/*/*.bbappend \
@@ -671,14 +671,14 @@
"
When the collection name is prefixed with "!" it will add the file pattern in case
- the layer is absent: ::
+ the layer is absent::
BBFILES_DYNAMIC += "\
!clang-layer:${LAYERDIR}/backfill/meta-clang/*/*/*.bb \
"
This next example shows an error message that occurs because invalid
- entries are found, which cause parsing to abort: ::
+ entries are found, which cause parsing to abort::
ERROR: BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:
/work/my-layer/bbappends/meta-security-isafw/*/*/*.bbappend
@@ -701,7 +701,7 @@
:term:`BBLAYERS`
Lists the layers to enable during the build. This variable is defined
in the ``bblayers.conf`` configuration file in the build directory.
- Here is an example: ::
+ Here is an example::
BBLAYERS = " \
/home/scottrif/poky/meta \
@@ -735,13 +735,13 @@
The following example uses a complete regular expression to tell
BitBake to ignore all recipe and recipe append files in the
- ``meta-ti/recipes-misc/`` directory: ::
+ ``meta-ti/recipes-misc/`` directory::
BBMASK = "meta-ti/recipes-misc/"
If you want to mask out multiple directories or recipes, you can
specify multiple regular expression fragments. This next example
- masks out multiple directories and individual recipes: ::
+ masks out multiple directories and individual recipes::
BBMASK += "/meta-ti/recipes-misc/ meta-ti/recipes-ti/packagegroup/"
BBMASK += "/meta-oe/recipes-support/"
@@ -762,7 +762,7 @@
``conf/local.conf`` configuration file.
As an example, the following line specifies three multiconfigs, each
- having a separate configuration file: ::
+ having a separate configuration file::
BBMULTIFONFIG = "configA configB configC"
@@ -783,7 +783,7 @@
If you run BitBake from a directory outside of the build directory,
you must be sure to set ``BBPATH`` to point to the build directory.
Set the variable as you would any environment variable and then run
- BitBake: ::
+ BitBake::
$ BBPATH="build_directory"
$ export BBPATH
@@ -852,7 +852,7 @@
Consider this simple example for two recipes named "a" and "b" that
produce similarly named packages. In this example, the ``DEPENDS``
- statement appears in the "a" recipe: ::
+ statement appears in the "a" recipe::
DEPENDS = "b"
@@ -1074,7 +1074,7 @@
recipes provide the same item. You should always suffix the variable
with the name of the provided item, and you should set it to the
:term:`PN` of the recipe to which you want to give
- precedence. Some examples: ::
+ precedence. Some examples::
PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto"
PREFERRED_PROVIDER_virtual/xserver = "xserver-xf86"
@@ -1086,11 +1086,11 @@
``PREFERRED_PROVIDERS`` is identical to
:term:`PREFERRED_PROVIDER`. However, the ``PREFERRED_PROVIDERS`` variable
lets you define preferences for multiple situations using the following
- form: ::
+ form::
PREFERRED_PROVIDERS = "xxx:yyy aaa:bbb ..."
- This form is a convenient replacement for the following: ::
+ This form is a convenient replacement for the following::
PREFERRED_PROVIDER_xxx = "yyy"
PREFERRED_PROVIDER_aaa = "bbb"
@@ -1106,7 +1106,7 @@
through the "``%``" character. You can use the character to match any
number of characters, which can be useful when specifying versions
that contain long revision numbers that potentially change. Here are
- two examples: ::
+ two examples::
PREFERRED_VERSION_python = "2.7.3"
PREFERRED_VERSION_linux-yocto = "4.12%"
@@ -1130,7 +1130,7 @@
Typically, you would add a specific server for the build system to
attempt before any others by adding something like the following to
- your configuration: ::
+ your configuration::
PREMIRRORS_prepend = "\
git://.*/.* http://www.yoctoproject.org/sources/ \n \
@@ -1152,7 +1152,7 @@
``DEPENDS``.
Consider the following example ``PROVIDES`` statement from a recipe
- file ``libav_0.8.11.bb``: ::
+ file ``libav_0.8.11.bb``::
PROVIDES += "libpostproc"
@@ -1175,7 +1175,7 @@
:term:`PRSERV_HOST`
The network based :term:`PR` service host and port.
- Following is an example of how the ``PRSERV_HOST`` variable is set: ::
+ Following is an example of how the ``PRSERV_HOST`` variable is set::
PRSERV_HOST = "localhost:0"
@@ -1196,7 +1196,7 @@
you should always use the variable in a form with an attached package
name. For example, suppose you are building a development package
that depends on the ``perl`` package. In this case, you would use the
- following ``RDEPENDS`` statement: ::
+ following ``RDEPENDS`` statement::
RDEPENDS_${PN}-dev += "perl"
@@ -1207,11 +1207,11 @@
BitBake supports specifying versioned dependencies. Although the
syntax varies depending on the packaging format, BitBake hides these
differences from you. Here is the general syntax to specify versions
- with the ``RDEPENDS`` variable: ::
+ with the ``RDEPENDS`` variable::
RDEPENDS_${PN} = "package (operator version)"
- For ``operator``, you can specify the following: ::
+ For ``operator``, you can specify the following::
=
<
@@ -1220,7 +1220,7 @@
>=
For example, the following sets up a dependency on version 1.2 or
- greater of the package ``foo``: ::
+ greater of the package ``foo``::
RDEPENDS_${PN} = "foo (>= 1.2)"
@@ -1249,7 +1249,7 @@
As with all package-controlling variables, you must always use the
variable in conjunction with a package name override. Here is an
- example: ::
+ example::
RPROVIDES_${PN} = "widget-abi-2"
@@ -1263,11 +1263,11 @@
BitBake supports specifying versioned recommends. Although the syntax
varies depending on the packaging format, BitBake hides these
differences from you. Here is the general syntax to specify versions
- with the ``RRECOMMENDS`` variable: ::
+ with the ``RRECOMMENDS`` variable::
RRECOMMENDS_${PN} = "package (operator version)"
- For ``operator``, you can specify the following: ::
+ For ``operator``, you can specify the following::
=
<
@@ -1276,7 +1276,7 @@
>=
For example, the following sets up a recommend on version
- 1.2 or greater of the package ``foo``: ::
+ 1.2 or greater of the package ``foo``::
RRECOMMENDS_${PN} = "foo (>= 1.2)"
diff --git a/poky/bitbake/lib/bb/__init__.py b/poky/bitbake/lib/bb/__init__.py
index 9f61dae..a144bd6 100644
--- a/poky/bitbake/lib/bb/__init__.py
+++ b/poky/bitbake/lib/bb/__init__.py
@@ -9,7 +9,7 @@
# SPDX-License-Identifier: GPL-2.0-only
#
-__version__ = "1.50.0"
+__version__ = "1.51.0"
import sys
if sys.version_info < (3, 5, 0):
diff --git a/poky/bitbake/lib/bb/asyncrpc/__init__.py b/poky/bitbake/lib/bb/asyncrpc/__init__.py
new file mode 100644
index 0000000..b2bec31
--- /dev/null
+++ b/poky/bitbake/lib/bb/asyncrpc/__init__.py
@@ -0,0 +1,31 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import itertools
+import json
+
+# The Python async server defaults to a 64K receive buffer, so we hardcode our
+# maximum chunk size. It would be better if the client and server reported to
+# each other what the maximum chunk sizes were, but that will slow down the
+# connection setup with a round trip delay so I'd rather not do that unless it
+# is necessary
+DEFAULT_MAX_CHUNK = 32 * 1024
+
+
+def chunkify(msg, max_chunk):
+ if len(msg) < max_chunk - 1:
+ yield ''.join((msg, "\n"))
+ else:
+ yield ''.join((json.dumps({
+ 'chunk-stream': None
+ }), "\n"))
+
+ args = [iter(msg)] * (max_chunk - 1)
+ for m in map(''.join, itertools.zip_longest(*args, fillvalue='')):
+ yield ''.join(itertools.chain(m, "\n"))
+ yield "\n"
+
+
+from .client import AsyncClient, Client
+from .serv import AsyncServer, AsyncServerConnection
diff --git a/poky/bitbake/lib/bb/asyncrpc/client.py b/poky/bitbake/lib/bb/asyncrpc/client.py
new file mode 100644
index 0000000..4cdad9a
--- /dev/null
+++ b/poky/bitbake/lib/bb/asyncrpc/client.py
@@ -0,0 +1,145 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import abc
+import asyncio
+import json
+import os
+import socket
+from . import chunkify, DEFAULT_MAX_CHUNK
+
+
+class AsyncClient(object):
+ def __init__(self, proto_name, proto_version, logger):
+ self.reader = None
+ self.writer = None
+ self.max_chunk = DEFAULT_MAX_CHUNK
+ self.proto_name = proto_name
+ self.proto_version = proto_version
+ self.logger = logger
+
+ async def connect_tcp(self, address, port):
+ async def connect_sock():
+ return await asyncio.open_connection(address, port)
+
+ self._connect_sock = connect_sock
+
+ async def connect_unix(self, path):
+ async def connect_sock():
+ return await asyncio.open_unix_connection(path)
+
+ self._connect_sock = connect_sock
+
+ async def setup_connection(self):
+ s = '%s %s\n\n' % (self.proto_name, self.proto_version)
+ self.writer.write(s.encode("utf-8"))
+ await self.writer.drain()
+
+ async def connect(self):
+ if self.reader is None or self.writer is None:
+ (self.reader, self.writer) = await self._connect_sock()
+ await self.setup_connection()
+
+ async def close(self):
+ self.reader = None
+
+ if self.writer is not None:
+ self.writer.close()
+ self.writer = None
+
+ async def _send_wrapper(self, proc):
+ count = 0
+ while True:
+ try:
+ await self.connect()
+ return await proc()
+ except (
+ OSError,
+ ConnectionError,
+ json.JSONDecodeError,
+ UnicodeDecodeError,
+ ) as e:
+ self.logger.warning("Error talking to server: %s" % e)
+ if count >= 3:
+ if not isinstance(e, ConnectionError):
+ raise ConnectionError(str(e))
+ raise e
+ await self.close()
+ count += 1
+
+ async def send_message(self, msg):
+ async def get_line():
+ line = await self.reader.readline()
+ if not line:
+ raise ConnectionError("Connection closed")
+
+ line = line.decode("utf-8")
+
+ if not line.endswith("\n"):
+ raise ConnectionError("Bad message %r" % msg)
+
+ return line
+
+ async def proc():
+ for c in chunkify(json.dumps(msg), self.max_chunk):
+ self.writer.write(c.encode("utf-8"))
+ await self.writer.drain()
+
+ l = await get_line()
+
+ m = json.loads(l)
+ if m and "chunk-stream" in m:
+ lines = []
+ while True:
+ l = (await get_line()).rstrip("\n")
+ if not l:
+ break
+ lines.append(l)
+
+ m = json.loads("".join(lines))
+
+ return m
+
+ return await self._send_wrapper(proc)
+
+
+class Client(object):
+ def __init__(self):
+ self.client = self._get_async_client()
+ self.loop = asyncio.new_event_loop()
+
+ self._add_methods('connect_tcp', 'close')
+
+ @abc.abstractmethod
+ def _get_async_client(self):
+ pass
+
+ def _get_downcall_wrapper(self, downcall):
+ def wrapper(*args, **kwargs):
+ return self.loop.run_until_complete(downcall(*args, **kwargs))
+
+ return wrapper
+
+ def _add_methods(self, *methods):
+ for m in methods:
+ downcall = getattr(self.client, m)
+ setattr(self, m, self._get_downcall_wrapper(downcall))
+
+ def connect_unix(self, path):
+ # AF_UNIX has path length issues so chdir here to workaround
+ cwd = os.getcwd()
+ try:
+ os.chdir(os.path.dirname(path))
+ self.loop.run_until_complete(self.client.connect_unix(os.path.basename(path)))
+ self.loop.run_until_complete(self.client.connect())
+ finally:
+ os.chdir(cwd)
+
+ @property
+ def max_chunk(self):
+ return self.client.max_chunk
+
+ @max_chunk.setter
+ def max_chunk(self, value):
+ self.client.max_chunk = value
diff --git a/poky/bitbake/lib/bb/asyncrpc/serv.py b/poky/bitbake/lib/bb/asyncrpc/serv.py
new file mode 100644
index 0000000..cb33846
--- /dev/null
+++ b/poky/bitbake/lib/bb/asyncrpc/serv.py
@@ -0,0 +1,218 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import abc
+import asyncio
+import json
+import os
+import signal
+import socket
+import sys
+from . import chunkify, DEFAULT_MAX_CHUNK
+
+
+class ClientError(Exception):
+ pass
+
+
+class ServerError(Exception):
+ pass
+
+
+class AsyncServerConnection(object):
+ def __init__(self, reader, writer, proto_name, logger):
+ self.reader = reader
+ self.writer = writer
+ self.proto_name = proto_name
+ self.max_chunk = DEFAULT_MAX_CHUNK
+ self.handlers = {
+ 'chunk-stream': self.handle_chunk,
+ }
+ self.logger = logger
+
+ async def process_requests(self):
+ try:
+ self.addr = self.writer.get_extra_info('peername')
+ self.logger.debug('Client %r connected' % (self.addr,))
+
+ # Read protocol and version
+ client_protocol = await self.reader.readline()
+ if client_protocol is None:
+ return
+
+ (client_proto_name, client_proto_version) = client_protocol.decode('utf-8').rstrip().split()
+ if client_proto_name != self.proto_name:
+ self.logger.debug('Rejecting invalid protocol %s' % (self.proto_name))
+ return
+
+ self.proto_version = tuple(int(v) for v in client_proto_version.split('.'))
+ if not self.validate_proto_version():
+ self.logger.debug('Rejecting invalid protocol version %s' % (client_proto_version))
+ return
+
+ # Read headers. Currently, no headers are implemented, so look for
+ # an empty line to signal the end of the headers
+ while True:
+ line = await self.reader.readline()
+ if line is None:
+ return
+
+ line = line.decode('utf-8').rstrip()
+ if not line:
+ break
+
+ # Handle messages
+ while True:
+ d = await self.read_message()
+ if d is None:
+ break
+ await self.dispatch_message(d)
+ await self.writer.drain()
+ except ClientError as e:
+ self.logger.error(str(e))
+ finally:
+ self.writer.close()
+
+ async def dispatch_message(self, msg):
+ for k in self.handlers.keys():
+ if k in msg:
+ self.logger.debug('Handling %s' % k)
+ await self.handlers[k](msg[k])
+ return
+
+ raise ClientError("Unrecognized command %r" % msg)
+
+ def write_message(self, msg):
+ for c in chunkify(json.dumps(msg), self.max_chunk):
+ self.writer.write(c.encode('utf-8'))
+
+ async def read_message(self):
+ l = await self.reader.readline()
+ if not l:
+ return None
+
+ try:
+ message = l.decode('utf-8')
+
+ if not message.endswith('\n'):
+ return None
+
+ return json.loads(message)
+ except (json.JSONDecodeError, UnicodeDecodeError) as e:
+ self.logger.error('Bad message from client: %r' % message)
+ raise e
+
+ async def handle_chunk(self, request):
+ lines = []
+ try:
+ while True:
+ l = await self.reader.readline()
+ l = l.rstrip(b"\n").decode("utf-8")
+ if not l:
+ break
+ lines.append(l)
+
+ msg = json.loads(''.join(lines))
+ except (json.JSONDecodeError, UnicodeDecodeError) as e:
+ self.logger.error('Bad message from client: %r' % lines)
+ raise e
+
+ if 'chunk-stream' in msg:
+ raise ClientError("Nested chunks are not allowed")
+
+ await self.dispatch_message(msg)
+
+
+class AsyncServer(object):
+ def __init__(self, logger, loop=None):
+ if loop is None:
+ self.loop = asyncio.new_event_loop()
+ self.close_loop = True
+ else:
+ self.loop = loop
+ self.close_loop = False
+
+ self._cleanup_socket = None
+ self.logger = logger
+
+ def start_tcp_server(self, host, port):
+ self.server = self.loop.run_until_complete(
+ asyncio.start_server(self.handle_client, host, port, loop=self.loop)
+ )
+
+ for s in self.server.sockets:
+ self.logger.info('Listening on %r' % (s.getsockname(),))
+ # Newer python does this automatically. Do it manually here for
+ # maximum compatibility
+ s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
+ s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1)
+
+ name = self.server.sockets[0].getsockname()
+ if self.server.sockets[0].family == socket.AF_INET6:
+ self.address = "[%s]:%d" % (name[0], name[1])
+ else:
+ self.address = "%s:%d" % (name[0], name[1])
+
+ def start_unix_server(self, path):
+ def cleanup():
+ os.unlink(path)
+
+ cwd = os.getcwd()
+ try:
+ # Work around path length limits in AF_UNIX
+ os.chdir(os.path.dirname(path))
+ self.server = self.loop.run_until_complete(
+ asyncio.start_unix_server(self.handle_client, os.path.basename(path), loop=self.loop)
+ )
+ finally:
+ os.chdir(cwd)
+
+ self.logger.info('Listening on %r' % path)
+
+ self._cleanup_socket = cleanup
+ self.address = "unix://%s" % os.path.abspath(path)
+
+ @abc.abstractmethod
+ def accept_client(self, reader, writer):
+ pass
+
+ async def handle_client(self, reader, writer):
+ # writer.transport.set_write_buffer_limits(0)
+ try:
+ client = self.accept_client(reader, writer)
+ await client.process_requests()
+ except Exception as e:
+ import traceback
+ self.logger.error('Error from client: %s' % str(e), exc_info=True)
+ traceback.print_exc()
+ writer.close()
+ self.logger.info('Client disconnected')
+
+ def run_loop_forever(self):
+ try:
+ self.loop.run_forever()
+ except KeyboardInterrupt:
+ pass
+
+ def signal_handler(self):
+ self.loop.stop()
+
+ def serve_forever(self):
+ asyncio.set_event_loop(self.loop)
+ try:
+ self.loop.add_signal_handler(signal.SIGTERM, self.signal_handler)
+
+ self.run_loop_forever()
+ self.server.close()
+
+ self.loop.run_until_complete(self.server.wait_closed())
+ self.logger.info('Server shutting down')
+ finally:
+ if self.close_loop:
+ if sys.version_info >= (3, 6):
+ self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+ self.loop.close()
+
+ if self._cleanup_socket is not None:
+ self._cleanup_socket()
diff --git a/poky/bitbake/lib/bb/command.py b/poky/bitbake/lib/bb/command.py
index dd77cdd..f530cf8 100644
--- a/poky/bitbake/lib/bb/command.py
+++ b/poky/bitbake/lib/bb/command.py
@@ -20,6 +20,7 @@
from collections import OrderedDict, defaultdict
+import io
import bb.event
import bb.cooker
import bb.remotedata
@@ -500,6 +501,17 @@
d = command.remotedatastores[dsindex].varhistory
return getattr(d, method)(*args, **kwargs)
+ def dataStoreConnectorVarHistCmdEmit(self, command, params):
+ dsindex = params[0]
+ var = params[1]
+ oval = params[2]
+ val = params[3]
+ d = command.remotedatastores[params[4]]
+
+ o = io.StringIO()
+ command.remotedatastores[dsindex].varhistory.emit(var, oval, val, o, d)
+ return o.getvalue()
+
def dataStoreConnectorIncHistCmd(self, command, params):
dsindex = params[0]
method = params[1]
diff --git a/poky/bitbake/lib/bb/fetch2/git.py b/poky/bitbake/lib/bb/fetch2/git.py
index e3ba80a..5e65c83 100644
--- a/poky/bitbake/lib/bb/fetch2/git.py
+++ b/poky/bitbake/lib/bb/fetch2/git.py
@@ -168,7 +168,11 @@
if len(branches) != len(ud.names):
raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
- ud.cloneflags = "-s -n"
+ ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1"
+
+ ud.cloneflags = "-n"
+ if not ud.noshared:
+ ud.cloneflags += " -s"
if ud.bareclone:
ud.cloneflags += " --mirror"
@@ -394,7 +398,7 @@
tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
try:
# Do the checkout. This implicitly involves a Git LFS fetch.
- self.unpack(ud, tmpdir, d)
+ Git.unpack(self, ud, tmpdir, d)
# Scoop up a copy of any stuff that Git LFS downloaded. Merge them into
# the bare clonedir.
diff --git a/poky/bitbake/lib/bb/fetch2/s3.py b/poky/bitbake/lib/bb/fetch2/s3.py
index ffca73c..6b8ffd5 100644
--- a/poky/bitbake/lib/bb/fetch2/s3.py
+++ b/poky/bitbake/lib/bb/fetch2/s3.py
@@ -18,10 +18,47 @@
import os
import bb
import urllib.request, urllib.parse, urllib.error
+import re
from bb.fetch2 import FetchMethod
from bb.fetch2 import FetchError
from bb.fetch2 import runfetchcmd
+def convertToBytes(value, unit):
+ value = float(value)
+ if (unit == "KiB"):
+ value = value*1024.0;
+ elif (unit == "MiB"):
+ value = value*1024.0*1024.0;
+ elif (unit == "GiB"):
+ value = value*1024.0*1024.0*1024.0;
+ return value
+
+class S3ProgressHandler(bb.progress.LineFilterProgressHandler):
+ """
+ Extract progress information from s3 cp output, e.g.:
+ Completed 5.1 KiB/8.8 GiB (12.0 MiB/s) with 1 file(s) remaining
+ """
+ def __init__(self, d):
+ super(S3ProgressHandler, self).__init__(d)
+ # Send an initial progress event so the bar gets shown
+ self._fire_progress(0)
+
+ def writeline(self, line):
+ percs = re.findall(r'^Completed (\d+.{0,1}\d*) (\w+)\/(\d+.{0,1}\d*) (\w+) (\(.+\)) with\s+', line)
+ if percs:
+ completed = (percs[-1][0])
+ completedUnit = (percs[-1][1])
+ total = (percs[-1][2])
+ totalUnit = (percs[-1][3])
+ completed = convertToBytes(completed, completedUnit)
+ total = convertToBytes(total, totalUnit)
+ progress = (completed/total)*100.0
+ rate = percs[-1][4]
+ self.update(progress, rate)
+ return False
+ return True
+
+
class S3(FetchMethod):
"""Class to fetch urls via 'aws s3'"""
@@ -52,7 +89,9 @@
cmd = '%s cp s3://%s%s %s' % (ud.basecmd, ud.host, ud.path, ud.localpath)
bb.fetch2.check_network_access(d, cmd, ud.url)
- runfetchcmd(cmd, d)
+
+ progresshandler = S3ProgressHandler(d)
+ runfetchcmd(cmd, d, False, log=progresshandler)
# Additional sanity checks copied from the wget class (although there
# are no known issues which mean these are required, treat the aws cli
diff --git a/poky/bitbake/lib/bb/fetch2/svn.py b/poky/bitbake/lib/bb/fetch2/svn.py
index 8856ef1..80102b4 100644
--- a/poky/bitbake/lib/bb/fetch2/svn.py
+++ b/poky/bitbake/lib/bb/fetch2/svn.py
@@ -86,7 +86,7 @@
if command == "info":
svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
elif command == "log1":
- svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
+ svncmd = "%s log --limit 1 --quiet %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
else:
suffix = ""
diff --git a/poky/bitbake/lib/bb/progress.py b/poky/bitbake/lib/bb/progress.py
index d051ba0..52d704d 100644
--- a/poky/bitbake/lib/bb/progress.py
+++ b/poky/bitbake/lib/bb/progress.py
@@ -94,12 +94,15 @@
while True:
breakpos = self._linebuffer.find('\n') + 1
if breakpos == 0:
- break
+ # for the case when the line with progress ends with only '\r'
+ breakpos = self._linebuffer.find('\r') + 1
+ if breakpos == 0:
+ break
line = self._linebuffer[:breakpos]
self._linebuffer = self._linebuffer[breakpos:]
# Drop any line feeds and anything that precedes them
lbreakpos = line.rfind('\r') + 1
- if lbreakpos:
+ if lbreakpos and lbreakpos != breakpos:
line = line[lbreakpos:]
if self.writeline(filter_color(line)):
super().write(line)
diff --git a/poky/bitbake/lib/bb/runqueue.py b/poky/bitbake/lib/bb/runqueue.py
index cd56a55..6c41fe6 100644
--- a/poky/bitbake/lib/bb/runqueue.py
+++ b/poky/bitbake/lib/bb/runqueue.py
@@ -2030,8 +2030,6 @@
logger.debug("%s didn't become valid, skipping setscene" % nexttask)
self.sq_task_failoutright(nexttask)
return True
- else:
- self.sqdata.outrightfail.remove(nexttask)
if nexttask in self.sqdata.outrightfail:
logger.debug2('No package found, so skipping setscene task %s', nexttask)
self.sq_task_failoutright(nexttask)
@@ -2296,10 +2294,16 @@
self.updated_taskhash_queue.remove((tid, unihash))
if unihash != self.rqdata.runtaskentries[tid].unihash:
- hashequiv_logger.verbose("Task %s unihash changed to %s" % (tid, unihash))
- self.rqdata.runtaskentries[tid].unihash = unihash
- bb.parse.siggen.set_unihash(tid, unihash)
- toprocess.add(tid)
+ # Make sure we rehash any other tasks with the same task hash that we're deferred against.
+ torehash = [tid]
+ for deftid in self.sq_deferred:
+ if self.sq_deferred[deftid] == tid:
+ torehash.append(deftid)
+ for hashtid in torehash:
+ hashequiv_logger.verbose("Task %s unihash changed to %s" % (hashtid, unihash))
+ self.rqdata.runtaskentries[hashtid].unihash = unihash
+ bb.parse.siggen.set_unihash(hashtid, unihash)
+ toprocess.add(hashtid)
# Work out all tasks which depend upon these
total = set()
@@ -2827,6 +2831,8 @@
sqdata.stamppresent.remove(tid)
if tid in sqdata.valid:
sqdata.valid.remove(tid)
+ if tid in sqdata.outrightfail:
+ sqdata.outrightfail.remove(tid)
noexec, stamppresent = check_setscene_stamps(tid, rqdata, rq, stampcache, noexecstamp=True)
@@ -2845,6 +2851,7 @@
sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False, summary=summary)
sqdata.hashes = {}
+ sqrq.sq_deferred = {}
for mc in sorted(sqdata.multiconfigs):
for tid in sorted(sqdata.sq_revdeps):
if mc_from_tid(tid) != mc:
@@ -2857,10 +2864,13 @@
continue
if tid in sqrq.scenequeue_notcovered:
continue
- sqdata.outrightfail.add(tid)
+ if tid in sqrq.scenequeue_covered:
+ continue
h = pending_hash_index(tid, rqdata)
if h not in sqdata.hashes:
+ if tid in tids:
+ sqdata.outrightfail.add(tid)
sqdata.hashes[h] = tid
else:
sqrq.sq_deferred[tid] = sqdata.hashes[h]
diff --git a/poky/bitbake/lib/bb/server/process.py b/poky/bitbake/lib/bb/server/process.py
index b27b4ae..3e99bce 100644
--- a/poky/bitbake/lib/bb/server/process.py
+++ b/poky/bitbake/lib/bb/server/process.py
@@ -509,7 +509,7 @@
os.set_inheritable(self.bitbake_lock.fileno(), True)
os.set_inheritable(self.readypipein, True)
serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server")
- os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname, str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
+ os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname, str(self.server_timeout or 0), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface):
diff --git a/poky/bitbake/lib/bb/siggen.py b/poky/bitbake/lib/bb/siggen.py
index 0d88c6e..07692e6 100644
--- a/poky/bitbake/lib/bb/siggen.py
+++ b/poky/bitbake/lib/bb/siggen.py
@@ -402,7 +402,7 @@
p = pickle.dump(data, stream, -1)
stream.flush()
os.chmod(tmpfile, 0o664)
- os.rename(tmpfile, sigfile)
+ bb.utils.rename(tmpfile, sigfile)
except (OSError, IOError) as err:
try:
os.unlink(tmpfile)
@@ -542,7 +542,7 @@
hashequiv_logger.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server))
else:
hashequiv_logger.debug2('No reported unihash for %s:%s from %s' % (tid, taskhash, self.server))
- except hashserv.client.HashConnectionError as e:
+ except ConnectionError as e:
bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
self.set_unihash(tid, unihash)
@@ -621,7 +621,7 @@
d.setVar('BB_UNIHASH', new_unihash)
else:
hashequiv_logger.debug('Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
- except hashserv.client.HashConnectionError as e:
+ except ConnectionError as e:
bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
finally:
if sigfile:
@@ -661,7 +661,7 @@
# TODO: What to do here?
hashequiv_logger.verbose('Task %s unihash reported as unwanted hash %s' % (tid, finalunihash))
- except hashserv.client.HashConnectionError as e:
+ except ConnectionError as e:
bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
return False
diff --git a/poky/bitbake/lib/bb/tests/fetch.py b/poky/bitbake/lib/bb/tests/fetch.py
index ddf6e97..9291ce4 100644
--- a/poky/bitbake/lib/bb/tests/fetch.py
+++ b/poky/bitbake/lib/bb/tests/fetch.py
@@ -390,6 +390,7 @@
if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
print("Not cleaning up %s. Please remove manually." % self.tempdir)
else:
+ bb.process.run('chmod u+rw -R %s' % self.tempdir)
bb.utils.prunedir(self.tempdir)
class MirrorUriTest(FetcherTest):
@@ -673,12 +674,14 @@
with self.assertRaises(bb.fetch2.UnpackError):
self.fetchUnpack(['file://a;subdir=/bin/sh'])
- def test_local_gitfetch_usehead(self):
+ def dummyGitTest(self, suffix):
# Create dummy local Git repo
src_dir = tempfile.mkdtemp(dir=self.tempdir,
prefix='gitfetch_localusehead_')
src_dir = os.path.abspath(src_dir)
bb.process.run("git init", cwd=src_dir)
+ bb.process.run("git config user.email 'you@example.com'", cwd=src_dir)
+ bb.process.run("git config user.name 'Your Name'", cwd=src_dir)
bb.process.run("git commit --allow-empty -m'Dummy commit'",
cwd=src_dir)
# Use other branch than master
@@ -690,7 +693,7 @@
# Fetch and check revision
self.d.setVar("SRCREV", "AUTOINC")
- url = "git://" + src_dir + ";protocol=file;usehead=1"
+ url = "git://" + src_dir + ";protocol=file;" + suffix
fetcher = bb.fetch.Fetch([url], self.d)
fetcher.download()
fetcher.unpack(self.unpackdir)
@@ -699,31 +702,23 @@
unpack_rev = stdout[0].strip()
self.assertEqual(orig_rev, unpack_rev)
+ def test_local_gitfetch_usehead(self):
+ self.dummyGitTest("usehead=1")
+
def test_local_gitfetch_usehead_withname(self):
- # Create dummy local Git repo
- src_dir = tempfile.mkdtemp(dir=self.tempdir,
- prefix='gitfetch_localusehead_')
- src_dir = os.path.abspath(src_dir)
- bb.process.run("git init", cwd=src_dir)
- bb.process.run("git commit --allow-empty -m'Dummy commit'",
- cwd=src_dir)
- # Use other branch than master
- bb.process.run("git checkout -b my-devel", cwd=src_dir)
- bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
- cwd=src_dir)
- stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
- orig_rev = stdout[0].strip()
+ self.dummyGitTest("usehead=1;name=newName")
- # Fetch and check revision
- self.d.setVar("SRCREV", "AUTOINC")
- url = "git://" + src_dir + ";protocol=file;usehead=1;name=newName"
- fetcher = bb.fetch.Fetch([url], self.d)
- fetcher.download()
- fetcher.unpack(self.unpackdir)
- stdout = bb.process.run("git rev-parse HEAD",
- cwd=os.path.join(self.unpackdir, 'git'))
- unpack_rev = stdout[0].strip()
- self.assertEqual(orig_rev, unpack_rev)
+ def test_local_gitfetch_shared(self):
+ self.dummyGitTest("usehead=1;name=sharedName")
+ alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+ self.assertTrue(os.path.exists(alt))
+
+ def test_local_gitfetch_noshared(self):
+ self.d.setVar('BB_GIT_NOSHARED', '1')
+ self.unpackdir += '_noshared'
+ self.dummyGitTest("usehead=1;name=noSharedName")
+ alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+ self.assertFalse(os.path.exists(alt))
class FetcherNoNetworkTest(FetcherTest):
def setUp(self):
@@ -1390,6 +1385,8 @@
self.gitdir = os.path.join(self.tempdir, 'gitshallow')
bb.utils.mkdirhier(self.gitdir)
bb.process.run('git init', cwd=self.gitdir)
+ bb.process.run('git config user.email "you@example.com"', cwd=self.gitdir)
+ bb.process.run('git config user.name "Your Name"', cwd=self.gitdir)
def assertRefs(self, expected_refs):
actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
@@ -1513,6 +1510,8 @@
bb.utils.mkdirhier(self.srcdir)
self.git('init', cwd=self.srcdir)
+ self.git('config user.email "you@example.com"', cwd=self.srcdir)
+ self.git('config user.name "Your Name"', cwd=self.srcdir)
self.d.setVar('WORKDIR', self.tempdir)
self.d.setVar('S', self.gitdir)
self.d.delVar('PREMIRRORS')
@@ -1594,6 +1593,7 @@
# fetch and unpack, from the shallow tarball
bb.utils.remove(self.gitdir, recurse=True)
+ bb.process.run('chmod u+w -R "%s"' % ud.clonedir)
bb.utils.remove(ud.clonedir, recurse=True)
bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
@@ -1746,6 +1746,8 @@
smdir = os.path.join(self.tempdir, 'gitsubmodule')
bb.utils.mkdirhier(smdir)
self.git('init', cwd=smdir)
+ self.git('config user.email "you@example.com"', cwd=smdir)
+ self.git('config user.name "Your Name"', cwd=smdir)
# Make this look like it was cloned from a remote...
self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
@@ -1776,6 +1778,8 @@
smdir = os.path.join(self.tempdir, 'gitsubmodule')
bb.utils.mkdirhier(smdir)
self.git('init', cwd=smdir)
+ self.git('config user.email "you@example.com"', cwd=smdir)
+ self.git('config user.name "Your Name"', cwd=smdir)
# Make this look like it was cloned from a remote...
self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
@@ -1794,7 +1798,7 @@
# Set up the mirror
mirrordir = os.path.join(self.tempdir, 'mirror')
- os.rename(self.dldir, mirrordir)
+ bb.utils.rename(self.dldir, mirrordir)
self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/\n' % mirrordir)
# Fetch from the mirror
@@ -1818,8 +1822,8 @@
self.git('annex init', cwd=self.srcdir)
open(os.path.join(self.srcdir, 'c'), 'w').close()
self.git('annex add c', cwd=self.srcdir)
- self.git('commit -m annex-c -a', cwd=self.srcdir)
- bb.process.run('chmod u+w -R %s' % os.path.join(self.srcdir, '.git', 'annex'))
+ self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir)
+ bb.process.run('chmod u+w -R %s' % self.srcdir)
uri = 'gitannex://%s;protocol=file;subdir=${S}' % self.srcdir
fetcher, ud = self.fetch_shallow(uri)
@@ -1912,7 +1916,7 @@
bb.utils.mkdirhier(mirrordir)
self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir)
- os.rename(os.path.join(self.dldir, mirrortarball),
+ bb.utils.rename(os.path.join(self.dldir, mirrortarball),
os.path.join(mirrordir, mirrortarball))
# Fetch from the mirror
@@ -2094,6 +2098,8 @@
bb.utils.mkdirhier(self.srcdir)
self.git('init', cwd=self.srcdir)
+ self.git('config user.email "you@example.com"', cwd=self.srcdir)
+ self.git('config user.name "Your Name"', cwd=self.srcdir)
with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs:
attrs.write('*.mp3 filter=lfs -text')
self.git(['add', '.gitattributes'], cwd=self.srcdir)
@@ -2634,3 +2640,29 @@
fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
fetcher.download()
self.assertTrue(os.path.exists(ud.localpath))
+
+class GitSharedTest(FetcherTest):
+ def setUp(self):
+ super(GitSharedTest, self).setUp()
+ self.recipe_url = "git://git.openembedded.org/bitbake"
+ self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
+
+ @skipIfNoNetwork()
+ def test_shared_unpack(self):
+ fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+ fetcher.download()
+ fetcher.unpack(self.unpackdir)
+ alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+ self.assertTrue(os.path.exists(alt))
+
+ @skipIfNoNetwork()
+ def test_noshared_unpack(self):
+ self.d.setVar('BB_GIT_NOSHARED', '1')
+ self.unpackdir += '_noshared'
+ fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+ fetcher.download()
+ fetcher.unpack(self.unpackdir)
+ alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+ self.assertFalse(os.path.exists(alt))
diff --git a/poky/bitbake/lib/bb/tinfoil.py b/poky/bitbake/lib/bb/tinfoil.py
index 796a98f..27a3415 100644
--- a/poky/bitbake/lib/bb/tinfoil.py
+++ b/poky/bitbake/lib/bb/tinfoil.py
@@ -52,6 +52,10 @@
def remoteCommand(self, cmd, *args, **kwargs):
return self.tinfoil.run_command('dataStoreConnectorVarHistCmd', self.dsindex, cmd, args, kwargs)
+ def emit(self, var, oval, val, o, d):
+ ret = self.tinfoil.run_command('dataStoreConnectorVarHistCmdEmit', self.dsindex, var, oval, val, d.dsindex)
+ o.write(ret)
+
def __getattr__(self, name):
if not hasattr(bb.data_smart.VariableHistory, name):
raise AttributeError("VariableHistory has no such method %s" % name)
diff --git a/poky/bitbake/lib/bb/ui/knotty.py b/poky/bitbake/lib/bb/ui/knotty.py
index 0efa614..65ff272 100644
--- a/poky/bitbake/lib/bb/ui/knotty.py
+++ b/poky/bitbake/lib/bb/ui/knotty.py
@@ -21,6 +21,7 @@
import struct
import copy
import atexit
+from itertools import groupby
from bb.ui import uihelper
@@ -539,6 +540,13 @@
except OSError:
pass
+ # Add the logging domains specified by the user on the command line
+ for (domainarg, iterator) in groupby(params.debug_domains):
+ dlevel = len(tuple(iterator))
+ l = logconfig["loggers"].setdefault("BitBake.%s" % domainarg, {})
+ l["level"] = logging.DEBUG - dlevel + 1
+ l.setdefault("handlers", []).extend(["BitBake.verbconsole"])
+
conf = bb.msg.setLoggingConfig(logconfig, logconfigfile)
if sys.stdin.isatty() and sys.stdout.isatty():
diff --git a/poky/bitbake/lib/bb/utils.py b/poky/bitbake/lib/bb/utils.py
index b282d09..6ba1d2a 100644
--- a/poky/bitbake/lib/bb/utils.py
+++ b/poky/bitbake/lib/bb/utils.py
@@ -782,7 +782,7 @@
if sstat[stat.ST_DEV] == dstat[stat.ST_DEV]:
try:
- os.rename(src, destpath)
+ bb.utils.rename(src, destpath)
renamefailed = 0
except Exception as e:
if e.errno != errno.EXDEV:
@@ -796,7 +796,7 @@
if stat.S_ISREG(sstat[stat.ST_MODE]):
try: # For safety copy then move it over.
shutil.copyfile(src, destpath + "#new")
- os.rename(destpath + "#new", destpath)
+ bb.utils.rename(destpath + "#new", destpath)
didcopy = 1
except Exception as e:
print('movefile: copy', src, '->', dest, 'failed.', e)
@@ -874,7 +874,7 @@
# For safety copy then move it over.
shutil.copyfile(src, dest + "#new")
- os.rename(dest + "#new", dest)
+ bb.utils.rename(dest + "#new", dest)
except Exception as e:
logger.warning("copyfile: copy %s to %s failed (%s)" % (src, dest, e))
return False
@@ -1669,3 +1669,15 @@
return False
return True
+
+# Wrapper around os.rename which can handle cross device problems
+# e.g. from container filesystems
+def rename(src, dst):
+ try:
+ os.rename(src, dst)
+ except OSError as err:
+ if err.errno == 18:
+ # Invalid cross-device link error
+ shutil.move(src, dst)
+ else:
+ raise err
diff --git a/poky/bitbake/lib/bblayers/layerindex.py b/poky/bitbake/lib/bblayers/layerindex.py
index b2f27b2..7936516 100644
--- a/poky/bitbake/lib/bblayers/layerindex.py
+++ b/poky/bitbake/lib/bblayers/layerindex.py
@@ -159,12 +159,17 @@
logger.plain(' recommended by: %s' % ' '.join(recommendedby))
if dependencies:
- fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR')
- if not fetchdir:
- logger.error("Cannot get BBLAYERS_FETCH_DIR")
- return 1
+ if args.fetchdir:
+ fetchdir = args.fetchdir
+ else:
+ fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR')
+ if not fetchdir:
+ logger.error("Cannot get BBLAYERS_FETCH_DIR")
+ return 1
+
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
+
addlayers = []
for deplayerbranch in dependencies:
@@ -206,6 +211,8 @@
"""
args.show_only = True
args.ignore = []
+ args.fetchdir = ""
+ args.shallow = True
self.do_layerindex_fetch(args)
def register_commands(self, sp):
@@ -214,6 +221,7 @@
parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch')
parser_layerindex_fetch.add_argument('-s', '--shallow', help='do only shallow clones (--depth=1)', action='store_true')
parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER')
+ parser_layerindex_fetch.add_argument('-f', '--fetchdir', help='directory to fetch the layer(s) into (will be created if it does not exist)')
parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch')
parser_layerindex_show_depends = self.add_command(sp, 'layerindex-show-depends', self.do_layerindex_show_depends, parserecipes=False)
diff --git a/poky/bitbake/lib/hashserv/client.py b/poky/bitbake/lib/hashserv/client.py
index e05c1eb..5311709 100644
--- a/poky/bitbake/lib/hashserv/client.py
+++ b/poky/bitbake/lib/hashserv/client.py
@@ -8,110 +8,26 @@
import logging
import socket
import os
-from . import chunkify, DEFAULT_MAX_CHUNK, create_async_client
+import bb.asyncrpc
+from . import create_async_client
logger = logging.getLogger("hashserv.client")
-class HashConnectionError(Exception):
- pass
-
-
-class AsyncClient(object):
+class AsyncClient(bb.asyncrpc.AsyncClient):
MODE_NORMAL = 0
MODE_GET_STREAM = 1
def __init__(self):
- self.reader = None
- self.writer = None
+ super().__init__('OEHASHEQUIV', '1.1', logger)
self.mode = self.MODE_NORMAL
- self.max_chunk = DEFAULT_MAX_CHUNK
- async def connect_tcp(self, address, port):
- async def connect_sock():
- return await asyncio.open_connection(address, port)
-
- self._connect_sock = connect_sock
-
- async def connect_unix(self, path):
- async def connect_sock():
- return await asyncio.open_unix_connection(path)
-
- self._connect_sock = connect_sock
-
- async def connect(self):
- if self.reader is None or self.writer is None:
- (self.reader, self.writer) = await self._connect_sock()
-
- self.writer.write("OEHASHEQUIV 1.1\n\n".encode("utf-8"))
- await self.writer.drain()
-
- cur_mode = self.mode
- self.mode = self.MODE_NORMAL
- await self._set_mode(cur_mode)
-
- async def close(self):
- self.reader = None
-
- if self.writer is not None:
- self.writer.close()
- self.writer = None
-
- async def _send_wrapper(self, proc):
- count = 0
- while True:
- try:
- await self.connect()
- return await proc()
- except (
- OSError,
- HashConnectionError,
- json.JSONDecodeError,
- UnicodeDecodeError,
- ) as e:
- logger.warning("Error talking to server: %s" % e)
- if count >= 3:
- if not isinstance(e, HashConnectionError):
- raise HashConnectionError(str(e))
- raise e
- await self.close()
- count += 1
-
- async def send_message(self, msg):
- async def get_line():
- line = await self.reader.readline()
- if not line:
- raise HashConnectionError("Connection closed")
-
- line = line.decode("utf-8")
-
- if not line.endswith("\n"):
- raise HashConnectionError("Bad message %r" % message)
-
- return line
-
- async def proc():
- for c in chunkify(json.dumps(msg), self.max_chunk):
- self.writer.write(c.encode("utf-8"))
- await self.writer.drain()
-
- l = await get_line()
-
- m = json.loads(l)
- if m and "chunk-stream" in m:
- lines = []
- while True:
- l = (await get_line()).rstrip("\n")
- if not l:
- break
- lines.append(l)
-
- m = json.loads("".join(lines))
-
- return m
-
- return await self._send_wrapper(proc)
+ async def setup_connection(self):
+ await super().setup_connection()
+ cur_mode = self.mode
+ self.mode = self.MODE_NORMAL
+ await self._set_mode(cur_mode)
async def send_stream(self, msg):
async def proc():
@@ -119,7 +35,7 @@
await self.writer.drain()
l = await self.reader.readline()
if not l:
- raise HashConnectionError("Connection closed")
+ raise ConnectionError("Connection closed")
return l.decode("utf-8").rstrip()
return await self._send_wrapper(proc)
@@ -128,11 +44,11 @@
if new_mode == self.MODE_NORMAL and self.mode == self.MODE_GET_STREAM:
r = await self.send_stream("END")
if r != "ok":
- raise HashConnectionError("Bad response from server %r" % r)
+ raise ConnectionError("Bad response from server %r" % r)
elif new_mode == self.MODE_GET_STREAM and self.mode == self.MODE_NORMAL:
r = await self.send_message({"get-stream": None})
if r != "ok":
- raise HashConnectionError("Bad response from server %r" % r)
+ raise ConnectionError("Bad response from server %r" % r)
elif new_mode != self.mode:
raise Exception(
"Undefined mode transition %r -> %r" % (self.mode, new_mode)
@@ -189,12 +105,10 @@
return (await self.send_message({"backfill-wait": None}))["tasks"]
-class Client(object):
+class Client(bb.asyncrpc.Client):
def __init__(self):
- self.client = AsyncClient()
- self.loop = asyncio.new_event_loop()
-
- for call in (
+ super().__init__()
+ self._add_methods(
"connect_tcp",
"close",
"get_unihash",
@@ -204,30 +118,7 @@
"get_stats",
"reset_stats",
"backfill_wait",
- ):
- downcall = getattr(self.client, call)
- setattr(self, call, self._get_downcall_wrapper(downcall))
+ )
- def _get_downcall_wrapper(self, downcall):
- def wrapper(*args, **kwargs):
- return self.loop.run_until_complete(downcall(*args, **kwargs))
-
- return wrapper
-
- def connect_unix(self, path):
- # AF_UNIX has path length issues so chdir here to workaround
- cwd = os.getcwd()
- try:
- os.chdir(os.path.dirname(path))
- self.loop.run_until_complete(self.client.connect_unix(os.path.basename(path)))
- self.loop.run_until_complete(self.client.connect())
- finally:
- os.chdir(cwd)
-
- @property
- def max_chunk(self):
- return self.client.max_chunk
-
- @max_chunk.setter
- def max_chunk(self, value):
- self.client.max_chunk = value
+ def _get_async_client(self):
+ return AsyncClient()
diff --git a/poky/bitbake/lib/hashserv/server.py b/poky/bitbake/lib/hashserv/server.py
index a0dc0c1..c941c0e 100644
--- a/poky/bitbake/lib/hashserv/server.py
+++ b/poky/bitbake/lib/hashserv/server.py
@@ -14,7 +14,9 @@
import socket
import sys
import time
-from . import chunkify, DEFAULT_MAX_CHUNK, create_async_client, TABLE_COLUMNS
+from . import create_async_client, TABLE_COLUMNS
+import bb.asyncrpc
+
logger = logging.getLogger('hashserv.server')
@@ -109,12 +111,6 @@
return {k: getattr(self, k) for k in ('num', 'total_time', 'max_time', 'average', 'stdev')}
-class ClientError(Exception):
- pass
-
-class ServerError(Exception):
- pass
-
def insert_task(cursor, data, ignore=False):
keys = sorted(data.keys())
query = '''INSERT%s INTO tasks_v2 (%s) VALUES (%s)''' % (
@@ -149,7 +145,7 @@
return d
-class ServerClient(object):
+class ServerClient(bb.asyncrpc.AsyncServerConnection):
FAST_QUERY = 'SELECT taskhash, method, unihash FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1'
ALL_QUERY = 'SELECT * FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1'
OUTHASH_QUERY = '''
@@ -168,21 +164,19 @@
'''
def __init__(self, reader, writer, db, request_stats, backfill_queue, upstream, read_only):
- self.reader = reader
- self.writer = writer
+ super().__init__(reader, writer, 'OEHASHEQUIV', logger)
self.db = db
self.request_stats = request_stats
- self.max_chunk = DEFAULT_MAX_CHUNK
+ self.max_chunk = bb.asyncrpc.DEFAULT_MAX_CHUNK
self.backfill_queue = backfill_queue
self.upstream = upstream
- self.handlers = {
+ self.handlers.update({
'get': self.handle_get,
'get-outhash': self.handle_get_outhash,
'get-stream': self.handle_get_stream,
'get-stats': self.handle_get_stats,
- 'chunk-stream': self.handle_chunk,
- }
+ })
if not read_only:
self.handlers.update({
@@ -192,56 +186,19 @@
'backfill-wait': self.handle_backfill_wait,
})
+ def validate_proto_version(self):
+ return (self.proto_version > (1, 0) and self.proto_version <= (1, 1))
+
async def process_requests(self):
if self.upstream is not None:
self.upstream_client = await create_async_client(self.upstream)
else:
self.upstream_client = None
- try:
+ await super().process_requests()
-
- self.addr = self.writer.get_extra_info('peername')
- logger.debug('Client %r connected' % (self.addr,))
-
- # Read protocol and version
- protocol = await self.reader.readline()
- if protocol is None:
- return
-
- (proto_name, proto_version) = protocol.decode('utf-8').rstrip().split()
- if proto_name != 'OEHASHEQUIV':
- return
-
- proto_version = tuple(int(v) for v in proto_version.split('.'))
- if proto_version < (1, 0) or proto_version > (1, 1):
- return
-
- # Read headers. Currently, no headers are implemented, so look for
- # an empty line to signal the end of the headers
- while True:
- line = await self.reader.readline()
- if line is None:
- return
-
- line = line.decode('utf-8').rstrip()
- if not line:
- break
-
- # Handle messages
- while True:
- d = await self.read_message()
- if d is None:
- break
- await self.dispatch_message(d)
- await self.writer.drain()
- except ClientError as e:
- logger.error(str(e))
- finally:
- if self.upstream_client is not None:
- await self.upstream_client.close()
-
- self.writer.close()
+ if self.upstream_client is not None:
+ await self.upstream_client.close()
async def dispatch_message(self, msg):
for k in self.handlers.keys():
@@ -255,47 +212,7 @@
await self.handlers[k](msg[k])
return
- raise ClientError("Unrecognized command %r" % msg)
-
- def write_message(self, msg):
- for c in chunkify(json.dumps(msg), self.max_chunk):
- self.writer.write(c.encode('utf-8'))
-
- async def read_message(self):
- l = await self.reader.readline()
- if not l:
- return None
-
- try:
- message = l.decode('utf-8')
-
- if not message.endswith('\n'):
- return None
-
- return json.loads(message)
- except (json.JSONDecodeError, UnicodeDecodeError) as e:
- logger.error('Bad message from client: %r' % message)
- raise e
-
- async def handle_chunk(self, request):
- lines = []
- try:
- while True:
- l = await self.reader.readline()
- l = l.rstrip(b"\n").decode("utf-8")
- if not l:
- break
- lines.append(l)
-
- msg = json.loads(''.join(lines))
- except (json.JSONDecodeError, UnicodeDecodeError) as e:
- logger.error('Bad message from client: %r' % message)
- raise e
-
- if 'chunk-stream' in msg:
- raise ClientError("Nested chunks are not allowed")
-
- await self.dispatch_message(msg)
+ raise bb.asyncrpc.ClientError("Unrecognized command %r" % msg)
async def handle_get(self, request):
method = request['method']
@@ -499,74 +416,20 @@
cursor.close()
-class Server(object):
+class Server(bb.asyncrpc.AsyncServer):
def __init__(self, db, loop=None, upstream=None, read_only=False):
if upstream and read_only:
- raise ServerError("Read-only hashserv cannot pull from an upstream server")
+ raise bb.asyncrpc.ServerError("Read-only hashserv cannot pull from an upstream server")
+
+ super().__init__(logger, loop)
self.request_stats = Stats()
self.db = db
-
- if loop is None:
- self.loop = asyncio.new_event_loop()
- self.close_loop = True
- else:
- self.loop = loop
- self.close_loop = False
-
self.upstream = upstream
self.read_only = read_only
- self._cleanup_socket = None
-
- def start_tcp_server(self, host, port):
- self.server = self.loop.run_until_complete(
- asyncio.start_server(self.handle_client, host, port, loop=self.loop)
- )
-
- for s in self.server.sockets:
- logger.info('Listening on %r' % (s.getsockname(),))
- # Newer python does this automatically. Do it manually here for
- # maximum compatibility
- s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
- s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1)
-
- name = self.server.sockets[0].getsockname()
- if self.server.sockets[0].family == socket.AF_INET6:
- self.address = "[%s]:%d" % (name[0], name[1])
- else:
- self.address = "%s:%d" % (name[0], name[1])
-
- def start_unix_server(self, path):
- def cleanup():
- os.unlink(path)
-
- cwd = os.getcwd()
- try:
- # Work around path length limits in AF_UNIX
- os.chdir(os.path.dirname(path))
- self.server = self.loop.run_until_complete(
- asyncio.start_unix_server(self.handle_client, os.path.basename(path), loop=self.loop)
- )
- finally:
- os.chdir(cwd)
-
- logger.info('Listening on %r' % path)
-
- self._cleanup_socket = cleanup
- self.address = "unix://%s" % os.path.abspath(path)
-
- async def handle_client(self, reader, writer):
- # writer.transport.set_write_buffer_limits(0)
- try:
- client = ServerClient(reader, writer, self.db, self.request_stats, self.backfill_queue, self.upstream, self.read_only)
- await client.process_requests()
- except Exception as e:
- import traceback
- logger.error('Error from client: %s' % str(e), exc_info=True)
- traceback.print_exc()
- writer.close()
- logger.info('Client disconnected')
+ def accept_client(self, reader, writer):
+ return ServerClient(reader, writer, self.db, self.request_stats, self.backfill_queue, self.upstream, self.read_only)
@contextmanager
def _backfill_worker(self):
@@ -597,31 +460,8 @@
else:
yield
- def serve_forever(self):
- def signal_handler():
- self.loop.stop()
+ def run_loop_forever(self):
+ self.backfill_queue = asyncio.Queue()
- asyncio.set_event_loop(self.loop)
- try:
- self.backfill_queue = asyncio.Queue()
-
- self.loop.add_signal_handler(signal.SIGTERM, signal_handler)
-
- with self._backfill_worker():
- try:
- self.loop.run_forever()
- except KeyboardInterrupt:
- pass
-
- self.server.close()
-
- self.loop.run_until_complete(self.server.wait_closed())
- logger.info('Server shutting down')
- finally:
- if self.close_loop:
- if sys.version_info >= (3, 6):
- self.loop.run_until_complete(self.loop.shutdown_asyncgens())
- self.loop.close()
-
- if self._cleanup_socket is not None:
- self._cleanup_socket()
+ with self._backfill_worker():
+ super().run_loop_forever()
diff --git a/poky/bitbake/lib/hashserv/tests.py b/poky/bitbake/lib/hashserv/tests.py
index 1a69648..e2b762d 100644
--- a/poky/bitbake/lib/hashserv/tests.py
+++ b/poky/bitbake/lib/hashserv/tests.py
@@ -6,7 +6,6 @@
#
from . import create_server, create_client
-from .client import HashConnectionError
import hashlib
import logging
import multiprocessing
@@ -277,7 +276,7 @@
outhash2 = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44'
unihash2 = '90e9bc1d1f094c51824adca7f8ea79a048d68824'
- with self.assertRaises(HashConnectionError):
+ with self.assertRaises(ConnectionError):
ro_client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
# Ensure that the database was not modified
diff --git a/poky/bitbake/lib/prserv/serv.py b/poky/bitbake/lib/prserv/serv.py
index 25dcf8a..5e322bf 100644
--- a/poky/bitbake/lib/prserv/serv.py
+++ b/poky/bitbake/lib/prserv/serv.py
@@ -5,8 +5,6 @@
import os,sys,logging
import signal, time
from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
-import threading
-import queue
import socket
import io
import sqlite3
@@ -14,14 +12,10 @@
import prserv
import prserv.db
import errno
-import select
+import multiprocessing
logger = logging.getLogger("BitBake.PRserv")
-if sys.hexversion < 0x020600F0:
- print("Sorry, python 2.6 or later is required.")
- sys.exit(1)
-
class Handler(SimpleXMLRPCRequestHandler):
def _dispatch(self,method,params):
try:
@@ -37,7 +31,7 @@
class PRServer(SimpleXMLRPCServer):
- def __init__(self, dbfile, logfile, interface, daemon=True):
+ def __init__(self, dbfile, logfile, interface):
''' constructor '''
try:
SimpleXMLRPCServer.__init__(self, interface,
@@ -50,57 +44,18 @@
raise PRServiceConfigError
self.dbfile=dbfile
- self.daemon=daemon
self.logfile=logfile
- self.working_thread=None
self.host, self.port = self.socket.getsockname()
- self.pidfile=PIDPREFIX % (self.host, self.port)
self.register_function(self.getPR, "getPR")
- self.register_function(self.quit, "quit")
self.register_function(self.ping, "ping")
self.register_function(self.export, "export")
- self.register_function(self.dump_db, "dump_db")
self.register_function(self.importone, "importone")
self.register_introspection_functions()
- self.quitpipein, self.quitpipeout = os.pipe()
-
- self.requestqueue = queue.Queue()
- self.handlerthread = threading.Thread(target = self.process_request_thread)
- self.handlerthread.daemon = False
-
- def process_request_thread(self):
- """Same as in BaseServer but as a thread.
-
- In addition, exception handling is done here.
-
- """
- iter_count = 1
+ self.iter_count = 0
# 60 iterations between syncs or sync if dirty every ~30 seconds
- iterations_between_sync = 60
-
- bb.utils.set_process_name("PRServ Handler")
-
- while not self.quitflag:
- try:
- (request, client_address) = self.requestqueue.get(True, 30)
- except queue.Empty:
- self.table.sync_if_dirty()
- continue
- if request is None:
- continue
- try:
- self.finish_request(request, client_address)
- self.shutdown_request(request)
- iter_count = (iter_count + 1) % iterations_between_sync
- if iter_count == 0:
- self.table.sync_if_dirty()
- except:
- self.handle_error(request, client_address)
- self.shutdown_request(request)
- self.table.sync()
- self.table.sync_if_dirty()
+ self.iterations_between_sync = 60
def sigint_handler(self, signum, stack):
if self.table:
@@ -109,11 +64,30 @@
def sigterm_handler(self, signum, stack):
if self.table:
self.table.sync()
- self.quit()
- self.requestqueue.put((None, None))
+ raise(SystemExit)
def process_request(self, request, client_address):
- self.requestqueue.put((request, client_address))
+ if request is None:
+ return
+ try:
+ self.finish_request(request, client_address)
+ self.shutdown_request(request)
+ self.iter_count = (self.iter_count + 1) % self.iterations_between_sync
+ if self.iter_count == 0:
+ self.table.sync_if_dirty()
+ except:
+ self.handle_error(request, client_address)
+ self.shutdown_request(request)
+ self.table.sync()
+ self.table.sync_if_dirty()
+
+ def serve_forever(self, poll_interval=0.5):
+ signal.signal(signal.SIGINT, self.sigint_handler)
+ signal.signal(signal.SIGTERM, self.sigterm_handler)
+
+ self.db = prserv.db.PRData(self.dbfile)
+ self.table = self.db["PRMAIN"]
+ return super().serve_forever(poll_interval)
def export(self, version=None, pkgarch=None, checksum=None, colinfo=True):
try:
@@ -122,31 +96,11 @@
logger.error(str(exc))
return None
- def dump_db(self):
- """
- Returns a script (string) that reconstructs the state of the
- entire database at the time this function is called. The script
- language is defined by the backing database engine, which is a
- function of server configuration.
- Returns None if the database engine does not support dumping to
- script or if some other error is encountered in processing.
- """
- buff = io.StringIO()
- try:
- self.table.sync()
- self.table.dump_db(buff)
- return buff.getvalue()
- except Exception as exc:
- logger.error(str(exc))
- return None
- finally:
- buff.close()
-
def importone(self, version, pkgarch, checksum, value):
return self.table.importone(version, pkgarch, checksum, value)
def ping(self):
- return not self.quitflag
+ return True
def getinfo(self):
return (self.host, self.port)
@@ -161,144 +115,6 @@
logger.error(str(exc))
return None
- def quit(self):
- self.quitflag=True
- os.write(self.quitpipeout, b"q")
- os.close(self.quitpipeout)
- return
-
- def work_forever(self,):
- self.quitflag = False
- # This timeout applies to the poll in TCPServer, we need the select
- # below to wake on our quit pipe closing. We only ever call into handle_request
- # if there is data there.
- self.timeout = 0.01
-
- bb.utils.set_process_name("PRServ")
-
- # DB connection must be created after all forks
- self.db = prserv.db.PRData(self.dbfile)
- self.table = self.db["PRMAIN"]
-
- logger.info("Started PRServer with DBfile: %s, IP: %s, PORT: %s, PID: %s" %
- (self.dbfile, self.host, self.port, str(os.getpid())))
-
- self.handlerthread.start()
- while not self.quitflag:
- ready = select.select([self.fileno(), self.quitpipein], [], [], 30)
- if self.quitflag:
- break
- if self.fileno() in ready[0]:
- self.handle_request()
- self.handlerthread.join()
- self.db.disconnect()
- logger.info("PRServer: stopping...")
- self.server_close()
- os.close(self.quitpipein)
- return
-
- def start(self):
- if self.daemon:
- pid = self.daemonize()
- else:
- pid = self.fork()
- self.pid = pid
-
- # Ensure both the parent sees this and the child from the work_forever log entry above
- logger.info("Started PRServer with DBfile: %s, IP: %s, PORT: %s, PID: %s" %
- (self.dbfile, self.host, self.port, str(pid)))
-
- def delpid(self):
- os.remove(self.pidfile)
-
- def daemonize(self):
- """
- See Advanced Programming in the UNIX, Sec 13.3
- """
- try:
- pid = os.fork()
- if pid > 0:
- os.waitpid(pid, 0)
- #parent return instead of exit to give control
- return pid
- except OSError as e:
- raise Exception("%s [%d]" % (e.strerror, e.errno))
-
- os.setsid()
- """
- fork again to make sure the daemon is not session leader,
- which prevents it from acquiring controlling terminal
- """
- try:
- pid = os.fork()
- if pid > 0: #parent
- os._exit(0)
- except OSError as e:
- raise Exception("%s [%d]" % (e.strerror, e.errno))
-
- self.cleanup_handles()
- os._exit(0)
-
- def fork(self):
- try:
- pid = os.fork()
- if pid > 0:
- self.socket.close() # avoid ResourceWarning in parent
- return pid
- except OSError as e:
- raise Exception("%s [%d]" % (e.strerror, e.errno))
-
- bb.utils.signal_on_parent_exit("SIGTERM")
- self.cleanup_handles()
- os._exit(0)
-
- def cleanup_handles(self):
- signal.signal(signal.SIGINT, self.sigint_handler)
- signal.signal(signal.SIGTERM, self.sigterm_handler)
- os.chdir("/")
-
- sys.stdout.flush()
- sys.stderr.flush()
-
- # We could be called from a python thread with io.StringIO as
- # stdout/stderr or it could be 'real' unix fd forking where we need
- # to physically close the fds to prevent the program launching us from
- # potentially hanging on a pipe. Handle both cases.
- si = open('/dev/null', 'r')
- try:
- os.dup2(si.fileno(),sys.stdin.fileno())
- except (AttributeError, io.UnsupportedOperation):
- sys.stdin = si
- so = open(self.logfile, 'a+')
- try:
- os.dup2(so.fileno(),sys.stdout.fileno())
- except (AttributeError, io.UnsupportedOperation):
- sys.stdout = so
- try:
- os.dup2(so.fileno(),sys.stderr.fileno())
- except (AttributeError, io.UnsupportedOperation):
- sys.stderr = so
-
- # Clear out all log handlers prior to the fork() to avoid calling
- # event handlers not part of the PRserver
- for logger_iter in logging.Logger.manager.loggerDict.keys():
- logging.getLogger(logger_iter).handlers = []
-
- # Ensure logging makes it to the logfile
- streamhandler = logging.StreamHandler()
- streamhandler.setLevel(logging.DEBUG)
- formatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
- streamhandler.setFormatter(formatter)
- logger.addHandler(streamhandler)
-
- # write pidfile
- pid = str(os.getpid())
- with open(self.pidfile, 'w') as pf:
- pf.write("%s\n" % pid)
-
- self.work_forever()
- self.delpid()
-
class PRServSingleton(object):
def __init__(self, dbfile, logfile, interface):
self.dbfile = dbfile
@@ -308,8 +124,10 @@
self.port = None
def start(self):
- self.prserv = PRServer(self.dbfile, self.logfile, self.interface, daemon=False)
- self.prserv.start()
+ self.prserv = PRServer(self.dbfile, self.logfile, self.interface)
+ self.process = multiprocessing.Process(target=self.prserv.serve_forever)
+ self.process.start()
+
self.host, self.port = self.prserv.getinfo()
def getinfo(self):
@@ -323,13 +141,6 @@
self.port = port
self.connection, self.transport = bb.server.xmlrpcclient._create_server(self.host, self.port)
- def terminate(self):
- try:
- logger.info("Terminating PRServer...")
- self.connection.quit()
- except Exception as exc:
- sys.stderr.write("%s\n" % str(exc))
-
def getPR(self, version, pkgarch, checksum):
return self.connection.getPR(version, pkgarch, checksum)
@@ -339,15 +150,82 @@
def export(self,version=None, pkgarch=None, checksum=None, colinfo=True):
return self.connection.export(version, pkgarch, checksum, colinfo)
- def dump_db(self):
- return self.connection.dump_db()
-
def importone(self, version, pkgarch, checksum, value):
return self.connection.importone(version, pkgarch, checksum, value)
def getinfo(self):
return self.host, self.port
+def run_as_daemon(func, pidfile, logfile):
+ """
+ See Advanced Programming in the UNIX, Sec 13.3
+ """
+ try:
+ pid = os.fork()
+ if pid > 0:
+ os.waitpid(pid, 0)
+ #parent return instead of exit to give control
+ return pid
+ except OSError as e:
+ raise Exception("%s [%d]" % (e.strerror, e.errno))
+
+ os.setsid()
+ """
+ fork again to make sure the daemon is not session leader,
+ which prevents it from acquiring controlling terminal
+ """
+ try:
+ pid = os.fork()
+ if pid > 0: #parent
+ os._exit(0)
+ except OSError as e:
+ raise Exception("%s [%d]" % (e.strerror, e.errno))
+
+ os.chdir("/")
+
+ sys.stdout.flush()
+ sys.stderr.flush()
+
+ # We could be called from a python thread with io.StringIO as
+ # stdout/stderr or it could be 'real' unix fd forking where we need
+ # to physically close the fds to prevent the program launching us from
+ # potentially hanging on a pipe. Handle both cases.
+ si = open('/dev/null', 'r')
+ try:
+ os.dup2(si.fileno(),sys.stdin.fileno())
+ except (AttributeError, io.UnsupportedOperation):
+ sys.stdin = si
+ so = open(logfile, 'a+')
+ try:
+ os.dup2(so.fileno(),sys.stdout.fileno())
+ except (AttributeError, io.UnsupportedOperation):
+ sys.stdout = so
+ try:
+ os.dup2(so.fileno(),sys.stderr.fileno())
+ except (AttributeError, io.UnsupportedOperation):
+ sys.stderr = so
+
+ # Clear out all log handlers prior to the fork() to avoid calling
+ # event handlers not part of the PRserver
+ for logger_iter in logging.Logger.manager.loggerDict.keys():
+ logging.getLogger(logger_iter).handlers = []
+
+ # Ensure logging makes it to the logfile
+ streamhandler = logging.StreamHandler()
+ streamhandler.setLevel(logging.DEBUG)
+ formatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
+ streamhandler.setFormatter(formatter)
+ logger.addHandler(streamhandler)
+
+ # write pidfile
+ pid = str(os.getpid())
+ with open(pidfile, 'w') as pf:
+ pf.write("%s\n" % pid)
+
+ func()
+ os.remove(pidfile)
+ os._exit(0)
+
def start_daemon(dbfile, host, port, logfile):
ip = socket.gethostbyname(host)
pidfile = PIDPREFIX % (ip, port)
@@ -363,7 +241,7 @@
return 1
server = PRServer(os.path.abspath(dbfile), os.path.abspath(logfile), (ip,port))
- server.start()
+ run_as_daemon(server.serve_forever, pidfile, os.path.abspath(logfile))
# Sometimes, the port (i.e. localhost:0) indicated by the user does not match with
# the one the server actually is listening, so at least warn the user about it
@@ -400,25 +278,13 @@
return 1
try:
- PRServerConnection(ip, port).terminate()
- except:
- logger.critical("Stop PRService %s:%d failed" % (host,port))
+ if is_running(pid):
+ print("Sending SIGTERM to pr-server.")
+ os.kill(pid, signal.SIGTERM)
+ time.sleep(0.1)
- try:
- if pid:
- wait_timeout = 0
- print("Waiting for pr-server to exit.")
- while is_running(pid) and wait_timeout < 50:
- time.sleep(0.1)
- wait_timeout += 1
-
- if is_running(pid):
- print("Sending SIGTERM to pr-server.")
- os.kill(pid,signal.SIGTERM)
- time.sleep(0.1)
-
- if os.path.exists(pidfile):
- os.remove(pidfile)
+ if os.path.exists(pidfile):
+ os.remove(pidfile)
except OSError as e:
err = str(e)
@@ -494,19 +360,14 @@
def auto_shutdown():
global singleton
- if singleton:
- host, port = singleton.getinfo()
- try:
- PRServerConnection(host, port).terminate()
- except:
- logger.critical("Stop PRService %s:%d failed" % (host,port))
-
- try:
- os.waitpid(singleton.prserv.pid, 0)
- except ChildProcessError:
- pass
+ if singleton and singleton.process:
+ singleton.process.terminate()
+ singleton.process.join()
singleton = None
def ping(host, port):
conn=PRServerConnection(host, port)
return conn.ping()
+
+def connect(host, port):
+ return PRServerConnection(host, port)