blob: 77bf61133ea32061c928e25a2fc263c640330da1 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# IceCream distributed compiling support
2#
3# Stages directories with symlinks from gcc/g++ to icecc, for both
4# native and cross compilers. Depending on each configure or compile,
5# the directories are added at the head of the PATH list and ICECC_CXX
6# and ICEC_CC are set.
7#
8# For the cross compiler, creates a tar.gz of our toolchain and sets
9# ICECC_VERSION accordingly.
10#
11# The class now handles all 3 different compile 'stages' (i.e native ,cross-kernel and target) creating the
12# necessary environment tar.gz file to be used by the remote machines.
13# It also supports meta-toolchain generation
14#
15# If ICECC_PATH is not set in local.conf then the class will try to locate it using 'bb.utils.which'
16# but nothing is sure ;)
17#
18# If ICECC_ENV_EXEC is set in local.conf, then it should point to the icecc-create-env script provided by the user
19# or the default one provided by icecc-create-env.bb will be used
20# (NOTE that this is a modified version of the script need it and *not the one that comes with icecc*
21#
22# User can specify if specific packages or packages belonging to class should not use icecc to distribute
23# compile jobs to remote machines, but handled locally, by defining ICECC_USER_CLASS_BL and ICECC_USER_PACKAGE_BL
24# with the appropriate values in local.conf. In addition the user can force to enable icecc for packages
25# which set an empty PARALLEL_MAKE variable by defining ICECC_USER_PACKAGE_WL.
26#
27#########################################################################################
28#Error checking is kept to minimum so double check any parameters you pass to the class
29###########################################################################################
30
31BB_HASHBASE_WHITELIST += "ICECC_PARALLEL_MAKE ICECC_DISABLED ICECC_USER_PACKAGE_BL ICECC_USER_CLASS_BL ICECC_USER_PACKAGE_WL ICECC_PATH ICECC_ENV_EXEC"
32
33ICECC_ENV_EXEC ?= "${STAGING_BINDIR_NATIVE}/icecc-create-env"
34
35def icecc_dep_prepend(d):
36 # INHIBIT_DEFAULT_DEPS doesn't apply to the patch command. Whether or not
37 # we need that built is the responsibility of the patch function / class, not
38 # the application.
39 if not d.getVar('INHIBIT_DEFAULT_DEPS', False):
40 return "icecc-create-env-native"
41 return ""
42
43DEPENDS_prepend += "${@icecc_dep_prepend(d)} "
44
Brad Bishop37a0e4d2017-12-04 01:01:44 -050045get_cross_kernel_cc[vardepsexclude] += "KERNEL_CC"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050046def get_cross_kernel_cc(bb,d):
47 kernel_cc = d.getVar('KERNEL_CC', False)
48
49 # evaluate the expression by the shell if necessary
50 if '`' in kernel_cc or '$(' in kernel_cc:
Patrick Williamsc0f7c042017-02-23 20:41:17 -060051 import subprocess
52 kernel_cc = subprocess.check_output("echo %s" % kernel_cc, shell=True).decode("utf-8")[:-1]
Patrick Williamsc124f4f2015-09-15 14:41:29 -050053
54 kernel_cc = d.expand(kernel_cc)
55 kernel_cc = kernel_cc.replace('ccache', '').strip()
56 kernel_cc = kernel_cc.split(' ')[0]
57 kernel_cc = kernel_cc.strip()
58 return kernel_cc
59
60def get_icecc(d):
61 return d.getVar('ICECC_PATH', False) or bb.utils.which(os.getenv("PATH"), "icecc")
62
63def create_path(compilers, bb, d):
64 """
65 Create Symlinks for the icecc in the staging directory
66 """
67 staging = os.path.join(d.expand('${STAGING_BINDIR}'), "ice")
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050068 if icecc_is_kernel(bb, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050069 staging += "-kernel"
70
71 #check if the icecc path is set by the user
72 icecc = get_icecc(d)
73
74 # Create the dir if necessary
75 try:
76 os.stat(staging)
77 except:
78 try:
79 os.makedirs(staging)
80 except:
81 pass
82
83 for compiler in compilers:
84 gcc_path = os.path.join(staging, compiler)
85 try:
86 os.stat(gcc_path)
87 except:
88 try:
89 os.symlink(icecc, gcc_path)
90 except:
91 pass
92
93 return staging
94
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050095def use_icecc(bb,d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050096 if d.getVar('ICECC_DISABLED', False) == "1":
97 # don't even try it, when explicitly disabled
98 return "no"
99
100 # allarch recipes don't use compiler
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500101 if icecc_is_allarch(bb, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500102 return "no"
103
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500104 pn = d.getVar('PN')
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500105
106 system_class_blacklist = []
107 user_class_blacklist = (d.getVar('ICECC_USER_CLASS_BL', False) or "none").split()
108 package_class_blacklist = system_class_blacklist + user_class_blacklist
109
110 for black in package_class_blacklist:
111 if bb.data.inherits_class(black, d):
112 bb.debug(1, "%s: class %s found in blacklist, disable icecc" % (pn, black))
113 return "no"
114
115 # "system" recipe blacklist contains a list of packages that can not distribute compile tasks
116 # for one reason or the other
117 # this is the old list (which doesn't seem to be valid anymore, because I was able to build
118 # all these with icecc enabled)
119 # system_package_blacklist = [ "uclibc", "glibc", "gcc", "bind", "u-boot", "dhcp-forwarder", "enchant", "connman", "orbit2" ]
120 # when adding new entry, please document why (how it failed) so that we can re-evaluate it later
121 # e.g. when there is new version
122 # building libgcc-initial with icecc fails with CPP sanity check error if host sysroot contains cross gcc built for another target tune/variant
123 system_package_blacklist = ["libgcc-initial"]
124 user_package_blacklist = (d.getVar('ICECC_USER_PACKAGE_BL', False) or "").split()
125 user_package_whitelist = (d.getVar('ICECC_USER_PACKAGE_WL', False) or "").split()
126 package_blacklist = system_package_blacklist + user_package_blacklist
127
128 if pn in package_blacklist:
129 bb.debug(1, "%s: found in blacklist, disable icecc" % pn)
130 return "no"
131
132 if pn in user_package_whitelist:
133 bb.debug(1, "%s: found in whitelist, enable icecc" % pn)
134 return "yes"
135
136 if d.getVar('PARALLEL_MAKE', False) == "":
137 bb.debug(1, "%s: has empty PARALLEL_MAKE, disable icecc" % pn)
138 return "no"
139
140 return "yes"
141
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500142def icecc_is_allarch(bb, d):
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500143 return d.getVar("PACKAGE_ARCH") == "all" or bb.data.inherits_class('allarch', d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500144
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500145def icecc_is_kernel(bb, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500146 return \
147 bb.data.inherits_class("kernel", d);
148
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500149def icecc_is_native(bb, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500150 return \
151 bb.data.inherits_class("cross", d) or \
152 bb.data.inherits_class("native", d);
153
154# Don't pollute allarch signatures with TARGET_FPU
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500155icecc_version[vardepsexclude] += "TARGET_FPU"
156def icecc_version(bb, d):
157 if use_icecc(bb, d) == "no":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500158 return ""
159
160 parallel = d.getVar('ICECC_PARALLEL_MAKE', False) or ""
161 if not d.getVar('PARALLEL_MAKE', False) == "" and parallel:
162 d.setVar("PARALLEL_MAKE", parallel)
163
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500164 if icecc_is_native(bb, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500165 archive_name = "local-host-env"
166 elif d.expand('${HOST_PREFIX}') == "":
167 bb.fatal(d.expand("${PN}"), " NULL prefix")
168 else:
169 prefix = d.expand('${HOST_PREFIX}' )
170 distro = d.expand('${DISTRO}')
171 target_sys = d.expand('${TARGET_SYS}')
172 float = d.getVar('TARGET_FPU', False) or "hard"
173 archive_name = prefix + distro + "-" + target_sys + "-" + float
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500174 if icecc_is_kernel(bb, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500175 archive_name += "-kernel"
176
177 import socket
178 ice_dir = d.expand('${STAGING_DIR_NATIVE}${prefix_native}')
179 tar_file = os.path.join(ice_dir, 'ice', archive_name + "-@VERSION@-" + socket.gethostname() + '.tar.gz')
180
181 return tar_file
182
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500183def icecc_path(bb,d):
184 if use_icecc(bb, d) == "no":
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500185 # don't create unnecessary directories when icecc is disabled
186 return
187
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500188 if icecc_is_kernel(bb, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500189 return create_path( [get_cross_kernel_cc(bb,d), ], bb, d)
190
191 else:
192 prefix = d.expand('${HOST_PREFIX}')
193 return create_path( [prefix+"gcc", prefix+"g++"], bb, d)
194
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500195def icecc_get_external_tool(bb, d, tool):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500196 external_toolchain_bindir = d.expand('${EXTERNAL_TOOLCHAIN}${bindir_cross}')
197 target_prefix = d.expand('${TARGET_PREFIX}')
198 return os.path.join(external_toolchain_bindir, '%s%s' % (target_prefix, tool))
199
200# Don't pollute native signatures with target TUNE_PKGARCH through STAGING_BINDIR_TOOLCHAIN
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500201icecc_get_tool[vardepsexclude] += "STAGING_BINDIR_TOOLCHAIN"
202def icecc_get_tool(bb, d, tool):
203 if icecc_is_native(bb, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500204 return bb.utils.which(os.getenv("PATH"), tool)
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500205 elif icecc_is_kernel(bb, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500206 return bb.utils.which(os.getenv("PATH"), get_cross_kernel_cc(bb, d))
207 else:
208 ice_dir = d.expand('${STAGING_BINDIR_TOOLCHAIN}')
209 target_sys = d.expand('${TARGET_SYS}')
210 tool_bin = os.path.join(ice_dir, "%s-%s" % (target_sys, tool))
211 if os.path.isfile(tool_bin):
212 return tool_bin
213 else:
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500214 external_tool_bin = icecc_get_external_tool(bb, d, tool)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500215 if os.path.isfile(external_tool_bin):
216 return external_tool_bin
217 else:
218 return ""
219
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500220def icecc_get_and_check_tool(bb, d, tool):
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500221 # Check that g++ or gcc is not a symbolic link to icecc binary in
222 # PATH or icecc-create-env script will silently create an invalid
223 # compiler environment package.
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500224 t = icecc_get_tool(bb, d, tool)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600225 if t:
226 import subprocess
227 link_path = subprocess.check_output("readlink -f %s" % t, shell=True).decode("utf-8")[:-1]
228 if link_path == get_icecc(d):
229 bb.error("%s is a symlink to %s in PATH and this prevents icecc from working" % (t, get_icecc(d)))
230 return ""
231 else:
232 return t
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500233 else:
234 return t
235
236wait_for_file() {
237 local TIME_ELAPSED=0
238 local FILE_TO_TEST=$1
239 local TIMEOUT=$2
240 until [ -f "$FILE_TO_TEST" ]
241 do
242 TIME_ELAPSED=`expr $TIME_ELAPSED + 1`
243 if [ $TIME_ELAPSED -gt $TIMEOUT ]
244 then
245 return 1
246 fi
247 sleep 1
248 done
249}
250
251def set_icecc_env():
252 # dummy python version of set_icecc_env
253 return
254
255set_icecc_env() {
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500256 if [ "${@use_icecc(bb, d)}" = "no" ]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500257 then
258 return
259 fi
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500260 ICECC_VERSION="${@icecc_version(bb, d)}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500261 if [ "x${ICECC_VERSION}" = "x" ]
262 then
263 bbwarn "Cannot use icecc: could not get ICECC_VERSION"
264 return
265 fi
266
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500267 ICE_PATH="${@icecc_path(bb, d)}"
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500268 if [ "x${ICE_PATH}" = "x" ]
269 then
270 bbwarn "Cannot use icecc: could not get ICE_PATH"
271 return
272 fi
273
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500274 ICECC_CC="${@icecc_get_and_check_tool(bb, d, "gcc")}"
275 ICECC_CXX="${@icecc_get_and_check_tool(bb, d, "g++")}"
276 # cannot use icecc_get_and_check_tool here because it assumes as without target_sys prefix
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500277 ICECC_WHICH_AS="${@bb.utils.which(os.getenv('PATH'), 'as')}"
278 if [ ! -x "${ICECC_CC}" -o ! -x "${ICECC_CXX}" ]
279 then
280 bbwarn "Cannot use icecc: could not get ICECC_CC or ICECC_CXX"
281 return
282 fi
283
284 ICE_VERSION=`$ICECC_CC -dumpversion`
285 ICECC_VERSION=`echo ${ICECC_VERSION} | sed -e "s/@VERSION@/$ICE_VERSION/g"`
286 if [ ! -x "${ICECC_ENV_EXEC}" ]
287 then
288 bbwarn "Cannot use icecc: invalid ICECC_ENV_EXEC"
289 return
290 fi
291
292 ICECC_AS="`${ICECC_CC} -print-prog-name=as`"
293 # for target recipes should return something like:
294 # /OE/tmp-eglibc/sysroots/x86_64-linux/usr/libexec/arm920tt-oe-linux-gnueabi/gcc/arm-oe-linux-gnueabi/4.8.2/as
295 # and just "as" for native, if it returns "as" in current directory (for whatever reason) use "as" from PATH
296 if [ "`dirname "${ICECC_AS}"`" = "." ]
297 then
298 ICECC_AS="${ICECC_WHICH_AS}"
299 fi
300
301 if [ ! -f "${ICECC_VERSION}.done" ]
302 then
303 mkdir -p "`dirname "${ICECC_VERSION}"`"
304
305 # the ICECC_VERSION generation step must be locked by a mutex
306 # in order to prevent race conditions
307 if flock -n "${ICECC_VERSION}.lock" \
308 ${ICECC_ENV_EXEC} "${ICECC_CC}" "${ICECC_CXX}" "${ICECC_AS}" "${ICECC_VERSION}"
309 then
310 touch "${ICECC_VERSION}.done"
311 elif [ ! wait_for_file "${ICECC_VERSION}.done" 30 ]
312 then
313 # locking failed so wait for ${ICECC_VERSION}.done to appear
314 bbwarn "Timeout waiting for ${ICECC_VERSION}.done"
315 return
316 fi
317 fi
318
319 export ICECC_VERSION ICECC_CC ICECC_CXX
320 export PATH="$ICE_PATH:$PATH"
321 export CCACHE_PATH="$PATH"
322
323 bbnote "Using icecc"
324}
325
326do_configure_prepend() {
327 set_icecc_env
328}
329
330do_compile_prepend() {
331 set_icecc_env
332}
333
334do_compile_kernelmodules_prepend() {
335 set_icecc_env
336}
337
338do_install_prepend() {
339 set_icecc_env
340}