blob: 0eb696ac78f4d4cdaba69716f75f6628cb90a724 [file] [log] [blame]
Brad Bishop316dfdd2018-06-25 12:45:53 -04001# reproducible_build.bbclass
2#
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08003# Sets SOURCE_DATE_EPOCH in each component's build environment.
4# Upstream components (generally) respect this environment variable,
5# using it in place of the "current" date and time.
6# See https://reproducible-builds.org/specs/source-date-epoch/
Brad Bishop316dfdd2018-06-25 12:45:53 -04007#
Brad Bishop1a4b7ee2018-12-16 17:11:34 -08008# After sources are unpacked but before they are patched, we set a reproducible value for SOURCE_DATE_EPOCH.
9# This value should be reproducible for anyone who builds the same revision from the same sources.
Brad Bishop316dfdd2018-06-25 12:45:53 -040010#
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080011# There are 4 ways we determine SOURCE_DATE_EPOCH:
Brad Bishop316dfdd2018-06-25 12:45:53 -040012#
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080013# 1. Use the value from __source_date_epoch.txt file if this file exists.
14# This file was most likely created in the previous build by one of the following methods 2,3,4.
15# Alternatively, it can be provided by a recipe via SRC_URI.
Brad Bishop316dfdd2018-06-25 12:45:53 -040016#
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080017# If the file does not exist:
18#
19# 2. If there is a git checkout, use the last git commit timestamp.
20# Git does not preserve file timestamps on checkout.
Brad Bishop316dfdd2018-06-25 12:45:53 -040021#
22# 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ...
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080023# This works for well-kept repositories distributed via tarball.
Brad Bishop316dfdd2018-06-25 12:45:53 -040024#
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080025# 4. If the above steps fail, use the modification time of the youngest file in the source tree.
Brad Bishop316dfdd2018-06-25 12:45:53 -040026#
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080027# Once the value of SOURCE_DATE_EPOCH is determined, it is stored in the recipe's SDE_FILE.
28# If none of these mechanisms are suitable, replace the do_deploy_source_date_epoch task
29# with recipe-specific functionality to write the appropriate SOURCE_DATE_EPOCH into the SDE_FILE.
30#
31# If this file is found by other tasks, the value is exported in the SOURCE_DATE_EPOCH variable.
32# SOURCE_DATE_EPOCH is set for all tasks that might use it (do_configure, do_compile, do_package, ...)
Brad Bishop316dfdd2018-06-25 12:45:53 -040033
34BUILD_REPRODUCIBLE_BINARIES ??= '1'
35inherit ${@oe.utils.ifelse(d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1', 'reproducible_build_simple', '')}
36
37SDE_DIR ="${WORKDIR}/source-date-epoch"
38SDE_FILE = "${SDE_DIR}/__source_date_epoch.txt"
39
40SSTATETASKS += "do_deploy_source_date_epoch"
41
42do_deploy_source_date_epoch () {
43 echo "Deploying SDE to ${SDE_DIR}."
44}
45
46python do_deploy_source_date_epoch_setscene () {
47 sstate_setscene(d)
48}
49
50do_deploy_source_date_epoch[dirs] = "${SDE_DIR}"
51do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DIR}"
52addtask do_deploy_source_date_epoch_setscene
53addtask do_deploy_source_date_epoch before do_configure after do_patch
54
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080055def get_source_date_epoch_from_known_files(d, sourcedir):
56 source_date_epoch = None
57 newest_file = None
Brad Bishop316dfdd2018-06-25 12:45:53 -040058 known_files = set(["NEWS", "ChangeLog", "Changelog", "CHANGES"])
59 for file in known_files:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080060 filepath = os.path.join(sourcedir, file)
Brad Bishop316dfdd2018-06-25 12:45:53 -040061 if os.path.isfile(filepath):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080062 mtime = int(os.lstat(filepath).st_mtime)
Brad Bishop316dfdd2018-06-25 12:45:53 -040063 # There may be more than one "known_file" present, if so, use the youngest one
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080064 if not source_date_epoch or mtime > source_date_epoch:
Brad Bishop316dfdd2018-06-25 12:45:53 -040065 source_date_epoch = mtime
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080066 newest_file = filepath
67 if newest_file:
68 bb.debug(1, "SOURCE_DATE_EPOCH taken from: %s" % newest_file)
Brad Bishop316dfdd2018-06-25 12:45:53 -040069 return source_date_epoch
70
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080071def find_git_folder(d, sourcedir):
72 # First guess: WORKDIR/git
73 # This is the default git fetcher unpack path
74 workdir = d.getVar('WORKDIR')
75 gitpath = os.path.join(workdir, "git/.git")
76 if os.path.isdir(gitpath):
77 return gitpath
78
79 # Second guess: ${S}
80 gitpath = os.path.join(sourcedir, ".git")
81 if os.path.isdir(gitpath):
82 return gitpath
83
84 # Perhaps there was a subpath or destsuffix specified.
85 # Go looking in the WORKDIR
86 exclude = set(["build", "image", "license-destdir", "patches", "pseudo",
87 "recipe-sysroot", "recipe-sysroot-native", "sysroot-destdir", "temp"])
88 for root, dirs, files in os.walk(workdir, topdown=True):
Brad Bishop316dfdd2018-06-25 12:45:53 -040089 dirs[:] = [d for d in dirs if d not in exclude]
90 if '.git' in dirs:
Brad Bishop316dfdd2018-06-25 12:45:53 -040091 return root
Brad Bishop316dfdd2018-06-25 12:45:53 -040092
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080093 bb.warn("Failed to find a git repository in WORKDIR: %s" % workdir)
94 return None
95
96def get_source_date_epoch_from_git(d, sourcedir):
97 source_date_epoch = None
98 if "git://" in d.getVar('SRC_URI'):
99 gitpath = find_git_folder(d, sourcedir)
100 if gitpath:
101 import subprocess
102 source_date_epoch = int(subprocess.check_output(['git','log','-1','--pretty=%ct'], cwd=gitpath))
103 bb.debug(1, "git repository: %s" % gitpath)
104 return source_date_epoch
105
106def get_source_date_epoch_from_youngest_file(d, sourcedir):
107 # Do it the hard way: check all files and find the youngest one...
108 source_date_epoch = None
109 newest_file = None
110 # Just in case S = WORKDIR
111 exclude = set(["build", "image", "license-destdir", "patches", "pseudo",
112 "recipe-sysroot", "recipe-sysroot-native", "sysroot-destdir", "temp"])
113 for root, dirs, files in os.walk(sourcedir, topdown=True):
114 files = [f for f in files if not f[0] == '.']
115 dirs[:] = [d for d in dirs if d not in exclude]
116
117 for fname in files:
118 filename = os.path.join(root, fname)
119 try:
120 mtime = int(os.lstat(filename).st_mtime)
121 except ValueError:
122 mtime = 0
123 if not source_date_epoch or mtime > source_date_epoch:
124 source_date_epoch = mtime
125 newest_file = filename
126
127 if newest_file:
128 bb.debug(1, "Newest file found: %s" % newest_file)
129 return source_date_epoch
130
131python do_create_source_date_epoch_stamp() {
Brad Bishop316dfdd2018-06-25 12:45:53 -0400132 epochfile = d.getVar('SDE_FILE')
133 if os.path.isfile(epochfile):
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800134 bb.debug(1, "Reusing SOURCE_DATE_EPOCH from: %s" % epochfile)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400135 return
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800136
137 sourcedir = d.getVar('S')
138 source_date_epoch = (
139 get_source_date_epoch_from_git(d, sourcedir) or
140 get_source_date_epoch_from_known_files(d, sourcedir) or
141 get_source_date_epoch_from_youngest_file(d, sourcedir) or
142 0 # Last resort
143 )
Brad Bishop316dfdd2018-06-25 12:45:53 -0400144 if source_date_epoch == 0:
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800145 # empty folder, not a single file ...
146 bb.debug(1, "No files found to determine SOURCE_DATE_EPOCH")
Brad Bishop316dfdd2018-06-25 12:45:53 -0400147
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800148 bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400149 bb.utils.mkdirhier(d.getVar('SDE_DIR'))
150 with open(epochfile, 'w') as f:
151 f.write(str(source_date_epoch))
152}
153
154BB_HASHBASE_WHITELIST += "SOURCE_DATE_EPOCH"
155
156python () {
157 if d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1':
158 d.appendVarFlag("do_unpack", "postfuncs", " do_create_source_date_epoch_stamp")
159 epochfile = d.getVar('SDE_FILE')
160 source_date_epoch = "0"
161 if os.path.isfile(epochfile):
162 with open(epochfile, 'r') as f:
163 source_date_epoch = f.read()
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800164 bb.debug(1, "SOURCE_DATE_EPOCH: %s" % source_date_epoch)
Brad Bishop316dfdd2018-06-25 12:45:53 -0400165 d.setVar('SOURCE_DATE_EPOCH', source_date_epoch)
166}