Andrew Geissler | 0903674 | 2021-06-25 14:25:14 -0500 | [diff] [blame] | 1 | From 27ed9962f5cb3afcc44d6c96c53277132a999712 Mon Sep 17 00:00:00 2001 |
| 2 | From: Alexander Kanavin <alex.kanavin@gmail.com> |
| 3 | Date: Mon, 14 Jun 2021 19:57:30 +0200 |
| 4 | Subject: [PATCH 6/6] reproducible |
| 5 | |
William A. Kennington III | ac69b48 | 2021-06-02 12:28:27 -0700 | [diff] [blame] | 6 | This patch fixes various things which make the build more reproducible. Some changes |
| 7 | here only change intermediate artefacts but that means when you have two build trees |
| 8 | giving differing results, the differences can be isolated more easily. The issues here |
| 9 | usually become apparent with longer paths. |
| 10 | |
| 11 | This was all debugged with: |
| 12 | TMPDIR = "${TOPDIR}/tmp" |
| 13 | vs. |
| 14 | TMPDIR = "${TOPDIR}/tmp-inital-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath-mylongpath" |
| 15 | |
| 16 | The patch specifically: |
| 17 | |
| 18 | * Sorts output in GNUmakefile |
Andrew Geissler | 0903674 | 2021-06-25 14:25:14 -0500 | [diff] [blame] | 19 | * Always generates indirect flags files used to avoid pathlength issues else the |
William A. Kennington III | ac69b48 | 2021-06-02 12:28:27 -0700 | [diff] [blame] | 20 | compile commands suddenly change when using longer paths |
| 21 | * Sorts the AutoGenTimeStamp file contents |
| 22 | * Makes the TargetDescBlock objects from BuildEngine sortable to allow the makefile fix |
Andrew Geissler | 0903674 | 2021-06-25 14:25:14 -0500 | [diff] [blame] | 23 | * Fix ElfConvert within GenFw so that only the basename of the binary being converted |
William A. Kennington III | ac69b48 | 2021-06-02 12:28:27 -0700 | [diff] [blame] | 24 | is used, else the output from "GenFw XXX.bin" differs from "GenFw /long/path/XXX.bin" |
| 25 | with sufficiently long paths |
| 26 | |
| 27 | Upstream-Status: Pending [At least some of this might be interesting to upstream] |
| 28 | Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> |
Andrew Geissler | 0903674 | 2021-06-25 14:25:14 -0500 | [diff] [blame] | 29 | Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com> |
| 30 | --- |
| 31 | BaseTools/Source/C/GenFw/Elf64Convert.c | 8 ++++--- |
| 32 | .../Source/Python/AutoGen/BuildEngine.py | 3 +++ |
| 33 | BaseTools/Source/Python/AutoGen/GenMake.py | 24 +++++++++---------- |
| 34 | .../Source/Python/AutoGen/ModuleAutoGen.py | 5 +++- |
| 35 | 4 files changed, 24 insertions(+), 16 deletions(-) |
William A. Kennington III | ac69b48 | 2021-06-02 12:28:27 -0700 | [diff] [blame] | 36 | |
Andrew Geissler | 0903674 | 2021-06-25 14:25:14 -0500 | [diff] [blame] | 37 | diff --git a/BaseTools/Source/C/GenFw/Elf64Convert.c b/BaseTools/Source/C/GenFw/Elf64Convert.c |
| 38 | index d097db8632..a87ae6f3d0 100644 |
| 39 | --- a/BaseTools/Source/C/GenFw/Elf64Convert.c |
| 40 | +++ b/BaseTools/Source/C/GenFw/Elf64Convert.c |
| 41 | @@ -14,6 +14,8 @@ SPDX-License-Identifier: BSD-2-Clause-Patent |
| 42 | #ifndef __GNUC__
|
| 43 | #include <windows.h>
|
| 44 | #include <io.h>
|
| 45 | +#else
|
| 46 | +#define _GNU_SOURCE
|
| 47 | #endif
|
| 48 | #include <assert.h>
|
| 49 | #include <stdio.h>
|
| 50 | @@ -769,7 +771,7 @@ ScanSections64 ( |
| 51 | }
|
| 52 | mCoffOffset = mDebugOffset + sizeof(EFI_IMAGE_DEBUG_DIRECTORY_ENTRY) +
|
| 53 | sizeof(EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY) +
|
| 54 | - strlen(mInImageName) + 1;
|
| 55 | + strlen(basename(mInImageName)) + 1;
|
| 56 |
|
| 57 | mCoffOffset = CoffAlign(mCoffOffset);
|
| 58 | if (SectionCount == 0) {
|
| 59 | @@ -1608,7 +1610,7 @@ WriteDebug64 ( |
| 60 | EFI_IMAGE_DEBUG_DIRECTORY_ENTRY *Dir;
|
| 61 | EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY *Nb10;
|
| 62 |
|
| 63 | - Len = strlen(mInImageName) + 1;
|
| 64 | + Len = strlen(basename(mInImageName)) + 1;
|
| 65 |
|
| 66 | Dir = (EFI_IMAGE_DEBUG_DIRECTORY_ENTRY*)(mCoffFile + mDebugOffset);
|
| 67 | Dir->Type = EFI_IMAGE_DEBUG_TYPE_CODEVIEW;
|
| 68 | @@ -1618,7 +1620,7 @@ WriteDebug64 ( |
| 69 |
|
| 70 | Nb10 = (EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY*)(Dir + 1);
|
| 71 | Nb10->Signature = CODEVIEW_SIGNATURE_NB10;
|
| 72 | - strcpy ((char *)(Nb10 + 1), mInImageName);
|
| 73 | + strcpy ((char *)(Nb10 + 1), basename(mInImageName));
|
| 74 |
|
| 75 |
|
| 76 | NtHdr = (EFI_IMAGE_OPTIONAL_HEADER_UNION *)(mCoffFile + mNtHdrOffset);
|
| 77 | diff --git a/BaseTools/Source/Python/AutoGen/BuildEngine.py b/BaseTools/Source/Python/AutoGen/BuildEngine.py |
| 78 | index 722fead75a..8f1c236970 100644 |
| 79 | --- a/BaseTools/Source/Python/AutoGen/BuildEngine.py |
| 80 | +++ b/BaseTools/Source/Python/AutoGen/BuildEngine.py |
| 81 | @@ -70,6 +70,9 @@ class TargetDescBlock(object): |
| 82 | else:
|
| 83 | return str(Other) == self.Target.Path
|
| 84 |
|
| 85 | + def __lt__(self, other):
|
| 86 | + return str(self) < str(other)
|
| 87 | +
|
| 88 | def AddInput(self, Input):
|
| 89 | if Input not in self.Inputs:
|
| 90 | self.Inputs.append(Input)
|
| 91 | diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py |
| 92 | index 961b2ab1c3..23c1592025 100755 |
| 93 | --- a/BaseTools/Source/Python/AutoGen/GenMake.py |
| 94 | +++ b/BaseTools/Source/Python/AutoGen/GenMake.py |
| 95 | @@ -575,7 +575,7 @@ cleanlib: |
William A. Kennington III | ac69b48 | 2021-06-02 12:28:27 -0700 | [diff] [blame] | 96 | os.remove(RespFileList)
|
| 97 |
|
| 98 | # convert source files and binary files to build targets
|
| 99 | - self.ResultFileList = [str(T.Target) for T in MyAgo.CodaTargetList]
|
| 100 | + self.ResultFileList = sorted([str(T.Target) for T in MyAgo.CodaTargetList])
|
| 101 | if len(self.ResultFileList) == 0 and len(MyAgo.SourceFileList) != 0:
|
| 102 | EdkLogger.error("build", AUTOGEN_ERROR, "Nothing to build",
|
| 103 | ExtraData="[%s]" % str(MyAgo))
|
Andrew Geissler | 0903674 | 2021-06-25 14:25:14 -0500 | [diff] [blame] | 104 | @@ -726,7 +726,7 @@ cleanlib: |
William A. Kennington III | ac69b48 | 2021-06-02 12:28:27 -0700 | [diff] [blame] | 105 | OutputFile = ''
|
| 106 | DepsFileList = []
|
| 107 |
|
| 108 | - for Cmd in self.GenFfsList:
|
| 109 | + for Cmd in sorted(self.GenFfsList):
|
| 110 | if Cmd[2]:
|
| 111 | for CopyCmd in Cmd[2]:
|
| 112 | Src, Dst = CopyCmd
|
Andrew Geissler | 0903674 | 2021-06-25 14:25:14 -0500 | [diff] [blame] | 113 | @@ -759,7 +759,7 @@ cleanlib: |
William A. Kennington III | ac69b48 | 2021-06-02 12:28:27 -0700 | [diff] [blame] | 114 | self.BuildTargetList.append('\t%s' % CmdString)
|
| 115 |
|
| 116 | self.ParseSecCmd(DepsFileList, Cmd[1])
|
| 117 | - for SecOutputFile, SecDepsFile, SecCmd in self.FfsOutputFileList :
|
| 118 | + for SecOutputFile, SecDepsFile, SecCmd in sorted(self.FfsOutputFileList):
|
| 119 | self.BuildTargetList.append('%s : %s' % (self.ReplaceMacro(SecOutputFile), self.ReplaceMacro(SecDepsFile)))
|
| 120 | self.BuildTargetList.append('\t%s' % self.ReplaceMacro(SecCmd))
|
| 121 | self.FfsOutputFileList = []
|
Andrew Geissler | 0903674 | 2021-06-25 14:25:14 -0500 | [diff] [blame] | 122 | @@ -798,13 +798,13 @@ cleanlib: |
William A. Kennington III | ac69b48 | 2021-06-02 12:28:27 -0700 | [diff] [blame] | 123 |
|
| 124 | def CommandExceedLimit(self):
|
| 125 | FlagDict = {
|
| 126 | - 'CC' : { 'Macro' : '$(CC_FLAGS)', 'Value' : False},
|
| 127 | - 'PP' : { 'Macro' : '$(PP_FLAGS)', 'Value' : False},
|
| 128 | - 'APP' : { 'Macro' : '$(APP_FLAGS)', 'Value' : False},
|
| 129 | - 'ASLPP' : { 'Macro' : '$(ASLPP_FLAGS)', 'Value' : False},
|
| 130 | - 'VFRPP' : { 'Macro' : '$(VFRPP_FLAGS)', 'Value' : False},
|
| 131 | - 'ASM' : { 'Macro' : '$(ASM_FLAGS)', 'Value' : False},
|
| 132 | - 'ASLCC' : { 'Macro' : '$(ASLCC_FLAGS)', 'Value' : False},
|
| 133 | + 'CC' : { 'Macro' : '$(CC_FLAGS)', 'Value' : True},
|
| 134 | + 'PP' : { 'Macro' : '$(PP_FLAGS)', 'Value' : True},
|
| 135 | + 'APP' : { 'Macro' : '$(APP_FLAGS)', 'Value' : True},
|
| 136 | + 'ASLPP' : { 'Macro' : '$(ASLPP_FLAGS)', 'Value' : True},
|
| 137 | + 'VFRPP' : { 'Macro' : '$(VFRPP_FLAGS)', 'Value' : True},
|
| 138 | + 'ASM' : { 'Macro' : '$(ASM_FLAGS)', 'Value' : True},
|
| 139 | + 'ASLCC' : { 'Macro' : '$(ASLCC_FLAGS)', 'Value' : True},
|
| 140 | }
|
| 141 |
|
| 142 | RespDict = {}
|
Andrew Geissler | 0903674 | 2021-06-25 14:25:14 -0500 | [diff] [blame] | 143 | @@ -1007,9 +1007,9 @@ cleanlib: |
William A. Kennington III | ac69b48 | 2021-06-02 12:28:27 -0700 | [diff] [blame] | 144 | if not self.ObjTargetDict.get(T.Target.SubDir):
|
| 145 | self.ObjTargetDict[T.Target.SubDir] = set()
|
| 146 | self.ObjTargetDict[T.Target.SubDir].add(NewFile)
|
| 147 | - for Type in self._AutoGenObject.Targets:
|
| 148 | + for Type in sorted(self._AutoGenObject.Targets):
|
| 149 | resp_file_number = 0
|
| 150 | - for T in self._AutoGenObject.Targets[Type]:
|
| 151 | + for T in sorted(self._AutoGenObject.Targets[Type]):
|
| 152 | # Generate related macros if needed
|
| 153 | if T.GenFileListMacro and T.FileListMacro not in self.FileListMacros:
|
| 154 | self.FileListMacros[T.FileListMacro] = []
|
Andrew Geissler | 0903674 | 2021-06-25 14:25:14 -0500 | [diff] [blame] | 155 | diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py |
| 156 | index d70b0d7ae8..25dca9a6df 100755 |
| 157 | --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py |
| 158 | +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py |
William A. Kennington III | ac69b48 | 2021-06-02 12:28:27 -0700 | [diff] [blame] | 159 | @@ -1484,6 +1484,9 @@ class ModuleAutoGen(AutoGen): |
| 160 | for File in Files:
|
| 161 | if File.lower().endswith('.pdb'):
|
| 162 | AsBuiltInfDict['binary_item'].append('DISPOSABLE|' + File)
|
| 163 | +
|
| 164 | + AsBuiltInfDict['binary_item'] = sorted(AsBuiltInfDict['binary_item'])
|
| 165 | +
|
| 166 | HeaderComments = self.Module.HeaderComments
|
| 167 | StartPos = 0
|
| 168 | for Index in range(len(HeaderComments)):
|
| 169 | @@ -1759,7 +1762,7 @@ class ModuleAutoGen(AutoGen): |
| 170 | if os.path.exists (self.TimeStampPath):
|
| 171 | os.remove (self.TimeStampPath)
|
| 172 |
|
| 173 | - SaveFileOnChange(self.TimeStampPath, "\n".join(FileSet), False)
|
| 174 | + SaveFileOnChange(self.TimeStampPath, "\n".join(sorted(FileSet)), False)
|
| 175 |
|
| 176 | # Ignore generating makefile when it is a binary module
|
| 177 | if self.IsBinaryModule:
|
Andrew Geissler | 0903674 | 2021-06-25 14:25:14 -0500 | [diff] [blame] | 178 | -- |
| 179 | 2.32.0 |
| 180 | |