blob: 1bbb99ac79426241a64813f3413cb3eb89b1a994 [file] [log] [blame]
Patrick Williamsac13d5f2023-11-24 18:59:46 -06001#
2# Copyright 2023 (C) Weidmueller GmbH & Co KG
3# Author: Lukas Funke <lukas.funke@weidmueller.com>
4#
5# Handle Go vendor support for offline builds
6#
7# When importing Go modules, Go downloads the imported modules using
8# a network (proxy) connection ahead of the compile stage. This contradicts
9# the yocto build concept of fetching every source ahead of build-time
10# and supporting offline builds.
11#
12# To support offline builds, we use Go 'vendoring': module dependencies are
13# downloaded during the fetch-phase and unpacked into the modules 'vendor'
14# folder. Additionally a manifest file is generated for the 'vendor' folder
15#
16
17inherit go-mod
18
19def go_src_uri(repo, version, path=None, subdir=None, \
20 vcs='git', replaces=None, pathmajor=None):
21
22 destsuffix = "git/src/import/vendor.fetch"
23 module_path = repo if not path else path
24
25 src_uri = "{}://{};name={}".format(vcs, repo, module_path.replace('/', '.'))
26 src_uri += ";destsuffix={}/{}@{}".format(destsuffix, repo, version)
27
28 if vcs == "git":
29 src_uri += ";nobranch=1;protocol=https"
30
31 src_uri += ";go_module_path={}".format(module_path)
32
33 if replaces:
34 src_uri += ";go_module_replacement={}".format(replaces)
35 if subdir:
36 src_uri += ";go_subdir={}".format(subdir)
37 if pathmajor:
38 src_uri += ";go_pathmajor={}".format(pathmajor)
39 src_uri += ";is_go_dependency=1"
40
41 return src_uri
42
43python do_vendor_unlink() {
44 go_import = d.getVar('GO_IMPORT')
45 source_dir = d.getVar('S')
46 linkname = os.path.join(source_dir, *['src', go_import, 'vendor'])
47
48 os.unlink(linkname)
49}
50
Patrick Williams56b44a92024-01-19 08:49:29 -060051addtask vendor_unlink before do_package after do_install
Patrick Williamsac13d5f2023-11-24 18:59:46 -060052
53python do_go_vendor() {
54 import shutil
55
56 src_uri = (d.getVar('SRC_URI') or "").split()
57
58 if not src_uri:
59 bb.fatal("SRC_URI is empty")
60
61 default_destsuffix = "git/src/import/vendor.fetch"
62 fetcher = bb.fetch2.Fetch(src_uri, d)
63 go_import = d.getVar('GO_IMPORT')
64 source_dir = d.getVar('S')
65
66 linkname = os.path.join(source_dir, *['src', go_import, 'vendor'])
67 vendor_dir = os.path.join(source_dir, *['src', 'import', 'vendor'])
68 import_dir = os.path.join(source_dir, *['src', 'import', 'vendor.fetch'])
69
70 if os.path.exists(vendor_dir):
71 # Nothing to do except re-establish link to actual vendor folder
72 if not os.path.exists(linkname):
73 os.symlink(vendor_dir, linkname)
74 return
75
76 bb.utils.mkdirhier(vendor_dir)
77
78 modules = {}
79
80 for url in fetcher.urls:
81 srcuri = fetcher.ud[url].host + fetcher.ud[url].path
82
83 # Skip non Go module src uris
84 if not fetcher.ud[url].parm.get('is_go_dependency'):
85 continue
86
87 destsuffix = fetcher.ud[url].parm.get('destsuffix')
88 # We derive the module repo / version in the following manner (exmaple):
89 #
90 # destsuffix = git/src/import/vendor.fetch/github.com/foo/bar@v1.2.3
91 # p = github.com/foo/bar@v1.2.3
92 # repo = github.com/foo/bar
93 # version = v1.2.3
94
95 p = destsuffix[len(default_destsuffix)+1:]
96 repo, version = p.split('@')
97
98 module_path = fetcher.ud[url].parm.get('go_module_path')
99
100 subdir = fetcher.ud[url].parm.get('go_subdir')
101 subdir = None if not subdir else subdir
102
103 pathMajor = fetcher.ud[url].parm.get('go_pathmajor')
104 pathMajor = None if not pathMajor else pathMajor.strip('/')
105
Patrick Williams56b44a92024-01-19 08:49:29 -0600106 if not (repo, version) in modules:
107 modules[(repo, version)] = {
Patrick Williamsac13d5f2023-11-24 18:59:46 -0600108 "repo_path": os.path.join(import_dir, p),
109 "module_path": module_path,
110 "subdir": subdir,
111 "pathMajor": pathMajor }
112
Patrick Williams56b44a92024-01-19 08:49:29 -0600113 for module_key, module in modules.items():
Patrick Williamsac13d5f2023-11-24 18:59:46 -0600114
115 # only take the version which is explicitly listed
116 # as a dependency in the go.mod
Patrick Williamsac13d5f2023-11-24 18:59:46 -0600117 module_path = module['module_path']
118 rootdir = module['repo_path']
119 subdir = module['subdir']
120 pathMajor = module['pathMajor']
121
122 src = rootdir
123
124 if subdir:
125 src = os.path.join(rootdir, subdir)
126
127 # If the module is released at major version 2 or higher, the module
128 # path must end with a major version suffix like /v2.
129 # This may or may not be part of the subdirectory name
130 #
131 # https://go.dev/ref/mod#modules-overview
132 if pathMajor:
133 tmp = os.path.join(src, pathMajor)
134 # source directory including major version path may or may not exist
135 if os.path.exists(tmp):
136 src = tmp
137
138 dst = os.path.join(vendor_dir, module_path)
139
140 bb.debug(1, "cp %s --> %s" % (src, dst))
Patrick Williams56b44a92024-01-19 08:49:29 -0600141 shutil.copytree(src, dst, symlinks=True, dirs_exist_ok=True, \
Patrick Williamsac13d5f2023-11-24 18:59:46 -0600142 ignore=shutil.ignore_patterns(".git", \
143 "vendor", \
144 "*._test.go"))
145
146 # If the root directory has a LICENSE file but not the subdir
147 # we copy the root license to the sub module since the license
148 # applies to all modules in the repository
149 # see https://go.dev/ref/mod#vcs-license
150 if subdir:
151 rootdirLicese = os.path.join(rootdir, "LICENSE")
152 subdirLicense = os.path.join(src, "LICENSE")
153
154 if not os.path.exists(subdir) and \
155 os.path.exists(rootdirLicese):
156 shutil.copy2(rootdirLicese, subdirLicense)
157
158 # Copy vendor manifest
159 modules_txt_src = os.path.join(d.getVar('WORKDIR'), "modules.txt")
160 bb.debug(1, "cp %s --> %s" % (modules_txt_src, vendor_dir))
161 shutil.copy2(modules_txt_src, vendor_dir)
162
163 # Clean up vendor dir
164 # We only require the modules in the modules_txt file
165 fetched_paths = set([os.path.relpath(x[0], vendor_dir) for x in os.walk(vendor_dir)])
166
167 # Remove toplevel dir
168 fetched_paths.remove('.')
169
170 vendored_paths = set()
Patrick Williams56b44a92024-01-19 08:49:29 -0600171 replaced_paths = dict()
Patrick Williamsac13d5f2023-11-24 18:59:46 -0600172 with open(modules_txt_src) as f:
173 for line in f:
174 if not line.startswith("#"):
175 line = line.strip()
176 vendored_paths.add(line)
177
178 # Add toplevel dirs into vendored dir, as we want to keep them
179 topdir = os.path.dirname(line)
180 while len(topdir):
181 if not topdir in vendored_paths:
182 vendored_paths.add(topdir)
183
184 topdir = os.path.dirname(topdir)
Patrick Williams56b44a92024-01-19 08:49:29 -0600185 else:
186 replaced_module = line.split("=>")
187 if len(replaced_module) > 1:
188 # This module has been replaced, use a local path
189 # we parse the line that has a pattern "# module-name [module-version] => local-path
190 actual_path = replaced_module[1].strip()
191 vendored_name = replaced_module[0].split()[1]
192 bb.debug(1, "added vendored name %s for actual path %s" % (vendored_name, actual_path))
193 replaced_paths[vendored_name] = actual_path
Patrick Williamsac13d5f2023-11-24 18:59:46 -0600194
195 for path in fetched_paths:
196 if path not in vendored_paths:
197 realpath = os.path.join(vendor_dir, path)
198 if os.path.exists(realpath):
199 shutil.rmtree(realpath)
200
Patrick Williams56b44a92024-01-19 08:49:29 -0600201 for vendored_name, replaced_path in replaced_paths.items():
202 symlink_target = os.path.join(source_dir, *['src', go_import, replaced_path])
203 symlink_name = os.path.join(vendor_dir, vendored_name)
204 bb.debug(1, "vendored name %s, symlink name %s" % (vendored_name, symlink_name))
205 os.symlink(symlink_target, symlink_name)
206
207 # Create a symlink to the actual directory
Patrick Williamsac13d5f2023-11-24 18:59:46 -0600208 os.symlink(vendor_dir, linkname)
209}
210
211addtask go_vendor before do_patch after do_unpack