blob: 971a5add4a5101194e3dafc18fb28bb58d9c9b8f [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001"""
2BitBake 'Fetch' implementation for svn.
3
4"""
5
6# Copyright (C) 2003, 2004 Chris Larson
7# Copyright (C) 2004 Marcin Juszkiewicz
8#
Brad Bishopc342db32019-05-15 21:57:59 -04009# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050010#
11# Based on functions from the base bb module, Copyright 2003 Holger Schurig
12
13import os
Patrick Williamsc124f4f2015-09-15 14:41:29 -050014import bb
15import re
Patrick Williamsc124f4f2015-09-15 14:41:29 -050016from bb.fetch2 import FetchMethod
17from bb.fetch2 import FetchError
18from bb.fetch2 import MissingParameterError
19from bb.fetch2 import runfetchcmd
20from bb.fetch2 import logger
21
22class Svn(FetchMethod):
23 """Class to fetch a module or modules from svn repositories"""
24 def supports(self, ud, d):
25 """
26 Check to see if a given url can be fetched with svn.
27 """
28 return ud.type in ['svn']
29
30 def urldata_init(self, ud, d):
31 """
32 init svn specific variable within url data
33 """
34 if not "module" in ud.parm:
35 raise MissingParameterError('module', ud.url)
36
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080037 ud.basecmd = d.getVar("FETCHCMD_svn") or "/usr/bin/env svn --non-interactive --trust-server-cert"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050038
39 ud.module = ud.parm["module"]
40
Patrick Williamsf1e5d692016-03-30 15:21:19 -050041 if not "path_spec" in ud.parm:
42 ud.path_spec = ud.module
43 else:
44 ud.path_spec = ud.parm["path_spec"]
45
Patrick Williamsc124f4f2015-09-15 14:41:29 -050046 # Create paths to svn checkouts
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080047 svndir = d.getVar("SVNDIR") or (d.getVar("DL_DIR") + "/svn")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050048 relpath = self._strip_leading_slashes(ud.path)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080049 ud.pkgdir = os.path.join(svndir, ud.host, relpath)
Andrew Geissler82c905d2020-04-13 13:39:40 -050050 ud.moddir = os.path.join(ud.pkgdir, ud.path_spec)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080051 # Protects the repository from concurrent updates, e.g. from two
52 # recipes fetching different revisions at the same time
53 ud.svnlock = os.path.join(ud.pkgdir, "svn.lock")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050054
Brad Bishop6e60e8b2018-02-01 10:27:11 -050055 ud.setup_revisions(d)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050056
57 if 'rev' in ud.parm:
58 ud.revision = ud.parm['rev']
59
Brad Bishop6e60e8b2018-02-01 10:27:11 -050060 ud.localfile = d.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision))
Patrick Williamsc124f4f2015-09-15 14:41:29 -050061
62 def _buildsvncommand(self, ud, d, command):
63 """
64 Build up an svn commandline based on ud
65 command is "fetch", "update", "info"
66 """
67
68 proto = ud.parm.get('protocol', 'svn')
69
Brad Bishop6e60e8b2018-02-01 10:27:11 -050070 svn_ssh = None
71 if proto == "svn+ssh" and "ssh" in ud.parm:
72 svn_ssh = ud.parm["ssh"]
Patrick Williamsc124f4f2015-09-15 14:41:29 -050073
74 svnroot = ud.host + ud.path
75
76 options = []
77
78 options.append("--no-auth-cache")
79
80 if ud.user:
81 options.append("--username %s" % ud.user)
82
83 if ud.pswd:
84 options.append("--password %s" % ud.pswd)
85
86 if command == "info":
87 svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
88 elif command == "log1":
89 svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
90 else:
91 suffix = ""
Brad Bishop15ae2502019-06-18 21:44:24 -040092
93 # externals may be either 'allowed' or 'nowarn', but not both. Allowed
94 # will not issue a warning, but will log to the debug buffer what has likely
95 # been downloaded by SVN.
96 if not ("externals" in ud.parm and ud.parm["externals"] == "allowed"):
97 options.append("--ignore-externals")
98
Patrick Williamsc124f4f2015-09-15 14:41:29 -050099 if ud.revision:
100 options.append("-r %s" % ud.revision)
101 suffix = "@%s" % (ud.revision)
102
103 if command == "fetch":
104 transportuser = ud.parm.get("transportuser", "")
Patrick Williamsf1e5d692016-03-30 15:21:19 -0500105 svncmd = "%s co %s %s://%s%s/%s%s %s" % (ud.basecmd, " ".join(options), proto, transportuser, svnroot, ud.module, suffix, ud.path_spec)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500106 elif command == "update":
107 svncmd = "%s update %s" % (ud.basecmd, " ".join(options))
108 else:
109 raise FetchError("Invalid svn command %s" % command, ud.url)
110
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500111 if svn_ssh:
112 svncmd = "SVN_SSH=\"%s\" %s" % (svn_ssh, svncmd)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500113
114 return svncmd
115
116 def download(self, ud, d):
117 """Fetch url"""
118
119 logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
120
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800121 lf = bb.utils.lockfile(ud.svnlock)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500122
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800123 try:
124 if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
Andrew Geissler82c905d2020-04-13 13:39:40 -0500125 svncmd = self._buildsvncommand(ud, d, "update")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800126 logger.info("Update " + ud.url)
127 # We need to attempt to run svn upgrade first in case its an older working format
128 try:
129 runfetchcmd(ud.basecmd + " upgrade", d, workdir=ud.moddir)
130 except FetchError:
131 pass
Andrew Geissler82c905d2020-04-13 13:39:40 -0500132 logger.debug(1, "Running %s", svncmd)
133 bb.fetch2.check_network_access(d, svncmd, ud.url)
134 runfetchcmd(svncmd, d, workdir=ud.moddir)
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800135 else:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500136 svncmd = self._buildsvncommand(ud, d, "fetch")
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800137 logger.info("Fetch " + ud.url)
138 # check out sources there
139 bb.utils.mkdirhier(ud.pkgdir)
Andrew Geissler82c905d2020-04-13 13:39:40 -0500140 logger.debug(1, "Running %s", svncmd)
141 bb.fetch2.check_network_access(d, svncmd, ud.url)
142 runfetchcmd(svncmd, d, workdir=ud.pkgdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500143
Brad Bishop15ae2502019-06-18 21:44:24 -0400144 if not ("externals" in ud.parm and ud.parm["externals"] == "nowarn"):
145 # Warn the user if this had externals (won't catch them all)
Brad Bishopa34c0302019-09-23 22:34:48 -0400146 output = runfetchcmd("svn propget svn:externals || true", d, workdir=ud.moddir)
Brad Bishop15ae2502019-06-18 21:44:24 -0400147 if output:
Andrew Geissler82c905d2020-04-13 13:39:40 -0500148 if "--ignore-externals" in svncmd.split():
Brad Bishop15ae2502019-06-18 21:44:24 -0400149 bb.warn("%s contains svn:externals." % ud.url)
150 bb.warn("These should be added to the recipe SRC_URI as necessary.")
151 bb.warn("svn fetch has ignored externals:\n%s" % output)
152 bb.warn("To disable this warning add ';externals=nowarn' to the url.")
153 else:
154 bb.debug(1, "svn repository has externals:\n%s" % output)
155
Brad Bishop1a4b7ee2018-12-16 17:11:34 -0800156 scmdata = ud.parm.get("scmdata", "")
157 if scmdata == "keep":
158 tar_flags = ""
159 else:
160 tar_flags = "--exclude='.svn'"
161
162 # tar them up to a defined filename
163 runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.path_spec), d,
164 cleanup=[ud.localpath], workdir=ud.pkgdir)
165 finally:
166 bb.utils.unlockfile(lf)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500167
168 def clean(self, ud, d):
169 """ Clean SVN specific files and dirs """
170
171 bb.utils.remove(ud.localpath)
172 bb.utils.remove(ud.moddir, True)
173
174
175 def supports_srcrev(self):
176 return True
177
178 def _revision_key(self, ud, d, name):
179 """
180 Return a unique key for the url
181 """
182 return "svn:" + ud.moddir
183
184 def _latest_revision(self, ud, d, name):
185 """
186 Return the latest upstream revision number
187 """
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500188 bb.fetch2.check_network_access(d, self._buildsvncommand(ud, d, "log1"), ud.url)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500189
190 output = runfetchcmd("LANG=C LC_ALL=C " + self._buildsvncommand(ud, d, "log1"), d, True)
191
192 # skip the first line, as per output of svn log
193 # then we expect the revision on the 2nd line
194 revision = re.search('^r([0-9]*)', output.splitlines()[1]).group(1)
195
196 return revision
197
198 def sortable_revision(self, ud, d, name):
199 """
200 Return a sortable revision number which in our case is the revision number
201 """
202
203 return False, self._build_revision(ud, d)
204
205 def _build_revision(self, ud, d):
206 return ud.revision