blob: e2a41a4a1287831f895aad644f0fce7424aec115 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001"""
Patrick Williamsc0f7c042017-02-23 20:41:17 -06002BitBake 'Fetch' implementation for perforce
Patrick Williamsc124f4f2015-09-15 14:41:29 -05003
Andrew Geisslerd25ed322020-06-27 00:28:28 -05004Supported SRC_URI options are:
5
6- module
7 The top-level location to fetch while preserving the remote paths
8
9 The value of module can point to either a directory or a file. The result,
10 in both cases, is that the fetcher will preserve all file paths starting
11 from the module path. That is, the top-level directory in the module value
12 will also be the top-level directory in P4DIR.
13
14- remotepath
15 If the value "keep" is given, the full depot location of each file is
16 preserved in P4DIR. This option overrides the effect of the module option.
17
Patrick Williamsc124f4f2015-09-15 14:41:29 -050018"""
19
20# Copyright (C) 2003, 2004 Chris Larson
Patrick Williamsc0f7c042017-02-23 20:41:17 -060021# Copyright (C) 2016 Kodak Alaris, Inc.
Patrick Williamsc124f4f2015-09-15 14:41:29 -050022#
Brad Bishopc342db32019-05-15 21:57:59 -040023# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -050024#
25# Based on functions from the base bb module, Copyright 2003 Holger Schurig
26
Patrick Williamsc124f4f2015-09-15 14:41:29 -050027import os
Patrick Williamsc124f4f2015-09-15 14:41:29 -050028import bb
Patrick Williamsc124f4f2015-09-15 14:41:29 -050029from bb.fetch2 import FetchMethod
30from bb.fetch2 import FetchError
31from bb.fetch2 import logger
32from bb.fetch2 import runfetchcmd
33
Andrew Geisslerd25ed322020-06-27 00:28:28 -050034class PerforceProgressHandler (bb.progress.BasicProgressHandler):
35 """
36 Implements basic progress information for perforce, based on the number of
37 files to be downloaded.
38
39 The p4 print command will print one line per file, therefore it can be used
40 to "count" the number of files already completed and give an indication of
41 the progress.
42 """
43 def __init__(self, d, num_files):
44 self._num_files = num_files
45 self._count = 0
46 super(PerforceProgressHandler, self).__init__(d)
47
48 # Send an initial progress event so the bar gets shown
49 self._fire_progress(-1)
50
51 def write(self, string):
52 self._count = self._count + 1
53
54 percent = int(100.0 * float(self._count) / float(self._num_files))
55
56 # In case something goes wrong, we try to preserve our sanity
57 if percent > 100:
58 percent = 100
59
60 self.update(percent)
61
62 super(PerforceProgressHandler, self).write(string)
63
Patrick Williamsc124f4f2015-09-15 14:41:29 -050064class Perforce(FetchMethod):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060065 """ Class to fetch from perforce repositories """
Patrick Williamsc124f4f2015-09-15 14:41:29 -050066 def supports(self, ud, d):
Patrick Williamsc0f7c042017-02-23 20:41:17 -060067 """ Check to see if a given url can be fetched with perforce. """
Patrick Williamsc124f4f2015-09-15 14:41:29 -050068 return ud.type in ['p4']
69
Patrick Williamsc124f4f2015-09-15 14:41:29 -050070 def urldata_init(self, ud, d):
Patrick Williamsc124f4f2015-09-15 14:41:29 -050071 """
Patrick Williamsc0f7c042017-02-23 20:41:17 -060072 Initialize perforce specific variables within url data. If P4CONFIG is
73 provided by the env, use it. If P4PORT is specified by the recipe, use
74 its values, which may override the settings in P4CONFIG.
Patrick Williamsc124f4f2015-09-15 14:41:29 -050075 """
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080076 ud.basecmd = d.getVar("FETCHCMD_p4") or "/usr/bin/env p4"
Patrick Williamsc124f4f2015-09-15 14:41:29 -050077
Brad Bishop1a4b7ee2018-12-16 17:11:34 -080078 ud.dldir = d.getVar("P4DIR") or (d.getVar("DL_DIR") + "/p4")
Patrick Williamsc124f4f2015-09-15 14:41:29 -050079
Patrick Williamsc0f7c042017-02-23 20:41:17 -060080 path = ud.url.split('://')[1]
81 path = path.split(';')[0]
82 delim = path.find('@');
83 if delim != -1:
84 (ud.user, ud.pswd) = path.split('@')[0].split(':')
85 ud.path = path.split('@')[1]
Patrick Williamsc124f4f2015-09-15 14:41:29 -050086 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -060087 ud.path = path
Patrick Williamsc124f4f2015-09-15 14:41:29 -050088
Patrick Williamsc0f7c042017-02-23 20:41:17 -060089 ud.usingp4config = False
Brad Bishop6e60e8b2018-02-01 10:27:11 -050090 p4port = d.getVar('P4PORT')
Patrick Williamsc124f4f2015-09-15 14:41:29 -050091
Patrick Williamsc0f7c042017-02-23 20:41:17 -060092 if p4port:
Andrew Geisslerd1e89492021-02-12 15:35:20 -060093 logger.debug('Using recipe provided P4PORT: %s' % p4port)
Patrick Williamsc0f7c042017-02-23 20:41:17 -060094 ud.host = p4port
95 else:
Andrew Geisslerd1e89492021-02-12 15:35:20 -060096 logger.debug('Trying to use P4CONFIG to automatically set P4PORT...')
Patrick Williamsc0f7c042017-02-23 20:41:17 -060097 ud.usingp4config = True
98 p4cmd = '%s info | grep "Server address"' % ud.basecmd
Brad Bishop6e60e8b2018-02-01 10:27:11 -050099 bb.fetch2.check_network_access(d, p4cmd, ud.url)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600100 ud.host = runfetchcmd(p4cmd, d, True)
101 ud.host = ud.host.split(': ')[1].strip()
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600102 logger.debug('Determined P4PORT to be: %s' % ud.host)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600103 if not ud.host:
104 raise FetchError('Could not determine P4PORT from P4CONFIG')
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500105
106 # Fetcher options
107 ud.module = ud.parm.get('module')
108 ud.keepremotepath = (ud.parm.get('remotepath', '') == 'keep')
109
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600110 if ud.path.find('/...') >= 0:
111 ud.pathisdir = True
112 else:
113 ud.pathisdir = False
114
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500115 # Avoid using the "/..." syntax in SRC_URI when a module value is given
116 if ud.pathisdir and ud.module:
117 raise FetchError('SRC_URI depot path cannot not end in /... when a module value is given')
118
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600119 cleanedpath = ud.path.replace('/...', '').replace('/', '.')
120 cleanedhost = ud.host.replace(':', '.')
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500121
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600122 cleanedmodule = ""
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500123 # Merge the path and module into the final depot location
124 if ud.module:
125 if ud.module.find('/') == 0:
126 raise FetchError('module cannot begin with /')
127 ud.path = os.path.join(ud.path, ud.module)
128
129 # Append the module path to the local pkg name
130 cleanedmodule = ud.module.replace('/...', '').replace('/', '.')
131 cleanedpath += '--%s' % cleanedmodule
132
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600133 ud.pkgdir = os.path.join(ud.dldir, cleanedhost, cleanedpath)
134
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500135 ud.setup_revisions(d)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600136
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600137 ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleandedmodule, ud.revision))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600138
139 def _buildp4command(self, ud, d, command, depot_filename=None):
140 """
141 Build a p4 commandline. Valid commands are "changes", "print", and
142 "files". depot_filename is the full path to the file in the depot
143 including the trailing '#rev' value.
144 """
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500145 p4opt = ""
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500146
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600147 if ud.user:
148 p4opt += ' -u "%s"' % (ud.user)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500149
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600150 if ud.pswd:
151 p4opt += ' -P "%s"' % (ud.pswd)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500152
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600153 if ud.host and not ud.usingp4config:
154 p4opt += ' -p %s' % (ud.host)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500155
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600156 if hasattr(ud, 'revision') and ud.revision:
157 pathnrev = '%s@%s' % (ud.path, ud.revision)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500158 else:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600159 pathnrev = '%s' % (ud.path)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500160
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600161 if depot_filename:
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500162 if ud.keepremotepath:
163 # preserve everything, remove the leading //
164 filename = depot_filename.lstrip('/')
165 elif ud.module:
166 # remove everything up to the module path
167 modulepath = ud.module.rstrip('/...')
168 filename = depot_filename[depot_filename.rfind(modulepath):]
169 elif ud.pathisdir:
170 # Remove leading (visible) path to obtain the filepath
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600171 filename = depot_filename[len(ud.path)-1:]
172 else:
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500173 # Remove everything, except the filename
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600174 filename = depot_filename[depot_filename.rfind('/'):]
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500175
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600176 filename = filename[:filename.find('#')] # Remove trailing '#rev'
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500177
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600178 if command == 'changes':
179 p4cmd = '%s%s changes -m 1 //%s' % (ud.basecmd, p4opt, pathnrev)
180 elif command == 'print':
Andrew Geissler82c905d2020-04-13 13:39:40 -0500181 if depot_filename is not None:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600182 p4cmd = '%s%s print -o "p4/%s" "%s"' % (ud.basecmd, p4opt, filename, depot_filename)
183 else:
184 raise FetchError('No depot file name provided to p4 %s' % command, ud.url)
185 elif command == 'files':
186 p4cmd = '%s%s files //%s' % (ud.basecmd, p4opt, pathnrev)
187 else:
188 raise FetchError('Invalid p4 command %s' % command, ud.url)
189
190 return p4cmd
191
192 def _p4listfiles(self, ud, d):
193 """
194 Return a list of the file names which are present in the depot using the
195 'p4 files' command, including trailing '#rev' file revision indicator
196 """
197 p4cmd = self._buildp4command(ud, d, 'files')
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500198 bb.fetch2.check_network_access(d, p4cmd, ud.url)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600199 p4fileslist = runfetchcmd(p4cmd, d, True)
200 p4fileslist = [f.rstrip() for f in p4fileslist.splitlines()]
201
202 if not p4fileslist:
203 raise FetchError('Unable to fetch listing of p4 files from %s@%s' % (ud.host, ud.path))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500204
205 count = 0
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600206 filelist = []
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500207
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600208 for filename in p4fileslist:
209 item = filename.split(' - ')
210 lastaction = item[1].split()
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600211 logger.debug('File: %s Last Action: %s' % (item[0], lastaction[0]))
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600212 if lastaction[0] == 'delete':
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500213 continue
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600214 filelist.append(item[0])
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500215
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600216 return filelist
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500217
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600218 def download(self, ud, d):
219 """ Get the list of files, fetch each one """
220 filelist = self._p4listfiles(ud, d)
221 if not filelist:
222 raise FetchError('No files found in depot %s@%s' % (ud.host, ud.path))
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500223
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600224 bb.utils.remove(ud.pkgdir, True)
225 bb.utils.mkdirhier(ud.pkgdir)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500226
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500227 progresshandler = PerforceProgressHandler(d, len(filelist))
228
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600229 for afile in filelist:
230 p4fetchcmd = self._buildp4command(ud, d, 'print', afile)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500231 bb.fetch2.check_network_access(d, p4fetchcmd, ud.url)
Andrew Geisslerd25ed322020-06-27 00:28:28 -0500232 runfetchcmd(p4fetchcmd, d, workdir=ud.pkgdir, log=progresshandler)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600233
234 runfetchcmd('tar -czf %s p4' % (ud.localpath), d, cleanup=[ud.localpath], workdir=ud.pkgdir)
235
236 def clean(self, ud, d):
237 """ Cleanup p4 specific files and dirs"""
238 bb.utils.remove(ud.localpath)
239 bb.utils.remove(ud.pkgdir, True)
240
241 def supports_srcrev(self):
242 return True
243
244 def _revision_key(self, ud, d, name):
245 """ Return a unique key for the url """
246 return 'p4:%s' % ud.pkgdir
247
248 def _latest_revision(self, ud, d, name):
249 """ Return the latest upstream scm revision number """
250 p4cmd = self._buildp4command(ud, d, "changes")
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500251 bb.fetch2.check_network_access(d, p4cmd, ud.url)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600252 tip = runfetchcmd(p4cmd, d, True)
253
254 if not tip:
255 raise FetchError('Could not determine the latest perforce changelist')
256
257 tipcset = tip.split(' ')[1]
Andrew Geisslerd1e89492021-02-12 15:35:20 -0600258 logger.debug('p4 tip found to be changelist %s' % tipcset)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600259 return tipcset
260
261 def sortable_revision(self, ud, d, name):
262 """ Return a sortable revision number """
263 return False, self._build_revision(ud, d)
264
265 def _build_revision(self, ud, d):
266 return ud.revision
267