blob: 84289208f447cc969244522cdc4d24f88e326757 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001# Local file checksum cache implementation
2#
3# Copyright (C) 2012 Intel Corporation
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050018import glob
19import operator
Patrick Williamsc124f4f2015-09-15 14:41:29 -050020import os
21import stat
Patrick Williamsc0f7c042017-02-23 20:41:17 -060022import pickle
Patrick Williamsc124f4f2015-09-15 14:41:29 -050023import bb.utils
24import logging
25from bb.cache import MultiProcessCache
26
27logger = logging.getLogger("BitBake.Cache")
28
Patrick Williamsc124f4f2015-09-15 14:41:29 -050029# mtime cache (non-persistent)
30# based upon the assumption that files do not change during bitbake run
31class FileMtimeCache(object):
32 cache = {}
33
34 def cached_mtime(self, f):
35 if f not in self.cache:
36 self.cache[f] = os.stat(f)[stat.ST_MTIME]
37 return self.cache[f]
38
39 def cached_mtime_noerror(self, f):
40 if f not in self.cache:
41 try:
42 self.cache[f] = os.stat(f)[stat.ST_MTIME]
43 except OSError:
44 return 0
45 return self.cache[f]
46
47 def update_mtime(self, f):
48 self.cache[f] = os.stat(f)[stat.ST_MTIME]
49 return self.cache[f]
50
51 def clear(self):
52 self.cache.clear()
53
54# Checksum + mtime cache (persistent)
55class FileChecksumCache(MultiProcessCache):
56 cache_file_name = "local_file_checksum_cache.dat"
57 CACHE_VERSION = 1
58
59 def __init__(self):
60 self.mtime_cache = FileMtimeCache()
61 MultiProcessCache.__init__(self)
62
63 def get_checksum(self, f):
64 entry = self.cachedata[0].get(f)
65 cmtime = self.mtime_cache.cached_mtime(f)
66 if entry:
67 (mtime, hashval) = entry
68 if cmtime == mtime:
69 return hashval
70 else:
71 bb.debug(2, "file %s changed mtime, recompute checksum" % f)
72
73 hashval = bb.utils.md5_file(f)
74 self.cachedata_extras[0][f] = (cmtime, hashval)
75 return hashval
76
77 def merge_data(self, source, dest):
78 for h in source[0]:
79 if h in dest:
80 (smtime, _) = source[0][h]
81 (dmtime, _) = dest[0][h]
82 if smtime > dmtime:
83 dest[0][h] = source[0][h]
84 else:
85 dest[0][h] = source[0][h]
Patrick Williamsd8c66bc2016-06-20 12:57:21 -050086
87 def get_checksums(self, filelist, pn):
88 """Get checksums for a list of files"""
89
90 def checksum_file(f):
91 try:
92 checksum = self.get_checksum(f)
93 except OSError as e:
94 bb.warn("Unable to get checksum for %s SRC_URI entry %s: %s" % (pn, os.path.basename(f), e))
95 return None
96 return checksum
97
98 def checksum_dir(pth):
99 # Handle directories recursively
100 dirchecksums = []
101 for root, dirs, files in os.walk(pth):
102 for name in files:
103 fullpth = os.path.join(root, name)
104 checksum = checksum_file(fullpth)
105 if checksum:
106 dirchecksums.append((fullpth, checksum))
107 return dirchecksums
108
109 checksums = []
110 for pth in filelist.split():
111 exist = pth.split(":")[1]
112 if exist == "False":
113 continue
114 pth = pth.split(":")[0]
115 if '*' in pth:
116 # Handle globs
117 for f in glob.glob(pth):
118 if os.path.isdir(f):
119 if not os.path.islink(f):
120 checksums.extend(checksum_dir(f))
121 else:
122 checksum = checksum_file(f)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600123 if checksum:
124 checksums.append((f, checksum))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500125 elif os.path.isdir(pth):
126 if not os.path.islink(pth):
127 checksums.extend(checksum_dir(pth))
128 else:
129 checksum = checksum_file(pth)
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600130 if checksum:
131 checksums.append((pth, checksum))
Patrick Williamsd8c66bc2016-06-20 12:57:21 -0500132
133 checksums.sort(key=operator.itemgetter(1))
134 return checksums