blob: e7c07264a8804f92c0be6f6fc17471ea895db5ba [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#
2# Copyright (C) 2012 Robert Yang
3#
Brad Bishopc342db32019-05-15 21:57:59 -04004# SPDX-License-Identifier: GPL-2.0-only
Patrick Williamsc124f4f2015-09-15 14:41:29 -05005#
Patrick Williamsc124f4f2015-09-15 14:41:29 -05006
Andrew Geissler82c905d2020-04-13 13:39:40 -05007import os, logging, re
Patrick Williamsc124f4f2015-09-15 14:41:29 -05008import bb
9logger = logging.getLogger("BitBake.Monitor")
10
11def printErr(info):
12 logger.error("%s\n Disk space monitor will NOT be enabled" % info)
13
14def convertGMK(unit):
15
16 """ Convert the space unit G, M, K, the unit is case-insensitive """
17
Brad Bishop19323692019-04-05 15:28:33 -040018 unitG = re.match(r'([1-9][0-9]*)[gG]\s?$', unit)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050019 if unitG:
20 return int(unitG.group(1)) * (1024 ** 3)
Brad Bishop19323692019-04-05 15:28:33 -040021 unitM = re.match(r'([1-9][0-9]*)[mM]\s?$', unit)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050022 if unitM:
23 return int(unitM.group(1)) * (1024 ** 2)
Brad Bishop19323692019-04-05 15:28:33 -040024 unitK = re.match(r'([1-9][0-9]*)[kK]\s?$', unit)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050025 if unitK:
26 return int(unitK.group(1)) * 1024
Brad Bishop19323692019-04-05 15:28:33 -040027 unitN = re.match(r'([1-9][0-9]*)\s?$', unit)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050028 if unitN:
29 return int(unitN.group(1))
30 else:
31 return None
32
33def getMountedDev(path):
34
35 """ Get the device mounted at the path, uses /proc/mounts """
36
37 # Get the mount point of the filesystem containing path
38 # st_dev is the ID of device containing file
39 parentDev = os.stat(path).st_dev
40 currentDev = parentDev
41 # When the current directory's device is different from the
42 # parent's, then the current directory is a mount point
43 while parentDev == currentDev:
44 mountPoint = path
45 # Use dirname to get the parent's directory
46 path = os.path.dirname(path)
47 # Reach the "/"
48 if path == mountPoint:
49 break
50 parentDev= os.stat(path).st_dev
51
52 try:
53 with open("/proc/mounts", "r") as ifp:
54 for line in ifp:
55 procLines = line.rstrip('\n').split()
56 if procLines[1] == mountPoint:
57 return procLines[0]
58 except EnvironmentError:
59 pass
60 return None
61
62def getDiskData(BBDirs, configuration):
63
64 """Prepare disk data for disk space monitor"""
65
66 # Save the device IDs, need the ID to be unique (the dictionary's key is
67 # unique), so that when more than one directory is located on the same
68 # device, we just monitor it once
69 devDict = {}
70 for pathSpaceInode in BBDirs.split():
71 # The input format is: "dir,space,inode", dir is a must, space
72 # and inode are optional
Brad Bishop19323692019-04-05 15:28:33 -040073 pathSpaceInodeRe = re.match(r'([^,]*),([^,]*),([^,]*),?(.*)', pathSpaceInode)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050074 if not pathSpaceInodeRe:
75 printErr("Invalid value in BB_DISKMON_DIRS: %s" % pathSpaceInode)
76 return None
77
78 action = pathSpaceInodeRe.group(1)
79 if action not in ("ABORT", "STOPTASKS", "WARN"):
80 printErr("Unknown disk space monitor action: %s" % action)
81 return None
82
83 path = os.path.realpath(pathSpaceInodeRe.group(2))
84 if not path:
85 printErr("Invalid path value in BB_DISKMON_DIRS: %s" % pathSpaceInode)
86 return None
87
88 # The disk space or inode is optional, but it should have a correct
89 # value once it is specified
90 minSpace = pathSpaceInodeRe.group(3)
91 if minSpace:
92 minSpace = convertGMK(minSpace)
93 if not minSpace:
94 printErr("Invalid disk space value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(3))
95 return None
96 else:
97 # None means that it is not specified
98 minSpace = None
99
100 minInode = pathSpaceInodeRe.group(4)
101 if minInode:
102 minInode = convertGMK(minInode)
103 if not minInode:
104 printErr("Invalid inode value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(4))
105 return None
106 else:
107 # None means that it is not specified
108 minInode = None
109
110 if minSpace is None and minInode is None:
111 printErr("No disk space or inode value in found BB_DISKMON_DIRS: %s" % pathSpaceInode)
112 return None
113 # mkdir for the directory since it may not exist, for example the
114 # DL_DIR may not exist at the very beginning
115 if not os.path.exists(path):
116 bb.utils.mkdirhier(path)
117 dev = getMountedDev(path)
118 # Use path/action as the key
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500119 devDict[(path, action)] = [dev, minSpace, minInode]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500120
121 return devDict
122
123def getInterval(configuration):
124
125 """ Get the disk space interval """
126
127 # The default value is 50M and 5K.
128 spaceDefault = 50 * 1024 * 1024
129 inodeDefault = 5 * 1024
130
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500131 interval = configuration.getVar("BB_DISKMON_WARNINTERVAL")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500132 if not interval:
133 return spaceDefault, inodeDefault
134 else:
135 # The disk space or inode interval is optional, but it should
136 # have a correct value once it is specified
Brad Bishop19323692019-04-05 15:28:33 -0400137 intervalRe = re.match(r'([^,]*),?\s*(.*)', interval)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500138 if intervalRe:
139 intervalSpace = intervalRe.group(1)
140 if intervalSpace:
141 intervalSpace = convertGMK(intervalSpace)
142 if not intervalSpace:
143 printErr("Invalid disk space interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(1))
144 return None, None
145 else:
146 intervalSpace = spaceDefault
147 intervalInode = intervalRe.group(2)
148 if intervalInode:
149 intervalInode = convertGMK(intervalInode)
150 if not intervalInode:
151 printErr("Invalid disk inode interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(2))
152 return None, None
153 else:
154 intervalInode = inodeDefault
155 return intervalSpace, intervalInode
156 else:
157 printErr("Invalid interval value in BB_DISKMON_WARNINTERVAL: %s" % interval)
158 return None, None
159
160class diskMonitor:
161
162 """Prepare the disk space monitor data"""
163
164 def __init__(self, configuration):
165
166 self.enableMonitor = False
167 self.configuration = configuration
168
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500169 BBDirs = configuration.getVar("BB_DISKMON_DIRS") or None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500170 if BBDirs:
171 self.devDict = getDiskData(BBDirs, configuration)
172 if self.devDict:
173 self.spaceInterval, self.inodeInterval = getInterval(configuration)
174 if self.spaceInterval and self.inodeInterval:
175 self.enableMonitor = True
176 # These are for saving the previous disk free space and inode, we
177 # use them to avoid printing too many warning messages
178 self.preFreeS = {}
179 self.preFreeI = {}
180 # This is for STOPTASKS and ABORT, to avoid printing the message
181 # repeatedly while waiting for the tasks to finish
182 self.checked = {}
183 for k in self.devDict:
184 self.preFreeS[k] = 0
185 self.preFreeI[k] = 0
186 self.checked[k] = False
187 if self.spaceInterval is None and self.inodeInterval is None:
188 self.enableMonitor = False
189
190 def check(self, rq):
191
192 """ Take action for the monitor """
193
194 if self.enableMonitor:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500195 diskUsage = {}
196 for k, attributes in self.devDict.items():
197 path, action = k
198 dev, minSpace, minInode = attributes
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500199
200 st = os.statvfs(path)
201
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500202 # The available free space, integer number
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500203 freeSpace = st.f_bavail * st.f_frsize
204
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500205 # Send all relevant information in the event.
206 freeSpaceRoot = st.f_bfree * st.f_frsize
207 totalSpace = st.f_blocks * st.f_frsize
208 diskUsage[dev] = bb.event.DiskUsageSample(freeSpace, freeSpaceRoot, totalSpace)
209
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500210 if minSpace and freeSpace < minSpace:
211 # Always show warning, the self.checked would always be False if the action is WARN
212 if self.preFreeS[k] == 0 or self.preFreeS[k] - freeSpace > self.spaceInterval and not self.checked[k]:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600213 logger.warning("The free space of %s (%s) is running low (%.3fGB left)" % \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500214 (path, dev, freeSpace / 1024 / 1024 / 1024.0))
215 self.preFreeS[k] = freeSpace
216
217 if action == "STOPTASKS" and not self.checked[k]:
218 logger.error("No new tasks can be executed since the disk space monitor action is \"STOPTASKS\"!")
219 self.checked[k] = True
220 rq.finish_runqueue(False)
221 bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
222 elif action == "ABORT" and not self.checked[k]:
223 logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
224 self.checked[k] = True
225 rq.finish_runqueue(True)
226 bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
227
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500228 # The free inodes, integer number
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500229 freeInode = st.f_favail
230
231 if minInode and freeInode < minInode:
232 # Some filesystems use dynamic inodes so can't run out
233 # (e.g. btrfs). This is reported by the inode count being 0.
234 if st.f_files == 0:
235 self.devDict[k][2] = None
236 continue
237 # Always show warning, the self.checked would always be False if the action is WARN
238 if self.preFreeI[k] == 0 or self.preFreeI[k] - freeInode > self.inodeInterval and not self.checked[k]:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600239 logger.warning("The free inode of %s (%s) is running low (%.3fK left)" % \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500240 (path, dev, freeInode / 1024.0))
241 self.preFreeI[k] = freeInode
242
243 if action == "STOPTASKS" and not self.checked[k]:
244 logger.error("No new tasks can be executed since the disk space monitor action is \"STOPTASKS\"!")
245 self.checked[k] = True
246 rq.finish_runqueue(False)
247 bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
248 elif action == "ABORT" and not self.checked[k]:
249 logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
250 self.checked[k] = True
251 rq.finish_runqueue(True)
252 bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500253
254 bb.event.fire(bb.event.MonitorDiskEvent(diskUsage), self.configuration)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500255 return