blob: 2ad1e615625599dc26d2fd2709f6498f880a5763 [file] [log] [blame]
Patrick Williamsc124f4f2015-09-15 14:41:29 -05001#!/usr/bin/env python
2# ex:ts=4:sw=4:sts=4:et
3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4#
5# Copyright (C) 2012 Robert Yang
6#
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License version 2 as
9# published by the Free Software Foundation.
10#
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along
17# with this program; if not, write to the Free Software Foundation, Inc.,
18# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20import os, logging, re, sys
21import bb
22logger = logging.getLogger("BitBake.Monitor")
23
24def printErr(info):
25 logger.error("%s\n Disk space monitor will NOT be enabled" % info)
26
27def convertGMK(unit):
28
29 """ Convert the space unit G, M, K, the unit is case-insensitive """
30
Brad Bishop19323692019-04-05 15:28:33 -040031 unitG = re.match(r'([1-9][0-9]*)[gG]\s?$', unit)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050032 if unitG:
33 return int(unitG.group(1)) * (1024 ** 3)
Brad Bishop19323692019-04-05 15:28:33 -040034 unitM = re.match(r'([1-9][0-9]*)[mM]\s?$', unit)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050035 if unitM:
36 return int(unitM.group(1)) * (1024 ** 2)
Brad Bishop19323692019-04-05 15:28:33 -040037 unitK = re.match(r'([1-9][0-9]*)[kK]\s?$', unit)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050038 if unitK:
39 return int(unitK.group(1)) * 1024
Brad Bishop19323692019-04-05 15:28:33 -040040 unitN = re.match(r'([1-9][0-9]*)\s?$', unit)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050041 if unitN:
42 return int(unitN.group(1))
43 else:
44 return None
45
46def getMountedDev(path):
47
48 """ Get the device mounted at the path, uses /proc/mounts """
49
50 # Get the mount point of the filesystem containing path
51 # st_dev is the ID of device containing file
52 parentDev = os.stat(path).st_dev
53 currentDev = parentDev
54 # When the current directory's device is different from the
55 # parent's, then the current directory is a mount point
56 while parentDev == currentDev:
57 mountPoint = path
58 # Use dirname to get the parent's directory
59 path = os.path.dirname(path)
60 # Reach the "/"
61 if path == mountPoint:
62 break
63 parentDev= os.stat(path).st_dev
64
65 try:
66 with open("/proc/mounts", "r") as ifp:
67 for line in ifp:
68 procLines = line.rstrip('\n').split()
69 if procLines[1] == mountPoint:
70 return procLines[0]
71 except EnvironmentError:
72 pass
73 return None
74
75def getDiskData(BBDirs, configuration):
76
77 """Prepare disk data for disk space monitor"""
78
79 # Save the device IDs, need the ID to be unique (the dictionary's key is
80 # unique), so that when more than one directory is located on the same
81 # device, we just monitor it once
82 devDict = {}
83 for pathSpaceInode in BBDirs.split():
84 # The input format is: "dir,space,inode", dir is a must, space
85 # and inode are optional
Brad Bishop19323692019-04-05 15:28:33 -040086 pathSpaceInodeRe = re.match(r'([^,]*),([^,]*),([^,]*),?(.*)', pathSpaceInode)
Patrick Williamsc124f4f2015-09-15 14:41:29 -050087 if not pathSpaceInodeRe:
88 printErr("Invalid value in BB_DISKMON_DIRS: %s" % pathSpaceInode)
89 return None
90
91 action = pathSpaceInodeRe.group(1)
92 if action not in ("ABORT", "STOPTASKS", "WARN"):
93 printErr("Unknown disk space monitor action: %s" % action)
94 return None
95
96 path = os.path.realpath(pathSpaceInodeRe.group(2))
97 if not path:
98 printErr("Invalid path value in BB_DISKMON_DIRS: %s" % pathSpaceInode)
99 return None
100
101 # The disk space or inode is optional, but it should have a correct
102 # value once it is specified
103 minSpace = pathSpaceInodeRe.group(3)
104 if minSpace:
105 minSpace = convertGMK(minSpace)
106 if not minSpace:
107 printErr("Invalid disk space value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(3))
108 return None
109 else:
110 # None means that it is not specified
111 minSpace = None
112
113 minInode = pathSpaceInodeRe.group(4)
114 if minInode:
115 minInode = convertGMK(minInode)
116 if not minInode:
117 printErr("Invalid inode value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(4))
118 return None
119 else:
120 # None means that it is not specified
121 minInode = None
122
123 if minSpace is None and minInode is None:
124 printErr("No disk space or inode value in found BB_DISKMON_DIRS: %s" % pathSpaceInode)
125 return None
126 # mkdir for the directory since it may not exist, for example the
127 # DL_DIR may not exist at the very beginning
128 if not os.path.exists(path):
129 bb.utils.mkdirhier(path)
130 dev = getMountedDev(path)
131 # Use path/action as the key
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500132 devDict[(path, action)] = [dev, minSpace, minInode]
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500133
134 return devDict
135
136def getInterval(configuration):
137
138 """ Get the disk space interval """
139
140 # The default value is 50M and 5K.
141 spaceDefault = 50 * 1024 * 1024
142 inodeDefault = 5 * 1024
143
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500144 interval = configuration.getVar("BB_DISKMON_WARNINTERVAL")
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500145 if not interval:
146 return spaceDefault, inodeDefault
147 else:
148 # The disk space or inode interval is optional, but it should
149 # have a correct value once it is specified
Brad Bishop19323692019-04-05 15:28:33 -0400150 intervalRe = re.match(r'([^,]*),?\s*(.*)', interval)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500151 if intervalRe:
152 intervalSpace = intervalRe.group(1)
153 if intervalSpace:
154 intervalSpace = convertGMK(intervalSpace)
155 if not intervalSpace:
156 printErr("Invalid disk space interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(1))
157 return None, None
158 else:
159 intervalSpace = spaceDefault
160 intervalInode = intervalRe.group(2)
161 if intervalInode:
162 intervalInode = convertGMK(intervalInode)
163 if not intervalInode:
164 printErr("Invalid disk inode interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(2))
165 return None, None
166 else:
167 intervalInode = inodeDefault
168 return intervalSpace, intervalInode
169 else:
170 printErr("Invalid interval value in BB_DISKMON_WARNINTERVAL: %s" % interval)
171 return None, None
172
173class diskMonitor:
174
175 """Prepare the disk space monitor data"""
176
177 def __init__(self, configuration):
178
179 self.enableMonitor = False
180 self.configuration = configuration
181
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500182 BBDirs = configuration.getVar("BB_DISKMON_DIRS") or None
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500183 if BBDirs:
184 self.devDict = getDiskData(BBDirs, configuration)
185 if self.devDict:
186 self.spaceInterval, self.inodeInterval = getInterval(configuration)
187 if self.spaceInterval and self.inodeInterval:
188 self.enableMonitor = True
189 # These are for saving the previous disk free space and inode, we
190 # use them to avoid printing too many warning messages
191 self.preFreeS = {}
192 self.preFreeI = {}
193 # This is for STOPTASKS and ABORT, to avoid printing the message
194 # repeatedly while waiting for the tasks to finish
195 self.checked = {}
196 for k in self.devDict:
197 self.preFreeS[k] = 0
198 self.preFreeI[k] = 0
199 self.checked[k] = False
200 if self.spaceInterval is None and self.inodeInterval is None:
201 self.enableMonitor = False
202
203 def check(self, rq):
204
205 """ Take action for the monitor """
206
207 if self.enableMonitor:
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500208 diskUsage = {}
209 for k, attributes in self.devDict.items():
210 path, action = k
211 dev, minSpace, minInode = attributes
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500212
213 st = os.statvfs(path)
214
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500215 # The available free space, integer number
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500216 freeSpace = st.f_bavail * st.f_frsize
217
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500218 # Send all relevant information in the event.
219 freeSpaceRoot = st.f_bfree * st.f_frsize
220 totalSpace = st.f_blocks * st.f_frsize
221 diskUsage[dev] = bb.event.DiskUsageSample(freeSpace, freeSpaceRoot, totalSpace)
222
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500223 if minSpace and freeSpace < minSpace:
224 # Always show warning, the self.checked would always be False if the action is WARN
225 if self.preFreeS[k] == 0 or self.preFreeS[k] - freeSpace > self.spaceInterval and not self.checked[k]:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600226 logger.warning("The free space of %s (%s) is running low (%.3fGB left)" % \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500227 (path, dev, freeSpace / 1024 / 1024 / 1024.0))
228 self.preFreeS[k] = freeSpace
229
230 if action == "STOPTASKS" and not self.checked[k]:
231 logger.error("No new tasks can be executed since the disk space monitor action is \"STOPTASKS\"!")
232 self.checked[k] = True
233 rq.finish_runqueue(False)
234 bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
235 elif action == "ABORT" and not self.checked[k]:
236 logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
237 self.checked[k] = True
238 rq.finish_runqueue(True)
239 bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
240
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500241 # The free inodes, integer number
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500242 freeInode = st.f_favail
243
244 if minInode and freeInode < minInode:
245 # Some filesystems use dynamic inodes so can't run out
246 # (e.g. btrfs). This is reported by the inode count being 0.
247 if st.f_files == 0:
248 self.devDict[k][2] = None
249 continue
250 # Always show warning, the self.checked would always be False if the action is WARN
251 if self.preFreeI[k] == 0 or self.preFreeI[k] - freeInode > self.inodeInterval and not self.checked[k]:
Patrick Williamsc0f7c042017-02-23 20:41:17 -0600252 logger.warning("The free inode of %s (%s) is running low (%.3fK left)" % \
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500253 (path, dev, freeInode / 1024.0))
254 self.preFreeI[k] = freeInode
255
256 if action == "STOPTASKS" and not self.checked[k]:
257 logger.error("No new tasks can be executed since the disk space monitor action is \"STOPTASKS\"!")
258 self.checked[k] = True
259 rq.finish_runqueue(False)
260 bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
261 elif action == "ABORT" and not self.checked[k]:
262 logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
263 self.checked[k] = True
264 rq.finish_runqueue(True)
265 bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
Brad Bishop6e60e8b2018-02-01 10:27:11 -0500266
267 bb.event.fire(bb.event.MonitorDiskEvent(diskUsage), self.configuration)
Patrick Williamsc124f4f2015-09-15 14:41:29 -0500268 return