Matt Spinler | 711d51d | 2019-11-06 09:36:51 -0600 | [diff] [blame] | 1 | /** |
| 2 | * Copyright © 2019 IBM Corporation |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 16 | #include "manager.hpp" |
| 17 | |
| 18 | #include "additional_data.hpp" |
Matt Spinler | 05c2c6c | 2019-12-18 14:02:09 -0600 | [diff] [blame] | 19 | #include "json_utils.hpp" |
Matt Spinler | 89fa082 | 2019-07-17 13:54:30 -0500 | [diff] [blame] | 20 | #include "pel.hpp" |
Matt Spinler | 1962e08 | 2020-08-05 13:44:53 -0500 | [diff] [blame] | 21 | #include "service_indicators.hpp" |
Matt Spinler | 89fa082 | 2019-07-17 13:54:30 -0500 | [diff] [blame] | 22 | |
Matt Spinler | 22421b9 | 2020-07-17 09:41:08 -0500 | [diff] [blame] | 23 | #include <fmt/format.h> |
Matt Spinler | ff9cec2 | 2020-07-15 13:06:35 -0500 | [diff] [blame] | 24 | #include <sys/inotify.h> |
Matt Spinler | 6b1a5c8 | 2020-01-07 08:48:53 -0600 | [diff] [blame] | 25 | #include <unistd.h> |
| 26 | |
Matt Spinler | 89fa082 | 2019-07-17 13:54:30 -0500 | [diff] [blame] | 27 | #include <filesystem> |
| 28 | #include <fstream> |
Matt Spinler | a34ab72 | 2019-12-16 10:39:32 -0600 | [diff] [blame] | 29 | #include <xyz/openbmc_project/Common/error.hpp> |
Matt Spinler | 56ad2a0 | 2020-03-26 14:00:52 -0500 | [diff] [blame] | 30 | #include <xyz/openbmc_project/Logging/Create/server.hpp> |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 31 | |
| 32 | namespace openpower |
| 33 | { |
| 34 | namespace pels |
| 35 | { |
| 36 | |
| 37 | using namespace phosphor::logging; |
Matt Spinler | 89fa082 | 2019-07-17 13:54:30 -0500 | [diff] [blame] | 38 | namespace fs = std::filesystem; |
Harisuddin Mohamed Isa | 0f717e1 | 2020-01-15 20:05:33 +0800 | [diff] [blame] | 39 | namespace rg = openpower::pels::message; |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 40 | |
Matt Spinler | a34ab72 | 2019-12-16 10:39:32 -0600 | [diff] [blame] | 41 | namespace common_error = sdbusplus::xyz::openbmc_project::Common::Error; |
| 42 | |
Matt Spinler | 56ad2a0 | 2020-03-26 14:00:52 -0500 | [diff] [blame] | 43 | using Create = sdbusplus::xyz::openbmc_project::Logging::server::Create; |
| 44 | |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 45 | namespace additional_data |
| 46 | { |
| 47 | constexpr auto rawPEL = "RAWPEL"; |
Matt Spinler | 19e7290 | 2020-01-24 11:05:20 -0600 | [diff] [blame] | 48 | constexpr auto esel = "ESEL"; |
Matt Spinler | 30ddc9f | 2020-07-16 15:39:59 -0500 | [diff] [blame] | 49 | constexpr auto error = "ERROR_NAME"; |
Matt Spinler | 19e7290 | 2020-01-24 11:05:20 -0600 | [diff] [blame] | 50 | } // namespace additional_data |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 51 | |
Matt Spinler | 30ddc9f | 2020-07-16 15:39:59 -0500 | [diff] [blame] | 52 | constexpr auto defaultLogMessage = "xyz.openbmc_project.Logging.Error.Default"; |
| 53 | |
Matt Spinler | ff9cec2 | 2020-07-15 13:06:35 -0500 | [diff] [blame] | 54 | Manager::~Manager() |
| 55 | { |
| 56 | if (_pelFileDeleteFD != -1) |
| 57 | { |
| 58 | if (_pelFileDeleteWatchFD != -1) |
| 59 | { |
| 60 | inotify_rm_watch(_pelFileDeleteFD, _pelFileDeleteWatchFD); |
| 61 | } |
| 62 | close(_pelFileDeleteFD); |
| 63 | } |
| 64 | } |
| 65 | |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 66 | void Manager::create(const std::string& message, uint32_t obmcLogID, |
| 67 | uint64_t timestamp, Entry::Level severity, |
| 68 | const std::vector<std::string>& additionalData, |
Matt Spinler | 56ad2a0 | 2020-03-26 14:00:52 -0500 | [diff] [blame] | 69 | const std::vector<std::string>& associations, |
| 70 | const FFDCEntries& ffdc) |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 71 | { |
| 72 | AdditionalData ad{additionalData}; |
| 73 | |
Matt Spinler | 19e7290 | 2020-01-24 11:05:20 -0600 | [diff] [blame] | 74 | // If a PEL was passed in via a filename or in an ESEL, |
| 75 | // use that. Otherwise, create one. |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 76 | auto rawPelPath = ad.getValue(additional_data::rawPEL); |
| 77 | if (rawPelPath) |
| 78 | { |
| 79 | addRawPEL(*rawPelPath, obmcLogID); |
| 80 | } |
| 81 | else |
| 82 | { |
Matt Spinler | 19e7290 | 2020-01-24 11:05:20 -0600 | [diff] [blame] | 83 | auto esel = ad.getValue(additional_data::esel); |
| 84 | if (esel) |
| 85 | { |
| 86 | addESELPEL(*esel, obmcLogID); |
| 87 | } |
| 88 | else |
| 89 | { |
| 90 | createPEL(message, obmcLogID, timestamp, severity, additionalData, |
Matt Spinler | 56ad2a0 | 2020-03-26 14:00:52 -0500 | [diff] [blame] | 91 | associations, ffdc); |
Matt Spinler | 19e7290 | 2020-01-24 11:05:20 -0600 | [diff] [blame] | 92 | } |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 93 | } |
Adriana Kobylak | e7d271a | 2020-12-07 14:32:44 -0600 | [diff] [blame] | 94 | |
| 95 | setEntryPath(obmcLogID); |
Vijay Lobo | cbc93a4 | 2021-05-20 19:04:07 -0500 | [diff] [blame] | 96 | setServiceProviderNotifyFlag(obmcLogID); |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 97 | } |
| 98 | |
| 99 | void Manager::addRawPEL(const std::string& rawPelPath, uint32_t obmcLogID) |
| 100 | { |
Matt Spinler | 89fa082 | 2019-07-17 13:54:30 -0500 | [diff] [blame] | 101 | if (fs::exists(rawPelPath)) |
| 102 | { |
| 103 | std::ifstream file(rawPelPath, std::ios::in | std::ios::binary); |
| 104 | |
| 105 | auto data = std::vector<uint8_t>(std::istreambuf_iterator<char>(file), |
| 106 | std::istreambuf_iterator<char>()); |
| 107 | if (file.fail()) |
| 108 | { |
| 109 | log<level::ERR>("Filesystem error reading a raw PEL", |
| 110 | entry("PELFILE=%s", rawPelPath.c_str()), |
| 111 | entry("OBMCLOGID=%d", obmcLogID)); |
| 112 | // TODO, Decide what to do here. Maybe nothing. |
| 113 | return; |
| 114 | } |
| 115 | |
| 116 | file.close(); |
| 117 | |
Matt Spinler | 19e7290 | 2020-01-24 11:05:20 -0600 | [diff] [blame] | 118 | addPEL(data, obmcLogID); |
Matt Spinler | 89fa082 | 2019-07-17 13:54:30 -0500 | [diff] [blame] | 119 | } |
| 120 | else |
| 121 | { |
| 122 | log<level::ERR>("Raw PEL file from BMC event log does not exist", |
| 123 | entry("PELFILE=%s", (rawPelPath).c_str()), |
| 124 | entry("OBMCLOGID=%d", obmcLogID)); |
| 125 | } |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 126 | } |
| 127 | |
Matt Spinler | 19e7290 | 2020-01-24 11:05:20 -0600 | [diff] [blame] | 128 | void Manager::addPEL(std::vector<uint8_t>& pelData, uint32_t obmcLogID) |
| 129 | { |
| 130 | |
| 131 | auto pel = std::make_unique<openpower::pels::PEL>(pelData, obmcLogID); |
| 132 | if (pel->valid()) |
| 133 | { |
Sumit Kumar | a1e4084 | 2021-06-23 09:52:25 -0500 | [diff] [blame] | 134 | // Assign Id other than to Hostbot PEL |
| 135 | if ((pel->privateHeader()).creatorID() != |
| 136 | static_cast<uint8_t>(CreatorID::hostboot)) |
| 137 | { |
| 138 | pel->assignID(); |
| 139 | } |
| 140 | |
| 141 | // PELs created by others still need this field set by us. |
Matt Spinler | 19e7290 | 2020-01-24 11:05:20 -0600 | [diff] [blame] | 142 | pel->setCommitTime(); |
| 143 | |
Sumit Kumar | 3160a54 | 2021-04-26 08:07:04 -0500 | [diff] [blame] | 144 | // Update System Info to Extended User Data |
| 145 | pel->updateSysInfoInExtendedUserDataSection(*_dataIface); |
| 146 | |
Matt Spinler | 19e7290 | 2020-01-24 11:05:20 -0600 | [diff] [blame] | 147 | try |
| 148 | { |
Matt Spinler | d0ab1cf | 2021-02-10 13:26:18 -0600 | [diff] [blame] | 149 | log<level::DEBUG>( |
Matt Spinler | 6321ba3 | 2020-07-17 09:58:19 -0500 | [diff] [blame] | 150 | fmt::format("Adding external PEL {:#x} (BMC ID {}) to repo", |
| 151 | pel->id(), obmcLogID) |
| 152 | .c_str()); |
Matt Spinler | 5f5352e | 2020-03-05 16:23:27 -0600 | [diff] [blame] | 153 | |
Matt Spinler | 19e7290 | 2020-01-24 11:05:20 -0600 | [diff] [blame] | 154 | _repo.add(pel); |
Matt Spinler | 7e727a3 | 2020-07-07 15:00:17 -0500 | [diff] [blame] | 155 | |
| 156 | if (_repo.sizeWarning()) |
| 157 | { |
| 158 | scheduleRepoPrune(); |
| 159 | } |
Matt Spinler | 1962e08 | 2020-08-05 13:44:53 -0500 | [diff] [blame] | 160 | |
| 161 | // Activate any resulting service indicators if necessary |
| 162 | auto policy = service_indicators::getPolicy(*_dataIface); |
| 163 | policy->activate(*pel); |
Matt Spinler | 19e7290 | 2020-01-24 11:05:20 -0600 | [diff] [blame] | 164 | } |
| 165 | catch (std::exception& e) |
| 166 | { |
| 167 | // Probably a full or r/o filesystem, not much we can do. |
| 168 | log<level::ERR>("Unable to add PEL to Repository", |
| 169 | entry("PEL_ID=0x%X", pel->id())); |
| 170 | } |
Andrew Geissler | 44fc316 | 2020-07-09 09:21:31 -0500 | [diff] [blame] | 171 | |
| 172 | // Check if firmware should quiesce system due to error |
| 173 | checkPelAndQuiesce(pel); |
Vijay Lobo | d354a39 | 2021-06-01 16:21:02 -0500 | [diff] [blame] | 174 | updateEventId(pel); |
Vijay Lobo | 593a4c6 | 2021-06-16 14:25:26 -0500 | [diff] [blame] | 175 | updateResolution(pel); |
Matt Spinler | 19e7290 | 2020-01-24 11:05:20 -0600 | [diff] [blame] | 176 | } |
| 177 | else |
| 178 | { |
| 179 | log<level::ERR>("Invalid PEL received from the host", |
| 180 | entry("OBMCLOGID=%d", obmcLogID)); |
| 181 | |
| 182 | AdditionalData ad; |
| 183 | ad.add("PLID", getNumberString("0x%08X", pel->plid())); |
| 184 | ad.add("OBMC_LOG_ID", std::to_string(obmcLogID)); |
| 185 | ad.add("PEL_SIZE", std::to_string(pelData.size())); |
| 186 | |
| 187 | std::string asciiString; |
| 188 | auto src = pel->primarySRC(); |
| 189 | if (src) |
| 190 | { |
| 191 | asciiString = (*src)->asciiString(); |
| 192 | } |
| 193 | |
| 194 | ad.add("SRC", asciiString); |
| 195 | |
| 196 | _eventLogger.log("org.open_power.Logging.Error.BadHostPEL", |
| 197 | Entry::Level::Error, ad); |
Matt Spinler | fe72189 | 2020-04-02 10:28:08 -0500 | [diff] [blame] | 198 | |
| 199 | // Save it to a file for debug in the lab. Just keep the latest. |
| 200 | // Not adding it to the PEL because it could already be max size |
| 201 | // and don't want to truncate an already invalid PEL. |
| 202 | std::ofstream pelFile{getPELRepoPath() / "badPEL"}; |
| 203 | pelFile.write(reinterpret_cast<const char*>(pelData.data()), |
| 204 | pelData.size()); |
Matt Spinler | 19e7290 | 2020-01-24 11:05:20 -0600 | [diff] [blame] | 205 | } |
| 206 | } |
| 207 | |
| 208 | void Manager::addESELPEL(const std::string& esel, uint32_t obmcLogID) |
| 209 | { |
| 210 | std::vector<uint8_t> data; |
| 211 | |
Matt Spinler | 5f5352e | 2020-03-05 16:23:27 -0600 | [diff] [blame] | 212 | log<level::DEBUG>("Adding PEL from ESEL", |
| 213 | entry("OBMC_LOG_ID=%d", obmcLogID)); |
| 214 | |
Matt Spinler | 19e7290 | 2020-01-24 11:05:20 -0600 | [diff] [blame] | 215 | try |
| 216 | { |
| 217 | data = std::move(eselToRawData(esel)); |
| 218 | } |
| 219 | catch (std::exception& e) |
| 220 | { |
| 221 | // Try to add it below anyway, so it follows the usual bad data path. |
| 222 | log<level::ERR>("Problems converting ESEL string to a byte vector"); |
| 223 | } |
| 224 | |
| 225 | addPEL(data, obmcLogID); |
| 226 | } |
| 227 | |
| 228 | std::vector<uint8_t> Manager::eselToRawData(const std::string& esel) |
| 229 | { |
| 230 | std::vector<uint8_t> data; |
| 231 | std::string byteString; |
| 232 | |
| 233 | // As the eSEL string looks like: "50 48 00 ab ..." there are 3 |
| 234 | // characters per raw byte, and since the actual PEL data starts |
| 235 | // at the 16th byte, the code will grab the PEL data starting at |
| 236 | // offset 48 in the string. |
| 237 | static constexpr size_t pelStart = 16 * 3; |
| 238 | |
| 239 | if (esel.size() <= pelStart) |
| 240 | { |
| 241 | log<level::ERR>("ESEL data too short", |
| 242 | entry("ESEL_SIZE=%d", esel.size())); |
| 243 | |
| 244 | throw std::length_error("ESEL data too short"); |
| 245 | } |
| 246 | |
| 247 | for (size_t i = pelStart; i < esel.size(); i += 3) |
| 248 | { |
| 249 | if (i + 1 < esel.size()) |
| 250 | { |
| 251 | byteString = esel.substr(i, 2); |
| 252 | data.push_back(std::stoi(byteString, nullptr, 16)); |
| 253 | } |
| 254 | else |
| 255 | { |
| 256 | log<level::ERR>("ESEL data too short", |
| 257 | entry("ESEL_SIZE=%d", esel.size())); |
| 258 | throw std::length_error("ESEL data too short"); |
| 259 | } |
| 260 | } |
| 261 | |
| 262 | return data; |
| 263 | } |
| 264 | |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 265 | void Manager::erase(uint32_t obmcLogID) |
| 266 | { |
Matt Spinler | 475e574 | 2019-07-18 16:09:49 -0500 | [diff] [blame] | 267 | Repository::LogID id{Repository::LogID::Obmc(obmcLogID)}; |
| 268 | |
| 269 | _repo.remove(id); |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 270 | } |
| 271 | |
Patrick Williams | d26fa3e | 2021-04-21 15:22:23 -0500 | [diff] [blame] | 272 | bool Manager::isDeleteProhibited(uint32_t /*obmcLogID*/) |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 273 | { |
| 274 | return false; |
| 275 | } |
| 276 | |
Matt Spinler | 56ad2a0 | 2020-03-26 14:00:52 -0500 | [diff] [blame] | 277 | PelFFDC Manager::convertToPelFFDC(const FFDCEntries& ffdc) |
| 278 | { |
| 279 | PelFFDC pelFFDC; |
| 280 | |
| 281 | std::for_each(ffdc.begin(), ffdc.end(), [&pelFFDC](const auto& f) { |
| 282 | PelFFDCfile pf; |
| 283 | pf.subType = std::get<ffdcSubtypePos>(f); |
| 284 | pf.version = std::get<ffdcVersionPos>(f); |
| 285 | pf.fd = std::get<ffdcFDPos>(f); |
| 286 | |
| 287 | switch (std::get<ffdcFormatPos>(f)) |
| 288 | { |
| 289 | case Create::FFDCFormat::JSON: |
| 290 | pf.format = UserDataFormat::json; |
| 291 | break; |
| 292 | case Create::FFDCFormat::CBOR: |
| 293 | pf.format = UserDataFormat::cbor; |
| 294 | break; |
| 295 | case Create::FFDCFormat::Text: |
| 296 | pf.format = UserDataFormat::text; |
| 297 | break; |
| 298 | case Create::FFDCFormat::Custom: |
| 299 | pf.format = UserDataFormat::custom; |
| 300 | break; |
| 301 | } |
| 302 | |
| 303 | pelFFDC.push_back(pf); |
| 304 | }); |
| 305 | |
| 306 | return pelFFDC; |
| 307 | } |
| 308 | |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 309 | void Manager::createPEL(const std::string& message, uint32_t obmcLogID, |
| 310 | uint64_t timestamp, |
| 311 | phosphor::logging::Entry::Level severity, |
| 312 | const std::vector<std::string>& additionalData, |
Patrick Williams | d26fa3e | 2021-04-21 15:22:23 -0500 | [diff] [blame] | 313 | const std::vector<std::string>& /*associations*/, |
Matt Spinler | 56ad2a0 | 2020-03-26 14:00:52 -0500 | [diff] [blame] | 314 | const FFDCEntries& ffdc) |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 315 | { |
Harisuddin Mohamed Isa | 0f717e1 | 2020-01-15 20:05:33 +0800 | [diff] [blame] | 316 | auto entry = _registry.lookup(message, rg::LookupType::name); |
Matt Spinler | 30ddc9f | 2020-07-16 15:39:59 -0500 | [diff] [blame] | 317 | auto pelFFDC = convertToPelFFDC(ffdc); |
| 318 | AdditionalData ad{additionalData}; |
Matt Spinler | 1d4c74a | 2019-12-16 14:40:21 -0600 | [diff] [blame] | 319 | std::string msg; |
Matt Spinler | 67456c2 | 2019-10-21 12:22:49 -0500 | [diff] [blame] | 320 | |
Matt Spinler | 30ddc9f | 2020-07-16 15:39:59 -0500 | [diff] [blame] | 321 | if (!entry) |
Matt Spinler | 67456c2 | 2019-10-21 12:22:49 -0500 | [diff] [blame] | 322 | { |
Matt Spinler | 30ddc9f | 2020-07-16 15:39:59 -0500 | [diff] [blame] | 323 | // Instead, get the default entry that means there is no |
| 324 | // other matching entry. This error will still use the |
| 325 | // AdditionalData values of the original error, and this |
| 326 | // code will add the error message value that wasn't found |
| 327 | // to this AD. This way, there will at least be a PEL, |
| 328 | // possibly with callouts, to allow users to debug the |
| 329 | // issue that caused the error even without its own PEL. |
Matt Spinler | 1d4c74a | 2019-12-16 14:40:21 -0600 | [diff] [blame] | 330 | msg = "Event not found in PEL message registry: " + message; |
| 331 | log<level::INFO>(msg.c_str()); |
Matt Spinler | 30ddc9f | 2020-07-16 15:39:59 -0500 | [diff] [blame] | 332 | |
| 333 | entry = _registry.lookup(defaultLogMessage, rg::LookupType::name); |
| 334 | if (!entry) |
| 335 | { |
| 336 | log<level::ERR>("Default event not found in PEL message registry"); |
| 337 | return; |
| 338 | } |
| 339 | |
| 340 | ad.add(additional_data::error, message); |
| 341 | } |
| 342 | |
| 343 | auto pel = std::make_unique<openpower::pels::PEL>( |
| 344 | *entry, obmcLogID, timestamp, severity, ad, pelFFDC, *_dataIface); |
| 345 | |
| 346 | _repo.add(pel); |
| 347 | |
| 348 | if (_repo.sizeWarning()) |
| 349 | { |
| 350 | scheduleRepoPrune(); |
| 351 | } |
| 352 | |
| 353 | auto src = pel->primarySRC(); |
| 354 | if (src) |
| 355 | { |
Matt Spinler | 22421b9 | 2020-07-17 09:41:08 -0500 | [diff] [blame] | 356 | auto msg = |
| 357 | fmt::format("Created PEL {:#x} (BMC ID {}) with SRC {}", pel->id(), |
| 358 | pel->obmcLogID(), (*src)->asciiString()); |
Matt Spinler | 30ddc9f | 2020-07-16 15:39:59 -0500 | [diff] [blame] | 359 | while (msg.back() == ' ') |
| 360 | { |
| 361 | msg.pop_back(); |
| 362 | } |
| 363 | log<level::INFO>(msg.c_str()); |
Matt Spinler | 1d4c74a | 2019-12-16 14:40:21 -0600 | [diff] [blame] | 364 | } |
Matt Spinler | 1962e08 | 2020-08-05 13:44:53 -0500 | [diff] [blame] | 365 | |
| 366 | // Activate any resulting service indicators if necessary |
| 367 | auto policy = service_indicators::getPolicy(*_dataIface); |
| 368 | policy->activate(*pel); |
Andrew Geissler | 44fc316 | 2020-07-09 09:21:31 -0500 | [diff] [blame] | 369 | |
| 370 | // Check if firmware should quiesce system due to error |
| 371 | checkPelAndQuiesce(pel); |
Vijay Lobo | d354a39 | 2021-06-01 16:21:02 -0500 | [diff] [blame] | 372 | updateEventId(pel); |
Vijay Lobo | 593a4c6 | 2021-06-16 14:25:26 -0500 | [diff] [blame] | 373 | updateResolution(pel); |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 374 | } |
| 375 | |
Matt Spinler | a34ab72 | 2019-12-16 10:39:32 -0600 | [diff] [blame] | 376 | sdbusplus::message::unix_fd Manager::getPEL(uint32_t pelID) |
| 377 | { |
| 378 | Repository::LogID id{Repository::LogID::Pel(pelID)}; |
| 379 | std::optional<int> fd; |
| 380 | |
Matt Spinler | 5f5352e | 2020-03-05 16:23:27 -0600 | [diff] [blame] | 381 | log<level::DEBUG>("getPEL", entry("PEL_ID=0x%X", pelID)); |
| 382 | |
Matt Spinler | a34ab72 | 2019-12-16 10:39:32 -0600 | [diff] [blame] | 383 | try |
| 384 | { |
| 385 | fd = _repo.getPELFD(id); |
| 386 | } |
| 387 | catch (std::exception& e) |
| 388 | { |
| 389 | throw common_error::InternalFailure(); |
| 390 | } |
| 391 | |
| 392 | if (!fd) |
| 393 | { |
| 394 | throw common_error::InvalidArgument(); |
| 395 | } |
| 396 | |
Matt Spinler | 6b1a5c8 | 2020-01-07 08:48:53 -0600 | [diff] [blame] | 397 | scheduleFDClose(*fd); |
| 398 | |
Matt Spinler | a34ab72 | 2019-12-16 10:39:32 -0600 | [diff] [blame] | 399 | return *fd; |
| 400 | } |
| 401 | |
Matt Spinler | 6b1a5c8 | 2020-01-07 08:48:53 -0600 | [diff] [blame] | 402 | void Manager::scheduleFDClose(int fd) |
| 403 | { |
| 404 | _fdCloserEventSource = std::make_unique<sdeventplus::source::Defer>( |
Matt Spinler | ff9cec2 | 2020-07-15 13:06:35 -0500 | [diff] [blame] | 405 | _event, std::bind(std::mem_fn(&Manager::closeFD), this, fd, |
| 406 | std::placeholders::_1)); |
Matt Spinler | 6b1a5c8 | 2020-01-07 08:48:53 -0600 | [diff] [blame] | 407 | } |
| 408 | |
Patrick Williams | d26fa3e | 2021-04-21 15:22:23 -0500 | [diff] [blame] | 409 | void Manager::closeFD(int fd, sdeventplus::source::EventBase& /*source*/) |
Matt Spinler | 6b1a5c8 | 2020-01-07 08:48:53 -0600 | [diff] [blame] | 410 | { |
| 411 | close(fd); |
| 412 | _fdCloserEventSource.reset(); |
| 413 | } |
| 414 | |
Matt Spinler | a34ab72 | 2019-12-16 10:39:32 -0600 | [diff] [blame] | 415 | std::vector<uint8_t> Manager::getPELFromOBMCID(uint32_t obmcLogID) |
| 416 | { |
| 417 | Repository::LogID id{Repository::LogID::Obmc(obmcLogID)}; |
| 418 | std::optional<std::vector<uint8_t>> data; |
| 419 | |
Matt Spinler | 5f5352e | 2020-03-05 16:23:27 -0600 | [diff] [blame] | 420 | log<level::DEBUG>("getPELFromOBMCID", entry("OBMC_LOG_ID=%d", obmcLogID)); |
| 421 | |
Matt Spinler | a34ab72 | 2019-12-16 10:39:32 -0600 | [diff] [blame] | 422 | try |
| 423 | { |
| 424 | data = _repo.getPELData(id); |
| 425 | } |
| 426 | catch (std::exception& e) |
| 427 | { |
| 428 | throw common_error::InternalFailure(); |
| 429 | } |
| 430 | |
| 431 | if (!data) |
| 432 | { |
| 433 | throw common_error::InvalidArgument(); |
| 434 | } |
| 435 | |
| 436 | return *data; |
| 437 | } |
| 438 | |
| 439 | void Manager::hostAck(uint32_t pelID) |
| 440 | { |
| 441 | Repository::LogID id{Repository::LogID::Pel(pelID)}; |
| 442 | |
Matt Spinler | 5f5352e | 2020-03-05 16:23:27 -0600 | [diff] [blame] | 443 | log<level::DEBUG>("HostAck", entry("PEL_ID=0x%X", pelID)); |
| 444 | |
Matt Spinler | a34ab72 | 2019-12-16 10:39:32 -0600 | [diff] [blame] | 445 | if (!_repo.hasPEL(id)) |
| 446 | { |
| 447 | throw common_error::InvalidArgument(); |
| 448 | } |
| 449 | |
| 450 | if (_hostNotifier) |
| 451 | { |
| 452 | _hostNotifier->ackPEL(pelID); |
| 453 | } |
| 454 | } |
| 455 | |
| 456 | void Manager::hostReject(uint32_t pelID, RejectionReason reason) |
| 457 | { |
| 458 | Repository::LogID id{Repository::LogID::Pel(pelID)}; |
| 459 | |
Matt Spinler | 5f5352e | 2020-03-05 16:23:27 -0600 | [diff] [blame] | 460 | log<level::DEBUG>("HostReject", entry("PEL_ID=0x%X", pelID), |
| 461 | entry("REASON=%d", static_cast<int>(reason))); |
| 462 | |
Matt Spinler | a34ab72 | 2019-12-16 10:39:32 -0600 | [diff] [blame] | 463 | if (!_repo.hasPEL(id)) |
| 464 | { |
| 465 | throw common_error::InvalidArgument(); |
| 466 | } |
| 467 | |
Matt Spinler | 05c2c6c | 2019-12-18 14:02:09 -0600 | [diff] [blame] | 468 | if (reason == RejectionReason::BadPEL) |
Matt Spinler | a34ab72 | 2019-12-16 10:39:32 -0600 | [diff] [blame] | 469 | { |
Matt Spinler | 05c2c6c | 2019-12-18 14:02:09 -0600 | [diff] [blame] | 470 | AdditionalData data; |
| 471 | data.add("BAD_ID", getNumberString("0x%08X", pelID)); |
| 472 | _eventLogger.log("org.open_power.Logging.Error.SentBadPELToHost", |
| 473 | Entry::Level::Informational, data); |
| 474 | if (_hostNotifier) |
Matt Spinler | a34ab72 | 2019-12-16 10:39:32 -0600 | [diff] [blame] | 475 | { |
| 476 | _hostNotifier->setBadPEL(pelID); |
| 477 | } |
Matt Spinler | 05c2c6c | 2019-12-18 14:02:09 -0600 | [diff] [blame] | 478 | } |
| 479 | else if ((reason == RejectionReason::HostFull) && _hostNotifier) |
| 480 | { |
| 481 | _hostNotifier->setHostFull(pelID); |
Matt Spinler | a34ab72 | 2019-12-16 10:39:32 -0600 | [diff] [blame] | 482 | } |
| 483 | } |
| 484 | |
Matt Spinler | 7e727a3 | 2020-07-07 15:00:17 -0500 | [diff] [blame] | 485 | void Manager::scheduleRepoPrune() |
| 486 | { |
Matt Spinler | 7e727a3 | 2020-07-07 15:00:17 -0500 | [diff] [blame] | 487 | _repoPrunerEventSource = std::make_unique<sdeventplus::source::Defer>( |
Matt Spinler | ff9cec2 | 2020-07-15 13:06:35 -0500 | [diff] [blame] | 488 | _event, std::bind(std::mem_fn(&Manager::pruneRepo), this, |
| 489 | std::placeholders::_1)); |
Matt Spinler | 7e727a3 | 2020-07-07 15:00:17 -0500 | [diff] [blame] | 490 | } |
| 491 | |
Patrick Williams | d26fa3e | 2021-04-21 15:22:23 -0500 | [diff] [blame] | 492 | void Manager::pruneRepo(sdeventplus::source::EventBase& /*source*/) |
Matt Spinler | 7e727a3 | 2020-07-07 15:00:17 -0500 | [diff] [blame] | 493 | { |
| 494 | auto idsToDelete = _repo.prune(); |
| 495 | |
| 496 | // Remove the OpenBMC event logs for the PELs that were just removed. |
| 497 | std::for_each(idsToDelete.begin(), idsToDelete.end(), |
| 498 | [this](auto id) { this->_logManager.erase(id); }); |
| 499 | |
| 500 | _repoPrunerEventSource.reset(); |
| 501 | } |
| 502 | |
Matt Spinler | ff9cec2 | 2020-07-15 13:06:35 -0500 | [diff] [blame] | 503 | void Manager::setupPELDeleteWatch() |
| 504 | { |
| 505 | _pelFileDeleteFD = inotify_init1(IN_NONBLOCK); |
| 506 | if (-1 == _pelFileDeleteFD) |
| 507 | { |
| 508 | auto e = errno; |
| 509 | std::string msg = |
| 510 | "inotify_init1 failed with errno " + std::to_string(e); |
| 511 | log<level::ERR>(msg.c_str()); |
| 512 | abort(); |
| 513 | } |
| 514 | |
| 515 | _pelFileDeleteWatchFD = inotify_add_watch( |
| 516 | _pelFileDeleteFD, _repo.repoPath().c_str(), IN_DELETE); |
| 517 | if (-1 == _pelFileDeleteWatchFD) |
| 518 | { |
| 519 | auto e = errno; |
| 520 | std::string msg = |
| 521 | "inotify_add_watch failed with error " + std::to_string(e); |
| 522 | log<level::ERR>(msg.c_str()); |
| 523 | abort(); |
| 524 | } |
| 525 | |
| 526 | _pelFileDeleteEventSource = std::make_unique<sdeventplus::source::IO>( |
| 527 | _event, _pelFileDeleteFD, EPOLLIN, |
| 528 | std::bind(std::mem_fn(&Manager::pelFileDeleted), this, |
| 529 | std::placeholders::_1, std::placeholders::_2, |
| 530 | std::placeholders::_3)); |
| 531 | } |
| 532 | |
Patrick Williams | d26fa3e | 2021-04-21 15:22:23 -0500 | [diff] [blame] | 533 | void Manager::pelFileDeleted(sdeventplus::source::IO& /*io*/, int /*fd*/, |
Matt Spinler | ff9cec2 | 2020-07-15 13:06:35 -0500 | [diff] [blame] | 534 | uint32_t revents) |
| 535 | { |
| 536 | if (!(revents & EPOLLIN)) |
| 537 | { |
| 538 | return; |
| 539 | } |
| 540 | |
| 541 | // An event for 1 PEL uses 48B. When all PELs are deleted at once, |
| 542 | // as many events as there is room for can be handled in one callback. |
| 543 | // A size of 2000 will allow 41 to be processed, with additional |
| 544 | // callbacks being needed to process the remaining ones. |
Matt Spinler | 9d59d58 | 2021-05-19 07:57:10 -0600 | [diff] [blame] | 545 | std::array<uint8_t, 2000> data{}; |
Matt Spinler | ff9cec2 | 2020-07-15 13:06:35 -0500 | [diff] [blame] | 546 | auto bytesRead = read(_pelFileDeleteFD, data.data(), data.size()); |
| 547 | if (bytesRead < 0) |
| 548 | { |
| 549 | auto e = errno; |
| 550 | std::string msg = "Failed reading data from inotify event, errno = " + |
| 551 | std::to_string(e); |
| 552 | log<level::ERR>(msg.c_str()); |
| 553 | abort(); |
| 554 | } |
| 555 | |
| 556 | auto offset = 0; |
| 557 | while (offset < bytesRead) |
| 558 | { |
| 559 | auto event = reinterpret_cast<inotify_event*>(&data[offset]); |
| 560 | if (event->mask & IN_DELETE) |
| 561 | { |
| 562 | std::string filename{event->name}; |
| 563 | |
| 564 | // Get the PEL ID from the filename and tell the |
| 565 | // repo it's been removed, and then delete the BMC |
| 566 | // event log if it's there. |
| 567 | auto pos = filename.find_first_of('_'); |
| 568 | if (pos != std::string::npos) |
| 569 | { |
| 570 | try |
| 571 | { |
| 572 | auto idString = filename.substr(pos + 1); |
| 573 | auto pelID = std::stoul(idString, nullptr, 16); |
| 574 | |
| 575 | Repository::LogID id{Repository::LogID::Pel(pelID)}; |
| 576 | auto removedLogID = _repo.remove(id); |
| 577 | if (removedLogID) |
| 578 | { |
| 579 | _logManager.erase(removedLogID->obmcID.id); |
| 580 | } |
| 581 | } |
| 582 | catch (const std::exception& e) |
| 583 | { |
| 584 | log<level::INFO>("Could not find PEL ID from its filename", |
| 585 | entry("FILENAME=%s", filename.c_str())); |
| 586 | } |
| 587 | } |
| 588 | } |
| 589 | |
| 590 | offset += offsetof(inotify_event, name) + event->len; |
| 591 | } |
| 592 | } |
Matt Spinler | 9cc3007 | 2020-09-16 15:39:34 -0500 | [diff] [blame] | 593 | |
| 594 | std::tuple<uint32_t, uint32_t> Manager::createPELWithFFDCFiles( |
| 595 | std::string message, Entry::Level severity, |
| 596 | std::map<std::string, std::string> additionalData, |
| 597 | std::vector<std::tuple< |
| 598 | sdbusplus::xyz::openbmc_project::Logging::server::Create::FFDCFormat, |
| 599 | uint8_t, uint8_t, sdbusplus::message::unix_fd>> |
| 600 | fFDC) |
| 601 | { |
Matt Spinler | 44893cc | 2020-08-26 11:34:17 -0500 | [diff] [blame] | 602 | _logManager.createWithFFDC(message, severity, additionalData, fFDC); |
| 603 | |
| 604 | return {_logManager.lastEntryID(), _repo.lastPelID()}; |
Matt Spinler | 9cc3007 | 2020-09-16 15:39:34 -0500 | [diff] [blame] | 605 | } |
| 606 | |
Andrew Geissler | 44fc316 | 2020-07-09 09:21:31 -0500 | [diff] [blame] | 607 | void Manager::checkPelAndQuiesce(std::unique_ptr<openpower::pels::PEL>& pel) |
| 608 | { |
Matt Spinler | b2abc04 | 2021-05-17 15:32:50 -0600 | [diff] [blame] | 609 | if ((pel->userHeader().severity() == |
| 610 | static_cast<uint8_t>(SeverityType::nonError)) || |
| 611 | (pel->userHeader().severity() == |
| 612 | static_cast<uint8_t>(SeverityType::recovered))) |
Andrew Geissler | 44fc316 | 2020-07-09 09:21:31 -0500 | [diff] [blame] | 613 | { |
Matt Spinler | b2abc04 | 2021-05-17 15:32:50 -0600 | [diff] [blame] | 614 | log<level::DEBUG>( |
| 615 | "PEL severity informational or recovered. no quiesce needed"); |
Andrew Geissler | 44fc316 | 2020-07-09 09:21:31 -0500 | [diff] [blame] | 616 | return; |
| 617 | } |
| 618 | if (!_logManager.isQuiesceOnErrorEnabled()) |
| 619 | { |
| 620 | log<level::DEBUG>("QuiesceOnHwError not enabled, no quiesce needed"); |
| 621 | return; |
| 622 | } |
| 623 | |
| 624 | // Now check if it has any type of callout |
| 625 | if (pel->isCalloutPresent()) |
| 626 | { |
Matt Spinler | b2abc04 | 2021-05-17 15:32:50 -0600 | [diff] [blame] | 627 | log<level::INFO>( |
| 628 | "QuiesceOnHwError enabled, PEL severity not nonError or recovered, " |
| 629 | "and callout is present"); |
Andrew Geissler | 44fc316 | 2020-07-09 09:21:31 -0500 | [diff] [blame] | 630 | |
| 631 | _logManager.quiesceOnError(pel->obmcLogID()); |
| 632 | } |
| 633 | } |
| 634 | |
Vijay Lobo | d354a39 | 2021-06-01 16:21:02 -0500 | [diff] [blame] | 635 | std::string Manager::getEventId(const openpower::pels::PEL& pel) const |
| 636 | { |
| 637 | std::string str; |
| 638 | auto src = pel.primarySRC(); |
| 639 | if (src) |
| 640 | { |
| 641 | const auto& hexwords = (*src)->hexwordData(); |
| 642 | |
| 643 | std::string refcode = (*src)->asciiString(); |
| 644 | size_t pos = refcode.find_last_not_of(0x20); |
| 645 | if (pos != std::string::npos) |
| 646 | { |
| 647 | refcode.erase(pos + 1); |
| 648 | } |
| 649 | str = refcode; |
| 650 | |
| 651 | for (auto& value : hexwords) |
| 652 | { |
| 653 | str += " "; |
| 654 | str += getNumberString("%08X", value); |
| 655 | } |
| 656 | } |
| 657 | return str; |
| 658 | } |
| 659 | |
| 660 | void Manager::updateEventId(std::unique_ptr<openpower::pels::PEL>& pel) |
| 661 | { |
| 662 | std::string eventIdStr = getEventId(*pel); |
| 663 | |
| 664 | auto entryN = _logManager.entries.find(pel->obmcLogID()); |
| 665 | if (entryN != _logManager.entries.end()) |
| 666 | { |
| 667 | entryN->second->eventId(eventIdStr); |
| 668 | } |
| 669 | } |
| 670 | |
Vijay Lobo | 593a4c6 | 2021-06-16 14:25:26 -0500 | [diff] [blame] | 671 | std::string Manager::getResolution(const openpower::pels::PEL& pel) const |
| 672 | { |
| 673 | std::string str; |
| 674 | std::string resolution; |
| 675 | auto src = pel.primarySRC(); |
| 676 | if (src) |
| 677 | { |
| 678 | // First extract the callout pointer and then go through |
| 679 | const auto& callouts = (*src)->callouts(); |
| 680 | namespace pv = openpower::pels::pel_values; |
| 681 | // All PELs dont have callout, check before parsing callout data |
| 682 | if (callouts) |
| 683 | { |
| 684 | const auto& entries = callouts->callouts(); |
| 685 | // Entry starts with index 1 |
| 686 | uint8_t index = 1; |
| 687 | for (auto& entry : entries) |
| 688 | { |
| 689 | resolution += std::to_string(index) + ". "; |
| 690 | // Adding Location code to resolution |
| 691 | if (!entry->locationCode().empty()) |
| 692 | resolution += |
| 693 | "Location Code: " + entry->locationCode() + ", "; |
| 694 | if (entry->fruIdentity()) |
| 695 | { |
| 696 | // Get priority and set the resolution string |
| 697 | str = pv::getValue(entry->priority(), |
| 698 | pel_values::calloutPriorityValues, |
| 699 | pel_values::registryNamePos); |
| 700 | str[0] = toupper(str[0]); |
| 701 | resolution += "Priority: " + str + ", "; |
| 702 | if (entry->fruIdentity()->getPN().has_value()) |
| 703 | { |
| 704 | resolution += |
| 705 | "PN: " + entry->fruIdentity()->getPN().value() + |
| 706 | ", "; |
| 707 | } |
| 708 | if (entry->fruIdentity()->getSN().has_value()) |
| 709 | { |
| 710 | resolution += |
| 711 | "SN: " + entry->fruIdentity()->getSN().value() + |
| 712 | ", "; |
| 713 | } |
| 714 | if (entry->fruIdentity()->getCCIN().has_value()) |
| 715 | { |
| 716 | resolution += |
| 717 | "CCIN: " + entry->fruIdentity()->getCCIN().value() + |
| 718 | ", "; |
| 719 | } |
| 720 | // Add the maintenance procedure |
| 721 | if (entry->fruIdentity()->getMaintProc().has_value()) |
| 722 | { |
| 723 | resolution += |
| 724 | "Procedure: " + |
| 725 | entry->fruIdentity()->getMaintProc().value() + ", "; |
| 726 | } |
| 727 | } |
| 728 | resolution.resize(resolution.size() - 2); |
| 729 | resolution += "\n"; |
| 730 | index++; |
| 731 | } |
| 732 | } |
| 733 | } |
| 734 | return resolution; |
| 735 | } |
| 736 | |
| 737 | void Manager::updateResolution(std::unique_ptr<openpower::pels::PEL>& pel) |
| 738 | { |
| 739 | std::string callouts = getResolution(*pel); |
| 740 | auto entryN = _logManager.entries.find(pel->obmcLogID()); |
| 741 | if (entryN != _logManager.entries.end()) |
| 742 | { |
| 743 | entryN->second->resolution(callouts); |
| 744 | } |
| 745 | } |
| 746 | |
Adriana Kobylak | e7d271a | 2020-12-07 14:32:44 -0600 | [diff] [blame] | 747 | void Manager::setEntryPath(uint32_t obmcLogID) |
| 748 | { |
| 749 | Repository::LogID id{Repository::LogID::Obmc(obmcLogID)}; |
| 750 | if (auto attributes = _repo.getPELAttributes(id); attributes) |
| 751 | { |
| 752 | auto& attr = attributes.value().get(); |
| 753 | auto entry = _logManager.entries.find(obmcLogID); |
| 754 | if (entry != _logManager.entries.end()) |
| 755 | { |
| 756 | entry->second->path(attr.path); |
| 757 | } |
| 758 | } |
| 759 | } |
| 760 | |
Vijay Lobo | cbc93a4 | 2021-05-20 19:04:07 -0500 | [diff] [blame] | 761 | void Manager::setServiceProviderNotifyFlag(uint32_t obmcLogID) |
| 762 | { |
| 763 | Repository::LogID id{Repository::LogID::Obmc(obmcLogID)}; |
| 764 | if (auto attributes = _repo.getPELAttributes(id); attributes) |
| 765 | { |
| 766 | auto& attr = attributes.value().get(); |
| 767 | auto entry = _logManager.entries.find(obmcLogID); |
| 768 | if (entry != _logManager.entries.end()) |
| 769 | { |
| 770 | if (attr.actionFlags.test(callHomeFlagBit)) |
| 771 | { |
| 772 | entry->second->serviceProviderNotify(true); |
| 773 | } |
| 774 | else |
| 775 | { |
| 776 | entry->second->serviceProviderNotify(false); |
| 777 | } |
| 778 | } |
| 779 | } |
| 780 | } |
| 781 | |
Ramesh Iyyar | f4203c4 | 2021-06-24 06:09:23 -0500 | [diff] [blame] | 782 | uint32_t Manager::getPELIdFromBMCLogId(uint32_t bmcLogId) |
| 783 | { |
| 784 | Repository::LogID id{Repository::LogID::Obmc(bmcLogId)}; |
| 785 | if (auto logId = _repo.getLogID(id); !logId.has_value()) |
| 786 | { |
| 787 | throw common_error::InvalidArgument(); |
| 788 | } |
| 789 | else |
| 790 | { |
| 791 | return logId->pelID.id; |
| 792 | } |
| 793 | } |
| 794 | |
Ramesh Iyyar | 530efbf | 2021-06-24 06:22:22 -0500 | [diff] [blame] | 795 | uint32_t Manager::getBMCLogIdFromPELId(uint32_t pelId) |
| 796 | { |
| 797 | Repository::LogID id{Repository::LogID::Pel(pelId)}; |
| 798 | if (auto logId = _repo.getLogID(id); !logId.has_value()) |
| 799 | { |
| 800 | throw common_error::InvalidArgument(); |
| 801 | } |
| 802 | else |
| 803 | { |
| 804 | return logId->obmcID.id; |
| 805 | } |
| 806 | } |
| 807 | |
Matt Spinler | 4e8078c | 2019-07-09 13:22:32 -0500 | [diff] [blame] | 808 | } // namespace pels |
| 809 | } // namespace openpower |