blob: c1344df041e1cb5dd62c1125a2363cbdbb8a350b [file] [log] [blame]
Jason M. Bills1490b142019-07-01 15:48:43 -07001/*
2// Copyright (c) 2019 Intel Corporation
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8// http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15*/
Jason M. Bills6a2cb692019-08-06 11:03:49 -070016#include <peci.h>
Chen Yugange6c0f1c2019-08-02 20:36:42 +080017#include <systemd/sd-journal.h>
18
Jason M. Bills6a2cb692019-08-06 11:03:49 -070019#include <bitset>
Jason M. Bills1490b142019-07-01 15:48:43 -070020#include <boost/asio/posix/stream_descriptor.hpp>
21#include <gpiod.hpp>
22#include <iostream>
23#include <sdbusplus/asio/object_server.hpp>
Jason M. Billsd1a19f62019-08-06 11:52:58 -070024#include <variant>
Jason M. Bills1490b142019-07-01 15:48:43 -070025
26namespace host_error_monitor
27{
28static boost::asio::io_service io;
29static std::shared_ptr<sdbusplus::asio::connection> conn;
Jason M. Billsc4b91f22019-11-26 17:04:50 -080030static std::shared_ptr<sdbusplus::asio::dbus_interface> hostErrorTimeoutIface;
Jason M. Bills1490b142019-07-01 15:48:43 -070031
32static bool hostOff = true;
33
Jason M. Billsc4b91f22019-11-26 17:04:50 -080034static size_t caterrTimeoutMs = 2000;
35const static constexpr size_t caterrTimeoutMsMax = 600000; // 10 minutes maximum
Jason M. Billscbf78532019-08-16 15:32:11 -070036const static constexpr size_t errTimeoutMs = 90000;
Jason M. Bills89922f82019-08-06 11:10:02 -070037const static constexpr size_t smiTimeoutMs = 90000;
Jason M. Bills1490b142019-07-01 15:48:43 -070038const static constexpr size_t crashdumpTimeoutS = 300;
39
40// Timers
41// Timer for CATERR asserted
42static boost::asio::steady_timer caterrAssertTimer(io);
Jason M. Bills8c584392019-08-19 11:05:51 -070043// Timer for ERR0 asserted
44static boost::asio::steady_timer err0AssertTimer(io);
Jason M. Bills75af3962019-08-19 11:07:17 -070045// Timer for ERR1 asserted
46static boost::asio::steady_timer err1AssertTimer(io);
Jason M. Bills6a2cb692019-08-06 11:03:49 -070047// Timer for ERR2 asserted
48static boost::asio::steady_timer err2AssertTimer(io);
Jason M. Bills89922f82019-08-06 11:10:02 -070049// Timer for SMI asserted
50static boost::asio::steady_timer smiAssertTimer(io);
Jason M. Bills1490b142019-07-01 15:48:43 -070051
52// GPIO Lines and Event Descriptors
53static gpiod::line caterrLine;
54static boost::asio::posix::stream_descriptor caterrEvent(io);
Jason M. Bills8c584392019-08-19 11:05:51 -070055static gpiod::line err0Line;
56static boost::asio::posix::stream_descriptor err0Event(io);
Jason M. Bills75af3962019-08-19 11:07:17 -070057static gpiod::line err1Line;
58static boost::asio::posix::stream_descriptor err1Event(io);
Jason M. Bills6a2cb692019-08-06 11:03:49 -070059static gpiod::line err2Line;
60static boost::asio::posix::stream_descriptor err2Event(io);
Jason M. Bills89922f82019-08-06 11:10:02 -070061static gpiod::line smiLine;
62static boost::asio::posix::stream_descriptor smiEvent(io);
Jason M. Bills45e87e02019-09-09 14:45:38 -070063static gpiod::line cpu1FIVRFaultLine;
Jason M. Bills78c5eed2019-08-28 14:00:40 -070064static gpiod::line cpu1ThermtripLine;
65static boost::asio::posix::stream_descriptor cpu1ThermtripEvent(io);
Jason M. Bills45e87e02019-09-09 14:45:38 -070066static gpiod::line cpu2FIVRFaultLine;
Jason M. Bills78c5eed2019-08-28 14:00:40 -070067static gpiod::line cpu2ThermtripLine;
68static boost::asio::posix::stream_descriptor cpu2ThermtripEvent(io);
Jason M. Bills250fa632019-08-28 15:58:25 -070069static gpiod::line cpu1VRHotLine;
70static boost::asio::posix::stream_descriptor cpu1VRHotEvent(io);
71static gpiod::line cpu2VRHotLine;
Jason M. Bills9647ba72019-08-29 14:19:19 -070072static boost::asio::posix::stream_descriptor cpu1MemABCDVRHotEvent(io);
73static gpiod::line cpu1MemEFGHVRHotLine;
74static boost::asio::posix::stream_descriptor cpu1MemEFGHVRHotEvent(io);
75static gpiod::line cpu2MemABCDVRHotLine;
Jason M. Bills250fa632019-08-28 15:58:25 -070076static boost::asio::posix::stream_descriptor cpu2VRHotEvent(io);
Jason M. Bills9647ba72019-08-29 14:19:19 -070077static gpiod::line cpu1MemABCDVRHotLine;
78static boost::asio::posix::stream_descriptor cpu2MemABCDVRHotEvent(io);
79static gpiod::line cpu2MemEFGHVRHotLine;
80static boost::asio::posix::stream_descriptor cpu2MemEFGHVRHotEvent(io);
Chen Yugange6c0f1c2019-08-02 20:36:42 +080081//----------------------------------
82// PCH_BMC_THERMTRIP function related definition
83//----------------------------------
Chen Yugange6c0f1c2019-08-02 20:36:42 +080084static gpiod::line pchThermtripLine;
85static boost::asio::posix::stream_descriptor pchThermtripEvent(io);
jayaprakash Mutyala009adbc2019-12-24 22:08:07 +000086//----------------------------------
87// CPU_MEM_THERM_EVENT function related definition
88//----------------------------------
89static gpiod::line cpu1MemtripLine;
90static boost::asio::posix::stream_descriptor cpu1MemtripEvent(io);
91static gpiod::line cpu2MemtripLine;
92static boost::asio::posix::stream_descriptor cpu2MemtripEvent(io);
Jason M. Bills1490b142019-07-01 15:48:43 -070093
Yong Li061eb032020-02-26 15:06:18 +080094// beep function for CPU error
95const static constexpr uint8_t beepCPUErr2 = 5;
96
97static void beep(const uint8_t& beepPriority)
98{
99 conn->async_method_call(
100 [](boost::system::error_code ec) {
101 if (ec)
102 {
103 std::cerr << "beep returned error with "
104 "async_method_call (ec = "
105 << ec << ")\n";
106 return;
107 }
108 },
109 "xyz.openbmc_project.BeepCode", "/xyz/openbmc_project/BeepCode",
110 "xyz.openbmc_project.BeepCode", "Beep", uint8_t(beepPriority));
111}
112
Jason M. Billsa3397932019-08-06 11:07:21 -0700113static void cpuIERRLog()
114{
115 sd_journal_send("MESSAGE=HostError: IERR", "PRIORITY=%i", LOG_INFO,
116 "REDFISH_MESSAGE_ID=%s", "OpenBMC.0.1.CPUError",
117 "REDFISH_MESSAGE_ARGS=%s", "IERR", NULL);
118}
119
120static void cpuIERRLog(const int cpuNum)
121{
122 std::string msg = "IERR on CPU " + std::to_string(cpuNum + 1);
123
124 sd_journal_send("MESSAGE=HostError: %s", msg.c_str(), "PRIORITY=%i",
125 LOG_INFO, "REDFISH_MESSAGE_ID=%s", "OpenBMC.0.1.CPUError",
126 "REDFISH_MESSAGE_ARGS=%s", msg.c_str(), NULL);
127}
128
129static void cpuIERRLog(const int cpuNum, const std::string& type)
130{
131 std::string msg = type + " IERR on CPU " + std::to_string(cpuNum + 1);
132
133 sd_journal_send("MESSAGE=HostError: %s", msg.c_str(), "PRIORITY=%i",
134 LOG_INFO, "REDFISH_MESSAGE_ID=%s", "OpenBMC.0.1.CPUError",
135 "REDFISH_MESSAGE_ARGS=%s", msg.c_str(), NULL);
136}
137
Jason M. Billscbf78532019-08-16 15:32:11 -0700138static void cpuERRXLog(const int errPin)
Jason M. Bills6a2cb692019-08-06 11:03:49 -0700139{
Jason M. Billscbf78532019-08-16 15:32:11 -0700140 std::string msg = "ERR" + std::to_string(errPin) + " Timeout";
141
142 sd_journal_send("MESSAGE=HostError: %s", msg.c_str(), "PRIORITY=%i",
143 LOG_INFO, "REDFISH_MESSAGE_ID=%s", "OpenBMC.0.1.CPUError",
144 "REDFISH_MESSAGE_ARGS=%s", msg.c_str(), NULL);
Jason M. Bills6a2cb692019-08-06 11:03:49 -0700145}
146
Jason M. Billscbf78532019-08-16 15:32:11 -0700147static void cpuERRXLog(const int errPin, const int cpuNum)
Jason M. Bills6a2cb692019-08-06 11:03:49 -0700148{
Jason M. Billscbf78532019-08-16 15:32:11 -0700149 std::string msg = "ERR" + std::to_string(errPin) + " Timeout on CPU " +
150 std::to_string(cpuNum + 1);
Jason M. Bills6a2cb692019-08-06 11:03:49 -0700151
152 sd_journal_send("MESSAGE=HostError: %s", msg.c_str(), "PRIORITY=%i",
153 LOG_INFO, "REDFISH_MESSAGE_ID=%s", "OpenBMC.0.1.CPUError",
154 "REDFISH_MESSAGE_ARGS=%s", msg.c_str(), NULL);
155}
156
Jason M. Bills89922f82019-08-06 11:10:02 -0700157static void smiTimeoutLog()
158{
159 sd_journal_send("MESSAGE=HostError: SMI Timeout", "PRIORITY=%i", LOG_INFO,
160 "REDFISH_MESSAGE_ID=%s", "OpenBMC.0.1.CPUError",
161 "REDFISH_MESSAGE_ARGS=%s", "SMI Timeout", NULL);
162}
163
Jason M. Bills45e87e02019-09-09 14:45:38 -0700164static void cpuBootFIVRFaultLog(const int cpuNum)
165{
166 std::string msg = "Boot FIVR Fault on CPU " + std::to_string(cpuNum);
167
168 sd_journal_send("MESSAGE=HostError: %s", msg.c_str(), "PRIORITY=%i",
169 LOG_INFO, "REDFISH_MESSAGE_ID=%s", "OpenBMC.0.1.CPUError",
170 "REDFISH_MESSAGE_ARGS=%s", msg.c_str(), NULL);
171}
172
Jason M. Bills78c5eed2019-08-28 14:00:40 -0700173static void cpuThermTripLog(const int cpuNum)
174{
175 std::string msg = "CPU " + std::to_string(cpuNum) + " thermal trip";
176
177 sd_journal_send("MESSAGE=HostError: %s", msg.c_str(), "PRIORITY=%i",
178 LOG_INFO, "REDFISH_MESSAGE_ID=%s",
179 "OpenBMC.0.1.CPUThermalTrip", "REDFISH_MESSAGE_ARGS=%d",
180 cpuNum, NULL);
181}
182
jayaprakash Mutyala009adbc2019-12-24 22:08:07 +0000183static void memThermTripLog(const int cpuNum)
184{
185 std::string cpuNumber = "CPU " + std::to_string(cpuNum);
186 std::string msg = cpuNumber + " Memory Thermal trip.";
187
188 sd_journal_send("MESSAGE=HostError: %s", msg.c_str(), "PRIORITY=%i",
189 LOG_ERR, "REDFISH_MESSAGE_ID=%s",
190 "OpenBMC.0.1.MemoryThermTrip", "REDFISH_MESSAGE_ARGS=%s",
191 cpuNumber.c_str(), NULL);
192}
193
Jason M. Bills250fa632019-08-28 15:58:25 -0700194static void cpuVRHotLog(const std::string& vr)
195{
196 std::string msg = vr + " Voltage Regulator Overheated.";
197
198 sd_journal_send("MESSAGE=HostError: %s", msg.c_str(), "PRIORITY=%i",
199 LOG_INFO, "REDFISH_MESSAGE_ID=%s",
200 "OpenBMC.0.1.VoltageRegulatorOverheated",
201 "REDFISH_MESSAGE_ARGS=%s", vr.c_str(), NULL);
202}
203
Jason M. Bills08866542019-08-16 12:04:19 -0700204static void ssbThermTripLog()
205{
206 sd_journal_send("MESSAGE=HostError: SSB thermal trip", "PRIORITY=%i",
207 LOG_INFO, "REDFISH_MESSAGE_ID=%s",
208 "OpenBMC.0.1.SsbThermalTrip", NULL);
209}
210
Jason M. Billsa15c2522019-08-16 10:01:44 -0700211static void initializeErrorState();
Jason M. Bills1490b142019-07-01 15:48:43 -0700212static void initializeHostState()
213{
214 conn->async_method_call(
215 [](boost::system::error_code ec,
216 const std::variant<std::string>& property) {
217 if (ec)
218 {
219 return;
220 }
221 const std::string* state = std::get_if<std::string>(&property);
222 if (state == nullptr)
223 {
224 std::cerr << "Unable to read host state value\n";
225 return;
226 }
227 hostOff = *state == "xyz.openbmc_project.State.Host.HostState.Off";
Jason M. Billsa15c2522019-08-16 10:01:44 -0700228 // If the system is on, initialize the error state
229 if (!hostOff)
230 {
231 initializeErrorState();
232 }
Jason M. Bills1490b142019-07-01 15:48:43 -0700233 },
234 "xyz.openbmc_project.State.Host", "/xyz/openbmc_project/state/host0",
235 "org.freedesktop.DBus.Properties", "Get",
236 "xyz.openbmc_project.State.Host", "CurrentHostState");
237}
238
239static std::shared_ptr<sdbusplus::bus::match::match> startHostStateMonitor()
240{
241 return std::make_shared<sdbusplus::bus::match::match>(
242 *conn,
243 "type='signal',interface='org.freedesktop.DBus.Properties',"
244 "member='PropertiesChanged',arg0namespace='xyz.openbmc_project.State."
245 "Host'",
246 [](sdbusplus::message::message& msg) {
247 std::string interfaceName;
248 boost::container::flat_map<std::string, std::variant<std::string>>
249 propertiesChanged;
250 std::string state;
251 try
252 {
253 msg.read(interfaceName, propertiesChanged);
254 state =
255 std::get<std::string>(propertiesChanged.begin()->second);
256 }
257 catch (std::exception& e)
258 {
259 std::cerr << "Unable to read host state\n";
260 return;
261 }
262 hostOff = state == "xyz.openbmc_project.State.Host.HostState.Off";
263
Jason M. Bills1490b142019-07-01 15:48:43 -0700264 if (hostOff)
265 {
Jason M. Billse94f5e12019-09-13 11:11:34 -0700266 // No host events should fire while off, so cancel any pending
267 // timers
Jason M. Bills1490b142019-07-01 15:48:43 -0700268 caterrAssertTimer.cancel();
Jason M. Bills8c584392019-08-19 11:05:51 -0700269 err0AssertTimer.cancel();
Jason M. Bills75af3962019-08-19 11:07:17 -0700270 err1AssertTimer.cancel();
Jason M. Bills6a2cb692019-08-06 11:03:49 -0700271 err2AssertTimer.cancel();
Jason M. Bills89922f82019-08-06 11:10:02 -0700272 smiAssertTimer.cancel();
Jason M. Bills1490b142019-07-01 15:48:43 -0700273 }
Jason M. Billse94f5e12019-09-13 11:11:34 -0700274 else
275 {
276 // Handle any initial errors when the host turns on
277 initializeErrorState();
278 }
Jason M. Bills1490b142019-07-01 15:48:43 -0700279 });
280}
281
282static bool requestGPIOEvents(
283 const std::string& name, const std::function<void()>& handler,
284 gpiod::line& gpioLine,
285 boost::asio::posix::stream_descriptor& gpioEventDescriptor)
286{
287 // Find the GPIO line
288 gpioLine = gpiod::find_line(name);
289 if (!gpioLine)
290 {
291 std::cerr << "Failed to find the " << name << " line\n";
292 return false;
293 }
294
295 try
296 {
297 gpioLine.request(
298 {"host-error-monitor", gpiod::line_request::EVENT_BOTH_EDGES});
299 }
300 catch (std::exception&)
301 {
302 std::cerr << "Failed to request events for " << name << "\n";
303 return false;
304 }
305
306 int gpioLineFd = gpioLine.event_get_fd();
307 if (gpioLineFd < 0)
308 {
309 std::cerr << "Failed to get " << name << " fd\n";
310 return false;
311 }
312
313 gpioEventDescriptor.assign(gpioLineFd);
314
315 gpioEventDescriptor.async_wait(
316 boost::asio::posix::stream_descriptor::wait_read,
317 [&name, handler](const boost::system::error_code ec) {
318 if (ec)
319 {
320 std::cerr << name << " fd handler error: " << ec.message()
321 << "\n";
322 return;
323 }
324 handler();
325 });
326 return true;
327}
328
Jason M. Bills45e87e02019-09-09 14:45:38 -0700329static bool requestGPIOInput(const std::string& name, gpiod::line& gpioLine)
330{
331 // Find the GPIO line
332 gpioLine = gpiod::find_line(name);
333 if (!gpioLine)
334 {
335 std::cerr << "Failed to find the " << name << " line.\n";
336 return false;
337 }
338
339 // Request GPIO input
340 try
341 {
342 gpioLine.request({__FUNCTION__, gpiod::line_request::DIRECTION_INPUT});
343 }
344 catch (std::exception&)
345 {
346 std::cerr << "Failed to request " << name << " input\n";
347 return false;
348 }
349
350 return true;
351}
352
Jason M. Bills1490b142019-07-01 15:48:43 -0700353static void startPowerCycle()
354{
355 conn->async_method_call(
356 [](boost::system::error_code ec) {
357 if (ec)
358 {
359 std::cerr << "failed to set Chassis State\n";
360 }
361 },
362 "xyz.openbmc_project.State.Chassis",
363 "/xyz/openbmc_project/state/chassis0",
364 "org.freedesktop.DBus.Properties", "Set",
365 "xyz.openbmc_project.State.Chassis", "RequestedPowerTransition",
366 std::variant<std::string>{
367 "xyz.openbmc_project.State.Chassis.Transition.PowerCycle"});
368}
369
Jason M. Billsb61766b2019-11-26 17:02:44 -0800370static void startCrashdumpAndRecovery(bool recoverSystem,
371 const std::string& triggerType)
Jason M. Bills1490b142019-07-01 15:48:43 -0700372{
373 std::cout << "Starting crashdump\n";
374 static std::shared_ptr<sdbusplus::bus::match::match> crashdumpCompleteMatch;
375 static boost::asio::steady_timer crashdumpTimer(io);
376
377 crashdumpCompleteMatch = std::make_shared<sdbusplus::bus::match::match>(
378 *conn,
379 "type='signal',interface='org.freedesktop.DBus.Properties',"
380 "member='PropertiesChanged',arg0namespace='com.intel.crashdump'",
381 [recoverSystem](sdbusplus::message::message& msg) {
382 crashdumpTimer.cancel();
383 std::cout << "Crashdump completed\n";
384 if (recoverSystem)
385 {
386 std::cout << "Recovering the system\n";
387 startPowerCycle();
388 }
389 crashdumpCompleteMatch.reset();
390 });
391
392 crashdumpTimer.expires_after(std::chrono::seconds(crashdumpTimeoutS));
393 crashdumpTimer.async_wait([](const boost::system::error_code ec) {
394 if (ec)
395 {
396 // operation_aborted is expected if timer is canceled
397 if (ec != boost::asio::error::operation_aborted)
398 {
399 std::cerr << "Crashdump async_wait failed: " << ec.message()
400 << "\n";
401 }
402 std::cout << "Crashdump timer canceled\n";
403 return;
404 }
405 std::cerr << "Crashdump failed to complete before timeout\n";
406 crashdumpCompleteMatch.reset();
407 });
408
409 conn->async_method_call(
410 [](boost::system::error_code ec) {
411 if (ec)
412 {
413 std::cerr << "failed to start Crashdump\n";
414 crashdumpTimer.cancel();
415 crashdumpCompleteMatch.reset();
416 }
417 },
418 "com.intel.crashdump", "/com/intel/crashdump",
Jason M. Billsb61766b2019-11-26 17:02:44 -0800419 "com.intel.crashdump.Stored", "GenerateStoredLog", triggerType);
Jason M. Bills1490b142019-07-01 15:48:43 -0700420}
421
Jason M. Billsd1a19f62019-08-06 11:52:58 -0700422static void incrementCPUErrorCount(int cpuNum)
423{
424 std::string propertyName = "ErrorCountCPU" + std::to_string(cpuNum + 1);
425
426 // Get the current count
427 conn->async_method_call(
428 [propertyName](boost::system::error_code ec,
429 const std::variant<uint8_t>& property) {
430 if (ec)
431 {
432 std::cerr << "Failed to read " << propertyName << ": "
433 << ec.message() << "\n";
434 return;
435 }
436 const uint8_t* errorCountVariant = std::get_if<uint8_t>(&property);
437 if (errorCountVariant == nullptr)
438 {
439 std::cerr << propertyName << " invalid\n";
440 return;
441 }
442 uint8_t errorCount = *errorCountVariant;
443 if (errorCount == std::numeric_limits<uint8_t>::max())
444 {
445 std::cerr << "Maximum error count reached\n";
446 return;
447 }
448 // Increment the count
449 errorCount++;
450 conn->async_method_call(
451 [propertyName](boost::system::error_code ec) {
452 if (ec)
453 {
454 std::cerr << "Failed to set " << propertyName << ": "
455 << ec.message() << "\n";
456 }
457 },
458 "xyz.openbmc_project.Settings",
459 "/xyz/openbmc_project/control/processor_error_config",
460 "org.freedesktop.DBus.Properties", "Set",
461 "xyz.openbmc_project.Control.Processor.ErrConfig", propertyName,
462 std::variant<uint8_t>{errorCount});
463 },
464 "xyz.openbmc_project.Settings",
465 "/xyz/openbmc_project/control/processor_error_config",
466 "org.freedesktop.DBus.Properties", "Get",
467 "xyz.openbmc_project.Control.Processor.ErrConfig", propertyName);
468}
469
Jason M. Billsa3397932019-08-06 11:07:21 -0700470static bool checkIERRCPUs()
471{
472 bool cpuIERRFound = false;
473 for (int cpu = 0, addr = MIN_CLIENT_ADDR; addr <= MAX_CLIENT_ADDR;
474 cpu++, addr++)
475 {
476 uint8_t cc = 0;
477 CPUModel model{};
478 uint8_t stepping = 0;
479 if (peci_GetCPUID(addr, &model, &stepping, &cc) != PECI_CC_SUCCESS)
480 {
481 std::cerr << "Cannot get CPUID!\n";
482 continue;
483 }
484
485 switch (model)
486 {
487 case skx:
488 {
489 // First check the MCA_ERR_SRC_LOG to see if this is the CPU
490 // that caused the IERR
491 uint32_t mcaErrSrcLog = 0;
492 if (peci_RdPkgConfig(addr, 0, 5, 4, (uint8_t*)&mcaErrSrcLog,
493 &cc) != PECI_CC_SUCCESS)
494 {
495 continue;
496 }
497 // Check MSMI_INTERNAL (20) and IERR_INTERNAL (27)
498 if ((mcaErrSrcLog & (1 << 20)) || (mcaErrSrcLog & (1 << 27)))
499 {
500 // TODO: Light the CPU fault LED?
501 cpuIERRFound = true;
Jason M. Billsd1a19f62019-08-06 11:52:58 -0700502 incrementCPUErrorCount(cpu);
Jason M. Billsa3397932019-08-06 11:07:21 -0700503 // Next check if it's a CPU/VR mismatch by reading the
504 // IA32_MC4_STATUS MSR (0x411)
505 uint64_t mc4Status = 0;
506 if (peci_RdIAMSR(addr, 0, 0x411, &mc4Status, &cc) !=
507 PECI_CC_SUCCESS)
508 {
509 continue;
510 }
511 // Check MSEC bits 31:24 for
512 // MCA_SVID_VCCIN_VR_ICC_MAX_FAILURE (0x40),
513 // MCA_SVID_VCCIN_VR_VOUT_FAILURE (0x42), or
514 // MCA_SVID_CPU_VR_CAPABILITY_ERROR (0x43)
515 if ((mc4Status & (0x40 << 24)) ||
516 (mc4Status & (0x42 << 24)) ||
517 (mc4Status & (0x43 << 24)))
518 {
519 cpuIERRLog(cpu, "CPU/VR Mismatch");
520 continue;
521 }
522
523 // Next check if it's a Core FIVR fault by looking for a
524 // non-zero value of CORE_FIVR_ERR_LOG (B(1) D30 F2 offset
525 // 80h)
526 uint32_t coreFIVRErrLog = 0;
527 if (peci_RdPCIConfigLocal(
528 addr, 1, 30, 2, 0x80, sizeof(uint32_t),
529 (uint8_t*)&coreFIVRErrLog, &cc) != PECI_CC_SUCCESS)
530 {
531 continue;
532 }
533 if (coreFIVRErrLog)
534 {
535 cpuIERRLog(cpu, "Core FIVR Fault");
536 continue;
537 }
538
539 // Next check if it's an Uncore FIVR fault by looking for a
540 // non-zero value of UNCORE_FIVR_ERR_LOG (B(1) D30 F2 offset
541 // 84h)
542 uint32_t uncoreFIVRErrLog = 0;
543 if (peci_RdPCIConfigLocal(addr, 1, 30, 2, 0x84,
544 sizeof(uint32_t),
545 (uint8_t*)&uncoreFIVRErrLog,
546 &cc) != PECI_CC_SUCCESS)
547 {
548 continue;
549 }
550 if (uncoreFIVRErrLog)
551 {
552 cpuIERRLog(cpu, "Uncore FIVR Fault");
553 continue;
554 }
555
556 // Last if CORE_FIVR_ERR_LOG and UNCORE_FIVR_ERR_LOG are
557 // both zero, but MSEC bits 31:24 have either
558 // MCA_FIVR_CATAS_OVERVOL_FAULT (0x51) or
559 // MCA_FIVR_CATAS_OVERCUR_FAULT (0x52), then log it as an
560 // uncore FIVR fault
561 if (!coreFIVRErrLog && !uncoreFIVRErrLog &&
562 ((mc4Status & (0x51 << 24)) ||
563 (mc4Status & (0x52 << 24))))
564 {
565 cpuIERRLog(cpu, "Uncore FIVR Fault");
566 continue;
567 }
568 cpuIERRLog(cpu);
569 }
570 break;
571 }
572 case icx:
573 {
574 // First check the MCA_ERR_SRC_LOG to see if this is the CPU
575 // that caused the IERR
576 uint32_t mcaErrSrcLog = 0;
577 if (peci_RdPkgConfig(addr, 0, 5, 4, (uint8_t*)&mcaErrSrcLog,
578 &cc) != PECI_CC_SUCCESS)
579 {
580 continue;
581 }
582 // Check MSMI_INTERNAL (20) and IERR_INTERNAL (27)
583 if ((mcaErrSrcLog & (1 << 20)) || (mcaErrSrcLog & (1 << 27)))
584 {
585 // TODO: Light the CPU fault LED?
586 cpuIERRFound = true;
Jason M. Billsd1a19f62019-08-06 11:52:58 -0700587 incrementCPUErrorCount(cpu);
Jason M. Billsa3397932019-08-06 11:07:21 -0700588 // Next check if it's a CPU/VR mismatch by reading the
589 // IA32_MC4_STATUS MSR (0x411)
590 uint64_t mc4Status = 0;
591 if (peci_RdIAMSR(addr, 0, 0x411, &mc4Status, &cc) !=
592 PECI_CC_SUCCESS)
593 {
594 continue;
595 }
596 // TODO: Update MSEC/MSCOD_31_24 check
597 // Check MSEC bits 31:24 for
598 // MCA_SVID_VCCIN_VR_ICC_MAX_FAILURE (0x40),
599 // MCA_SVID_VCCIN_VR_VOUT_FAILURE (0x42), or
600 // MCA_SVID_CPU_VR_CAPABILITY_ERROR (0x43)
601 if ((mc4Status & (0x40 << 24)) ||
602 (mc4Status & (0x42 << 24)) ||
603 (mc4Status & (0x43 << 24)))
604 {
605 cpuIERRLog(cpu, "CPU/VR Mismatch");
606 continue;
607 }
608
609 // Next check if it's a Core FIVR fault by looking for a
610 // non-zero value of CORE_FIVR_ERR_LOG (B(31) D30 F2 offsets
611 // C0h and C4h) (Note: Bus 31 is accessed on PECI as bus 14)
612 uint32_t coreFIVRErrLog0 = 0;
613 uint32_t coreFIVRErrLog1 = 0;
614 if (peci_RdEndPointConfigPciLocal(
615 addr, 0, 14, 30, 2, 0xC0, sizeof(uint32_t),
616 (uint8_t*)&coreFIVRErrLog0, &cc) != PECI_CC_SUCCESS)
617 {
618 continue;
619 }
620 if (peci_RdEndPointConfigPciLocal(
621 addr, 0, 14, 30, 2, 0xC4, sizeof(uint32_t),
622 (uint8_t*)&coreFIVRErrLog1, &cc) != PECI_CC_SUCCESS)
623 {
624 continue;
625 }
626 if (coreFIVRErrLog0 || coreFIVRErrLog1)
627 {
628 cpuIERRLog(cpu, "Core FIVR Fault");
629 continue;
630 }
631
632 // Next check if it's an Uncore FIVR fault by looking for a
633 // non-zero value of UNCORE_FIVR_ERR_LOG (B(31) D30 F2
634 // offset 84h) (Note: Bus 31 is accessed on PECI as bus 14)
635 uint32_t uncoreFIVRErrLog = 0;
636 if (peci_RdEndPointConfigPciLocal(
637 addr, 0, 14, 30, 2, 0x84, sizeof(uint32_t),
638 (uint8_t*)&uncoreFIVRErrLog,
639 &cc) != PECI_CC_SUCCESS)
640 {
641 continue;
642 }
643 if (uncoreFIVRErrLog)
644 {
645 cpuIERRLog(cpu, "Uncore FIVR Fault");
646 continue;
647 }
648
649 // TODO: Update MSEC/MSCOD_31_24 check
650 // Last if CORE_FIVR_ERR_LOG and UNCORE_FIVR_ERR_LOG are
651 // both zero, but MSEC bits 31:24 have either
652 // MCA_FIVR_CATAS_OVERVOL_FAULT (0x51) or
653 // MCA_FIVR_CATAS_OVERCUR_FAULT (0x52), then log it as an
654 // uncore FIVR fault
655 if (!coreFIVRErrLog0 && !coreFIVRErrLog1 &&
656 !uncoreFIVRErrLog &&
657 ((mc4Status & (0x51 << 24)) ||
658 (mc4Status & (0x52 << 24))))
659 {
660 cpuIERRLog(cpu, "Uncore FIVR Fault");
661 continue;
662 }
663 cpuIERRLog(cpu);
664 }
665 break;
666 }
667 }
668 }
669 return cpuIERRFound;
670}
671
Jason M. Billsa15c2522019-08-16 10:01:44 -0700672static void caterrAssertHandler()
673{
Jason M. Billsa15c2522019-08-16 10:01:44 -0700674 caterrAssertTimer.expires_after(std::chrono::milliseconds(caterrTimeoutMs));
675 caterrAssertTimer.async_wait([](const boost::system::error_code ec) {
676 if (ec)
677 {
678 // operation_aborted is expected if timer is canceled
679 // before completion.
680 if (ec != boost::asio::error::operation_aborted)
681 {
682 std::cerr << "caterr timeout async_wait failed: "
683 << ec.message() << "\n";
684 }
Jason M. Billsa15c2522019-08-16 10:01:44 -0700685 return;
686 }
Jason M. Billsa3397932019-08-06 11:07:21 -0700687 std::cerr << "CATERR asserted for " << std::to_string(caterrTimeoutMs)
688 << " ms\n";
689 if (!checkIERRCPUs())
690 {
691 cpuIERRLog();
692 }
Jason M. Billsa15c2522019-08-16 10:01:44 -0700693 conn->async_method_call(
694 [](boost::system::error_code ec,
695 const std::variant<bool>& property) {
696 if (ec)
697 {
698 return;
699 }
700 const bool* reset = std::get_if<bool>(&property);
701 if (reset == nullptr)
702 {
703 std::cerr << "Unable to read reset on CATERR value\n";
704 return;
705 }
Jason M. Billsb61766b2019-11-26 17:02:44 -0800706 startCrashdumpAndRecovery(*reset, "IERR");
Jason M. Billsa15c2522019-08-16 10:01:44 -0700707 },
708 "xyz.openbmc_project.Settings",
709 "/xyz/openbmc_project/control/processor_error_config",
710 "org.freedesktop.DBus.Properties", "Get",
711 "xyz.openbmc_project.Control.Processor.ErrConfig", "ResetOnCATERR");
712 });
713}
714
Jason M. Bills1490b142019-07-01 15:48:43 -0700715static void caterrHandler()
716{
717 if (!hostOff)
718 {
719 gpiod::line_event gpioLineEvent = caterrLine.event_read();
720
721 bool caterr =
722 gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE;
723 if (caterr)
724 {
Jason M. Billsa15c2522019-08-16 10:01:44 -0700725 caterrAssertHandler();
Jason M. Bills1490b142019-07-01 15:48:43 -0700726 }
727 else
728 {
729 caterrAssertTimer.cancel();
730 }
731 }
732 caterrEvent.async_wait(boost::asio::posix::stream_descriptor::wait_read,
733 [](const boost::system::error_code ec) {
734 if (ec)
735 {
736 std::cerr << "caterr handler error: "
737 << ec.message() << "\n";
738 return;
739 }
740 caterrHandler();
741 });
742}
Jason M. Bills78c5eed2019-08-28 14:00:40 -0700743
Jason M. Billse94f5e12019-09-13 11:11:34 -0700744static void cpu1ThermtripAssertHandler()
745{
Jason M. Bills45e87e02019-09-09 14:45:38 -0700746 if (cpu1FIVRFaultLine.get_value() == 0)
747 {
748 cpuBootFIVRFaultLog(1);
749 }
750 else
751 {
752 cpuThermTripLog(1);
753 }
Jason M. Billse94f5e12019-09-13 11:11:34 -0700754}
755
Jason M. Bills78c5eed2019-08-28 14:00:40 -0700756static void cpu1ThermtripHandler()
757{
758 if (!hostOff)
759 {
760 gpiod::line_event gpioLineEvent = cpu1ThermtripLine.event_read();
761
762 bool cpu1Thermtrip =
763 gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE;
764 if (cpu1Thermtrip)
765 {
Jason M. Billse94f5e12019-09-13 11:11:34 -0700766 cpu1ThermtripAssertHandler();
Jason M. Bills78c5eed2019-08-28 14:00:40 -0700767 }
768 }
769 cpu1ThermtripEvent.async_wait(
770 boost::asio::posix::stream_descriptor::wait_read,
771 [](const boost::system::error_code ec) {
772 if (ec)
773 {
774 std::cerr << "CPU 1 Thermtrip handler error: " << ec.message()
775 << "\n";
776 return;
777 }
778 cpu1ThermtripHandler();
779 });
780}
781
jayaprakash Mutyala009adbc2019-12-24 22:08:07 +0000782static void cpu1MemtripHandler()
783{
784 if (!hostOff)
785 {
786 gpiod::line_event gpioLineEvent = cpu1MemtripLine.event_read();
787
788 bool cpu1Memtrip =
789 gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE;
790 if (cpu1Memtrip)
791 {
792 memThermTripLog(1);
793 }
794 }
795 cpu1MemtripEvent.async_wait(
796 boost::asio::posix::stream_descriptor::wait_read,
797 [](const boost::system::error_code ec) {
798 if (ec)
799 {
800 std::cerr << "CPU 1 Memory Thermaltrip handler error: "
801 << ec.message() << "\n";
802 return;
803 }
804 cpu1MemtripHandler();
805 });
806}
807
Jason M. Billse94f5e12019-09-13 11:11:34 -0700808static void cpu2ThermtripAssertHandler()
809{
Jason M. Bills45e87e02019-09-09 14:45:38 -0700810 if (cpu2FIVRFaultLine.get_value() == 0)
811 {
812 cpuBootFIVRFaultLog(2);
813 }
814 else
815 {
816 cpuThermTripLog(2);
817 }
Jason M. Billse94f5e12019-09-13 11:11:34 -0700818}
819
Jason M. Bills78c5eed2019-08-28 14:00:40 -0700820static void cpu2ThermtripHandler()
821{
822 if (!hostOff)
823 {
824 gpiod::line_event gpioLineEvent = cpu2ThermtripLine.event_read();
825
826 bool cpu2Thermtrip =
827 gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE;
828 if (cpu2Thermtrip)
829 {
Jason M. Billse94f5e12019-09-13 11:11:34 -0700830 cpu2ThermtripAssertHandler();
Jason M. Bills78c5eed2019-08-28 14:00:40 -0700831 }
832 }
833 cpu2ThermtripEvent.async_wait(
834 boost::asio::posix::stream_descriptor::wait_read,
835 [](const boost::system::error_code ec) {
836 if (ec)
837 {
838 std::cerr << "CPU 2 Thermtrip handler error: " << ec.message()
839 << "\n";
840 return;
841 }
842 cpu2ThermtripHandler();
843 });
844}
845
jayaprakash Mutyala009adbc2019-12-24 22:08:07 +0000846static void cpu2MemtripHandler()
847{
848 if (!hostOff)
849 {
850 gpiod::line_event gpioLineEvent = cpu2MemtripLine.event_read();
851
852 bool cpu2Memtrip =
853 gpioLineEvent.event_type == gpiod::line_event::RISING_EDGE;
854 if (cpu2Memtrip)
855 {
856 memThermTripLog(2);
857 }
858 }
859 cpu2MemtripEvent.async_wait(
860 boost::asio::posix::stream_descriptor::wait_read,
861 [](const boost::system::error_code ec) {
862 if (ec)
863 {
864 std::cerr << "CPU 2 Memory Thermaltrip handler error: "
865 << ec.message() << "\n";
866 return;
867 }
868 cpu2MemtripHandler();
869 });
870}
871
Jason M. Billse94f5e12019-09-13 11:11:34 -0700872static void cpu1VRHotAssertHandler()
873{
874 cpuVRHotLog("CPU 1");
875}
876
Jason M. Bills250fa632019-08-28 15:58:25 -0700877static void cpu1VRHotHandler()
878{
879 if (!hostOff)
880 {
881 gpiod::line_event gpioLineEvent = cpu1VRHotLine.event_read();
882
883 bool cpu1VRHot =
884 gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE;
885 if (cpu1VRHot)
886 {
Jason M. Billse94f5e12019-09-13 11:11:34 -0700887 cpu1VRHotAssertHandler();
Jason M. Bills250fa632019-08-28 15:58:25 -0700888 }
889 }
890 cpu1VRHotEvent.async_wait(boost::asio::posix::stream_descriptor::wait_read,
891 [](const boost::system::error_code ec) {
892 if (ec)
893 {
894 std::cerr << "CPU 1 VRHot handler error: "
895 << ec.message() << "\n";
896 return;
897 }
898 cpu1VRHotHandler();
899 });
900}
901
Jason M. Billse94f5e12019-09-13 11:11:34 -0700902static void cpu1MemABCDVRHotAssertHandler()
903{
904 cpuVRHotLog("CPU 1 Memory ABCD");
905}
906
Jason M. Bills9647ba72019-08-29 14:19:19 -0700907static void cpu1MemABCDVRHotHandler()
908{
909 if (!hostOff)
910 {
911 gpiod::line_event gpioLineEvent = cpu1MemABCDVRHotLine.event_read();
912
913 bool cpu1MemABCDVRHot =
914 gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE;
915 if (cpu1MemABCDVRHot)
916 {
Jason M. Billse94f5e12019-09-13 11:11:34 -0700917 cpu1MemABCDVRHotAssertHandler();
Jason M. Bills9647ba72019-08-29 14:19:19 -0700918 }
919 }
920 cpu1MemABCDVRHotEvent.async_wait(
921 boost::asio::posix::stream_descriptor::wait_read,
922 [](const boost::system::error_code ec) {
923 if (ec)
924 {
925 std::cerr << "CPU 1 Memory ABCD VRHot handler error: "
926 << ec.message() << "\n";
927 return;
928 }
929 cpu1MemABCDVRHotHandler();
930 });
931}
932
Jason M. Billse94f5e12019-09-13 11:11:34 -0700933static void cpu1MemEFGHVRHotAssertHandler()
934{
935 cpuVRHotLog("CPU 1 Memory EFGH");
936}
937
Jason M. Bills9647ba72019-08-29 14:19:19 -0700938static void cpu1MemEFGHVRHotHandler()
939{
940 if (!hostOff)
941 {
942 gpiod::line_event gpioLineEvent = cpu1MemEFGHVRHotLine.event_read();
943
944 bool cpu1MemEFGHVRHot =
945 gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE;
946 if (cpu1MemEFGHVRHot)
947 {
Jason M. Billse94f5e12019-09-13 11:11:34 -0700948 cpu1MemEFGHVRHotAssertHandler();
Jason M. Bills9647ba72019-08-29 14:19:19 -0700949 }
950 }
951 cpu1MemEFGHVRHotEvent.async_wait(
952 boost::asio::posix::stream_descriptor::wait_read,
953 [](const boost::system::error_code ec) {
954 if (ec)
955 {
956 std::cerr << "CPU 1 Memory EFGH VRHot handler error: "
957 << ec.message() << "\n";
958 return;
959 }
960 cpu1MemEFGHVRHotHandler();
961 });
962}
963
Jason M. Billse94f5e12019-09-13 11:11:34 -0700964static void cpu2VRHotAssertHandler()
965{
966 cpuVRHotLog("CPU 2");
967}
968
Jason M. Bills250fa632019-08-28 15:58:25 -0700969static void cpu2VRHotHandler()
970{
971 if (!hostOff)
972 {
973 gpiod::line_event gpioLineEvent = cpu2VRHotLine.event_read();
974
975 bool cpu2VRHot =
976 gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE;
977 if (cpu2VRHot)
978 {
Jason M. Billse94f5e12019-09-13 11:11:34 -0700979 cpu2VRHotAssertHandler();
Jason M. Bills250fa632019-08-28 15:58:25 -0700980 }
981 }
982 cpu2VRHotEvent.async_wait(boost::asio::posix::stream_descriptor::wait_read,
983 [](const boost::system::error_code ec) {
984 if (ec)
985 {
986 std::cerr << "CPU 2 VRHot handler error: "
987 << ec.message() << "\n";
988 return;
989 }
990 cpu2VRHotHandler();
991 });
992}
993
Jason M. Billse94f5e12019-09-13 11:11:34 -0700994static void cpu2MemABCDVRHotAssertHandler()
995{
996 cpuVRHotLog("CPU 2 Memory ABCD");
997}
998
Jason M. Bills9647ba72019-08-29 14:19:19 -0700999static void cpu2MemABCDVRHotHandler()
1000{
1001 if (!hostOff)
1002 {
1003 gpiod::line_event gpioLineEvent = cpu2MemABCDVRHotLine.event_read();
1004
1005 bool cpu2MemABCDVRHot =
1006 gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE;
1007 if (cpu2MemABCDVRHot)
1008 {
Jason M. Billse94f5e12019-09-13 11:11:34 -07001009 cpu2MemABCDVRHotAssertHandler();
Jason M. Bills9647ba72019-08-29 14:19:19 -07001010 }
1011 }
1012 cpu2MemABCDVRHotEvent.async_wait(
1013 boost::asio::posix::stream_descriptor::wait_read,
1014 [](const boost::system::error_code ec) {
1015 if (ec)
1016 {
1017 std::cerr << "CPU 2 Memory ABCD VRHot handler error: "
1018 << ec.message() << "\n";
1019 return;
1020 }
1021 cpu2MemABCDVRHotHandler();
1022 });
1023}
1024
Jason M. Billse94f5e12019-09-13 11:11:34 -07001025static void cpu2MemEFGHVRHotAssertHandler()
1026{
1027 cpuVRHotLog("CPU 2 Memory EFGH");
1028}
1029
Jason M. Bills9647ba72019-08-29 14:19:19 -07001030static void cpu2MemEFGHVRHotHandler()
1031{
1032 if (!hostOff)
1033 {
1034 gpiod::line_event gpioLineEvent = cpu2MemEFGHVRHotLine.event_read();
1035
1036 bool cpu2MemEFGHVRHot =
1037 gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE;
1038 if (cpu2MemEFGHVRHot)
1039 {
Jason M. Billse94f5e12019-09-13 11:11:34 -07001040 cpu2MemEFGHVRHotAssertHandler();
Jason M. Bills9647ba72019-08-29 14:19:19 -07001041 }
1042 }
1043 cpu2MemEFGHVRHotEvent.async_wait(
1044 boost::asio::posix::stream_descriptor::wait_read,
1045 [](const boost::system::error_code ec) {
1046 if (ec)
1047 {
1048 std::cerr << "CPU 2 Memory EFGH VRHot handler error: "
1049 << ec.message() << "\n";
1050 return;
1051 }
1052 cpu2MemEFGHVRHotHandler();
1053 });
1054}
1055
Chen Yugange6c0f1c2019-08-02 20:36:42 +08001056static void pchThermtripHandler()
1057{
1058 if (!hostOff)
1059 {
1060 gpiod::line_event gpioLineEvent = pchThermtripLine.event_read();
1061
1062 bool pchThermtrip =
1063 gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE;
1064 if (pchThermtrip)
1065 {
Jason M. Bills08866542019-08-16 12:04:19 -07001066 ssbThermTripLog();
Chen Yugange6c0f1c2019-08-02 20:36:42 +08001067 }
1068 }
1069 pchThermtripEvent.async_wait(
1070 boost::asio::posix::stream_descriptor::wait_read,
1071 [](const boost::system::error_code ec) {
1072 if (ec)
1073 {
1074 std::cerr << "PCH Thermal trip handler error: " << ec.message()
1075 << "\n";
1076 return;
1077 }
1078 pchThermtripHandler();
1079 });
1080}
1081
Jason M. Billscbf78532019-08-16 15:32:11 -07001082static std::bitset<MAX_CPUS> checkERRPinCPUs(const int errPin)
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001083{
Jason M. Billscbf78532019-08-16 15:32:11 -07001084 int errPinSts = (1 << errPin);
1085 std::bitset<MAX_CPUS> errPinCPUs = 0;
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001086 for (int cpu = 0, addr = MIN_CLIENT_ADDR; addr <= MAX_CLIENT_ADDR;
1087 cpu++, addr++)
1088 {
1089 if (peci_Ping(addr) == PECI_CC_SUCCESS)
1090 {
1091 uint8_t cc = 0;
1092 CPUModel model{};
1093 uint8_t stepping = 0;
1094 if (peci_GetCPUID(addr, &model, &stepping, &cc) != PECI_CC_SUCCESS)
1095 {
1096 std::cerr << "Cannot get CPUID!\n";
1097 continue;
1098 }
1099
1100 switch (model)
1101 {
1102 case skx:
1103 {
1104 // Check the ERRPINSTS to see if this is the CPU that caused
Jason M. Billscbf78532019-08-16 15:32:11 -07001105 // the ERRx (B(0) D8 F0 offset 210h)
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001106 uint32_t errpinsts = 0;
1107 if (peci_RdPCIConfigLocal(
1108 addr, 0, 8, 0, 0x210, sizeof(uint32_t),
1109 (uint8_t*)&errpinsts, &cc) == PECI_CC_SUCCESS)
1110 {
Jason M. Billscbf78532019-08-16 15:32:11 -07001111 errPinCPUs[cpu] = (errpinsts & errPinSts) != 0;
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001112 }
1113 break;
1114 }
1115 case icx:
1116 {
1117 // Check the ERRPINSTS to see if this is the CPU that caused
Jason M. Billscbf78532019-08-16 15:32:11 -07001118 // the ERRx (B(30) D0 F3 offset 274h) (Note: Bus 30 is
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001119 // accessed on PECI as bus 13)
1120 uint32_t errpinsts = 0;
1121 if (peci_RdEndPointConfigPciLocal(
1122 addr, 0, 13, 0, 3, 0x274, sizeof(uint32_t),
1123 (uint8_t*)&errpinsts, &cc) == PECI_CC_SUCCESS)
1124 {
Jason M. Billscbf78532019-08-16 15:32:11 -07001125 errPinCPUs[cpu] = (errpinsts & errPinSts) != 0;
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001126 }
1127 break;
1128 }
1129 }
1130 }
1131 }
Jason M. Billscbf78532019-08-16 15:32:11 -07001132 return errPinCPUs;
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001133}
1134
Jason M. Billscbf78532019-08-16 15:32:11 -07001135static void errXAssertHandler(const int errPin,
1136 boost::asio::steady_timer& errXAssertTimer)
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001137{
Jason M. Billscbf78532019-08-16 15:32:11 -07001138 // ERRx status is not guaranteed through the timeout, so save which
1139 // CPUs have it asserted
1140 std::bitset<MAX_CPUS> errPinCPUs = checkERRPinCPUs(errPin);
1141 errXAssertTimer.expires_after(std::chrono::milliseconds(errTimeoutMs));
1142 errXAssertTimer.async_wait([errPin, errPinCPUs](
1143 const boost::system::error_code ec) {
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001144 if (ec)
1145 {
1146 // operation_aborted is expected if timer is canceled before
1147 // completion.
1148 if (ec != boost::asio::error::operation_aborted)
1149 {
1150 std::cerr << "err2 timeout async_wait failed: " << ec.message()
1151 << "\n";
1152 }
1153 return;
1154 }
Jason M. Billscbf78532019-08-16 15:32:11 -07001155 std::cerr << "ERR" << std::to_string(errPin) << " asserted for "
1156 << std::to_string(errTimeoutMs) << " ms\n";
1157 if (errPinCPUs.count())
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001158 {
Jason M. Billscbf78532019-08-16 15:32:11 -07001159 for (int i = 0; i < errPinCPUs.size(); i++)
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001160 {
Jason M. Billscbf78532019-08-16 15:32:11 -07001161 if (errPinCPUs[i])
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001162 {
Jason M. Billscbf78532019-08-16 15:32:11 -07001163 cpuERRXLog(errPin, i);
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001164 }
1165 }
1166 }
1167 else
1168 {
Jason M. Billscbf78532019-08-16 15:32:11 -07001169 cpuERRXLog(errPin);
1170 }
1171 });
1172}
1173
Jason M. Bills8c584392019-08-19 11:05:51 -07001174static void err0AssertHandler()
1175{
1176 // Handle the standard ERR0 detection and logging
1177 const static constexpr int err0 = 0;
1178 errXAssertHandler(err0, err0AssertTimer);
1179}
1180
1181static void err0Handler()
1182{
1183 if (!hostOff)
1184 {
1185 gpiod::line_event gpioLineEvent = err0Line.event_read();
1186
1187 bool err0 = gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE;
1188 if (err0)
1189 {
1190 err0AssertHandler();
1191 }
1192 else
1193 {
1194 err0AssertTimer.cancel();
1195 }
1196 }
1197 err0Event.async_wait(boost::asio::posix::stream_descriptor::wait_read,
1198 [](const boost::system::error_code ec) {
1199 if (ec)
1200 {
1201 std::cerr
1202 << "err0 handler error: " << ec.message()
1203 << "\n";
1204 return;
1205 }
1206 err0Handler();
1207 });
1208}
1209
Jason M. Bills75af3962019-08-19 11:07:17 -07001210static void err1AssertHandler()
1211{
1212 // Handle the standard ERR1 detection and logging
1213 const static constexpr int err1 = 1;
1214 errXAssertHandler(err1, err1AssertTimer);
1215}
1216
1217static void err1Handler()
1218{
1219 if (!hostOff)
1220 {
1221 gpiod::line_event gpioLineEvent = err1Line.event_read();
1222
1223 bool err1 = gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE;
1224 if (err1)
1225 {
1226 err1AssertHandler();
1227 }
1228 else
1229 {
1230 err1AssertTimer.cancel();
1231 }
1232 }
1233 err1Event.async_wait(boost::asio::posix::stream_descriptor::wait_read,
1234 [](const boost::system::error_code ec) {
1235 if (ec)
1236 {
1237 std::cerr
1238 << "err1 handler error: " << ec.message()
1239 << "\n";
1240 return;
1241 }
1242 err1Handler();
1243 });
1244}
1245
Jason M. Billscbf78532019-08-16 15:32:11 -07001246static void err2AssertHandler()
1247{
1248 // Handle the standard ERR2 detection and logging
1249 const static constexpr int err2 = 2;
1250 errXAssertHandler(err2, err2AssertTimer);
1251 // Also handle reset for ERR2
1252 err2AssertTimer.async_wait([](const boost::system::error_code ec) {
1253 if (ec)
1254 {
1255 // operation_aborted is expected if timer is canceled before
1256 // completion.
1257 if (ec != boost::asio::error::operation_aborted)
1258 {
1259 std::cerr << "err2 timeout async_wait failed: " << ec.message()
1260 << "\n";
1261 }
1262 return;
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001263 }
1264 conn->async_method_call(
1265 [](boost::system::error_code ec,
1266 const std::variant<bool>& property) {
1267 if (ec)
1268 {
1269 return;
1270 }
1271 const bool* reset = std::get_if<bool>(&property);
1272 if (reset == nullptr)
1273 {
1274 std::cerr << "Unable to read reset on ERR2 value\n";
1275 return;
1276 }
Jason M. Billsb61766b2019-11-26 17:02:44 -08001277 startCrashdumpAndRecovery(*reset, "ERR2 Timeout");
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001278 },
1279 "xyz.openbmc_project.Settings",
1280 "/xyz/openbmc_project/control/processor_error_config",
1281 "org.freedesktop.DBus.Properties", "Get",
1282 "xyz.openbmc_project.Control.Processor.ErrConfig", "ResetOnERR2");
Yong Li061eb032020-02-26 15:06:18 +08001283
1284 beep(beepCPUErr2);
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001285 });
1286}
1287
1288static void err2Handler()
1289{
1290 if (!hostOff)
1291 {
1292 gpiod::line_event gpioLineEvent = err2Line.event_read();
1293
1294 bool err2 = gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE;
1295 if (err2)
1296 {
1297 err2AssertHandler();
1298 }
1299 else
1300 {
1301 err2AssertTimer.cancel();
1302 }
1303 }
1304 err2Event.async_wait(boost::asio::posix::stream_descriptor::wait_read,
1305 [](const boost::system::error_code ec) {
1306 if (ec)
1307 {
1308 std::cerr
1309 << "err2 handler error: " << ec.message()
1310 << "\n";
1311 return;
1312 }
1313 err2Handler();
1314 });
1315}
1316
Jason M. Bills89922f82019-08-06 11:10:02 -07001317static void smiAssertHandler()
1318{
1319 smiAssertTimer.expires_after(std::chrono::milliseconds(smiTimeoutMs));
1320 smiAssertTimer.async_wait([](const boost::system::error_code ec) {
1321 if (ec)
1322 {
1323 // operation_aborted is expected if timer is canceled before
1324 // completion.
1325 if (ec != boost::asio::error::operation_aborted)
1326 {
1327 std::cerr << "smi timeout async_wait failed: " << ec.message()
1328 << "\n";
1329 }
1330 return;
1331 }
1332 std::cerr << "SMI asserted for " << std::to_string(smiTimeoutMs)
1333 << " ms\n";
1334 smiTimeoutLog();
1335 conn->async_method_call(
1336 [](boost::system::error_code ec,
1337 const std::variant<bool>& property) {
1338 if (ec)
1339 {
1340 return;
1341 }
1342 const bool* reset = std::get_if<bool>(&property);
1343 if (reset == nullptr)
1344 {
1345 std::cerr << "Unable to read reset on SMI value\n";
1346 return;
1347 }
Jason M. Bills94785442020-01-07 15:22:09 -08001348#ifdef HOST_ERROR_CRASHDUMP_ON_SMI_TIMEOUT
Jason M. Billsb61766b2019-11-26 17:02:44 -08001349 startCrashdumpAndRecovery(*reset, "SMI Timeout");
Jason M. Bills94785442020-01-07 15:22:09 -08001350#else
1351 if (*reset)
1352 {
1353 std::cout << "Recovering the system\n";
1354 startPowerCycle();
1355 }
1356#endif
Jason M. Bills89922f82019-08-06 11:10:02 -07001357 },
1358 "xyz.openbmc_project.Settings",
1359 "/xyz/openbmc_project/control/bmc_reset_disables",
1360 "org.freedesktop.DBus.Properties", "Get",
1361 "xyz.openbmc_project.Control.ResetDisables", "ResetOnSMI");
1362 });
1363}
1364
1365static void smiHandler()
1366{
1367 if (!hostOff)
1368 {
1369 gpiod::line_event gpioLineEvent = smiLine.event_read();
1370
1371 bool smi = gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE;
1372 if (smi)
1373 {
1374 smiAssertHandler();
1375 }
1376 else
1377 {
1378 smiAssertTimer.cancel();
1379 }
1380 }
1381 smiEvent.async_wait(boost::asio::posix::stream_descriptor::wait_read,
1382 [](const boost::system::error_code ec) {
1383 if (ec)
1384 {
1385 std::cerr
1386 << "smi handler error: " << ec.message()
1387 << "\n";
1388 return;
1389 }
1390 smiHandler();
1391 });
1392}
1393
Jason M. Billsa15c2522019-08-16 10:01:44 -07001394static void initializeErrorState()
1395{
1396 // Handle CPU_CATERR if it's asserted now
1397 if (caterrLine.get_value() == 0)
1398 {
1399 caterrAssertHandler();
1400 }
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001401
Jason M. Bills8c584392019-08-19 11:05:51 -07001402 // Handle CPU_ERR0 if it's asserted now
1403 if (err0Line.get_value() == 0)
1404 {
1405 err0AssertHandler();
1406 }
1407
Jason M. Bills75af3962019-08-19 11:07:17 -07001408 // Handle CPU_ERR1 if it's asserted now
1409 if (err1Line.get_value() == 0)
1410 {
1411 err1AssertHandler();
1412 }
1413
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001414 // Handle CPU_ERR2 if it's asserted now
1415 if (err2Line.get_value() == 0)
1416 {
1417 err2AssertHandler();
1418 }
Jason M. Bills89922f82019-08-06 11:10:02 -07001419
1420 // Handle SMI if it's asserted now
1421 if (smiLine.get_value() == 0)
1422 {
1423 smiAssertHandler();
1424 }
Jason M. Bills08866542019-08-16 12:04:19 -07001425
Jason M. Billse94f5e12019-09-13 11:11:34 -07001426 // Handle CPU1_THERMTRIP if it's asserted now
1427 if (cpu1ThermtripLine.get_value() == 0)
1428 {
1429 cpu1ThermtripAssertHandler();
1430 }
1431
1432 // Handle CPU2_THERMTRIP if it's asserted now
1433 if (cpu2ThermtripLine.get_value() == 0)
1434 {
1435 cpu2ThermtripAssertHandler();
1436 }
1437
jayaprakash Mutyala009adbc2019-12-24 22:08:07 +00001438 // Handle CPU1_MEM_THERM_EVENT (CPU1 DIMM Thermal trip) if it's asserted now
1439 if (cpu1MemtripLine.get_value() == 0)
1440 {
1441 memThermTripLog(1);
1442 }
1443
1444 // Handle CPU2_MEM_THERM_EVENT (CPU2 DIMM Thermal trip) if it's asserted now
1445 if (cpu2MemtripLine.get_value() == 0)
1446 {
1447 memThermTripLog(2);
1448 }
1449
Jason M. Billse94f5e12019-09-13 11:11:34 -07001450 // Handle CPU1_VRHOT if it's asserted now
1451 if (cpu1VRHotLine.get_value() == 0)
1452 {
1453 cpu1VRHotAssertHandler();
1454 }
1455
1456 // Handle CPU1_MEM_ABCD_VRHOT if it's asserted now
1457 if (cpu1MemABCDVRHotLine.get_value() == 0)
1458 {
1459 cpu1MemABCDVRHotAssertHandler();
1460 }
1461
1462 // Handle CPU1_MEM_EFGH_VRHOT if it's asserted now
1463 if (cpu1MemEFGHVRHotLine.get_value() == 0)
1464 {
1465 cpu1MemEFGHVRHotAssertHandler();
1466 }
1467
1468 // Handle CPU2_VRHOT if it's asserted now
1469 if (cpu2VRHotLine.get_value() == 0)
1470 {
1471 cpu2VRHotAssertHandler();
1472 }
1473
1474 // Handle CPU2_MEM_ABCD_VRHOT if it's asserted now
1475 if (cpu2MemABCDVRHotLine.get_value() == 0)
1476 {
1477 cpu2MemABCDVRHotAssertHandler();
1478 }
1479
1480 // Handle CPU2_MEM_EFGH_VRHOT if it's asserted now
1481 if (cpu2MemEFGHVRHotLine.get_value() == 0)
1482 {
1483 cpu2MemEFGHVRHotAssertHandler();
1484 }
1485
Jason M. Bills08866542019-08-16 12:04:19 -07001486 // Handle PCH_BMC_THERMTRIP if it's asserted now
1487 if (pchThermtripLine.get_value() == 0)
1488 {
1489 ssbThermTripLog();
1490 }
Jason M. Billsa15c2522019-08-16 10:01:44 -07001491}
Jason M. Bills1490b142019-07-01 15:48:43 -07001492} // namespace host_error_monitor
1493
1494int main(int argc, char* argv[])
1495{
1496 // setup connection to dbus
1497 host_error_monitor::conn =
1498 std::make_shared<sdbusplus::asio::connection>(host_error_monitor::io);
1499
Jason M. Billsc4b91f22019-11-26 17:04:50 -08001500 // Host Error Monitor Service
Jason M. Bills1490b142019-07-01 15:48:43 -07001501 host_error_monitor::conn->request_name(
1502 "xyz.openbmc_project.HostErrorMonitor");
1503 sdbusplus::asio::object_server server =
1504 sdbusplus::asio::object_server(host_error_monitor::conn);
1505
Jason M. Billsc4b91f22019-11-26 17:04:50 -08001506 // Restart Cause Interface
1507 host_error_monitor::hostErrorTimeoutIface =
1508 server.add_interface("/xyz/openbmc_project/host_error_monitor",
1509 "xyz.openbmc_project.HostErrorMonitor.Timeout");
1510
1511 host_error_monitor::hostErrorTimeoutIface->register_property(
1512 "IERRTimeoutMs", host_error_monitor::caterrTimeoutMs,
1513 [](const std::size_t& requested, std::size_t& resp) {
1514 if (requested > host_error_monitor::caterrTimeoutMsMax)
1515 {
1516 std::cerr << "IERRTimeoutMs update to " << requested
1517 << "ms rejected. Cannot be greater than "
1518 << host_error_monitor::caterrTimeoutMsMax << "ms.\n";
1519 return 0;
1520 }
1521 std::cerr << "IERRTimeoutMs updated to " << requested << "ms\n";
1522 host_error_monitor::caterrTimeoutMs = requested;
1523 resp = requested;
1524 return 1;
1525 },
1526 [](std::size_t& resp) { return host_error_monitor::caterrTimeoutMs; });
1527 host_error_monitor::hostErrorTimeoutIface->initialize();
1528
Jason M. Bills1490b142019-07-01 15:48:43 -07001529 // Start tracking host state
1530 std::shared_ptr<sdbusplus::bus::match::match> hostStateMonitor =
1531 host_error_monitor::startHostStateMonitor();
1532
1533 // Initialize the host state
1534 host_error_monitor::initializeHostState();
1535
1536 // Request CPU_CATERR GPIO events
1537 if (!host_error_monitor::requestGPIOEvents(
1538 "CPU_CATERR", host_error_monitor::caterrHandler,
1539 host_error_monitor::caterrLine, host_error_monitor::caterrEvent))
1540 {
1541 return -1;
1542 }
1543
Jason M. Bills8c584392019-08-19 11:05:51 -07001544 // Request CPU_ERR0 GPIO events
1545 if (!host_error_monitor::requestGPIOEvents(
1546 "CPU_ERR0", host_error_monitor::err0Handler,
1547 host_error_monitor::err0Line, host_error_monitor::err0Event))
1548 {
1549 return -1;
1550 }
1551
Jason M. Bills75af3962019-08-19 11:07:17 -07001552 // Request CPU_ERR1 GPIO events
1553 if (!host_error_monitor::requestGPIOEvents(
1554 "CPU_ERR1", host_error_monitor::err1Handler,
1555 host_error_monitor::err1Line, host_error_monitor::err1Event))
1556 {
1557 return -1;
1558 }
1559
Jason M. Bills6a2cb692019-08-06 11:03:49 -07001560 // Request CPU_ERR2 GPIO events
1561 if (!host_error_monitor::requestGPIOEvents(
1562 "CPU_ERR2", host_error_monitor::err2Handler,
1563 host_error_monitor::err2Line, host_error_monitor::err2Event))
1564 {
1565 return -1;
1566 }
1567
Jason M. Bills89922f82019-08-06 11:10:02 -07001568 // Request SMI GPIO events
1569 if (!host_error_monitor::requestGPIOEvents(
1570 "SMI", host_error_monitor::smiHandler, host_error_monitor::smiLine,
1571 host_error_monitor::smiEvent))
1572 {
1573 return -1;
1574 }
1575
Jason M. Bills45e87e02019-09-09 14:45:38 -07001576 // Request CPU1_FIVR_FAULT GPIO input
1577 if (!host_error_monitor::requestGPIOInput(
1578 "CPU1_FIVR_FAULT", host_error_monitor::cpu1FIVRFaultLine))
1579 {
1580 return -1;
1581 }
1582
Jason M. Bills78c5eed2019-08-28 14:00:40 -07001583 // Request CPU1_THERMTRIP GPIO events
1584 if (!host_error_monitor::requestGPIOEvents(
1585 "CPU1_THERMTRIP", host_error_monitor::cpu1ThermtripHandler,
1586 host_error_monitor::cpu1ThermtripLine,
1587 host_error_monitor::cpu1ThermtripEvent))
1588 {
1589 return -1;
1590 }
1591
Jason M. Bills45e87e02019-09-09 14:45:38 -07001592 // Request CPU2_FIVR_FAULT GPIO input
1593 if (!host_error_monitor::requestGPIOInput(
1594 "CPU2_FIVR_FAULT", host_error_monitor::cpu2FIVRFaultLine))
1595 {
1596 return -1;
1597 }
1598
Jason M. Bills78c5eed2019-08-28 14:00:40 -07001599 // Request CPU2_THERMTRIP GPIO events
1600 if (!host_error_monitor::requestGPIOEvents(
1601 "CPU2_THERMTRIP", host_error_monitor::cpu2ThermtripHandler,
1602 host_error_monitor::cpu2ThermtripLine,
1603 host_error_monitor::cpu2ThermtripEvent))
1604 {
1605 return -1;
1606 }
1607
Jason M. Bills250fa632019-08-28 15:58:25 -07001608 // Request CPU1_VRHOT GPIO events
1609 if (!host_error_monitor::requestGPIOEvents(
1610 "CPU1_VRHOT", host_error_monitor::cpu1VRHotHandler,
1611 host_error_monitor::cpu1VRHotLine,
1612 host_error_monitor::cpu1VRHotEvent))
1613 {
1614 return -1;
1615 }
1616
Jason M. Bills9647ba72019-08-29 14:19:19 -07001617 // Request CPU1_MEM_ABCD_VRHOT GPIO events
1618 if (!host_error_monitor::requestGPIOEvents(
1619 "CPU1_MEM_ABCD_VRHOT", host_error_monitor::cpu1MemABCDVRHotHandler,
1620 host_error_monitor::cpu1MemABCDVRHotLine,
1621 host_error_monitor::cpu1MemABCDVRHotEvent))
1622 {
1623 return -1;
1624 }
1625
1626 // Request CPU1_MEM_EFGH_VRHOT GPIO events
1627 if (!host_error_monitor::requestGPIOEvents(
1628 "CPU1_MEM_EFGH_VRHOT", host_error_monitor::cpu1MemEFGHVRHotHandler,
1629 host_error_monitor::cpu1MemEFGHVRHotLine,
1630 host_error_monitor::cpu1MemEFGHVRHotEvent))
1631 {
1632 return -1;
1633 }
1634
Jason M. Bills250fa632019-08-28 15:58:25 -07001635 // Request CPU2_VRHOT GPIO events
1636 if (!host_error_monitor::requestGPIOEvents(
1637 "CPU2_VRHOT", host_error_monitor::cpu2VRHotHandler,
1638 host_error_monitor::cpu2VRHotLine,
1639 host_error_monitor::cpu2VRHotEvent))
1640 {
1641 return -1;
1642 }
1643
Jason M. Bills9647ba72019-08-29 14:19:19 -07001644 // Request CPU2_MEM_ABCD_VRHOT GPIO events
1645 if (!host_error_monitor::requestGPIOEvents(
1646 "CPU2_MEM_ABCD_VRHOT", host_error_monitor::cpu2MemABCDVRHotHandler,
1647 host_error_monitor::cpu2MemABCDVRHotLine,
1648 host_error_monitor::cpu2MemABCDVRHotEvent))
1649 {
1650 return -1;
1651 }
1652
1653 // Request CPU2_MEM_EFGH_VRHOT GPIO events
1654 if (!host_error_monitor::requestGPIOEvents(
1655 "CPU2_MEM_EFGH_VRHOT", host_error_monitor::cpu2MemEFGHVRHotHandler,
1656 host_error_monitor::cpu2MemEFGHVRHotLine,
1657 host_error_monitor::cpu2MemEFGHVRHotEvent))
1658 {
1659 return -1;
1660 }
1661
Chen Yugange6c0f1c2019-08-02 20:36:42 +08001662 // Request PCH_BMC_THERMTRIP GPIO events
1663 if (!host_error_monitor::requestGPIOEvents(
1664 "PCH_BMC_THERMTRIP", host_error_monitor::pchThermtripHandler,
1665 host_error_monitor::pchThermtripLine,
1666 host_error_monitor::pchThermtripEvent))
1667 {
1668 return -1;
1669 }
1670
jayaprakash Mutyala009adbc2019-12-24 22:08:07 +00001671 // Request CPU1_MEM_THERM_EVENT GPIO events
1672 if (!host_error_monitor::requestGPIOEvents(
1673 "CPU1_MEM_THERM_EVENT", host_error_monitor::cpu1MemtripHandler,
1674 host_error_monitor::cpu1MemtripLine,
1675 host_error_monitor::cpu1MemtripEvent))
1676 {
1677 return -1;
1678 }
1679
1680 // Request CPU2_MEM_THERM_EVENT GPIO events
1681 if (!host_error_monitor::requestGPIOEvents(
1682 "CPU2_MEM_THERM_EVENT", host_error_monitor::cpu2MemtripHandler,
1683 host_error_monitor::cpu2MemtripLine,
1684 host_error_monitor::cpu2MemtripEvent))
1685 {
1686 return -1;
1687 }
1688
Jason M. Bills1490b142019-07-01 15:48:43 -07001689 host_error_monitor::io.run();
1690
1691 return 0;
1692}