Add retry to sendFile handler
In cases where the ipmi interface is interrupted, we want to be able to
recover automatically without having to retrigger it again. For example,
this may happen if the ipminet interface gets reconfigure in the middle
of transfering an image.
The inital goal was to have the retry for ipminet only with and restart
at the remaining data instead of from the beginning. The issue there is
that we needed to restart the ipmi blob session to write again and doing
so will clear out the existing written data. When trying to write to
existing session will caused the update to be stalled and required a
ipmi restart to recover.
Tested:
images is able to be tranfered fully and validated after we interrupted
the ipminet interface in the middle.
Change-Id: Id734f6a92625bc6a1256fea010fb4b068f7bf1d5
Signed-off-by: Willy Tu <wltu@google.com>
diff --git a/tools/handler.cpp b/tools/handler.cpp
index 2b847a9..cc73e13 100644
--- a/tools/handler.cpp
+++ b/tools/handler.cpp
@@ -70,27 +70,49 @@
return true;
}
-void UpdateHandler::sendFile(const std::string& target, const std::string& path)
+void UpdateHandler::retrySendFile(const std::string& target,
+ const std::string& path)
{
auto supported = handler->supportedType();
+ auto session =
+ openBlob(blob, target,
+ static_cast<std::uint16_t>(supported) |
+ static_cast<std::uint16_t>(
+ ipmi_flash::FirmwareFlags::UpdateFlags::openWrite));
- try
+ if (!handler->sendContents(path, *session))
{
- auto session = openBlob(
- blob, target,
- static_cast<std::uint16_t>(supported) |
- static_cast<std::uint16_t>(
- ipmi_flash::FirmwareFlags::UpdateFlags::openWrite));
-
- if (!handler->sendContents(path, *session))
- {
- throw ToolException("Failed to send contents of " + path);
- }
+ throw ToolException("Failed to send contents of " + path);
}
- catch (const ipmiblob::BlobException& b)
+}
+
+void UpdateHandler::sendFile(const std::string& target, const std::string& path)
+{
+ const uint8_t retryCount = 3;
+ uint8_t i = 1;
+ while (true)
{
- throw ToolException("blob exception received: " +
- std::string(b.what()));
+ try
+ {
+ retrySendFile(target, path);
+ return;
+ }
+ catch (const ipmiblob::BlobException& b)
+ {
+ throw ToolException("blob exception received: " +
+ std::string(b.what()));
+ }
+ catch (const ToolException& t)
+ {
+ uint8_t remains = retryCount - i;
+ std::fprintf(
+ stderr,
+ "tool exception received: %s: Retrying it %u more times\n",
+ t.what(), remains);
+ if (remains == 0)
+ throw;
+ }
+ ++i;
}
}
diff --git a/tools/handler.hpp b/tools/handler.hpp
index f4bf843..4d2fe50 100644
--- a/tools/handler.hpp
+++ b/tools/handler.hpp
@@ -76,6 +76,11 @@
void sendFile(const std::string& target, const std::string& path) override;
/**
+ * @throw ToolException on failure.
+ */
+ void retrySendFile(const std::string& target, const std::string& path);
+
+ /**
* @throw ToolException on failure (TODO: throw on timeout.)
*/
bool verifyFile(const std::string& target, bool ignoreStatus) override;
diff --git a/tools/test/tools_updater_unittest.cpp b/tools/test/tools_updater_unittest.cpp
index cf6cf50..e04cde8 100644
--- a/tools/test/tools_updater_unittest.cpp
+++ b/tools/test/tools_updater_unittest.cpp
@@ -108,21 +108,47 @@
ipmi_flash::FirmwareFlags::UpdateFlags::openWrite);
EXPECT_CALL(handlerMock, supportedType())
- .WillOnce(Return(ipmi_flash::FirmwareFlags::UpdateFlags::lpc));
+ .WillRepeatedly(Return(ipmi_flash::FirmwareFlags::UpdateFlags::lpc));
EXPECT_CALL(blobMock, openBlob(ipmi_flash::staticLayoutBlobId, supported))
- .WillOnce(Return(session));
+ .WillRepeatedly(Return(session));
EXPECT_CALL(handlerMock, sendContents(firmwareImage, session))
- .WillOnce(Return(false));
+ .WillRepeatedly(Return(false));
- EXPECT_CALL(blobMock, closeBlob(session)).Times(1);
+ EXPECT_CALL(blobMock, closeBlob(session)).Times(3);
EXPECT_THROW(
updater.sendFile(ipmi_flash::staticLayoutBlobId, firmwareImage),
ToolException);
}
+TEST_F(UpdateHandlerTest, SendFileHandlerPassWithRetries)
+{
+ std::string firmwareImage = "image.bin";
+
+ std::uint16_t supported =
+ static_cast<std::uint16_t>(
+ ipmi_flash::FirmwareFlags::UpdateFlags::lpc) |
+ static_cast<std::uint16_t>(
+ ipmi_flash::FirmwareFlags::UpdateFlags::openWrite);
+
+ EXPECT_CALL(handlerMock, supportedType())
+ .WillRepeatedly(Return(ipmi_flash::FirmwareFlags::UpdateFlags::lpc));
+
+ EXPECT_CALL(blobMock, openBlob(ipmi_flash::staticLayoutBlobId, supported))
+ .WillRepeatedly(Return(session));
+
+ EXPECT_CALL(handlerMock, sendContents(firmwareImage, session))
+ .WillOnce(Return(false))
+ .WillOnce(Return(false))
+ .WillOnce(Return(true));
+
+ EXPECT_CALL(blobMock, closeBlob(session)).Times(3);
+
+ updater.sendFile(ipmi_flash::staticLayoutBlobId, firmwareImage);
+}
+
TEST_F(UpdateHandlerTest, VerifyFileHandleReturnsTrueOnSuccess)
{
EXPECT_CALL(blobMock, openBlob(ipmi_flash::verifyBlobId, _))