Remove subscription for TerminateAfterRetries policy

Redfish Data Model [1] section 6.42.5.2 and EventDestination schema [2]
specify that the subscription is terminated under the following policy
and the conditions.

DeliveryRetryPolicy:
- `TerminateAfterRetries`:
```
  This value shall indicate the subscription is terminated after the
  maximum number of retries is reached, specified by the
  DeliveryRetryAttempts property in the event service.
```

This implements this policy to delete  subscription of the stale client
connections after trying `DeliveryRetryAttempts` when
`DeliveryRetryPolicy == TerminateAfterRetries`.

Tested:
1) Subscription with blocked communication between bmcweb and listener

- Run Redfish Event Listener [3] to subscribe events with
 `DeliveryRetryPolicy == TerminateAfterRetries`

- Check the subscription creation
```
 curl -k -X GET https://${bmc}/redfish/v1/EventService/Subscriptions/
```

- Generate an event and check the delivery to the listener.
For example,
```
 curl -k -X POST https://${bmc}/redfish/v1/EventService/Actions/EventService.SubmitTestEvent
```

- Block the communication between bmcweb and listener
(or modify the listener not to delete the subscription at its exit, and
kill the listener so that its subscription is still alive)

- If the above task is already finished, generate more events and wait
for the sufficient time till `DeliveryRetryAttempts`.

- bmcweb journal log may contain the entries like
```
Sep 20 10:47:55 p10bmc bmcwebd[286]: [ERROR http_client.hpp:444] Maximum number of retries reached. https://9.3.62.209:8080/Redfish-Event-Listener
Sep 20 10:47:55 p10bmc bmcwebd[286]: [ERROR event_service_manager.hpp:1459]  Subscription 590587653 is deleted after MaxRetryAttempts
```

- Check the subscription again if the subscription is deleted.

2) Redfish Validator passes

[1] https://www.dmtf.org/sites/default/files/standards/documents/DSP0268_2024.3.html
[2] https://github.com/openbmc/bmcweb/blob/878edd599b1706ec8ffe6c3d81ba7cb3534f6393/redfish-core/schema/dmtf/csdl/EventDestination_v1.xml#L857
[3] https://github.com/DMTF/Redfish-Event-Listener

Change-Id: I6e41288995cbb6e37e17a7ef1be093abb7ce54b9
Signed-off-by: Myung Bae <myungbae@us.ibm.com>
diff --git a/http/http_client.hpp b/http/http_client.hpp
index 546d92c..b48fc6a 100644
--- a/http/http_client.hpp
+++ b/http/http_client.hpp
@@ -841,6 +841,27 @@
         // Initialize the pool with a single connection
         addConnection();
     }
+
+    // Check whether all connections are terminated
+    bool areAllConnectionsTerminated()
+    {
+        if (connections.empty())
+        {
+            BMCWEB_LOG_DEBUG("There are no connections for pool id:{}", id);
+            return false;
+        }
+        for (const auto& conn : connections)
+        {
+            if (conn != nullptr && conn->state != ConnState::terminated)
+            {
+                BMCWEB_LOG_DEBUG(
+                    "Not all connections of pool id:{} are terminated", id);
+                return false;
+            }
+        }
+        BMCWEB_LOG_INFO("All connections of pool id:{} are terminated", id);
+        return true;
+    }
 };
 
 class HttpClient
@@ -914,5 +935,22 @@
         pool.first->second->sendData(std::move(data), destUrl, httpHeader, verb,
                                      resHandler);
     }
+
+    // Test whether all connections are terminated (after MaxRetryAttempts)
+    bool isTerminated()
+    {
+        for (const auto& pool : connectionPools)
+        {
+            if (pool.second != nullptr &&
+                !pool.second->areAllConnectionsTerminated())
+            {
+                BMCWEB_LOG_DEBUG(
+                    "Not all of client connections are terminated");
+                return false;
+            }
+        }
+        BMCWEB_LOG_DEBUG("All client connections are terminated");
+        return true;
+    }
 };
 } // namespace crow
diff --git a/redfish-core/include/event_service_manager.hpp b/redfish-core/include/event_service_manager.hpp
index 7c20e10..66e92d5 100644
--- a/redfish-core/include/event_service_manager.hpp
+++ b/redfish-core/include/event_service_manager.hpp
@@ -252,7 +252,7 @@
 
 } // namespace event_log
 
-class Subscription
+class Subscription : public std::enable_shared_from_this<Subscription>
 {
   public:
     Subscription(const Subscription&) = delete;
@@ -277,6 +277,36 @@
 
     ~Subscription() = default;
 
+    // callback for subscription sendData
+    void resHandler(const std::shared_ptr<Subscription>& /*unused*/,
+                    const crow::Response& res)
+    {
+        BMCWEB_LOG_DEBUG("Response handled with return code: {}",
+                         res.resultInt());
+
+        if (!client)
+        {
+            BMCWEB_LOG_ERROR(
+                "Http client wasn't filled but http client callback was called.");
+            return;
+        }
+
+        if (userSub.retryPolicy != "TerminateAfterRetries")
+        {
+            return;
+        }
+        if (client->isTerminated())
+        {
+            if (deleter)
+            {
+                BMCWEB_LOG_INFO(
+                    "Subscription {} is deleted after MaxRetryAttempts",
+                    userSub.id);
+                deleter();
+            }
+        }
+    }
+
     bool sendEventToSubscriber(std::string&& msg)
     {
         persistent_data::EventServiceConfig eventServiceConfig =
@@ -289,11 +319,13 @@
 
         if (client)
         {
-            client->sendData(std::move(msg), userSub.destinationUrl,
-                             static_cast<ensuressl::VerifyCertificate>(
-                                 userSub.verifyCertificate),
-                             userSub.httpHeaders,
-                             boost::beast::http::verb::post);
+            client->sendDataWithCallback(
+                std::move(msg), userSub.destinationUrl,
+                static_cast<ensuressl::VerifyCertificate>(
+                    userSub.verifyCertificate),
+                userSub.httpHeaders, boost::beast::http::verb::post,
+                std::bind_front(&Subscription::resHandler, this,
+                                shared_from_this()));
             return true;
         }
 
@@ -476,6 +508,7 @@
     }
 
     persistent_data::UserSubscription userSub;
+    std::function<void()> deleter;
 
   private:
     uint64_t eventSeqNum = 1;
@@ -564,8 +597,12 @@
             }
             std::shared_ptr<Subscription> subValue =
                 std::make_shared<Subscription>(newSub, *url, ioc);
+            std::string id = subValue->userSub.id;
+            subValue->deleter = [id]() {
+                EventServiceManager::getInstance().deleteSubscription(id);
+            };
 
-            subscriptionsMap.insert(std::pair(subValue->userSub.id, subValue));
+            subscriptionsMap.emplace(id, subValue);
 
             updateNoOfSubscribersCount();
 
@@ -875,7 +912,9 @@
         addPushSubscription(const std::shared_ptr<Subscription>& subValue)
     {
         std::string id = addSubscriptionInternal(subValue);
-
+        subValue->deleter = [id]() {
+            EventServiceManager::getInstance().deleteSubscription(id);
+        };
         updateSubscriptionData();
         return id;
     }