diff --git a/src/ethernet_interface.cpp b/src/ethernet_interface.cpp
index 4664655..e5a3110 100644
--- a/src/ethernet_interface.cpp
+++ b/src/ethernet_interface.cpp
@@ -213,20 +213,31 @@
     }
 }
 
+void EthernetInterface::addStaticNeigh(const NeighborInfo& info)
+{
+    if ((info.state & NUD_PERMANENT) == 0)
+    {
+        return;
+    }
+    if (!info.mac)
+    {
+        auto msg = fmt::format("Missing neighbor mac for `{}` on {}\n",
+                               info.addr, interfaceName());
+        log<level::ERR>(msg.c_str());
+        return;
+    }
+    staticNeighbors.emplace(
+        info.addr, std::make_unique<Neighbor>(bus, std::string_view(objPath),
+                                              *this, info.addr, *info.mac,
+                                              Neighbor::State::Permanent));
+}
+
 void EthernetInterface::createStaticNeighborObjects()
 {
     staticNeighbors.clear();
     for (const auto& neighbor : system::getNeighbors({.ifidx = ifIdx}))
     {
-        if (!neighbor.mac || (neighbor.state & NUD_PERMANENT) == 0)
-        {
-            continue;
-        }
-        staticNeighbors.emplace(
-            neighbor.addr,
-            std::make_unique<Neighbor>(bus, std::string_view(objPath), *this,
-                                       neighbor.addr, *neighbor.mac,
-                                       Neighbor::State::Permanent));
+        addStaticNeigh(neighbor);
     }
 }
 
@@ -320,7 +331,7 @@
                                    lladdr, Neighbor::State::Permanent));
 
     writeConfigurationFile();
-    manager.reloadConfigs();
+    manager.reloadConfigsNoRefresh();
 
     return it->second->getObjPath();
 }
diff --git a/src/ethernet_interface.hpp b/src/ethernet_interface.hpp
index f14e069..fcb45f6 100644
--- a/src/ethernet_interface.hpp
+++ b/src/ethernet_interface.hpp
@@ -101,6 +101,7 @@
     std::unordered_map<InAddrAny, std::unique_ptr<Neighbor>> staticNeighbors;
 
     void addAddr(const AddressInfo& info);
+    void addStaticNeigh(const NeighborInfo& info);
 
     /** @brief Updates the interface information based on new InterfaceInfo */
     void updateInfo(const system::InterfaceInfo& info);
diff --git a/src/neighbor.cpp b/src/neighbor.cpp
index 0f5140e..58334b2 100644
--- a/src/neighbor.cpp
+++ b/src/neighbor.cpp
@@ -57,7 +57,7 @@
     }
 
     parent.writeConfigurationFile();
-    parent.manager.reloadConfigs();
+    parent.manager.reloadConfigsNoRefresh();
 }
 
 using sdbusplus::xyz::openbmc_project::Common::Error::NotAllowed;
diff --git a/src/rtnetlink_server.cpp b/src/rtnetlink_server.cpp
index 2a0ab38..d226efa 100644
--- a/src/rtnetlink_server.cpp
+++ b/src/rtnetlink_server.cpp
@@ -29,25 +29,13 @@
 using phosphor::logging::level;
 using phosphor::logging::log;
 
-static bool shouldRefresh(const struct nlmsghdr& hdr,
-                          std::string_view data) noexcept
+static bool shouldRefresh(const struct nlmsghdr& hdr, std::string_view) noexcept
 {
     switch (hdr.nlmsg_type)
     {
         case RTM_NEWLINK:
         case RTM_DELLINK:
             return true;
-        case RTM_NEWNEIGH:
-        case RTM_DELNEIGH:
-        {
-            if (data.size() < sizeof(ndmsg))
-            {
-                return false;
-            }
-            const auto& ndm = *reinterpret_cast<const ndmsg*>(data.data());
-            // We only want to refresh for static neighbors
-            return ndm.ndm_state & NUD_PERMANENT;
-        }
     }
     return false;
 }
@@ -121,6 +109,26 @@
     }
 }
 
+static void neighhandler(Manager& m, bool n, std::string_view data)
+{
+    auto info = netlink::neighFromRtm(data);
+    auto it = m.interfacesByIdx.find(info.ifidx);
+    if (it == m.interfacesByIdx.end())
+    {
+        auto msg = fmt::format("Interface `{}` not found for addr", info.ifidx);
+        log<level::ERR>(msg.c_str(), entry("IFIDX=%u", info.ifidx));
+        return;
+    }
+    if (n)
+    {
+        it->second->addStaticNeigh(info);
+    }
+    else
+    {
+        it->second->staticNeighbors.erase(info.addr);
+    }
+}
+
 static void handler(Manager& m, const nlmsghdr& hdr, std::string_view data)
 {
     if (shouldRefresh(hdr, data) && !refreshObjectTimer->isEnabled())
@@ -141,6 +149,12 @@
         case RTM_DELADDR:
             addrhandler(m, false, data);
             break;
+        case RTM_NEWNEIGH:
+            neighhandler(m, true, data);
+            break;
+        case RTM_DELNEIGH:
+            neighhandler(m, false, data);
+            break;
     }
 }