core: Validate binding MTU before allowing transmission

Don't accept MTU values like 0, which will cause the packetisation loop
to fail in unacceptable ways (OOM).

Signed-off-by: Andrew Jeffery <andrew@aj.id.au>
Change-Id: I00d563cd91f718e695ed28d2b9978fc38566c4ae
diff --git a/core.c b/core.c
index 9cc6666..0aca12b 100644
--- a/core.c
+++ b/core.c
@@ -611,6 +611,13 @@
 		if (!enable)
 			return;
 
+		if (binding->pkt_size < MCTP_PACKET_SIZE(MCTP_BTU)) {
+			mctp_prerr("Cannot start %s binding with invalid MTU: %zu",
+				   binding->name,
+				   MCTP_BODY_SIZE(binding->pkt_size));
+			return;
+		}
+
 		bus->state = mctp_bus_state_tx_enabled;
 		mctp_prinfo("%s binding started", binding->name);
 		return;
@@ -643,7 +650,14 @@
 	if (bus->state == mctp_bus_state_constructed)
 		return -ENXIO;
 
-	max_payload_len = bus->binding->pkt_size - sizeof(*hdr);
+	max_payload_len = MCTP_BODY_SIZE(bus->binding->pkt_size);
+
+	{
+		const bool valid_mtu = max_payload_len >= MCTP_BTU;
+		assert(valid_mtu);
+		if (!valid_mtu)
+			return -EINVAL;
+	}
 
 	mctp_prdebug("%s: Generating packets for transmission of %zu byte message from %hhu to %hhu",
 		     __func__, msg_len, src, dest);