core: Define return value behaviours for binding Tx callbacks

Binding Tx callbacks must return 0 upon success or a negative error code
on failure. Some error codes invoke specific error handling behaviours.
If a binding Tx callback returns the following negative error codes:

1. EMSGSIZE: The packet whose transmission failed is dequeued from the
   transmit queue and dropped, as it will never be successfully
   transmitted

2. EBUSY: The packet whose transmission failed remains queued for a
   subsequent attempt.

This prevents Tx queue stalls for bindings such as astlpc where
reinitialisation can renegotiate the Tx buffer size to a lower value
than the size of packets already in the Tx queue. Previously the
implementation in core failed to discard the packet from the binding Tx
queue if transmission of the head packet was not possible.

Signed-off-by: Andrew Jeffery <andrew@aj.id.au>
Change-Id: I7dde22bd7340f2c028adf5112d7d4ab78cac66a6
diff --git a/astlpc.c b/astlpc.c
index 0e7e9f8..455276d 100644
--- a/astlpc.c
+++ b/astlpc.c
@@ -821,9 +821,10 @@
 		       __func__, len, hdr->src, hdr->dest, hdr->flags_seq_tag);
 
 	if (len > astlpc->proto->body_size(astlpc->layout.tx.size)) {
-		astlpc_prwarn(astlpc, "invalid TX len %" PRIu32 ": %" PRIu32, len,
-				astlpc->proto->body_size(astlpc->layout.tx.size));
-		return -1;
+		astlpc_prwarn(astlpc, "invalid TX len %" PRIu32 ": %" PRIu32,
+			      len,
+			      astlpc->proto->body_size(astlpc->layout.tx.size));
+		return -EMSGSIZE;
 	}
 
 	mctp_binding_set_tx_enabled(b, false);
diff --git a/core.c b/core.c
index e878175..2f87692 100644
--- a/core.c
+++ b/core.c
@@ -694,13 +694,26 @@
 		int rc;
 
 		rc = mctp_packet_tx(bus, pkt);
-		if (rc)
+		switch (rc) {
+		/* If transmission succeded, or */
+		case 0:
+		/* If the packet is somehow too large */
+		case -EMSGSIZE:
+			/* Drop the packet */
+			bus->tx_queue_head = pkt->next;
+			mctp_pktbuf_free(pkt);
 			break;
 
-		bus->tx_queue_head = pkt->next;
-		mctp_pktbuf_free(pkt);
+		/* If the binding was busy, or */
+		case -EBUSY:
+		/* Some other unknown error occurred */
+		default:
+			/* Make sure the tail pointer is consistent and retry later */
+			goto cleanup_tail;
+		};
 	}
 
+cleanup_tail:
 	if (!bus->tx_queue_head)
 		bus->tx_queue_tail = NULL;
 
diff --git a/libmctp.h b/libmctp.h
index 497d536..8e13c29 100644
--- a/libmctp.h
+++ b/libmctp.h
@@ -107,6 +107,14 @@
 		    uint8_t msg_tag, void *msg, size_t msg_len);
 
 /* hardware bindings */
+
+/**
+ * @tx: Binding function to transmit one packet on the interface
+ *      Return:
+ *      * 0 - Success, pktbuf can be released
+ *	* -EMSGSIZE - Packet exceeds binding MTU, pktbuf must be dropped
+ *	* -EBUSY - Packet unable to be transmitted, pktbuf must be retained
+ */
 struct mctp_binding {
 	const char *name;
 	uint8_t version;
diff --git a/serial.c b/serial.c
index 5e37507..35c4a02 100644
--- a/serial.c
+++ b/serial.c
@@ -154,7 +154,7 @@
 
 	len = mctp_serial_pkt_escape(pkt, NULL);
 	if (len + sizeof(*hdr) + sizeof(*tlr) > sizeof(serial->txbuf))
-		return -1;
+		return -EMSGSIZE;
 
 	mctp_serial_pkt_escape(pkt, buf);