net: stmmac: Add Tx HWTS support to XDP ZC

This patch enables transmit hardware timestamp support to XDP zero copy
via XDP Tx metadata framework.

This patchset is tested with tools/testing/selftests/bpf/xdp_hw_metadata
on Intel Tiger Lake platform. Below are the test steps and results.

Command on DUT:
sudo ./xdp_hw_metadata <interface name>
sudo hwstamp_ctl -i <interface name> -t 1 -r 1

Command on Link Partner:
echo -n xdp | nc -u -q1 <destination IPv4 addr> 9091

Result:
xsk_ring_cons__peek: 1
0x55bbbf08b6d0: rx_desc[2]->addr=8c100 addr=8c100 comp_addr=8c100 EoP
No rx_hash err=-95
rx_timestamp:  1677762688429141540 (sec:1677762688.4291)
HW RX-time:   1677762688429141540 (sec:1677762688.4291) delta to User RX-time sec:0.0003 (250.665 usec)
XDP RX-time:   1677762688429375597 (sec:1677762688.4294) delta to User RX-time sec:0.0000 (16.608 usec)
0x55bbbf08b6d0: ping-pong with csum=561c (want f488) csum_start=34 csum_offset=6
0x55bbbf08b6d0: complete tx idx=2 addr=2008
tx_timestamp:  1677762688431127273 (sec:1677762688.4311)
HW TX-complete-time:   1677762688431127273 (sec:1677762688.4311) delta to User TX-complete-time sec:0.0083 (8331.655 usec)
XDP RX-time:   1677762688429375597 (sec:1677762688.4294) delta to User TX-complete-time sec:0.0101 (10083.331 usec)
HW RX-time:   1677762688429141540 (sec:1677762688.4291) delta to HW TX-complete-time sec:0.0020 (1985.733 usec)
0x55bbbf08b6d0: complete rx idx=130 addr=8c100

Signed-off-by: Song Yoong Siang <yoong.siang.song@intel.com>
Signed-off-by: Stanislav Fomichev <sdf@google.com>
Link: https://lore.kernel.org/r/20231127190319.1190813-6-sdf@google.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
index cd7a9768de5f12960cd88167a5483fbc24cf06d8..686c94c2e8a7282c8639ffbf5bd6022cc16648a2 100644 (file)
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h
@@ -51,6 +51,7 @@ struct stmmac_tx_info {
        bool last_segment;
        bool is_jumbo;
        enum stmmac_txbuf_type buf_type;
+       struct xsk_tx_metadata_compl xsk_meta;
 };
 
 #define STMMAC_TBS_AVAIL       BIT(0)
@@ -100,6 +101,17 @@ struct stmmac_xdp_buff {
        struct dma_desc *ndesc;
 };
 
+struct stmmac_metadata_request {
+       struct stmmac_priv *priv;
+       struct dma_desc *tx_desc;
+       bool *set_ic;
+};
+
+struct stmmac_xsk_tx_complete {
+       struct stmmac_priv *priv;
+       struct dma_desc *desc;
+};
+
 struct stmmac_rx_queue {
        u32 rx_count_frames;
        u32 queue_index;

diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
index 8964fc8a955fd11e9036254a7fbe67ecbaf1efa8..c2ac88aaffed66d547537fc4c22944f5e430a1b9 100644 (file)
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
@@ -2430,6 +2430,46 @@ static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
        }
 }
 
+static void stmmac_xsk_request_timestamp(void *_priv)
+{
+       struct stmmac_metadata_request *meta_req = _priv;
+
+       stmmac_enable_tx_timestamp(meta_req->priv, meta_req->tx_desc);
+       *meta_req->set_ic = true;
+}
+
+static u64 stmmac_xsk_fill_timestamp(void *_priv)
+{
+       struct stmmac_xsk_tx_complete *tx_compl = _priv;
+       struct stmmac_priv *priv = tx_compl->priv;
+       struct dma_desc *desc = tx_compl->desc;
+       bool found = false;
+       u64 ns = 0;
+
+       if (!priv->hwts_tx_en)
+               return 0;
+
+       /* check tx tstamp status */
+       if (stmmac_get_tx_timestamp_status(priv, desc)) {
+               stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
+               found = true;
+       } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {
+               found = true;
+       }
+
+       if (found) {
+               ns -= priv->plat->cdc_error_adj;
+               return ns_to_ktime(ns);
+       }
+
+       return 0;
+}
+
+static const struct xsk_tx_metadata_ops stmmac_xsk_tx_metadata_ops = {
+       .tmo_request_timestamp          = stmmac_xsk_request_timestamp,
+       .tmo_fill_timestamp             = stmmac_xsk_fill_timestamp,
+};
+
 static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
 {
        struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue);
@@ -2449,6 +2489,8 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
        budget = min(budget, stmmac_tx_avail(priv, queue));
 
        while (budget-- > 0) {
+               struct stmmac_metadata_request meta_req;
+               struct xsk_tx_metadata *meta = NULL;
                dma_addr_t dma_addr;
                bool set_ic;
 
@@ -2472,6 +2514,7 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
                        tx_desc = tx_q->dma_tx + entry;
 
                dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr);
+               meta = xsk_buff_get_metadata(pool, xdp_desc.addr);
                xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len);
 
                tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX;
@@ -2499,6 +2542,11 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
                else
                        set_ic = false;
 
+               meta_req.priv = priv;
+               meta_req.tx_desc = tx_desc;
+               meta_req.set_ic = &set_ic;
+               xsk_tx_metadata_request(meta, &stmmac_xsk_tx_metadata_ops,
+                                       &meta_req);
                if (set_ic) {
                        tx_q->tx_count_frames = 0;
                        stmmac_set_tx_ic(priv, tx_desc);
@@ -2511,6 +2559,9 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
 
                stmmac_enable_dma_transmission(priv, priv->ioaddr);
 
+               xsk_tx_metadata_to_compl(meta,
+                                        &tx_q->tx_skbuff_dma[entry].xsk_meta);
+
                tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size);
                entry = tx_q->cur_tx;
        }
@@ -2620,8 +2671,18 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue,
                        } else {
                                tx_packets++;
                        }
-                       if (skb)
+                       if (skb) {
                                stmmac_get_tx_hwtstamp(priv, p, skb);
+                       } else {
+                               struct stmmac_xsk_tx_complete tx_compl = {
+                                       .priv = priv,
+                                       .desc = p,
+                               };
+
+                               xsk_tx_metadata_complete(&tx_q->tx_skbuff_dma[entry].xsk_meta,
+                                                        &stmmac_xsk_tx_metadata_ops,
+                                                        &tx_compl);
+                       }
                }
 
                if (likely(tx_q->tx_skbuff_dma[entry].buf &&
@@ -7464,6 +7525,7 @@ int stmmac_dvr_probe(struct device *device,
        ndev->netdev_ops = &stmmac_netdev_ops;
 
        ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops;
+       ndev->xsk_tx_metadata_ops = &stmmac_xsk_tx_metadata_ops;
 
        ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
                            NETIF_F_RXCSUM;