RDMA/rxe: Let destroy qp succeed with stuck packet

In some situations a sent packet may get queued in the NIC longer than
than timeout of a ULP. Currently if this happens the ULP may try to reset
the link by destroying the qp and setting up an alternate connection but
will fail because the rxe driver is waiting for the packet to finish
getting sent and be returned to the skb destructor function where the qp
reference holding things up will be dropped. This patch modifies the way
that the qp is passed to the destructor to pass the qp index and not a qp
pointer.  Then the destructor will attempt to lookup the qp from its index
and if it fails exit early. This requires taking a reference on the struct
sock rather than the qp allowing the qp to be destroyed while the sk is
still around waiting for the packet to finish.

Link: https://lore.kernel.org/r/20240329145513.35381-15-rpearsonhpe@gmail.com
Signed-off-by: Bob Pearson <rpearsonhpe@gmail.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index b58eab75df9787cc5e4d56a10d996054b49e64df..ca9a82e1c4c7ea96946861cef6606e3190cde16f 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -345,25 +345,44 @@ int rxe_prepare(struct rxe_av *av, struct rxe_pkt_info *pkt,
 
 static void rxe_skb_tx_dtor(struct sk_buff *skb)
 {
-       struct sock *sk = skb->sk;
-       struct rxe_qp *qp = sk->sk_user_data;
-       int skb_out = atomic_dec_return(&qp->skb_out);
+       struct net_device *ndev = skb->dev;
+       struct rxe_dev *rxe;
+       unsigned int qp_index;
+       struct rxe_qp *qp;
+       int skb_out;
+
+       rxe = rxe_get_dev_from_net(ndev);
+       if (!rxe && is_vlan_dev(ndev))
+               rxe = rxe_get_dev_from_net(vlan_dev_real_dev(ndev));
+       if (WARN_ON(!rxe))
+               return;
 
-       if (unlikely(qp->need_req_skb &&
-                    skb_out < RXE_INFLIGHT_SKBS_PER_QP_LOW))
+       qp_index = (int)(uintptr_t)skb->sk->sk_user_data;
+       if (!qp_index)
+               return;
+
+       qp = rxe_pool_get_index(&rxe->qp_pool, qp_index);
+       if (!qp)
+               goto put_dev;
+
+       skb_out = atomic_dec_return(&qp->skb_out);
+       if (qp->need_req_skb && skb_out < RXE_INFLIGHT_SKBS_PER_QP_LOW)
                rxe_sched_task(&qp->send_task);
 
        rxe_put(qp);
+put_dev:
+       ib_device_put(&rxe->ib_dev);
+       sock_put(skb->sk);
 }
 
 static int rxe_send(struct sk_buff *skb, struct rxe_pkt_info *pkt)
 {
        int err;
+       struct sock *sk = pkt->qp->sk->sk;
 
+       sock_hold(sk);
+       skb->sk = sk;
        skb->destructor = rxe_skb_tx_dtor;
-       skb->sk = pkt->qp->sk->sk;
-
-       rxe_get(pkt->qp);
        atomic_inc(&pkt->qp->skb_out);
 
        if (skb->protocol == htons(ETH_P_IP))
@@ -379,12 +398,13 @@ static int rxe_send(struct sk_buff *skb, struct rxe_pkt_info *pkt)
  */
 static int rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt)
 {
+       struct sock *sk = pkt->qp->sk->sk;
+
        memcpy(SKB_TO_PKT(skb), pkt, sizeof(*pkt));
 
+       sock_hold(sk);
+       skb->sk = sk;
        skb->destructor = rxe_skb_tx_dtor;
-       skb->sk = pkt->qp->sk->sk;
-
-       rxe_get(pkt->qp);
        atomic_inc(&pkt->qp->skb_out);
 
        if (skb->protocol == htons(ETH_P_IP))

diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c
index c7d99063594bcb8d6fc414eca842711dd2a71768..d2f7b5195c19dd2df34f3c24f3031e946804eb6d 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe_qp.c
+++ b/drivers/infiniband/sw/rxe/rxe_qp.c
@@ -244,7 +244,7 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,
        err = sock_create_kern(&init_net, AF_INET, SOCK_DGRAM, 0, &qp->sk);
        if (err < 0)
                return err;
-       qp->sk->sk->sk_user_data = qp;
+       qp->sk->sk->sk_user_data = (void *)(uintptr_t)qp->elem.index;
 
        /* pick a source UDP port number for this QP based on
         * the source QPN. this spreads traffic for different QPs