RDMA/efa: Expose device statistics

Expose hardware statistics through the sysfs api:
/sys/class/infiniband/efa_0/hw_counters/*.
/sys/class/infiniband/efa_0/ports/1/hw_counters/*.

Reviewed-by: Firas JahJah <firasj@amazon.com>
Reviewed-by: Yossi Leybovich <sleybo@amazon.com>
Signed-off-by: Gal Pressman <galpress@amazon.com>
Link: https://lore.kernel.org/r/20190725130353.11544-1-galpress@amazon.com
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h
index 119f8efec56474340bb6a7cb9f13fee456e4e67d..2283e432693ea0771804915c2c25c8eae4298ffc 100644 (file)
--- a/drivers/infiniband/hw/efa/efa.h
+++ b/drivers/infiniband/hw/efa/efa.h
@@ -156,5 +156,8 @@ int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
                  int qp_attr_mask, struct ib_udata *udata);
 enum rdma_link_layer efa_port_link_layer(struct ib_device *ibdev,
                                         u8 port_num);
+struct rdma_hw_stats *efa_alloc_hw_stats(struct ib_device *ibdev, u8 port_num);
+int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
+                    u8 port_num, int index);
 
 #endif /* _EFA_H_ */

diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c
index 62345d8abf3ca20bde1ffbe79a1ab8253fb2a81c..16b115df63e8f65c671cb205f0380d880a3797f9 100644 (file)
--- a/drivers/infiniband/hw/efa/efa_com_cmd.c
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.c
@@ -702,3 +702,38 @@ int efa_com_dealloc_uar(struct efa_com_dev *edev,
 
        return 0;
 }
+
+int efa_com_get_stats(struct efa_com_dev *edev,
+                     struct efa_com_get_stats_params *params,
+                     union efa_com_get_stats_result *result)
+{
+       struct efa_com_admin_queue *aq = &edev->aq;
+       struct efa_admin_aq_get_stats_cmd cmd = {};
+       struct efa_admin_acq_get_stats_resp resp;
+       int err;
+
+       cmd.aq_common_descriptor.opcode = EFA_ADMIN_GET_STATS;
+       cmd.type = params->type;
+       cmd.scope = params->scope;
+       cmd.scope_modifier = params->scope_modifier;
+
+       err = efa_com_cmd_exec(aq,
+                              (struct efa_admin_aq_entry *)&cmd,
+                              sizeof(cmd),
+                              (struct efa_admin_acq_entry *)&resp,
+                              sizeof(resp));
+       if (err) {
+               ibdev_err(edev->efa_dev,
+                         "Failed to get stats type-%u scope-%u.%u [%d]\n",
+                         cmd.type, cmd.scope, cmd.scope_modifier, err);
+               return err;
+       }
+
+       result->basic_stats.tx_bytes = resp.basic_stats.tx_bytes;
+       result->basic_stats.tx_pkts = resp.basic_stats.tx_pkts;
+       result->basic_stats.rx_bytes = resp.basic_stats.rx_bytes;
+       result->basic_stats.rx_pkts = resp.basic_stats.rx_pkts;
+       result->basic_stats.rx_drops = resp.basic_stats.rx_drops;
+
+       return 0;
+}

diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h
index a1174380462c15569e8d63b7c3a89d5db60cb3c0..7f6c13052f497699b5086ce6c8c6eebd4aa3ba07 100644 (file)
--- a/drivers/infiniband/hw/efa/efa_com_cmd.h
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.h
@@ -225,6 +225,26 @@ struct efa_com_dealloc_uar_params {
        u16 uarn;
 };
 
+struct efa_com_get_stats_params {
+       /* see enum efa_admin_get_stats_type */
+       u8 type;
+       /* see enum efa_admin_get_stats_scope */
+       u8 scope;
+       u16 scope_modifier;
+};
+
+struct efa_com_basic_stats {
+       u64 tx_bytes;
+       u64 tx_pkts;
+       u64 rx_bytes;
+       u64 rx_pkts;
+       u64 rx_drops;
+};
+
+union efa_com_get_stats_result {
+       struct efa_com_basic_stats basic_stats;
+};
+
 void efa_com_set_dma_addr(dma_addr_t addr, u32 *addr_high, u32 *addr_low);
 int efa_com_create_qp(struct efa_com_dev *edev,
                      struct efa_com_create_qp_params *params,
@@ -266,5 +286,8 @@ int efa_com_alloc_uar(struct efa_com_dev *edev,
                      struct efa_com_alloc_uar_result *result);
 int efa_com_dealloc_uar(struct efa_com_dev *edev,
                        struct efa_com_dealloc_uar_params *params);
+int efa_com_get_stats(struct efa_com_dev *edev,
+                     struct efa_com_get_stats_params *params,
+                     union efa_com_get_stats_result *result);
 
 #endif /* _EFA_COM_CMD_H_ */

diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c
index dd1c6d49466f5837fa4c33f1cf368e7cfdc77277..83858f7e83d0f2c2d8dd016d08f514d7af376783 100644 (file)
--- a/drivers/infiniband/hw/efa/efa_main.c
+++ b/drivers/infiniband/hw/efa/efa_main.c
@@ -201,6 +201,7 @@ static const struct ib_device_ops efa_dev_ops = {
        .driver_id = RDMA_DRIVER_EFA,
        .uverbs_abi_ver = EFA_UVERBS_ABI_VERSION,
 
+       .alloc_hw_stats = efa_alloc_hw_stats,
        .alloc_pd = efa_alloc_pd,
        .alloc_ucontext = efa_alloc_ucontext,
        .create_ah = efa_create_ah,
@@ -212,6 +213,7 @@ static const struct ib_device_ops efa_dev_ops = {
        .destroy_ah = efa_destroy_ah,
        .destroy_cq = efa_destroy_cq,
        .destroy_qp = efa_destroy_qp,
+       .get_hw_stats = efa_get_hw_stats,
        .get_link_layer = efa_port_link_layer,
        .get_port_immutable = efa_get_port_immutable,
        .mmap = efa_mmap,

diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
index df77bc312a25388bee6e84c95f317daf542352cb..32d3b3deabce29486790f1267c5ac17aa147d5e3 100644 (file)
--- a/drivers/infiniband/hw/efa/efa_verbs.c
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -41,6 +41,33 @@ static inline u64 get_mmap_key(const struct efa_mmap_entry *efa)
               ((u64)efa->mmap_page << PAGE_SHIFT);
 }
 
+#define EFA_DEFINE_STATS(op) \
+       op(EFA_TX_BYTES, "tx_bytes") \
+       op(EFA_TX_PKTS, "tx_pkts") \
+       op(EFA_RX_BYTES, "rx_bytes") \
+       op(EFA_RX_PKTS, "rx_pkts") \
+       op(EFA_RX_DROPS, "rx_drops") \
+       op(EFA_SUBMITTED_CMDS, "submitted_cmds") \
+       op(EFA_COMPLETED_CMDS, "completed_cmds") \
+       op(EFA_NO_COMPLETION_CMDS, "no_completion_cmds") \
+       op(EFA_KEEP_ALIVE_RCVD, "keep_alive_rcvd") \
+       op(EFA_ALLOC_PD_ERR, "alloc_pd_err") \
+       op(EFA_CREATE_QP_ERR, "create_qp_err") \
+       op(EFA_REG_MR_ERR, "reg_mr_err") \
+       op(EFA_ALLOC_UCONTEXT_ERR, "alloc_ucontext_err") \
+       op(EFA_CREATE_AH_ERR, "create_ah_err")
+
+#define EFA_STATS_ENUM(ename, name) ename,
+#define EFA_STATS_STR(ename, name) [ename] = name,
+
+enum efa_hw_stats {
+       EFA_DEFINE_STATS(EFA_STATS_ENUM)
+};
+
+static const char *const efa_stats_names[] = {
+       EFA_DEFINE_STATS(EFA_STATS_STR)
+};
+
 #define EFA_CHUNK_PAYLOAD_SHIFT       12
 #define EFA_CHUNK_PAYLOAD_SIZE        BIT(EFA_CHUNK_PAYLOAD_SHIFT)
 #define EFA_CHUNK_PAYLOAD_PTR_SIZE    8
@@ -1727,6 +1754,54 @@ void efa_destroy_ah(struct ib_ah *ibah, u32 flags)
        efa_ah_destroy(dev, ah);
 }
 
+struct rdma_hw_stats *efa_alloc_hw_stats(struct ib_device *ibdev, u8 port_num)
+{
+       return rdma_alloc_hw_stats_struct(efa_stats_names,
+                                         ARRAY_SIZE(efa_stats_names),
+                                         RDMA_HW_STATS_DEFAULT_LIFESPAN);
+}
+
+int efa_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
+                    u8 port_num, int index)
+{
+       struct efa_com_get_stats_params params = {};
+       union efa_com_get_stats_result result;
+       struct efa_dev *dev = to_edev(ibdev);
+       struct efa_com_basic_stats *bs;
+       struct efa_com_stats_admin *as;
+       struct efa_stats *s;
+       int err;
+
+       params.type = EFA_ADMIN_GET_STATS_TYPE_BASIC;
+       params.scope = EFA_ADMIN_GET_STATS_SCOPE_ALL;
+
+       err = efa_com_get_stats(&dev->edev, &params, &result);
+       if (err)
+               return err;
+
+       bs = &result.basic_stats;
+       stats->value[EFA_TX_BYTES] = bs->tx_bytes;
+       stats->value[EFA_TX_PKTS] = bs->tx_pkts;
+       stats->value[EFA_RX_BYTES] = bs->rx_bytes;
+       stats->value[EFA_RX_PKTS] = bs->rx_pkts;
+       stats->value[EFA_RX_DROPS] = bs->rx_drops;
+
+       as = &dev->edev.aq.stats;
+       stats->value[EFA_SUBMITTED_CMDS] = atomic64_read(&as->submitted_cmd);
+       stats->value[EFA_COMPLETED_CMDS] = atomic64_read(&as->completed_cmd);
+       stats->value[EFA_NO_COMPLETION_CMDS] = atomic64_read(&as->no_completion);
+
+       s = &dev->stats;
+       stats->value[EFA_KEEP_ALIVE_RCVD] = atomic64_read(&s->keep_alive_rcvd);
+       stats->value[EFA_ALLOC_PD_ERR] = atomic64_read(&s->sw_stats.alloc_pd_err);
+       stats->value[EFA_CREATE_QP_ERR] = atomic64_read(&s->sw_stats.create_qp_err);
+       stats->value[EFA_REG_MR_ERR] = atomic64_read(&s->sw_stats.reg_mr_err);
+       stats->value[EFA_ALLOC_UCONTEXT_ERR] = atomic64_read(&s->sw_stats.alloc_ucontext_err);
+       stats->value[EFA_CREATE_AH_ERR] = atomic64_read(&s->sw_stats.create_ah_err);
+
+       return ARRAY_SIZE(efa_stats_names);
+}
+
 enum rdma_link_layer efa_port_link_layer(struct ib_device *ibdev,
                                         u8 port_num)
 {