SUNRPC: Use per-CPU counters to tally server RPC counts

 - Improves counting accuracy
 - Reduces cross-CPU memory traffic

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>

diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c
index 0b28a6cf93032886b11bb8f2869bae35716bf3c5..1da00230860c20b6de09a603a036f41ba7143617 100644 (file)
--- a/fs/lockd/svc.c
+++ b/fs/lockd/svc.c
@@ -721,7 +721,7 @@ out_encode_err:
 /*
  * Define NLM program and procedures
  */
-static unsigned int nlmsvc_version1_count[17];
+static DEFINE_PER_CPU_ALIGNED(unsigned long, nlmsvc_version1_count[17]);
 static const struct svc_version        nlmsvc_version1 = {
        .vs_vers        = 1,
        .vs_nproc       = 17,
@@ -730,26 +730,31 @@ static const struct svc_version   nlmsvc_version1 = {
        .vs_dispatch    = nlmsvc_dispatch,
        .vs_xdrsize     = NLMSVC_XDRSIZE,
 };
-static unsigned int nlmsvc_version3_count[24];
+
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nlmsvc_version3_count[ARRAY_SIZE(nlmsvc_procedures)]);
 static const struct svc_version        nlmsvc_version3 = {
        .vs_vers        = 3,
-       .vs_nproc       = 24,
+       .vs_nproc       = ARRAY_SIZE(nlmsvc_procedures),
        .vs_proc        = nlmsvc_procedures,
        .vs_count       = nlmsvc_version3_count,
        .vs_dispatch    = nlmsvc_dispatch,
        .vs_xdrsize     = NLMSVC_XDRSIZE,
 };
+
 #ifdef CONFIG_LOCKD_V4
-static unsigned int nlmsvc_version4_count[24];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nlmsvc_version4_count[ARRAY_SIZE(nlmsvc_procedures4)]);
 static const struct svc_version        nlmsvc_version4 = {
        .vs_vers        = 4,
-       .vs_nproc       = 24,
+       .vs_nproc       = ARRAY_SIZE(nlmsvc_procedures4),
        .vs_proc        = nlmsvc_procedures4,
        .vs_count       = nlmsvc_version4_count,
        .vs_dispatch    = nlmsvc_dispatch,
        .vs_xdrsize     = NLMSVC_XDRSIZE,
 };
 #endif
+
 static const struct svc_version *nlmsvc_version[] = {
        [1] = &nlmsvc_version1,
        [3] = &nlmsvc_version3,

diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
index 13b2af55497d43f3dbf9b4a9ad99da5fd1c77be2..321af81c456e2a6e68462bc683f2973d58dc4687 100644 (file)
--- a/fs/nfs/callback_xdr.c
+++ b/fs/nfs/callback_xdr.c
@@ -1069,7 +1069,8 @@ static const struct svc_procedure nfs4_callback_procedures1[] = {
        }
 };
 
-static unsigned int nfs4_callback_count1[ARRAY_SIZE(nfs4_callback_procedures1)];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nfs4_callback_count1[ARRAY_SIZE(nfs4_callback_procedures1)]);
 const struct svc_version nfs4_callback_version1 = {
        .vs_vers = 1,
        .vs_nproc = ARRAY_SIZE(nfs4_callback_procedures1),
@@ -1081,7 +1082,8 @@ const struct svc_version nfs4_callback_version1 = {
        .vs_need_cong_ctrl = true,
 };
 
-static unsigned int nfs4_callback_count4[ARRAY_SIZE(nfs4_callback_procedures1)];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nfs4_callback_count4[ARRAY_SIZE(nfs4_callback_procedures1)]);
 const struct svc_version nfs4_callback_version4 = {
        .vs_vers = 4,
        .vs_nproc = ARRAY_SIZE(nfs4_callback_procedures1),

diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c
index 1457f59f447a415f1e2e518045655f1c56257bfe..dea55883e0996f4bf6570eab84a906227d967380 100644 (file)
--- a/fs/nfsd/nfs2acl.c
+++ b/fs/nfsd/nfs2acl.c
@@ -377,10 +377,11 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = {
        },
 };
 
-static unsigned int nfsd_acl_count2[ARRAY_SIZE(nfsd_acl_procedures2)];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nfsd_acl_count2[ARRAY_SIZE(nfsd_acl_procedures2)]);
 const struct svc_version nfsd_acl_version2 = {
        .vs_vers        = 2,
-       .vs_nproc       = 5,
+       .vs_nproc       = ARRAY_SIZE(nfsd_acl_procedures2),
        .vs_proc        = nfsd_acl_procedures2,
        .vs_count       = nfsd_acl_count2,
        .vs_dispatch    = nfsd_dispatch,

diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c
index 647108138e8abf401ff0044ebb06086b73a297bb..bf1c52c1bdab54e02af561df2a6332a105a80574 100644 (file)
--- a/fs/nfsd/nfs3acl.c
+++ b/fs/nfsd/nfs3acl.c
@@ -266,10 +266,11 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = {
        },
 };
 
-static unsigned int nfsd_acl_count3[ARRAY_SIZE(nfsd_acl_procedures3)];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nfsd_acl_count3[ARRAY_SIZE(nfsd_acl_procedures3)]);
 const struct svc_version nfsd_acl_version3 = {
        .vs_vers        = 3,
-       .vs_nproc       = 3,
+       .vs_nproc       = ARRAY_SIZE(nfsd_acl_procedures3),
        .vs_proc        = nfsd_acl_procedures3,
        .vs_count       = nfsd_acl_count3,
        .vs_dispatch    = nfsd_dispatch,

diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c
index d01b29aba66237ca43755d0f739cbe38585cac89..1c73921c02e086225dd081023c032eb2ce43ab7b 100644 (file)
--- a/fs/nfsd/nfs3proc.c
+++ b/fs/nfsd/nfs3proc.c
@@ -1064,10 +1064,11 @@ static const struct svc_procedure nfsd_procedures3[22] = {
        },
 };
 
-static unsigned int nfsd_count3[ARRAY_SIZE(nfsd_procedures3)];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nfsd_count3[ARRAY_SIZE(nfsd_procedures3)]);
 const struct svc_version nfsd_version3 = {
        .vs_vers        = 3,
-       .vs_nproc       = 22,
+       .vs_nproc       = ARRAY_SIZE(nfsd_procedures3),
        .vs_proc        = nfsd_procedures3,
        .vs_dispatch    = nfsd_dispatch,
        .vs_count       = nfsd_count3,

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index dbaf33398c827599e163e392680ac20eab5b97dd..9435136d7de6b8a0dfd54f363a0f8b1529574689 100644 (file)
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -3596,12 +3596,13 @@ static const struct svc_procedure nfsd_procedures4[2] = {
        },
 };
 
-static unsigned int nfsd_count3[ARRAY_SIZE(nfsd_procedures4)];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nfsd_count4[ARRAY_SIZE(nfsd_procedures4)]);
 const struct svc_version nfsd_version4 = {
        .vs_vers                = 4,
-       .vs_nproc               = 2,
+       .vs_nproc               = ARRAY_SIZE(nfsd_procedures4),
        .vs_proc                = nfsd_procedures4,
-       .vs_count               = nfsd_count3,
+       .vs_count               = nfsd_count4,
        .vs_dispatch            = nfsd_dispatch,
        .vs_xdrsize             = NFS4_SVC_XDRSIZE,
        .vs_rpcb_optnl          = true,

diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c
index 9744443c396522af6bf9ae34eca57c60a133cc8f..9e6069e9c9f1d7a0c77712c69d2b0d99e9b07037 100644 (file)
--- a/fs/nfsd/nfsproc.c
+++ b/fs/nfsd/nfsproc.c
@@ -838,11 +838,11 @@ static const struct svc_procedure nfsd_procedures2[18] = {
        },
 };
 
-
-static unsigned int nfsd_count2[ARRAY_SIZE(nfsd_procedures2)];
+static DEFINE_PER_CPU_ALIGNED(unsigned long,
+                             nfsd_count2[ARRAY_SIZE(nfsd_procedures2)]);
 const struct svc_version nfsd_version2 = {
        .vs_vers        = 2,
-       .vs_nproc       = 18,
+       .vs_nproc       = ARRAY_SIZE(nfsd_procedures2),
        .vs_proc        = nfsd_procedures2,
        .vs_count       = nfsd_count2,
        .vs_dispatch    = nfsd_dispatch,

diff --git a/include/linux/lockd/lockd.h b/include/linux/lockd/lockd.h
index 70ce419e270935f7b42e40ee3a658bb015ffc304..84de6b7c994885cddd13bfa57a47bbc787e50c71 100644 (file)
--- a/include/linux/lockd/lockd.h
+++ b/include/linux/lockd/lockd.h
@@ -196,9 +196,9 @@ struct nlm_block {
  * Global variables
  */
 extern const struct rpc_program        nlm_program;
-extern const struct svc_procedure nlmsvc_procedures[];
+extern const struct svc_procedure nlmsvc_procedures[24];
 #ifdef CONFIG_LOCKD_V4
-extern const struct svc_procedure nlmsvc_procedures4[];
+extern const struct svc_procedure nlmsvc_procedures4[24];
 #endif
 extern int                     nlmsvc_grace_period;
 extern unsigned long           nlmsvc_timeout;

diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
index 392d2d2620fac60d6e7a76375a0e9a04f4121965..1b078c9a2d60718d33c032bfc842de574cdc1fd6 100644 (file)
--- a/include/linux/sunrpc/svc.h
+++ b/include/linux/sunrpc/svc.h
@@ -377,7 +377,7 @@ struct svc_version {
        u32                     vs_vers;        /* version number */
        u32                     vs_nproc;       /* number of procedures */
        const struct svc_procedure *vs_proc;    /* per-procedure info */
-       unsigned int            *vs_count;      /* call counts */
+       unsigned long __percpu  *vs_count;      /* call counts */
        u32                     vs_xdrsize;     /* xdrsize needed for this version */
 
        /* Don't register with rpcbind */

diff --git a/net/sunrpc/stats.c b/net/sunrpc/stats.c
index 52908f9e6eab54572f6852d725daef152163a77d..65fc1297c6dfa4fcd96857a5a0753e78a6f5ebb8 100644 (file)
--- a/net/sunrpc/stats.c
+++ b/net/sunrpc/stats.c
@@ -83,7 +83,8 @@ void svc_seq_show(struct seq_file *seq, const struct svc_stat *statp)
 {
        const struct svc_program *prog = statp->program;
        const struct svc_version *vers;
-       unsigned int i, j;
+       unsigned int i, j, k;
+       unsigned long count;
 
        seq_printf(seq,
                "net %u %u %u %u\n",
@@ -104,8 +105,12 @@ void svc_seq_show(struct seq_file *seq, const struct svc_stat *statp)
                if (!vers)
                        continue;
                seq_printf(seq, "proc%d %u", i, vers->vs_nproc);
-               for (j = 0; j < vers->vs_nproc; j++)
-                       seq_printf(seq, " %u", vers->vs_count[j]);
+               for (j = 0; j < vers->vs_nproc; j++) {
+                       count = 0;
+                       for_each_possible_cpu(k)
+                               count += per_cpu(vers->vs_count[j], k);
+                       seq_printf(seq, " %lu", count);
+               }
                seq_putc(seq, '\n');
        }
 }

diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index 7b208e063334f50d748b3e651897eb408ce7c040..5dc298c30ea8a1cb73e444f336e2791e8c3a22e4 100644 (file)
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -1208,7 +1208,7 @@ svc_generic_init_request(struct svc_rqst *rqstp,
        memset(rqstp->rq_resp, 0, procp->pc_ressize);
 
        /* Bump per-procedure stats counter */
-       versp->vs_count[rqstp->rq_proc]++;
+       this_cpu_inc(versp->vs_count[rqstp->rq_proc]);
 
        ret->dispatch = versp->vs_dispatch;
        return rpc_success;