1 // SPDX-License-Identifier: GPL-2.0
3 * Request reply cache. This is currently a global cache, but this may
4 * change in the future and be a per-client cache.
6 * This code is heavily inspired by the 44BSD implementation, although
7 * it does things a bit differently.
9 * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
12 #include <linux/sunrpc/svc_xprt.h>
13 #include <linux/slab.h>
14 #include <linux/vmalloc.h>
15 #include <linux/sunrpc/addr.h>
16 #include <linux/highmem.h>
17 #include <linux/log2.h>
18 #include <linux/hash.h>
19 #include <net/checksum.h>
26 * We use this value to determine the number of hash buckets from the max
27 * cache size, the idea being that when the cache is at its maximum number
28 * of entries, then this should be the average number of entries per bucket.
30 #define TARGET_BUCKET_SIZE 64
32 struct nfsd_drc_bucket
{
33 struct rb_root rb_head
;
34 struct list_head lru_head
;
35 spinlock_t cache_lock
;
38 static struct kmem_cache
*drc_slab
;
40 static int nfsd_cache_append(struct svc_rqst
*rqstp
, struct kvec
*vec
);
41 static unsigned long nfsd_reply_cache_count(struct shrinker
*shrink
,
42 struct shrink_control
*sc
);
43 static unsigned long nfsd_reply_cache_scan(struct shrinker
*shrink
,
44 struct shrink_control
*sc
);
47 * Put a cap on the size of the DRC based on the amount of available
48 * low memory in the machine.
60 * ...with a hard cap of 256k entries. In the worst case, each entry will be
61 * ~1k, so the above numbers should give a rough max of the amount of memory
64 * XXX: these limits are per-container, so memory used will increase
65 * linearly with number of containers. Maybe that's OK.
68 nfsd_cache_size_limit(void)
71 unsigned long low_pages
= totalram_pages() - totalhigh_pages();
73 limit
= (16 * int_sqrt(low_pages
)) << (PAGE_SHIFT
-10);
74 return min_t(unsigned int, limit
, 256*1024);
78 * Compute the number of hash buckets we need. Divide the max cachesize by
79 * the "target" max bucket size, and round up to next power of two.
82 nfsd_hashsize(unsigned int limit
)
84 return roundup_pow_of_two(limit
/ TARGET_BUCKET_SIZE
);
88 nfsd_cache_hash(__be32 xid
, struct nfsd_net
*nn
)
90 return hash_32(be32_to_cpu(xid
), nn
->maskbits
);
93 static struct svc_cacherep
*
94 nfsd_reply_cache_alloc(struct svc_rqst
*rqstp
, __wsum csum
,
97 struct svc_cacherep
*rp
;
99 rp
= kmem_cache_alloc(drc_slab
, GFP_KERNEL
);
101 rp
->c_state
= RC_UNUSED
;
102 rp
->c_type
= RC_NOCACHE
;
103 RB_CLEAR_NODE(&rp
->c_node
);
104 INIT_LIST_HEAD(&rp
->c_lru
);
106 memset(&rp
->c_key
, 0, sizeof(rp
->c_key
));
107 rp
->c_key
.k_xid
= rqstp
->rq_xid
;
108 rp
->c_key
.k_proc
= rqstp
->rq_proc
;
109 rpc_copy_addr((struct sockaddr
*)&rp
->c_key
.k_addr
, svc_addr(rqstp
));
110 rpc_set_port((struct sockaddr
*)&rp
->c_key
.k_addr
, rpc_get_port(svc_addr(rqstp
)));
111 rp
->c_key
.k_prot
= rqstp
->rq_prot
;
112 rp
->c_key
.k_vers
= rqstp
->rq_vers
;
113 rp
->c_key
.k_len
= rqstp
->rq_arg
.len
;
114 rp
->c_key
.k_csum
= csum
;
120 nfsd_reply_cache_free_locked(struct nfsd_drc_bucket
*b
, struct svc_cacherep
*rp
,
123 if (rp
->c_type
== RC_REPLBUFF
&& rp
->c_replvec
.iov_base
) {
124 nfsd_stats_drc_mem_usage_sub(nn
, rp
->c_replvec
.iov_len
);
125 kfree(rp
->c_replvec
.iov_base
);
127 if (rp
->c_state
!= RC_UNUSED
) {
128 rb_erase(&rp
->c_node
, &b
->rb_head
);
129 list_del(&rp
->c_lru
);
130 atomic_dec(&nn
->num_drc_entries
);
131 nfsd_stats_drc_mem_usage_sub(nn
, sizeof(*rp
));
133 kmem_cache_free(drc_slab
, rp
);
137 nfsd_reply_cache_free(struct nfsd_drc_bucket
*b
, struct svc_cacherep
*rp
,
140 spin_lock(&b
->cache_lock
);
141 nfsd_reply_cache_free_locked(b
, rp
, nn
);
142 spin_unlock(&b
->cache_lock
);
145 int nfsd_drc_slab_create(void)
147 drc_slab
= kmem_cache_create("nfsd_drc",
148 sizeof(struct svc_cacherep
), 0, 0, NULL
);
149 return drc_slab
? 0: -ENOMEM
;
152 void nfsd_drc_slab_free(void)
154 kmem_cache_destroy(drc_slab
);
157 static int nfsd_reply_cache_stats_init(struct nfsd_net
*nn
)
159 return nfsd_percpu_counters_init(nn
->counter
, NFSD_NET_COUNTERS_NUM
);
162 static void nfsd_reply_cache_stats_destroy(struct nfsd_net
*nn
)
164 nfsd_percpu_counters_destroy(nn
->counter
, NFSD_NET_COUNTERS_NUM
);
167 int nfsd_reply_cache_init(struct nfsd_net
*nn
)
169 unsigned int hashsize
;
173 nn
->max_drc_entries
= nfsd_cache_size_limit();
174 atomic_set(&nn
->num_drc_entries
, 0);
175 hashsize
= nfsd_hashsize(nn
->max_drc_entries
);
176 nn
->maskbits
= ilog2(hashsize
);
178 status
= nfsd_reply_cache_stats_init(nn
);
182 nn
->nfsd_reply_cache_shrinker
.scan_objects
= nfsd_reply_cache_scan
;
183 nn
->nfsd_reply_cache_shrinker
.count_objects
= nfsd_reply_cache_count
;
184 nn
->nfsd_reply_cache_shrinker
.seeks
= 1;
185 status
= register_shrinker(&nn
->nfsd_reply_cache_shrinker
);
187 goto out_stats_destroy
;
189 nn
->drc_hashtbl
= kvzalloc(array_size(hashsize
,
190 sizeof(*nn
->drc_hashtbl
)), GFP_KERNEL
);
191 if (!nn
->drc_hashtbl
)
194 for (i
= 0; i
< hashsize
; i
++) {
195 INIT_LIST_HEAD(&nn
->drc_hashtbl
[i
].lru_head
);
196 spin_lock_init(&nn
->drc_hashtbl
[i
].cache_lock
);
198 nn
->drc_hashsize
= hashsize
;
202 unregister_shrinker(&nn
->nfsd_reply_cache_shrinker
);
204 nfsd_reply_cache_stats_destroy(nn
);
206 printk(KERN_ERR
"nfsd: failed to allocate reply cache\n");
210 void nfsd_reply_cache_shutdown(struct nfsd_net
*nn
)
212 struct svc_cacherep
*rp
;
215 nfsd_reply_cache_stats_destroy(nn
);
216 unregister_shrinker(&nn
->nfsd_reply_cache_shrinker
);
218 for (i
= 0; i
< nn
->drc_hashsize
; i
++) {
219 struct list_head
*head
= &nn
->drc_hashtbl
[i
].lru_head
;
220 while (!list_empty(head
)) {
221 rp
= list_first_entry(head
, struct svc_cacherep
, c_lru
);
222 nfsd_reply_cache_free_locked(&nn
->drc_hashtbl
[i
],
227 kvfree(nn
->drc_hashtbl
);
228 nn
->drc_hashtbl
= NULL
;
229 nn
->drc_hashsize
= 0;
234 * Move cache entry to end of LRU list, and queue the cleaner to run if it's
235 * not already scheduled.
238 lru_put_end(struct nfsd_drc_bucket
*b
, struct svc_cacherep
*rp
)
240 rp
->c_timestamp
= jiffies
;
241 list_move_tail(&rp
->c_lru
, &b
->lru_head
);
245 prune_bucket(struct nfsd_drc_bucket
*b
, struct nfsd_net
*nn
)
247 struct svc_cacherep
*rp
, *tmp
;
250 list_for_each_entry_safe(rp
, tmp
, &b
->lru_head
, c_lru
) {
252 * Don't free entries attached to calls that are still
253 * in-progress, but do keep scanning the list.
255 if (rp
->c_state
== RC_INPROG
)
257 if (atomic_read(&nn
->num_drc_entries
) <= nn
->max_drc_entries
&&
258 time_before(jiffies
, rp
->c_timestamp
+ RC_EXPIRE
))
260 nfsd_reply_cache_free_locked(b
, rp
, nn
);
267 * Walk the LRU list and prune off entries that are older than RC_EXPIRE.
268 * Also prune the oldest ones when the total exceeds the max number of entries.
271 prune_cache_entries(struct nfsd_net
*nn
)
276 for (i
= 0; i
< nn
->drc_hashsize
; i
++) {
277 struct nfsd_drc_bucket
*b
= &nn
->drc_hashtbl
[i
];
279 if (list_empty(&b
->lru_head
))
281 spin_lock(&b
->cache_lock
);
282 freed
+= prune_bucket(b
, nn
);
283 spin_unlock(&b
->cache_lock
);
289 nfsd_reply_cache_count(struct shrinker
*shrink
, struct shrink_control
*sc
)
291 struct nfsd_net
*nn
= container_of(shrink
,
292 struct nfsd_net
, nfsd_reply_cache_shrinker
);
294 return atomic_read(&nn
->num_drc_entries
);
298 nfsd_reply_cache_scan(struct shrinker
*shrink
, struct shrink_control
*sc
)
300 struct nfsd_net
*nn
= container_of(shrink
,
301 struct nfsd_net
, nfsd_reply_cache_shrinker
);
303 return prune_cache_entries(nn
);
306 * Walk an xdr_buf and get a CRC for at most the first RC_CSUMLEN bytes
309 nfsd_cache_csum(struct svc_rqst
*rqstp
)
314 struct xdr_buf
*buf
= &rqstp
->rq_arg
;
315 const unsigned char *p
= buf
->head
[0].iov_base
;
316 size_t csum_len
= min_t(size_t, buf
->head
[0].iov_len
+ buf
->page_len
,
318 size_t len
= min(buf
->head
[0].iov_len
, csum_len
);
320 /* rq_arg.head first */
321 csum
= csum_partial(p
, len
, 0);
324 /* Continue into page array */
325 idx
= buf
->page_base
/ PAGE_SIZE
;
326 base
= buf
->page_base
& ~PAGE_MASK
;
328 p
= page_address(buf
->pages
[idx
]) + base
;
329 len
= min_t(size_t, PAGE_SIZE
- base
, csum_len
);
330 csum
= csum_partial(p
, len
, csum
);
339 nfsd_cache_key_cmp(const struct svc_cacherep
*key
,
340 const struct svc_cacherep
*rp
, struct nfsd_net
*nn
)
342 if (key
->c_key
.k_xid
== rp
->c_key
.k_xid
&&
343 key
->c_key
.k_csum
!= rp
->c_key
.k_csum
) {
344 nfsd_stats_payload_misses_inc(nn
);
345 trace_nfsd_drc_mismatch(nn
, key
, rp
);
348 return memcmp(&key
->c_key
, &rp
->c_key
, sizeof(key
->c_key
));
352 * Search the request hash for an entry that matches the given rqstp.
353 * Must be called with cache_lock held. Returns the found entry or
354 * inserts an empty key on failure.
356 static struct svc_cacherep
*
357 nfsd_cache_insert(struct nfsd_drc_bucket
*b
, struct svc_cacherep
*key
,
360 struct svc_cacherep
*rp
, *ret
= key
;
361 struct rb_node
**p
= &b
->rb_head
.rb_node
,
363 unsigned int entries
= 0;
369 rp
= rb_entry(parent
, struct svc_cacherep
, c_node
);
371 cmp
= nfsd_cache_key_cmp(key
, rp
, nn
);
373 p
= &parent
->rb_left
;
375 p
= &parent
->rb_right
;
381 rb_link_node(&key
->c_node
, parent
, p
);
382 rb_insert_color(&key
->c_node
, &b
->rb_head
);
384 /* tally hash chain length stats */
385 if (entries
> nn
->longest_chain
) {
386 nn
->longest_chain
= entries
;
387 nn
->longest_chain_cachesize
= atomic_read(&nn
->num_drc_entries
);
388 } else if (entries
== nn
->longest_chain
) {
389 /* prefer to keep the smallest cachesize possible here */
390 nn
->longest_chain_cachesize
= min_t(unsigned int,
391 nn
->longest_chain_cachesize
,
392 atomic_read(&nn
->num_drc_entries
));
400 * nfsd_cache_lookup - Find an entry in the duplicate reply cache
401 * @rqstp: Incoming Call to find
403 * Try to find an entry matching the current call in the cache. When none
404 * is found, we try to grab the oldest expired entry off the LRU list. If
405 * a suitable one isn't there, then drop the cache_lock and allocate a
406 * new one, then search again in case one got inserted while this thread
407 * didn't hold the lock.
410 * %RC_DOIT: Process the request normally
411 * %RC_REPLY: Reply from cache
412 * %RC_DROPIT: Do not process the request further
414 int nfsd_cache_lookup(struct svc_rqst
*rqstp
)
416 struct nfsd_net
*nn
= net_generic(SVC_NET(rqstp
), nfsd_net_id
);
417 struct svc_cacherep
*rp
, *found
;
418 __be32 xid
= rqstp
->rq_xid
;
420 u32 hash
= nfsd_cache_hash(xid
, nn
);
421 struct nfsd_drc_bucket
*b
= &nn
->drc_hashtbl
[hash
];
422 int type
= rqstp
->rq_cachetype
;
425 rqstp
->rq_cacherep
= NULL
;
426 if (type
== RC_NOCACHE
) {
427 nfsd_stats_rc_nocache_inc();
431 csum
= nfsd_cache_csum(rqstp
);
434 * Since the common case is a cache miss followed by an insert,
435 * preallocate an entry.
437 rp
= nfsd_reply_cache_alloc(rqstp
, csum
, nn
);
441 spin_lock(&b
->cache_lock
);
442 found
= nfsd_cache_insert(b
, rp
, nn
);
444 nfsd_reply_cache_free_locked(NULL
, rp
, nn
);
449 nfsd_stats_rc_misses_inc();
450 rqstp
->rq_cacherep
= rp
;
451 rp
->c_state
= RC_INPROG
;
453 atomic_inc(&nn
->num_drc_entries
);
454 nfsd_stats_drc_mem_usage_add(nn
, sizeof(*rp
));
456 /* go ahead and prune the cache */
460 spin_unlock(&b
->cache_lock
);
465 /* We found a matching entry which is either in progress or done. */
466 nfsd_stats_rc_hits_inc();
469 /* Request being processed */
470 if (rp
->c_state
== RC_INPROG
)
473 /* From the hall of fame of impractical attacks:
474 * Is this a user who tries to snoop on the cache? */
476 if (!test_bit(RQ_SECURE
, &rqstp
->rq_flags
) && rp
->c_secure
)
479 /* Compose RPC reply header */
480 switch (rp
->c_type
) {
484 svc_putu32(&rqstp
->rq_res
.head
[0], rp
->c_replstat
);
488 if (!nfsd_cache_append(rqstp
, &rp
->c_replvec
))
489 goto out_unlock
; /* should not happen */
493 WARN_ONCE(1, "nfsd: bad repcache type %d\n", rp
->c_type
);
497 trace_nfsd_drc_found(nn
, rqstp
, rtn
);
502 * nfsd_cache_update - Update an entry in the duplicate reply cache.
503 * @rqstp: svc_rqst with a finished Reply
504 * @cachetype: which cache to update
505 * @statp: Reply's status code
507 * This is called from nfsd_dispatch when the procedure has been
508 * executed and the complete reply is in rqstp->rq_res.
510 * We're copying around data here rather than swapping buffers because
511 * the toplevel loop requires max-sized buffers, which would be a waste
512 * of memory for a cache with a max reply size of 100 bytes (diropokres).
514 * If we should start to use different types of cache entries tailored
515 * specifically for attrstat and fh's, we may save even more space.
517 * Also note that a cachetype of RC_NOCACHE can legally be passed when
518 * nfsd failed to encode a reply that otherwise would have been cached.
519 * In this case, nfsd_cache_update is called with statp == NULL.
521 void nfsd_cache_update(struct svc_rqst
*rqstp
, int cachetype
, __be32
*statp
)
523 struct nfsd_net
*nn
= net_generic(SVC_NET(rqstp
), nfsd_net_id
);
524 struct svc_cacherep
*rp
= rqstp
->rq_cacherep
;
525 struct kvec
*resv
= &rqstp
->rq_res
.head
[0], *cachv
;
527 struct nfsd_drc_bucket
*b
;
534 hash
= nfsd_cache_hash(rp
->c_key
.k_xid
, nn
);
535 b
= &nn
->drc_hashtbl
[hash
];
537 len
= resv
->iov_len
- ((char*)statp
- (char*)resv
->iov_base
);
540 /* Don't cache excessive amounts of data and XDR failures */
541 if (!statp
|| len
> (256 >> 2)) {
542 nfsd_reply_cache_free(b
, rp
, nn
);
549 printk("nfsd: RC_REPLSTAT/reply len %d!\n",len
);
550 rp
->c_replstat
= *statp
;
553 cachv
= &rp
->c_replvec
;
555 cachv
->iov_base
= kmalloc(bufsize
, GFP_KERNEL
);
556 if (!cachv
->iov_base
) {
557 nfsd_reply_cache_free(b
, rp
, nn
);
560 cachv
->iov_len
= bufsize
;
561 memcpy(cachv
->iov_base
, statp
, bufsize
);
564 nfsd_reply_cache_free(b
, rp
, nn
);
567 spin_lock(&b
->cache_lock
);
568 nfsd_stats_drc_mem_usage_add(nn
, bufsize
);
570 rp
->c_secure
= test_bit(RQ_SECURE
, &rqstp
->rq_flags
);
571 rp
->c_type
= cachetype
;
572 rp
->c_state
= RC_DONE
;
573 spin_unlock(&b
->cache_lock
);
578 * Copy cached reply to current reply buffer. Should always fit.
579 * FIXME as reply is in a page, we should just attach the page, and
580 * keep a refcount....
583 nfsd_cache_append(struct svc_rqst
*rqstp
, struct kvec
*data
)
585 struct kvec
*vec
= &rqstp
->rq_res
.head
[0];
587 if (vec
->iov_len
+ data
->iov_len
> PAGE_SIZE
) {
588 printk(KERN_WARNING
"nfsd: cached reply too large (%zd).\n",
592 memcpy((char*)vec
->iov_base
+ vec
->iov_len
, data
->iov_base
, data
->iov_len
);
593 vec
->iov_len
+= data
->iov_len
;
598 * Note that fields may be added, removed or reordered in the future. Programs
599 * scraping this file for info should test the labels to ensure they're
600 * getting the correct field.
602 static int nfsd_reply_cache_stats_show(struct seq_file
*m
, void *v
)
604 struct nfsd_net
*nn
= m
->private;
606 seq_printf(m
, "max entries: %u\n", nn
->max_drc_entries
);
607 seq_printf(m
, "num entries: %u\n",
608 atomic_read(&nn
->num_drc_entries
));
609 seq_printf(m
, "hash buckets: %u\n", 1 << nn
->maskbits
);
610 seq_printf(m
, "mem usage: %lld\n",
611 percpu_counter_sum_positive(&nn
->counter
[NFSD_NET_DRC_MEM_USAGE
]));
612 seq_printf(m
, "cache hits: %lld\n",
613 percpu_counter_sum_positive(&nfsdstats
.counter
[NFSD_STATS_RC_HITS
]));
614 seq_printf(m
, "cache misses: %lld\n",
615 percpu_counter_sum_positive(&nfsdstats
.counter
[NFSD_STATS_RC_MISSES
]));
616 seq_printf(m
, "not cached: %lld\n",
617 percpu_counter_sum_positive(&nfsdstats
.counter
[NFSD_STATS_RC_NOCACHE
]));
618 seq_printf(m
, "payload misses: %lld\n",
619 percpu_counter_sum_positive(&nn
->counter
[NFSD_NET_PAYLOAD_MISSES
]));
620 seq_printf(m
, "longest chain len: %u\n", nn
->longest_chain
);
621 seq_printf(m
, "cachesize at longest: %u\n", nn
->longest_chain_cachesize
);
625 int nfsd_reply_cache_stats_open(struct inode
*inode
, struct file
*file
)
627 struct nfsd_net
*nn
= net_generic(file_inode(file
)->i_sb
->s_fs_info
,
630 return single_open(file
, nfsd_reply_cache_stats_show
, nn
);