]>
git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - fs/erofs/pcpubuf.c
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) Gao Xiang <xiang@kernel.org>
5 * For low-latency decompression algorithms (e.g. lz4), reserve consecutive
6 * per-CPU virtual memory (in pages) in advance to store such inplace I/O
7 * data if inplace decompression is failed (due to unmet inplace margin for
12 struct erofs_pcpubuf
{
19 static DEFINE_PER_CPU(struct erofs_pcpubuf
, erofs_pcb
);
21 void *erofs_get_pcpubuf(unsigned int requiredpages
)
24 struct erofs_pcpubuf
*pcb
= &get_cpu_var(erofs_pcb
);
26 raw_spin_lock(&pcb
->lock
);
27 /* check if the per-CPU buffer is too small */
28 if (requiredpages
> pcb
->nrpages
) {
29 raw_spin_unlock(&pcb
->lock
);
30 put_cpu_var(erofs_pcb
);
31 /* (for sparse checker) pretend pcb->lock is still taken */
38 void erofs_put_pcpubuf(void *ptr
) __releases(pcb
->lock
)
40 struct erofs_pcpubuf
*pcb
= &per_cpu(erofs_pcb
, smp_processor_id());
42 DBG_BUGON(pcb
->ptr
!= ptr
);
43 raw_spin_unlock(&pcb
->lock
);
44 put_cpu_var(erofs_pcb
);
47 /* the next step: support per-CPU page buffers hotplug */
48 int erofs_pcpubuf_growsize(unsigned int nrpages
)
50 static DEFINE_MUTEX(pcb_resize_mutex
);
51 static unsigned int pcb_nrpages
;
53 int delta
, cpu
, ret
, i
;
55 mutex_lock(&pcb_resize_mutex
);
56 delta
= nrpages
- pcb_nrpages
;
58 /* avoid shrinking pcpubuf, since no idea how many fses rely on */
62 for_each_possible_cpu(cpu
) {
63 struct erofs_pcpubuf
*pcb
= &per_cpu(erofs_pcb
, cpu
);
64 struct page
**pages
, **oldpages
;
67 pages
= kmalloc_array(nrpages
, sizeof(*pages
), GFP_KERNEL
);
73 for (i
= 0; i
< nrpages
; ++i
) {
74 pages
[i
] = erofs_allocpage(&pagepool
, GFP_KERNEL
);
81 ptr
= vmap(pages
, nrpages
, VM_MAP
, PAGE_KERNEL
);
87 raw_spin_lock(&pcb
->lock
);
90 oldpages
= pcb
->pages
;
93 pcb
->nrpages
= nrpages
;
94 raw_spin_unlock(&pcb
->lock
);
105 list_add(&oldpages
[--i
]->lru
, &pagepool
);
110 pcb_nrpages
= nrpages
;
111 put_pages_list(&pagepool
);
113 mutex_unlock(&pcb_resize_mutex
);
117 void erofs_pcpubuf_init(void)
121 for_each_possible_cpu(cpu
) {
122 struct erofs_pcpubuf
*pcb
= &per_cpu(erofs_pcb
, cpu
);
124 raw_spin_lock_init(&pcb
->lock
);
128 void erofs_pcpubuf_exit(void)
132 for_each_possible_cpu(cpu
) {
133 struct erofs_pcpubuf
*pcb
= &per_cpu(erofs_pcb
, cpu
);
142 for (i
= 0; i
< pcb
->nrpages
; ++i
)
144 put_page(pcb
->pages
[i
]);