1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright 2014 IBM Corp.
7 #include <linux/slab.h>
8 #include <linux/file.h>
10 #include <linux/module.h>
11 #include <linux/mount.h>
12 #include <linux/sched/mm.h>
13 #include <linux/mmu_context.h>
18 * Since we want to track memory mappings to be able to force-unmap
19 * when the AFU is no longer reachable, we need an inode. For devices
20 * opened through the cxl user API, this is not a problem, but a
21 * userland process can also get a cxl fd through the cxl_get_fd()
22 * API, which is used by the cxlflash driver.
24 * Therefore we implement our own simple pseudo-filesystem and inode
25 * allocator. We don't use the anonymous inode, as we need the
26 * meta-data associated with it (address_space) and it is shared by
27 * other drivers/processes, so it could lead to cxl unmapping VMAs
28 * from random processes.
31 #define CXL_PSEUDO_FS_MAGIC 0x1697697f
33 static int cxl_fs_cnt
;
34 static struct vfsmount
*cxl_vfs_mount
;
36 static const struct dentry_operations cxl_fs_dops
= {
37 .d_dname
= simple_dname
,
40 static struct dentry
*cxl_fs_mount(struct file_system_type
*fs_type
, int flags
,
41 const char *dev_name
, void *data
)
43 return mount_pseudo(fs_type
, "cxl:", NULL
, &cxl_fs_dops
,
47 static struct file_system_type cxl_fs_type
= {
50 .mount
= cxl_fs_mount
,
51 .kill_sb
= kill_anon_super
,
55 void cxl_release_mapping(struct cxl_context
*ctx
)
57 if (ctx
->kernelapi
&& ctx
->mapping
)
58 simple_release_fs(&cxl_vfs_mount
, &cxl_fs_cnt
);
61 static struct file
*cxl_getfile(const char *name
,
62 const struct file_operations
*fops
,
63 void *priv
, int flags
)
69 /* strongly inspired by anon_inode_getfile() */
71 if (fops
->owner
&& !try_module_get(fops
->owner
))
72 return ERR_PTR(-ENOENT
);
74 rc
= simple_pin_fs(&cxl_fs_type
, &cxl_vfs_mount
, &cxl_fs_cnt
);
76 pr_err("Cannot mount cxl pseudo filesystem: %d\n", rc
);
81 inode
= alloc_anon_inode(cxl_vfs_mount
->mnt_sb
);
83 file
= ERR_CAST(inode
);
87 file
= alloc_file_pseudo(inode
, cxl_vfs_mount
, name
,
88 flags
& (O_ACCMODE
| O_NONBLOCK
), fops
);
92 file
->private_data
= priv
;
99 simple_release_fs(&cxl_vfs_mount
, &cxl_fs_cnt
);
101 module_put(fops
->owner
);
105 struct cxl_context
*cxl_dev_context_init(struct pci_dev
*dev
)
108 struct cxl_context
*ctx
;
111 afu
= cxl_pci_to_afu(dev
);
113 return ERR_CAST(afu
);
115 ctx
= cxl_context_alloc();
117 return ERR_PTR(-ENOMEM
);
119 ctx
->kernelapi
= true;
121 /* Make it a slave context. We can promote it later? */
122 rc
= cxl_context_init(ctx
, afu
, false);
132 EXPORT_SYMBOL_GPL(cxl_dev_context_init
);
134 struct cxl_context
*cxl_get_context(struct pci_dev
*dev
)
136 return dev
->dev
.archdata
.cxl_ctx
;
138 EXPORT_SYMBOL_GPL(cxl_get_context
);
140 int cxl_release_context(struct cxl_context
*ctx
)
142 if (ctx
->status
>= STARTED
)
145 cxl_context_free(ctx
);
149 EXPORT_SYMBOL_GPL(cxl_release_context
);
151 static irq_hw_number_t
cxl_find_afu_irq(struct cxl_context
*ctx
, int num
)
156 for (r
= 0; r
< CXL_IRQ_RANGES
; r
++) {
157 range
= ctx
->irqs
.range
[r
];
159 return ctx
->irqs
.offset
[r
] + num
;
167 int cxl_set_priv(struct cxl_context
*ctx
, void *priv
)
176 EXPORT_SYMBOL_GPL(cxl_set_priv
);
178 void *cxl_get_priv(struct cxl_context
*ctx
)
181 return ERR_PTR(-EINVAL
);
185 EXPORT_SYMBOL_GPL(cxl_get_priv
);
187 int cxl_allocate_afu_irqs(struct cxl_context
*ctx
, int num
)
190 irq_hw_number_t hwirq
;
193 num
= ctx
->afu
->pp_irqs
;
194 res
= afu_allocate_irqs(ctx
, num
);
198 if (!cpu_has_feature(CPU_FTR_HVMODE
)) {
199 /* In a guest, the PSL interrupt is not multiplexed. It was
200 * allocated above, and we need to set its handler
202 hwirq
= cxl_find_afu_irq(ctx
, 0);
204 cxl_map_irq(ctx
->afu
->adapter
, hwirq
, cxl_ops
->psl_interrupt
, ctx
, "psl");
207 if (ctx
->status
== STARTED
) {
208 if (cxl_ops
->update_ivtes
)
209 cxl_ops
->update_ivtes(ctx
);
210 else WARN(1, "BUG: cxl_allocate_afu_irqs must be called prior to starting the context on this platform\n");
215 EXPORT_SYMBOL_GPL(cxl_allocate_afu_irqs
);
217 void cxl_free_afu_irqs(struct cxl_context
*ctx
)
219 irq_hw_number_t hwirq
;
222 if (!cpu_has_feature(CPU_FTR_HVMODE
)) {
223 hwirq
= cxl_find_afu_irq(ctx
, 0);
225 virq
= irq_find_mapping(NULL
, hwirq
);
227 cxl_unmap_irq(virq
, ctx
);
230 afu_irq_name_free(ctx
);
231 cxl_ops
->release_irq_ranges(&ctx
->irqs
, ctx
->afu
->adapter
);
233 EXPORT_SYMBOL_GPL(cxl_free_afu_irqs
);
235 int cxl_map_afu_irq(struct cxl_context
*ctx
, int num
,
236 irq_handler_t handler
, void *cookie
, char *name
)
238 irq_hw_number_t hwirq
;
241 * Find interrupt we are to register.
243 hwirq
= cxl_find_afu_irq(ctx
, num
);
247 return cxl_map_irq(ctx
->afu
->adapter
, hwirq
, handler
, cookie
, name
);
249 EXPORT_SYMBOL_GPL(cxl_map_afu_irq
);
251 void cxl_unmap_afu_irq(struct cxl_context
*ctx
, int num
, void *cookie
)
253 irq_hw_number_t hwirq
;
256 hwirq
= cxl_find_afu_irq(ctx
, num
);
260 virq
= irq_find_mapping(NULL
, hwirq
);
262 cxl_unmap_irq(virq
, cookie
);
264 EXPORT_SYMBOL_GPL(cxl_unmap_afu_irq
);
268 * Code here similar to afu_ioctl_start_work().
270 int cxl_start_context(struct cxl_context
*ctx
, u64 wed
,
271 struct task_struct
*task
)
276 pr_devel("%s: pe: %i\n", __func__
, ctx
->pe
);
278 mutex_lock(&ctx
->status_mutex
);
279 if (ctx
->status
== STARTED
)
280 goto out
; /* already started */
283 * Increment the mapped context count for adapter. This also checks
284 * if adapter_context_lock is taken.
286 rc
= cxl_adapter_context_get(ctx
->afu
->adapter
);
291 ctx
->pid
= get_task_pid(task
, PIDTYPE_PID
);
294 /* acquire a reference to the task's mm */
295 ctx
->mm
= get_task_mm(current
);
297 /* ensure this mm_struct can't be freed */
298 cxl_context_mm_count_get(ctx
);
301 /* decrement the use count from above */
303 /* make TLBIs for this context global */
304 mm_context_add_copro(ctx
->mm
);
309 * Increment driver use count. Enables global TLBIs for hash
310 * and callbacks to handle the segment table
314 /* See the comment in afu_ioctl_start_work() */
317 if ((rc
= cxl_ops
->attach_process(ctx
, kernel
, wed
, 0))) {
320 cxl_adapter_context_put(ctx
->afu
->adapter
);
323 cxl_context_mm_count_put(ctx
);
325 mm_context_remove_copro(ctx
->mm
);
330 ctx
->status
= STARTED
;
332 mutex_unlock(&ctx
->status_mutex
);
335 EXPORT_SYMBOL_GPL(cxl_start_context
);
337 int cxl_process_element(struct cxl_context
*ctx
)
339 return ctx
->external_pe
;
341 EXPORT_SYMBOL_GPL(cxl_process_element
);
343 /* Stop a context. Returns 0 on success, otherwise -Errno */
344 int cxl_stop_context(struct cxl_context
*ctx
)
346 return __detach_context(ctx
);
348 EXPORT_SYMBOL_GPL(cxl_stop_context
);
350 void cxl_set_master(struct cxl_context
*ctx
)
354 EXPORT_SYMBOL_GPL(cxl_set_master
);
356 /* wrappers around afu_* file ops which are EXPORTED */
357 int cxl_fd_open(struct inode
*inode
, struct file
*file
)
359 return afu_open(inode
, file
);
361 EXPORT_SYMBOL_GPL(cxl_fd_open
);
362 int cxl_fd_release(struct inode
*inode
, struct file
*file
)
364 return afu_release(inode
, file
);
366 EXPORT_SYMBOL_GPL(cxl_fd_release
);
367 long cxl_fd_ioctl(struct file
*file
, unsigned int cmd
, unsigned long arg
)
369 return afu_ioctl(file
, cmd
, arg
);
371 EXPORT_SYMBOL_GPL(cxl_fd_ioctl
);
372 int cxl_fd_mmap(struct file
*file
, struct vm_area_struct
*vm
)
374 return afu_mmap(file
, vm
);
376 EXPORT_SYMBOL_GPL(cxl_fd_mmap
);
377 __poll_t
cxl_fd_poll(struct file
*file
, struct poll_table_struct
*poll
)
379 return afu_poll(file
, poll
);
381 EXPORT_SYMBOL_GPL(cxl_fd_poll
);
382 ssize_t
cxl_fd_read(struct file
*file
, char __user
*buf
, size_t count
,
385 return afu_read(file
, buf
, count
, off
);
387 EXPORT_SYMBOL_GPL(cxl_fd_read
);
389 #define PATCH_FOPS(NAME) if (!fops->NAME) fops->NAME = afu_fops.NAME
391 /* Get a struct file and fd for a context and attach the ops */
392 struct file
*cxl_get_fd(struct cxl_context
*ctx
, struct file_operations
*fops
,
396 int rc
, flags
, fdtmp
;
399 /* only allow one per context */
401 return ERR_PTR(-EEXIST
);
403 flags
= O_RDWR
| O_CLOEXEC
;
405 /* This code is similar to anon_inode_getfd() */
406 rc
= get_unused_fd_flags(flags
);
412 * Patch the file ops. Needs to be careful that this is rentrant safe.
419 PATCH_FOPS(unlocked_ioctl
);
420 PATCH_FOPS(compat_ioctl
);
422 } else /* use default ops */
423 fops
= (struct file_operations
*)&afu_fops
;
425 name
= kasprintf(GFP_KERNEL
, "cxl:%d", ctx
->pe
);
426 file
= cxl_getfile(name
, fops
, ctx
, flags
);
431 cxl_context_set_mapping(ctx
, file
->f_mapping
);
436 put_unused_fd(fdtmp
);
439 EXPORT_SYMBOL_GPL(cxl_get_fd
);
441 struct cxl_context
*cxl_fops_get_context(struct file
*file
)
443 return file
->private_data
;
445 EXPORT_SYMBOL_GPL(cxl_fops_get_context
);
447 void cxl_set_driver_ops(struct cxl_context
*ctx
,
448 struct cxl_afu_driver_ops
*ops
)
450 WARN_ON(!ops
->fetch_event
|| !ops
->event_delivered
);
451 atomic_set(&ctx
->afu_driver_events
, 0);
452 ctx
->afu_driver_ops
= ops
;
454 EXPORT_SYMBOL_GPL(cxl_set_driver_ops
);
456 void cxl_context_events_pending(struct cxl_context
*ctx
,
457 unsigned int new_events
)
459 atomic_add(new_events
, &ctx
->afu_driver_events
);
460 wake_up_all(&ctx
->wq
);
462 EXPORT_SYMBOL_GPL(cxl_context_events_pending
);
464 int cxl_start_work(struct cxl_context
*ctx
,
465 struct cxl_ioctl_start_work
*work
)
469 /* code taken from afu_ioctl_start_work */
470 if (!(work
->flags
& CXL_START_WORK_NUM_IRQS
))
471 work
->num_interrupts
= ctx
->afu
->pp_irqs
;
472 else if ((work
->num_interrupts
< ctx
->afu
->pp_irqs
) ||
473 (work
->num_interrupts
> ctx
->afu
->irqs_max
)) {
477 rc
= afu_register_irqs(ctx
, work
->num_interrupts
);
481 rc
= cxl_start_context(ctx
, work
->work_element_descriptor
, current
);
483 afu_release_irqs(ctx
, ctx
);
489 EXPORT_SYMBOL_GPL(cxl_start_work
);
491 void __iomem
*cxl_psa_map(struct cxl_context
*ctx
)
493 if (ctx
->status
!= STARTED
)
496 pr_devel("%s: psn_phys%llx size:%llx\n",
497 __func__
, ctx
->psn_phys
, ctx
->psn_size
);
498 return ioremap(ctx
->psn_phys
, ctx
->psn_size
);
500 EXPORT_SYMBOL_GPL(cxl_psa_map
);
502 void cxl_psa_unmap(void __iomem
*addr
)
506 EXPORT_SYMBOL_GPL(cxl_psa_unmap
);
508 int cxl_afu_reset(struct cxl_context
*ctx
)
510 struct cxl_afu
*afu
= ctx
->afu
;
513 rc
= cxl_ops
->afu_reset(afu
);
517 return cxl_ops
->afu_check_and_enable(afu
);
519 EXPORT_SYMBOL_GPL(cxl_afu_reset
);
521 void cxl_perst_reloads_same_image(struct cxl_afu
*afu
,
522 bool perst_reloads_same_image
)
524 afu
->adapter
->perst_same_image
= perst_reloads_same_image
;
526 EXPORT_SYMBOL_GPL(cxl_perst_reloads_same_image
);
528 ssize_t
cxl_read_adapter_vpd(struct pci_dev
*dev
, void *buf
, size_t count
)
530 struct cxl_afu
*afu
= cxl_pci_to_afu(dev
);
534 return cxl_ops
->read_adapter_vpd(afu
->adapter
, buf
, count
);
536 EXPORT_SYMBOL_GPL(cxl_read_adapter_vpd
);