1 // SPDX-License-Identifier: GPL-2.0+
5 * COMEDI - Linux Control and Measurement Device Interface
6 * Copyright (C) 1997-2000 David A. Schleef <ds@schleef.org>
7 * Copyright (C) 2002 Frank Mori Hess <fmhess@users.sourceforge.net>
10 #include <linux/vmalloc.h>
11 #include <linux/slab.h>
13 #include "comedidev.h"
14 #include "comedi_internal.h"
16 #ifdef PAGE_KERNEL_NOCACHE
17 #define COMEDI_PAGE_PROTECTION PAGE_KERNEL_NOCACHE
19 #define COMEDI_PAGE_PROTECTION PAGE_KERNEL
22 static void comedi_buf_map_kref_release(struct kref
*kref
)
24 struct comedi_buf_map
*bm
=
25 container_of(kref
, struct comedi_buf_map
, refcount
);
26 struct comedi_buf_page
*buf
;
30 if (bm
->dma_dir
!= DMA_NONE
) {
32 * DMA buffer was allocated as a single block.
33 * Address is in page_list[0].
35 buf
= &bm
->page_list
[0];
36 dma_free_coherent(bm
->dma_hw_dev
,
37 PAGE_SIZE
* bm
->n_pages
,
38 buf
->virt_addr
, buf
->dma_addr
);
40 for (i
= 0; i
< bm
->n_pages
; i
++) {
41 buf
= &bm
->page_list
[i
];
42 ClearPageReserved(virt_to_page(buf
->virt_addr
));
43 free_page((unsigned long)buf
->virt_addr
);
48 if (bm
->dma_dir
!= DMA_NONE
)
49 put_device(bm
->dma_hw_dev
);
53 static void __comedi_buf_free(struct comedi_device
*dev
,
54 struct comedi_subdevice
*s
)
56 struct comedi_async
*async
= s
->async
;
57 struct comedi_buf_map
*bm
;
60 if (async
->prealloc_buf
) {
61 if (s
->async_dma_dir
== DMA_NONE
)
62 vunmap(async
->prealloc_buf
);
63 async
->prealloc_buf
= NULL
;
64 async
->prealloc_bufsz
= 0;
67 spin_lock_irqsave(&s
->spin_lock
, flags
);
69 async
->buf_map
= NULL
;
70 spin_unlock_irqrestore(&s
->spin_lock
, flags
);
71 comedi_buf_map_put(bm
);
74 static struct comedi_buf_map
*
75 comedi_buf_map_alloc(struct comedi_device
*dev
, enum dma_data_direction dma_dir
,
78 struct comedi_buf_map
*bm
;
79 struct comedi_buf_page
*buf
;
82 bm
= kzalloc(sizeof(*bm
), GFP_KERNEL
);
86 kref_init(&bm
->refcount
);
87 bm
->dma_dir
= dma_dir
;
88 if (bm
->dma_dir
!= DMA_NONE
) {
89 /* Need ref to hardware device to free buffer later. */
90 bm
->dma_hw_dev
= get_device(dev
->hw_dev
);
93 bm
->page_list
= vzalloc(sizeof(*buf
) * n_pages
);
97 if (bm
->dma_dir
!= DMA_NONE
) {
102 * Currently, the DMA buffer needs to be allocated as a
103 * single block so that it can be mmap()'ed.
105 virt_addr
= dma_alloc_coherent(bm
->dma_hw_dev
,
106 PAGE_SIZE
* n_pages
, &dma_addr
,
111 for (i
= 0; i
< n_pages
; i
++) {
112 buf
= &bm
->page_list
[i
];
113 buf
->virt_addr
= virt_addr
+ (i
<< PAGE_SHIFT
);
114 buf
->dma_addr
= dma_addr
+ (i
<< PAGE_SHIFT
);
119 for (i
= 0; i
< n_pages
; i
++) {
120 buf
= &bm
->page_list
[i
];
121 buf
->virt_addr
= (void *)get_zeroed_page(GFP_KERNEL
);
125 SetPageReserved(virt_to_page(buf
->virt_addr
));
136 comedi_buf_map_put(bm
);
140 static void __comedi_buf_alloc(struct comedi_device
*dev
,
141 struct comedi_subdevice
*s
,
142 unsigned int n_pages
)
144 struct comedi_async
*async
= s
->async
;
145 struct page
**pages
= NULL
;
146 struct comedi_buf_map
*bm
;
147 struct comedi_buf_page
*buf
;
151 if (!IS_ENABLED(CONFIG_HAS_DMA
) && s
->async_dma_dir
!= DMA_NONE
) {
152 dev_err(dev
->class_dev
,
153 "dma buffer allocation not supported\n");
157 bm
= comedi_buf_map_alloc(dev
, s
->async_dma_dir
, n_pages
);
161 spin_lock_irqsave(&s
->spin_lock
, flags
);
163 spin_unlock_irqrestore(&s
->spin_lock
, flags
);
165 if (bm
->dma_dir
!= DMA_NONE
) {
167 * DMA buffer was allocated as a single block.
168 * Address is in page_list[0].
170 buf
= &bm
->page_list
[0];
171 async
->prealloc_buf
= buf
->virt_addr
;
173 pages
= vmalloc(sizeof(struct page
*) * n_pages
);
177 for (i
= 0; i
< n_pages
; i
++) {
178 buf
= &bm
->page_list
[i
];
179 pages
[i
] = virt_to_page(buf
->virt_addr
);
182 /* vmap the pages to prealloc_buf */
183 async
->prealloc_buf
= vmap(pages
, n_pages
, VM_MAP
,
184 COMEDI_PAGE_PROTECTION
);
190 void comedi_buf_map_get(struct comedi_buf_map
*bm
)
193 kref_get(&bm
->refcount
);
196 int comedi_buf_map_put(struct comedi_buf_map
*bm
)
199 return kref_put(&bm
->refcount
, comedi_buf_map_kref_release
);
203 /* helper for "access" vm operation */
204 int comedi_buf_map_access(struct comedi_buf_map
*bm
, unsigned long offset
,
205 void *buf
, int len
, int write
)
207 unsigned int pgoff
= offset_in_page(offset
);
208 unsigned long pg
= offset
>> PAGE_SHIFT
;
211 while (done
< len
&& pg
< bm
->n_pages
) {
212 int l
= min_t(int, len
- done
, PAGE_SIZE
- pgoff
);
213 void *b
= bm
->page_list
[pg
].virt_addr
+ pgoff
;
227 /* returns s->async->buf_map and increments its kref refcount */
228 struct comedi_buf_map
*
229 comedi_buf_map_from_subdev_get(struct comedi_subdevice
*s
)
231 struct comedi_async
*async
= s
->async
;
232 struct comedi_buf_map
*bm
= NULL
;
238 spin_lock_irqsave(&s
->spin_lock
, flags
);
240 /* only want it if buffer pages allocated */
241 if (bm
&& bm
->n_pages
)
242 comedi_buf_map_get(bm
);
245 spin_unlock_irqrestore(&s
->spin_lock
, flags
);
250 bool comedi_buf_is_mmapped(struct comedi_subdevice
*s
)
252 struct comedi_buf_map
*bm
= s
->async
->buf_map
;
254 return bm
&& (kref_read(&bm
->refcount
) > 1);
257 int comedi_buf_alloc(struct comedi_device
*dev
, struct comedi_subdevice
*s
,
258 unsigned long new_size
)
260 struct comedi_async
*async
= s
->async
;
262 lockdep_assert_held(&dev
->mutex
);
264 /* Round up new_size to multiple of PAGE_SIZE */
265 new_size
= (new_size
+ PAGE_SIZE
- 1) & PAGE_MASK
;
267 /* if no change is required, do nothing */
268 if (async
->prealloc_buf
&& async
->prealloc_bufsz
== new_size
)
271 /* deallocate old buffer */
272 __comedi_buf_free(dev
, s
);
274 /* allocate new buffer */
276 unsigned int n_pages
= new_size
>> PAGE_SHIFT
;
278 __comedi_buf_alloc(dev
, s
, n_pages
);
280 if (!async
->prealloc_buf
) {
281 /* allocation failed */
282 __comedi_buf_free(dev
, s
);
286 async
->prealloc_bufsz
= new_size
;
291 void comedi_buf_reset(struct comedi_subdevice
*s
)
293 struct comedi_async
*async
= s
->async
;
295 async
->buf_write_alloc_count
= 0;
296 async
->buf_write_count
= 0;
297 async
->buf_read_alloc_count
= 0;
298 async
->buf_read_count
= 0;
300 async
->buf_write_ptr
= 0;
301 async
->buf_read_ptr
= 0;
304 async
->scans_done
= 0;
305 async
->scan_progress
= 0;
306 async
->munge_chan
= 0;
307 async
->munge_count
= 0;
308 async
->munge_ptr
= 0;
313 static unsigned int comedi_buf_write_n_unalloc(struct comedi_subdevice
*s
)
315 struct comedi_async
*async
= s
->async
;
316 unsigned int free_end
= async
->buf_read_count
+ async
->prealloc_bufsz
;
318 return free_end
- async
->buf_write_alloc_count
;
321 unsigned int comedi_buf_write_n_available(struct comedi_subdevice
*s
)
323 struct comedi_async
*async
= s
->async
;
324 unsigned int free_end
= async
->buf_read_count
+ async
->prealloc_bufsz
;
326 return free_end
- async
->buf_write_count
;
330 * comedi_buf_write_alloc() - Reserve buffer space for writing
331 * @s: COMEDI subdevice.
332 * @nbytes: Maximum space to reserve in bytes.
334 * Reserve up to @nbytes bytes of space to be written in the COMEDI acquisition
335 * data buffer associated with the subdevice. The amount reserved is limited
336 * by the space available.
338 * Return: The amount of space reserved in bytes.
340 unsigned int comedi_buf_write_alloc(struct comedi_subdevice
*s
,
343 struct comedi_async
*async
= s
->async
;
344 unsigned int unalloc
= comedi_buf_write_n_unalloc(s
);
346 if (nbytes
> unalloc
)
349 async
->buf_write_alloc_count
+= nbytes
;
352 * ensure the async buffer 'counts' are read and updated
353 * before we write data to the write-alloc'ed buffer space
359 EXPORT_SYMBOL_GPL(comedi_buf_write_alloc
);
362 * munging is applied to data by core as it passes between user
365 static unsigned int comedi_buf_munge(struct comedi_subdevice
*s
,
366 unsigned int num_bytes
)
368 struct comedi_async
*async
= s
->async
;
369 unsigned int count
= 0;
370 const unsigned int num_sample_bytes
= comedi_bytes_per_sample(s
);
372 if (!s
->munge
|| (async
->cmd
.flags
& CMDF_RAWDATA
)) {
373 async
->munge_count
+= num_bytes
;
377 /* don't munge partial samples */
378 num_bytes
-= num_bytes
% num_sample_bytes
;
379 while (count
< num_bytes
) {
380 int block_size
= num_bytes
- count
;
381 unsigned int buf_end
;
383 buf_end
= async
->prealloc_bufsz
- async
->munge_ptr
;
384 if (block_size
> buf_end
)
385 block_size
= buf_end
;
387 s
->munge(s
->device
, s
,
388 async
->prealloc_buf
+ async
->munge_ptr
,
389 block_size
, async
->munge_chan
);
392 * ensure data is munged in buffer before the
393 * async buffer munge_count is incremented
397 async
->munge_chan
+= block_size
/ num_sample_bytes
;
398 async
->munge_chan
%= async
->cmd
.chanlist_len
;
399 async
->munge_count
+= block_size
;
400 async
->munge_ptr
+= block_size
;
401 async
->munge_ptr
%= async
->prealloc_bufsz
;
408 unsigned int comedi_buf_write_n_allocated(struct comedi_subdevice
*s
)
410 struct comedi_async
*async
= s
->async
;
412 return async
->buf_write_alloc_count
- async
->buf_write_count
;
416 * comedi_buf_write_free() - Free buffer space after it is written
417 * @s: COMEDI subdevice.
418 * @nbytes: Maximum space to free in bytes.
420 * Free up to @nbytes bytes of space previously reserved for writing in the
421 * COMEDI acquisition data buffer associated with the subdevice. The amount of
422 * space freed is limited to the amount that was reserved. The freed space is
423 * assumed to have been filled with sample data by the writer.
425 * If the samples in the freed space need to be "munged", do so here. The
426 * freed space becomes available for allocation by the reader.
428 * Return: The amount of space freed in bytes.
430 unsigned int comedi_buf_write_free(struct comedi_subdevice
*s
,
433 struct comedi_async
*async
= s
->async
;
434 unsigned int allocated
= comedi_buf_write_n_allocated(s
);
436 if (nbytes
> allocated
)
439 async
->buf_write_count
+= nbytes
;
440 async
->buf_write_ptr
+= nbytes
;
441 comedi_buf_munge(s
, async
->buf_write_count
- async
->munge_count
);
442 if (async
->buf_write_ptr
>= async
->prealloc_bufsz
)
443 async
->buf_write_ptr
%= async
->prealloc_bufsz
;
447 EXPORT_SYMBOL_GPL(comedi_buf_write_free
);
450 * comedi_buf_read_n_available() - Determine amount of readable buffer space
451 * @s: COMEDI subdevice.
453 * Determine the amount of readable buffer space in the COMEDI acquisition data
454 * buffer associated with the subdevice. The readable buffer space is that
455 * which has been freed by the writer and "munged" to the sample data format
456 * expected by COMEDI if necessary.
458 * Return: The amount of readable buffer space.
460 unsigned int comedi_buf_read_n_available(struct comedi_subdevice
*s
)
462 struct comedi_async
*async
= s
->async
;
463 unsigned int num_bytes
;
468 num_bytes
= async
->munge_count
- async
->buf_read_count
;
471 * ensure the async buffer 'counts' are read before we
472 * attempt to read data from the buffer
478 EXPORT_SYMBOL_GPL(comedi_buf_read_n_available
);
481 * comedi_buf_read_alloc() - Reserve buffer space for reading
482 * @s: COMEDI subdevice.
483 * @nbytes: Maximum space to reserve in bytes.
485 * Reserve up to @nbytes bytes of previously written and "munged" buffer space
486 * for reading in the COMEDI acquisition data buffer associated with the
487 * subdevice. The amount reserved is limited to the space available. The
488 * reader can read from the reserved space and then free it. A reader is also
489 * allowed to read from the space before reserving it as long as it determines
490 * the amount of readable data available, but the space needs to be marked as
491 * reserved before it can be freed.
493 * Return: The amount of space reserved in bytes.
495 unsigned int comedi_buf_read_alloc(struct comedi_subdevice
*s
,
498 struct comedi_async
*async
= s
->async
;
499 unsigned int available
;
501 available
= async
->munge_count
- async
->buf_read_alloc_count
;
502 if (nbytes
> available
)
505 async
->buf_read_alloc_count
+= nbytes
;
508 * ensure the async buffer 'counts' are read before we
509 * attempt to read data from the read-alloc'ed buffer space
515 EXPORT_SYMBOL_GPL(comedi_buf_read_alloc
);
517 static unsigned int comedi_buf_read_n_allocated(struct comedi_async
*async
)
519 return async
->buf_read_alloc_count
- async
->buf_read_count
;
523 * comedi_buf_read_free() - Free buffer space after it has been read
524 * @s: COMEDI subdevice.
525 * @nbytes: Maximum space to free in bytes.
527 * Free up to @nbytes bytes of buffer space previously reserved for reading in
528 * the COMEDI acquisition data buffer associated with the subdevice. The
529 * amount of space freed is limited to the amount that was reserved.
531 * The freed space becomes available for allocation by the writer.
533 * Return: The amount of space freed in bytes.
535 unsigned int comedi_buf_read_free(struct comedi_subdevice
*s
,
538 struct comedi_async
*async
= s
->async
;
539 unsigned int allocated
;
542 * ensure data has been read out of buffer before
543 * the async read count is incremented
547 allocated
= comedi_buf_read_n_allocated(async
);
548 if (nbytes
> allocated
)
551 async
->buf_read_count
+= nbytes
;
552 async
->buf_read_ptr
+= nbytes
;
553 async
->buf_read_ptr
%= async
->prealloc_bufsz
;
556 EXPORT_SYMBOL_GPL(comedi_buf_read_free
);
558 static void comedi_buf_memcpy_to(struct comedi_subdevice
*s
,
559 const void *data
, unsigned int num_bytes
)
561 struct comedi_async
*async
= s
->async
;
562 unsigned int write_ptr
= async
->buf_write_ptr
;
565 unsigned int block_size
;
567 if (write_ptr
+ num_bytes
> async
->prealloc_bufsz
)
568 block_size
= async
->prealloc_bufsz
- write_ptr
;
570 block_size
= num_bytes
;
572 memcpy(async
->prealloc_buf
+ write_ptr
, data
, block_size
);
575 num_bytes
-= block_size
;
581 static void comedi_buf_memcpy_from(struct comedi_subdevice
*s
,
582 void *dest
, unsigned int nbytes
)
585 struct comedi_async
*async
= s
->async
;
586 unsigned int read_ptr
= async
->buf_read_ptr
;
589 unsigned int block_size
;
591 src
= async
->prealloc_buf
+ read_ptr
;
593 if (nbytes
>= async
->prealloc_bufsz
- read_ptr
)
594 block_size
= async
->prealloc_bufsz
- read_ptr
;
598 memcpy(dest
, src
, block_size
);
599 nbytes
-= block_size
;
606 * comedi_buf_write_samples() - Write sample data to COMEDI buffer
607 * @s: COMEDI subdevice.
608 * @data: Pointer to source samples.
609 * @nsamples: Number of samples to write.
611 * Write up to @nsamples samples to the COMEDI acquisition data buffer
612 * associated with the subdevice, mark it as written and update the
613 * acquisition scan progress. If there is not enough room for the specified
614 * number of samples, the number of samples written is limited to the number
615 * that will fit and the %COMEDI_CB_OVERFLOW event flag is set to cause the
616 * acquisition to terminate with an overrun error. Set the %COMEDI_CB_BLOCK
617 * event flag if any samples are written to cause waiting tasks to be woken
618 * when the event flags are processed.
620 * Return: The amount of data written in bytes.
622 unsigned int comedi_buf_write_samples(struct comedi_subdevice
*s
,
623 const void *data
, unsigned int nsamples
)
625 unsigned int max_samples
;
629 * Make sure there is enough room in the buffer for all the samples.
630 * If not, clamp the nsamples to the number that will fit, flag the
631 * buffer overrun and add the samples that fit.
633 max_samples
= comedi_bytes_to_samples(s
, comedi_buf_write_n_unalloc(s
));
634 if (nsamples
> max_samples
) {
635 dev_warn(s
->device
->class_dev
, "buffer overrun\n");
636 s
->async
->events
|= COMEDI_CB_OVERFLOW
;
637 nsamples
= max_samples
;
643 nbytes
= comedi_buf_write_alloc(s
,
644 comedi_samples_to_bytes(s
, nsamples
));
645 comedi_buf_memcpy_to(s
, data
, nbytes
);
646 comedi_buf_write_free(s
, nbytes
);
647 comedi_inc_scan_progress(s
, nbytes
);
648 s
->async
->events
|= COMEDI_CB_BLOCK
;
652 EXPORT_SYMBOL_GPL(comedi_buf_write_samples
);
655 * comedi_buf_read_samples() - Read sample data from COMEDI buffer
656 * @s: COMEDI subdevice.
657 * @data: Pointer to destination.
658 * @nsamples: Maximum number of samples to read.
660 * Read up to @nsamples samples from the COMEDI acquisition data buffer
661 * associated with the subdevice, mark it as read and update the acquisition
662 * scan progress. Limit the number of samples read to the number available.
663 * Set the %COMEDI_CB_BLOCK event flag if any samples are read to cause waiting
664 * tasks to be woken when the event flags are processed.
666 * Return: The amount of data read in bytes.
668 unsigned int comedi_buf_read_samples(struct comedi_subdevice
*s
,
669 void *data
, unsigned int nsamples
)
671 unsigned int max_samples
;
674 /* clamp nsamples to the number of full samples available */
675 max_samples
= comedi_bytes_to_samples(s
,
676 comedi_buf_read_n_available(s
));
677 if (nsamples
> max_samples
)
678 nsamples
= max_samples
;
683 nbytes
= comedi_buf_read_alloc(s
,
684 comedi_samples_to_bytes(s
, nsamples
));
685 comedi_buf_memcpy_from(s
, data
, nbytes
);
686 comedi_buf_read_free(s
, nbytes
);
687 comedi_inc_scan_progress(s
, nbytes
);
688 s
->async
->events
|= COMEDI_CB_BLOCK
;
692 EXPORT_SYMBOL_GPL(comedi_buf_read_samples
);