2 * Support for Medifield PNW Camera Imaging ISP subsystem.
4 * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
6 * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License version
10 * 2 as published by the Free Software Foundation.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
24 * This file contains entry functions for memory management of ISP driver
26 #include <linux/kernel.h>
27 #include <linux/types.h>
29 #include <linux/highmem.h> /* for kmap */
30 #include <linux/io.h> /* for page_to_phys */
31 #include <linux/sysfs.h>
34 #include "hmm/hmm_pool.h"
35 #include "hmm/hmm_bo.h"
37 #include "atomisp_internal.h"
38 #include "asm/cacheflush.h"
39 #include "mmu/isp_mmu.h"
40 #include "mmu/sh_mmu_mrfld.h"
43 #include <asm/ssse3.h>
46 struct hmm_bo_device bo_device
;
47 struct hmm_pool dynamic_pool
;
48 struct hmm_pool reserved_pool
;
49 static ia_css_ptr dummy_ptr
;
50 struct _hmm_mem_stat hmm_mem_stat
;
52 const char *hmm_bo_type_strings
[HMM_BO_LAST
] = {
61 static ssize_t
bo_show(struct device
*dev
, struct device_attribute
*attr
,
62 char *buf
, struct list_head
*bo_list
, bool active
)
65 struct hmm_buffer_object
*bo
;
68 long total
[HMM_BO_LAST
] = { 0 };
69 long count
[HMM_BO_LAST
] = { 0 };
73 ret
= scnprintf(buf
, PAGE_SIZE
, "type pgnr\n");
79 spin_lock_irqsave(&bo_device
.list_lock
, flags
);
80 list_for_each_entry(bo
, bo_list
, list
) {
81 if ((active
&& (bo
->status
& HMM_BO_ALLOCED
)) ||
82 (!active
&& !(bo
->status
& HMM_BO_ALLOCED
))) {
83 ret
= scnprintf(buf
+ index1
, PAGE_SIZE
- index1
,
85 hmm_bo_type_strings
[bo
->type
], bo
->pgnr
);
87 total
[bo
->type
] += bo
->pgnr
;
93 spin_unlock_irqrestore(&bo_device
.list_lock
, flags
);
95 for (i
= 0; i
< HMM_BO_LAST
; i
++) {
97 ret
= scnprintf(buf
+ index1
+ index2
,
98 PAGE_SIZE
- index1
- index2
,
99 "%ld %s buffer objects: %ld KB\n",
100 count
[i
], hmm_bo_type_strings
[i
], total
[i
] * 4);
106 /* Add trailing zero, not included by scnprintf */
107 return index1
+ index2
+ 1;
110 static ssize_t
active_bo_show(struct device
*dev
,
111 struct device_attribute
*attr
,
114 return bo_show(dev
, attr
, buf
, &bo_device
.entire_bo_list
, true);
117 static ssize_t
free_bo_show(struct device
*dev
,
118 struct device_attribute
*attr
,
121 return bo_show(dev
, attr
, buf
, &bo_device
.entire_bo_list
, false);
124 static ssize_t
reserved_pool_show(struct device
*dev
,
125 struct device_attribute
*attr
,
130 struct hmm_reserved_pool_info
*pinfo
= reserved_pool
.pool_info
;
133 if (!pinfo
|| !pinfo
->initialized
)
136 spin_lock_irqsave(&pinfo
->list_lock
, flags
);
137 ret
= scnprintf(buf
, PAGE_SIZE
, "%d out of %d pages available\n",
138 pinfo
->index
, pinfo
->pgnr
);
139 spin_unlock_irqrestore(&pinfo
->list_lock
, flags
);
142 ret
++; /* Add trailing zero, not included by scnprintf */
147 static ssize_t
dynamic_pool_show(struct device
*dev
,
148 struct device_attribute
*attr
,
153 struct hmm_dynamic_pool_info
*pinfo
= dynamic_pool
.pool_info
;
156 if (!pinfo
|| !pinfo
->initialized
)
159 spin_lock_irqsave(&pinfo
->list_lock
, flags
);
160 ret
= scnprintf(buf
, PAGE_SIZE
, "%d (max %d) pages available\n",
161 pinfo
->pgnr
, pinfo
->pool_size
);
162 spin_unlock_irqrestore(&pinfo
->list_lock
, flags
);
165 ret
++; /* Add trailing zero, not included by scnprintf */
170 static DEVICE_ATTR(active_bo
, S_IRUGO
, active_bo_show
, NULL
);
171 static DEVICE_ATTR(free_bo
, S_IRUGO
, free_bo_show
, NULL
);
172 static DEVICE_ATTR(reserved_pool
, S_IRUGO
, reserved_pool_show
, NULL
);
173 static DEVICE_ATTR(dynamic_pool
, S_IRUGO
, dynamic_pool_show
, NULL
);
175 static struct attribute
*sysfs_attrs_ctrl
[] = {
176 &dev_attr_active_bo
.attr
,
177 &dev_attr_free_bo
.attr
,
178 &dev_attr_reserved_pool
.attr
,
179 &dev_attr_dynamic_pool
.attr
,
183 static struct attribute_group atomisp_attribute_group
[] = {
184 {.attrs
= sysfs_attrs_ctrl
},
191 ret
= hmm_bo_device_init(&bo_device
, &sh_mmu_mrfld
,
192 ISP_VM_START
, ISP_VM_SIZE
);
194 dev_err(atomisp_dev
, "hmm_bo_device_init failed.\n");
197 * As hmm use NULL to indicate invalid ISP virtual address,
198 * and ISP_VM_START is defined to 0 too, so we allocate
199 * one piece of dummy memory, which should return value 0,
200 * at the beginning, to avoid hmm_alloc return 0 in the
201 * further allocation.
203 dummy_ptr
= hmm_alloc(1, HMM_BO_PRIVATE
, 0, 0, HMM_UNCACHED
);
206 ret
= sysfs_create_group(&atomisp_dev
->kobj
,
207 atomisp_attribute_group
);
210 "%s Failed to create sysfs\n", __func__
);
216 void hmm_cleanup(void)
218 sysfs_remove_group(&atomisp_dev
->kobj
, atomisp_attribute_group
);
221 * free dummy memory first
226 hmm_bo_device_exit(&bo_device
);
229 ia_css_ptr
hmm_alloc(size_t bytes
, enum hmm_bo_type type
,
230 int from_highmem
, void *userptr
, bool cached
)
233 struct hmm_buffer_object
*bo
;
236 /*Get page number from size*/
237 pgnr
= size_to_pgnr_ceil(bytes
);
239 /*Buffer object structure init*/
240 bo
= hmm_bo_alloc(&bo_device
, pgnr
);
242 dev_err(atomisp_dev
, "hmm_bo_create failed.\n");
246 /*Allocate pages for memory*/
247 ret
= hmm_bo_alloc_pages(bo
, type
, from_highmem
, userptr
, cached
);
250 "hmm_bo_alloc_pages failed.\n");
254 /*Combind the virtual address and pages togather*/
255 ret
= hmm_bo_bind(bo
);
257 dev_err(atomisp_dev
, "hmm_bo_bind failed.\n");
261 hmm_mem_stat
.tol_cnt
+= pgnr
;
266 hmm_bo_free_pages(bo
);
273 void hmm_free(ia_css_ptr virt
)
275 struct hmm_buffer_object
*bo
;
277 bo
= hmm_bo_device_search_start(&bo_device
, (unsigned int)virt
);
281 "can not find buffer object start with "
282 "address 0x%x\n", (unsigned int)virt
);
286 hmm_mem_stat
.tol_cnt
-= bo
->pgnr
;
290 hmm_bo_free_pages(bo
);
295 static inline int hmm_check_bo(struct hmm_buffer_object
*bo
, unsigned int ptr
)
299 "can not find buffer object contains "
300 "address 0x%x\n", ptr
);
304 if (!hmm_bo_page_allocated(bo
)) {
306 "buffer object has no page allocated.\n");
310 if (!hmm_bo_allocated(bo
)) {
312 "buffer object has no virtual address"
313 " space allocated.\n");
320 /*Read function in ISP memory management*/
321 static int load_and_flush_by_kmap(ia_css_ptr virt
, void *data
, unsigned int bytes
)
323 struct hmm_buffer_object
*bo
;
324 unsigned int idx
, offset
, len
;
328 bo
= hmm_bo_device_search_in_range(&bo_device
, virt
);
329 ret
= hmm_check_bo(bo
, virt
);
335 idx
= (virt
- bo
->start
) >> PAGE_SHIFT
;
336 offset
= (virt
- bo
->start
) - (idx
<< PAGE_SHIFT
);
338 src
= (char *)kmap(bo
->page_obj
[idx
].page
);
341 "kmap buffer object page failed: "
342 "pg_idx = %d\n", idx
);
348 if ((bytes
+ offset
) >= PAGE_SIZE
) {
349 len
= PAGE_SIZE
- offset
;
356 virt
+= len
; /* update virt for next loop */
361 _ssse3_memcpy(des
, src
, len
);
363 memcpy(des
, src
, len
);
368 clflush_cache_range(src
, len
);
370 kunmap(bo
->page_obj
[idx
].page
);
376 /*Read function in ISP memory management*/
377 static int load_and_flush(ia_css_ptr virt
, void *data
, unsigned int bytes
)
379 struct hmm_buffer_object
*bo
;
382 bo
= hmm_bo_device_search_in_range(&bo_device
, virt
);
383 ret
= hmm_check_bo(bo
, virt
);
387 if (bo
->status
& HMM_BO_VMAPED
|| bo
->status
& HMM_BO_VMAPED_CACHED
) {
388 void *src
= bo
->vmap_addr
;
390 src
+= (virt
- bo
->start
);
392 _ssse3_memcpy(data
, src
, bytes
);
394 memcpy(data
, src
, bytes
);
396 if (bo
->status
& HMM_BO_VMAPED_CACHED
)
397 clflush_cache_range(src
, bytes
);
401 vptr
= hmm_bo_vmap(bo
, true);
403 return load_and_flush_by_kmap(virt
, data
, bytes
);
405 vptr
= vptr
+ (virt
- bo
->start
);
408 _ssse3_memcpy(data
, vptr
, bytes
);
410 memcpy(data
, vptr
, bytes
);
412 clflush_cache_range(vptr
, bytes
);
419 /*Read function in ISP memory management*/
420 int hmm_load(ia_css_ptr virt
, void *data
, unsigned int bytes
)
424 "hmm_load NULL argument\n");
427 return load_and_flush(virt
, data
, bytes
);
430 /*Flush hmm data from the data cache*/
431 int hmm_flush(ia_css_ptr virt
, unsigned int bytes
)
433 return load_and_flush(virt
, NULL
, bytes
);
436 /*Write function in ISP memory management*/
437 int hmm_store(ia_css_ptr virt
, const void *data
, unsigned int bytes
)
439 struct hmm_buffer_object
*bo
;
440 unsigned int idx
, offset
, len
;
444 bo
= hmm_bo_device_search_in_range(&bo_device
, virt
);
445 ret
= hmm_check_bo(bo
, virt
);
449 if (bo
->status
& HMM_BO_VMAPED
|| bo
->status
& HMM_BO_VMAPED_CACHED
) {
450 void *dst
= bo
->vmap_addr
;
452 dst
+= (virt
- bo
->start
);
454 _ssse3_memcpy(dst
, data
, bytes
);
456 memcpy(dst
, data
, bytes
);
458 if (bo
->status
& HMM_BO_VMAPED_CACHED
)
459 clflush_cache_range(dst
, bytes
);
463 vptr
= hmm_bo_vmap(bo
, true);
465 vptr
= vptr
+ (virt
- bo
->start
);
468 _ssse3_memcpy(vptr
, data
, bytes
);
470 memcpy(vptr
, data
, bytes
);
472 clflush_cache_range(vptr
, bytes
);
480 idx
= (virt
- bo
->start
) >> PAGE_SHIFT
;
481 offset
= (virt
- bo
->start
) - (idx
<< PAGE_SHIFT
);
484 des
= (char *)kmap_atomic(bo
->page_obj
[idx
].page
);
486 des
= (char *)kmap(bo
->page_obj
[idx
].page
);
490 "kmap buffer object page failed: "
491 "pg_idx = %d\n", idx
);
497 if ((bytes
+ offset
) >= PAGE_SIZE
) {
498 len
= PAGE_SIZE
- offset
;
508 _ssse3_memcpy(des
, src
, len
);
510 memcpy(des
, src
, len
);
514 clflush_cache_range(des
, len
);
518 * Note: kunmap_atomic requires return addr from
519 * kmap_atomic, not the page. See linux/highmem.h
521 kunmap_atomic(des
- offset
);
523 kunmap(bo
->page_obj
[idx
].page
);
529 /*memset function in ISP memory management*/
530 int hmm_set(ia_css_ptr virt
, int c
, unsigned int bytes
)
532 struct hmm_buffer_object
*bo
;
533 unsigned int idx
, offset
, len
;
537 bo
= hmm_bo_device_search_in_range(&bo_device
, virt
);
538 ret
= hmm_check_bo(bo
, virt
);
542 if (bo
->status
& HMM_BO_VMAPED
|| bo
->status
& HMM_BO_VMAPED_CACHED
) {
543 void *dst
= bo
->vmap_addr
;
545 dst
+= (virt
- bo
->start
);
546 memset(dst
, c
, bytes
);
548 if (bo
->status
& HMM_BO_VMAPED_CACHED
)
549 clflush_cache_range(dst
, bytes
);
553 vptr
= hmm_bo_vmap(bo
, true);
555 vptr
= vptr
+ (virt
- bo
->start
);
556 memset((void *)vptr
, c
, bytes
);
557 clflush_cache_range(vptr
, bytes
);
564 idx
= (virt
- bo
->start
) >> PAGE_SHIFT
;
565 offset
= (virt
- bo
->start
) - (idx
<< PAGE_SHIFT
);
567 des
= (char *)kmap(bo
->page_obj
[idx
].page
);
570 "kmap buffer object page failed: "
571 "pg_idx = %d\n", idx
);
576 if ((bytes
+ offset
) >= PAGE_SIZE
) {
577 len
= PAGE_SIZE
- offset
;
588 clflush_cache_range(des
, len
);
590 kunmap(bo
->page_obj
[idx
].page
);
596 /*Virtual address to physical address convert*/
597 phys_addr_t
hmm_virt_to_phys(ia_css_ptr virt
)
599 unsigned int idx
, offset
;
600 struct hmm_buffer_object
*bo
;
602 bo
= hmm_bo_device_search_in_range(&bo_device
, virt
);
605 "can not find buffer object contains address 0x%x\n",
610 idx
= (virt
- bo
->start
) >> PAGE_SHIFT
;
611 offset
= (virt
- bo
->start
) - (idx
<< PAGE_SHIFT
);
613 return page_to_phys(bo
->page_obj
[idx
].page
) + offset
;
616 int hmm_mmap(struct vm_area_struct
*vma
, ia_css_ptr virt
)
618 struct hmm_buffer_object
*bo
;
620 bo
= hmm_bo_device_search_start(&bo_device
, virt
);
623 "can not find buffer object start with address 0x%x\n",
628 return hmm_bo_mmap(vma
, bo
);
631 /*Map ISP virtual address into IA virtual address*/
632 void *hmm_vmap(ia_css_ptr virt
, bool cached
)
634 struct hmm_buffer_object
*bo
;
637 bo
= hmm_bo_device_search_in_range(&bo_device
, virt
);
640 "can not find buffer object contains address 0x%x\n",
645 ptr
= hmm_bo_vmap(bo
, cached
);
647 return ptr
+ (virt
- bo
->start
);
652 /* Flush the memory which is mapped as cached memory through hmm_vmap */
653 void hmm_flush_vmap(ia_css_ptr virt
)
655 struct hmm_buffer_object
*bo
;
657 bo
= hmm_bo_device_search_in_range(&bo_device
, virt
);
659 dev_warn(atomisp_dev
,
660 "can not find buffer object contains address 0x%x\n",
665 hmm_bo_flush_vmap(bo
);
668 void hmm_vunmap(ia_css_ptr virt
)
670 struct hmm_buffer_object
*bo
;
672 bo
= hmm_bo_device_search_in_range(&bo_device
, virt
);
674 dev_warn(atomisp_dev
,
675 "can not find buffer object contains address 0x%x\n",
680 return hmm_bo_vunmap(bo
);
683 int hmm_pool_register(unsigned int pool_size
,
684 enum hmm_pool_type pool_type
)
687 case HMM_POOL_TYPE_RESERVED
:
688 reserved_pool
.pops
= &reserved_pops
;
689 return reserved_pool
.pops
->pool_init(&reserved_pool
.pool_info
,
691 case HMM_POOL_TYPE_DYNAMIC
:
692 dynamic_pool
.pops
= &dynamic_pops
;
693 return dynamic_pool
.pops
->pool_init(&dynamic_pool
.pool_info
,
696 dev_err(atomisp_dev
, "invalid pool type.\n");
701 void hmm_pool_unregister(enum hmm_pool_type pool_type
)
704 case HMM_POOL_TYPE_RESERVED
:
705 if (reserved_pool
.pops
&& reserved_pool
.pops
->pool_exit
)
706 reserved_pool
.pops
->pool_exit(&reserved_pool
.pool_info
);
708 case HMM_POOL_TYPE_DYNAMIC
:
709 if (dynamic_pool
.pops
&& dynamic_pool
.pops
->pool_exit
)
710 dynamic_pool
.pops
->pool_exit(&dynamic_pool
.pool_info
);
713 dev_err(atomisp_dev
, "invalid pool type.\n");
720 void *hmm_isp_vaddr_to_host_vaddr(ia_css_ptr ptr
, bool cached
)
722 return hmm_vmap(ptr
, cached
);
723 /* vmunmap will be done in hmm_bo_release() */
726 ia_css_ptr
hmm_host_vaddr_to_hrt_vaddr(const void *ptr
)
728 struct hmm_buffer_object
*bo
;
730 bo
= hmm_bo_device_search_vmap_start(&bo_device
, ptr
);
735 "can not find buffer object whose kernel virtual address is %p\n",
740 void hmm_show_mem_stat(const char *func
, const int line
)
742 trace_printk("tol_cnt=%d usr_size=%d res_size=%d res_cnt=%d sys_size=%d dyc_thr=%d dyc_size=%d.\n",
743 hmm_mem_stat
.tol_cnt
,
744 hmm_mem_stat
.usr_size
, hmm_mem_stat
.res_size
,
745 hmm_mem_stat
.res_cnt
, hmm_mem_stat
.sys_size
,
746 hmm_mem_stat
.dyc_thr
, hmm_mem_stat
.dyc_size
);
749 void hmm_init_mem_stat(int res_pgnr
, int dyc_en
, int dyc_pgnr
)
751 hmm_mem_stat
.res_size
= res_pgnr
;
752 /* If reserved mem pool is not enabled, set its "mem stat" values as -1. */
753 if (0 == hmm_mem_stat
.res_size
) {
754 hmm_mem_stat
.res_size
= -1;
755 hmm_mem_stat
.res_cnt
= -1;
758 /* If dynamic memory pool is not enabled, set its "mem stat" values as -1. */
760 hmm_mem_stat
.dyc_size
= -1;
761 hmm_mem_stat
.dyc_thr
= -1;
763 hmm_mem_stat
.dyc_size
= 0;
764 hmm_mem_stat
.dyc_thr
= dyc_pgnr
;
766 hmm_mem_stat
.usr_size
= 0;
767 hmm_mem_stat
.sys_size
= 0;
768 hmm_mem_stat
.tol_cnt
= 0;