2 * PS3 address space management.
4 * Copyright (C) 2006 Sony Computer Entertainment Inc.
5 * Copyright 2006 Sony Corp.
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; version 2 of the License.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 #include <linux/kernel.h>
22 #include <linux/export.h>
23 #include <linux/memblock.h>
24 #include <linux/slab.h>
26 #include <asm/cell-regs.h>
27 #include <asm/firmware.h>
30 #include <asm/lv1call.h>
31 #include <asm/setup.h>
36 #define DBG udbg_printf
42 #if defined(CONFIG_PS3_DYNAMIC_DMA)
55 static unsigned long make_page_sizes(unsigned long a
, unsigned long b
)
57 return (a
<< 56) | (b
<< 48);
61 ALLOCATE_MEMORY_TRY_ALT_UNIT
= 0X04,
62 ALLOCATE_MEMORY_ADDR_ZERO
= 0X08,
65 /* valid htab sizes are {18,19,20} = 256K, 512K, 1M */
68 HTAB_SIZE_MAX
= 20U, /* HV limit of 1MB */
69 HTAB_SIZE_MIN
= 18U, /* CPU limit of 256KB */
72 /*============================================================================*/
73 /* virtual address space routines */
74 /*============================================================================*/
77 * struct mem_region - memory region structure
79 * @size: size in bytes
80 * @offset: difference between base and rm.size
81 * @destroy: flag if region should be destroyed upon shutdown
92 * struct map - address space state variables holder
93 * @total: total memory available as reported by HV
94 * @vas_id - HV virtual address space id
95 * @htab_size: htab size in bytes
97 * The HV virtual address space (vas) allows for hotplug memory regions.
98 * Memory regions can be created and destroyed in the vas at runtime.
99 * @rm: real mode (bootmem) region
100 * @r1: highmem region(s)
103 * virt_addr: a cpu 'translated' effective address
104 * phys_addr: an address in what Linux thinks is the physical address space
105 * lpar_addr: an address in the HV virtual address space
106 * bus_addr: an io controller 'translated' address on a device bus
113 struct mem_region rm
;
114 struct mem_region r1
;
117 #define debug_dump_map(x) _debug_dump_map(x, __func__, __LINE__)
118 static void __maybe_unused
_debug_dump_map(const struct map
*m
,
119 const char *func
, int line
)
121 DBG("%s:%d: map.total = %llxh\n", func
, line
, m
->total
);
122 DBG("%s:%d: map.rm.size = %llxh\n", func
, line
, m
->rm
.size
);
123 DBG("%s:%d: map.vas_id = %llu\n", func
, line
, m
->vas_id
);
124 DBG("%s:%d: map.htab_size = %llxh\n", func
, line
, m
->htab_size
);
125 DBG("%s:%d: map.r1.base = %llxh\n", func
, line
, m
->r1
.base
);
126 DBG("%s:%d: map.r1.offset = %lxh\n", func
, line
, m
->r1
.offset
);
127 DBG("%s:%d: map.r1.size = %llxh\n", func
, line
, m
->r1
.size
);
130 static struct map map
;
133 * ps3_mm_phys_to_lpar - translate a linux physical address to lpar address
134 * @phys_addr: linux physical address
137 unsigned long ps3_mm_phys_to_lpar(unsigned long phys_addr
)
139 BUG_ON(is_kernel_addr(phys_addr
));
140 return (phys_addr
< map
.rm
.size
|| phys_addr
>= map
.total
)
141 ? phys_addr
: phys_addr
+ map
.r1
.offset
;
144 EXPORT_SYMBOL(ps3_mm_phys_to_lpar
);
147 * ps3_mm_vas_create - create the virtual address space
150 void __init
ps3_mm_vas_create(unsigned long* htab_size
)
159 result
= lv1_query_logical_partition_address_region_info(0,
160 &start_address
, &size
, &access_right
, &max_page_size
,
164 DBG("%s:%d: lv1_query_logical_partition_address_region_info "
165 "failed: %s\n", __func__
, __LINE__
,
170 if (max_page_size
< PAGE_SHIFT_16M
) {
171 DBG("%s:%d: bad max_page_size %llxh\n", __func__
, __LINE__
,
176 BUILD_BUG_ON(CONFIG_PS3_HTAB_SIZE
> HTAB_SIZE_MAX
);
177 BUILD_BUG_ON(CONFIG_PS3_HTAB_SIZE
< HTAB_SIZE_MIN
);
179 result
= lv1_construct_virtual_address_space(CONFIG_PS3_HTAB_SIZE
,
180 2, make_page_sizes(PAGE_SHIFT_16M
, PAGE_SHIFT_64K
),
181 &map
.vas_id
, &map
.htab_size
);
184 DBG("%s:%d: lv1_construct_virtual_address_space failed: %s\n",
185 __func__
, __LINE__
, ps3_result(result
));
189 result
= lv1_select_virtual_address_space(map
.vas_id
);
192 DBG("%s:%d: lv1_select_virtual_address_space failed: %s\n",
193 __func__
, __LINE__
, ps3_result(result
));
197 *htab_size
= map
.htab_size
;
199 debug_dump_map(&map
);
204 panic("ps3_mm_vas_create failed");
208 * ps3_mm_vas_destroy -
211 void ps3_mm_vas_destroy(void)
215 DBG("%s:%d: map.vas_id = %llu\n", __func__
, __LINE__
, map
.vas_id
);
218 result
= lv1_select_virtual_address_space(0);
220 result
= lv1_destruct_virtual_address_space(map
.vas_id
);
226 static int ps3_mm_get_repository_highmem(struct mem_region
*r
)
230 /* Assume a single highmem region. */
232 result
= ps3_repository_read_highmem_info(0, &r
->base
, &r
->size
);
237 if (!r
->base
|| !r
->size
) {
242 r
->offset
= r
->base
- map
.rm
.size
;
244 DBG("%s:%d: Found high region in repository: %llxh %llxh\n",
245 __func__
, __LINE__
, r
->base
, r
->size
);
250 DBG("%s:%d: No high region in repository.\n", __func__
, __LINE__
);
252 r
->size
= r
->base
= r
->offset
= 0;
256 static int ps3_mm_set_repository_highmem(const struct mem_region
*r
)
258 /* Assume a single highmem region. */
260 return r
? ps3_repository_write_highmem_info(0, r
->base
, r
->size
) :
261 ps3_repository_write_highmem_info(0, 0, 0);
265 * ps3_mm_region_create - create a memory region in the vas
266 * @r: pointer to a struct mem_region to accept initialized values
267 * @size: requested region size
269 * This implementation creates the region with the vas large page size.
270 * @size is rounded down to a multiple of the vas large page size.
273 static int ps3_mm_region_create(struct mem_region
*r
, unsigned long size
)
278 r
->size
= _ALIGN_DOWN(size
, 1 << PAGE_SHIFT_16M
);
280 DBG("%s:%d requested %lxh\n", __func__
, __LINE__
, size
);
281 DBG("%s:%d actual %llxh\n", __func__
, __LINE__
, r
->size
);
282 DBG("%s:%d difference %llxh (%lluMB)\n", __func__
, __LINE__
,
283 size
- r
->size
, (size
- r
->size
) / 1024 / 1024);
286 DBG("%s:%d: size == 0\n", __func__
, __LINE__
);
291 result
= lv1_allocate_memory(r
->size
, PAGE_SHIFT_16M
, 0,
292 ALLOCATE_MEMORY_TRY_ALT_UNIT
, &r
->base
, &muid
);
294 if (result
|| r
->base
< map
.rm
.size
) {
295 DBG("%s:%d: lv1_allocate_memory failed: %s\n",
296 __func__
, __LINE__
, ps3_result(result
));
301 r
->offset
= r
->base
- map
.rm
.size
;
305 r
->size
= r
->base
= r
->offset
= 0;
310 * ps3_mm_region_destroy - destroy a memory region
311 * @r: pointer to struct mem_region
314 static void ps3_mm_region_destroy(struct mem_region
*r
)
319 pr_info("%s:%d: Not destroying high region: %llxh %llxh\n",
320 __func__
, __LINE__
, r
->base
, r
->size
);
324 DBG("%s:%d: r->base = %llxh\n", __func__
, __LINE__
, r
->base
);
327 result
= lv1_release_memory(r
->base
);
329 r
->size
= r
->base
= r
->offset
= 0;
330 map
.total
= map
.rm
.size
;
332 ps3_mm_set_repository_highmem(NULL
);
335 /*============================================================================*/
337 /*============================================================================*/
340 * dma_sb_lpar_to_bus - Translate an lpar address to ioc mapped bus address.
341 * @r: pointer to dma region structure
342 * @lpar_addr: HV lpar address
345 static unsigned long dma_sb_lpar_to_bus(struct ps3_dma_region
*r
,
346 unsigned long lpar_addr
)
348 if (lpar_addr
>= map
.rm
.size
)
349 lpar_addr
-= map
.r1
.offset
;
350 BUG_ON(lpar_addr
< r
->offset
);
351 BUG_ON(lpar_addr
>= r
->offset
+ r
->len
);
352 return r
->bus_addr
+ lpar_addr
- r
->offset
;
355 #define dma_dump_region(_a) _dma_dump_region(_a, __func__, __LINE__)
356 static void __maybe_unused
_dma_dump_region(const struct ps3_dma_region
*r
,
357 const char *func
, int line
)
359 DBG("%s:%d: dev %llu:%llu\n", func
, line
, r
->dev
->bus_id
,
361 DBG("%s:%d: page_size %u\n", func
, line
, r
->page_size
);
362 DBG("%s:%d: bus_addr %lxh\n", func
, line
, r
->bus_addr
);
363 DBG("%s:%d: len %lxh\n", func
, line
, r
->len
);
364 DBG("%s:%d: offset %lxh\n", func
, line
, r
->offset
);
368 * dma_chunk - A chunk of dma pages mapped by the io controller.
369 * @region - The dma region that owns this chunk.
370 * @lpar_addr: Starting lpar address of the area to map.
371 * @bus_addr: Starting ioc bus address of the area to map.
372 * @len: Length in bytes of the area to map.
373 * @link: A struct list_head used with struct ps3_dma_region.chunk_list, the
374 * list of all chuncks owned by the region.
376 * This implementation uses a very simple dma page manager
377 * based on the dma_chunk structure. This scheme assumes
378 * that all drivers use very well behaved dma ops.
382 struct ps3_dma_region
*region
;
383 unsigned long lpar_addr
;
384 unsigned long bus_addr
;
386 struct list_head link
;
387 unsigned int usage_count
;
390 #define dma_dump_chunk(_a) _dma_dump_chunk(_a, __func__, __LINE__)
391 static void _dma_dump_chunk (const struct dma_chunk
* c
, const char* func
,
394 DBG("%s:%d: r.dev %llu:%llu\n", func
, line
,
395 c
->region
->dev
->bus_id
, c
->region
->dev
->dev_id
);
396 DBG("%s:%d: r.bus_addr %lxh\n", func
, line
, c
->region
->bus_addr
);
397 DBG("%s:%d: r.page_size %u\n", func
, line
, c
->region
->page_size
);
398 DBG("%s:%d: r.len %lxh\n", func
, line
, c
->region
->len
);
399 DBG("%s:%d: r.offset %lxh\n", func
, line
, c
->region
->offset
);
400 DBG("%s:%d: c.lpar_addr %lxh\n", func
, line
, c
->lpar_addr
);
401 DBG("%s:%d: c.bus_addr %lxh\n", func
, line
, c
->bus_addr
);
402 DBG("%s:%d: c.len %lxh\n", func
, line
, c
->len
);
405 static struct dma_chunk
* dma_find_chunk(struct ps3_dma_region
*r
,
406 unsigned long bus_addr
, unsigned long len
)
409 unsigned long aligned_bus
= _ALIGN_DOWN(bus_addr
, 1 << r
->page_size
);
410 unsigned long aligned_len
= _ALIGN_UP(len
+bus_addr
-aligned_bus
,
413 list_for_each_entry(c
, &r
->chunk_list
.head
, link
) {
415 if (aligned_bus
>= c
->bus_addr
&&
416 aligned_bus
+ aligned_len
<= c
->bus_addr
+ c
->len
)
420 if (aligned_bus
+ aligned_len
<= c
->bus_addr
)
424 if (aligned_bus
>= c
->bus_addr
+ c
->len
)
427 /* we don't handle the multi-chunk case for now */
434 static struct dma_chunk
*dma_find_chunk_lpar(struct ps3_dma_region
*r
,
435 unsigned long lpar_addr
, unsigned long len
)
438 unsigned long aligned_lpar
= _ALIGN_DOWN(lpar_addr
, 1 << r
->page_size
);
439 unsigned long aligned_len
= _ALIGN_UP(len
+ lpar_addr
- aligned_lpar
,
442 list_for_each_entry(c
, &r
->chunk_list
.head
, link
) {
444 if (c
->lpar_addr
<= aligned_lpar
&&
445 aligned_lpar
< c
->lpar_addr
+ c
->len
) {
446 if (aligned_lpar
+ aligned_len
<= c
->lpar_addr
+ c
->len
)
454 if (aligned_lpar
+ aligned_len
<= c
->lpar_addr
) {
458 if (c
->lpar_addr
+ c
->len
<= aligned_lpar
) {
465 static int dma_sb_free_chunk(struct dma_chunk
*c
)
470 result
= lv1_unmap_device_dma_region(c
->region
->dev
->bus_id
,
471 c
->region
->dev
->dev_id
, c
->bus_addr
, c
->len
);
479 static int dma_ioc0_free_chunk(struct dma_chunk
*c
)
483 unsigned long offset
;
484 struct ps3_dma_region
*r
= c
->region
;
486 DBG("%s:start\n", __func__
);
487 for (iopage
= 0; iopage
< (c
->len
>> r
->page_size
); iopage
++) {
488 offset
= (1 << r
->page_size
) * iopage
;
489 /* put INVALID entry */
490 result
= lv1_put_iopte(0,
491 c
->bus_addr
+ offset
,
492 c
->lpar_addr
+ offset
,
495 DBG("%s: bus=%#lx, lpar=%#lx, ioid=%d\n", __func__
,
496 c
->bus_addr
+ offset
,
497 c
->lpar_addr
+ offset
,
501 DBG("%s:%d: lv1_put_iopte failed: %s\n", __func__
,
502 __LINE__
, ps3_result(result
));
506 DBG("%s:end\n", __func__
);
511 * dma_sb_map_pages - Maps dma pages into the io controller bus address space.
512 * @r: Pointer to a struct ps3_dma_region.
513 * @phys_addr: Starting physical address of the area to map.
514 * @len: Length in bytes of the area to map.
515 * c_out: A pointer to receive an allocated struct dma_chunk for this area.
517 * This is the lowest level dma mapping routine, and is the one that will
518 * make the HV call to add the pages into the io controller address space.
521 static int dma_sb_map_pages(struct ps3_dma_region
*r
, unsigned long phys_addr
,
522 unsigned long len
, struct dma_chunk
**c_out
, u64 iopte_flag
)
527 c
= kzalloc(sizeof(*c
), GFP_ATOMIC
);
534 c
->lpar_addr
= ps3_mm_phys_to_lpar(phys_addr
);
535 c
->bus_addr
= dma_sb_lpar_to_bus(r
, c
->lpar_addr
);
538 BUG_ON(iopte_flag
!= 0xf800000000000000UL
);
539 result
= lv1_map_device_dma_region(c
->region
->dev
->bus_id
,
540 c
->region
->dev
->dev_id
, c
->lpar_addr
,
541 c
->bus_addr
, c
->len
, iopte_flag
);
543 DBG("%s:%d: lv1_map_device_dma_region failed: %s\n",
544 __func__
, __LINE__
, ps3_result(result
));
548 list_add(&c
->link
, &r
->chunk_list
.head
);
557 DBG(" <- %s:%d\n", __func__
, __LINE__
);
561 static int dma_ioc0_map_pages(struct ps3_dma_region
*r
, unsigned long phys_addr
,
562 unsigned long len
, struct dma_chunk
**c_out
,
566 struct dma_chunk
*c
, *last
;
568 unsigned long offset
;
570 DBG(KERN_ERR
"%s: phy=%#lx, lpar%#lx, len=%#lx\n", __func__
,
571 phys_addr
, ps3_mm_phys_to_lpar(phys_addr
), len
);
572 c
= kzalloc(sizeof(*c
), GFP_ATOMIC
);
580 c
->lpar_addr
= ps3_mm_phys_to_lpar(phys_addr
);
581 /* allocate IO address */
582 if (list_empty(&r
->chunk_list
.head
)) {
584 c
->bus_addr
= r
->bus_addr
;
586 /* derive from last bus addr*/
587 last
= list_entry(r
->chunk_list
.head
.next
,
588 struct dma_chunk
, link
);
589 c
->bus_addr
= last
->bus_addr
+ last
->len
;
590 DBG("%s: last bus=%#lx, len=%#lx\n", __func__
,
591 last
->bus_addr
, last
->len
);
594 /* FIXME: check whether length exceeds region size */
596 /* build ioptes for the area */
597 pages
= len
>> r
->page_size
;
598 DBG("%s: pgsize=%#x len=%#lx pages=%#x iopteflag=%#llx\n", __func__
,
599 r
->page_size
, r
->len
, pages
, iopte_flag
);
600 for (iopage
= 0; iopage
< pages
; iopage
++) {
601 offset
= (1 << r
->page_size
) * iopage
;
602 result
= lv1_put_iopte(0,
603 c
->bus_addr
+ offset
,
604 c
->lpar_addr
+ offset
,
608 pr_warn("%s:%d: lv1_put_iopte failed: %s\n",
609 __func__
, __LINE__
, ps3_result(result
));
612 DBG("%s: pg=%d bus=%#lx, lpar=%#lx, ioid=%#x\n", __func__
,
613 iopage
, c
->bus_addr
+ offset
, c
->lpar_addr
+ offset
,
617 /* be sure that last allocated one is inserted at head */
618 list_add(&c
->link
, &r
->chunk_list
.head
);
621 DBG("%s: end\n", __func__
);
625 for (iopage
--; 0 <= iopage
; iopage
--) {
627 c
->bus_addr
+ offset
,
628 c
->lpar_addr
+ offset
,
639 * dma_sb_region_create - Create a device dma region.
640 * @r: Pointer to a struct ps3_dma_region.
642 * This is the lowest level dma region create routine, and is the one that
643 * will make the HV call to create the region.
646 static int dma_sb_region_create(struct ps3_dma_region
*r
)
651 DBG(" -> %s:%d:\n", __func__
, __LINE__
);
655 if (!r
->dev
->bus_id
) {
656 pr_info("%s:%d: %llu:%llu no dma\n", __func__
, __LINE__
,
657 r
->dev
->bus_id
, r
->dev
->dev_id
);
661 DBG("%s:%u: len = 0x%lx, page_size = %u, offset = 0x%lx\n", __func__
,
662 __LINE__
, r
->len
, r
->page_size
, r
->offset
);
665 BUG_ON(!r
->page_size
);
666 BUG_ON(!r
->region_ops
);
668 INIT_LIST_HEAD(&r
->chunk_list
.head
);
669 spin_lock_init(&r
->chunk_list
.lock
);
671 result
= lv1_allocate_device_dma_region(r
->dev
->bus_id
, r
->dev
->dev_id
,
672 roundup_pow_of_two(r
->len
), r
->page_size
, r
->region_type
,
674 r
->bus_addr
= bus_addr
;
677 DBG("%s:%d: lv1_allocate_device_dma_region failed: %s\n",
678 __func__
, __LINE__
, ps3_result(result
));
679 r
->len
= r
->bus_addr
= 0;
685 static int dma_ioc0_region_create(struct ps3_dma_region
*r
)
690 INIT_LIST_HEAD(&r
->chunk_list
.head
);
691 spin_lock_init(&r
->chunk_list
.lock
);
693 result
= lv1_allocate_io_segment(0,
697 r
->bus_addr
= bus_addr
;
699 DBG("%s:%d: lv1_allocate_io_segment failed: %s\n",
700 __func__
, __LINE__
, ps3_result(result
));
701 r
->len
= r
->bus_addr
= 0;
703 DBG("%s: len=%#lx, pg=%d, bus=%#lx\n", __func__
,
704 r
->len
, r
->page_size
, r
->bus_addr
);
709 * dma_region_free - Free a device dma region.
710 * @r: Pointer to a struct ps3_dma_region.
712 * This is the lowest level dma region free routine, and is the one that
713 * will make the HV call to free the region.
716 static int dma_sb_region_free(struct ps3_dma_region
*r
)
720 struct dma_chunk
*tmp
;
724 if (!r
->dev
->bus_id
) {
725 pr_info("%s:%d: %llu:%llu no dma\n", __func__
, __LINE__
,
726 r
->dev
->bus_id
, r
->dev
->dev_id
);
730 list_for_each_entry_safe(c
, tmp
, &r
->chunk_list
.head
, link
) {
732 dma_sb_free_chunk(c
);
735 result
= lv1_free_device_dma_region(r
->dev
->bus_id
, r
->dev
->dev_id
,
739 DBG("%s:%d: lv1_free_device_dma_region failed: %s\n",
740 __func__
, __LINE__
, ps3_result(result
));
747 static int dma_ioc0_region_free(struct ps3_dma_region
*r
)
750 struct dma_chunk
*c
, *n
;
752 DBG("%s: start\n", __func__
);
753 list_for_each_entry_safe(c
, n
, &r
->chunk_list
.head
, link
) {
755 dma_ioc0_free_chunk(c
);
758 result
= lv1_release_io_segment(0, r
->bus_addr
);
761 DBG("%s:%d: lv1_free_device_dma_region failed: %s\n",
762 __func__
, __LINE__
, ps3_result(result
));
765 DBG("%s: end\n", __func__
);
771 * dma_sb_map_area - Map an area of memory into a device dma region.
772 * @r: Pointer to a struct ps3_dma_region.
773 * @virt_addr: Starting virtual address of the area to map.
774 * @len: Length in bytes of the area to map.
775 * @bus_addr: A pointer to return the starting ioc bus address of the area to
778 * This is the common dma mapping routine.
781 static int dma_sb_map_area(struct ps3_dma_region
*r
, unsigned long virt_addr
,
782 unsigned long len
, dma_addr_t
*bus_addr
,
788 unsigned long phys_addr
= is_kernel_addr(virt_addr
) ? __pa(virt_addr
)
790 unsigned long aligned_phys
= _ALIGN_DOWN(phys_addr
, 1 << r
->page_size
);
791 unsigned long aligned_len
= _ALIGN_UP(len
+ phys_addr
- aligned_phys
,
793 *bus_addr
= dma_sb_lpar_to_bus(r
, ps3_mm_phys_to_lpar(phys_addr
));
795 if (!USE_DYNAMIC_DMA
) {
796 unsigned long lpar_addr
= ps3_mm_phys_to_lpar(phys_addr
);
797 DBG(" -> %s:%d\n", __func__
, __LINE__
);
798 DBG("%s:%d virt_addr %lxh\n", __func__
, __LINE__
,
800 DBG("%s:%d phys_addr %lxh\n", __func__
, __LINE__
,
802 DBG("%s:%d lpar_addr %lxh\n", __func__
, __LINE__
,
804 DBG("%s:%d len %lxh\n", __func__
, __LINE__
, len
);
805 DBG("%s:%d bus_addr %llxh (%lxh)\n", __func__
, __LINE__
,
809 spin_lock_irqsave(&r
->chunk_list
.lock
, flags
);
810 c
= dma_find_chunk(r
, *bus_addr
, len
);
813 DBG("%s:%d: reusing mapped chunk", __func__
, __LINE__
);
816 spin_unlock_irqrestore(&r
->chunk_list
.lock
, flags
);
820 result
= dma_sb_map_pages(r
, aligned_phys
, aligned_len
, &c
, iopte_flag
);
824 DBG("%s:%d: dma_sb_map_pages failed (%d)\n",
825 __func__
, __LINE__
, result
);
826 spin_unlock_irqrestore(&r
->chunk_list
.lock
, flags
);
832 spin_unlock_irqrestore(&r
->chunk_list
.lock
, flags
);
836 static int dma_ioc0_map_area(struct ps3_dma_region
*r
, unsigned long virt_addr
,
837 unsigned long len
, dma_addr_t
*bus_addr
,
843 unsigned long phys_addr
= is_kernel_addr(virt_addr
) ? __pa(virt_addr
)
845 unsigned long aligned_phys
= _ALIGN_DOWN(phys_addr
, 1 << r
->page_size
);
846 unsigned long aligned_len
= _ALIGN_UP(len
+ phys_addr
- aligned_phys
,
849 DBG(KERN_ERR
"%s: vaddr=%#lx, len=%#lx\n", __func__
,
851 DBG(KERN_ERR
"%s: ph=%#lx a_ph=%#lx a_l=%#lx\n", __func__
,
852 phys_addr
, aligned_phys
, aligned_len
);
854 spin_lock_irqsave(&r
->chunk_list
.lock
, flags
);
855 c
= dma_find_chunk_lpar(r
, ps3_mm_phys_to_lpar(phys_addr
), len
);
860 *bus_addr
= c
->bus_addr
+ phys_addr
- aligned_phys
;
862 spin_unlock_irqrestore(&r
->chunk_list
.lock
, flags
);
866 result
= dma_ioc0_map_pages(r
, aligned_phys
, aligned_len
, &c
,
871 DBG("%s:%d: dma_ioc0_map_pages failed (%d)\n",
872 __func__
, __LINE__
, result
);
873 spin_unlock_irqrestore(&r
->chunk_list
.lock
, flags
);
876 *bus_addr
= c
->bus_addr
+ phys_addr
- aligned_phys
;
877 DBG("%s: va=%#lx pa=%#lx a_pa=%#lx bus=%#llx\n", __func__
,
878 virt_addr
, phys_addr
, aligned_phys
, *bus_addr
);
881 spin_unlock_irqrestore(&r
->chunk_list
.lock
, flags
);
886 * dma_sb_unmap_area - Unmap an area of memory from a device dma region.
887 * @r: Pointer to a struct ps3_dma_region.
888 * @bus_addr: The starting ioc bus address of the area to unmap.
889 * @len: Length in bytes of the area to unmap.
891 * This is the common dma unmap routine.
894 static int dma_sb_unmap_area(struct ps3_dma_region
*r
, dma_addr_t bus_addr
,
900 spin_lock_irqsave(&r
->chunk_list
.lock
, flags
);
901 c
= dma_find_chunk(r
, bus_addr
, len
);
904 unsigned long aligned_bus
= _ALIGN_DOWN(bus_addr
,
906 unsigned long aligned_len
= _ALIGN_UP(len
+ bus_addr
907 - aligned_bus
, 1 << r
->page_size
);
908 DBG("%s:%d: not found: bus_addr %llxh\n",
909 __func__
, __LINE__
, bus_addr
);
910 DBG("%s:%d: not found: len %lxh\n",
911 __func__
, __LINE__
, len
);
912 DBG("%s:%d: not found: aligned_bus %lxh\n",
913 __func__
, __LINE__
, aligned_bus
);
914 DBG("%s:%d: not found: aligned_len %lxh\n",
915 __func__
, __LINE__
, aligned_len
);
921 if (!c
->usage_count
) {
923 dma_sb_free_chunk(c
);
926 spin_unlock_irqrestore(&r
->chunk_list
.lock
, flags
);
930 static int dma_ioc0_unmap_area(struct ps3_dma_region
*r
,
931 dma_addr_t bus_addr
, unsigned long len
)
936 DBG("%s: start a=%#llx l=%#lx\n", __func__
, bus_addr
, len
);
937 spin_lock_irqsave(&r
->chunk_list
.lock
, flags
);
938 c
= dma_find_chunk(r
, bus_addr
, len
);
941 unsigned long aligned_bus
= _ALIGN_DOWN(bus_addr
,
943 unsigned long aligned_len
= _ALIGN_UP(len
+ bus_addr
946 DBG("%s:%d: not found: bus_addr %llxh\n",
947 __func__
, __LINE__
, bus_addr
);
948 DBG("%s:%d: not found: len %lxh\n",
949 __func__
, __LINE__
, len
);
950 DBG("%s:%d: not found: aligned_bus %lxh\n",
951 __func__
, __LINE__
, aligned_bus
);
952 DBG("%s:%d: not found: aligned_len %lxh\n",
953 __func__
, __LINE__
, aligned_len
);
959 if (!c
->usage_count
) {
961 dma_ioc0_free_chunk(c
);
964 spin_unlock_irqrestore(&r
->chunk_list
.lock
, flags
);
965 DBG("%s: end\n", __func__
);
970 * dma_sb_region_create_linear - Setup a linear dma mapping for a device.
971 * @r: Pointer to a struct ps3_dma_region.
973 * This routine creates an HV dma region for the device and maps all available
974 * ram into the io controller bus address space.
977 static int dma_sb_region_create_linear(struct ps3_dma_region
*r
)
980 unsigned long virt_addr
, len
;
983 if (r
->len
> 16*1024*1024) { /* FIXME: need proper fix */
984 /* force 16M dma pages for linear mapping */
985 if (r
->page_size
!= PS3_DMA_16M
) {
986 pr_info("%s:%d: forcing 16M pages for linear map\n",
988 r
->page_size
= PS3_DMA_16M
;
989 r
->len
= _ALIGN_UP(r
->len
, 1 << r
->page_size
);
993 result
= dma_sb_region_create(r
);
996 if (r
->offset
< map
.rm
.size
) {
997 /* Map (part of) 1st RAM chunk */
998 virt_addr
= map
.rm
.base
+ r
->offset
;
999 len
= map
.rm
.size
- r
->offset
;
1002 result
= dma_sb_map_area(r
, virt_addr
, len
, &tmp
,
1003 CBE_IOPTE_PP_W
| CBE_IOPTE_PP_R
| CBE_IOPTE_SO_RW
|
1008 if (r
->offset
+ r
->len
> map
.rm
.size
) {
1009 /* Map (part of) 2nd RAM chunk */
1010 virt_addr
= map
.rm
.size
;
1012 if (r
->offset
>= map
.rm
.size
)
1013 virt_addr
+= r
->offset
- map
.rm
.size
;
1015 len
-= map
.rm
.size
- r
->offset
;
1016 result
= dma_sb_map_area(r
, virt_addr
, len
, &tmp
,
1017 CBE_IOPTE_PP_W
| CBE_IOPTE_PP_R
| CBE_IOPTE_SO_RW
|
1026 * dma_sb_region_free_linear - Free a linear dma mapping for a device.
1027 * @r: Pointer to a struct ps3_dma_region.
1029 * This routine will unmap all mapped areas and free the HV dma region.
1032 static int dma_sb_region_free_linear(struct ps3_dma_region
*r
)
1035 dma_addr_t bus_addr
;
1036 unsigned long len
, lpar_addr
;
1038 if (r
->offset
< map
.rm
.size
) {
1039 /* Unmap (part of) 1st RAM chunk */
1040 lpar_addr
= map
.rm
.base
+ r
->offset
;
1041 len
= map
.rm
.size
- r
->offset
;
1044 bus_addr
= dma_sb_lpar_to_bus(r
, lpar_addr
);
1045 result
= dma_sb_unmap_area(r
, bus_addr
, len
);
1049 if (r
->offset
+ r
->len
> map
.rm
.size
) {
1050 /* Unmap (part of) 2nd RAM chunk */
1051 lpar_addr
= map
.r1
.base
;
1053 if (r
->offset
>= map
.rm
.size
)
1054 lpar_addr
+= r
->offset
- map
.rm
.size
;
1056 len
-= map
.rm
.size
- r
->offset
;
1057 bus_addr
= dma_sb_lpar_to_bus(r
, lpar_addr
);
1058 result
= dma_sb_unmap_area(r
, bus_addr
, len
);
1062 result
= dma_sb_region_free(r
);
1069 * dma_sb_map_area_linear - Map an area of memory into a device dma region.
1070 * @r: Pointer to a struct ps3_dma_region.
1071 * @virt_addr: Starting virtual address of the area to map.
1072 * @len: Length in bytes of the area to map.
1073 * @bus_addr: A pointer to return the starting ioc bus address of the area to
1076 * This routine just returns the corresponding bus address. Actual mapping
1077 * occurs in dma_region_create_linear().
1080 static int dma_sb_map_area_linear(struct ps3_dma_region
*r
,
1081 unsigned long virt_addr
, unsigned long len
, dma_addr_t
*bus_addr
,
1084 unsigned long phys_addr
= is_kernel_addr(virt_addr
) ? __pa(virt_addr
)
1086 *bus_addr
= dma_sb_lpar_to_bus(r
, ps3_mm_phys_to_lpar(phys_addr
));
1091 * dma_unmap_area_linear - Unmap an area of memory from a device dma region.
1092 * @r: Pointer to a struct ps3_dma_region.
1093 * @bus_addr: The starting ioc bus address of the area to unmap.
1094 * @len: Length in bytes of the area to unmap.
1096 * This routine does nothing. Unmapping occurs in dma_sb_region_free_linear().
1099 static int dma_sb_unmap_area_linear(struct ps3_dma_region
*r
,
1100 dma_addr_t bus_addr
, unsigned long len
)
1105 static const struct ps3_dma_region_ops ps3_dma_sb_region_ops
= {
1106 .create
= dma_sb_region_create
,
1107 .free
= dma_sb_region_free
,
1108 .map
= dma_sb_map_area
,
1109 .unmap
= dma_sb_unmap_area
1112 static const struct ps3_dma_region_ops ps3_dma_sb_region_linear_ops
= {
1113 .create
= dma_sb_region_create_linear
,
1114 .free
= dma_sb_region_free_linear
,
1115 .map
= dma_sb_map_area_linear
,
1116 .unmap
= dma_sb_unmap_area_linear
1119 static const struct ps3_dma_region_ops ps3_dma_ioc0_region_ops
= {
1120 .create
= dma_ioc0_region_create
,
1121 .free
= dma_ioc0_region_free
,
1122 .map
= dma_ioc0_map_area
,
1123 .unmap
= dma_ioc0_unmap_area
1126 int ps3_dma_region_init(struct ps3_system_bus_device
*dev
,
1127 struct ps3_dma_region
*r
, enum ps3_dma_page_size page_size
,
1128 enum ps3_dma_region_type region_type
, void *addr
, unsigned long len
)
1130 unsigned long lpar_addr
;
1132 lpar_addr
= addr
? ps3_mm_phys_to_lpar(__pa(addr
)) : 0;
1135 r
->page_size
= page_size
;
1136 r
->region_type
= region_type
;
1137 r
->offset
= lpar_addr
;
1138 if (r
->offset
>= map
.rm
.size
)
1139 r
->offset
-= map
.r1
.offset
;
1140 r
->len
= len
? len
: _ALIGN_UP(map
.total
, 1 << r
->page_size
);
1142 switch (dev
->dev_type
) {
1143 case PS3_DEVICE_TYPE_SB
:
1144 r
->region_ops
= (USE_DYNAMIC_DMA
)
1145 ? &ps3_dma_sb_region_ops
1146 : &ps3_dma_sb_region_linear_ops
;
1148 case PS3_DEVICE_TYPE_IOC0
:
1149 r
->region_ops
= &ps3_dma_ioc0_region_ops
;
1157 EXPORT_SYMBOL(ps3_dma_region_init
);
1159 int ps3_dma_region_create(struct ps3_dma_region
*r
)
1162 BUG_ON(!r
->region_ops
);
1163 BUG_ON(!r
->region_ops
->create
);
1164 return r
->region_ops
->create(r
);
1166 EXPORT_SYMBOL(ps3_dma_region_create
);
1168 int ps3_dma_region_free(struct ps3_dma_region
*r
)
1171 BUG_ON(!r
->region_ops
);
1172 BUG_ON(!r
->region_ops
->free
);
1173 return r
->region_ops
->free(r
);
1175 EXPORT_SYMBOL(ps3_dma_region_free
);
1177 int ps3_dma_map(struct ps3_dma_region
*r
, unsigned long virt_addr
,
1178 unsigned long len
, dma_addr_t
*bus_addr
,
1181 return r
->region_ops
->map(r
, virt_addr
, len
, bus_addr
, iopte_flag
);
1184 int ps3_dma_unmap(struct ps3_dma_region
*r
, dma_addr_t bus_addr
,
1187 return r
->region_ops
->unmap(r
, bus_addr
, len
);
1190 /*============================================================================*/
1191 /* system startup routines */
1192 /*============================================================================*/
1195 * ps3_mm_init - initialize the address space state variables
1198 void __init
ps3_mm_init(void)
1202 DBG(" -> %s:%d\n", __func__
, __LINE__
);
1204 result
= ps3_repository_read_mm_info(&map
.rm
.base
, &map
.rm
.size
,
1208 panic("ps3_repository_read_mm_info() failed");
1210 map
.rm
.offset
= map
.rm
.base
;
1211 map
.vas_id
= map
.htab_size
= 0;
1213 /* this implementation assumes map.rm.base is zero */
1215 BUG_ON(map
.rm
.base
);
1216 BUG_ON(!map
.rm
.size
);
1218 /* Check if we got the highmem region from an earlier boot step */
1220 if (ps3_mm_get_repository_highmem(&map
.r1
)) {
1221 result
= ps3_mm_region_create(&map
.r1
, map
.total
- map
.rm
.size
);
1224 ps3_mm_set_repository_highmem(&map
.r1
);
1227 /* correct map.total for the real total amount of memory we use */
1228 map
.total
= map
.rm
.size
+ map
.r1
.size
;
1231 DBG("%s:%d: No highmem region found\n", __func__
, __LINE__
);
1233 DBG("%s:%d: Adding highmem region: %llxh %llxh\n",
1234 __func__
, __LINE__
, map
.rm
.size
,
1235 map
.total
- map
.rm
.size
);
1236 memblock_add(map
.rm
.size
, map
.total
- map
.rm
.size
);
1239 DBG(" <- %s:%d\n", __func__
, __LINE__
);
1243 * ps3_mm_shutdown - final cleanup of address space
1246 void ps3_mm_shutdown(void)
1248 ps3_mm_region_destroy(&map
.r1
);