2 * SN Platform GRU Driver
4 * FILE OPERATIONS & DRIVER INITIALIZATION
6 * This file supports the user system call for file open, close, mmap, etc.
7 * This also incudes the driver initialization code.
9 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 #include <linux/module.h>
27 #include <linux/kernel.h>
28 #include <linux/errno.h>
29 #include <linux/slab.h>
32 #include <linux/smp_lock.h>
33 #include <linux/spinlock.h>
34 #include <linux/device.h>
35 #include <linux/miscdevice.h>
36 #include <linux/interrupt.h>
37 #include <linux/proc_fs.h>
38 #include <linux/uaccess.h>
41 #include "grutables.h"
43 #if defined CONFIG_X86_64
44 #include <asm/genapic.h>
46 #define IS_UV() is_uv_system()
47 #elif defined CONFIG_IA64
48 #include <asm/system.h>
49 #include <asm/sn/simulator.h>
50 /* temp support for running on hardware simulator */
51 #define IS_UV() IS_MEDUSA() || ia64_platform_is("uv")
56 #include <asm/uv/uv_hub.h>
57 #include <asm/uv/uv_mmrs.h>
59 struct gru_blade_state
*gru_base
[GRU_MAX_BLADES
] __read_mostly
;
60 unsigned long gru_start_paddr
, gru_end_paddr __read_mostly
;
61 struct gru_stats_s gru_stats
;
63 /* Guaranteed user available resources on each node */
64 static int max_user_cbrs
, max_user_dsr_bytes
;
66 static struct file_operations gru_fops
;
67 static struct miscdevice gru_miscdev
;
73 * Called when unmapping a device mapping. Frees all gru resources
74 * and tables belonging to the vma.
76 static void gru_vma_close(struct vm_area_struct
*vma
)
78 struct gru_vma_data
*vdata
;
79 struct gru_thread_state
*gts
;
80 struct list_head
*entry
, *next
;
82 if (!vma
->vm_private_data
)
85 vdata
= vma
->vm_private_data
;
86 vma
->vm_private_data
= NULL
;
87 gru_dbg(grudev
, "vma %p, file %p, vdata %p\n", vma
, vma
->vm_file
,
89 list_for_each_safe(entry
, next
, &vdata
->vd_head
) {
91 list_entry(entry
, struct gru_thread_state
, ts_next
);
92 list_del(>s
->ts_next
);
93 mutex_lock(>s
->ts_ctxlock
);
95 gru_unload_context(gts
, 0);
96 mutex_unlock(>s
->ts_ctxlock
);
106 * Called when mmaping the device. Initializes the vma with a fault handler
107 * and private data structure necessary to allocate, track, and free the
110 static int gru_file_mmap(struct file
*file
, struct vm_area_struct
*vma
)
112 if ((vma
->vm_flags
& (VM_SHARED
| VM_WRITE
)) != (VM_SHARED
| VM_WRITE
))
115 if (vma
->vm_start
& (GRU_GSEG_PAGESIZE
- 1) ||
116 vma
->vm_end
& (GRU_GSEG_PAGESIZE
- 1))
120 (VM_IO
| VM_DONTCOPY
| VM_LOCKED
| VM_DONTEXPAND
| VM_PFNMAP
|
122 vma
->vm_page_prot
= PAGE_SHARED
;
123 vma
->vm_ops
= &gru_vm_ops
;
125 vma
->vm_private_data
= gru_alloc_vma_data(vma
, 0);
126 if (!vma
->vm_private_data
)
129 gru_dbg(grudev
, "file %p, vaddr 0x%lx, vma %p, vdata %p\n",
130 file
, vma
->vm_start
, vma
, vma
->vm_private_data
);
135 * Create a new GRU context
137 static int gru_create_new_context(unsigned long arg
)
139 struct gru_create_context_req req
;
140 struct vm_area_struct
*vma
;
141 struct gru_vma_data
*vdata
;
145 if (copy_from_user(&req
, (void __user
*)arg
, sizeof(req
)))
148 if (req
.data_segment_bytes
== 0 ||
149 req
.data_segment_bytes
> max_user_dsr_bytes
)
151 if (!req
.control_blocks
|| !req
.maximum_thread_count
||
152 req
.control_blocks
> max_user_cbrs
)
155 if (!(req
.options
& GRU_OPT_MISS_MASK
))
156 req
.options
|= GRU_OPT_MISS_FMM_INTR
;
158 down_write(¤t
->mm
->mmap_sem
);
159 vma
= gru_find_vma(req
.gseg
);
161 vdata
= vma
->vm_private_data
;
162 vdata
->vd_user_options
= req
.options
;
163 vdata
->vd_dsr_au_count
=
164 GRU_DS_BYTES_TO_AU(req
.data_segment_bytes
);
165 vdata
->vd_cbr_au_count
= GRU_CB_COUNT_TO_AU(req
.control_blocks
);
168 up_write(¤t
->mm
->mmap_sem
);
174 * Get GRU configuration info (temp - for emulator testing)
176 static long gru_get_config_info(unsigned long arg
)
178 struct gru_config_info info
;
181 if (num_online_nodes() > 1 &&
182 (uv_node_to_blade_id(1) == uv_node_to_blade_id(0)))
186 info
.cpus
= num_online_cpus();
187 info
.nodes
= num_online_nodes();
188 info
.blades
= info
.nodes
/ nodesperblade
;
189 info
.chiplets
= GRU_CHIPLETS_PER_BLADE
* info
.blades
;
191 if (copy_to_user((void __user
*)arg
, &info
, sizeof(info
)))
197 * Get GRU chiplet status
199 static long gru_get_chiplet_status(unsigned long arg
)
201 struct gru_state
*gru
;
202 struct gru_chiplet_info info
;
204 if (copy_from_user(&info
, (void __user
*)arg
, sizeof(info
)))
208 info
.node
= numa_node_id();
209 if (info
.node
>= num_possible_nodes() ||
210 info
.chiplet
>= GRU_CHIPLETS_PER_HUB
||
211 info
.node
< 0 || info
.chiplet
< 0)
214 info
.blade
= uv_node_to_blade_id(info
.node
);
215 gru
= get_gru(info
.blade
, info
.chiplet
);
217 info
.total_dsr_bytes
= GRU_NUM_DSR_BYTES
;
218 info
.total_cbr
= GRU_NUM_CB
;
219 info
.total_user_dsr_bytes
= GRU_NUM_DSR_BYTES
-
220 gru
->gs_reserved_dsr_bytes
;
221 info
.total_user_cbr
= GRU_NUM_CB
- gru
->gs_reserved_cbrs
;
222 info
.free_user_dsr_bytes
= hweight64(gru
->gs_dsr_map
) *
224 info
.free_user_cbr
= hweight64(gru
->gs_cbr_map
) * GRU_CBR_AU_SIZE
;
226 if (copy_to_user((void __user
*)arg
, &info
, sizeof(info
)))
232 * gru_file_unlocked_ioctl
234 * Called to update file attributes via IOCTL calls.
236 static long gru_file_unlocked_ioctl(struct file
*file
, unsigned int req
,
241 gru_dbg(grudev
, "file %p\n", file
);
244 case GRU_CREATE_CONTEXT
:
245 err
= gru_create_new_context(arg
);
247 case GRU_SET_TASK_SLICE
:
248 err
= gru_set_task_slice(arg
);
250 case GRU_USER_GET_EXCEPTION_DETAIL
:
251 err
= gru_get_exception_detail(arg
);
253 case GRU_USER_UNLOAD_CONTEXT
:
254 err
= gru_user_unload_context(arg
);
256 case GRU_GET_CHIPLET_STATUS
:
257 err
= gru_get_chiplet_status(arg
);
259 case GRU_USER_FLUSH_TLB
:
260 err
= gru_user_flush_tlb(arg
);
262 case GRU_USER_CALL_OS
:
263 err
= gru_handle_user_call_os(arg
);
265 case GRU_GET_CONFIG_INFO
:
266 err
= gru_get_config_info(arg
);
273 * Called at init time to build tables for all GRUs that are present in the
276 static void gru_init_chiplet(struct gru_state
*gru
, unsigned long paddr
,
277 void *vaddr
, int nid
, int bid
, int grunum
)
279 spin_lock_init(&gru
->gs_lock
);
280 spin_lock_init(&gru
->gs_asid_lock
);
281 gru
->gs_gru_base_paddr
= paddr
;
282 gru
->gs_gru_base_vaddr
= vaddr
;
283 gru
->gs_gid
= bid
* GRU_CHIPLETS_PER_BLADE
+ grunum
;
284 gru
->gs_blade
= gru_base
[bid
];
285 gru
->gs_blade_id
= bid
;
286 gru
->gs_cbr_map
= (GRU_CBR_AU
== 64) ? ~0 : (1UL << GRU_CBR_AU
) - 1;
287 gru
->gs_dsr_map
= (1UL << GRU_DSR_AU
) - 1;
288 gru_tgh_flush_init(gru
);
289 gru_dbg(grudev
, "bid %d, nid %d, gru %x, vaddr %p (0x%lx)\n",
290 bid
, nid
, gru
->gs_gid
, gru
->gs_gru_base_vaddr
,
291 gru
->gs_gru_base_paddr
);
292 gru_kservices_init(gru
);
295 static int gru_init_tables(unsigned long gru_base_paddr
, void *gru_base_vaddr
)
297 int pnode
, nid
, bid
, chip
;
298 int cbrs
, dsrbytes
, n
;
299 int order
= get_order(sizeof(struct gru_blade_state
));
301 struct gru_state
*gru
;
305 max_user_cbrs
= GRU_NUM_CB
;
306 max_user_dsr_bytes
= GRU_NUM_DSR_BYTES
;
307 for_each_online_node(nid
) {
308 bid
= uv_node_to_blade_id(nid
);
309 pnode
= uv_node_to_pnode(nid
);
312 page
= alloc_pages_node(nid
, GFP_KERNEL
, order
);
315 gru_base
[bid
] = page_address(page
);
316 memset(gru_base
[bid
], 0, sizeof(struct gru_blade_state
));
317 gru_base
[bid
]->bs_lru_gru
= &gru_base
[bid
]->bs_grus
[0];
318 spin_lock_init(&gru_base
[bid
]->bs_lock
);
322 for (gru
= gru_base
[bid
]->bs_grus
, chip
= 0;
323 chip
< GRU_CHIPLETS_PER_BLADE
;
325 paddr
= gru_chiplet_paddr(gru_base_paddr
, pnode
, chip
);
326 vaddr
= gru_chiplet_vaddr(gru_base_vaddr
, pnode
, chip
);
327 gru_init_chiplet(gru
, paddr
, vaddr
, bid
, nid
, chip
);
328 n
= hweight64(gru
->gs_cbr_map
) * GRU_CBR_AU_SIZE
;
330 n
= hweight64(gru
->gs_dsr_map
) * GRU_DSR_AU_BYTES
;
331 dsrbytes
= max(dsrbytes
, n
);
333 max_user_cbrs
= min(max_user_cbrs
, cbrs
);
334 max_user_dsr_bytes
= min(max_user_dsr_bytes
, dsrbytes
);
340 for (nid
--; nid
>= 0; nid
--)
341 free_pages((unsigned long)gru_base
[nid
], order
);
347 static int get_base_irq(void)
352 #elif defined CONFIG_X86_64
354 static void noop(unsigned int irq
)
358 static struct irq_chip gru_chip
= {
365 static int get_base_irq(void)
367 set_irq_chip(IRQ_GRU
, &gru_chip
);
368 set_irq_chip(IRQ_GRU
+ 1, &gru_chip
);
376 * Called at boot or module load time to initialize the GRUs.
378 static int __init
gru_init(void)
382 void *gru_start_vaddr
;
387 #if defined CONFIG_IA64
388 gru_start_paddr
= 0xd000000000UL
; /* ZZZZZZZZZZZZZZZZZZZ fixme */
390 gru_start_paddr
= uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR
) &
394 gru_start_vaddr
= __va(gru_start_paddr
);
395 gru_end_paddr
= gru_start_paddr
+ MAX_NUMNODES
* GRU_SIZE
;
396 printk(KERN_INFO
"GRU space: 0x%lx - 0x%lx\n",
397 gru_start_paddr
, gru_end_paddr
);
398 irq
= get_base_irq();
399 for (chip
= 0; chip
< GRU_CHIPLETS_PER_BLADE
; chip
++) {
400 ret
= request_irq(irq
+ chip
, gru_intr
, 0, id
, NULL
);
402 printk(KERN_ERR
"%s: request_irq failed\n",
408 ret
= misc_register(&gru_miscdev
);
410 printk(KERN_ERR
"%s: misc_register failed\n",
415 ret
= gru_proc_init();
417 printk(KERN_ERR
"%s: proc init failed\n", GRU_DRIVER_ID_STR
);
421 ret
= gru_init_tables(gru_start_paddr
, gru_start_vaddr
);
423 printk(KERN_ERR
"%s: init tables failed\n", GRU_DRIVER_ID_STR
);
427 printk(KERN_INFO
"%s: v%s\n", GRU_DRIVER_ID_STR
,
428 GRU_DRIVER_VERSION_STR
);
434 misc_deregister(&gru_miscdev
);
436 for (--chip
; chip
>= 0; chip
--)
437 free_irq(irq
+ chip
, NULL
);
442 static void __exit
gru_exit(void)
445 int order
= get_order(sizeof(struct gru_state
) *
446 GRU_CHIPLETS_PER_BLADE
);
451 for (i
= 0; i
< GRU_CHIPLETS_PER_BLADE
; i
++)
452 free_irq(IRQ_GRU
+ i
, NULL
);
454 for (bid
= 0; bid
< GRU_MAX_BLADES
; bid
++)
455 free_pages((unsigned long)gru_base
[bid
], order
);
457 misc_deregister(&gru_miscdev
);
461 static struct file_operations gru_fops
= {
462 .owner
= THIS_MODULE
,
463 .unlocked_ioctl
= gru_file_unlocked_ioctl
,
464 .mmap
= gru_file_mmap
,
467 static struct miscdevice gru_miscdev
= {
468 .minor
= MISC_DYNAMIC_MINOR
,
473 struct vm_operations_struct gru_vm_ops
= {
474 .close
= gru_vma_close
,
478 module_init(gru_init
);
479 module_exit(gru_exit
);
481 module_param(gru_options
, ulong
, 0644);
482 MODULE_PARM_DESC(gru_options
, "Various debug options");
484 MODULE_AUTHOR("Silicon Graphics, Inc.");
485 MODULE_LICENSE("GPL");
486 MODULE_DESCRIPTION(GRU_DRIVER_ID_STR GRU_DRIVER_VERSION_STR
);
487 MODULE_VERSION(GRU_DRIVER_VERSION_STR
);