4 * Copyright (c) 2010-2015, NVIDIA Corporation.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 #include <linux/dma-mapping.h>
20 #include <linux/err.h>
21 #include <linux/host1x.h>
22 #include <linux/kref.h>
23 #include <linux/module.h>
24 #include <linux/scatterlist.h>
25 #include <linux/slab.h>
26 #include <linux/vmalloc.h>
27 #include <trace/events/host1x.h>
34 struct host1x_job
*host1x_job_alloc(struct host1x_channel
*ch
,
35 u32 num_cmdbufs
, u32 num_relocs
,
38 struct host1x_job
*job
= NULL
;
39 unsigned int num_unpins
= num_cmdbufs
+ num_relocs
;
43 /* Check that we're not going to overflow */
44 total
= sizeof(struct host1x_job
) +
45 (u64
)num_relocs
* sizeof(struct host1x_reloc
) +
46 (u64
)num_unpins
* sizeof(struct host1x_job_unpin_data
) +
47 (u64
)num_waitchks
* sizeof(struct host1x_waitchk
) +
48 (u64
)num_cmdbufs
* sizeof(struct host1x_job_gather
) +
49 (u64
)num_unpins
* sizeof(dma_addr_t
) +
50 (u64
)num_unpins
* sizeof(u32
*);
51 if (total
> ULONG_MAX
)
54 mem
= job
= kzalloc(total
, GFP_KERNEL
);
61 /* Redistribute memory to the structs */
62 mem
+= sizeof(struct host1x_job
);
63 job
->relocarray
= num_relocs
? mem
: NULL
;
64 mem
+= num_relocs
* sizeof(struct host1x_reloc
);
65 job
->unpins
= num_unpins
? mem
: NULL
;
66 mem
+= num_unpins
* sizeof(struct host1x_job_unpin_data
);
67 job
->waitchk
= num_waitchks
? mem
: NULL
;
68 mem
+= num_waitchks
* sizeof(struct host1x_waitchk
);
69 job
->gathers
= num_cmdbufs
? mem
: NULL
;
70 mem
+= num_cmdbufs
* sizeof(struct host1x_job_gather
);
71 job
->addr_phys
= num_unpins
? mem
: NULL
;
73 job
->reloc_addr_phys
= job
->addr_phys
;
74 job
->gather_addr_phys
= &job
->addr_phys
[num_relocs
];
78 EXPORT_SYMBOL(host1x_job_alloc
);
80 struct host1x_job
*host1x_job_get(struct host1x_job
*job
)
85 EXPORT_SYMBOL(host1x_job_get
);
87 static void job_free(struct kref
*ref
)
89 struct host1x_job
*job
= container_of(ref
, struct host1x_job
, ref
);
94 void host1x_job_put(struct host1x_job
*job
)
96 kref_put(&job
->ref
, job_free
);
98 EXPORT_SYMBOL(host1x_job_put
);
100 void host1x_job_add_gather(struct host1x_job
*job
, struct host1x_bo
*bo
,
101 u32 words
, u32 offset
)
103 struct host1x_job_gather
*cur_gather
= &job
->gathers
[job
->num_gathers
];
105 cur_gather
->words
= words
;
107 cur_gather
->offset
= offset
;
110 EXPORT_SYMBOL(host1x_job_add_gather
);
113 * NULL an already satisfied WAIT_SYNCPT host method, by patching its
114 * args in the command stream. The method data is changed to reference
115 * a reserved (never given out or incr) HOST1X_SYNCPT_RESERVED syncpt
116 * with a matching threshold value of 0, so is guaranteed to be popped
119 static void host1x_syncpt_patch_offset(struct host1x_syncpt
*sp
,
120 struct host1x_bo
*h
, u32 offset
)
122 void *patch_addr
= NULL
;
125 patch_addr
= host1x_bo_kmap(h
, offset
>> PAGE_SHIFT
);
127 host1x_syncpt_patch_wait(sp
,
128 patch_addr
+ (offset
& ~PAGE_MASK
));
129 host1x_bo_kunmap(h
, offset
>> PAGE_SHIFT
, patch_addr
);
131 pr_err("Could not map cmdbuf for wait check\n");
135 * Check driver supplied waitchk structs for syncpt thresholds
136 * that have already been satisfied and NULL the comparison (to
137 * avoid a wrap condition in the HW).
139 static int do_waitchks(struct host1x_job
*job
, struct host1x
*host
,
140 struct host1x_bo
*patch
)
144 /* compare syncpt vs wait threshold */
145 for (i
= 0; i
< job
->num_waitchk
; i
++) {
146 struct host1x_waitchk
*wait
= &job
->waitchk
[i
];
147 struct host1x_syncpt
*sp
=
148 host1x_syncpt_get(host
, wait
->syncpt_id
);
150 /* validate syncpt id */
151 if (wait
->syncpt_id
> host1x_syncpt_nb_pts(host
))
154 /* skip all other gathers */
155 if (patch
!= wait
->bo
)
158 trace_host1x_syncpt_wait_check(wait
->bo
, wait
->offset
,
159 wait
->syncpt_id
, wait
->thresh
,
160 host1x_syncpt_read_min(sp
));
162 if (host1x_syncpt_is_expired(sp
, wait
->thresh
)) {
164 "drop WAIT id %u (%s) thresh 0x%x, min 0x%x\n",
165 wait
->syncpt_id
, sp
->name
, wait
->thresh
,
166 host1x_syncpt_read_min(sp
));
168 host1x_syncpt_patch_offset(sp
, patch
, wait
->offset
);
177 static unsigned int pin_job(struct host1x
*host
, struct host1x_job
*job
)
184 for (i
= 0; i
< job
->num_relocs
; i
++) {
185 struct host1x_reloc
*reloc
= &job
->relocarray
[i
];
186 struct sg_table
*sgt
;
187 dma_addr_t phys_addr
;
189 reloc
->target
.bo
= host1x_bo_get(reloc
->target
.bo
);
190 if (!reloc
->target
.bo
) {
195 phys_addr
= host1x_bo_pin(reloc
->target
.bo
, &sgt
);
201 job
->addr_phys
[job
->num_unpins
] = phys_addr
;
202 job
->unpins
[job
->num_unpins
].bo
= reloc
->target
.bo
;
203 job
->unpins
[job
->num_unpins
].sgt
= sgt
;
207 for (i
= 0; i
< job
->num_gathers
; i
++) {
208 struct host1x_job_gather
*g
= &job
->gathers
[i
];
209 size_t gather_size
= 0;
210 struct scatterlist
*sg
;
211 struct sg_table
*sgt
;
212 dma_addr_t phys_addr
;
217 g
->bo
= host1x_bo_get(g
->bo
);
223 phys_addr
= host1x_bo_pin(g
->bo
, &sgt
);
229 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL
) && host
->domain
) {
230 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, j
)
231 gather_size
+= sg
->length
;
232 gather_size
= iova_align(&host
->iova
, gather_size
);
234 shift
= iova_shift(&host
->iova
);
235 alloc
= alloc_iova(&host
->iova
, gather_size
>> shift
,
236 host
->iova_end
>> shift
, true);
242 err
= iommu_map_sg(host
->domain
,
243 iova_dma_addr(&host
->iova
, alloc
),
244 sgt
->sgl
, sgt
->nents
, IOMMU_READ
);
246 __free_iova(&host
->iova
, alloc
);
251 job
->addr_phys
[job
->num_unpins
] =
252 iova_dma_addr(&host
->iova
, alloc
);
253 job
->unpins
[job
->num_unpins
].size
= gather_size
;
255 job
->addr_phys
[job
->num_unpins
] = phys_addr
;
258 job
->gather_addr_phys
[i
] = job
->addr_phys
[job
->num_unpins
];
260 job
->unpins
[job
->num_unpins
].bo
= g
->bo
;
261 job
->unpins
[job
->num_unpins
].sgt
= sgt
;
268 host1x_job_unpin(job
);
272 static int do_relocs(struct host1x_job
*job
, struct host1x_bo
*cmdbuf
)
276 void *cmdbuf_page_addr
= NULL
;
278 /* pin & patch the relocs for one gather */
279 for (i
= 0; i
< job
->num_relocs
; i
++) {
280 struct host1x_reloc
*reloc
= &job
->relocarray
[i
];
281 u32 reloc_addr
= (job
->reloc_addr_phys
[i
] +
282 reloc
->target
.offset
) >> reloc
->shift
;
285 /* skip all other gathers */
286 if (cmdbuf
!= reloc
->cmdbuf
.bo
)
289 if (last_page
!= reloc
->cmdbuf
.offset
>> PAGE_SHIFT
) {
290 if (cmdbuf_page_addr
)
291 host1x_bo_kunmap(cmdbuf
, last_page
,
294 cmdbuf_page_addr
= host1x_bo_kmap(cmdbuf
,
295 reloc
->cmdbuf
.offset
>> PAGE_SHIFT
);
296 last_page
= reloc
->cmdbuf
.offset
>> PAGE_SHIFT
;
298 if (unlikely(!cmdbuf_page_addr
)) {
299 pr_err("Could not map cmdbuf for relocation\n");
304 target
= cmdbuf_page_addr
+ (reloc
->cmdbuf
.offset
& ~PAGE_MASK
);
305 *target
= reloc_addr
;
308 if (cmdbuf_page_addr
)
309 host1x_bo_kunmap(cmdbuf
, last_page
, cmdbuf_page_addr
);
314 static bool check_reloc(struct host1x_reloc
*reloc
, struct host1x_bo
*cmdbuf
,
317 offset
*= sizeof(u32
);
319 if (reloc
->cmdbuf
.bo
!= cmdbuf
|| reloc
->cmdbuf
.offset
!= offset
)
325 struct host1x_firewall
{
326 struct host1x_job
*job
;
329 unsigned int num_relocs
;
330 struct host1x_reloc
*reloc
;
332 struct host1x_bo
*cmdbuf
;
342 static int check_register(struct host1x_firewall
*fw
, unsigned long offset
)
344 if (fw
->job
->is_addr_reg(fw
->dev
, fw
->class, offset
)) {
348 if (!check_reloc(fw
->reloc
, fw
->cmdbuf
, fw
->offset
))
358 static int check_mask(struct host1x_firewall
*fw
)
369 ret
= check_register(fw
, reg
);
383 static int check_incr(struct host1x_firewall
*fw
)
385 u32 count
= fw
->count
;
393 ret
= check_register(fw
, reg
);
406 static int check_nonincr(struct host1x_firewall
*fw
)
408 u32 count
= fw
->count
;
415 ret
= check_register(fw
, fw
->reg
);
427 static int validate(struct host1x_firewall
*fw
, struct host1x_job_gather
*g
)
429 u32
*cmdbuf_base
= (u32
*)fw
->job
->gather_copy_mapped
+
430 (g
->offset
/ sizeof(u32
));
433 if (!fw
->job
->is_addr_reg
)
436 fw
->words
= g
->words
;
440 while (fw
->words
&& !err
) {
441 u32 word
= cmdbuf_base
[fw
->offset
];
442 u32 opcode
= (word
& 0xf0000000) >> 28;
452 fw
->class = word
>> 6 & 0x3ff;
453 fw
->mask
= word
& 0x3f;
454 fw
->reg
= word
>> 16 & 0xfff;
455 err
= check_mask(fw
);
460 fw
->reg
= word
>> 16 & 0xfff;
461 fw
->count
= word
& 0xffff;
462 err
= check_incr(fw
);
468 fw
->reg
= word
>> 16 & 0xfff;
469 fw
->count
= word
& 0xffff;
470 err
= check_nonincr(fw
);
476 fw
->mask
= word
& 0xffff;
477 fw
->reg
= word
>> 16 & 0xfff;
478 err
= check_mask(fw
);
496 static inline int copy_gathers(struct host1x_job
*job
, struct device
*dev
)
498 struct host1x_firewall fw
;
505 fw
.reloc
= job
->relocarray
;
506 fw
.num_relocs
= job
->num_relocs
;
509 for (i
= 0; i
< job
->num_gathers
; i
++) {
510 struct host1x_job_gather
*g
= &job
->gathers
[i
];
512 size
+= g
->words
* sizeof(u32
);
515 job
->gather_copy_mapped
= dma_alloc_wc(dev
, size
, &job
->gather_copy
,
517 if (!job
->gather_copy_mapped
) {
518 job
->gather_copy_mapped
= NULL
;
522 job
->gather_copy_size
= size
;
524 for (i
= 0; i
< job
->num_gathers
; i
++) {
525 struct host1x_job_gather
*g
= &job
->gathers
[i
];
528 /* Copy the gather */
529 gather
= host1x_bo_mmap(g
->bo
);
530 memcpy(job
->gather_copy_mapped
+ offset
, gather
+ g
->offset
,
531 g
->words
* sizeof(u32
));
532 host1x_bo_munmap(g
->bo
, gather
);
534 /* Store the location in the buffer */
535 g
->base
= job
->gather_copy
;
538 /* Validate the job */
539 if (validate(&fw
, g
))
542 offset
+= g
->words
* sizeof(u32
);
545 /* No relocs should remain at this point */
552 int host1x_job_pin(struct host1x_job
*job
, struct device
*dev
)
556 struct host1x
*host
= dev_get_drvdata(dev
->parent
);
557 DECLARE_BITMAP(waitchk_mask
, host1x_syncpt_nb_pts(host
));
559 bitmap_zero(waitchk_mask
, host1x_syncpt_nb_pts(host
));
560 for (i
= 0; i
< job
->num_waitchk
; i
++) {
561 u32 syncpt_id
= job
->waitchk
[i
].syncpt_id
;
563 if (syncpt_id
< host1x_syncpt_nb_pts(host
))
564 set_bit(syncpt_id
, waitchk_mask
);
567 /* get current syncpt values for waitchk */
568 for_each_set_bit(i
, waitchk_mask
, host1x_syncpt_nb_pts(host
))
569 host1x_syncpt_load(host
->syncpt
+ i
);
572 err
= pin_job(host
, job
);
577 for (i
= 0; i
< job
->num_gathers
; i
++) {
578 struct host1x_job_gather
*g
= &job
->gathers
[i
];
580 /* process each gather mem only once */
584 g
->base
= job
->gather_addr_phys
[i
];
586 for (j
= i
+ 1; j
< job
->num_gathers
; j
++) {
587 if (job
->gathers
[j
].bo
== g
->bo
) {
588 job
->gathers
[j
].handled
= true;
589 job
->gathers
[j
].base
= g
->base
;
593 err
= do_relocs(job
, g
->bo
);
597 err
= do_waitchks(job
, host
, g
->bo
);
602 if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL
) && !err
) {
603 err
= copy_gathers(job
, dev
);
605 host1x_job_unpin(job
);
615 EXPORT_SYMBOL(host1x_job_pin
);
617 void host1x_job_unpin(struct host1x_job
*job
)
619 struct host1x
*host
= dev_get_drvdata(job
->channel
->dev
->parent
);
622 for (i
= 0; i
< job
->num_unpins
; i
++) {
623 struct host1x_job_unpin_data
*unpin
= &job
->unpins
[i
];
625 if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL
) && host
->domain
) {
626 iommu_unmap(host
->domain
, job
->addr_phys
[i
],
628 free_iova(&host
->iova
,
629 iova_pfn(&host
->iova
, job
->addr_phys
[i
]));
632 host1x_bo_unpin(unpin
->bo
, unpin
->sgt
);
633 host1x_bo_put(unpin
->bo
);
638 if (job
->gather_copy_size
)
639 dma_free_wc(job
->channel
->dev
, job
->gather_copy_size
,
640 job
->gather_copy_mapped
, job
->gather_copy
);
642 EXPORT_SYMBOL(host1x_job_unpin
);
645 * Debug routine used to dump job entries
647 void host1x_job_dump(struct device
*dev
, struct host1x_job
*job
)
649 dev_dbg(dev
, " SYNCPT_ID %d\n", job
->syncpt_id
);
650 dev_dbg(dev
, " SYNCPT_VAL %d\n", job
->syncpt_end
);
651 dev_dbg(dev
, " FIRST_GET 0x%x\n", job
->first_get
);
652 dev_dbg(dev
, " TIMEOUT %d\n", job
->timeout
);
653 dev_dbg(dev
, " NUM_SLOTS %d\n", job
->num_slots
);
654 dev_dbg(dev
, " NUM_HANDLES %d\n", job
->num_unpins
);