]>
Commit | Line | Data |
---|---|---|
ad0dfdfd MP |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Copyright (c) 2011-2012, The Linux Foundation. All rights reserved. | |
941943cf PG |
4 | * |
5 | * Description: CoreSight Embedded Trace Buffer driver | |
fdfc0d8a PP |
6 | */ |
7 | ||
f973d88b | 8 | #include <linux/atomic.h> |
fdfc0d8a | 9 | #include <linux/kernel.h> |
fdfc0d8a PP |
10 | #include <linux/init.h> |
11 | #include <linux/types.h> | |
12 | #include <linux/device.h> | |
13 | #include <linux/io.h> | |
14 | #include <linux/err.h> | |
15 | #include <linux/fs.h> | |
16 | #include <linux/miscdevice.h> | |
17 | #include <linux/uaccess.h> | |
18 | #include <linux/slab.h> | |
19 | #include <linux/spinlock.h> | |
1b19f59d | 20 | #include <linux/pm_runtime.h> |
fdfc0d8a PP |
21 | #include <linux/seq_file.h> |
22 | #include <linux/coresight.h> | |
23 | #include <linux/amba/bus.h> | |
267b2c23 | 24 | #include <linux/clk.h> |
2997aa40 MP |
25 | #include <linux/circ_buf.h> |
26 | #include <linux/mm.h> | |
27 | #include <linux/perf_event.h> | |
28 | ||
fdfc0d8a PP |
29 | |
30 | #include "coresight-priv.h" | |
3d6e8935 | 31 | #include "coresight-etm-perf.h" |
fdfc0d8a PP |
32 | |
33 | #define ETB_RAM_DEPTH_REG 0x004 | |
34 | #define ETB_STATUS_REG 0x00c | |
35 | #define ETB_RAM_READ_DATA_REG 0x010 | |
36 | #define ETB_RAM_READ_POINTER 0x014 | |
37 | #define ETB_RAM_WRITE_POINTER 0x018 | |
38 | #define ETB_TRG 0x01c | |
39 | #define ETB_CTL_REG 0x020 | |
40 | #define ETB_RWD_REG 0x024 | |
41 | #define ETB_FFSR 0x300 | |
42 | #define ETB_FFCR 0x304 | |
43 | #define ETB_ITMISCOP0 0xee0 | |
44 | #define ETB_ITTRFLINACK 0xee4 | |
45 | #define ETB_ITTRFLIN 0xee8 | |
46 | #define ETB_ITATBDATA0 0xeeC | |
47 | #define ETB_ITATBCTR2 0xef0 | |
48 | #define ETB_ITATBCTR1 0xef4 | |
49 | #define ETB_ITATBCTR0 0xef8 | |
50 | ||
51 | /* register description */ | |
52 | /* STS - 0x00C */ | |
53 | #define ETB_STATUS_RAM_FULL BIT(0) | |
54 | /* CTL - 0x020 */ | |
55 | #define ETB_CTL_CAPT_EN BIT(0) | |
56 | /* FFCR - 0x304 */ | |
57 | #define ETB_FFCR_EN_FTC BIT(0) | |
58 | #define ETB_FFCR_FON_MAN BIT(6) | |
59 | #define ETB_FFCR_STOP_FI BIT(12) | |
60 | #define ETB_FFCR_STOP_TRIGGER BIT(13) | |
61 | ||
62 | #define ETB_FFCR_BIT 6 | |
63 | #define ETB_FFSR_BIT 1 | |
64 | #define ETB_FRAME_SIZE_WORDS 4 | |
65 | ||
0f5f9b6b SP |
66 | DEFINE_CORESIGHT_DEVLIST(etb_devs, "etb"); |
67 | ||
fdfc0d8a PP |
68 | /** |
69 | * struct etb_drvdata - specifics associated to an ETB component | |
70 | * @base: memory mapped base address for this component. | |
267b2c23 | 71 | * @atclk: optional clock for the core parts of the ETB. |
fdfc0d8a PP |
72 | * @csdev: component vitals needed by the framework. |
73 | * @miscdev: specifics to handle "/dev/xyz.etb" entry. | |
fdfc0d8a | 74 | * @spinlock: only one at a time pls. |
27b10da8 | 75 | * @reading: synchronise user space access to etb buffer. |
75d7dbd3 MP |
76 | * @pid: Process ID of the process being monitored by the session |
77 | * that is using this component. | |
fdfc0d8a | 78 | * @buf: area of memory where ETB buffer content gets sent. |
d43b8ec5 | 79 | * @mode: this ETB is being used. |
fdfc0d8a | 80 | * @buffer_depth: size of @buf. |
fdfc0d8a PP |
81 | * @trigger_cntr: amount of words to store after a trigger. |
82 | */ | |
83 | struct etb_drvdata { | |
84 | void __iomem *base; | |
267b2c23 | 85 | struct clk *atclk; |
fdfc0d8a PP |
86 | struct coresight_device *csdev; |
87 | struct miscdevice miscdev; | |
fdfc0d8a | 88 | spinlock_t spinlock; |
27b10da8 | 89 | local_t reading; |
75d7dbd3 | 90 | pid_t pid; |
fdfc0d8a | 91 | u8 *buf; |
d43b8ec5 | 92 | u32 mode; |
fdfc0d8a | 93 | u32 buffer_depth; |
fdfc0d8a PP |
94 | u32 trigger_cntr; |
95 | }; | |
96 | ||
3d6e8935 SP |
97 | static int etb_set_buffer(struct coresight_device *csdev, |
98 | struct perf_output_handle *handle); | |
99 | ||
850ccfe3 | 100 | static inline unsigned int etb_get_buffer_depth(struct etb_drvdata *drvdata) |
fdfc0d8a | 101 | { |
850ccfe3 | 102 | return readl_relaxed(drvdata->base + ETB_RAM_DEPTH_REG); |
fdfc0d8a PP |
103 | } |
104 | ||
62563e84 | 105 | static void __etb_enable_hw(struct etb_drvdata *drvdata) |
fdfc0d8a PP |
106 | { |
107 | int i; | |
108 | u32 depth; | |
109 | ||
110 | CS_UNLOCK(drvdata->base); | |
111 | ||
112 | depth = drvdata->buffer_depth; | |
113 | /* reset write RAM pointer address */ | |
114 | writel_relaxed(0x0, drvdata->base + ETB_RAM_WRITE_POINTER); | |
115 | /* clear entire RAM buffer */ | |
116 | for (i = 0; i < depth; i++) | |
117 | writel_relaxed(0x0, drvdata->base + ETB_RWD_REG); | |
118 | ||
119 | /* reset write RAM pointer address */ | |
120 | writel_relaxed(0x0, drvdata->base + ETB_RAM_WRITE_POINTER); | |
121 | /* reset read RAM pointer address */ | |
122 | writel_relaxed(0x0, drvdata->base + ETB_RAM_READ_POINTER); | |
123 | ||
124 | writel_relaxed(drvdata->trigger_cntr, drvdata->base + ETB_TRG); | |
125 | writel_relaxed(ETB_FFCR_EN_FTC | ETB_FFCR_STOP_TRIGGER, | |
126 | drvdata->base + ETB_FFCR); | |
127 | /* ETB trace capture enable */ | |
128 | writel_relaxed(ETB_CTL_CAPT_EN, drvdata->base + ETB_CTL_REG); | |
129 | ||
130 | CS_LOCK(drvdata->base); | |
131 | } | |
132 | ||
62563e84 SP |
133 | static int etb_enable_hw(struct etb_drvdata *drvdata) |
134 | { | |
acaf5a06 MP |
135 | int rc = coresight_claim_device(drvdata->base); |
136 | ||
137 | if (rc) | |
138 | return rc; | |
139 | ||
62563e84 SP |
140 | __etb_enable_hw(drvdata); |
141 | return 0; | |
142 | } | |
143 | ||
d4989fe8 | 144 | static int etb_enable_sysfs(struct coresight_device *csdev) |
fdfc0d8a | 145 | { |
3d6e8935 | 146 | int ret = 0; |
fdfc0d8a | 147 | unsigned long flags; |
e827d455 MP |
148 | struct etb_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); |
149 | ||
d43b8ec5 | 150 | spin_lock_irqsave(&drvdata->spinlock, flags); |
3d6e8935 | 151 | |
d4989fe8 | 152 | /* Don't messup with perf sessions. */ |
d43b8ec5 MP |
153 | if (drvdata->mode == CS_MODE_PERF) { |
154 | ret = -EBUSY; | |
155 | goto out; | |
156 | } | |
e827d455 | 157 | |
f973d88b MP |
158 | if (drvdata->mode == CS_MODE_DISABLED) { |
159 | ret = etb_enable_hw(drvdata); | |
160 | if (ret) | |
161 | goto out; | |
987d1e8d | 162 | |
62563e84 | 163 | drvdata->mode = CS_MODE_SYSFS; |
f973d88b | 164 | } |
d4989fe8 | 165 | |
f973d88b | 166 | atomic_inc(csdev->refcnt); |
d4989fe8 MP |
167 | out: |
168 | spin_unlock_irqrestore(&drvdata->spinlock, flags); | |
169 | return ret; | |
170 | } | |
171 | ||
172 | static int etb_enable_perf(struct coresight_device *csdev, void *data) | |
173 | { | |
174 | int ret = 0; | |
75d7dbd3 | 175 | pid_t pid; |
d4989fe8 MP |
176 | unsigned long flags; |
177 | struct etb_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); | |
75d7dbd3 | 178 | struct perf_output_handle *handle = data; |
22b2beaa | 179 | struct cs_buffers *buf = etm_perf_sink_config(handle); |
d4989fe8 MP |
180 | |
181 | spin_lock_irqsave(&drvdata->spinlock, flags); | |
182 | ||
75d7dbd3 MP |
183 | /* No need to continue if the component is already in used by sysFS. */ |
184 | if (drvdata->mode == CS_MODE_SYSFS) { | |
185 | ret = -EBUSY; | |
186 | goto out; | |
187 | } | |
188 | ||
189 | /* Get a handle on the pid of the process to monitor */ | |
22b2beaa | 190 | pid = buf->pid; |
75d7dbd3 MP |
191 | |
192 | if (drvdata->pid != -1 && drvdata->pid != pid) { | |
d4989fe8 | 193 | ret = -EBUSY; |
e827d455 | 194 | goto out; |
d4989fe8 | 195 | } |
fdfc0d8a | 196 | |
75d7dbd3 MP |
197 | /* |
198 | * No HW configuration is needed if the sink is already in | |
199 | * use for this session. | |
200 | */ | |
201 | if (drvdata->pid == pid) { | |
202 | atomic_inc(csdev->refcnt); | |
203 | goto out; | |
204 | } | |
205 | ||
d43b8ec5 MP |
206 | /* |
207 | * We don't have an internal state to clean up if we fail to setup | |
208 | * the perf buffer. So we can perform the step before we turn the | |
209 | * ETB on and leave without cleaning up. | |
210 | */ | |
75d7dbd3 | 211 | ret = etb_set_buffer(csdev, handle); |
d4989fe8 MP |
212 | if (ret) |
213 | goto out; | |
d43b8ec5 | 214 | |
62563e84 | 215 | ret = etb_enable_hw(drvdata); |
f973d88b | 216 | if (!ret) { |
75d7dbd3 MP |
217 | /* Associate with monitored process. */ |
218 | drvdata->pid = pid; | |
62563e84 | 219 | drvdata->mode = CS_MODE_PERF; |
f973d88b MP |
220 | atomic_inc(csdev->refcnt); |
221 | } | |
fdfc0d8a | 222 | |
e827d455 | 223 | out: |
d43b8ec5 | 224 | spin_unlock_irqrestore(&drvdata->spinlock, flags); |
3d6e8935 | 225 | return ret; |
fdfc0d8a PP |
226 | } |
227 | ||
d4989fe8 MP |
228 | static int etb_enable(struct coresight_device *csdev, u32 mode, void *data) |
229 | { | |
230 | int ret; | |
d4989fe8 MP |
231 | |
232 | switch (mode) { | |
233 | case CS_MODE_SYSFS: | |
234 | ret = etb_enable_sysfs(csdev); | |
235 | break; | |
236 | case CS_MODE_PERF: | |
237 | ret = etb_enable_perf(csdev, data); | |
238 | break; | |
239 | default: | |
240 | ret = -EINVAL; | |
241 | break; | |
242 | } | |
243 | ||
244 | if (ret) | |
245 | return ret; | |
246 | ||
7f84712a | 247 | dev_dbg(&csdev->dev, "ETB enabled\n"); |
d4989fe8 MP |
248 | return 0; |
249 | } | |
250 | ||
acaf5a06 | 251 | static void __etb_disable_hw(struct etb_drvdata *drvdata) |
fdfc0d8a PP |
252 | { |
253 | u32 ffcr; | |
7f84712a | 254 | struct device *dev = &drvdata->csdev->dev; |
fdfc0d8a PP |
255 | |
256 | CS_UNLOCK(drvdata->base); | |
257 | ||
258 | ffcr = readl_relaxed(drvdata->base + ETB_FFCR); | |
259 | /* stop formatter when a stop has completed */ | |
260 | ffcr |= ETB_FFCR_STOP_FI; | |
261 | writel_relaxed(ffcr, drvdata->base + ETB_FFCR); | |
262 | /* manually generate a flush of the system */ | |
263 | ffcr |= ETB_FFCR_FON_MAN; | |
264 | writel_relaxed(ffcr, drvdata->base + ETB_FFCR); | |
265 | ||
266 | if (coresight_timeout(drvdata->base, ETB_FFCR, ETB_FFCR_BIT, 0)) { | |
7f84712a | 267 | dev_err(dev, |
67337e8d | 268 | "timeout while waiting for completion of Manual Flush\n"); |
fdfc0d8a PP |
269 | } |
270 | ||
271 | /* disable trace capture */ | |
272 | writel_relaxed(0x0, drvdata->base + ETB_CTL_REG); | |
273 | ||
274 | if (coresight_timeout(drvdata->base, ETB_FFSR, ETB_FFSR_BIT, 1)) { | |
7f84712a | 275 | dev_err(dev, |
67337e8d | 276 | "timeout while waiting for Formatter to Stop\n"); |
fdfc0d8a PP |
277 | } |
278 | ||
279 | CS_LOCK(drvdata->base); | |
280 | } | |
281 | ||
282 | static void etb_dump_hw(struct etb_drvdata *drvdata) | |
283 | { | |
0c3fc4d5 | 284 | bool lost = false; |
fdfc0d8a PP |
285 | int i; |
286 | u8 *buf_ptr; | |
287 | u32 read_data, depth; | |
288 | u32 read_ptr, write_ptr; | |
289 | u32 frame_off, frame_endoff; | |
7f84712a | 290 | struct device *dev = &drvdata->csdev->dev; |
fdfc0d8a PP |
291 | |
292 | CS_UNLOCK(drvdata->base); | |
293 | ||
294 | read_ptr = readl_relaxed(drvdata->base + ETB_RAM_READ_POINTER); | |
295 | write_ptr = readl_relaxed(drvdata->base + ETB_RAM_WRITE_POINTER); | |
296 | ||
297 | frame_off = write_ptr % ETB_FRAME_SIZE_WORDS; | |
298 | frame_endoff = ETB_FRAME_SIZE_WORDS - frame_off; | |
299 | if (frame_off) { | |
7f84712a | 300 | dev_err(dev, |
fdfc0d8a PP |
301 | "write_ptr: %lu not aligned to formatter frame size\n", |
302 | (unsigned long)write_ptr); | |
7f84712a | 303 | dev_err(dev, "frameoff: %lu, frame_endoff: %lu\n", |
fdfc0d8a PP |
304 | (unsigned long)frame_off, (unsigned long)frame_endoff); |
305 | write_ptr += frame_endoff; | |
306 | } | |
307 | ||
308 | if ((readl_relaxed(drvdata->base + ETB_STATUS_REG) | |
0c3fc4d5 | 309 | & ETB_STATUS_RAM_FULL) == 0) { |
fdfc0d8a | 310 | writel_relaxed(0x0, drvdata->base + ETB_RAM_READ_POINTER); |
0c3fc4d5 | 311 | } else { |
fdfc0d8a | 312 | writel_relaxed(write_ptr, drvdata->base + ETB_RAM_READ_POINTER); |
0c3fc4d5 MP |
313 | lost = true; |
314 | } | |
fdfc0d8a PP |
315 | |
316 | depth = drvdata->buffer_depth; | |
317 | buf_ptr = drvdata->buf; | |
318 | for (i = 0; i < depth; i++) { | |
319 | read_data = readl_relaxed(drvdata->base + | |
320 | ETB_RAM_READ_DATA_REG); | |
4f871a9f MP |
321 | *(u32 *)buf_ptr = read_data; |
322 | buf_ptr += 4; | |
fdfc0d8a PP |
323 | } |
324 | ||
6f755e85 SP |
325 | if (lost) |
326 | coresight_insert_barrier_packet(drvdata->buf); | |
327 | ||
fdfc0d8a PP |
328 | if (frame_off) { |
329 | buf_ptr -= (frame_endoff * 4); | |
330 | for (i = 0; i < frame_endoff; i++) { | |
331 | *buf_ptr++ = 0x0; | |
332 | *buf_ptr++ = 0x0; | |
333 | *buf_ptr++ = 0x0; | |
334 | *buf_ptr++ = 0x0; | |
335 | } | |
336 | } | |
337 | ||
338 | writel_relaxed(read_ptr, drvdata->base + ETB_RAM_READ_POINTER); | |
339 | ||
340 | CS_LOCK(drvdata->base); | |
341 | } | |
342 | ||
acaf5a06 MP |
343 | static void etb_disable_hw(struct etb_drvdata *drvdata) |
344 | { | |
345 | __etb_disable_hw(drvdata); | |
346 | etb_dump_hw(drvdata); | |
347 | coresight_disclaim_device(drvdata->base); | |
348 | } | |
349 | ||
6c817a95 | 350 | static int etb_disable(struct coresight_device *csdev) |
fdfc0d8a PP |
351 | { |
352 | struct etb_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); | |
353 | unsigned long flags; | |
354 | ||
355 | spin_lock_irqsave(&drvdata->spinlock, flags); | |
fdfc0d8a | 356 | |
f973d88b MP |
357 | if (atomic_dec_return(csdev->refcnt)) { |
358 | spin_unlock_irqrestore(&drvdata->spinlock, flags); | |
359 | return -EBUSY; | |
360 | } | |
361 | ||
12dfc9e0 MP |
362 | /* Complain if we (somehow) got out of sync */ |
363 | WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED); | |
364 | etb_disable_hw(drvdata); | |
75d7dbd3 MP |
365 | /* Dissociate from monitored process. */ |
366 | drvdata->pid = -1; | |
12dfc9e0 | 367 | drvdata->mode = CS_MODE_DISABLED; |
d43b8ec5 | 368 | spin_unlock_irqrestore(&drvdata->spinlock, flags); |
e827d455 | 369 | |
7f84712a | 370 | dev_dbg(&csdev->dev, "ETB disabled\n"); |
6c817a95 | 371 | return 0; |
fdfc0d8a PP |
372 | } |
373 | ||
a0f08a6a MP |
374 | static void *etb_alloc_buffer(struct coresight_device *csdev, |
375 | struct perf_event *event, void **pages, | |
376 | int nr_pages, bool overwrite) | |
2997aa40 | 377 | { |
730766ba | 378 | int node; |
2997aa40 MP |
379 | struct cs_buffers *buf; |
380 | ||
730766ba | 381 | node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu); |
2997aa40 MP |
382 | |
383 | buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node); | |
384 | if (!buf) | |
385 | return NULL; | |
386 | ||
22b2beaa | 387 | buf->pid = task_pid_nr(event->owner); |
2997aa40 MP |
388 | buf->snapshot = overwrite; |
389 | buf->nr_pages = nr_pages; | |
390 | buf->data_pages = pages; | |
391 | ||
392 | return buf; | |
393 | } | |
394 | ||
395 | static void etb_free_buffer(void *config) | |
396 | { | |
397 | struct cs_buffers *buf = config; | |
398 | ||
399 | kfree(buf); | |
400 | } | |
401 | ||
402 | static int etb_set_buffer(struct coresight_device *csdev, | |
3d6e8935 | 403 | struct perf_output_handle *handle) |
2997aa40 MP |
404 | { |
405 | int ret = 0; | |
406 | unsigned long head; | |
3d6e8935 SP |
407 | struct cs_buffers *buf = etm_perf_sink_config(handle); |
408 | ||
409 | if (!buf) | |
410 | return -EINVAL; | |
2997aa40 MP |
411 | |
412 | /* wrap head around to the amount of space we have */ | |
413 | head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1); | |
414 | ||
415 | /* find the page to write to */ | |
416 | buf->cur = head / PAGE_SIZE; | |
417 | ||
418 | /* and offset within that page */ | |
419 | buf->offset = head % PAGE_SIZE; | |
420 | ||
421 | local_set(&buf->data_size, 0); | |
422 | ||
423 | return ret; | |
424 | } | |
425 | ||
7ec786ad | 426 | static unsigned long etb_update_buffer(struct coresight_device *csdev, |
2997aa40 MP |
427 | struct perf_output_handle *handle, |
428 | void *sink_config) | |
429 | { | |
cfd9f630 | 430 | bool lost = false; |
2997aa40 MP |
431 | int i, cur; |
432 | u8 *buf_ptr; | |
0c3fc4d5 | 433 | const u32 *barrier; |
2997aa40 | 434 | u32 read_ptr, write_ptr, capacity; |
7ec786ad | 435 | u32 status, read_data; |
75d7dbd3 | 436 | unsigned long offset, to_read = 0, flags; |
2997aa40 MP |
437 | struct cs_buffers *buf = sink_config; |
438 | struct etb_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); | |
439 | ||
440 | if (!buf) | |
7ec786ad | 441 | return 0; |
2997aa40 MP |
442 | |
443 | capacity = drvdata->buffer_depth * ETB_FRAME_SIZE_WORDS; | |
444 | ||
0916447c | 445 | spin_lock_irqsave(&drvdata->spinlock, flags); |
75d7dbd3 MP |
446 | |
447 | /* Don't do anything if another tracer is using this sink */ | |
448 | if (atomic_read(csdev->refcnt) != 1) | |
449 | goto out; | |
450 | ||
acaf5a06 | 451 | __etb_disable_hw(drvdata); |
1655a3d6 | 452 | CS_UNLOCK(drvdata->base); |
2997aa40 MP |
453 | |
454 | /* unit is in words, not bytes */ | |
455 | read_ptr = readl_relaxed(drvdata->base + ETB_RAM_READ_POINTER); | |
456 | write_ptr = readl_relaxed(drvdata->base + ETB_RAM_WRITE_POINTER); | |
457 | ||
458 | /* | |
459 | * Entries should be aligned to the frame size. If they are not | |
63a5c022 | 460 | * go back to the last alignment point to give decoding tools a |
2997aa40 MP |
461 | * chance to fix things. |
462 | */ | |
463 | if (write_ptr % ETB_FRAME_SIZE_WORDS) { | |
7f84712a | 464 | dev_err(&csdev->dev, |
2997aa40 MP |
465 | "write_ptr: %lu not aligned to formatter frame size\n", |
466 | (unsigned long)write_ptr); | |
467 | ||
468 | write_ptr &= ~(ETB_FRAME_SIZE_WORDS - 1); | |
cfd9f630 | 469 | lost = true; |
2997aa40 MP |
470 | } |
471 | ||
472 | /* | |
473 | * Get a hold of the status register and see if a wrap around | |
474 | * has occurred. If so adjust things accordingly. Otherwise | |
475 | * start at the beginning and go until the write pointer has | |
476 | * been reached. | |
477 | */ | |
478 | status = readl_relaxed(drvdata->base + ETB_STATUS_REG); | |
479 | if (status & ETB_STATUS_RAM_FULL) { | |
cfd9f630 | 480 | lost = true; |
2997aa40 MP |
481 | to_read = capacity; |
482 | read_ptr = write_ptr; | |
483 | } else { | |
484 | to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->buffer_depth); | |
485 | to_read *= ETB_FRAME_SIZE_WORDS; | |
486 | } | |
487 | ||
488 | /* | |
489 | * Make sure we don't overwrite data that hasn't been consumed yet. | |
490 | * It is entirely possible that the HW buffer has more data than the | |
491 | * ring buffer can currently handle. If so adjust the start address | |
492 | * to take only the last traces. | |
493 | * | |
494 | * In snapshot mode we are looking to get the latest traces only and as | |
495 | * such, we don't care about not overwriting data that hasn't been | |
496 | * processed by user space. | |
497 | */ | |
498 | if (!buf->snapshot && to_read > handle->size) { | |
499 | u32 mask = ~(ETB_FRAME_SIZE_WORDS - 1); | |
500 | ||
501 | /* The new read pointer must be frame size aligned */ | |
b5af0a26 | 502 | to_read = handle->size & mask; |
2997aa40 MP |
503 | /* |
504 | * Move the RAM read pointer up, keeping in mind that | |
505 | * everything is in frame size units. | |
506 | */ | |
507 | read_ptr = (write_ptr + drvdata->buffer_depth) - | |
508 | to_read / ETB_FRAME_SIZE_WORDS; | |
509 | /* Wrap around if need be*/ | |
bedffda8 MP |
510 | if (read_ptr > (drvdata->buffer_depth - 1)) |
511 | read_ptr -= drvdata->buffer_depth; | |
2997aa40 | 512 | /* let the decoder know we've skipped ahead */ |
cfd9f630 | 513 | lost = true; |
2997aa40 MP |
514 | } |
515 | ||
5aafd9bf MP |
516 | /* |
517 | * Don't set the TRUNCATED flag in snapshot mode because 1) the | |
518 | * captured buffer is expected to be truncated and 2) a full buffer | |
519 | * prevents the event from being re-enabled by the perf core, | |
520 | * resulting in stale data being send to user space. | |
521 | */ | |
522 | if (!buf->snapshot && lost) | |
cfd9f630 MP |
523 | perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); |
524 | ||
2997aa40 MP |
525 | /* finally tell HW where we want to start reading from */ |
526 | writel_relaxed(read_ptr, drvdata->base + ETB_RAM_READ_POINTER); | |
527 | ||
528 | cur = buf->cur; | |
529 | offset = buf->offset; | |
92fc7d81 | 530 | barrier = coresight_barrier_pkt; |
0c3fc4d5 | 531 | |
2997aa40 MP |
532 | for (i = 0; i < to_read; i += 4) { |
533 | buf_ptr = buf->data_pages[cur] + offset; | |
534 | read_data = readl_relaxed(drvdata->base + | |
535 | ETB_RAM_READ_DATA_REG); | |
6f755e85 | 536 | if (lost && i < CORESIGHT_BARRIER_PKT_SIZE) { |
0c3fc4d5 MP |
537 | read_data = *barrier; |
538 | barrier++; | |
539 | } | |
540 | ||
4f871a9f MP |
541 | *(u32 *)buf_ptr = read_data; |
542 | buf_ptr += 4; | |
2997aa40 MP |
543 | |
544 | offset += 4; | |
545 | if (offset >= PAGE_SIZE) { | |
546 | offset = 0; | |
547 | cur++; | |
548 | /* wrap around at the end of the buffer */ | |
549 | cur &= buf->nr_pages - 1; | |
550 | } | |
551 | } | |
552 | ||
553 | /* reset ETB buffer for next run */ | |
554 | writel_relaxed(0x0, drvdata->base + ETB_RAM_READ_POINTER); | |
555 | writel_relaxed(0x0, drvdata->base + ETB_RAM_WRITE_POINTER); | |
556 | ||
557 | /* | |
514e5150 MP |
558 | * In snapshot mode we simply increment the head by the number of byte |
559 | * that were written. User space function cs_etm_find_snapshot() will | |
560 | * figure out how many bytes to get from the AUX buffer based on the | |
561 | * position of the head. | |
2997aa40 | 562 | */ |
514e5150 MP |
563 | if (buf->snapshot) |
564 | handle->head += to_read; | |
565 | ||
acaf5a06 | 566 | __etb_enable_hw(drvdata); |
2997aa40 | 567 | CS_LOCK(drvdata->base); |
75d7dbd3 | 568 | out: |
0916447c | 569 | spin_unlock_irqrestore(&drvdata->spinlock, flags); |
7ec786ad SP |
570 | |
571 | return to_read; | |
2997aa40 MP |
572 | } |
573 | ||
fdfc0d8a PP |
574 | static const struct coresight_ops_sink etb_sink_ops = { |
575 | .enable = etb_enable, | |
576 | .disable = etb_disable, | |
2997aa40 MP |
577 | .alloc_buffer = etb_alloc_buffer, |
578 | .free_buffer = etb_free_buffer, | |
2997aa40 | 579 | .update_buffer = etb_update_buffer, |
fdfc0d8a PP |
580 | }; |
581 | ||
582 | static const struct coresight_ops etb_cs_ops = { | |
583 | .sink_ops = &etb_sink_ops, | |
584 | }; | |
585 | ||
586 | static void etb_dump(struct etb_drvdata *drvdata) | |
587 | { | |
588 | unsigned long flags; | |
589 | ||
590 | spin_lock_irqsave(&drvdata->spinlock, flags); | |
d43b8ec5 | 591 | if (drvdata->mode == CS_MODE_SYSFS) { |
acaf5a06 | 592 | __etb_disable_hw(drvdata); |
fdfc0d8a | 593 | etb_dump_hw(drvdata); |
acaf5a06 | 594 | __etb_enable_hw(drvdata); |
fdfc0d8a PP |
595 | } |
596 | spin_unlock_irqrestore(&drvdata->spinlock, flags); | |
597 | ||
7f84712a | 598 | dev_dbg(&drvdata->csdev->dev, "ETB dumped\n"); |
fdfc0d8a PP |
599 | } |
600 | ||
601 | static int etb_open(struct inode *inode, struct file *file) | |
602 | { | |
603 | struct etb_drvdata *drvdata = container_of(file->private_data, | |
604 | struct etb_drvdata, miscdev); | |
605 | ||
27b10da8 | 606 | if (local_cmpxchg(&drvdata->reading, 0, 1)) |
fdfc0d8a PP |
607 | return -EBUSY; |
608 | ||
7f84712a | 609 | dev_dbg(&drvdata->csdev->dev, "%s: successfully opened\n", __func__); |
fdfc0d8a PP |
610 | return 0; |
611 | } | |
612 | ||
613 | static ssize_t etb_read(struct file *file, char __user *data, | |
614 | size_t len, loff_t *ppos) | |
615 | { | |
616 | u32 depth; | |
617 | struct etb_drvdata *drvdata = container_of(file->private_data, | |
618 | struct etb_drvdata, miscdev); | |
7f84712a | 619 | struct device *dev = &drvdata->csdev->dev; |
fdfc0d8a PP |
620 | |
621 | etb_dump(drvdata); | |
622 | ||
623 | depth = drvdata->buffer_depth; | |
624 | if (*ppos + len > depth * 4) | |
625 | len = depth * 4 - *ppos; | |
626 | ||
627 | if (copy_to_user(data, drvdata->buf + *ppos, len)) { | |
7f84712a SP |
628 | dev_dbg(dev, |
629 | "%s: copy_to_user failed\n", __func__); | |
fdfc0d8a PP |
630 | return -EFAULT; |
631 | } | |
632 | ||
633 | *ppos += len; | |
634 | ||
7f84712a | 635 | dev_dbg(dev, "%s: %zu bytes copied, %d bytes left\n", |
72f641fe | 636 | __func__, len, (int)(depth * 4 - *ppos)); |
fdfc0d8a PP |
637 | return len; |
638 | } | |
639 | ||
640 | static int etb_release(struct inode *inode, struct file *file) | |
641 | { | |
642 | struct etb_drvdata *drvdata = container_of(file->private_data, | |
643 | struct etb_drvdata, miscdev); | |
27b10da8 | 644 | local_set(&drvdata->reading, 0); |
fdfc0d8a | 645 | |
7f84712a | 646 | dev_dbg(&drvdata->csdev->dev, "%s: released\n", __func__); |
fdfc0d8a PP |
647 | return 0; |
648 | } | |
649 | ||
650 | static const struct file_operations etb_fops = { | |
651 | .owner = THIS_MODULE, | |
652 | .open = etb_open, | |
653 | .read = etb_read, | |
654 | .release = etb_release, | |
655 | .llseek = no_llseek, | |
656 | }; | |
657 | ||
47675f6a SP |
658 | #define coresight_etb10_reg(name, offset) \ |
659 | coresight_simple_reg32(struct etb_drvdata, name, offset) | |
660 | ||
661 | coresight_etb10_reg(rdp, ETB_RAM_DEPTH_REG); | |
662 | coresight_etb10_reg(sts, ETB_STATUS_REG); | |
663 | coresight_etb10_reg(rrp, ETB_RAM_READ_POINTER); | |
664 | coresight_etb10_reg(rwp, ETB_RAM_WRITE_POINTER); | |
665 | coresight_etb10_reg(trg, ETB_TRG); | |
666 | coresight_etb10_reg(ctl, ETB_CTL_REG); | |
667 | coresight_etb10_reg(ffsr, ETB_FFSR); | |
668 | coresight_etb10_reg(ffcr, ETB_FFCR); | |
ad352acb MP |
669 | |
670 | static struct attribute *coresight_etb_mgmt_attrs[] = { | |
671 | &dev_attr_rdp.attr, | |
672 | &dev_attr_sts.attr, | |
673 | &dev_attr_rrp.attr, | |
674 | &dev_attr_rwp.attr, | |
675 | &dev_attr_trg.attr, | |
676 | &dev_attr_ctl.attr, | |
677 | &dev_attr_ffsr.attr, | |
678 | &dev_attr_ffcr.attr, | |
679 | NULL, | |
680 | }; | |
fdfc0d8a PP |
681 | |
682 | static ssize_t trigger_cntr_show(struct device *dev, | |
683 | struct device_attribute *attr, char *buf) | |
684 | { | |
685 | struct etb_drvdata *drvdata = dev_get_drvdata(dev->parent); | |
686 | unsigned long val = drvdata->trigger_cntr; | |
687 | ||
688 | return sprintf(buf, "%#lx\n", val); | |
689 | } | |
690 | ||
691 | static ssize_t trigger_cntr_store(struct device *dev, | |
692 | struct device_attribute *attr, | |
693 | const char *buf, size_t size) | |
694 | { | |
695 | int ret; | |
696 | unsigned long val; | |
697 | struct etb_drvdata *drvdata = dev_get_drvdata(dev->parent); | |
698 | ||
699 | ret = kstrtoul(buf, 16, &val); | |
700 | if (ret) | |
701 | return ret; | |
702 | ||
703 | drvdata->trigger_cntr = val; | |
704 | return size; | |
705 | } | |
706 | static DEVICE_ATTR_RW(trigger_cntr); | |
707 | ||
708 | static struct attribute *coresight_etb_attrs[] = { | |
709 | &dev_attr_trigger_cntr.attr, | |
fdfc0d8a PP |
710 | NULL, |
711 | }; | |
ad352acb MP |
712 | |
713 | static const struct attribute_group coresight_etb_group = { | |
714 | .attrs = coresight_etb_attrs, | |
715 | }; | |
716 | ||
717 | static const struct attribute_group coresight_etb_mgmt_group = { | |
718 | .attrs = coresight_etb_mgmt_attrs, | |
719 | .name = "mgmt", | |
720 | }; | |
721 | ||
1c33c65c | 722 | static const struct attribute_group *coresight_etb_groups[] = { |
ad352acb MP |
723 | &coresight_etb_group, |
724 | &coresight_etb_mgmt_group, | |
725 | NULL, | |
726 | }; | |
fdfc0d8a PP |
727 | |
728 | static int etb_probe(struct amba_device *adev, const struct amba_id *id) | |
729 | { | |
730 | int ret; | |
731 | void __iomem *base; | |
732 | struct device *dev = &adev->dev; | |
733 | struct coresight_platform_data *pdata = NULL; | |
734 | struct etb_drvdata *drvdata; | |
735 | struct resource *res = &adev->res; | |
9486295a | 736 | struct coresight_desc desc = { 0 }; |
fdfc0d8a | 737 | |
0f5f9b6b SP |
738 | desc.name = coresight_alloc_device_name(&etb_devs, dev); |
739 | if (!desc.name) | |
740 | return -ENOMEM; | |
741 | ||
fdfc0d8a PP |
742 | drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); |
743 | if (!drvdata) | |
744 | return -ENOMEM; | |
745 | ||
267b2c23 LW |
746 | drvdata->atclk = devm_clk_get(&adev->dev, "atclk"); /* optional */ |
747 | if (!IS_ERR(drvdata->atclk)) { | |
748 | ret = clk_prepare_enable(drvdata->atclk); | |
749 | if (ret) | |
750 | return ret; | |
751 | } | |
fdfc0d8a PP |
752 | dev_set_drvdata(dev, drvdata); |
753 | ||
754 | /* validity for the resource is already checked by the AMBA core */ | |
755 | base = devm_ioremap_resource(dev, res); | |
756 | if (IS_ERR(base)) | |
757 | return PTR_ERR(base); | |
758 | ||
759 | drvdata->base = base; | |
760 | ||
761 | spin_lock_init(&drvdata->spinlock); | |
762 | ||
d786a47d | 763 | drvdata->buffer_depth = etb_get_buffer_depth(drvdata); |
fdfc0d8a | 764 | |
f5da7cb2 | 765 | if (drvdata->buffer_depth & 0x80000000) |
fdfc0d8a PP |
766 | return -EINVAL; |
767 | ||
a86854d0 KC |
768 | drvdata->buf = devm_kcalloc(dev, |
769 | drvdata->buffer_depth, 4, GFP_KERNEL); | |
0f9df80e | 770 | if (!drvdata->buf) |
fdfc0d8a PP |
771 | return -ENOMEM; |
772 | ||
75d7dbd3 MP |
773 | /* This device is not associated with a session */ |
774 | drvdata->pid = -1; | |
775 | ||
af7cfd0f SP |
776 | pdata = coresight_get_platform_data(dev); |
777 | if (IS_ERR(pdata)) | |
778 | return PTR_ERR(pdata); | |
779 | adev->dev.platform_data = pdata; | |
780 | ||
9486295a SP |
781 | desc.type = CORESIGHT_DEV_TYPE_SINK; |
782 | desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_BUFFER; | |
783 | desc.ops = &etb_cs_ops; | |
784 | desc.pdata = pdata; | |
785 | desc.dev = dev; | |
786 | desc.groups = coresight_etb_groups; | |
787 | drvdata->csdev = coresight_register(&desc); | |
fdfc0d8a PP |
788 | if (IS_ERR(drvdata->csdev)) |
789 | return PTR_ERR(drvdata->csdev); | |
790 | ||
2ede79a6 | 791 | drvdata->miscdev.name = desc.name; |
fdfc0d8a PP |
792 | drvdata->miscdev.minor = MISC_DYNAMIC_MINOR; |
793 | drvdata->miscdev.fops = &etb_fops; | |
794 | ret = misc_register(&drvdata->miscdev); | |
795 | if (ret) | |
796 | goto err_misc_register; | |
797 | ||
850ccfe3 | 798 | pm_runtime_put(&adev->dev); |
fdfc0d8a PP |
799 | return 0; |
800 | ||
801 | err_misc_register: | |
802 | coresight_unregister(drvdata->csdev); | |
803 | return ret; | |
804 | } | |
805 | ||
529c4451 KP |
806 | static int __exit etb_remove(struct amba_device *adev) |
807 | { | |
808 | struct etb_drvdata *drvdata = dev_get_drvdata(&adev->dev); | |
809 | ||
810 | /* | |
811 | * Since misc_open() holds a refcount on the f_ops, which is | |
812 | * etb fops in this case, device is there until last file | |
813 | * handler to this device is closed. | |
814 | */ | |
815 | misc_deregister(&drvdata->miscdev); | |
816 | coresight_unregister(drvdata->csdev); | |
817 | ||
818 | return 0; | |
819 | } | |
820 | ||
267b2c23 LW |
821 | #ifdef CONFIG_PM |
822 | static int etb_runtime_suspend(struct device *dev) | |
823 | { | |
824 | struct etb_drvdata *drvdata = dev_get_drvdata(dev); | |
825 | ||
826 | if (drvdata && !IS_ERR(drvdata->atclk)) | |
827 | clk_disable_unprepare(drvdata->atclk); | |
828 | ||
829 | return 0; | |
830 | } | |
831 | ||
832 | static int etb_runtime_resume(struct device *dev) | |
833 | { | |
834 | struct etb_drvdata *drvdata = dev_get_drvdata(dev); | |
835 | ||
836 | if (drvdata && !IS_ERR(drvdata->atclk)) | |
837 | clk_prepare_enable(drvdata->atclk); | |
838 | ||
839 | return 0; | |
840 | } | |
841 | #endif | |
842 | ||
843 | static const struct dev_pm_ops etb_dev_pm_ops = { | |
844 | SET_RUNTIME_PM_OPS(etb_runtime_suspend, etb_runtime_resume, NULL) | |
845 | }; | |
846 | ||
5959f3d7 | 847 | static const struct amba_id etb_ids[] = { |
fdfc0d8a | 848 | { |
0bbb194c SP |
849 | .id = 0x000bb907, |
850 | .mask = 0x000fffff, | |
fdfc0d8a PP |
851 | }, |
852 | { 0, 0}, | |
853 | }; | |
854 | ||
529c4451 KP |
855 | MODULE_DEVICE_TABLE(amba, etb_ids); |
856 | ||
fdfc0d8a PP |
857 | static struct amba_driver etb_driver = { |
858 | .drv = { | |
859 | .name = "coresight-etb10", | |
860 | .owner = THIS_MODULE, | |
267b2c23 | 861 | .pm = &etb_dev_pm_ops, |
b15f0fb6 | 862 | .suppress_bind_attrs = true, |
267b2c23 | 863 | |
fdfc0d8a PP |
864 | }, |
865 | .probe = etb_probe, | |
529c4451 | 866 | .remove = etb_remove, |
fdfc0d8a PP |
867 | .id_table = etb_ids, |
868 | }; | |
529c4451 KP |
869 | |
870 | module_amba_driver(etb_driver); | |
871 | ||
872 | MODULE_AUTHOR("Pratik Patel <pratikp@codeaurora.org>"); | |
873 | MODULE_AUTHOR("Mathieu Poirier <mathieu.poirier@linaro.org>"); | |
874 | MODULE_DESCRIPTION("Arm CoreSight Embedded Trace Buffer driver"); | |
875 | MODULE_LICENSE("GPL v2"); |