2 * Driver for Marvell PPv2 network controller for Armada 375 SoC.
4 * Copyright (C) 2014 Marvell
6 * Marcin Wojtas <mw@semihalf.com>
8 * This file is licensed under the terms of the GNU General Public
9 * License version 2. This program is licensed "as is" without any
10 * warranty of any kind, whether express or implied.
13 #include <linux/kernel.h>
14 #include <linux/netdevice.h>
15 #include <linux/etherdevice.h>
16 #include <linux/platform_device.h>
17 #include <linux/skbuff.h>
18 #include <linux/inetdevice.h>
19 #include <linux/mbus.h>
20 #include <linux/module.h>
21 #include <linux/mfd/syscon.h>
22 #include <linux/interrupt.h>
23 #include <linux/cpumask.h>
25 #include <linux/of_irq.h>
26 #include <linux/of_mdio.h>
27 #include <linux/of_net.h>
28 #include <linux/of_address.h>
29 #include <linux/of_device.h>
30 #include <linux/phy.h>
31 #include <linux/phy/phy.h>
32 #include <linux/clk.h>
33 #include <linux/hrtimer.h>
34 #include <linux/ktime.h>
35 #include <linux/regmap.h>
36 #include <uapi/linux/ppp_defs.h>
42 #define MVPP2_RX_DATA_FIFO_SIZE_REG(port) (0x00 + 4 * (port))
43 #define MVPP2_RX_ATTR_FIFO_SIZE_REG(port) (0x20 + 4 * (port))
44 #define MVPP2_RX_MIN_PKT_SIZE_REG 0x60
45 #define MVPP2_RX_FIFO_INIT_REG 0x64
46 #define MVPP22_TX_FIFO_SIZE_REG(port) (0x8860 + 4 * (port))
48 /* RX DMA Top Registers */
49 #define MVPP2_RX_CTRL_REG(port) (0x140 + 4 * (port))
50 #define MVPP2_RX_LOW_LATENCY_PKT_SIZE(s) (((s) & 0xfff) << 16)
51 #define MVPP2_RX_USE_PSEUDO_FOR_CSUM_MASK BIT(31)
52 #define MVPP2_POOL_BUF_SIZE_REG(pool) (0x180 + 4 * (pool))
53 #define MVPP2_POOL_BUF_SIZE_OFFSET 5
54 #define MVPP2_RXQ_CONFIG_REG(rxq) (0x800 + 4 * (rxq))
55 #define MVPP2_SNOOP_PKT_SIZE_MASK 0x1ff
56 #define MVPP2_SNOOP_BUF_HDR_MASK BIT(9)
57 #define MVPP2_RXQ_POOL_SHORT_OFFS 20
58 #define MVPP21_RXQ_POOL_SHORT_MASK 0x700000
59 #define MVPP22_RXQ_POOL_SHORT_MASK 0xf00000
60 #define MVPP2_RXQ_POOL_LONG_OFFS 24
61 #define MVPP21_RXQ_POOL_LONG_MASK 0x7000000
62 #define MVPP22_RXQ_POOL_LONG_MASK 0xf000000
63 #define MVPP2_RXQ_PACKET_OFFSET_OFFS 28
64 #define MVPP2_RXQ_PACKET_OFFSET_MASK 0x70000000
65 #define MVPP2_RXQ_DISABLE_MASK BIT(31)
67 /* Parser Registers */
68 #define MVPP2_PRS_INIT_LOOKUP_REG 0x1000
69 #define MVPP2_PRS_PORT_LU_MAX 0xf
70 #define MVPP2_PRS_PORT_LU_MASK(port) (0xff << ((port) * 4))
71 #define MVPP2_PRS_PORT_LU_VAL(port, val) ((val) << ((port) * 4))
72 #define MVPP2_PRS_INIT_OFFS_REG(port) (0x1004 + ((port) & 4))
73 #define MVPP2_PRS_INIT_OFF_MASK(port) (0x3f << (((port) % 4) * 8))
74 #define MVPP2_PRS_INIT_OFF_VAL(port, val) ((val) << (((port) % 4) * 8))
75 #define MVPP2_PRS_MAX_LOOP_REG(port) (0x100c + ((port) & 4))
76 #define MVPP2_PRS_MAX_LOOP_MASK(port) (0xff << (((port) % 4) * 8))
77 #define MVPP2_PRS_MAX_LOOP_VAL(port, val) ((val) << (((port) % 4) * 8))
78 #define MVPP2_PRS_TCAM_IDX_REG 0x1100
79 #define MVPP2_PRS_TCAM_DATA_REG(idx) (0x1104 + (idx) * 4)
80 #define MVPP2_PRS_TCAM_INV_MASK BIT(31)
81 #define MVPP2_PRS_SRAM_IDX_REG 0x1200
82 #define MVPP2_PRS_SRAM_DATA_REG(idx) (0x1204 + (idx) * 4)
83 #define MVPP2_PRS_TCAM_CTRL_REG 0x1230
84 #define MVPP2_PRS_TCAM_EN_MASK BIT(0)
87 #define MVPP22_RSS_INDEX 0x1500
88 #define MVPP22_RSS_INDEX_TABLE_ENTRY(idx) (idx)
89 #define MVPP22_RSS_INDEX_TABLE(idx) ((idx) << 8)
90 #define MVPP22_RSS_INDEX_QUEUE(idx) ((idx) << 16)
91 #define MVPP22_RSS_TABLE_ENTRY 0x1508
92 #define MVPP22_RSS_TABLE 0x1510
93 #define MVPP22_RSS_TABLE_POINTER(p) (p)
94 #define MVPP22_RSS_WIDTH 0x150c
96 /* Classifier Registers */
97 #define MVPP2_CLS_MODE_REG 0x1800
98 #define MVPP2_CLS_MODE_ACTIVE_MASK BIT(0)
99 #define MVPP2_CLS_PORT_WAY_REG 0x1810
100 #define MVPP2_CLS_PORT_WAY_MASK(port) (1 << (port))
101 #define MVPP2_CLS_LKP_INDEX_REG 0x1814
102 #define MVPP2_CLS_LKP_INDEX_WAY_OFFS 6
103 #define MVPP2_CLS_LKP_TBL_REG 0x1818
104 #define MVPP2_CLS_LKP_TBL_RXQ_MASK 0xff
105 #define MVPP2_CLS_LKP_TBL_LOOKUP_EN_MASK BIT(25)
106 #define MVPP2_CLS_FLOW_INDEX_REG 0x1820
107 #define MVPP2_CLS_FLOW_TBL0_REG 0x1824
108 #define MVPP2_CLS_FLOW_TBL1_REG 0x1828
109 #define MVPP2_CLS_FLOW_TBL2_REG 0x182c
110 #define MVPP2_CLS_OVERSIZE_RXQ_LOW_REG(port) (0x1980 + ((port) * 4))
111 #define MVPP2_CLS_OVERSIZE_RXQ_LOW_BITS 3
112 #define MVPP2_CLS_OVERSIZE_RXQ_LOW_MASK 0x7
113 #define MVPP2_CLS_SWFWD_P2HQ_REG(port) (0x19b0 + ((port) * 4))
114 #define MVPP2_CLS_SWFWD_PCTRL_REG 0x19d0
115 #define MVPP2_CLS_SWFWD_PCTRL_MASK(port) (1 << (port))
117 /* Descriptor Manager Top Registers */
118 #define MVPP2_RXQ_NUM_REG 0x2040
119 #define MVPP2_RXQ_DESC_ADDR_REG 0x2044
120 #define MVPP22_DESC_ADDR_OFFS 8
121 #define MVPP2_RXQ_DESC_SIZE_REG 0x2048
122 #define MVPP2_RXQ_DESC_SIZE_MASK 0x3ff0
123 #define MVPP2_RXQ_STATUS_UPDATE_REG(rxq) (0x3000 + 4 * (rxq))
124 #define MVPP2_RXQ_NUM_PROCESSED_OFFSET 0
125 #define MVPP2_RXQ_NUM_NEW_OFFSET 16
126 #define MVPP2_RXQ_STATUS_REG(rxq) (0x3400 + 4 * (rxq))
127 #define MVPP2_RXQ_OCCUPIED_MASK 0x3fff
128 #define MVPP2_RXQ_NON_OCCUPIED_OFFSET 16
129 #define MVPP2_RXQ_NON_OCCUPIED_MASK 0x3fff0000
130 #define MVPP2_RXQ_THRESH_REG 0x204c
131 #define MVPP2_OCCUPIED_THRESH_OFFSET 0
132 #define MVPP2_OCCUPIED_THRESH_MASK 0x3fff
133 #define MVPP2_RXQ_INDEX_REG 0x2050
134 #define MVPP2_TXQ_NUM_REG 0x2080
135 #define MVPP2_TXQ_DESC_ADDR_REG 0x2084
136 #define MVPP2_TXQ_DESC_SIZE_REG 0x2088
137 #define MVPP2_TXQ_DESC_SIZE_MASK 0x3ff0
138 #define MVPP2_TXQ_THRESH_REG 0x2094
139 #define MVPP2_TXQ_THRESH_OFFSET 16
140 #define MVPP2_TXQ_THRESH_MASK 0x3fff
141 #define MVPP2_AGGR_TXQ_UPDATE_REG 0x2090
142 #define MVPP2_TXQ_INDEX_REG 0x2098
143 #define MVPP2_TXQ_PREF_BUF_REG 0x209c
144 #define MVPP2_PREF_BUF_PTR(desc) ((desc) & 0xfff)
145 #define MVPP2_PREF_BUF_SIZE_4 (BIT(12) | BIT(13))
146 #define MVPP2_PREF_BUF_SIZE_16 (BIT(12) | BIT(14))
147 #define MVPP2_PREF_BUF_THRESH(val) ((val) << 17)
148 #define MVPP2_TXQ_DRAIN_EN_MASK BIT(31)
149 #define MVPP2_TXQ_PENDING_REG 0x20a0
150 #define MVPP2_TXQ_PENDING_MASK 0x3fff
151 #define MVPP2_TXQ_INT_STATUS_REG 0x20a4
152 #define MVPP2_TXQ_SENT_REG(txq) (0x3c00 + 4 * (txq))
153 #define MVPP2_TRANSMITTED_COUNT_OFFSET 16
154 #define MVPP2_TRANSMITTED_COUNT_MASK 0x3fff0000
155 #define MVPP2_TXQ_RSVD_REQ_REG 0x20b0
156 #define MVPP2_TXQ_RSVD_REQ_Q_OFFSET 16
157 #define MVPP2_TXQ_RSVD_RSLT_REG 0x20b4
158 #define MVPP2_TXQ_RSVD_RSLT_MASK 0x3fff
159 #define MVPP2_TXQ_RSVD_CLR_REG 0x20b8
160 #define MVPP2_TXQ_RSVD_CLR_OFFSET 16
161 #define MVPP2_AGGR_TXQ_DESC_ADDR_REG(cpu) (0x2100 + 4 * (cpu))
162 #define MVPP22_AGGR_TXQ_DESC_ADDR_OFFS 8
163 #define MVPP2_AGGR_TXQ_DESC_SIZE_REG(cpu) (0x2140 + 4 * (cpu))
164 #define MVPP2_AGGR_TXQ_DESC_SIZE_MASK 0x3ff0
165 #define MVPP2_AGGR_TXQ_STATUS_REG(cpu) (0x2180 + 4 * (cpu))
166 #define MVPP2_AGGR_TXQ_PENDING_MASK 0x3fff
167 #define MVPP2_AGGR_TXQ_INDEX_REG(cpu) (0x21c0 + 4 * (cpu))
169 /* MBUS bridge registers */
170 #define MVPP2_WIN_BASE(w) (0x4000 + ((w) << 2))
171 #define MVPP2_WIN_SIZE(w) (0x4020 + ((w) << 2))
172 #define MVPP2_WIN_REMAP(w) (0x4040 + ((w) << 2))
173 #define MVPP2_BASE_ADDR_ENABLE 0x4060
175 /* AXI Bridge Registers */
176 #define MVPP22_AXI_BM_WR_ATTR_REG 0x4100
177 #define MVPP22_AXI_BM_RD_ATTR_REG 0x4104
178 #define MVPP22_AXI_AGGRQ_DESCR_RD_ATTR_REG 0x4110
179 #define MVPP22_AXI_TXQ_DESCR_WR_ATTR_REG 0x4114
180 #define MVPP22_AXI_TXQ_DESCR_RD_ATTR_REG 0x4118
181 #define MVPP22_AXI_RXQ_DESCR_WR_ATTR_REG 0x411c
182 #define MVPP22_AXI_RX_DATA_WR_ATTR_REG 0x4120
183 #define MVPP22_AXI_TX_DATA_RD_ATTR_REG 0x4130
184 #define MVPP22_AXI_RD_NORMAL_CODE_REG 0x4150
185 #define MVPP22_AXI_RD_SNOOP_CODE_REG 0x4154
186 #define MVPP22_AXI_WR_NORMAL_CODE_REG 0x4160
187 #define MVPP22_AXI_WR_SNOOP_CODE_REG 0x4164
189 /* Values for AXI Bridge registers */
190 #define MVPP22_AXI_ATTR_CACHE_OFFS 0
191 #define MVPP22_AXI_ATTR_DOMAIN_OFFS 12
193 #define MVPP22_AXI_CODE_CACHE_OFFS 0
194 #define MVPP22_AXI_CODE_DOMAIN_OFFS 4
196 #define MVPP22_AXI_CODE_CACHE_NON_CACHE 0x3
197 #define MVPP22_AXI_CODE_CACHE_WR_CACHE 0x7
198 #define MVPP22_AXI_CODE_CACHE_RD_CACHE 0xb
200 #define MVPP22_AXI_CODE_DOMAIN_OUTER_DOM 2
201 #define MVPP22_AXI_CODE_DOMAIN_SYSTEM 3
203 /* Interrupt Cause and Mask registers */
204 #define MVPP2_ISR_TX_THRESHOLD_REG(port) (0x5140 + 4 * (port))
205 #define MVPP2_MAX_ISR_TX_THRESHOLD 0xfffff0
207 #define MVPP2_ISR_RX_THRESHOLD_REG(rxq) (0x5200 + 4 * (rxq))
208 #define MVPP2_MAX_ISR_RX_THRESHOLD 0xfffff0
209 #define MVPP21_ISR_RXQ_GROUP_REG(port) (0x5400 + 4 * (port))
211 #define MVPP22_ISR_RXQ_GROUP_INDEX_REG 0x5400
212 #define MVPP22_ISR_RXQ_GROUP_INDEX_SUBGROUP_MASK 0xf
213 #define MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_MASK 0x380
214 #define MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_OFFSET 7
216 #define MVPP22_ISR_RXQ_GROUP_INDEX_SUBGROUP_MASK 0xf
217 #define MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_MASK 0x380
219 #define MVPP22_ISR_RXQ_SUB_GROUP_CONFIG_REG 0x5404
220 #define MVPP22_ISR_RXQ_SUB_GROUP_STARTQ_MASK 0x1f
221 #define MVPP22_ISR_RXQ_SUB_GROUP_SIZE_MASK 0xf00
222 #define MVPP22_ISR_RXQ_SUB_GROUP_SIZE_OFFSET 8
224 #define MVPP2_ISR_ENABLE_REG(port) (0x5420 + 4 * (port))
225 #define MVPP2_ISR_ENABLE_INTERRUPT(mask) ((mask) & 0xffff)
226 #define MVPP2_ISR_DISABLE_INTERRUPT(mask) (((mask) << 16) & 0xffff0000)
227 #define MVPP2_ISR_RX_TX_CAUSE_REG(port) (0x5480 + 4 * (port))
228 #define MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK 0xffff
229 #define MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK 0xff0000
230 #define MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_OFFSET 16
231 #define MVPP2_CAUSE_RX_FIFO_OVERRUN_MASK BIT(24)
232 #define MVPP2_CAUSE_FCS_ERR_MASK BIT(25)
233 #define MVPP2_CAUSE_TX_FIFO_UNDERRUN_MASK BIT(26)
234 #define MVPP2_CAUSE_TX_EXCEPTION_SUM_MASK BIT(29)
235 #define MVPP2_CAUSE_RX_EXCEPTION_SUM_MASK BIT(30)
236 #define MVPP2_CAUSE_MISC_SUM_MASK BIT(31)
237 #define MVPP2_ISR_RX_TX_MASK_REG(port) (0x54a0 + 4 * (port))
238 #define MVPP2_ISR_PON_RX_TX_MASK_REG 0x54bc
239 #define MVPP2_PON_CAUSE_RXQ_OCCUP_DESC_ALL_MASK 0xffff
240 #define MVPP2_PON_CAUSE_TXP_OCCUP_DESC_ALL_MASK 0x3fc00000
241 #define MVPP2_PON_CAUSE_MISC_SUM_MASK BIT(31)
242 #define MVPP2_ISR_MISC_CAUSE_REG 0x55b0
244 /* Buffer Manager registers */
245 #define MVPP2_BM_POOL_BASE_REG(pool) (0x6000 + ((pool) * 4))
246 #define MVPP2_BM_POOL_BASE_ADDR_MASK 0xfffff80
247 #define MVPP2_BM_POOL_SIZE_REG(pool) (0x6040 + ((pool) * 4))
248 #define MVPP2_BM_POOL_SIZE_MASK 0xfff0
249 #define MVPP2_BM_POOL_READ_PTR_REG(pool) (0x6080 + ((pool) * 4))
250 #define MVPP2_BM_POOL_GET_READ_PTR_MASK 0xfff0
251 #define MVPP2_BM_POOL_PTRS_NUM_REG(pool) (0x60c0 + ((pool) * 4))
252 #define MVPP2_BM_POOL_PTRS_NUM_MASK 0xfff0
253 #define MVPP2_BM_BPPI_READ_PTR_REG(pool) (0x6100 + ((pool) * 4))
254 #define MVPP2_BM_BPPI_PTRS_NUM_REG(pool) (0x6140 + ((pool) * 4))
255 #define MVPP2_BM_BPPI_PTR_NUM_MASK 0x7ff
256 #define MVPP2_BM_BPPI_PREFETCH_FULL_MASK BIT(16)
257 #define MVPP2_BM_POOL_CTRL_REG(pool) (0x6200 + ((pool) * 4))
258 #define MVPP2_BM_START_MASK BIT(0)
259 #define MVPP2_BM_STOP_MASK BIT(1)
260 #define MVPP2_BM_STATE_MASK BIT(4)
261 #define MVPP2_BM_LOW_THRESH_OFFS 8
262 #define MVPP2_BM_LOW_THRESH_MASK 0x7f00
263 #define MVPP2_BM_LOW_THRESH_VALUE(val) ((val) << \
264 MVPP2_BM_LOW_THRESH_OFFS)
265 #define MVPP2_BM_HIGH_THRESH_OFFS 16
266 #define MVPP2_BM_HIGH_THRESH_MASK 0x7f0000
267 #define MVPP2_BM_HIGH_THRESH_VALUE(val) ((val) << \
268 MVPP2_BM_HIGH_THRESH_OFFS)
269 #define MVPP2_BM_INTR_CAUSE_REG(pool) (0x6240 + ((pool) * 4))
270 #define MVPP2_BM_RELEASED_DELAY_MASK BIT(0)
271 #define MVPP2_BM_ALLOC_FAILED_MASK BIT(1)
272 #define MVPP2_BM_BPPE_EMPTY_MASK BIT(2)
273 #define MVPP2_BM_BPPE_FULL_MASK BIT(3)
274 #define MVPP2_BM_AVAILABLE_BP_LOW_MASK BIT(4)
275 #define MVPP2_BM_INTR_MASK_REG(pool) (0x6280 + ((pool) * 4))
276 #define MVPP2_BM_PHY_ALLOC_REG(pool) (0x6400 + ((pool) * 4))
277 #define MVPP2_BM_PHY_ALLOC_GRNTD_MASK BIT(0)
278 #define MVPP2_BM_VIRT_ALLOC_REG 0x6440
279 #define MVPP22_BM_ADDR_HIGH_ALLOC 0x6444
280 #define MVPP22_BM_ADDR_HIGH_PHYS_MASK 0xff
281 #define MVPP22_BM_ADDR_HIGH_VIRT_MASK 0xff00
282 #define MVPP22_BM_ADDR_HIGH_VIRT_SHIFT 8
283 #define MVPP2_BM_PHY_RLS_REG(pool) (0x6480 + ((pool) * 4))
284 #define MVPP2_BM_PHY_RLS_MC_BUFF_MASK BIT(0)
285 #define MVPP2_BM_PHY_RLS_PRIO_EN_MASK BIT(1)
286 #define MVPP2_BM_PHY_RLS_GRNTD_MASK BIT(2)
287 #define MVPP2_BM_VIRT_RLS_REG 0x64c0
288 #define MVPP22_BM_ADDR_HIGH_RLS_REG 0x64c4
289 #define MVPP22_BM_ADDR_HIGH_PHYS_RLS_MASK 0xff
290 #define MVPP22_BM_ADDR_HIGH_VIRT_RLS_MASK 0xff00
291 #define MVPP22_BM_ADDR_HIGH_VIRT_RLS_SHIFT 8
293 /* TX Scheduler registers */
294 #define MVPP2_TXP_SCHED_PORT_INDEX_REG 0x8000
295 #define MVPP2_TXP_SCHED_Q_CMD_REG 0x8004
296 #define MVPP2_TXP_SCHED_ENQ_MASK 0xff
297 #define MVPP2_TXP_SCHED_DISQ_OFFSET 8
298 #define MVPP2_TXP_SCHED_CMD_1_REG 0x8010
299 #define MVPP2_TXP_SCHED_PERIOD_REG 0x8018
300 #define MVPP2_TXP_SCHED_MTU_REG 0x801c
301 #define MVPP2_TXP_MTU_MAX 0x7FFFF
302 #define MVPP2_TXP_SCHED_REFILL_REG 0x8020
303 #define MVPP2_TXP_REFILL_TOKENS_ALL_MASK 0x7ffff
304 #define MVPP2_TXP_REFILL_PERIOD_ALL_MASK 0x3ff00000
305 #define MVPP2_TXP_REFILL_PERIOD_MASK(v) ((v) << 20)
306 #define MVPP2_TXP_SCHED_TOKEN_SIZE_REG 0x8024
307 #define MVPP2_TXP_TOKEN_SIZE_MAX 0xffffffff
308 #define MVPP2_TXQ_SCHED_REFILL_REG(q) (0x8040 + ((q) << 2))
309 #define MVPP2_TXQ_REFILL_TOKENS_ALL_MASK 0x7ffff
310 #define MVPP2_TXQ_REFILL_PERIOD_ALL_MASK 0x3ff00000
311 #define MVPP2_TXQ_REFILL_PERIOD_MASK(v) ((v) << 20)
312 #define MVPP2_TXQ_SCHED_TOKEN_SIZE_REG(q) (0x8060 + ((q) << 2))
313 #define MVPP2_TXQ_TOKEN_SIZE_MAX 0x7fffffff
314 #define MVPP2_TXQ_SCHED_TOKEN_CNTR_REG(q) (0x8080 + ((q) << 2))
315 #define MVPP2_TXQ_TOKEN_CNTR_MAX 0xffffffff
317 /* TX general registers */
318 #define MVPP2_TX_SNOOP_REG 0x8800
319 #define MVPP2_TX_PORT_FLUSH_REG 0x8810
320 #define MVPP2_TX_PORT_FLUSH_MASK(port) (1 << (port))
323 #define MVPP2_SRC_ADDR_MIDDLE 0x24
324 #define MVPP2_SRC_ADDR_HIGH 0x28
325 #define MVPP2_PHY_AN_CFG0_REG 0x34
326 #define MVPP2_PHY_AN_STOP_SMI0_MASK BIT(7)
327 #define MVPP2_MNG_EXTENDED_GLOBAL_CTRL_REG 0x305c
328 #define MVPP2_EXT_GLOBAL_CTRL_DEFAULT 0x27
330 /* Per-port registers */
331 #define MVPP2_GMAC_CTRL_0_REG 0x0
332 #define MVPP2_GMAC_PORT_EN_MASK BIT(0)
333 #define MVPP2_GMAC_PORT_TYPE_MASK BIT(1)
334 #define MVPP2_GMAC_MAX_RX_SIZE_OFFS 2
335 #define MVPP2_GMAC_MAX_RX_SIZE_MASK 0x7ffc
336 #define MVPP2_GMAC_MIB_CNTR_EN_MASK BIT(15)
337 #define MVPP2_GMAC_CTRL_1_REG 0x4
338 #define MVPP2_GMAC_PERIODIC_XON_EN_MASK BIT(1)
339 #define MVPP2_GMAC_GMII_LB_EN_MASK BIT(5)
340 #define MVPP2_GMAC_PCS_LB_EN_BIT 6
341 #define MVPP2_GMAC_PCS_LB_EN_MASK BIT(6)
342 #define MVPP2_GMAC_SA_LOW_OFFS 7
343 #define MVPP2_GMAC_CTRL_2_REG 0x8
344 #define MVPP2_GMAC_INBAND_AN_MASK BIT(0)
345 #define MVPP2_GMAC_FLOW_CTRL_MASK GENMASK(2, 1)
346 #define MVPP2_GMAC_PCS_ENABLE_MASK BIT(3)
347 #define MVPP2_GMAC_INTERNAL_CLK_MASK BIT(4)
348 #define MVPP2_GMAC_DISABLE_PADDING BIT(5)
349 #define MVPP2_GMAC_PORT_RESET_MASK BIT(6)
350 #define MVPP2_GMAC_AUTONEG_CONFIG 0xc
351 #define MVPP2_GMAC_FORCE_LINK_DOWN BIT(0)
352 #define MVPP2_GMAC_FORCE_LINK_PASS BIT(1)
353 #define MVPP2_GMAC_IN_BAND_AUTONEG BIT(2)
354 #define MVPP2_GMAC_IN_BAND_AUTONEG_BYPASS BIT(3)
355 #define MVPP2_GMAC_CONFIG_MII_SPEED BIT(5)
356 #define MVPP2_GMAC_CONFIG_GMII_SPEED BIT(6)
357 #define MVPP2_GMAC_AN_SPEED_EN BIT(7)
358 #define MVPP2_GMAC_FC_ADV_EN BIT(9)
359 #define MVPP2_GMAC_FLOW_CTRL_AUTONEG BIT(11)
360 #define MVPP2_GMAC_CONFIG_FULL_DUPLEX BIT(12)
361 #define MVPP2_GMAC_AN_DUPLEX_EN BIT(13)
362 #define MVPP2_GMAC_STATUS0 0x10
363 #define MVPP2_GMAC_STATUS0_LINK_UP BIT(0)
364 #define MVPP2_GMAC_PORT_FIFO_CFG_1_REG 0x1c
365 #define MVPP2_GMAC_TX_FIFO_MIN_TH_OFFS 6
366 #define MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK 0x1fc0
367 #define MVPP2_GMAC_TX_FIFO_MIN_TH_MASK(v) (((v) << 6) & \
368 MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK)
369 #define MVPP22_GMAC_INT_STAT 0x20
370 #define MVPP22_GMAC_INT_STAT_LINK BIT(1)
371 #define MVPP22_GMAC_INT_MASK 0x24
372 #define MVPP22_GMAC_INT_MASK_LINK_STAT BIT(1)
373 #define MVPP22_GMAC_CTRL_4_REG 0x90
374 #define MVPP22_CTRL4_EXT_PIN_GMII_SEL BIT(0)
375 #define MVPP22_CTRL4_DP_CLK_SEL BIT(5)
376 #define MVPP22_CTRL4_SYNC_BYPASS_DIS BIT(6)
377 #define MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE BIT(7)
378 #define MVPP22_GMAC_INT_SUM_MASK 0xa4
379 #define MVPP22_GMAC_INT_SUM_MASK_LINK_STAT BIT(1)
381 /* Per-port XGMAC registers. PPv2.2 only, only for GOP port 0,
382 * relative to port->base.
384 #define MVPP22_XLG_CTRL0_REG 0x100
385 #define MVPP22_XLG_CTRL0_PORT_EN BIT(0)
386 #define MVPP22_XLG_CTRL0_MAC_RESET_DIS BIT(1)
387 #define MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN BIT(7)
388 #define MVPP22_XLG_CTRL0_MIB_CNT_DIS BIT(14)
389 #define MVPP22_XLG_CTRL1_REG 0x104
390 #define MVPP22_XLG_CTRL1_FRAMESIZELIMIT_OFFS 0
391 #define MVPP22_XLG_CTRL1_FRAMESIZELIMIT_MASK 0x1fff
392 #define MVPP22_XLG_STATUS 0x10c
393 #define MVPP22_XLG_STATUS_LINK_UP BIT(0)
394 #define MVPP22_XLG_INT_STAT 0x114
395 #define MVPP22_XLG_INT_STAT_LINK BIT(1)
396 #define MVPP22_XLG_INT_MASK 0x118
397 #define MVPP22_XLG_INT_MASK_LINK BIT(1)
398 #define MVPP22_XLG_CTRL3_REG 0x11c
399 #define MVPP22_XLG_CTRL3_MACMODESELECT_MASK (7 << 13)
400 #define MVPP22_XLG_CTRL3_MACMODESELECT_GMAC (0 << 13)
401 #define MVPP22_XLG_CTRL3_MACMODESELECT_10G (1 << 13)
402 #define MVPP22_XLG_EXT_INT_MASK 0x15c
403 #define MVPP22_XLG_EXT_INT_MASK_XLG BIT(1)
404 #define MVPP22_XLG_EXT_INT_MASK_GIG BIT(2)
405 #define MVPP22_XLG_CTRL4_REG 0x184
406 #define MVPP22_XLG_CTRL4_FWD_FC BIT(5)
407 #define MVPP22_XLG_CTRL4_FWD_PFC BIT(6)
408 #define MVPP22_XLG_CTRL4_MACMODSELECT_GMAC BIT(12)
410 /* SMI registers. PPv2.2 only, relative to priv->iface_base. */
411 #define MVPP22_SMI_MISC_CFG_REG 0x1204
412 #define MVPP22_SMI_POLLING_EN BIT(10)
414 #define MVPP22_GMAC_BASE(port) (0x7000 + (port) * 0x1000 + 0xe00)
416 #define MVPP2_CAUSE_TXQ_SENT_DESC_ALL_MASK 0xff
418 /* Descriptor ring Macros */
419 #define MVPP2_QUEUE_NEXT_DESC(q, index) \
420 (((index) < (q)->last_desc) ? ((index) + 1) : 0)
422 /* XPCS registers. PPv2.2 only */
423 #define MVPP22_MPCS_BASE(port) (0x7000 + (port) * 0x1000)
424 #define MVPP22_MPCS_CTRL 0x14
425 #define MVPP22_MPCS_CTRL_FWD_ERR_CONN BIT(10)
426 #define MVPP22_MPCS_CLK_RESET 0x14c
427 #define MAC_CLK_RESET_SD_TX BIT(0)
428 #define MAC_CLK_RESET_SD_RX BIT(1)
429 #define MAC_CLK_RESET_MAC BIT(2)
430 #define MVPP22_MPCS_CLK_RESET_DIV_RATIO(n) ((n) << 4)
431 #define MVPP22_MPCS_CLK_RESET_DIV_SET BIT(11)
433 /* XPCS registers. PPv2.2 only */
434 #define MVPP22_XPCS_BASE(port) (0x7400 + (port) * 0x1000)
435 #define MVPP22_XPCS_CFG0 0x0
436 #define MVPP22_XPCS_CFG0_PCS_MODE(n) ((n) << 3)
437 #define MVPP22_XPCS_CFG0_ACTIVE_LANE(n) ((n) << 5)
439 /* System controller registers. Accessed through a regmap. */
440 #define GENCONF_SOFT_RESET1 0x1108
441 #define GENCONF_SOFT_RESET1_GOP BIT(6)
442 #define GENCONF_PORT_CTRL0 0x1110
443 #define GENCONF_PORT_CTRL0_BUS_WIDTH_SELECT BIT(1)
444 #define GENCONF_PORT_CTRL0_RX_DATA_SAMPLE BIT(29)
445 #define GENCONF_PORT_CTRL0_CLK_DIV_PHASE_CLR BIT(31)
446 #define GENCONF_PORT_CTRL1 0x1114
447 #define GENCONF_PORT_CTRL1_EN(p) BIT(p)
448 #define GENCONF_PORT_CTRL1_RESET(p) (BIT(p) << 28)
449 #define GENCONF_CTRL0 0x1120
450 #define GENCONF_CTRL0_PORT0_RGMII BIT(0)
451 #define GENCONF_CTRL0_PORT1_RGMII_MII BIT(1)
452 #define GENCONF_CTRL0_PORT1_RGMII BIT(2)
454 /* Various constants */
457 #define MVPP2_TXDONE_COAL_PKTS_THRESH 15
458 #define MVPP2_TXDONE_HRTIMER_PERIOD_NS 1000000UL
459 #define MVPP2_TXDONE_COAL_USEC 1000
460 #define MVPP2_RX_COAL_PKTS 32
461 #define MVPP2_RX_COAL_USEC 100
463 /* The two bytes Marvell header. Either contains a special value used
464 * by Marvell switches when a specific hardware mode is enabled (not
465 * supported by this driver) or is filled automatically by zeroes on
466 * the RX side. Those two bytes being at the front of the Ethernet
467 * header, they allow to have the IP header aligned on a 4 bytes
468 * boundary automatically: the hardware skips those two bytes on its
471 #define MVPP2_MH_SIZE 2
472 #define MVPP2_ETH_TYPE_LEN 2
473 #define MVPP2_PPPOE_HDR_SIZE 8
474 #define MVPP2_VLAN_TAG_LEN 4
476 /* Lbtd 802.3 type */
477 #define MVPP2_IP_LBDT_TYPE 0xfffa
479 #define MVPP2_TX_CSUM_MAX_SIZE 9800
481 /* Timeout constants */
482 #define MVPP2_TX_DISABLE_TIMEOUT_MSEC 1000
483 #define MVPP2_TX_PENDING_TIMEOUT_MSEC 1000
485 #define MVPP2_TX_MTU_MAX 0x7ffff
487 /* Maximum number of T-CONTs of PON port */
488 #define MVPP2_MAX_TCONT 16
490 /* Maximum number of supported ports */
491 #define MVPP2_MAX_PORTS 4
493 /* Maximum number of TXQs used by single port */
494 #define MVPP2_MAX_TXQ 8
496 /* MVPP2_MAX_TSO_SEGS is the maximum number of fragments to allow in the GSO
497 * skb. As we need a maxium of two descriptors per fragments (1 header, 1 data),
498 * multiply this value by two to count the maximum number of skb descs needed.
500 #define MVPP2_MAX_TSO_SEGS 300
501 #define MVPP2_MAX_SKB_DESCS (MVPP2_MAX_TSO_SEGS * 2 + MAX_SKB_FRAGS)
503 /* Dfault number of RXQs in use */
504 #define MVPP2_DEFAULT_RXQ 4
506 /* Max number of Rx descriptors */
507 #define MVPP2_MAX_RXD 128
509 /* Max number of Tx descriptors */
510 #define MVPP2_MAX_TXD 1024
512 /* Amount of Tx descriptors that can be reserved at once by CPU */
513 #define MVPP2_CPU_DESC_CHUNK 64
515 /* Max number of Tx descriptors in each aggregated queue */
516 #define MVPP2_AGGR_TXQ_SIZE 256
518 /* Descriptor aligned size */
519 #define MVPP2_DESC_ALIGNED_SIZE 32
521 /* Descriptor alignment mask */
522 #define MVPP2_TX_DESC_ALIGN (MVPP2_DESC_ALIGNED_SIZE - 1)
524 /* RX FIFO constants */
525 #define MVPP2_RX_FIFO_PORT_DATA_SIZE_32KB 0x8000
526 #define MVPP2_RX_FIFO_PORT_DATA_SIZE_8KB 0x2000
527 #define MVPP2_RX_FIFO_PORT_DATA_SIZE_4KB 0x1000
528 #define MVPP2_RX_FIFO_PORT_ATTR_SIZE_32KB 0x200
529 #define MVPP2_RX_FIFO_PORT_ATTR_SIZE_8KB 0x80
530 #define MVPP2_RX_FIFO_PORT_ATTR_SIZE_4KB 0x40
531 #define MVPP2_RX_FIFO_PORT_MIN_PKT 0x80
533 /* TX FIFO constants */
534 #define MVPP22_TX_FIFO_DATA_SIZE_10KB 0xa
535 #define MVPP22_TX_FIFO_DATA_SIZE_3KB 0x3
537 /* RX buffer constants */
538 #define MVPP2_SKB_SHINFO_SIZE \
539 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
541 #define MVPP2_RX_PKT_SIZE(mtu) \
542 ALIGN((mtu) + MVPP2_MH_SIZE + MVPP2_VLAN_TAG_LEN + \
543 ETH_HLEN + ETH_FCS_LEN, cache_line_size())
545 #define MVPP2_RX_BUF_SIZE(pkt_size) ((pkt_size) + NET_SKB_PAD)
546 #define MVPP2_RX_TOTAL_SIZE(buf_size) ((buf_size) + MVPP2_SKB_SHINFO_SIZE)
547 #define MVPP2_RX_MAX_PKT_SIZE(total_size) \
548 ((total_size) - NET_SKB_PAD - MVPP2_SKB_SHINFO_SIZE)
550 #define MVPP2_BIT_TO_BYTE(bit) ((bit) / 8)
552 /* IPv6 max L3 address size */
553 #define MVPP2_MAX_L3_ADDR_SIZE 16
556 #define MVPP2_F_LOOPBACK BIT(0)
558 /* Marvell tag types */
559 enum mvpp2_tag_type
{
560 MVPP2_TAG_TYPE_NONE
= 0,
561 MVPP2_TAG_TYPE_MH
= 1,
562 MVPP2_TAG_TYPE_DSA
= 2,
563 MVPP2_TAG_TYPE_EDSA
= 3,
564 MVPP2_TAG_TYPE_VLAN
= 4,
565 MVPP2_TAG_TYPE_LAST
= 5
568 /* Parser constants */
569 #define MVPP2_PRS_TCAM_SRAM_SIZE 256
570 #define MVPP2_PRS_TCAM_WORDS 6
571 #define MVPP2_PRS_SRAM_WORDS 4
572 #define MVPP2_PRS_FLOW_ID_SIZE 64
573 #define MVPP2_PRS_FLOW_ID_MASK 0x3f
574 #define MVPP2_PRS_TCAM_ENTRY_INVALID 1
575 #define MVPP2_PRS_TCAM_DSA_TAGGED_BIT BIT(5)
576 #define MVPP2_PRS_IPV4_HEAD 0x40
577 #define MVPP2_PRS_IPV4_HEAD_MASK 0xf0
578 #define MVPP2_PRS_IPV4_MC 0xe0
579 #define MVPP2_PRS_IPV4_MC_MASK 0xf0
580 #define MVPP2_PRS_IPV4_BC_MASK 0xff
581 #define MVPP2_PRS_IPV4_IHL 0x5
582 #define MVPP2_PRS_IPV4_IHL_MASK 0xf
583 #define MVPP2_PRS_IPV6_MC 0xff
584 #define MVPP2_PRS_IPV6_MC_MASK 0xff
585 #define MVPP2_PRS_IPV6_HOP_MASK 0xff
586 #define MVPP2_PRS_TCAM_PROTO_MASK 0xff
587 #define MVPP2_PRS_TCAM_PROTO_MASK_L 0x3f
588 #define MVPP2_PRS_DBL_VLANS_MAX 100
591 * - lookup ID - 4 bits
593 * - additional information - 1 byte
594 * - header data - 8 bytes
595 * The fields are represented by MVPP2_PRS_TCAM_DATA_REG(5)->(0).
597 #define MVPP2_PRS_AI_BITS 8
598 #define MVPP2_PRS_PORT_MASK 0xff
599 #define MVPP2_PRS_LU_MASK 0xf
600 #define MVPP2_PRS_TCAM_DATA_BYTE(offs) \
601 (((offs) - ((offs) % 2)) * 2 + ((offs) % 2))
602 #define MVPP2_PRS_TCAM_DATA_BYTE_EN(offs) \
603 (((offs) * 2) - ((offs) % 2) + 2)
604 #define MVPP2_PRS_TCAM_AI_BYTE 16
605 #define MVPP2_PRS_TCAM_PORT_BYTE 17
606 #define MVPP2_PRS_TCAM_LU_BYTE 20
607 #define MVPP2_PRS_TCAM_EN_OFFS(offs) ((offs) + 2)
608 #define MVPP2_PRS_TCAM_INV_WORD 5
609 /* Tcam entries ID */
610 #define MVPP2_PE_DROP_ALL 0
611 #define MVPP2_PE_FIRST_FREE_TID 1
612 #define MVPP2_PE_LAST_FREE_TID (MVPP2_PRS_TCAM_SRAM_SIZE - 31)
613 #define MVPP2_PE_IP6_EXT_PROTO_UN (MVPP2_PRS_TCAM_SRAM_SIZE - 30)
614 #define MVPP2_PE_MAC_MC_IP6 (MVPP2_PRS_TCAM_SRAM_SIZE - 29)
615 #define MVPP2_PE_IP6_ADDR_UN (MVPP2_PRS_TCAM_SRAM_SIZE - 28)
616 #define MVPP2_PE_IP4_ADDR_UN (MVPP2_PRS_TCAM_SRAM_SIZE - 27)
617 #define MVPP2_PE_LAST_DEFAULT_FLOW (MVPP2_PRS_TCAM_SRAM_SIZE - 26)
618 #define MVPP2_PE_FIRST_DEFAULT_FLOW (MVPP2_PRS_TCAM_SRAM_SIZE - 19)
619 #define MVPP2_PE_EDSA_TAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 18)
620 #define MVPP2_PE_EDSA_UNTAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 17)
621 #define MVPP2_PE_DSA_TAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 16)
622 #define MVPP2_PE_DSA_UNTAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 15)
623 #define MVPP2_PE_ETYPE_EDSA_TAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 14)
624 #define MVPP2_PE_ETYPE_EDSA_UNTAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 13)
625 #define MVPP2_PE_ETYPE_DSA_TAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 12)
626 #define MVPP2_PE_ETYPE_DSA_UNTAGGED (MVPP2_PRS_TCAM_SRAM_SIZE - 11)
627 #define MVPP2_PE_MH_DEFAULT (MVPP2_PRS_TCAM_SRAM_SIZE - 10)
628 #define MVPP2_PE_DSA_DEFAULT (MVPP2_PRS_TCAM_SRAM_SIZE - 9)
629 #define MVPP2_PE_IP6_PROTO_UN (MVPP2_PRS_TCAM_SRAM_SIZE - 8)
630 #define MVPP2_PE_IP4_PROTO_UN (MVPP2_PRS_TCAM_SRAM_SIZE - 7)
631 #define MVPP2_PE_ETH_TYPE_UN (MVPP2_PRS_TCAM_SRAM_SIZE - 6)
632 #define MVPP2_PE_VLAN_DBL (MVPP2_PRS_TCAM_SRAM_SIZE - 5)
633 #define MVPP2_PE_VLAN_NONE (MVPP2_PRS_TCAM_SRAM_SIZE - 4)
634 #define MVPP2_PE_MAC_MC_ALL (MVPP2_PRS_TCAM_SRAM_SIZE - 3)
635 #define MVPP2_PE_MAC_PROMISCUOUS (MVPP2_PRS_TCAM_SRAM_SIZE - 2)
636 #define MVPP2_PE_MAC_NON_PROMISCUOUS (MVPP2_PRS_TCAM_SRAM_SIZE - 1)
639 * The fields are represented by MVPP2_PRS_TCAM_DATA_REG(3)->(0).
641 #define MVPP2_PRS_SRAM_RI_OFFS 0
642 #define MVPP2_PRS_SRAM_RI_WORD 0
643 #define MVPP2_PRS_SRAM_RI_CTRL_OFFS 32
644 #define MVPP2_PRS_SRAM_RI_CTRL_WORD 1
645 #define MVPP2_PRS_SRAM_RI_CTRL_BITS 32
646 #define MVPP2_PRS_SRAM_SHIFT_OFFS 64
647 #define MVPP2_PRS_SRAM_SHIFT_SIGN_BIT 72
648 #define MVPP2_PRS_SRAM_UDF_OFFS 73
649 #define MVPP2_PRS_SRAM_UDF_BITS 8
650 #define MVPP2_PRS_SRAM_UDF_MASK 0xff
651 #define MVPP2_PRS_SRAM_UDF_SIGN_BIT 81
652 #define MVPP2_PRS_SRAM_UDF_TYPE_OFFS 82
653 #define MVPP2_PRS_SRAM_UDF_TYPE_MASK 0x7
654 #define MVPP2_PRS_SRAM_UDF_TYPE_L3 1
655 #define MVPP2_PRS_SRAM_UDF_TYPE_L4 4
656 #define MVPP2_PRS_SRAM_OP_SEL_SHIFT_OFFS 85
657 #define MVPP2_PRS_SRAM_OP_SEL_SHIFT_MASK 0x3
658 #define MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD 1
659 #define MVPP2_PRS_SRAM_OP_SEL_SHIFT_IP4_ADD 2
660 #define MVPP2_PRS_SRAM_OP_SEL_SHIFT_IP6_ADD 3
661 #define MVPP2_PRS_SRAM_OP_SEL_UDF_OFFS 87
662 #define MVPP2_PRS_SRAM_OP_SEL_UDF_BITS 2
663 #define MVPP2_PRS_SRAM_OP_SEL_UDF_MASK 0x3
664 #define MVPP2_PRS_SRAM_OP_SEL_UDF_ADD 0
665 #define MVPP2_PRS_SRAM_OP_SEL_UDF_IP4_ADD 2
666 #define MVPP2_PRS_SRAM_OP_SEL_UDF_IP6_ADD 3
667 #define MVPP2_PRS_SRAM_OP_SEL_BASE_OFFS 89
668 #define MVPP2_PRS_SRAM_AI_OFFS 90
669 #define MVPP2_PRS_SRAM_AI_CTRL_OFFS 98
670 #define MVPP2_PRS_SRAM_AI_CTRL_BITS 8
671 #define MVPP2_PRS_SRAM_AI_MASK 0xff
672 #define MVPP2_PRS_SRAM_NEXT_LU_OFFS 106
673 #define MVPP2_PRS_SRAM_NEXT_LU_MASK 0xf
674 #define MVPP2_PRS_SRAM_LU_DONE_BIT 110
675 #define MVPP2_PRS_SRAM_LU_GEN_BIT 111
677 /* Sram result info bits assignment */
678 #define MVPP2_PRS_RI_MAC_ME_MASK 0x1
679 #define MVPP2_PRS_RI_DSA_MASK 0x2
680 #define MVPP2_PRS_RI_VLAN_MASK (BIT(2) | BIT(3))
681 #define MVPP2_PRS_RI_VLAN_NONE 0x0
682 #define MVPP2_PRS_RI_VLAN_SINGLE BIT(2)
683 #define MVPP2_PRS_RI_VLAN_DOUBLE BIT(3)
684 #define MVPP2_PRS_RI_VLAN_TRIPLE (BIT(2) | BIT(3))
685 #define MVPP2_PRS_RI_CPU_CODE_MASK 0x70
686 #define MVPP2_PRS_RI_CPU_CODE_RX_SPEC BIT(4)
687 #define MVPP2_PRS_RI_L2_CAST_MASK (BIT(9) | BIT(10))
688 #define MVPP2_PRS_RI_L2_UCAST 0x0
689 #define MVPP2_PRS_RI_L2_MCAST BIT(9)
690 #define MVPP2_PRS_RI_L2_BCAST BIT(10)
691 #define MVPP2_PRS_RI_PPPOE_MASK 0x800
692 #define MVPP2_PRS_RI_L3_PROTO_MASK (BIT(12) | BIT(13) | BIT(14))
693 #define MVPP2_PRS_RI_L3_UN 0x0
694 #define MVPP2_PRS_RI_L3_IP4 BIT(12)
695 #define MVPP2_PRS_RI_L3_IP4_OPT BIT(13)
696 #define MVPP2_PRS_RI_L3_IP4_OTHER (BIT(12) | BIT(13))
697 #define MVPP2_PRS_RI_L3_IP6 BIT(14)
698 #define MVPP2_PRS_RI_L3_IP6_EXT (BIT(12) | BIT(14))
699 #define MVPP2_PRS_RI_L3_ARP (BIT(13) | BIT(14))
700 #define MVPP2_PRS_RI_L3_ADDR_MASK (BIT(15) | BIT(16))
701 #define MVPP2_PRS_RI_L3_UCAST 0x0
702 #define MVPP2_PRS_RI_L3_MCAST BIT(15)
703 #define MVPP2_PRS_RI_L3_BCAST (BIT(15) | BIT(16))
704 #define MVPP2_PRS_RI_IP_FRAG_MASK 0x20000
705 #define MVPP2_PRS_RI_IP_FRAG_TRUE BIT(17)
706 #define MVPP2_PRS_RI_UDF3_MASK 0x300000
707 #define MVPP2_PRS_RI_UDF3_RX_SPECIAL BIT(21)
708 #define MVPP2_PRS_RI_L4_PROTO_MASK 0x1c00000
709 #define MVPP2_PRS_RI_L4_TCP BIT(22)
710 #define MVPP2_PRS_RI_L4_UDP BIT(23)
711 #define MVPP2_PRS_RI_L4_OTHER (BIT(22) | BIT(23))
712 #define MVPP2_PRS_RI_UDF7_MASK 0x60000000
713 #define MVPP2_PRS_RI_UDF7_IP6_LITE BIT(29)
714 #define MVPP2_PRS_RI_DROP_MASK 0x80000000
716 /* Sram additional info bits assignment */
717 #define MVPP2_PRS_IPV4_DIP_AI_BIT BIT(0)
718 #define MVPP2_PRS_IPV6_NO_EXT_AI_BIT BIT(0)
719 #define MVPP2_PRS_IPV6_EXT_AI_BIT BIT(1)
720 #define MVPP2_PRS_IPV6_EXT_AH_AI_BIT BIT(2)
721 #define MVPP2_PRS_IPV6_EXT_AH_LEN_AI_BIT BIT(3)
722 #define MVPP2_PRS_IPV6_EXT_AH_L4_AI_BIT BIT(4)
723 #define MVPP2_PRS_SINGLE_VLAN_AI 0
724 #define MVPP2_PRS_DBL_VLAN_AI_BIT BIT(7)
727 #define MVPP2_PRS_TAGGED true
728 #define MVPP2_PRS_UNTAGGED false
729 #define MVPP2_PRS_EDSA true
730 #define MVPP2_PRS_DSA false
732 /* MAC entries, shadow udf */
734 MVPP2_PRS_UDF_MAC_DEF
,
735 MVPP2_PRS_UDF_MAC_RANGE
,
736 MVPP2_PRS_UDF_L2_DEF
,
737 MVPP2_PRS_UDF_L2_DEF_COPY
,
738 MVPP2_PRS_UDF_L2_USER
,
742 enum mvpp2_prs_lookup
{
756 enum mvpp2_prs_l3_cast
{
757 MVPP2_PRS_L3_UNI_CAST
,
758 MVPP2_PRS_L3_MULTI_CAST
,
759 MVPP2_PRS_L3_BROAD_CAST
762 /* Classifier constants */
763 #define MVPP2_CLS_FLOWS_TBL_SIZE 512
764 #define MVPP2_CLS_FLOWS_TBL_DATA_WORDS 3
765 #define MVPP2_CLS_LKP_TBL_SIZE 64
766 #define MVPP2_CLS_RX_QUEUES 256
769 #define MVPP22_RSS_TABLE_ENTRIES 32
772 #define MVPP2_BM_POOLS_NUM 8
773 #define MVPP2_BM_LONG_BUF_NUM 1024
774 #define MVPP2_BM_SHORT_BUF_NUM 2048
775 #define MVPP2_BM_POOL_SIZE_MAX (16*1024 - MVPP2_BM_POOL_PTR_ALIGN/4)
776 #define MVPP2_BM_POOL_PTR_ALIGN 128
777 #define MVPP2_BM_SWF_LONG_POOL(port) ((port > 2) ? 2 : port)
778 #define MVPP2_BM_SWF_SHORT_POOL 3
780 /* BM cookie (32 bits) definition */
781 #define MVPP2_BM_COOKIE_POOL_OFFS 8
782 #define MVPP2_BM_COOKIE_CPU_OFFS 24
784 /* BM short pool packet size
785 * These value assure that for SWF the total number
786 * of bytes allocated for each buffer will be 512
788 #define MVPP2_BM_SHORT_PKT_SIZE MVPP2_RX_MAX_PKT_SIZE(512)
790 #define MVPP21_ADDR_SPACE_SZ 0
791 #define MVPP22_ADDR_SPACE_SZ SZ_64K
793 #define MVPP2_MAX_THREADS 8
794 #define MVPP2_MAX_QVECS MVPP2_MAX_THREADS
802 /* GMAC MIB Counters register definitions */
803 #define MVPP21_MIB_COUNTERS_OFFSET 0x1000
804 #define MVPP21_MIB_COUNTERS_PORT_SZ 0x400
805 #define MVPP22_MIB_COUNTERS_OFFSET 0x0
806 #define MVPP22_MIB_COUNTERS_PORT_SZ 0x100
808 #define MVPP2_MIB_GOOD_OCTETS_RCVD 0x0
809 #define MVPP2_MIB_BAD_OCTETS_RCVD 0x8
810 #define MVPP2_MIB_CRC_ERRORS_SENT 0xc
811 #define MVPP2_MIB_UNICAST_FRAMES_RCVD 0x10
812 #define MVPP2_MIB_BROADCAST_FRAMES_RCVD 0x18
813 #define MVPP2_MIB_MULTICAST_FRAMES_RCVD 0x1c
814 #define MVPP2_MIB_FRAMES_64_OCTETS 0x20
815 #define MVPP2_MIB_FRAMES_65_TO_127_OCTETS 0x24
816 #define MVPP2_MIB_FRAMES_128_TO_255_OCTETS 0x28
817 #define MVPP2_MIB_FRAMES_256_TO_511_OCTETS 0x2c
818 #define MVPP2_MIB_FRAMES_512_TO_1023_OCTETS 0x30
819 #define MVPP2_MIB_FRAMES_1024_TO_MAX_OCTETS 0x34
820 #define MVPP2_MIB_GOOD_OCTETS_SENT 0x38
821 #define MVPP2_MIB_UNICAST_FRAMES_SENT 0x40
822 #define MVPP2_MIB_MULTICAST_FRAMES_SENT 0x48
823 #define MVPP2_MIB_BROADCAST_FRAMES_SENT 0x4c
824 #define MVPP2_MIB_FC_SENT 0x54
825 #define MVPP2_MIB_FC_RCVD 0x58
826 #define MVPP2_MIB_RX_FIFO_OVERRUN 0x5c
827 #define MVPP2_MIB_UNDERSIZE_RCVD 0x60
828 #define MVPP2_MIB_FRAGMENTS_RCVD 0x64
829 #define MVPP2_MIB_OVERSIZE_RCVD 0x68
830 #define MVPP2_MIB_JABBER_RCVD 0x6c
831 #define MVPP2_MIB_MAC_RCV_ERROR 0x70
832 #define MVPP2_MIB_BAD_CRC_EVENT 0x74
833 #define MVPP2_MIB_COLLISION 0x78
834 #define MVPP2_MIB_LATE_COLLISION 0x7c
836 #define MVPP2_MIB_COUNTERS_STATS_DELAY (1 * HZ)
838 #define MVPP2_DESC_DMA_MASK DMA_BIT_MASK(40)
842 /* Shared Packet Processor resources */
844 /* Shared registers' base addresses */
845 void __iomem
*lms_base
;
846 void __iomem
*iface_base
;
848 /* On PPv2.2, each "software thread" can access the base
849 * register through a separate address space, each 64 KB apart
850 * from each other. Typically, such address spaces will be
853 void __iomem
*swth_base
[MVPP2_MAX_THREADS
];
855 /* On PPv2.2, some port control registers are located into the system
856 * controller space. These registers are accessible through a regmap.
858 struct regmap
*sysctrl_base
;
866 /* List of pointers to port structures */
868 struct mvpp2_port
**port_list
;
870 /* Aggregated TXQs */
871 struct mvpp2_tx_queue
*aggr_txqs
;
874 struct mvpp2_bm_pool
*bm_pools
;
876 /* PRS shadow table */
877 struct mvpp2_prs_shadow
*prs_shadow
;
878 /* PRS auxiliary table for double vlan entries control */
879 bool *prs_double_vlans
;
885 enum { MVPP21
, MVPP22
} hw_version
;
887 /* Maximum number of RXQs per port */
888 unsigned int max_port_rxqs
;
890 /* Workqueue to gather hardware statistics */
892 struct workqueue_struct
*stats_queue
;
895 struct mvpp2_pcpu_stats
{
896 struct u64_stats_sync syncp
;
903 /* Per-CPU port control */
904 struct mvpp2_port_pcpu
{
905 struct hrtimer tx_done_timer
;
906 bool timer_scheduled
;
907 /* Tasklet for egress finalization */
908 struct tasklet_struct tx_done_tasklet
;
911 struct mvpp2_queue_vector
{
913 struct napi_struct napi
;
914 enum { MVPP2_QUEUE_VECTOR_SHARED
, MVPP2_QUEUE_VECTOR_PRIVATE
} type
;
919 u32 pending_cause_rx
;
920 struct mvpp2_port
*port
;
926 /* Index of the port from the "group of ports" complex point
935 /* Per-port registers' base address */
937 void __iomem
*stats_base
;
939 struct mvpp2_rx_queue
**rxqs
;
941 struct mvpp2_tx_queue
**txqs
;
943 struct net_device
*dev
;
947 /* Per-CPU port control */
948 struct mvpp2_port_pcpu __percpu
*pcpu
;
955 struct mvpp2_pcpu_stats __percpu
*stats
;
958 /* Per-port work and its lock to gather hardware statistics */
959 struct mutex gather_stats_lock
;
960 struct delayed_work stats_work
;
962 phy_interface_t phy_interface
;
963 struct device_node
*phy_node
;
969 struct mvpp2_bm_pool
*pool_long
;
970 struct mvpp2_bm_pool
*pool_short
;
972 /* Index of first port's physical RXQ */
975 struct mvpp2_queue_vector qvecs
[MVPP2_MAX_QVECS
];
982 /* The mvpp2_tx_desc and mvpp2_rx_desc structures describe the
983 * layout of the transmit and reception DMA descriptors, and their
984 * layout is therefore defined by the hardware design
987 #define MVPP2_TXD_L3_OFF_SHIFT 0
988 #define MVPP2_TXD_IP_HLEN_SHIFT 8
989 #define MVPP2_TXD_L4_CSUM_FRAG BIT(13)
990 #define MVPP2_TXD_L4_CSUM_NOT BIT(14)
991 #define MVPP2_TXD_IP_CSUM_DISABLE BIT(15)
992 #define MVPP2_TXD_PADDING_DISABLE BIT(23)
993 #define MVPP2_TXD_L4_UDP BIT(24)
994 #define MVPP2_TXD_L3_IP6 BIT(26)
995 #define MVPP2_TXD_L_DESC BIT(28)
996 #define MVPP2_TXD_F_DESC BIT(29)
998 #define MVPP2_RXD_ERR_SUMMARY BIT(15)
999 #define MVPP2_RXD_ERR_CODE_MASK (BIT(13) | BIT(14))
1000 #define MVPP2_RXD_ERR_CRC 0x0
1001 #define MVPP2_RXD_ERR_OVERRUN BIT(13)
1002 #define MVPP2_RXD_ERR_RESOURCE (BIT(13) | BIT(14))
1003 #define MVPP2_RXD_BM_POOL_ID_OFFS 16
1004 #define MVPP2_RXD_BM_POOL_ID_MASK (BIT(16) | BIT(17) | BIT(18))
1005 #define MVPP2_RXD_HWF_SYNC BIT(21)
1006 #define MVPP2_RXD_L4_CSUM_OK BIT(22)
1007 #define MVPP2_RXD_IP4_HEADER_ERR BIT(24)
1008 #define MVPP2_RXD_L4_TCP BIT(25)
1009 #define MVPP2_RXD_L4_UDP BIT(26)
1010 #define MVPP2_RXD_L3_IP4 BIT(28)
1011 #define MVPP2_RXD_L3_IP6 BIT(30)
1012 #define MVPP2_RXD_BUF_HDR BIT(31)
1014 /* HW TX descriptor for PPv2.1 */
1015 struct mvpp21_tx_desc
{
1016 u32 command
; /* Options used by HW for packet transmitting.*/
1017 u8 packet_offset
; /* the offset from the buffer beginning */
1018 u8 phys_txq
; /* destination queue ID */
1019 u16 data_size
; /* data size of transmitted packet in bytes */
1020 u32 buf_dma_addr
; /* physical addr of transmitted buffer */
1021 u32 buf_cookie
; /* cookie for access to TX buffer in tx path */
1022 u32 reserved1
[3]; /* hw_cmd (for future use, BM, PON, PNC) */
1023 u32 reserved2
; /* reserved (for future use) */
1026 /* HW RX descriptor for PPv2.1 */
1027 struct mvpp21_rx_desc
{
1028 u32 status
; /* info about received packet */
1029 u16 reserved1
; /* parser_info (for future use, PnC) */
1030 u16 data_size
; /* size of received packet in bytes */
1031 u32 buf_dma_addr
; /* physical address of the buffer */
1032 u32 buf_cookie
; /* cookie for access to RX buffer in rx path */
1033 u16 reserved2
; /* gem_port_id (for future use, PON) */
1034 u16 reserved3
; /* csum_l4 (for future use, PnC) */
1035 u8 reserved4
; /* bm_qset (for future use, BM) */
1037 u16 reserved6
; /* classify_info (for future use, PnC) */
1038 u32 reserved7
; /* flow_id (for future use, PnC) */
1042 /* HW TX descriptor for PPv2.2 */
1043 struct mvpp22_tx_desc
{
1049 u64 buf_dma_addr_ptp
;
1050 u64 buf_cookie_misc
;
1053 /* HW RX descriptor for PPv2.2 */
1054 struct mvpp22_rx_desc
{
1060 u64 buf_dma_addr_key_hash
;
1061 u64 buf_cookie_misc
;
1064 /* Opaque type used by the driver to manipulate the HW TX and RX
1067 struct mvpp2_tx_desc
{
1069 struct mvpp21_tx_desc pp21
;
1070 struct mvpp22_tx_desc pp22
;
1074 struct mvpp2_rx_desc
{
1076 struct mvpp21_rx_desc pp21
;
1077 struct mvpp22_rx_desc pp22
;
1081 struct mvpp2_txq_pcpu_buf
{
1082 /* Transmitted SKB */
1083 struct sk_buff
*skb
;
1085 /* Physical address of transmitted buffer */
1088 /* Size transmitted */
1092 /* Per-CPU Tx queue control */
1093 struct mvpp2_txq_pcpu
{
1096 /* Number of Tx DMA descriptors in the descriptor ring */
1099 /* Number of currently used Tx DMA descriptor in the
1107 /* Number of Tx DMA descriptors reserved for each CPU */
1110 /* Infos about transmitted buffers */
1111 struct mvpp2_txq_pcpu_buf
*buffs
;
1113 /* Index of last TX DMA descriptor that was inserted */
1116 /* Index of the TX DMA descriptor to be cleaned up */
1119 /* DMA buffer for TSO headers */
1121 dma_addr_t tso_headers_dma
;
1124 struct mvpp2_tx_queue
{
1125 /* Physical number of this Tx queue */
1128 /* Logical number of this Tx queue */
1131 /* Number of Tx DMA descriptors in the descriptor ring */
1134 /* Number of currently used Tx DMA descriptor in the descriptor ring */
1137 /* Per-CPU control of physical Tx queues */
1138 struct mvpp2_txq_pcpu __percpu
*pcpu
;
1142 /* Virtual address of thex Tx DMA descriptors array */
1143 struct mvpp2_tx_desc
*descs
;
1145 /* DMA address of the Tx DMA descriptors array */
1146 dma_addr_t descs_dma
;
1148 /* Index of the last Tx DMA descriptor */
1151 /* Index of the next Tx DMA descriptor to process */
1152 int next_desc_to_proc
;
1155 struct mvpp2_rx_queue
{
1156 /* RX queue number, in the range 0-31 for physical RXQs */
1159 /* Num of rx descriptors in the rx descriptor ring */
1165 /* Virtual address of the RX DMA descriptors array */
1166 struct mvpp2_rx_desc
*descs
;
1168 /* DMA address of the RX DMA descriptors array */
1169 dma_addr_t descs_dma
;
1171 /* Index of the last RX DMA descriptor */
1174 /* Index of the next RX DMA descriptor to process */
1175 int next_desc_to_proc
;
1177 /* ID of port to which physical RXQ is mapped */
1180 /* Port's logic RXQ number to which physical RXQ is mapped */
1184 union mvpp2_prs_tcam_entry
{
1185 u32 word
[MVPP2_PRS_TCAM_WORDS
];
1186 u8 byte
[MVPP2_PRS_TCAM_WORDS
* 4];
1189 union mvpp2_prs_sram_entry
{
1190 u32 word
[MVPP2_PRS_SRAM_WORDS
];
1191 u8 byte
[MVPP2_PRS_SRAM_WORDS
* 4];
1194 struct mvpp2_prs_entry
{
1196 union mvpp2_prs_tcam_entry tcam
;
1197 union mvpp2_prs_sram_entry sram
;
1200 struct mvpp2_prs_shadow
{
1207 /* User defined offset */
1215 struct mvpp2_cls_flow_entry
{
1217 u32 data
[MVPP2_CLS_FLOWS_TBL_DATA_WORDS
];
1220 struct mvpp2_cls_lookup_entry
{
1226 struct mvpp2_bm_pool
{
1227 /* Pool number in the range 0-7 */
1229 enum mvpp2_bm_type type
;
1231 /* Buffer Pointers Pool External (BPPE) size */
1233 /* BPPE size in bytes */
1235 /* Number of buffers for this pool */
1237 /* Pool buffer size */
1243 /* BPPE virtual base address */
1245 /* BPPE DMA base address */
1246 dma_addr_t dma_addr
;
1248 /* Ports using BM pool */
1252 #define IS_TSO_HEADER(txq_pcpu, addr) \
1253 ((addr) >= (txq_pcpu)->tso_headers_dma && \
1254 (addr) < (txq_pcpu)->tso_headers_dma + \
1255 (txq_pcpu)->size * TSO_HEADER_SIZE)
1258 #define MVPP2_QDIST_SINGLE_MODE 0
1259 #define MVPP2_QDIST_MULTI_MODE 1
1261 static int queue_mode
= MVPP2_QDIST_SINGLE_MODE
;
1263 module_param(queue_mode
, int, 0444);
1264 MODULE_PARM_DESC(queue_mode
, "Set queue_mode (single=0, multi=1)");
1266 #define MVPP2_DRIVER_NAME "mvpp2"
1267 #define MVPP2_DRIVER_VERSION "1.0"
1269 /* Utility/helper methods */
1271 static void mvpp2_write(struct mvpp2
*priv
, u32 offset
, u32 data
)
1273 writel(data
, priv
->swth_base
[0] + offset
);
1276 static u32
mvpp2_read(struct mvpp2
*priv
, u32 offset
)
1278 return readl(priv
->swth_base
[0] + offset
);
1281 /* These accessors should be used to access:
1283 * - per-CPU registers, where each CPU has its own copy of the
1286 * MVPP2_BM_VIRT_ALLOC_REG
1287 * MVPP2_BM_ADDR_HIGH_ALLOC
1288 * MVPP22_BM_ADDR_HIGH_RLS_REG
1289 * MVPP2_BM_VIRT_RLS_REG
1290 * MVPP2_ISR_RX_TX_CAUSE_REG
1291 * MVPP2_ISR_RX_TX_MASK_REG
1293 * MVPP2_AGGR_TXQ_UPDATE_REG
1294 * MVPP2_TXQ_RSVD_REQ_REG
1295 * MVPP2_TXQ_RSVD_RSLT_REG
1296 * MVPP2_TXQ_SENT_REG
1299 * - global registers that must be accessed through a specific CPU
1300 * window, because they are related to an access to a per-CPU
1303 * MVPP2_BM_PHY_ALLOC_REG (related to MVPP2_BM_VIRT_ALLOC_REG)
1304 * MVPP2_BM_PHY_RLS_REG (related to MVPP2_BM_VIRT_RLS_REG)
1305 * MVPP2_RXQ_THRESH_REG (related to MVPP2_RXQ_NUM_REG)
1306 * MVPP2_RXQ_DESC_ADDR_REG (related to MVPP2_RXQ_NUM_REG)
1307 * MVPP2_RXQ_DESC_SIZE_REG (related to MVPP2_RXQ_NUM_REG)
1308 * MVPP2_RXQ_INDEX_REG (related to MVPP2_RXQ_NUM_REG)
1309 * MVPP2_TXQ_PENDING_REG (related to MVPP2_TXQ_NUM_REG)
1310 * MVPP2_TXQ_DESC_ADDR_REG (related to MVPP2_TXQ_NUM_REG)
1311 * MVPP2_TXQ_DESC_SIZE_REG (related to MVPP2_TXQ_NUM_REG)
1312 * MVPP2_TXQ_INDEX_REG (related to MVPP2_TXQ_NUM_REG)
1313 * MVPP2_TXQ_PENDING_REG (related to MVPP2_TXQ_NUM_REG)
1314 * MVPP2_TXQ_PREF_BUF_REG (related to MVPP2_TXQ_NUM_REG)
1315 * MVPP2_TXQ_PREF_BUF_REG (related to MVPP2_TXQ_NUM_REG)
1317 static void mvpp2_percpu_write(struct mvpp2
*priv
, int cpu
,
1318 u32 offset
, u32 data
)
1320 writel(data
, priv
->swth_base
[cpu
] + offset
);
1323 static u32
mvpp2_percpu_read(struct mvpp2
*priv
, int cpu
,
1326 return readl(priv
->swth_base
[cpu
] + offset
);
1329 static dma_addr_t
mvpp2_txdesc_dma_addr_get(struct mvpp2_port
*port
,
1330 struct mvpp2_tx_desc
*tx_desc
)
1332 if (port
->priv
->hw_version
== MVPP21
)
1333 return tx_desc
->pp21
.buf_dma_addr
;
1335 return tx_desc
->pp22
.buf_dma_addr_ptp
& MVPP2_DESC_DMA_MASK
;
1338 static void mvpp2_txdesc_dma_addr_set(struct mvpp2_port
*port
,
1339 struct mvpp2_tx_desc
*tx_desc
,
1340 dma_addr_t dma_addr
)
1342 dma_addr_t addr
, offset
;
1344 addr
= dma_addr
& ~MVPP2_TX_DESC_ALIGN
;
1345 offset
= dma_addr
& MVPP2_TX_DESC_ALIGN
;
1347 if (port
->priv
->hw_version
== MVPP21
) {
1348 tx_desc
->pp21
.buf_dma_addr
= addr
;
1349 tx_desc
->pp21
.packet_offset
= offset
;
1351 u64 val
= (u64
)addr
;
1353 tx_desc
->pp22
.buf_dma_addr_ptp
&= ~MVPP2_DESC_DMA_MASK
;
1354 tx_desc
->pp22
.buf_dma_addr_ptp
|= val
;
1355 tx_desc
->pp22
.packet_offset
= offset
;
1359 static size_t mvpp2_txdesc_size_get(struct mvpp2_port
*port
,
1360 struct mvpp2_tx_desc
*tx_desc
)
1362 if (port
->priv
->hw_version
== MVPP21
)
1363 return tx_desc
->pp21
.data_size
;
1365 return tx_desc
->pp22
.data_size
;
1368 static void mvpp2_txdesc_size_set(struct mvpp2_port
*port
,
1369 struct mvpp2_tx_desc
*tx_desc
,
1372 if (port
->priv
->hw_version
== MVPP21
)
1373 tx_desc
->pp21
.data_size
= size
;
1375 tx_desc
->pp22
.data_size
= size
;
1378 static void mvpp2_txdesc_txq_set(struct mvpp2_port
*port
,
1379 struct mvpp2_tx_desc
*tx_desc
,
1382 if (port
->priv
->hw_version
== MVPP21
)
1383 tx_desc
->pp21
.phys_txq
= txq
;
1385 tx_desc
->pp22
.phys_txq
= txq
;
1388 static void mvpp2_txdesc_cmd_set(struct mvpp2_port
*port
,
1389 struct mvpp2_tx_desc
*tx_desc
,
1390 unsigned int command
)
1392 if (port
->priv
->hw_version
== MVPP21
)
1393 tx_desc
->pp21
.command
= command
;
1395 tx_desc
->pp22
.command
= command
;
1398 static unsigned int mvpp2_txdesc_offset_get(struct mvpp2_port
*port
,
1399 struct mvpp2_tx_desc
*tx_desc
)
1401 if (port
->priv
->hw_version
== MVPP21
)
1402 return tx_desc
->pp21
.packet_offset
;
1404 return tx_desc
->pp22
.packet_offset
;
1407 static dma_addr_t
mvpp2_rxdesc_dma_addr_get(struct mvpp2_port
*port
,
1408 struct mvpp2_rx_desc
*rx_desc
)
1410 if (port
->priv
->hw_version
== MVPP21
)
1411 return rx_desc
->pp21
.buf_dma_addr
;
1413 return rx_desc
->pp22
.buf_dma_addr_key_hash
& MVPP2_DESC_DMA_MASK
;
1416 static unsigned long mvpp2_rxdesc_cookie_get(struct mvpp2_port
*port
,
1417 struct mvpp2_rx_desc
*rx_desc
)
1419 if (port
->priv
->hw_version
== MVPP21
)
1420 return rx_desc
->pp21
.buf_cookie
;
1422 return rx_desc
->pp22
.buf_cookie_misc
& MVPP2_DESC_DMA_MASK
;
1425 static size_t mvpp2_rxdesc_size_get(struct mvpp2_port
*port
,
1426 struct mvpp2_rx_desc
*rx_desc
)
1428 if (port
->priv
->hw_version
== MVPP21
)
1429 return rx_desc
->pp21
.data_size
;
1431 return rx_desc
->pp22
.data_size
;
1434 static u32
mvpp2_rxdesc_status_get(struct mvpp2_port
*port
,
1435 struct mvpp2_rx_desc
*rx_desc
)
1437 if (port
->priv
->hw_version
== MVPP21
)
1438 return rx_desc
->pp21
.status
;
1440 return rx_desc
->pp22
.status
;
1443 static void mvpp2_txq_inc_get(struct mvpp2_txq_pcpu
*txq_pcpu
)
1445 txq_pcpu
->txq_get_index
++;
1446 if (txq_pcpu
->txq_get_index
== txq_pcpu
->size
)
1447 txq_pcpu
->txq_get_index
= 0;
1450 static void mvpp2_txq_inc_put(struct mvpp2_port
*port
,
1451 struct mvpp2_txq_pcpu
*txq_pcpu
,
1452 struct sk_buff
*skb
,
1453 struct mvpp2_tx_desc
*tx_desc
)
1455 struct mvpp2_txq_pcpu_buf
*tx_buf
=
1456 txq_pcpu
->buffs
+ txq_pcpu
->txq_put_index
;
1458 tx_buf
->size
= mvpp2_txdesc_size_get(port
, tx_desc
);
1459 tx_buf
->dma
= mvpp2_txdesc_dma_addr_get(port
, tx_desc
) +
1460 mvpp2_txdesc_offset_get(port
, tx_desc
);
1461 txq_pcpu
->txq_put_index
++;
1462 if (txq_pcpu
->txq_put_index
== txq_pcpu
->size
)
1463 txq_pcpu
->txq_put_index
= 0;
1466 /* Get number of physical egress port */
1467 static inline int mvpp2_egress_port(struct mvpp2_port
*port
)
1469 return MVPP2_MAX_TCONT
+ port
->id
;
1472 /* Get number of physical TXQ */
1473 static inline int mvpp2_txq_phys(int port
, int txq
)
1475 return (MVPP2_MAX_TCONT
+ port
) * MVPP2_MAX_TXQ
+ txq
;
1478 /* Parser configuration routines */
1480 /* Update parser tcam and sram hw entries */
1481 static int mvpp2_prs_hw_write(struct mvpp2
*priv
, struct mvpp2_prs_entry
*pe
)
1485 if (pe
->index
> MVPP2_PRS_TCAM_SRAM_SIZE
- 1)
1488 /* Clear entry invalidation bit */
1489 pe
->tcam
.word
[MVPP2_PRS_TCAM_INV_WORD
] &= ~MVPP2_PRS_TCAM_INV_MASK
;
1491 /* Write tcam index - indirect access */
1492 mvpp2_write(priv
, MVPP2_PRS_TCAM_IDX_REG
, pe
->index
);
1493 for (i
= 0; i
< MVPP2_PRS_TCAM_WORDS
; i
++)
1494 mvpp2_write(priv
, MVPP2_PRS_TCAM_DATA_REG(i
), pe
->tcam
.word
[i
]);
1496 /* Write sram index - indirect access */
1497 mvpp2_write(priv
, MVPP2_PRS_SRAM_IDX_REG
, pe
->index
);
1498 for (i
= 0; i
< MVPP2_PRS_SRAM_WORDS
; i
++)
1499 mvpp2_write(priv
, MVPP2_PRS_SRAM_DATA_REG(i
), pe
->sram
.word
[i
]);
1504 /* Read tcam entry from hw */
1505 static int mvpp2_prs_hw_read(struct mvpp2
*priv
, struct mvpp2_prs_entry
*pe
)
1509 if (pe
->index
> MVPP2_PRS_TCAM_SRAM_SIZE
- 1)
1512 /* Write tcam index - indirect access */
1513 mvpp2_write(priv
, MVPP2_PRS_TCAM_IDX_REG
, pe
->index
);
1515 pe
->tcam
.word
[MVPP2_PRS_TCAM_INV_WORD
] = mvpp2_read(priv
,
1516 MVPP2_PRS_TCAM_DATA_REG(MVPP2_PRS_TCAM_INV_WORD
));
1517 if (pe
->tcam
.word
[MVPP2_PRS_TCAM_INV_WORD
] & MVPP2_PRS_TCAM_INV_MASK
)
1518 return MVPP2_PRS_TCAM_ENTRY_INVALID
;
1520 for (i
= 0; i
< MVPP2_PRS_TCAM_WORDS
; i
++)
1521 pe
->tcam
.word
[i
] = mvpp2_read(priv
, MVPP2_PRS_TCAM_DATA_REG(i
));
1523 /* Write sram index - indirect access */
1524 mvpp2_write(priv
, MVPP2_PRS_SRAM_IDX_REG
, pe
->index
);
1525 for (i
= 0; i
< MVPP2_PRS_SRAM_WORDS
; i
++)
1526 pe
->sram
.word
[i
] = mvpp2_read(priv
, MVPP2_PRS_SRAM_DATA_REG(i
));
1531 /* Invalidate tcam hw entry */
1532 static void mvpp2_prs_hw_inv(struct mvpp2
*priv
, int index
)
1534 /* Write index - indirect access */
1535 mvpp2_write(priv
, MVPP2_PRS_TCAM_IDX_REG
, index
);
1536 mvpp2_write(priv
, MVPP2_PRS_TCAM_DATA_REG(MVPP2_PRS_TCAM_INV_WORD
),
1537 MVPP2_PRS_TCAM_INV_MASK
);
1540 /* Enable shadow table entry and set its lookup ID */
1541 static void mvpp2_prs_shadow_set(struct mvpp2
*priv
, int index
, int lu
)
1543 priv
->prs_shadow
[index
].valid
= true;
1544 priv
->prs_shadow
[index
].lu
= lu
;
1547 /* Update ri fields in shadow table entry */
1548 static void mvpp2_prs_shadow_ri_set(struct mvpp2
*priv
, int index
,
1549 unsigned int ri
, unsigned int ri_mask
)
1551 priv
->prs_shadow
[index
].ri_mask
= ri_mask
;
1552 priv
->prs_shadow
[index
].ri
= ri
;
1555 /* Update lookup field in tcam sw entry */
1556 static void mvpp2_prs_tcam_lu_set(struct mvpp2_prs_entry
*pe
, unsigned int lu
)
1558 int enable_off
= MVPP2_PRS_TCAM_EN_OFFS(MVPP2_PRS_TCAM_LU_BYTE
);
1560 pe
->tcam
.byte
[MVPP2_PRS_TCAM_LU_BYTE
] = lu
;
1561 pe
->tcam
.byte
[enable_off
] = MVPP2_PRS_LU_MASK
;
1564 /* Update mask for single port in tcam sw entry */
1565 static void mvpp2_prs_tcam_port_set(struct mvpp2_prs_entry
*pe
,
1566 unsigned int port
, bool add
)
1568 int enable_off
= MVPP2_PRS_TCAM_EN_OFFS(MVPP2_PRS_TCAM_PORT_BYTE
);
1571 pe
->tcam
.byte
[enable_off
] &= ~(1 << port
);
1573 pe
->tcam
.byte
[enable_off
] |= 1 << port
;
1576 /* Update port map in tcam sw entry */
1577 static void mvpp2_prs_tcam_port_map_set(struct mvpp2_prs_entry
*pe
,
1580 unsigned char port_mask
= MVPP2_PRS_PORT_MASK
;
1581 int enable_off
= MVPP2_PRS_TCAM_EN_OFFS(MVPP2_PRS_TCAM_PORT_BYTE
);
1583 pe
->tcam
.byte
[MVPP2_PRS_TCAM_PORT_BYTE
] = 0;
1584 pe
->tcam
.byte
[enable_off
] &= ~port_mask
;
1585 pe
->tcam
.byte
[enable_off
] |= ~ports
& MVPP2_PRS_PORT_MASK
;
1588 /* Obtain port map from tcam sw entry */
1589 static unsigned int mvpp2_prs_tcam_port_map_get(struct mvpp2_prs_entry
*pe
)
1591 int enable_off
= MVPP2_PRS_TCAM_EN_OFFS(MVPP2_PRS_TCAM_PORT_BYTE
);
1593 return ~(pe
->tcam
.byte
[enable_off
]) & MVPP2_PRS_PORT_MASK
;
1596 /* Set byte of data and its enable bits in tcam sw entry */
1597 static void mvpp2_prs_tcam_data_byte_set(struct mvpp2_prs_entry
*pe
,
1598 unsigned int offs
, unsigned char byte
,
1599 unsigned char enable
)
1601 pe
->tcam
.byte
[MVPP2_PRS_TCAM_DATA_BYTE(offs
)] = byte
;
1602 pe
->tcam
.byte
[MVPP2_PRS_TCAM_DATA_BYTE_EN(offs
)] = enable
;
1605 /* Get byte of data and its enable bits from tcam sw entry */
1606 static void mvpp2_prs_tcam_data_byte_get(struct mvpp2_prs_entry
*pe
,
1607 unsigned int offs
, unsigned char *byte
,
1608 unsigned char *enable
)
1610 *byte
= pe
->tcam
.byte
[MVPP2_PRS_TCAM_DATA_BYTE(offs
)];
1611 *enable
= pe
->tcam
.byte
[MVPP2_PRS_TCAM_DATA_BYTE_EN(offs
)];
1614 /* Compare tcam data bytes with a pattern */
1615 static bool mvpp2_prs_tcam_data_cmp(struct mvpp2_prs_entry
*pe
, int offs
,
1618 int off
= MVPP2_PRS_TCAM_DATA_BYTE(offs
);
1621 tcam_data
= (pe
->tcam
.byte
[off
+ 1] << 8) | pe
->tcam
.byte
[off
];
1622 if (tcam_data
!= data
)
1627 /* Update ai bits in tcam sw entry */
1628 static void mvpp2_prs_tcam_ai_update(struct mvpp2_prs_entry
*pe
,
1629 unsigned int bits
, unsigned int enable
)
1631 int i
, ai_idx
= MVPP2_PRS_TCAM_AI_BYTE
;
1633 for (i
= 0; i
< MVPP2_PRS_AI_BITS
; i
++) {
1635 if (!(enable
& BIT(i
)))
1639 pe
->tcam
.byte
[ai_idx
] |= 1 << i
;
1641 pe
->tcam
.byte
[ai_idx
] &= ~(1 << i
);
1644 pe
->tcam
.byte
[MVPP2_PRS_TCAM_EN_OFFS(ai_idx
)] |= enable
;
1647 /* Get ai bits from tcam sw entry */
1648 static int mvpp2_prs_tcam_ai_get(struct mvpp2_prs_entry
*pe
)
1650 return pe
->tcam
.byte
[MVPP2_PRS_TCAM_AI_BYTE
];
1653 /* Set ethertype in tcam sw entry */
1654 static void mvpp2_prs_match_etype(struct mvpp2_prs_entry
*pe
, int offset
,
1655 unsigned short ethertype
)
1657 mvpp2_prs_tcam_data_byte_set(pe
, offset
+ 0, ethertype
>> 8, 0xff);
1658 mvpp2_prs_tcam_data_byte_set(pe
, offset
+ 1, ethertype
& 0xff, 0xff);
1661 /* Set bits in sram sw entry */
1662 static void mvpp2_prs_sram_bits_set(struct mvpp2_prs_entry
*pe
, int bit_num
,
1665 pe
->sram
.byte
[MVPP2_BIT_TO_BYTE(bit_num
)] |= (val
<< (bit_num
% 8));
1668 /* Clear bits in sram sw entry */
1669 static void mvpp2_prs_sram_bits_clear(struct mvpp2_prs_entry
*pe
, int bit_num
,
1672 pe
->sram
.byte
[MVPP2_BIT_TO_BYTE(bit_num
)] &= ~(val
<< (bit_num
% 8));
1675 /* Update ri bits in sram sw entry */
1676 static void mvpp2_prs_sram_ri_update(struct mvpp2_prs_entry
*pe
,
1677 unsigned int bits
, unsigned int mask
)
1681 for (i
= 0; i
< MVPP2_PRS_SRAM_RI_CTRL_BITS
; i
++) {
1682 int ri_off
= MVPP2_PRS_SRAM_RI_OFFS
;
1684 if (!(mask
& BIT(i
)))
1688 mvpp2_prs_sram_bits_set(pe
, ri_off
+ i
, 1);
1690 mvpp2_prs_sram_bits_clear(pe
, ri_off
+ i
, 1);
1692 mvpp2_prs_sram_bits_set(pe
, MVPP2_PRS_SRAM_RI_CTRL_OFFS
+ i
, 1);
1696 /* Obtain ri bits from sram sw entry */
1697 static int mvpp2_prs_sram_ri_get(struct mvpp2_prs_entry
*pe
)
1699 return pe
->sram
.word
[MVPP2_PRS_SRAM_RI_WORD
];
1702 /* Update ai bits in sram sw entry */
1703 static void mvpp2_prs_sram_ai_update(struct mvpp2_prs_entry
*pe
,
1704 unsigned int bits
, unsigned int mask
)
1707 int ai_off
= MVPP2_PRS_SRAM_AI_OFFS
;
1709 for (i
= 0; i
< MVPP2_PRS_SRAM_AI_CTRL_BITS
; i
++) {
1711 if (!(mask
& BIT(i
)))
1715 mvpp2_prs_sram_bits_set(pe
, ai_off
+ i
, 1);
1717 mvpp2_prs_sram_bits_clear(pe
, ai_off
+ i
, 1);
1719 mvpp2_prs_sram_bits_set(pe
, MVPP2_PRS_SRAM_AI_CTRL_OFFS
+ i
, 1);
1723 /* Read ai bits from sram sw entry */
1724 static int mvpp2_prs_sram_ai_get(struct mvpp2_prs_entry
*pe
)
1727 int ai_off
= MVPP2_BIT_TO_BYTE(MVPP2_PRS_SRAM_AI_OFFS
);
1728 int ai_en_off
= ai_off
+ 1;
1729 int ai_shift
= MVPP2_PRS_SRAM_AI_OFFS
% 8;
1731 bits
= (pe
->sram
.byte
[ai_off
] >> ai_shift
) |
1732 (pe
->sram
.byte
[ai_en_off
] << (8 - ai_shift
));
1737 /* In sram sw entry set lookup ID field of the tcam key to be used in the next
1740 static void mvpp2_prs_sram_next_lu_set(struct mvpp2_prs_entry
*pe
,
1743 int sram_next_off
= MVPP2_PRS_SRAM_NEXT_LU_OFFS
;
1745 mvpp2_prs_sram_bits_clear(pe
, sram_next_off
,
1746 MVPP2_PRS_SRAM_NEXT_LU_MASK
);
1747 mvpp2_prs_sram_bits_set(pe
, sram_next_off
, lu
);
1750 /* In the sram sw entry set sign and value of the next lookup offset
1751 * and the offset value generated to the classifier
1753 static void mvpp2_prs_sram_shift_set(struct mvpp2_prs_entry
*pe
, int shift
,
1758 mvpp2_prs_sram_bits_set(pe
, MVPP2_PRS_SRAM_SHIFT_SIGN_BIT
, 1);
1761 mvpp2_prs_sram_bits_clear(pe
, MVPP2_PRS_SRAM_SHIFT_SIGN_BIT
, 1);
1765 pe
->sram
.byte
[MVPP2_BIT_TO_BYTE(MVPP2_PRS_SRAM_SHIFT_OFFS
)] =
1766 (unsigned char)shift
;
1768 /* Reset and set operation */
1769 mvpp2_prs_sram_bits_clear(pe
, MVPP2_PRS_SRAM_OP_SEL_SHIFT_OFFS
,
1770 MVPP2_PRS_SRAM_OP_SEL_SHIFT_MASK
);
1771 mvpp2_prs_sram_bits_set(pe
, MVPP2_PRS_SRAM_OP_SEL_SHIFT_OFFS
, op
);
1773 /* Set base offset as current */
1774 mvpp2_prs_sram_bits_clear(pe
, MVPP2_PRS_SRAM_OP_SEL_BASE_OFFS
, 1);
1777 /* In the sram sw entry set sign and value of the user defined offset
1778 * generated to the classifier
1780 static void mvpp2_prs_sram_offset_set(struct mvpp2_prs_entry
*pe
,
1781 unsigned int type
, int offset
,
1786 mvpp2_prs_sram_bits_set(pe
, MVPP2_PRS_SRAM_UDF_SIGN_BIT
, 1);
1787 offset
= 0 - offset
;
1789 mvpp2_prs_sram_bits_clear(pe
, MVPP2_PRS_SRAM_UDF_SIGN_BIT
, 1);
1793 mvpp2_prs_sram_bits_clear(pe
, MVPP2_PRS_SRAM_UDF_OFFS
,
1794 MVPP2_PRS_SRAM_UDF_MASK
);
1795 mvpp2_prs_sram_bits_set(pe
, MVPP2_PRS_SRAM_UDF_OFFS
, offset
);
1796 pe
->sram
.byte
[MVPP2_BIT_TO_BYTE(MVPP2_PRS_SRAM_UDF_OFFS
+
1797 MVPP2_PRS_SRAM_UDF_BITS
)] &=
1798 ~(MVPP2_PRS_SRAM_UDF_MASK
>> (8 - (MVPP2_PRS_SRAM_UDF_OFFS
% 8)));
1799 pe
->sram
.byte
[MVPP2_BIT_TO_BYTE(MVPP2_PRS_SRAM_UDF_OFFS
+
1800 MVPP2_PRS_SRAM_UDF_BITS
)] |=
1801 (offset
>> (8 - (MVPP2_PRS_SRAM_UDF_OFFS
% 8)));
1803 /* Set offset type */
1804 mvpp2_prs_sram_bits_clear(pe
, MVPP2_PRS_SRAM_UDF_TYPE_OFFS
,
1805 MVPP2_PRS_SRAM_UDF_TYPE_MASK
);
1806 mvpp2_prs_sram_bits_set(pe
, MVPP2_PRS_SRAM_UDF_TYPE_OFFS
, type
);
1808 /* Set offset operation */
1809 mvpp2_prs_sram_bits_clear(pe
, MVPP2_PRS_SRAM_OP_SEL_UDF_OFFS
,
1810 MVPP2_PRS_SRAM_OP_SEL_UDF_MASK
);
1811 mvpp2_prs_sram_bits_set(pe
, MVPP2_PRS_SRAM_OP_SEL_UDF_OFFS
, op
);
1813 pe
->sram
.byte
[MVPP2_BIT_TO_BYTE(MVPP2_PRS_SRAM_OP_SEL_UDF_OFFS
+
1814 MVPP2_PRS_SRAM_OP_SEL_UDF_BITS
)] &=
1815 ~(MVPP2_PRS_SRAM_OP_SEL_UDF_MASK
>>
1816 (8 - (MVPP2_PRS_SRAM_OP_SEL_UDF_OFFS
% 8)));
1818 pe
->sram
.byte
[MVPP2_BIT_TO_BYTE(MVPP2_PRS_SRAM_OP_SEL_UDF_OFFS
+
1819 MVPP2_PRS_SRAM_OP_SEL_UDF_BITS
)] |=
1820 (op
>> (8 - (MVPP2_PRS_SRAM_OP_SEL_UDF_OFFS
% 8)));
1822 /* Set base offset as current */
1823 mvpp2_prs_sram_bits_clear(pe
, MVPP2_PRS_SRAM_OP_SEL_BASE_OFFS
, 1);
1826 /* Find parser flow entry */
1827 static struct mvpp2_prs_entry
*mvpp2_prs_flow_find(struct mvpp2
*priv
, int flow
)
1829 struct mvpp2_prs_entry
*pe
;
1832 pe
= kzalloc(sizeof(*pe
), GFP_KERNEL
);
1835 mvpp2_prs_tcam_lu_set(pe
, MVPP2_PRS_LU_FLOWS
);
1837 /* Go through the all entires with MVPP2_PRS_LU_FLOWS */
1838 for (tid
= MVPP2_PRS_TCAM_SRAM_SIZE
- 1; tid
>= 0; tid
--) {
1841 if (!priv
->prs_shadow
[tid
].valid
||
1842 priv
->prs_shadow
[tid
].lu
!= MVPP2_PRS_LU_FLOWS
)
1846 mvpp2_prs_hw_read(priv
, pe
);
1847 bits
= mvpp2_prs_sram_ai_get(pe
);
1849 /* Sram store classification lookup ID in AI bits [5:0] */
1850 if ((bits
& MVPP2_PRS_FLOW_ID_MASK
) == flow
)
1858 /* Return first free tcam index, seeking from start to end */
1859 static int mvpp2_prs_tcam_first_free(struct mvpp2
*priv
, unsigned char start
,
1867 if (end
>= MVPP2_PRS_TCAM_SRAM_SIZE
)
1868 end
= MVPP2_PRS_TCAM_SRAM_SIZE
- 1;
1870 for (tid
= start
; tid
<= end
; tid
++) {
1871 if (!priv
->prs_shadow
[tid
].valid
)
1878 /* Enable/disable dropping all mac da's */
1879 static void mvpp2_prs_mac_drop_all_set(struct mvpp2
*priv
, int port
, bool add
)
1881 struct mvpp2_prs_entry pe
;
1883 if (priv
->prs_shadow
[MVPP2_PE_DROP_ALL
].valid
) {
1884 /* Entry exist - update port only */
1885 pe
.index
= MVPP2_PE_DROP_ALL
;
1886 mvpp2_prs_hw_read(priv
, &pe
);
1888 /* Entry doesn't exist - create new */
1889 memset(&pe
, 0, sizeof(pe
));
1890 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_MAC
);
1891 pe
.index
= MVPP2_PE_DROP_ALL
;
1893 /* Non-promiscuous mode for all ports - DROP unknown packets */
1894 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_DROP_MASK
,
1895 MVPP2_PRS_RI_DROP_MASK
);
1897 mvpp2_prs_sram_bits_set(&pe
, MVPP2_PRS_SRAM_LU_GEN_BIT
, 1);
1898 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_FLOWS
);
1900 /* Update shadow table */
1901 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_MAC
);
1903 /* Mask all ports */
1904 mvpp2_prs_tcam_port_map_set(&pe
, 0);
1907 /* Update port mask */
1908 mvpp2_prs_tcam_port_set(&pe
, port
, add
);
1910 mvpp2_prs_hw_write(priv
, &pe
);
1913 /* Set port to promiscuous mode */
1914 static void mvpp2_prs_mac_promisc_set(struct mvpp2
*priv
, int port
, bool add
)
1916 struct mvpp2_prs_entry pe
;
1918 /* Promiscuous mode - Accept unknown packets */
1920 if (priv
->prs_shadow
[MVPP2_PE_MAC_PROMISCUOUS
].valid
) {
1921 /* Entry exist - update port only */
1922 pe
.index
= MVPP2_PE_MAC_PROMISCUOUS
;
1923 mvpp2_prs_hw_read(priv
, &pe
);
1925 /* Entry doesn't exist - create new */
1926 memset(&pe
, 0, sizeof(pe
));
1927 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_MAC
);
1928 pe
.index
= MVPP2_PE_MAC_PROMISCUOUS
;
1930 /* Continue - set next lookup */
1931 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_DSA
);
1933 /* Set result info bits */
1934 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L2_UCAST
,
1935 MVPP2_PRS_RI_L2_CAST_MASK
);
1937 /* Shift to ethertype */
1938 mvpp2_prs_sram_shift_set(&pe
, 2 * ETH_ALEN
,
1939 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
1941 /* Mask all ports */
1942 mvpp2_prs_tcam_port_map_set(&pe
, 0);
1944 /* Update shadow table */
1945 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_MAC
);
1948 /* Update port mask */
1949 mvpp2_prs_tcam_port_set(&pe
, port
, add
);
1951 mvpp2_prs_hw_write(priv
, &pe
);
1954 /* Accept multicast */
1955 static void mvpp2_prs_mac_multi_set(struct mvpp2
*priv
, int port
, int index
,
1958 struct mvpp2_prs_entry pe
;
1959 unsigned char da_mc
;
1961 /* Ethernet multicast address first byte is
1962 * 0x01 for IPv4 and 0x33 for IPv6
1964 da_mc
= (index
== MVPP2_PE_MAC_MC_ALL
) ? 0x01 : 0x33;
1966 if (priv
->prs_shadow
[index
].valid
) {
1967 /* Entry exist - update port only */
1969 mvpp2_prs_hw_read(priv
, &pe
);
1971 /* Entry doesn't exist - create new */
1972 memset(&pe
, 0, sizeof(pe
));
1973 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_MAC
);
1976 /* Continue - set next lookup */
1977 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_DSA
);
1979 /* Set result info bits */
1980 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L2_MCAST
,
1981 MVPP2_PRS_RI_L2_CAST_MASK
);
1983 /* Update tcam entry data first byte */
1984 mvpp2_prs_tcam_data_byte_set(&pe
, 0, da_mc
, 0xff);
1986 /* Shift to ethertype */
1987 mvpp2_prs_sram_shift_set(&pe
, 2 * ETH_ALEN
,
1988 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
1990 /* Mask all ports */
1991 mvpp2_prs_tcam_port_map_set(&pe
, 0);
1993 /* Update shadow table */
1994 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_MAC
);
1997 /* Update port mask */
1998 mvpp2_prs_tcam_port_set(&pe
, port
, add
);
2000 mvpp2_prs_hw_write(priv
, &pe
);
2003 /* Set entry for dsa packets */
2004 static void mvpp2_prs_dsa_tag_set(struct mvpp2
*priv
, int port
, bool add
,
2005 bool tagged
, bool extend
)
2007 struct mvpp2_prs_entry pe
;
2011 tid
= tagged
? MVPP2_PE_EDSA_TAGGED
: MVPP2_PE_EDSA_UNTAGGED
;
2014 tid
= tagged
? MVPP2_PE_DSA_TAGGED
: MVPP2_PE_DSA_UNTAGGED
;
2018 if (priv
->prs_shadow
[tid
].valid
) {
2019 /* Entry exist - update port only */
2021 mvpp2_prs_hw_read(priv
, &pe
);
2023 /* Entry doesn't exist - create new */
2024 memset(&pe
, 0, sizeof(pe
));
2025 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_DSA
);
2028 /* Shift 4 bytes if DSA tag or 8 bytes in case of EDSA tag*/
2029 mvpp2_prs_sram_shift_set(&pe
, shift
,
2030 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
2032 /* Update shadow table */
2033 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_DSA
);
2036 /* Set tagged bit in DSA tag */
2037 mvpp2_prs_tcam_data_byte_set(&pe
, 0,
2038 MVPP2_PRS_TCAM_DSA_TAGGED_BIT
,
2039 MVPP2_PRS_TCAM_DSA_TAGGED_BIT
);
2040 /* Clear all ai bits for next iteration */
2041 mvpp2_prs_sram_ai_update(&pe
, 0,
2042 MVPP2_PRS_SRAM_AI_MASK
);
2043 /* If packet is tagged continue check vlans */
2044 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_VLAN
);
2046 /* Set result info bits to 'no vlans' */
2047 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_VLAN_NONE
,
2048 MVPP2_PRS_RI_VLAN_MASK
);
2049 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_L2
);
2052 /* Mask all ports */
2053 mvpp2_prs_tcam_port_map_set(&pe
, 0);
2056 /* Update port mask */
2057 mvpp2_prs_tcam_port_set(&pe
, port
, add
);
2059 mvpp2_prs_hw_write(priv
, &pe
);
2062 /* Set entry for dsa ethertype */
2063 static void mvpp2_prs_dsa_tag_ethertype_set(struct mvpp2
*priv
, int port
,
2064 bool add
, bool tagged
, bool extend
)
2066 struct mvpp2_prs_entry pe
;
2067 int tid
, shift
, port_mask
;
2070 tid
= tagged
? MVPP2_PE_ETYPE_EDSA_TAGGED
:
2071 MVPP2_PE_ETYPE_EDSA_UNTAGGED
;
2075 tid
= tagged
? MVPP2_PE_ETYPE_DSA_TAGGED
:
2076 MVPP2_PE_ETYPE_DSA_UNTAGGED
;
2077 port_mask
= MVPP2_PRS_PORT_MASK
;
2081 if (priv
->prs_shadow
[tid
].valid
) {
2082 /* Entry exist - update port only */
2084 mvpp2_prs_hw_read(priv
, &pe
);
2086 /* Entry doesn't exist - create new */
2087 memset(&pe
, 0, sizeof(pe
));
2088 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_DSA
);
2092 mvpp2_prs_match_etype(&pe
, 0, ETH_P_EDSA
);
2093 mvpp2_prs_match_etype(&pe
, 2, 0);
2095 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_DSA_MASK
,
2096 MVPP2_PRS_RI_DSA_MASK
);
2097 /* Shift ethertype + 2 byte reserved + tag*/
2098 mvpp2_prs_sram_shift_set(&pe
, 2 + MVPP2_ETH_TYPE_LEN
+ shift
,
2099 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
2101 /* Update shadow table */
2102 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_DSA
);
2105 /* Set tagged bit in DSA tag */
2106 mvpp2_prs_tcam_data_byte_set(&pe
,
2107 MVPP2_ETH_TYPE_LEN
+ 2 + 3,
2108 MVPP2_PRS_TCAM_DSA_TAGGED_BIT
,
2109 MVPP2_PRS_TCAM_DSA_TAGGED_BIT
);
2110 /* Clear all ai bits for next iteration */
2111 mvpp2_prs_sram_ai_update(&pe
, 0,
2112 MVPP2_PRS_SRAM_AI_MASK
);
2113 /* If packet is tagged continue check vlans */
2114 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_VLAN
);
2116 /* Set result info bits to 'no vlans' */
2117 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_VLAN_NONE
,
2118 MVPP2_PRS_RI_VLAN_MASK
);
2119 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_L2
);
2121 /* Mask/unmask all ports, depending on dsa type */
2122 mvpp2_prs_tcam_port_map_set(&pe
, port_mask
);
2125 /* Update port mask */
2126 mvpp2_prs_tcam_port_set(&pe
, port
, add
);
2128 mvpp2_prs_hw_write(priv
, &pe
);
2131 /* Search for existing single/triple vlan entry */
2132 static struct mvpp2_prs_entry
*mvpp2_prs_vlan_find(struct mvpp2
*priv
,
2133 unsigned short tpid
, int ai
)
2135 struct mvpp2_prs_entry
*pe
;
2138 pe
= kzalloc(sizeof(*pe
), GFP_KERNEL
);
2141 mvpp2_prs_tcam_lu_set(pe
, MVPP2_PRS_LU_VLAN
);
2143 /* Go through the all entries with MVPP2_PRS_LU_VLAN */
2144 for (tid
= MVPP2_PE_FIRST_FREE_TID
;
2145 tid
<= MVPP2_PE_LAST_FREE_TID
; tid
++) {
2146 unsigned int ri_bits
, ai_bits
;
2149 if (!priv
->prs_shadow
[tid
].valid
||
2150 priv
->prs_shadow
[tid
].lu
!= MVPP2_PRS_LU_VLAN
)
2155 mvpp2_prs_hw_read(priv
, pe
);
2156 match
= mvpp2_prs_tcam_data_cmp(pe
, 0, swab16(tpid
));
2161 ri_bits
= mvpp2_prs_sram_ri_get(pe
);
2162 ri_bits
&= MVPP2_PRS_RI_VLAN_MASK
;
2164 /* Get current ai value from tcam */
2165 ai_bits
= mvpp2_prs_tcam_ai_get(pe
);
2166 /* Clear double vlan bit */
2167 ai_bits
&= ~MVPP2_PRS_DBL_VLAN_AI_BIT
;
2172 if (ri_bits
== MVPP2_PRS_RI_VLAN_SINGLE
||
2173 ri_bits
== MVPP2_PRS_RI_VLAN_TRIPLE
)
2181 /* Add/update single/triple vlan entry */
2182 static int mvpp2_prs_vlan_add(struct mvpp2
*priv
, unsigned short tpid
, int ai
,
2183 unsigned int port_map
)
2185 struct mvpp2_prs_entry
*pe
;
2189 pe
= mvpp2_prs_vlan_find(priv
, tpid
, ai
);
2192 /* Create new tcam entry */
2193 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_LAST_FREE_TID
,
2194 MVPP2_PE_FIRST_FREE_TID
);
2198 pe
= kzalloc(sizeof(*pe
), GFP_KERNEL
);
2202 /* Get last double vlan tid */
2203 for (tid_aux
= MVPP2_PE_LAST_FREE_TID
;
2204 tid_aux
>= MVPP2_PE_FIRST_FREE_TID
; tid_aux
--) {
2205 unsigned int ri_bits
;
2207 if (!priv
->prs_shadow
[tid_aux
].valid
||
2208 priv
->prs_shadow
[tid_aux
].lu
!= MVPP2_PRS_LU_VLAN
)
2211 pe
->index
= tid_aux
;
2212 mvpp2_prs_hw_read(priv
, pe
);
2213 ri_bits
= mvpp2_prs_sram_ri_get(pe
);
2214 if ((ri_bits
& MVPP2_PRS_RI_VLAN_MASK
) ==
2215 MVPP2_PRS_RI_VLAN_DOUBLE
)
2219 if (tid
<= tid_aux
) {
2224 memset(pe
, 0, sizeof(*pe
));
2225 mvpp2_prs_tcam_lu_set(pe
, MVPP2_PRS_LU_VLAN
);
2228 mvpp2_prs_match_etype(pe
, 0, tpid
);
2230 mvpp2_prs_sram_next_lu_set(pe
, MVPP2_PRS_LU_L2
);
2231 /* Shift 4 bytes - skip 1 vlan tag */
2232 mvpp2_prs_sram_shift_set(pe
, MVPP2_VLAN_TAG_LEN
,
2233 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
2234 /* Clear all ai bits for next iteration */
2235 mvpp2_prs_sram_ai_update(pe
, 0, MVPP2_PRS_SRAM_AI_MASK
);
2237 if (ai
== MVPP2_PRS_SINGLE_VLAN_AI
) {
2238 mvpp2_prs_sram_ri_update(pe
, MVPP2_PRS_RI_VLAN_SINGLE
,
2239 MVPP2_PRS_RI_VLAN_MASK
);
2241 ai
|= MVPP2_PRS_DBL_VLAN_AI_BIT
;
2242 mvpp2_prs_sram_ri_update(pe
, MVPP2_PRS_RI_VLAN_TRIPLE
,
2243 MVPP2_PRS_RI_VLAN_MASK
);
2245 mvpp2_prs_tcam_ai_update(pe
, ai
, MVPP2_PRS_SRAM_AI_MASK
);
2247 mvpp2_prs_shadow_set(priv
, pe
->index
, MVPP2_PRS_LU_VLAN
);
2249 /* Update ports' mask */
2250 mvpp2_prs_tcam_port_map_set(pe
, port_map
);
2252 mvpp2_prs_hw_write(priv
, pe
);
2259 /* Get first free double vlan ai number */
2260 static int mvpp2_prs_double_vlan_ai_free_get(struct mvpp2
*priv
)
2264 for (i
= 1; i
< MVPP2_PRS_DBL_VLANS_MAX
; i
++) {
2265 if (!priv
->prs_double_vlans
[i
])
2272 /* Search for existing double vlan entry */
2273 static struct mvpp2_prs_entry
*mvpp2_prs_double_vlan_find(struct mvpp2
*priv
,
2274 unsigned short tpid1
,
2275 unsigned short tpid2
)
2277 struct mvpp2_prs_entry
*pe
;
2280 pe
= kzalloc(sizeof(*pe
), GFP_KERNEL
);
2283 mvpp2_prs_tcam_lu_set(pe
, MVPP2_PRS_LU_VLAN
);
2285 /* Go through the all entries with MVPP2_PRS_LU_VLAN */
2286 for (tid
= MVPP2_PE_FIRST_FREE_TID
;
2287 tid
<= MVPP2_PE_LAST_FREE_TID
; tid
++) {
2288 unsigned int ri_mask
;
2291 if (!priv
->prs_shadow
[tid
].valid
||
2292 priv
->prs_shadow
[tid
].lu
!= MVPP2_PRS_LU_VLAN
)
2296 mvpp2_prs_hw_read(priv
, pe
);
2298 match
= mvpp2_prs_tcam_data_cmp(pe
, 0, swab16(tpid1
))
2299 && mvpp2_prs_tcam_data_cmp(pe
, 4, swab16(tpid2
));
2304 ri_mask
= mvpp2_prs_sram_ri_get(pe
) & MVPP2_PRS_RI_VLAN_MASK
;
2305 if (ri_mask
== MVPP2_PRS_RI_VLAN_DOUBLE
)
2313 /* Add or update double vlan entry */
2314 static int mvpp2_prs_double_vlan_add(struct mvpp2
*priv
, unsigned short tpid1
,
2315 unsigned short tpid2
,
2316 unsigned int port_map
)
2318 struct mvpp2_prs_entry
*pe
;
2319 int tid_aux
, tid
, ai
, ret
= 0;
2321 pe
= mvpp2_prs_double_vlan_find(priv
, tpid1
, tpid2
);
2324 /* Create new tcam entry */
2325 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
2326 MVPP2_PE_LAST_FREE_TID
);
2330 pe
= kzalloc(sizeof(*pe
), GFP_KERNEL
);
2334 /* Set ai value for new double vlan entry */
2335 ai
= mvpp2_prs_double_vlan_ai_free_get(priv
);
2341 /* Get first single/triple vlan tid */
2342 for (tid_aux
= MVPP2_PE_FIRST_FREE_TID
;
2343 tid_aux
<= MVPP2_PE_LAST_FREE_TID
; tid_aux
++) {
2344 unsigned int ri_bits
;
2346 if (!priv
->prs_shadow
[tid_aux
].valid
||
2347 priv
->prs_shadow
[tid_aux
].lu
!= MVPP2_PRS_LU_VLAN
)
2350 pe
->index
= tid_aux
;
2351 mvpp2_prs_hw_read(priv
, pe
);
2352 ri_bits
= mvpp2_prs_sram_ri_get(pe
);
2353 ri_bits
&= MVPP2_PRS_RI_VLAN_MASK
;
2354 if (ri_bits
== MVPP2_PRS_RI_VLAN_SINGLE
||
2355 ri_bits
== MVPP2_PRS_RI_VLAN_TRIPLE
)
2359 if (tid
>= tid_aux
) {
2364 memset(pe
, 0, sizeof(*pe
));
2365 mvpp2_prs_tcam_lu_set(pe
, MVPP2_PRS_LU_VLAN
);
2368 priv
->prs_double_vlans
[ai
] = true;
2370 mvpp2_prs_match_etype(pe
, 0, tpid1
);
2371 mvpp2_prs_match_etype(pe
, 4, tpid2
);
2373 mvpp2_prs_sram_next_lu_set(pe
, MVPP2_PRS_LU_VLAN
);
2374 /* Shift 8 bytes - skip 2 vlan tags */
2375 mvpp2_prs_sram_shift_set(pe
, 2 * MVPP2_VLAN_TAG_LEN
,
2376 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
2377 mvpp2_prs_sram_ri_update(pe
, MVPP2_PRS_RI_VLAN_DOUBLE
,
2378 MVPP2_PRS_RI_VLAN_MASK
);
2379 mvpp2_prs_sram_ai_update(pe
, ai
| MVPP2_PRS_DBL_VLAN_AI_BIT
,
2380 MVPP2_PRS_SRAM_AI_MASK
);
2382 mvpp2_prs_shadow_set(priv
, pe
->index
, MVPP2_PRS_LU_VLAN
);
2385 /* Update ports' mask */
2386 mvpp2_prs_tcam_port_map_set(pe
, port_map
);
2387 mvpp2_prs_hw_write(priv
, pe
);
2393 /* IPv4 header parsing for fragmentation and L4 offset */
2394 static int mvpp2_prs_ip4_proto(struct mvpp2
*priv
, unsigned short proto
,
2395 unsigned int ri
, unsigned int ri_mask
)
2397 struct mvpp2_prs_entry pe
;
2400 if ((proto
!= IPPROTO_TCP
) && (proto
!= IPPROTO_UDP
) &&
2401 (proto
!= IPPROTO_IGMP
))
2404 /* Not fragmented packet */
2405 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
2406 MVPP2_PE_LAST_FREE_TID
);
2410 memset(&pe
, 0, sizeof(pe
));
2411 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_IP4
);
2414 /* Set next lu to IPv4 */
2415 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_IP4
);
2416 mvpp2_prs_sram_shift_set(&pe
, 12, MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
2418 mvpp2_prs_sram_offset_set(&pe
, MVPP2_PRS_SRAM_UDF_TYPE_L4
,
2419 sizeof(struct iphdr
) - 4,
2420 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD
);
2421 mvpp2_prs_sram_ai_update(&pe
, MVPP2_PRS_IPV4_DIP_AI_BIT
,
2422 MVPP2_PRS_IPV4_DIP_AI_BIT
);
2423 mvpp2_prs_sram_ri_update(&pe
, ri
, ri_mask
| MVPP2_PRS_RI_IP_FRAG_MASK
);
2425 mvpp2_prs_tcam_data_byte_set(&pe
, 2, 0x00,
2426 MVPP2_PRS_TCAM_PROTO_MASK_L
);
2427 mvpp2_prs_tcam_data_byte_set(&pe
, 3, 0x00,
2428 MVPP2_PRS_TCAM_PROTO_MASK
);
2430 mvpp2_prs_tcam_data_byte_set(&pe
, 5, proto
, MVPP2_PRS_TCAM_PROTO_MASK
);
2431 mvpp2_prs_tcam_ai_update(&pe
, 0, MVPP2_PRS_IPV4_DIP_AI_BIT
);
2432 /* Unmask all ports */
2433 mvpp2_prs_tcam_port_map_set(&pe
, MVPP2_PRS_PORT_MASK
);
2435 /* Update shadow table and hw entry */
2436 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_IP4
);
2437 mvpp2_prs_hw_write(priv
, &pe
);
2439 /* Fragmented packet */
2440 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
2441 MVPP2_PE_LAST_FREE_TID
);
2446 /* Clear ri before updating */
2447 pe
.sram
.word
[MVPP2_PRS_SRAM_RI_WORD
] = 0x0;
2448 pe
.sram
.word
[MVPP2_PRS_SRAM_RI_CTRL_WORD
] = 0x0;
2449 mvpp2_prs_sram_ri_update(&pe
, ri
, ri_mask
);
2451 mvpp2_prs_sram_ri_update(&pe
, ri
| MVPP2_PRS_RI_IP_FRAG_TRUE
,
2452 ri_mask
| MVPP2_PRS_RI_IP_FRAG_MASK
);
2454 mvpp2_prs_tcam_data_byte_set(&pe
, 2, 0x00, 0x0);
2455 mvpp2_prs_tcam_data_byte_set(&pe
, 3, 0x00, 0x0);
2457 /* Update shadow table and hw entry */
2458 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_IP4
);
2459 mvpp2_prs_hw_write(priv
, &pe
);
2464 /* IPv4 L3 multicast or broadcast */
2465 static int mvpp2_prs_ip4_cast(struct mvpp2
*priv
, unsigned short l3_cast
)
2467 struct mvpp2_prs_entry pe
;
2470 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
2471 MVPP2_PE_LAST_FREE_TID
);
2475 memset(&pe
, 0, sizeof(pe
));
2476 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_IP4
);
2480 case MVPP2_PRS_L3_MULTI_CAST
:
2481 mvpp2_prs_tcam_data_byte_set(&pe
, 0, MVPP2_PRS_IPV4_MC
,
2482 MVPP2_PRS_IPV4_MC_MASK
);
2483 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L3_MCAST
,
2484 MVPP2_PRS_RI_L3_ADDR_MASK
);
2486 case MVPP2_PRS_L3_BROAD_CAST
:
2487 mask
= MVPP2_PRS_IPV4_BC_MASK
;
2488 mvpp2_prs_tcam_data_byte_set(&pe
, 0, mask
, mask
);
2489 mvpp2_prs_tcam_data_byte_set(&pe
, 1, mask
, mask
);
2490 mvpp2_prs_tcam_data_byte_set(&pe
, 2, mask
, mask
);
2491 mvpp2_prs_tcam_data_byte_set(&pe
, 3, mask
, mask
);
2492 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L3_BCAST
,
2493 MVPP2_PRS_RI_L3_ADDR_MASK
);
2499 /* Finished: go to flowid generation */
2500 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_FLOWS
);
2501 mvpp2_prs_sram_bits_set(&pe
, MVPP2_PRS_SRAM_LU_GEN_BIT
, 1);
2503 mvpp2_prs_tcam_ai_update(&pe
, MVPP2_PRS_IPV4_DIP_AI_BIT
,
2504 MVPP2_PRS_IPV4_DIP_AI_BIT
);
2505 /* Unmask all ports */
2506 mvpp2_prs_tcam_port_map_set(&pe
, MVPP2_PRS_PORT_MASK
);
2508 /* Update shadow table and hw entry */
2509 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_IP4
);
2510 mvpp2_prs_hw_write(priv
, &pe
);
2515 /* Set entries for protocols over IPv6 */
2516 static int mvpp2_prs_ip6_proto(struct mvpp2
*priv
, unsigned short proto
,
2517 unsigned int ri
, unsigned int ri_mask
)
2519 struct mvpp2_prs_entry pe
;
2522 if ((proto
!= IPPROTO_TCP
) && (proto
!= IPPROTO_UDP
) &&
2523 (proto
!= IPPROTO_ICMPV6
) && (proto
!= IPPROTO_IPIP
))
2526 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
2527 MVPP2_PE_LAST_FREE_TID
);
2531 memset(&pe
, 0, sizeof(pe
));
2532 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_IP6
);
2535 /* Finished: go to flowid generation */
2536 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_FLOWS
);
2537 mvpp2_prs_sram_bits_set(&pe
, MVPP2_PRS_SRAM_LU_GEN_BIT
, 1);
2538 mvpp2_prs_sram_ri_update(&pe
, ri
, ri_mask
);
2539 mvpp2_prs_sram_offset_set(&pe
, MVPP2_PRS_SRAM_UDF_TYPE_L4
,
2540 sizeof(struct ipv6hdr
) - 6,
2541 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD
);
2543 mvpp2_prs_tcam_data_byte_set(&pe
, 0, proto
, MVPP2_PRS_TCAM_PROTO_MASK
);
2544 mvpp2_prs_tcam_ai_update(&pe
, MVPP2_PRS_IPV6_NO_EXT_AI_BIT
,
2545 MVPP2_PRS_IPV6_NO_EXT_AI_BIT
);
2546 /* Unmask all ports */
2547 mvpp2_prs_tcam_port_map_set(&pe
, MVPP2_PRS_PORT_MASK
);
2550 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_IP6
);
2551 mvpp2_prs_hw_write(priv
, &pe
);
2556 /* IPv6 L3 multicast entry */
2557 static int mvpp2_prs_ip6_cast(struct mvpp2
*priv
, unsigned short l3_cast
)
2559 struct mvpp2_prs_entry pe
;
2562 if (l3_cast
!= MVPP2_PRS_L3_MULTI_CAST
)
2565 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
2566 MVPP2_PE_LAST_FREE_TID
);
2570 memset(&pe
, 0, sizeof(pe
));
2571 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_IP6
);
2574 /* Finished: go to flowid generation */
2575 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_IP6
);
2576 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L3_MCAST
,
2577 MVPP2_PRS_RI_L3_ADDR_MASK
);
2578 mvpp2_prs_sram_ai_update(&pe
, MVPP2_PRS_IPV6_NO_EXT_AI_BIT
,
2579 MVPP2_PRS_IPV6_NO_EXT_AI_BIT
);
2580 /* Shift back to IPv6 NH */
2581 mvpp2_prs_sram_shift_set(&pe
, -18, MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
2583 mvpp2_prs_tcam_data_byte_set(&pe
, 0, MVPP2_PRS_IPV6_MC
,
2584 MVPP2_PRS_IPV6_MC_MASK
);
2585 mvpp2_prs_tcam_ai_update(&pe
, 0, MVPP2_PRS_IPV6_NO_EXT_AI_BIT
);
2586 /* Unmask all ports */
2587 mvpp2_prs_tcam_port_map_set(&pe
, MVPP2_PRS_PORT_MASK
);
2589 /* Update shadow table and hw entry */
2590 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_IP6
);
2591 mvpp2_prs_hw_write(priv
, &pe
);
2596 /* Parser per-port initialization */
2597 static void mvpp2_prs_hw_port_init(struct mvpp2
*priv
, int port
, int lu_first
,
2598 int lu_max
, int offset
)
2603 val
= mvpp2_read(priv
, MVPP2_PRS_INIT_LOOKUP_REG
);
2604 val
&= ~MVPP2_PRS_PORT_LU_MASK(port
);
2605 val
|= MVPP2_PRS_PORT_LU_VAL(port
, lu_first
);
2606 mvpp2_write(priv
, MVPP2_PRS_INIT_LOOKUP_REG
, val
);
2608 /* Set maximum number of loops for packet received from port */
2609 val
= mvpp2_read(priv
, MVPP2_PRS_MAX_LOOP_REG(port
));
2610 val
&= ~MVPP2_PRS_MAX_LOOP_MASK(port
);
2611 val
|= MVPP2_PRS_MAX_LOOP_VAL(port
, lu_max
);
2612 mvpp2_write(priv
, MVPP2_PRS_MAX_LOOP_REG(port
), val
);
2614 /* Set initial offset for packet header extraction for the first
2617 val
= mvpp2_read(priv
, MVPP2_PRS_INIT_OFFS_REG(port
));
2618 val
&= ~MVPP2_PRS_INIT_OFF_MASK(port
);
2619 val
|= MVPP2_PRS_INIT_OFF_VAL(port
, offset
);
2620 mvpp2_write(priv
, MVPP2_PRS_INIT_OFFS_REG(port
), val
);
2623 /* Default flow entries initialization for all ports */
2624 static void mvpp2_prs_def_flow_init(struct mvpp2
*priv
)
2626 struct mvpp2_prs_entry pe
;
2629 for (port
= 0; port
< MVPP2_MAX_PORTS
; port
++) {
2630 memset(&pe
, 0, sizeof(pe
));
2631 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_FLOWS
);
2632 pe
.index
= MVPP2_PE_FIRST_DEFAULT_FLOW
- port
;
2634 /* Mask all ports */
2635 mvpp2_prs_tcam_port_map_set(&pe
, 0);
2638 mvpp2_prs_sram_ai_update(&pe
, port
, MVPP2_PRS_FLOW_ID_MASK
);
2639 mvpp2_prs_sram_bits_set(&pe
, MVPP2_PRS_SRAM_LU_DONE_BIT
, 1);
2641 /* Update shadow table and hw entry */
2642 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_FLOWS
);
2643 mvpp2_prs_hw_write(priv
, &pe
);
2647 /* Set default entry for Marvell Header field */
2648 static void mvpp2_prs_mh_init(struct mvpp2
*priv
)
2650 struct mvpp2_prs_entry pe
;
2652 memset(&pe
, 0, sizeof(pe
));
2654 pe
.index
= MVPP2_PE_MH_DEFAULT
;
2655 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_MH
);
2656 mvpp2_prs_sram_shift_set(&pe
, MVPP2_MH_SIZE
,
2657 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
2658 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_MAC
);
2660 /* Unmask all ports */
2661 mvpp2_prs_tcam_port_map_set(&pe
, MVPP2_PRS_PORT_MASK
);
2663 /* Update shadow table and hw entry */
2664 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_MH
);
2665 mvpp2_prs_hw_write(priv
, &pe
);
2668 /* Set default entires (place holder) for promiscuous, non-promiscuous and
2669 * multicast MAC addresses
2671 static void mvpp2_prs_mac_init(struct mvpp2
*priv
)
2673 struct mvpp2_prs_entry pe
;
2675 memset(&pe
, 0, sizeof(pe
));
2677 /* Non-promiscuous mode for all ports - DROP unknown packets */
2678 pe
.index
= MVPP2_PE_MAC_NON_PROMISCUOUS
;
2679 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_MAC
);
2681 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_DROP_MASK
,
2682 MVPP2_PRS_RI_DROP_MASK
);
2683 mvpp2_prs_sram_bits_set(&pe
, MVPP2_PRS_SRAM_LU_GEN_BIT
, 1);
2684 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_FLOWS
);
2686 /* Unmask all ports */
2687 mvpp2_prs_tcam_port_map_set(&pe
, MVPP2_PRS_PORT_MASK
);
2689 /* Update shadow table and hw entry */
2690 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_MAC
);
2691 mvpp2_prs_hw_write(priv
, &pe
);
2693 /* place holders only - no ports */
2694 mvpp2_prs_mac_drop_all_set(priv
, 0, false);
2695 mvpp2_prs_mac_promisc_set(priv
, 0, false);
2696 mvpp2_prs_mac_multi_set(priv
, 0, MVPP2_PE_MAC_MC_ALL
, false);
2697 mvpp2_prs_mac_multi_set(priv
, 0, MVPP2_PE_MAC_MC_IP6
, false);
2700 /* Set default entries for various types of dsa packets */
2701 static void mvpp2_prs_dsa_init(struct mvpp2
*priv
)
2703 struct mvpp2_prs_entry pe
;
2705 /* None tagged EDSA entry - place holder */
2706 mvpp2_prs_dsa_tag_set(priv
, 0, false, MVPP2_PRS_UNTAGGED
,
2709 /* Tagged EDSA entry - place holder */
2710 mvpp2_prs_dsa_tag_set(priv
, 0, false, MVPP2_PRS_TAGGED
, MVPP2_PRS_EDSA
);
2712 /* None tagged DSA entry - place holder */
2713 mvpp2_prs_dsa_tag_set(priv
, 0, false, MVPP2_PRS_UNTAGGED
,
2716 /* Tagged DSA entry - place holder */
2717 mvpp2_prs_dsa_tag_set(priv
, 0, false, MVPP2_PRS_TAGGED
, MVPP2_PRS_DSA
);
2719 /* None tagged EDSA ethertype entry - place holder*/
2720 mvpp2_prs_dsa_tag_ethertype_set(priv
, 0, false,
2721 MVPP2_PRS_UNTAGGED
, MVPP2_PRS_EDSA
);
2723 /* Tagged EDSA ethertype entry - place holder*/
2724 mvpp2_prs_dsa_tag_ethertype_set(priv
, 0, false,
2725 MVPP2_PRS_TAGGED
, MVPP2_PRS_EDSA
);
2727 /* None tagged DSA ethertype entry */
2728 mvpp2_prs_dsa_tag_ethertype_set(priv
, 0, true,
2729 MVPP2_PRS_UNTAGGED
, MVPP2_PRS_DSA
);
2731 /* Tagged DSA ethertype entry */
2732 mvpp2_prs_dsa_tag_ethertype_set(priv
, 0, true,
2733 MVPP2_PRS_TAGGED
, MVPP2_PRS_DSA
);
2735 /* Set default entry, in case DSA or EDSA tag not found */
2736 memset(&pe
, 0, sizeof(pe
));
2737 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_DSA
);
2738 pe
.index
= MVPP2_PE_DSA_DEFAULT
;
2739 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_VLAN
);
2742 mvpp2_prs_sram_shift_set(&pe
, 0, MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
2743 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_MAC
);
2745 /* Clear all sram ai bits for next iteration */
2746 mvpp2_prs_sram_ai_update(&pe
, 0, MVPP2_PRS_SRAM_AI_MASK
);
2748 /* Unmask all ports */
2749 mvpp2_prs_tcam_port_map_set(&pe
, MVPP2_PRS_PORT_MASK
);
2751 mvpp2_prs_hw_write(priv
, &pe
);
2754 /* Match basic ethertypes */
2755 static int mvpp2_prs_etype_init(struct mvpp2
*priv
)
2757 struct mvpp2_prs_entry pe
;
2760 /* Ethertype: PPPoE */
2761 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
2762 MVPP2_PE_LAST_FREE_TID
);
2766 memset(&pe
, 0, sizeof(pe
));
2767 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_L2
);
2770 mvpp2_prs_match_etype(&pe
, 0, ETH_P_PPP_SES
);
2772 mvpp2_prs_sram_shift_set(&pe
, MVPP2_PPPOE_HDR_SIZE
,
2773 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
2774 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_PPPOE
);
2775 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_PPPOE_MASK
,
2776 MVPP2_PRS_RI_PPPOE_MASK
);
2778 /* Update shadow table and hw entry */
2779 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_L2
);
2780 priv
->prs_shadow
[pe
.index
].udf
= MVPP2_PRS_UDF_L2_DEF
;
2781 priv
->prs_shadow
[pe
.index
].finish
= false;
2782 mvpp2_prs_shadow_ri_set(priv
, pe
.index
, MVPP2_PRS_RI_PPPOE_MASK
,
2783 MVPP2_PRS_RI_PPPOE_MASK
);
2784 mvpp2_prs_hw_write(priv
, &pe
);
2786 /* Ethertype: ARP */
2787 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
2788 MVPP2_PE_LAST_FREE_TID
);
2792 memset(&pe
, 0, sizeof(pe
));
2793 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_L2
);
2796 mvpp2_prs_match_etype(&pe
, 0, ETH_P_ARP
);
2798 /* Generate flow in the next iteration*/
2799 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_FLOWS
);
2800 mvpp2_prs_sram_bits_set(&pe
, MVPP2_PRS_SRAM_LU_GEN_BIT
, 1);
2801 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L3_ARP
,
2802 MVPP2_PRS_RI_L3_PROTO_MASK
);
2804 mvpp2_prs_sram_offset_set(&pe
, MVPP2_PRS_SRAM_UDF_TYPE_L3
,
2806 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD
);
2808 /* Update shadow table and hw entry */
2809 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_L2
);
2810 priv
->prs_shadow
[pe
.index
].udf
= MVPP2_PRS_UDF_L2_DEF
;
2811 priv
->prs_shadow
[pe
.index
].finish
= true;
2812 mvpp2_prs_shadow_ri_set(priv
, pe
.index
, MVPP2_PRS_RI_L3_ARP
,
2813 MVPP2_PRS_RI_L3_PROTO_MASK
);
2814 mvpp2_prs_hw_write(priv
, &pe
);
2816 /* Ethertype: LBTD */
2817 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
2818 MVPP2_PE_LAST_FREE_TID
);
2822 memset(&pe
, 0, sizeof(pe
));
2823 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_L2
);
2826 mvpp2_prs_match_etype(&pe
, 0, MVPP2_IP_LBDT_TYPE
);
2828 /* Generate flow in the next iteration*/
2829 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_FLOWS
);
2830 mvpp2_prs_sram_bits_set(&pe
, MVPP2_PRS_SRAM_LU_GEN_BIT
, 1);
2831 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_CPU_CODE_RX_SPEC
|
2832 MVPP2_PRS_RI_UDF3_RX_SPECIAL
,
2833 MVPP2_PRS_RI_CPU_CODE_MASK
|
2834 MVPP2_PRS_RI_UDF3_MASK
);
2836 mvpp2_prs_sram_offset_set(&pe
, MVPP2_PRS_SRAM_UDF_TYPE_L3
,
2838 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD
);
2840 /* Update shadow table and hw entry */
2841 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_L2
);
2842 priv
->prs_shadow
[pe
.index
].udf
= MVPP2_PRS_UDF_L2_DEF
;
2843 priv
->prs_shadow
[pe
.index
].finish
= true;
2844 mvpp2_prs_shadow_ri_set(priv
, pe
.index
, MVPP2_PRS_RI_CPU_CODE_RX_SPEC
|
2845 MVPP2_PRS_RI_UDF3_RX_SPECIAL
,
2846 MVPP2_PRS_RI_CPU_CODE_MASK
|
2847 MVPP2_PRS_RI_UDF3_MASK
);
2848 mvpp2_prs_hw_write(priv
, &pe
);
2850 /* Ethertype: IPv4 without options */
2851 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
2852 MVPP2_PE_LAST_FREE_TID
);
2856 memset(&pe
, 0, sizeof(pe
));
2857 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_L2
);
2860 mvpp2_prs_match_etype(&pe
, 0, ETH_P_IP
);
2861 mvpp2_prs_tcam_data_byte_set(&pe
, MVPP2_ETH_TYPE_LEN
,
2862 MVPP2_PRS_IPV4_HEAD
| MVPP2_PRS_IPV4_IHL
,
2863 MVPP2_PRS_IPV4_HEAD_MASK
|
2864 MVPP2_PRS_IPV4_IHL_MASK
);
2866 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_IP4
);
2867 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L3_IP4
,
2868 MVPP2_PRS_RI_L3_PROTO_MASK
);
2869 /* Skip eth_type + 4 bytes of IP header */
2870 mvpp2_prs_sram_shift_set(&pe
, MVPP2_ETH_TYPE_LEN
+ 4,
2871 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
2873 mvpp2_prs_sram_offset_set(&pe
, MVPP2_PRS_SRAM_UDF_TYPE_L3
,
2875 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD
);
2877 /* Update shadow table and hw entry */
2878 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_L2
);
2879 priv
->prs_shadow
[pe
.index
].udf
= MVPP2_PRS_UDF_L2_DEF
;
2880 priv
->prs_shadow
[pe
.index
].finish
= false;
2881 mvpp2_prs_shadow_ri_set(priv
, pe
.index
, MVPP2_PRS_RI_L3_IP4
,
2882 MVPP2_PRS_RI_L3_PROTO_MASK
);
2883 mvpp2_prs_hw_write(priv
, &pe
);
2885 /* Ethertype: IPv4 with options */
2886 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
2887 MVPP2_PE_LAST_FREE_TID
);
2893 /* Clear tcam data before updating */
2894 pe
.tcam
.byte
[MVPP2_PRS_TCAM_DATA_BYTE(MVPP2_ETH_TYPE_LEN
)] = 0x0;
2895 pe
.tcam
.byte
[MVPP2_PRS_TCAM_DATA_BYTE_EN(MVPP2_ETH_TYPE_LEN
)] = 0x0;
2897 mvpp2_prs_tcam_data_byte_set(&pe
, MVPP2_ETH_TYPE_LEN
,
2898 MVPP2_PRS_IPV4_HEAD
,
2899 MVPP2_PRS_IPV4_HEAD_MASK
);
2901 /* Clear ri before updating */
2902 pe
.sram
.word
[MVPP2_PRS_SRAM_RI_WORD
] = 0x0;
2903 pe
.sram
.word
[MVPP2_PRS_SRAM_RI_CTRL_WORD
] = 0x0;
2904 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L3_IP4_OPT
,
2905 MVPP2_PRS_RI_L3_PROTO_MASK
);
2907 /* Update shadow table and hw entry */
2908 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_L2
);
2909 priv
->prs_shadow
[pe
.index
].udf
= MVPP2_PRS_UDF_L2_DEF
;
2910 priv
->prs_shadow
[pe
.index
].finish
= false;
2911 mvpp2_prs_shadow_ri_set(priv
, pe
.index
, MVPP2_PRS_RI_L3_IP4_OPT
,
2912 MVPP2_PRS_RI_L3_PROTO_MASK
);
2913 mvpp2_prs_hw_write(priv
, &pe
);
2915 /* Ethertype: IPv6 without options */
2916 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
2917 MVPP2_PE_LAST_FREE_TID
);
2921 memset(&pe
, 0, sizeof(pe
));
2922 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_L2
);
2925 mvpp2_prs_match_etype(&pe
, 0, ETH_P_IPV6
);
2927 /* Skip DIP of IPV6 header */
2928 mvpp2_prs_sram_shift_set(&pe
, MVPP2_ETH_TYPE_LEN
+ 8 +
2929 MVPP2_MAX_L3_ADDR_SIZE
,
2930 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
2931 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_IP6
);
2932 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L3_IP6
,
2933 MVPP2_PRS_RI_L3_PROTO_MASK
);
2935 mvpp2_prs_sram_offset_set(&pe
, MVPP2_PRS_SRAM_UDF_TYPE_L3
,
2937 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD
);
2939 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_L2
);
2940 priv
->prs_shadow
[pe
.index
].udf
= MVPP2_PRS_UDF_L2_DEF
;
2941 priv
->prs_shadow
[pe
.index
].finish
= false;
2942 mvpp2_prs_shadow_ri_set(priv
, pe
.index
, MVPP2_PRS_RI_L3_IP6
,
2943 MVPP2_PRS_RI_L3_PROTO_MASK
);
2944 mvpp2_prs_hw_write(priv
, &pe
);
2946 /* Default entry for MVPP2_PRS_LU_L2 - Unknown ethtype */
2947 memset(&pe
, 0, sizeof(struct mvpp2_prs_entry
));
2948 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_L2
);
2949 pe
.index
= MVPP2_PE_ETH_TYPE_UN
;
2951 /* Unmask all ports */
2952 mvpp2_prs_tcam_port_map_set(&pe
, MVPP2_PRS_PORT_MASK
);
2954 /* Generate flow in the next iteration*/
2955 mvpp2_prs_sram_bits_set(&pe
, MVPP2_PRS_SRAM_LU_GEN_BIT
, 1);
2956 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_FLOWS
);
2957 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L3_UN
,
2958 MVPP2_PRS_RI_L3_PROTO_MASK
);
2959 /* Set L3 offset even it's unknown L3 */
2960 mvpp2_prs_sram_offset_set(&pe
, MVPP2_PRS_SRAM_UDF_TYPE_L3
,
2962 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD
);
2964 /* Update shadow table and hw entry */
2965 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_L2
);
2966 priv
->prs_shadow
[pe
.index
].udf
= MVPP2_PRS_UDF_L2_DEF
;
2967 priv
->prs_shadow
[pe
.index
].finish
= true;
2968 mvpp2_prs_shadow_ri_set(priv
, pe
.index
, MVPP2_PRS_RI_L3_UN
,
2969 MVPP2_PRS_RI_L3_PROTO_MASK
);
2970 mvpp2_prs_hw_write(priv
, &pe
);
2975 /* Configure vlan entries and detect up to 2 successive VLAN tags.
2982 static int mvpp2_prs_vlan_init(struct platform_device
*pdev
, struct mvpp2
*priv
)
2984 struct mvpp2_prs_entry pe
;
2987 priv
->prs_double_vlans
= devm_kcalloc(&pdev
->dev
, sizeof(bool),
2988 MVPP2_PRS_DBL_VLANS_MAX
,
2990 if (!priv
->prs_double_vlans
)
2993 /* Double VLAN: 0x8100, 0x88A8 */
2994 err
= mvpp2_prs_double_vlan_add(priv
, ETH_P_8021Q
, ETH_P_8021AD
,
2995 MVPP2_PRS_PORT_MASK
);
2999 /* Double VLAN: 0x8100, 0x8100 */
3000 err
= mvpp2_prs_double_vlan_add(priv
, ETH_P_8021Q
, ETH_P_8021Q
,
3001 MVPP2_PRS_PORT_MASK
);
3005 /* Single VLAN: 0x88a8 */
3006 err
= mvpp2_prs_vlan_add(priv
, ETH_P_8021AD
, MVPP2_PRS_SINGLE_VLAN_AI
,
3007 MVPP2_PRS_PORT_MASK
);
3011 /* Single VLAN: 0x8100 */
3012 err
= mvpp2_prs_vlan_add(priv
, ETH_P_8021Q
, MVPP2_PRS_SINGLE_VLAN_AI
,
3013 MVPP2_PRS_PORT_MASK
);
3017 /* Set default double vlan entry */
3018 memset(&pe
, 0, sizeof(pe
));
3019 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_VLAN
);
3020 pe
.index
= MVPP2_PE_VLAN_DBL
;
3022 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_L2
);
3023 /* Clear ai for next iterations */
3024 mvpp2_prs_sram_ai_update(&pe
, 0, MVPP2_PRS_SRAM_AI_MASK
);
3025 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_VLAN_DOUBLE
,
3026 MVPP2_PRS_RI_VLAN_MASK
);
3028 mvpp2_prs_tcam_ai_update(&pe
, MVPP2_PRS_DBL_VLAN_AI_BIT
,
3029 MVPP2_PRS_DBL_VLAN_AI_BIT
);
3030 /* Unmask all ports */
3031 mvpp2_prs_tcam_port_map_set(&pe
, MVPP2_PRS_PORT_MASK
);
3033 /* Update shadow table and hw entry */
3034 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_VLAN
);
3035 mvpp2_prs_hw_write(priv
, &pe
);
3037 /* Set default vlan none entry */
3038 memset(&pe
, 0, sizeof(pe
));
3039 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_VLAN
);
3040 pe
.index
= MVPP2_PE_VLAN_NONE
;
3042 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_L2
);
3043 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_VLAN_NONE
,
3044 MVPP2_PRS_RI_VLAN_MASK
);
3046 /* Unmask all ports */
3047 mvpp2_prs_tcam_port_map_set(&pe
, MVPP2_PRS_PORT_MASK
);
3049 /* Update shadow table and hw entry */
3050 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_VLAN
);
3051 mvpp2_prs_hw_write(priv
, &pe
);
3056 /* Set entries for PPPoE ethertype */
3057 static int mvpp2_prs_pppoe_init(struct mvpp2
*priv
)
3059 struct mvpp2_prs_entry pe
;
3062 /* IPv4 over PPPoE with options */
3063 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
3064 MVPP2_PE_LAST_FREE_TID
);
3068 memset(&pe
, 0, sizeof(pe
));
3069 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_PPPOE
);
3072 mvpp2_prs_match_etype(&pe
, 0, PPP_IP
);
3074 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_IP4
);
3075 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L3_IP4_OPT
,
3076 MVPP2_PRS_RI_L3_PROTO_MASK
);
3077 /* Skip eth_type + 4 bytes of IP header */
3078 mvpp2_prs_sram_shift_set(&pe
, MVPP2_ETH_TYPE_LEN
+ 4,
3079 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
3081 mvpp2_prs_sram_offset_set(&pe
, MVPP2_PRS_SRAM_UDF_TYPE_L3
,
3083 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD
);
3085 /* Update shadow table and hw entry */
3086 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_PPPOE
);
3087 mvpp2_prs_hw_write(priv
, &pe
);
3089 /* IPv4 over PPPoE without options */
3090 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
3091 MVPP2_PE_LAST_FREE_TID
);
3097 mvpp2_prs_tcam_data_byte_set(&pe
, MVPP2_ETH_TYPE_LEN
,
3098 MVPP2_PRS_IPV4_HEAD
| MVPP2_PRS_IPV4_IHL
,
3099 MVPP2_PRS_IPV4_HEAD_MASK
|
3100 MVPP2_PRS_IPV4_IHL_MASK
);
3102 /* Clear ri before updating */
3103 pe
.sram
.word
[MVPP2_PRS_SRAM_RI_WORD
] = 0x0;
3104 pe
.sram
.word
[MVPP2_PRS_SRAM_RI_CTRL_WORD
] = 0x0;
3105 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L3_IP4
,
3106 MVPP2_PRS_RI_L3_PROTO_MASK
);
3108 /* Update shadow table and hw entry */
3109 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_PPPOE
);
3110 mvpp2_prs_hw_write(priv
, &pe
);
3112 /* IPv6 over PPPoE */
3113 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
3114 MVPP2_PE_LAST_FREE_TID
);
3118 memset(&pe
, 0, sizeof(pe
));
3119 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_PPPOE
);
3122 mvpp2_prs_match_etype(&pe
, 0, PPP_IPV6
);
3124 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_IP6
);
3125 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L3_IP6
,
3126 MVPP2_PRS_RI_L3_PROTO_MASK
);
3127 /* Skip eth_type + 4 bytes of IPv6 header */
3128 mvpp2_prs_sram_shift_set(&pe
, MVPP2_ETH_TYPE_LEN
+ 4,
3129 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
3131 mvpp2_prs_sram_offset_set(&pe
, MVPP2_PRS_SRAM_UDF_TYPE_L3
,
3133 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD
);
3135 /* Update shadow table and hw entry */
3136 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_PPPOE
);
3137 mvpp2_prs_hw_write(priv
, &pe
);
3139 /* Non-IP over PPPoE */
3140 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
3141 MVPP2_PE_LAST_FREE_TID
);
3145 memset(&pe
, 0, sizeof(pe
));
3146 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_PPPOE
);
3149 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L3_UN
,
3150 MVPP2_PRS_RI_L3_PROTO_MASK
);
3152 /* Finished: go to flowid generation */
3153 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_FLOWS
);
3154 mvpp2_prs_sram_bits_set(&pe
, MVPP2_PRS_SRAM_LU_GEN_BIT
, 1);
3155 /* Set L3 offset even if it's unknown L3 */
3156 mvpp2_prs_sram_offset_set(&pe
, MVPP2_PRS_SRAM_UDF_TYPE_L3
,
3158 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD
);
3160 /* Update shadow table and hw entry */
3161 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_PPPOE
);
3162 mvpp2_prs_hw_write(priv
, &pe
);
3167 /* Initialize entries for IPv4 */
3168 static int mvpp2_prs_ip4_init(struct mvpp2
*priv
)
3170 struct mvpp2_prs_entry pe
;
3173 /* Set entries for TCP, UDP and IGMP over IPv4 */
3174 err
= mvpp2_prs_ip4_proto(priv
, IPPROTO_TCP
, MVPP2_PRS_RI_L4_TCP
,
3175 MVPP2_PRS_RI_L4_PROTO_MASK
);
3179 err
= mvpp2_prs_ip4_proto(priv
, IPPROTO_UDP
, MVPP2_PRS_RI_L4_UDP
,
3180 MVPP2_PRS_RI_L4_PROTO_MASK
);
3184 err
= mvpp2_prs_ip4_proto(priv
, IPPROTO_IGMP
,
3185 MVPP2_PRS_RI_CPU_CODE_RX_SPEC
|
3186 MVPP2_PRS_RI_UDF3_RX_SPECIAL
,
3187 MVPP2_PRS_RI_CPU_CODE_MASK
|
3188 MVPP2_PRS_RI_UDF3_MASK
);
3192 /* IPv4 Broadcast */
3193 err
= mvpp2_prs_ip4_cast(priv
, MVPP2_PRS_L3_BROAD_CAST
);
3197 /* IPv4 Multicast */
3198 err
= mvpp2_prs_ip4_cast(priv
, MVPP2_PRS_L3_MULTI_CAST
);
3202 /* Default IPv4 entry for unknown protocols */
3203 memset(&pe
, 0, sizeof(pe
));
3204 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_IP4
);
3205 pe
.index
= MVPP2_PE_IP4_PROTO_UN
;
3207 /* Set next lu to IPv4 */
3208 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_IP4
);
3209 mvpp2_prs_sram_shift_set(&pe
, 12, MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
3211 mvpp2_prs_sram_offset_set(&pe
, MVPP2_PRS_SRAM_UDF_TYPE_L4
,
3212 sizeof(struct iphdr
) - 4,
3213 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD
);
3214 mvpp2_prs_sram_ai_update(&pe
, MVPP2_PRS_IPV4_DIP_AI_BIT
,
3215 MVPP2_PRS_IPV4_DIP_AI_BIT
);
3216 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L4_OTHER
,
3217 MVPP2_PRS_RI_L4_PROTO_MASK
);
3219 mvpp2_prs_tcam_ai_update(&pe
, 0, MVPP2_PRS_IPV4_DIP_AI_BIT
);
3220 /* Unmask all ports */
3221 mvpp2_prs_tcam_port_map_set(&pe
, MVPP2_PRS_PORT_MASK
);
3223 /* Update shadow table and hw entry */
3224 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_IP4
);
3225 mvpp2_prs_hw_write(priv
, &pe
);
3227 /* Default IPv4 entry for unicast address */
3228 memset(&pe
, 0, sizeof(pe
));
3229 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_IP4
);
3230 pe
.index
= MVPP2_PE_IP4_ADDR_UN
;
3232 /* Finished: go to flowid generation */
3233 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_FLOWS
);
3234 mvpp2_prs_sram_bits_set(&pe
, MVPP2_PRS_SRAM_LU_GEN_BIT
, 1);
3235 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L3_UCAST
,
3236 MVPP2_PRS_RI_L3_ADDR_MASK
);
3238 mvpp2_prs_tcam_ai_update(&pe
, MVPP2_PRS_IPV4_DIP_AI_BIT
,
3239 MVPP2_PRS_IPV4_DIP_AI_BIT
);
3240 /* Unmask all ports */
3241 mvpp2_prs_tcam_port_map_set(&pe
, MVPP2_PRS_PORT_MASK
);
3243 /* Update shadow table and hw entry */
3244 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_IP4
);
3245 mvpp2_prs_hw_write(priv
, &pe
);
3250 /* Initialize entries for IPv6 */
3251 static int mvpp2_prs_ip6_init(struct mvpp2
*priv
)
3253 struct mvpp2_prs_entry pe
;
3256 /* Set entries for TCP, UDP and ICMP over IPv6 */
3257 err
= mvpp2_prs_ip6_proto(priv
, IPPROTO_TCP
,
3258 MVPP2_PRS_RI_L4_TCP
,
3259 MVPP2_PRS_RI_L4_PROTO_MASK
);
3263 err
= mvpp2_prs_ip6_proto(priv
, IPPROTO_UDP
,
3264 MVPP2_PRS_RI_L4_UDP
,
3265 MVPP2_PRS_RI_L4_PROTO_MASK
);
3269 err
= mvpp2_prs_ip6_proto(priv
, IPPROTO_ICMPV6
,
3270 MVPP2_PRS_RI_CPU_CODE_RX_SPEC
|
3271 MVPP2_PRS_RI_UDF3_RX_SPECIAL
,
3272 MVPP2_PRS_RI_CPU_CODE_MASK
|
3273 MVPP2_PRS_RI_UDF3_MASK
);
3277 /* IPv4 is the last header. This is similar case as 6-TCP or 17-UDP */
3278 /* Result Info: UDF7=1, DS lite */
3279 err
= mvpp2_prs_ip6_proto(priv
, IPPROTO_IPIP
,
3280 MVPP2_PRS_RI_UDF7_IP6_LITE
,
3281 MVPP2_PRS_RI_UDF7_MASK
);
3285 /* IPv6 multicast */
3286 err
= mvpp2_prs_ip6_cast(priv
, MVPP2_PRS_L3_MULTI_CAST
);
3290 /* Entry for checking hop limit */
3291 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
3292 MVPP2_PE_LAST_FREE_TID
);
3296 memset(&pe
, 0, sizeof(pe
));
3297 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_IP6
);
3300 /* Finished: go to flowid generation */
3301 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_FLOWS
);
3302 mvpp2_prs_sram_bits_set(&pe
, MVPP2_PRS_SRAM_LU_GEN_BIT
, 1);
3303 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L3_UN
|
3304 MVPP2_PRS_RI_DROP_MASK
,
3305 MVPP2_PRS_RI_L3_PROTO_MASK
|
3306 MVPP2_PRS_RI_DROP_MASK
);
3308 mvpp2_prs_tcam_data_byte_set(&pe
, 1, 0x00, MVPP2_PRS_IPV6_HOP_MASK
);
3309 mvpp2_prs_tcam_ai_update(&pe
, MVPP2_PRS_IPV6_NO_EXT_AI_BIT
,
3310 MVPP2_PRS_IPV6_NO_EXT_AI_BIT
);
3312 /* Update shadow table and hw entry */
3313 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_IP4
);
3314 mvpp2_prs_hw_write(priv
, &pe
);
3316 /* Default IPv6 entry for unknown protocols */
3317 memset(&pe
, 0, sizeof(pe
));
3318 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_IP6
);
3319 pe
.index
= MVPP2_PE_IP6_PROTO_UN
;
3321 /* Finished: go to flowid generation */
3322 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_FLOWS
);
3323 mvpp2_prs_sram_bits_set(&pe
, MVPP2_PRS_SRAM_LU_GEN_BIT
, 1);
3324 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L4_OTHER
,
3325 MVPP2_PRS_RI_L4_PROTO_MASK
);
3326 /* Set L4 offset relatively to our current place */
3327 mvpp2_prs_sram_offset_set(&pe
, MVPP2_PRS_SRAM_UDF_TYPE_L4
,
3328 sizeof(struct ipv6hdr
) - 4,
3329 MVPP2_PRS_SRAM_OP_SEL_UDF_ADD
);
3331 mvpp2_prs_tcam_ai_update(&pe
, MVPP2_PRS_IPV6_NO_EXT_AI_BIT
,
3332 MVPP2_PRS_IPV6_NO_EXT_AI_BIT
);
3333 /* Unmask all ports */
3334 mvpp2_prs_tcam_port_map_set(&pe
, MVPP2_PRS_PORT_MASK
);
3336 /* Update shadow table and hw entry */
3337 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_IP4
);
3338 mvpp2_prs_hw_write(priv
, &pe
);
3340 /* Default IPv6 entry for unknown ext protocols */
3341 memset(&pe
, 0, sizeof(struct mvpp2_prs_entry
));
3342 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_IP6
);
3343 pe
.index
= MVPP2_PE_IP6_EXT_PROTO_UN
;
3345 /* Finished: go to flowid generation */
3346 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_FLOWS
);
3347 mvpp2_prs_sram_bits_set(&pe
, MVPP2_PRS_SRAM_LU_GEN_BIT
, 1);
3348 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L4_OTHER
,
3349 MVPP2_PRS_RI_L4_PROTO_MASK
);
3351 mvpp2_prs_tcam_ai_update(&pe
, MVPP2_PRS_IPV6_EXT_AI_BIT
,
3352 MVPP2_PRS_IPV6_EXT_AI_BIT
);
3353 /* Unmask all ports */
3354 mvpp2_prs_tcam_port_map_set(&pe
, MVPP2_PRS_PORT_MASK
);
3356 /* Update shadow table and hw entry */
3357 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_IP4
);
3358 mvpp2_prs_hw_write(priv
, &pe
);
3360 /* Default IPv6 entry for unicast address */
3361 memset(&pe
, 0, sizeof(struct mvpp2_prs_entry
));
3362 mvpp2_prs_tcam_lu_set(&pe
, MVPP2_PRS_LU_IP6
);
3363 pe
.index
= MVPP2_PE_IP6_ADDR_UN
;
3365 /* Finished: go to IPv6 again */
3366 mvpp2_prs_sram_next_lu_set(&pe
, MVPP2_PRS_LU_IP6
);
3367 mvpp2_prs_sram_ri_update(&pe
, MVPP2_PRS_RI_L3_UCAST
,
3368 MVPP2_PRS_RI_L3_ADDR_MASK
);
3369 mvpp2_prs_sram_ai_update(&pe
, MVPP2_PRS_IPV6_NO_EXT_AI_BIT
,
3370 MVPP2_PRS_IPV6_NO_EXT_AI_BIT
);
3371 /* Shift back to IPV6 NH */
3372 mvpp2_prs_sram_shift_set(&pe
, -18, MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
3374 mvpp2_prs_tcam_ai_update(&pe
, 0, MVPP2_PRS_IPV6_NO_EXT_AI_BIT
);
3375 /* Unmask all ports */
3376 mvpp2_prs_tcam_port_map_set(&pe
, MVPP2_PRS_PORT_MASK
);
3378 /* Update shadow table and hw entry */
3379 mvpp2_prs_shadow_set(priv
, pe
.index
, MVPP2_PRS_LU_IP6
);
3380 mvpp2_prs_hw_write(priv
, &pe
);
3385 /* Parser default initialization */
3386 static int mvpp2_prs_default_init(struct platform_device
*pdev
,
3391 /* Enable tcam table */
3392 mvpp2_write(priv
, MVPP2_PRS_TCAM_CTRL_REG
, MVPP2_PRS_TCAM_EN_MASK
);
3394 /* Clear all tcam and sram entries */
3395 for (index
= 0; index
< MVPP2_PRS_TCAM_SRAM_SIZE
; index
++) {
3396 mvpp2_write(priv
, MVPP2_PRS_TCAM_IDX_REG
, index
);
3397 for (i
= 0; i
< MVPP2_PRS_TCAM_WORDS
; i
++)
3398 mvpp2_write(priv
, MVPP2_PRS_TCAM_DATA_REG(i
), 0);
3400 mvpp2_write(priv
, MVPP2_PRS_SRAM_IDX_REG
, index
);
3401 for (i
= 0; i
< MVPP2_PRS_SRAM_WORDS
; i
++)
3402 mvpp2_write(priv
, MVPP2_PRS_SRAM_DATA_REG(i
), 0);
3405 /* Invalidate all tcam entries */
3406 for (index
= 0; index
< MVPP2_PRS_TCAM_SRAM_SIZE
; index
++)
3407 mvpp2_prs_hw_inv(priv
, index
);
3409 priv
->prs_shadow
= devm_kcalloc(&pdev
->dev
, MVPP2_PRS_TCAM_SRAM_SIZE
,
3410 sizeof(*priv
->prs_shadow
),
3412 if (!priv
->prs_shadow
)
3415 /* Always start from lookup = 0 */
3416 for (index
= 0; index
< MVPP2_MAX_PORTS
; index
++)
3417 mvpp2_prs_hw_port_init(priv
, index
, MVPP2_PRS_LU_MH
,
3418 MVPP2_PRS_PORT_LU_MAX
, 0);
3420 mvpp2_prs_def_flow_init(priv
);
3422 mvpp2_prs_mh_init(priv
);
3424 mvpp2_prs_mac_init(priv
);
3426 mvpp2_prs_dsa_init(priv
);
3428 err
= mvpp2_prs_etype_init(priv
);
3432 err
= mvpp2_prs_vlan_init(pdev
, priv
);
3436 err
= mvpp2_prs_pppoe_init(priv
);
3440 err
= mvpp2_prs_ip6_init(priv
);
3444 err
= mvpp2_prs_ip4_init(priv
);
3451 /* Compare MAC DA with tcam entry data */
3452 static bool mvpp2_prs_mac_range_equals(struct mvpp2_prs_entry
*pe
,
3453 const u8
*da
, unsigned char *mask
)
3455 unsigned char tcam_byte
, tcam_mask
;
3458 for (index
= 0; index
< ETH_ALEN
; index
++) {
3459 mvpp2_prs_tcam_data_byte_get(pe
, index
, &tcam_byte
, &tcam_mask
);
3460 if (tcam_mask
!= mask
[index
])
3463 if ((tcam_mask
& tcam_byte
) != (da
[index
] & mask
[index
]))
3470 /* Find tcam entry with matched pair <MAC DA, port> */
3471 static struct mvpp2_prs_entry
*
3472 mvpp2_prs_mac_da_range_find(struct mvpp2
*priv
, int pmap
, const u8
*da
,
3473 unsigned char *mask
, int udf_type
)
3475 struct mvpp2_prs_entry
*pe
;
3478 pe
= kzalloc(sizeof(*pe
), GFP_ATOMIC
);
3481 mvpp2_prs_tcam_lu_set(pe
, MVPP2_PRS_LU_MAC
);
3483 /* Go through the all entires with MVPP2_PRS_LU_MAC */
3484 for (tid
= MVPP2_PE_FIRST_FREE_TID
;
3485 tid
<= MVPP2_PE_LAST_FREE_TID
; tid
++) {
3486 unsigned int entry_pmap
;
3488 if (!priv
->prs_shadow
[tid
].valid
||
3489 (priv
->prs_shadow
[tid
].lu
!= MVPP2_PRS_LU_MAC
) ||
3490 (priv
->prs_shadow
[tid
].udf
!= udf_type
))
3494 mvpp2_prs_hw_read(priv
, pe
);
3495 entry_pmap
= mvpp2_prs_tcam_port_map_get(pe
);
3497 if (mvpp2_prs_mac_range_equals(pe
, da
, mask
) &&
3506 /* Update parser's mac da entry */
3507 static int mvpp2_prs_mac_da_accept(struct mvpp2
*priv
, int port
,
3508 const u8
*da
, bool add
)
3510 struct mvpp2_prs_entry
*pe
;
3511 unsigned int pmap
, len
, ri
;
3512 unsigned char mask
[ETH_ALEN
] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
3515 /* Scan TCAM and see if entry with this <MAC DA, port> already exist */
3516 pe
= mvpp2_prs_mac_da_range_find(priv
, (1 << port
), da
, mask
,
3517 MVPP2_PRS_UDF_MAC_DEF
);
3524 /* Create new TCAM entry */
3525 /* Find first range mac entry*/
3526 for (tid
= MVPP2_PE_FIRST_FREE_TID
;
3527 tid
<= MVPP2_PE_LAST_FREE_TID
; tid
++)
3528 if (priv
->prs_shadow
[tid
].valid
&&
3529 (priv
->prs_shadow
[tid
].lu
== MVPP2_PRS_LU_MAC
) &&
3530 (priv
->prs_shadow
[tid
].udf
==
3531 MVPP2_PRS_UDF_MAC_RANGE
))
3534 /* Go through the all entries from first to last */
3535 tid
= mvpp2_prs_tcam_first_free(priv
, MVPP2_PE_FIRST_FREE_TID
,
3540 pe
= kzalloc(sizeof(*pe
), GFP_ATOMIC
);
3543 mvpp2_prs_tcam_lu_set(pe
, MVPP2_PRS_LU_MAC
);
3546 /* Mask all ports */
3547 mvpp2_prs_tcam_port_map_set(pe
, 0);
3550 /* Update port mask */
3551 mvpp2_prs_tcam_port_set(pe
, port
, add
);
3553 /* Invalidate the entry if no ports are left enabled */
3554 pmap
= mvpp2_prs_tcam_port_map_get(pe
);
3560 mvpp2_prs_hw_inv(priv
, pe
->index
);
3561 priv
->prs_shadow
[pe
->index
].valid
= false;
3566 /* Continue - set next lookup */
3567 mvpp2_prs_sram_next_lu_set(pe
, MVPP2_PRS_LU_DSA
);
3569 /* Set match on DA */
3572 mvpp2_prs_tcam_data_byte_set(pe
, len
, da
[len
], 0xff);
3574 /* Set result info bits */
3575 if (is_broadcast_ether_addr(da
))
3576 ri
= MVPP2_PRS_RI_L2_BCAST
;
3577 else if (is_multicast_ether_addr(da
))
3578 ri
= MVPP2_PRS_RI_L2_MCAST
;
3580 ri
= MVPP2_PRS_RI_L2_UCAST
| MVPP2_PRS_RI_MAC_ME_MASK
;
3582 mvpp2_prs_sram_ri_update(pe
, ri
, MVPP2_PRS_RI_L2_CAST_MASK
|
3583 MVPP2_PRS_RI_MAC_ME_MASK
);
3584 mvpp2_prs_shadow_ri_set(priv
, pe
->index
, ri
, MVPP2_PRS_RI_L2_CAST_MASK
|
3585 MVPP2_PRS_RI_MAC_ME_MASK
);
3587 /* Shift to ethertype */
3588 mvpp2_prs_sram_shift_set(pe
, 2 * ETH_ALEN
,
3589 MVPP2_PRS_SRAM_OP_SEL_SHIFT_ADD
);
3591 /* Update shadow table and hw entry */
3592 priv
->prs_shadow
[pe
->index
].udf
= MVPP2_PRS_UDF_MAC_DEF
;
3593 mvpp2_prs_shadow_set(priv
, pe
->index
, MVPP2_PRS_LU_MAC
);
3594 mvpp2_prs_hw_write(priv
, pe
);
3601 static int mvpp2_prs_update_mac_da(struct net_device
*dev
, const u8
*da
)
3603 struct mvpp2_port
*port
= netdev_priv(dev
);
3606 /* Remove old parser entry */
3607 err
= mvpp2_prs_mac_da_accept(port
->priv
, port
->id
, dev
->dev_addr
,
3612 /* Add new parser entry */
3613 err
= mvpp2_prs_mac_da_accept(port
->priv
, port
->id
, da
, true);
3617 /* Set addr in the device */
3618 ether_addr_copy(dev
->dev_addr
, da
);
3623 /* Delete all port's multicast simple (not range) entries */
3624 static void mvpp2_prs_mcast_del_all(struct mvpp2
*priv
, int port
)
3626 struct mvpp2_prs_entry pe
;
3629 for (tid
= MVPP2_PE_FIRST_FREE_TID
;
3630 tid
<= MVPP2_PE_LAST_FREE_TID
; tid
++) {
3631 unsigned char da
[ETH_ALEN
], da_mask
[ETH_ALEN
];
3633 if (!priv
->prs_shadow
[tid
].valid
||
3634 (priv
->prs_shadow
[tid
].lu
!= MVPP2_PRS_LU_MAC
) ||
3635 (priv
->prs_shadow
[tid
].udf
!= MVPP2_PRS_UDF_MAC_DEF
))
3638 /* Only simple mac entries */
3640 mvpp2_prs_hw_read(priv
, &pe
);
3642 /* Read mac addr from entry */
3643 for (index
= 0; index
< ETH_ALEN
; index
++)
3644 mvpp2_prs_tcam_data_byte_get(&pe
, index
, &da
[index
],
3647 if (is_multicast_ether_addr(da
) && !is_broadcast_ether_addr(da
))
3648 /* Delete this entry */
3649 mvpp2_prs_mac_da_accept(priv
, port
, da
, false);
3653 static int mvpp2_prs_tag_mode_set(struct mvpp2
*priv
, int port
, int type
)
3656 case MVPP2_TAG_TYPE_EDSA
:
3657 /* Add port to EDSA entries */
3658 mvpp2_prs_dsa_tag_set(priv
, port
, true,
3659 MVPP2_PRS_TAGGED
, MVPP2_PRS_EDSA
);
3660 mvpp2_prs_dsa_tag_set(priv
, port
, true,
3661 MVPP2_PRS_UNTAGGED
, MVPP2_PRS_EDSA
);
3662 /* Remove port from DSA entries */
3663 mvpp2_prs_dsa_tag_set(priv
, port
, false,
3664 MVPP2_PRS_TAGGED
, MVPP2_PRS_DSA
);
3665 mvpp2_prs_dsa_tag_set(priv
, port
, false,
3666 MVPP2_PRS_UNTAGGED
, MVPP2_PRS_DSA
);
3669 case MVPP2_TAG_TYPE_DSA
:
3670 /* Add port to DSA entries */
3671 mvpp2_prs_dsa_tag_set(priv
, port
, true,
3672 MVPP2_PRS_TAGGED
, MVPP2_PRS_DSA
);
3673 mvpp2_prs_dsa_tag_set(priv
, port
, true,
3674 MVPP2_PRS_UNTAGGED
, MVPP2_PRS_DSA
);
3675 /* Remove port from EDSA entries */
3676 mvpp2_prs_dsa_tag_set(priv
, port
, false,
3677 MVPP2_PRS_TAGGED
, MVPP2_PRS_EDSA
);
3678 mvpp2_prs_dsa_tag_set(priv
, port
, false,
3679 MVPP2_PRS_UNTAGGED
, MVPP2_PRS_EDSA
);
3682 case MVPP2_TAG_TYPE_MH
:
3683 case MVPP2_TAG_TYPE_NONE
:
3684 /* Remove port form EDSA and DSA entries */
3685 mvpp2_prs_dsa_tag_set(priv
, port
, false,
3686 MVPP2_PRS_TAGGED
, MVPP2_PRS_DSA
);
3687 mvpp2_prs_dsa_tag_set(priv
, port
, false,
3688 MVPP2_PRS_UNTAGGED
, MVPP2_PRS_DSA
);
3689 mvpp2_prs_dsa_tag_set(priv
, port
, false,
3690 MVPP2_PRS_TAGGED
, MVPP2_PRS_EDSA
);
3691 mvpp2_prs_dsa_tag_set(priv
, port
, false,
3692 MVPP2_PRS_UNTAGGED
, MVPP2_PRS_EDSA
);
3696 if ((type
< 0) || (type
> MVPP2_TAG_TYPE_EDSA
))
3703 /* Set prs flow for the port */
3704 static int mvpp2_prs_def_flow(struct mvpp2_port
*port
)
3706 struct mvpp2_prs_entry
*pe
;
3709 pe
= mvpp2_prs_flow_find(port
->priv
, port
->id
);
3711 /* Such entry not exist */
3713 /* Go through the all entires from last to first */
3714 tid
= mvpp2_prs_tcam_first_free(port
->priv
,
3715 MVPP2_PE_LAST_FREE_TID
,
3716 MVPP2_PE_FIRST_FREE_TID
);
3720 pe
= kzalloc(sizeof(*pe
), GFP_KERNEL
);
3724 mvpp2_prs_tcam_lu_set(pe
, MVPP2_PRS_LU_FLOWS
);
3728 mvpp2_prs_sram_ai_update(pe
, port
->id
, MVPP2_PRS_FLOW_ID_MASK
);
3729 mvpp2_prs_sram_bits_set(pe
, MVPP2_PRS_SRAM_LU_DONE_BIT
, 1);
3731 /* Update shadow table */
3732 mvpp2_prs_shadow_set(port
->priv
, pe
->index
, MVPP2_PRS_LU_FLOWS
);
3735 mvpp2_prs_tcam_port_map_set(pe
, (1 << port
->id
));
3736 mvpp2_prs_hw_write(port
->priv
, pe
);
3742 /* Classifier configuration routines */
3744 /* Update classification flow table registers */
3745 static void mvpp2_cls_flow_write(struct mvpp2
*priv
,
3746 struct mvpp2_cls_flow_entry
*fe
)
3748 mvpp2_write(priv
, MVPP2_CLS_FLOW_INDEX_REG
, fe
->index
);
3749 mvpp2_write(priv
, MVPP2_CLS_FLOW_TBL0_REG
, fe
->data
[0]);
3750 mvpp2_write(priv
, MVPP2_CLS_FLOW_TBL1_REG
, fe
->data
[1]);
3751 mvpp2_write(priv
, MVPP2_CLS_FLOW_TBL2_REG
, fe
->data
[2]);
3754 /* Update classification lookup table register */
3755 static void mvpp2_cls_lookup_write(struct mvpp2
*priv
,
3756 struct mvpp2_cls_lookup_entry
*le
)
3760 val
= (le
->way
<< MVPP2_CLS_LKP_INDEX_WAY_OFFS
) | le
->lkpid
;
3761 mvpp2_write(priv
, MVPP2_CLS_LKP_INDEX_REG
, val
);
3762 mvpp2_write(priv
, MVPP2_CLS_LKP_TBL_REG
, le
->data
);
3765 /* Classifier default initialization */
3766 static void mvpp2_cls_init(struct mvpp2
*priv
)
3768 struct mvpp2_cls_lookup_entry le
;
3769 struct mvpp2_cls_flow_entry fe
;
3772 /* Enable classifier */
3773 mvpp2_write(priv
, MVPP2_CLS_MODE_REG
, MVPP2_CLS_MODE_ACTIVE_MASK
);
3775 /* Clear classifier flow table */
3776 memset(&fe
.data
, 0, sizeof(fe
.data
));
3777 for (index
= 0; index
< MVPP2_CLS_FLOWS_TBL_SIZE
; index
++) {
3779 mvpp2_cls_flow_write(priv
, &fe
);
3782 /* Clear classifier lookup table */
3784 for (index
= 0; index
< MVPP2_CLS_LKP_TBL_SIZE
; index
++) {
3787 mvpp2_cls_lookup_write(priv
, &le
);
3790 mvpp2_cls_lookup_write(priv
, &le
);
3794 static void mvpp2_cls_port_config(struct mvpp2_port
*port
)
3796 struct mvpp2_cls_lookup_entry le
;
3799 /* Set way for the port */
3800 val
= mvpp2_read(port
->priv
, MVPP2_CLS_PORT_WAY_REG
);
3801 val
&= ~MVPP2_CLS_PORT_WAY_MASK(port
->id
);
3802 mvpp2_write(port
->priv
, MVPP2_CLS_PORT_WAY_REG
, val
);
3804 /* Pick the entry to be accessed in lookup ID decoding table
3805 * according to the way and lkpid.
3807 le
.lkpid
= port
->id
;
3811 /* Set initial CPU queue for receiving packets */
3812 le
.data
&= ~MVPP2_CLS_LKP_TBL_RXQ_MASK
;
3813 le
.data
|= port
->first_rxq
;
3815 /* Disable classification engines */
3816 le
.data
&= ~MVPP2_CLS_LKP_TBL_LOOKUP_EN_MASK
;
3818 /* Update lookup ID table entry */
3819 mvpp2_cls_lookup_write(port
->priv
, &le
);
3822 /* Set CPU queue number for oversize packets */
3823 static void mvpp2_cls_oversize_rxq_set(struct mvpp2_port
*port
)
3827 mvpp2_write(port
->priv
, MVPP2_CLS_OVERSIZE_RXQ_LOW_REG(port
->id
),
3828 port
->first_rxq
& MVPP2_CLS_OVERSIZE_RXQ_LOW_MASK
);
3830 mvpp2_write(port
->priv
, MVPP2_CLS_SWFWD_P2HQ_REG(port
->id
),
3831 (port
->first_rxq
>> MVPP2_CLS_OVERSIZE_RXQ_LOW_BITS
));
3833 val
= mvpp2_read(port
->priv
, MVPP2_CLS_SWFWD_PCTRL_REG
);
3834 val
|= MVPP2_CLS_SWFWD_PCTRL_MASK(port
->id
);
3835 mvpp2_write(port
->priv
, MVPP2_CLS_SWFWD_PCTRL_REG
, val
);
3838 static void *mvpp2_frag_alloc(const struct mvpp2_bm_pool
*pool
)
3840 if (likely(pool
->frag_size
<= PAGE_SIZE
))
3841 return netdev_alloc_frag(pool
->frag_size
);
3843 return kmalloc(pool
->frag_size
, GFP_ATOMIC
);
3846 static void mvpp2_frag_free(const struct mvpp2_bm_pool
*pool
, void *data
)
3848 if (likely(pool
->frag_size
<= PAGE_SIZE
))
3849 skb_free_frag(data
);
3854 /* Buffer Manager configuration routines */
3857 static int mvpp2_bm_pool_create(struct platform_device
*pdev
,
3859 struct mvpp2_bm_pool
*bm_pool
, int size
)
3863 /* Number of buffer pointers must be a multiple of 16, as per
3864 * hardware constraints
3866 if (!IS_ALIGNED(size
, 16))
3869 /* PPv2.1 needs 8 bytes per buffer pointer, PPv2.2 needs 16
3870 * bytes per buffer pointer
3872 if (priv
->hw_version
== MVPP21
)
3873 bm_pool
->size_bytes
= 2 * sizeof(u32
) * size
;
3875 bm_pool
->size_bytes
= 2 * sizeof(u64
) * size
;
3877 bm_pool
->virt_addr
= dma_alloc_coherent(&pdev
->dev
, bm_pool
->size_bytes
,
3880 if (!bm_pool
->virt_addr
)
3883 if (!IS_ALIGNED((unsigned long)bm_pool
->virt_addr
,
3884 MVPP2_BM_POOL_PTR_ALIGN
)) {
3885 dma_free_coherent(&pdev
->dev
, bm_pool
->size_bytes
,
3886 bm_pool
->virt_addr
, bm_pool
->dma_addr
);
3887 dev_err(&pdev
->dev
, "BM pool %d is not %d bytes aligned\n",
3888 bm_pool
->id
, MVPP2_BM_POOL_PTR_ALIGN
);
3892 mvpp2_write(priv
, MVPP2_BM_POOL_BASE_REG(bm_pool
->id
),
3893 lower_32_bits(bm_pool
->dma_addr
));
3894 mvpp2_write(priv
, MVPP2_BM_POOL_SIZE_REG(bm_pool
->id
), size
);
3896 val
= mvpp2_read(priv
, MVPP2_BM_POOL_CTRL_REG(bm_pool
->id
));
3897 val
|= MVPP2_BM_START_MASK
;
3898 mvpp2_write(priv
, MVPP2_BM_POOL_CTRL_REG(bm_pool
->id
), val
);
3900 bm_pool
->type
= MVPP2_BM_FREE
;
3901 bm_pool
->size
= size
;
3902 bm_pool
->pkt_size
= 0;
3903 bm_pool
->buf_num
= 0;
3908 /* Set pool buffer size */
3909 static void mvpp2_bm_pool_bufsize_set(struct mvpp2
*priv
,
3910 struct mvpp2_bm_pool
*bm_pool
,
3915 bm_pool
->buf_size
= buf_size
;
3917 val
= ALIGN(buf_size
, 1 << MVPP2_POOL_BUF_SIZE_OFFSET
);
3918 mvpp2_write(priv
, MVPP2_POOL_BUF_SIZE_REG(bm_pool
->id
), val
);
3921 static void mvpp2_bm_bufs_get_addrs(struct device
*dev
, struct mvpp2
*priv
,
3922 struct mvpp2_bm_pool
*bm_pool
,
3923 dma_addr_t
*dma_addr
,
3924 phys_addr_t
*phys_addr
)
3926 int cpu
= get_cpu();
3928 *dma_addr
= mvpp2_percpu_read(priv
, cpu
,
3929 MVPP2_BM_PHY_ALLOC_REG(bm_pool
->id
));
3930 *phys_addr
= mvpp2_percpu_read(priv
, cpu
, MVPP2_BM_VIRT_ALLOC_REG
);
3932 if (priv
->hw_version
== MVPP22
) {
3934 u32 dma_addr_highbits
, phys_addr_highbits
;
3936 val
= mvpp2_percpu_read(priv
, cpu
, MVPP22_BM_ADDR_HIGH_ALLOC
);
3937 dma_addr_highbits
= (val
& MVPP22_BM_ADDR_HIGH_PHYS_MASK
);
3938 phys_addr_highbits
= (val
& MVPP22_BM_ADDR_HIGH_VIRT_MASK
) >>
3939 MVPP22_BM_ADDR_HIGH_VIRT_SHIFT
;
3941 if (sizeof(dma_addr_t
) == 8)
3942 *dma_addr
|= (u64
)dma_addr_highbits
<< 32;
3944 if (sizeof(phys_addr_t
) == 8)
3945 *phys_addr
|= (u64
)phys_addr_highbits
<< 32;
3951 /* Free all buffers from the pool */
3952 static void mvpp2_bm_bufs_free(struct device
*dev
, struct mvpp2
*priv
,
3953 struct mvpp2_bm_pool
*bm_pool
)
3957 for (i
= 0; i
< bm_pool
->buf_num
; i
++) {
3958 dma_addr_t buf_dma_addr
;
3959 phys_addr_t buf_phys_addr
;
3962 mvpp2_bm_bufs_get_addrs(dev
, priv
, bm_pool
,
3963 &buf_dma_addr
, &buf_phys_addr
);
3965 dma_unmap_single(dev
, buf_dma_addr
,
3966 bm_pool
->buf_size
, DMA_FROM_DEVICE
);
3968 data
= (void *)phys_to_virt(buf_phys_addr
);
3972 mvpp2_frag_free(bm_pool
, data
);
3975 /* Update BM driver with number of buffers removed from pool */
3976 bm_pool
->buf_num
-= i
;
3980 static int mvpp2_bm_pool_destroy(struct platform_device
*pdev
,
3982 struct mvpp2_bm_pool
*bm_pool
)
3986 mvpp2_bm_bufs_free(&pdev
->dev
, priv
, bm_pool
);
3987 if (bm_pool
->buf_num
) {
3988 WARN(1, "cannot free all buffers in pool %d\n", bm_pool
->id
);
3992 val
= mvpp2_read(priv
, MVPP2_BM_POOL_CTRL_REG(bm_pool
->id
));
3993 val
|= MVPP2_BM_STOP_MASK
;
3994 mvpp2_write(priv
, MVPP2_BM_POOL_CTRL_REG(bm_pool
->id
), val
);
3996 dma_free_coherent(&pdev
->dev
, bm_pool
->size_bytes
,
4002 static int mvpp2_bm_pools_init(struct platform_device
*pdev
,
4006 struct mvpp2_bm_pool
*bm_pool
;
4008 /* Create all pools with maximum size */
4009 size
= MVPP2_BM_POOL_SIZE_MAX
;
4010 for (i
= 0; i
< MVPP2_BM_POOLS_NUM
; i
++) {
4011 bm_pool
= &priv
->bm_pools
[i
];
4013 err
= mvpp2_bm_pool_create(pdev
, priv
, bm_pool
, size
);
4015 goto err_unroll_pools
;
4016 mvpp2_bm_pool_bufsize_set(priv
, bm_pool
, 0);
4021 dev_err(&pdev
->dev
, "failed to create BM pool %d, size %d\n", i
, size
);
4022 for (i
= i
- 1; i
>= 0; i
--)
4023 mvpp2_bm_pool_destroy(pdev
, priv
, &priv
->bm_pools
[i
]);
4027 static int mvpp2_bm_init(struct platform_device
*pdev
, struct mvpp2
*priv
)
4031 for (i
= 0; i
< MVPP2_BM_POOLS_NUM
; i
++) {
4032 /* Mask BM all interrupts */
4033 mvpp2_write(priv
, MVPP2_BM_INTR_MASK_REG(i
), 0);
4034 /* Clear BM cause register */
4035 mvpp2_write(priv
, MVPP2_BM_INTR_CAUSE_REG(i
), 0);
4038 /* Allocate and initialize BM pools */
4039 priv
->bm_pools
= devm_kcalloc(&pdev
->dev
, MVPP2_BM_POOLS_NUM
,
4040 sizeof(*priv
->bm_pools
), GFP_KERNEL
);
4041 if (!priv
->bm_pools
)
4044 err
= mvpp2_bm_pools_init(pdev
, priv
);
4050 /* Attach long pool to rxq */
4051 static void mvpp2_rxq_long_pool_set(struct mvpp2_port
*port
,
4052 int lrxq
, int long_pool
)
4057 /* Get queue physical ID */
4058 prxq
= port
->rxqs
[lrxq
]->id
;
4060 if (port
->priv
->hw_version
== MVPP21
)
4061 mask
= MVPP21_RXQ_POOL_LONG_MASK
;
4063 mask
= MVPP22_RXQ_POOL_LONG_MASK
;
4065 val
= mvpp2_read(port
->priv
, MVPP2_RXQ_CONFIG_REG(prxq
));
4067 val
|= (long_pool
<< MVPP2_RXQ_POOL_LONG_OFFS
) & mask
;
4068 mvpp2_write(port
->priv
, MVPP2_RXQ_CONFIG_REG(prxq
), val
);
4071 /* Attach short pool to rxq */
4072 static void mvpp2_rxq_short_pool_set(struct mvpp2_port
*port
,
4073 int lrxq
, int short_pool
)
4078 /* Get queue physical ID */
4079 prxq
= port
->rxqs
[lrxq
]->id
;
4081 if (port
->priv
->hw_version
== MVPP21
)
4082 mask
= MVPP21_RXQ_POOL_SHORT_MASK
;
4084 mask
= MVPP22_RXQ_POOL_SHORT_MASK
;
4086 val
= mvpp2_read(port
->priv
, MVPP2_RXQ_CONFIG_REG(prxq
));
4088 val
|= (short_pool
<< MVPP2_RXQ_POOL_SHORT_OFFS
) & mask
;
4089 mvpp2_write(port
->priv
, MVPP2_RXQ_CONFIG_REG(prxq
), val
);
4092 static void *mvpp2_buf_alloc(struct mvpp2_port
*port
,
4093 struct mvpp2_bm_pool
*bm_pool
,
4094 dma_addr_t
*buf_dma_addr
,
4095 phys_addr_t
*buf_phys_addr
,
4098 dma_addr_t dma_addr
;
4101 data
= mvpp2_frag_alloc(bm_pool
);
4105 dma_addr
= dma_map_single(port
->dev
->dev
.parent
, data
,
4106 MVPP2_RX_BUF_SIZE(bm_pool
->pkt_size
),
4108 if (unlikely(dma_mapping_error(port
->dev
->dev
.parent
, dma_addr
))) {
4109 mvpp2_frag_free(bm_pool
, data
);
4112 *buf_dma_addr
= dma_addr
;
4113 *buf_phys_addr
= virt_to_phys(data
);
4118 /* Release buffer to BM */
4119 static inline void mvpp2_bm_pool_put(struct mvpp2_port
*port
, int pool
,
4120 dma_addr_t buf_dma_addr
,
4121 phys_addr_t buf_phys_addr
)
4123 int cpu
= get_cpu();
4125 if (port
->priv
->hw_version
== MVPP22
) {
4128 if (sizeof(dma_addr_t
) == 8)
4129 val
|= upper_32_bits(buf_dma_addr
) &
4130 MVPP22_BM_ADDR_HIGH_PHYS_RLS_MASK
;
4132 if (sizeof(phys_addr_t
) == 8)
4133 val
|= (upper_32_bits(buf_phys_addr
)
4134 << MVPP22_BM_ADDR_HIGH_VIRT_RLS_SHIFT
) &
4135 MVPP22_BM_ADDR_HIGH_VIRT_RLS_MASK
;
4137 mvpp2_percpu_write(port
->priv
, cpu
,
4138 MVPP22_BM_ADDR_HIGH_RLS_REG
, val
);
4141 /* MVPP2_BM_VIRT_RLS_REG is not interpreted by HW, and simply
4142 * returned in the "cookie" field of the RX
4143 * descriptor. Instead of storing the virtual address, we
4144 * store the physical address
4146 mvpp2_percpu_write(port
->priv
, cpu
,
4147 MVPP2_BM_VIRT_RLS_REG
, buf_phys_addr
);
4148 mvpp2_percpu_write(port
->priv
, cpu
,
4149 MVPP2_BM_PHY_RLS_REG(pool
), buf_dma_addr
);
4154 /* Allocate buffers for the pool */
4155 static int mvpp2_bm_bufs_add(struct mvpp2_port
*port
,
4156 struct mvpp2_bm_pool
*bm_pool
, int buf_num
)
4158 int i
, buf_size
, total_size
;
4159 dma_addr_t dma_addr
;
4160 phys_addr_t phys_addr
;
4163 buf_size
= MVPP2_RX_BUF_SIZE(bm_pool
->pkt_size
);
4164 total_size
= MVPP2_RX_TOTAL_SIZE(buf_size
);
4167 (buf_num
+ bm_pool
->buf_num
> bm_pool
->size
)) {
4168 netdev_err(port
->dev
,
4169 "cannot allocate %d buffers for pool %d\n",
4170 buf_num
, bm_pool
->id
);
4174 for (i
= 0; i
< buf_num
; i
++) {
4175 buf
= mvpp2_buf_alloc(port
, bm_pool
, &dma_addr
,
4176 &phys_addr
, GFP_KERNEL
);
4180 mvpp2_bm_pool_put(port
, bm_pool
->id
, dma_addr
,
4184 /* Update BM driver with number of buffers added to pool */
4185 bm_pool
->buf_num
+= i
;
4187 netdev_dbg(port
->dev
,
4188 "%s pool %d: pkt_size=%4d, buf_size=%4d, total_size=%4d\n",
4189 bm_pool
->type
== MVPP2_BM_SWF_SHORT
? "short" : " long",
4190 bm_pool
->id
, bm_pool
->pkt_size
, buf_size
, total_size
);
4192 netdev_dbg(port
->dev
,
4193 "%s pool %d: %d of %d buffers added\n",
4194 bm_pool
->type
== MVPP2_BM_SWF_SHORT
? "short" : " long",
4195 bm_pool
->id
, i
, buf_num
);
4199 /* Notify the driver that BM pool is being used as specific type and return the
4200 * pool pointer on success
4202 static struct mvpp2_bm_pool
*
4203 mvpp2_bm_pool_use(struct mvpp2_port
*port
, int pool
, enum mvpp2_bm_type type
,
4206 struct mvpp2_bm_pool
*new_pool
= &port
->priv
->bm_pools
[pool
];
4209 if (new_pool
->type
!= MVPP2_BM_FREE
&& new_pool
->type
!= type
) {
4210 netdev_err(port
->dev
, "mixing pool types is forbidden\n");
4214 if (new_pool
->type
== MVPP2_BM_FREE
)
4215 new_pool
->type
= type
;
4217 /* Allocate buffers in case BM pool is used as long pool, but packet
4218 * size doesn't match MTU or BM pool hasn't being used yet
4220 if (((type
== MVPP2_BM_SWF_LONG
) && (pkt_size
> new_pool
->pkt_size
)) ||
4221 (new_pool
->pkt_size
== 0)) {
4224 /* Set default buffer number or free all the buffers in case
4225 * the pool is not empty
4227 pkts_num
= new_pool
->buf_num
;
4229 pkts_num
= type
== MVPP2_BM_SWF_LONG
?
4230 MVPP2_BM_LONG_BUF_NUM
:
4231 MVPP2_BM_SHORT_BUF_NUM
;
4233 mvpp2_bm_bufs_free(port
->dev
->dev
.parent
,
4234 port
->priv
, new_pool
);
4236 new_pool
->pkt_size
= pkt_size
;
4237 new_pool
->frag_size
=
4238 SKB_DATA_ALIGN(MVPP2_RX_BUF_SIZE(pkt_size
)) +
4239 MVPP2_SKB_SHINFO_SIZE
;
4241 /* Allocate buffers for this pool */
4242 num
= mvpp2_bm_bufs_add(port
, new_pool
, pkts_num
);
4243 if (num
!= pkts_num
) {
4244 WARN(1, "pool %d: %d of %d allocated\n",
4245 new_pool
->id
, num
, pkts_num
);
4250 mvpp2_bm_pool_bufsize_set(port
->priv
, new_pool
,
4251 MVPP2_RX_BUF_SIZE(new_pool
->pkt_size
));
4256 /* Initialize pools for swf */
4257 static int mvpp2_swf_bm_pool_init(struct mvpp2_port
*port
)
4261 if (!port
->pool_long
) {
4263 mvpp2_bm_pool_use(port
, MVPP2_BM_SWF_LONG_POOL(port
->id
),
4266 if (!port
->pool_long
)
4269 port
->pool_long
->port_map
|= (1 << port
->id
);
4271 for (rxq
= 0; rxq
< port
->nrxqs
; rxq
++)
4272 mvpp2_rxq_long_pool_set(port
, rxq
, port
->pool_long
->id
);
4275 if (!port
->pool_short
) {
4277 mvpp2_bm_pool_use(port
, MVPP2_BM_SWF_SHORT_POOL
,
4279 MVPP2_BM_SHORT_PKT_SIZE
);
4280 if (!port
->pool_short
)
4283 port
->pool_short
->port_map
|= (1 << port
->id
);
4285 for (rxq
= 0; rxq
< port
->nrxqs
; rxq
++)
4286 mvpp2_rxq_short_pool_set(port
, rxq
,
4287 port
->pool_short
->id
);
4293 static int mvpp2_bm_update_mtu(struct net_device
*dev
, int mtu
)
4295 struct mvpp2_port
*port
= netdev_priv(dev
);
4296 struct mvpp2_bm_pool
*port_pool
= port
->pool_long
;
4297 int num
, pkts_num
= port_pool
->buf_num
;
4298 int pkt_size
= MVPP2_RX_PKT_SIZE(mtu
);
4300 /* Update BM pool with new buffer size */
4301 mvpp2_bm_bufs_free(dev
->dev
.parent
, port
->priv
, port_pool
);
4302 if (port_pool
->buf_num
) {
4303 WARN(1, "cannot free all buffers in pool %d\n", port_pool
->id
);
4307 port_pool
->pkt_size
= pkt_size
;
4308 port_pool
->frag_size
= SKB_DATA_ALIGN(MVPP2_RX_BUF_SIZE(pkt_size
)) +
4309 MVPP2_SKB_SHINFO_SIZE
;
4310 num
= mvpp2_bm_bufs_add(port
, port_pool
, pkts_num
);
4311 if (num
!= pkts_num
) {
4312 WARN(1, "pool %d: %d of %d allocated\n",
4313 port_pool
->id
, num
, pkts_num
);
4317 mvpp2_bm_pool_bufsize_set(port
->priv
, port_pool
,
4318 MVPP2_RX_BUF_SIZE(port_pool
->pkt_size
));
4320 netdev_update_features(dev
);
4324 static inline void mvpp2_interrupts_enable(struct mvpp2_port
*port
)
4326 int i
, sw_thread_mask
= 0;
4328 for (i
= 0; i
< port
->nqvecs
; i
++)
4329 sw_thread_mask
|= port
->qvecs
[i
].sw_thread_mask
;
4331 mvpp2_write(port
->priv
, MVPP2_ISR_ENABLE_REG(port
->id
),
4332 MVPP2_ISR_ENABLE_INTERRUPT(sw_thread_mask
));
4335 static inline void mvpp2_interrupts_disable(struct mvpp2_port
*port
)
4337 int i
, sw_thread_mask
= 0;
4339 for (i
= 0; i
< port
->nqvecs
; i
++)
4340 sw_thread_mask
|= port
->qvecs
[i
].sw_thread_mask
;
4342 mvpp2_write(port
->priv
, MVPP2_ISR_ENABLE_REG(port
->id
),
4343 MVPP2_ISR_DISABLE_INTERRUPT(sw_thread_mask
));
4346 static inline void mvpp2_qvec_interrupt_enable(struct mvpp2_queue_vector
*qvec
)
4348 struct mvpp2_port
*port
= qvec
->port
;
4350 mvpp2_write(port
->priv
, MVPP2_ISR_ENABLE_REG(port
->id
),
4351 MVPP2_ISR_ENABLE_INTERRUPT(qvec
->sw_thread_mask
));
4354 static inline void mvpp2_qvec_interrupt_disable(struct mvpp2_queue_vector
*qvec
)
4356 struct mvpp2_port
*port
= qvec
->port
;
4358 mvpp2_write(port
->priv
, MVPP2_ISR_ENABLE_REG(port
->id
),
4359 MVPP2_ISR_DISABLE_INTERRUPT(qvec
->sw_thread_mask
));
4362 /* Mask the current CPU's Rx/Tx interrupts
4363 * Called by on_each_cpu(), guaranteed to run with migration disabled,
4364 * using smp_processor_id() is OK.
4366 static void mvpp2_interrupts_mask(void *arg
)
4368 struct mvpp2_port
*port
= arg
;
4370 mvpp2_percpu_write(port
->priv
, smp_processor_id(),
4371 MVPP2_ISR_RX_TX_MASK_REG(port
->id
), 0);
4374 /* Unmask the current CPU's Rx/Tx interrupts.
4375 * Called by on_each_cpu(), guaranteed to run with migration disabled,
4376 * using smp_processor_id() is OK.
4378 static void mvpp2_interrupts_unmask(void *arg
)
4380 struct mvpp2_port
*port
= arg
;
4383 val
= MVPP2_CAUSE_MISC_SUM_MASK
|
4384 MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK
;
4385 if (port
->has_tx_irqs
)
4386 val
|= MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK
;
4388 mvpp2_percpu_write(port
->priv
, smp_processor_id(),
4389 MVPP2_ISR_RX_TX_MASK_REG(port
->id
), val
);
4393 mvpp2_shared_interrupt_mask_unmask(struct mvpp2_port
*port
, bool mask
)
4398 if (port
->priv
->hw_version
!= MVPP22
)
4404 val
= MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK
;
4406 for (i
= 0; i
< port
->nqvecs
; i
++) {
4407 struct mvpp2_queue_vector
*v
= port
->qvecs
+ i
;
4409 if (v
->type
!= MVPP2_QUEUE_VECTOR_SHARED
)
4412 mvpp2_percpu_write(port
->priv
, v
->sw_thread_id
,
4413 MVPP2_ISR_RX_TX_MASK_REG(port
->id
), val
);
4417 /* Port configuration routines */
4419 static void mvpp22_gop_init_rgmii(struct mvpp2_port
*port
)
4421 struct mvpp2
*priv
= port
->priv
;
4424 regmap_read(priv
->sysctrl_base
, GENCONF_PORT_CTRL0
, &val
);
4425 val
|= GENCONF_PORT_CTRL0_BUS_WIDTH_SELECT
;
4426 regmap_write(priv
->sysctrl_base
, GENCONF_PORT_CTRL0
, val
);
4428 regmap_read(priv
->sysctrl_base
, GENCONF_CTRL0
, &val
);
4429 if (port
->gop_id
== 2)
4430 val
|= GENCONF_CTRL0_PORT0_RGMII
| GENCONF_CTRL0_PORT1_RGMII
;
4431 else if (port
->gop_id
== 3)
4432 val
|= GENCONF_CTRL0_PORT1_RGMII_MII
;
4433 regmap_write(priv
->sysctrl_base
, GENCONF_CTRL0
, val
);
4436 static void mvpp22_gop_init_sgmii(struct mvpp2_port
*port
)
4438 struct mvpp2
*priv
= port
->priv
;
4441 regmap_read(priv
->sysctrl_base
, GENCONF_PORT_CTRL0
, &val
);
4442 val
|= GENCONF_PORT_CTRL0_BUS_WIDTH_SELECT
|
4443 GENCONF_PORT_CTRL0_RX_DATA_SAMPLE
;
4444 regmap_write(priv
->sysctrl_base
, GENCONF_PORT_CTRL0
, val
);
4446 if (port
->gop_id
> 1) {
4447 regmap_read(priv
->sysctrl_base
, GENCONF_CTRL0
, &val
);
4448 if (port
->gop_id
== 2)
4449 val
&= ~GENCONF_CTRL0_PORT0_RGMII
;
4450 else if (port
->gop_id
== 3)
4451 val
&= ~GENCONF_CTRL0_PORT1_RGMII_MII
;
4452 regmap_write(priv
->sysctrl_base
, GENCONF_CTRL0
, val
);
4456 static void mvpp22_gop_init_10gkr(struct mvpp2_port
*port
)
4458 struct mvpp2
*priv
= port
->priv
;
4459 void __iomem
*mpcs
= priv
->iface_base
+ MVPP22_MPCS_BASE(port
->gop_id
);
4460 void __iomem
*xpcs
= priv
->iface_base
+ MVPP22_XPCS_BASE(port
->gop_id
);
4464 val
= readl(xpcs
+ MVPP22_XPCS_CFG0
);
4465 val
&= ~(MVPP22_XPCS_CFG0_PCS_MODE(0x3) |
4466 MVPP22_XPCS_CFG0_ACTIVE_LANE(0x3));
4467 val
|= MVPP22_XPCS_CFG0_ACTIVE_LANE(2);
4468 writel(val
, xpcs
+ MVPP22_XPCS_CFG0
);
4471 val
= readl(mpcs
+ MVPP22_MPCS_CTRL
);
4472 val
&= ~MVPP22_MPCS_CTRL_FWD_ERR_CONN
;
4473 writel(val
, mpcs
+ MVPP22_MPCS_CTRL
);
4475 val
= readl(mpcs
+ MVPP22_MPCS_CLK_RESET
);
4476 val
&= ~(MVPP22_MPCS_CLK_RESET_DIV_RATIO(0x7) | MAC_CLK_RESET_MAC
|
4477 MAC_CLK_RESET_SD_RX
| MAC_CLK_RESET_SD_TX
);
4478 val
|= MVPP22_MPCS_CLK_RESET_DIV_RATIO(1);
4479 writel(val
, mpcs
+ MVPP22_MPCS_CLK_RESET
);
4481 val
&= ~MVPP22_MPCS_CLK_RESET_DIV_SET
;
4482 val
|= MAC_CLK_RESET_MAC
| MAC_CLK_RESET_SD_RX
| MAC_CLK_RESET_SD_TX
;
4483 writel(val
, mpcs
+ MVPP22_MPCS_CLK_RESET
);
4486 static int mvpp22_gop_init(struct mvpp2_port
*port
)
4488 struct mvpp2
*priv
= port
->priv
;
4491 if (!priv
->sysctrl_base
)
4494 switch (port
->phy_interface
) {
4495 case PHY_INTERFACE_MODE_RGMII
:
4496 case PHY_INTERFACE_MODE_RGMII_ID
:
4497 case PHY_INTERFACE_MODE_RGMII_RXID
:
4498 case PHY_INTERFACE_MODE_RGMII_TXID
:
4499 if (port
->gop_id
== 0)
4501 mvpp22_gop_init_rgmii(port
);
4503 case PHY_INTERFACE_MODE_SGMII
:
4504 mvpp22_gop_init_sgmii(port
);
4506 case PHY_INTERFACE_MODE_10GKR
:
4507 if (port
->gop_id
!= 0)
4509 mvpp22_gop_init_10gkr(port
);
4512 goto unsupported_conf
;
4515 regmap_read(priv
->sysctrl_base
, GENCONF_PORT_CTRL1
, &val
);
4516 val
|= GENCONF_PORT_CTRL1_RESET(port
->gop_id
) |
4517 GENCONF_PORT_CTRL1_EN(port
->gop_id
);
4518 regmap_write(priv
->sysctrl_base
, GENCONF_PORT_CTRL1
, val
);
4520 regmap_read(priv
->sysctrl_base
, GENCONF_PORT_CTRL0
, &val
);
4521 val
|= GENCONF_PORT_CTRL0_CLK_DIV_PHASE_CLR
;
4522 regmap_write(priv
->sysctrl_base
, GENCONF_PORT_CTRL0
, val
);
4524 regmap_read(priv
->sysctrl_base
, GENCONF_SOFT_RESET1
, &val
);
4525 val
|= GENCONF_SOFT_RESET1_GOP
;
4526 regmap_write(priv
->sysctrl_base
, GENCONF_SOFT_RESET1
, val
);
4532 netdev_err(port
->dev
, "Invalid port configuration\n");
4536 static void mvpp22_gop_unmask_irq(struct mvpp2_port
*port
)
4540 if (phy_interface_mode_is_rgmii(port
->phy_interface
) ||
4541 port
->phy_interface
== PHY_INTERFACE_MODE_SGMII
) {
4542 /* Enable the GMAC link status irq for this port */
4543 val
= readl(port
->base
+ MVPP22_GMAC_INT_SUM_MASK
);
4544 val
|= MVPP22_GMAC_INT_SUM_MASK_LINK_STAT
;
4545 writel(val
, port
->base
+ MVPP22_GMAC_INT_SUM_MASK
);
4548 if (port
->gop_id
== 0) {
4549 /* Enable the XLG/GIG irqs for this port */
4550 val
= readl(port
->base
+ MVPP22_XLG_EXT_INT_MASK
);
4551 if (port
->phy_interface
== PHY_INTERFACE_MODE_10GKR
)
4552 val
|= MVPP22_XLG_EXT_INT_MASK_XLG
;
4554 val
|= MVPP22_XLG_EXT_INT_MASK_GIG
;
4555 writel(val
, port
->base
+ MVPP22_XLG_EXT_INT_MASK
);
4559 static void mvpp22_gop_mask_irq(struct mvpp2_port
*port
)
4563 if (port
->gop_id
== 0) {
4564 val
= readl(port
->base
+ MVPP22_XLG_EXT_INT_MASK
);
4565 val
&= ~(MVPP22_XLG_EXT_INT_MASK_XLG
|
4566 MVPP22_XLG_EXT_INT_MASK_GIG
);
4567 writel(val
, port
->base
+ MVPP22_XLG_EXT_INT_MASK
);
4570 if (phy_interface_mode_is_rgmii(port
->phy_interface
) ||
4571 port
->phy_interface
== PHY_INTERFACE_MODE_SGMII
) {
4572 val
= readl(port
->base
+ MVPP22_GMAC_INT_SUM_MASK
);
4573 val
&= ~MVPP22_GMAC_INT_SUM_MASK_LINK_STAT
;
4574 writel(val
, port
->base
+ MVPP22_GMAC_INT_SUM_MASK
);
4578 static void mvpp22_gop_setup_irq(struct mvpp2_port
*port
)
4582 if (phy_interface_mode_is_rgmii(port
->phy_interface
) ||
4583 port
->phy_interface
== PHY_INTERFACE_MODE_SGMII
) {
4584 val
= readl(port
->base
+ MVPP22_GMAC_INT_MASK
);
4585 val
|= MVPP22_GMAC_INT_MASK_LINK_STAT
;
4586 writel(val
, port
->base
+ MVPP22_GMAC_INT_MASK
);
4589 if (port
->gop_id
== 0) {
4590 val
= readl(port
->base
+ MVPP22_XLG_INT_MASK
);
4591 val
|= MVPP22_XLG_INT_MASK_LINK
;
4592 writel(val
, port
->base
+ MVPP22_XLG_INT_MASK
);
4595 mvpp22_gop_unmask_irq(port
);
4598 static int mvpp22_comphy_init(struct mvpp2_port
*port
)
4606 switch (port
->phy_interface
) {
4607 case PHY_INTERFACE_MODE_SGMII
:
4608 mode
= PHY_MODE_SGMII
;
4610 case PHY_INTERFACE_MODE_10GKR
:
4611 mode
= PHY_MODE_10GKR
;
4617 ret
= phy_set_mode(port
->comphy
, mode
);
4621 return phy_power_on(port
->comphy
);
4624 static void mvpp2_port_mii_gmac_configure_mode(struct mvpp2_port
*port
)
4628 if (port
->phy_interface
== PHY_INTERFACE_MODE_SGMII
) {
4629 val
= readl(port
->base
+ MVPP22_GMAC_CTRL_4_REG
);
4630 val
|= MVPP22_CTRL4_SYNC_BYPASS_DIS
| MVPP22_CTRL4_DP_CLK_SEL
|
4631 MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE
;
4632 val
&= ~MVPP22_CTRL4_EXT_PIN_GMII_SEL
;
4633 writel(val
, port
->base
+ MVPP22_GMAC_CTRL_4_REG
);
4634 } else if (phy_interface_mode_is_rgmii(port
->phy_interface
)) {
4635 val
= readl(port
->base
+ MVPP22_GMAC_CTRL_4_REG
);
4636 val
|= MVPP22_CTRL4_EXT_PIN_GMII_SEL
|
4637 MVPP22_CTRL4_SYNC_BYPASS_DIS
|
4638 MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE
;
4639 val
&= ~MVPP22_CTRL4_DP_CLK_SEL
;
4640 writel(val
, port
->base
+ MVPP22_GMAC_CTRL_4_REG
);
4643 /* The port is connected to a copper PHY */
4644 val
= readl(port
->base
+ MVPP2_GMAC_CTRL_0_REG
);
4645 val
&= ~MVPP2_GMAC_PORT_TYPE_MASK
;
4646 writel(val
, port
->base
+ MVPP2_GMAC_CTRL_0_REG
);
4648 val
= readl(port
->base
+ MVPP2_GMAC_AUTONEG_CONFIG
);
4649 val
|= MVPP2_GMAC_IN_BAND_AUTONEG_BYPASS
|
4650 MVPP2_GMAC_AN_SPEED_EN
| MVPP2_GMAC_FLOW_CTRL_AUTONEG
|
4651 MVPP2_GMAC_AN_DUPLEX_EN
;
4652 if (port
->phy_interface
== PHY_INTERFACE_MODE_SGMII
)
4653 val
|= MVPP2_GMAC_IN_BAND_AUTONEG
;
4654 writel(val
, port
->base
+ MVPP2_GMAC_AUTONEG_CONFIG
);
4657 static void mvpp2_port_mii_gmac_configure(struct mvpp2_port
*port
)
4661 /* Force link down */
4662 val
= readl(port
->base
+ MVPP2_GMAC_AUTONEG_CONFIG
);
4663 val
&= ~MVPP2_GMAC_FORCE_LINK_PASS
;
4664 val
|= MVPP2_GMAC_FORCE_LINK_DOWN
;
4665 writel(val
, port
->base
+ MVPP2_GMAC_AUTONEG_CONFIG
);
4667 /* Set the GMAC in a reset state */
4668 val
= readl(port
->base
+ MVPP2_GMAC_CTRL_2_REG
);
4669 val
|= MVPP2_GMAC_PORT_RESET_MASK
;
4670 writel(val
, port
->base
+ MVPP2_GMAC_CTRL_2_REG
);
4672 /* Configure the PCS and in-band AN */
4673 val
= readl(port
->base
+ MVPP2_GMAC_CTRL_2_REG
);
4674 if (port
->phy_interface
== PHY_INTERFACE_MODE_SGMII
) {
4675 val
|= MVPP2_GMAC_INBAND_AN_MASK
| MVPP2_GMAC_PCS_ENABLE_MASK
;
4676 } else if (phy_interface_mode_is_rgmii(port
->phy_interface
)) {
4677 val
&= ~MVPP2_GMAC_PCS_ENABLE_MASK
;
4679 writel(val
, port
->base
+ MVPP2_GMAC_CTRL_2_REG
);
4681 mvpp2_port_mii_gmac_configure_mode(port
);
4683 /* Unset the GMAC reset state */
4684 val
= readl(port
->base
+ MVPP2_GMAC_CTRL_2_REG
);
4685 val
&= ~MVPP2_GMAC_PORT_RESET_MASK
;
4686 writel(val
, port
->base
+ MVPP2_GMAC_CTRL_2_REG
);
4688 /* Stop forcing link down */
4689 val
= readl(port
->base
+ MVPP2_GMAC_AUTONEG_CONFIG
);
4690 val
&= ~MVPP2_GMAC_FORCE_LINK_DOWN
;
4691 writel(val
, port
->base
+ MVPP2_GMAC_AUTONEG_CONFIG
);
4694 static void mvpp2_port_mii_xlg_configure(struct mvpp2_port
*port
)
4698 if (port
->gop_id
!= 0)
4701 val
= readl(port
->base
+ MVPP22_XLG_CTRL0_REG
);
4702 val
|= MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN
;
4703 writel(val
, port
->base
+ MVPP22_XLG_CTRL0_REG
);
4705 val
= readl(port
->base
+ MVPP22_XLG_CTRL4_REG
);
4706 val
&= ~MVPP22_XLG_CTRL4_MACMODSELECT_GMAC
;
4707 val
|= MVPP22_XLG_CTRL4_FWD_FC
| MVPP22_XLG_CTRL4_FWD_PFC
;
4708 writel(val
, port
->base
+ MVPP22_XLG_CTRL4_REG
);
4711 static void mvpp22_port_mii_set(struct mvpp2_port
*port
)
4715 /* Only GOP port 0 has an XLG MAC */
4716 if (port
->gop_id
== 0) {
4717 val
= readl(port
->base
+ MVPP22_XLG_CTRL3_REG
);
4718 val
&= ~MVPP22_XLG_CTRL3_MACMODESELECT_MASK
;
4720 if (port
->phy_interface
== PHY_INTERFACE_MODE_XAUI
||
4721 port
->phy_interface
== PHY_INTERFACE_MODE_10GKR
)
4722 val
|= MVPP22_XLG_CTRL3_MACMODESELECT_10G
;
4724 val
|= MVPP22_XLG_CTRL3_MACMODESELECT_GMAC
;
4726 writel(val
, port
->base
+ MVPP22_XLG_CTRL3_REG
);
4730 static void mvpp2_port_mii_set(struct mvpp2_port
*port
)
4732 if (port
->priv
->hw_version
== MVPP22
)
4733 mvpp22_port_mii_set(port
);
4735 if (phy_interface_mode_is_rgmii(port
->phy_interface
) ||
4736 port
->phy_interface
== PHY_INTERFACE_MODE_SGMII
)
4737 mvpp2_port_mii_gmac_configure(port
);
4738 else if (port
->phy_interface
== PHY_INTERFACE_MODE_10GKR
)
4739 mvpp2_port_mii_xlg_configure(port
);
4742 static void mvpp2_port_fc_adv_enable(struct mvpp2_port
*port
)
4746 val
= readl(port
->base
+ MVPP2_GMAC_AUTONEG_CONFIG
);
4747 val
|= MVPP2_GMAC_FC_ADV_EN
;
4748 writel(val
, port
->base
+ MVPP2_GMAC_AUTONEG_CONFIG
);
4751 static void mvpp2_port_enable(struct mvpp2_port
*port
)
4755 /* Only GOP port 0 has an XLG MAC */
4756 if (port
->gop_id
== 0 &&
4757 (port
->phy_interface
== PHY_INTERFACE_MODE_XAUI
||
4758 port
->phy_interface
== PHY_INTERFACE_MODE_10GKR
)) {
4759 val
= readl(port
->base
+ MVPP22_XLG_CTRL0_REG
);
4760 val
|= MVPP22_XLG_CTRL0_PORT_EN
|
4761 MVPP22_XLG_CTRL0_MAC_RESET_DIS
;
4762 val
&= ~MVPP22_XLG_CTRL0_MIB_CNT_DIS
;
4763 writel(val
, port
->base
+ MVPP22_XLG_CTRL0_REG
);
4765 val
= readl(port
->base
+ MVPP2_GMAC_CTRL_0_REG
);
4766 val
|= MVPP2_GMAC_PORT_EN_MASK
;
4767 val
|= MVPP2_GMAC_MIB_CNTR_EN_MASK
;
4768 writel(val
, port
->base
+ MVPP2_GMAC_CTRL_0_REG
);
4772 static void mvpp2_port_disable(struct mvpp2_port
*port
)
4776 /* Only GOP port 0 has an XLG MAC */
4777 if (port
->gop_id
== 0 &&
4778 (port
->phy_interface
== PHY_INTERFACE_MODE_XAUI
||
4779 port
->phy_interface
== PHY_INTERFACE_MODE_10GKR
)) {
4780 val
= readl(port
->base
+ MVPP22_XLG_CTRL0_REG
);
4781 val
&= ~(MVPP22_XLG_CTRL0_PORT_EN
|
4782 MVPP22_XLG_CTRL0_MAC_RESET_DIS
);
4783 writel(val
, port
->base
+ MVPP22_XLG_CTRL0_REG
);
4785 val
= readl(port
->base
+ MVPP2_GMAC_CTRL_0_REG
);
4786 val
&= ~(MVPP2_GMAC_PORT_EN_MASK
);
4787 writel(val
, port
->base
+ MVPP2_GMAC_CTRL_0_REG
);
4791 /* Set IEEE 802.3x Flow Control Xon Packet Transmission Mode */
4792 static void mvpp2_port_periodic_xon_disable(struct mvpp2_port
*port
)
4796 val
= readl(port
->base
+ MVPP2_GMAC_CTRL_1_REG
) &
4797 ~MVPP2_GMAC_PERIODIC_XON_EN_MASK
;
4798 writel(val
, port
->base
+ MVPP2_GMAC_CTRL_1_REG
);
4801 /* Configure loopback port */
4802 static void mvpp2_port_loopback_set(struct mvpp2_port
*port
)
4806 val
= readl(port
->base
+ MVPP2_GMAC_CTRL_1_REG
);
4808 if (port
->speed
== 1000)
4809 val
|= MVPP2_GMAC_GMII_LB_EN_MASK
;
4811 val
&= ~MVPP2_GMAC_GMII_LB_EN_MASK
;
4813 if (port
->phy_interface
== PHY_INTERFACE_MODE_SGMII
)
4814 val
|= MVPP2_GMAC_PCS_LB_EN_MASK
;
4816 val
&= ~MVPP2_GMAC_PCS_LB_EN_MASK
;
4818 writel(val
, port
->base
+ MVPP2_GMAC_CTRL_1_REG
);
4821 struct mvpp2_ethtool_counter
{
4822 unsigned int offset
;
4823 const char string
[ETH_GSTRING_LEN
];
4827 static u64
mvpp2_read_count(struct mvpp2_port
*port
,
4828 const struct mvpp2_ethtool_counter
*counter
)
4832 val
= readl(port
->stats_base
+ counter
->offset
);
4833 if (counter
->reg_is_64b
)
4834 val
+= (u64
)readl(port
->stats_base
+ counter
->offset
+ 4) << 32;
4839 /* Due to the fact that software statistics and hardware statistics are, by
4840 * design, incremented at different moments in the chain of packet processing,
4841 * it is very likely that incoming packets could have been dropped after being
4842 * counted by hardware but before reaching software statistics (most probably
4843 * multicast packets), and in the oppposite way, during transmission, FCS bytes
4844 * are added in between as well as TSO skb will be split and header bytes added.
4845 * Hence, statistics gathered from userspace with ifconfig (software) and
4846 * ethtool (hardware) cannot be compared.
4848 static const struct mvpp2_ethtool_counter mvpp2_ethtool_regs
[] = {
4849 { MVPP2_MIB_GOOD_OCTETS_RCVD
, "good_octets_received", true },
4850 { MVPP2_MIB_BAD_OCTETS_RCVD
, "bad_octets_received" },
4851 { MVPP2_MIB_CRC_ERRORS_SENT
, "crc_errors_sent" },
4852 { MVPP2_MIB_UNICAST_FRAMES_RCVD
, "unicast_frames_received" },
4853 { MVPP2_MIB_BROADCAST_FRAMES_RCVD
, "broadcast_frames_received" },
4854 { MVPP2_MIB_MULTICAST_FRAMES_RCVD
, "multicast_frames_received" },
4855 { MVPP2_MIB_FRAMES_64_OCTETS
, "frames_64_octets" },
4856 { MVPP2_MIB_FRAMES_65_TO_127_OCTETS
, "frames_65_to_127_octet" },
4857 { MVPP2_MIB_FRAMES_128_TO_255_OCTETS
, "frames_128_to_255_octet" },
4858 { MVPP2_MIB_FRAMES_256_TO_511_OCTETS
, "frames_256_to_511_octet" },
4859 { MVPP2_MIB_FRAMES_512_TO_1023_OCTETS
, "frames_512_to_1023_octet" },
4860 { MVPP2_MIB_FRAMES_1024_TO_MAX_OCTETS
, "frames_1024_to_max_octet" },
4861 { MVPP2_MIB_GOOD_OCTETS_SENT
, "good_octets_sent", true },
4862 { MVPP2_MIB_UNICAST_FRAMES_SENT
, "unicast_frames_sent" },
4863 { MVPP2_MIB_MULTICAST_FRAMES_SENT
, "multicast_frames_sent" },
4864 { MVPP2_MIB_BROADCAST_FRAMES_SENT
, "broadcast_frames_sent" },
4865 { MVPP2_MIB_FC_SENT
, "fc_sent" },
4866 { MVPP2_MIB_FC_RCVD
, "fc_received" },
4867 { MVPP2_MIB_RX_FIFO_OVERRUN
, "rx_fifo_overrun" },
4868 { MVPP2_MIB_UNDERSIZE_RCVD
, "undersize_received" },
4869 { MVPP2_MIB_FRAGMENTS_RCVD
, "fragments_received" },
4870 { MVPP2_MIB_OVERSIZE_RCVD
, "oversize_received" },
4871 { MVPP2_MIB_JABBER_RCVD
, "jabber_received" },
4872 { MVPP2_MIB_MAC_RCV_ERROR
, "mac_receive_error" },
4873 { MVPP2_MIB_BAD_CRC_EVENT
, "bad_crc_event" },
4874 { MVPP2_MIB_COLLISION
, "collision" },
4875 { MVPP2_MIB_LATE_COLLISION
, "late_collision" },
4878 static void mvpp2_ethtool_get_strings(struct net_device
*netdev
, u32 sset
,
4881 if (sset
== ETH_SS_STATS
) {
4884 for (i
= 0; i
< ARRAY_SIZE(mvpp2_ethtool_regs
); i
++)
4885 memcpy(data
+ i
* ETH_GSTRING_LEN
,
4886 &mvpp2_ethtool_regs
[i
].string
, ETH_GSTRING_LEN
);
4890 static void mvpp2_gather_hw_statistics(struct work_struct
*work
)
4892 struct delayed_work
*del_work
= to_delayed_work(work
);
4893 struct mvpp2_port
*port
= container_of(del_work
, struct mvpp2_port
,
4898 mutex_lock(&port
->gather_stats_lock
);
4900 pstats
= port
->ethtool_stats
;
4901 for (i
= 0; i
< ARRAY_SIZE(mvpp2_ethtool_regs
); i
++)
4902 *pstats
++ += mvpp2_read_count(port
, &mvpp2_ethtool_regs
[i
]);
4904 /* No need to read again the counters right after this function if it
4905 * was called asynchronously by the user (ie. use of ethtool).
4907 cancel_delayed_work(&port
->stats_work
);
4908 queue_delayed_work(port
->priv
->stats_queue
, &port
->stats_work
,
4909 MVPP2_MIB_COUNTERS_STATS_DELAY
);
4911 mutex_unlock(&port
->gather_stats_lock
);
4914 static void mvpp2_ethtool_get_stats(struct net_device
*dev
,
4915 struct ethtool_stats
*stats
, u64
*data
)
4917 struct mvpp2_port
*port
= netdev_priv(dev
);
4919 /* Update statistics for the given port, then take the lock to avoid
4920 * concurrent accesses on the ethtool_stats structure during its copy.
4922 mvpp2_gather_hw_statistics(&port
->stats_work
.work
);
4924 mutex_lock(&port
->gather_stats_lock
);
4925 memcpy(data
, port
->ethtool_stats
,
4926 sizeof(u64
) * ARRAY_SIZE(mvpp2_ethtool_regs
));
4927 mutex_unlock(&port
->gather_stats_lock
);
4930 static int mvpp2_ethtool_get_sset_count(struct net_device
*dev
, int sset
)
4932 if (sset
== ETH_SS_STATS
)
4933 return ARRAY_SIZE(mvpp2_ethtool_regs
);
4938 static void mvpp2_port_reset(struct mvpp2_port
*port
)
4943 /* Read the GOP statistics to reset the hardware counters */
4944 for (i
= 0; i
< ARRAY_SIZE(mvpp2_ethtool_regs
); i
++)
4945 mvpp2_read_count(port
, &mvpp2_ethtool_regs
[i
]);
4947 val
= readl(port
->base
+ MVPP2_GMAC_CTRL_2_REG
) &
4948 ~MVPP2_GMAC_PORT_RESET_MASK
;
4949 writel(val
, port
->base
+ MVPP2_GMAC_CTRL_2_REG
);
4951 while (readl(port
->base
+ MVPP2_GMAC_CTRL_2_REG
) &
4952 MVPP2_GMAC_PORT_RESET_MASK
)
4956 /* Change maximum receive size of the port */
4957 static inline void mvpp2_gmac_max_rx_size_set(struct mvpp2_port
*port
)
4961 val
= readl(port
->base
+ MVPP2_GMAC_CTRL_0_REG
);
4962 val
&= ~MVPP2_GMAC_MAX_RX_SIZE_MASK
;
4963 val
|= (((port
->pkt_size
- MVPP2_MH_SIZE
) / 2) <<
4964 MVPP2_GMAC_MAX_RX_SIZE_OFFS
);
4965 writel(val
, port
->base
+ MVPP2_GMAC_CTRL_0_REG
);
4968 /* Change maximum receive size of the port */
4969 static inline void mvpp2_xlg_max_rx_size_set(struct mvpp2_port
*port
)
4973 val
= readl(port
->base
+ MVPP22_XLG_CTRL1_REG
);
4974 val
&= ~MVPP22_XLG_CTRL1_FRAMESIZELIMIT_MASK
;
4975 val
|= ((port
->pkt_size
- MVPP2_MH_SIZE
) / 2) <<
4976 MVPP22_XLG_CTRL1_FRAMESIZELIMIT_OFFS
;
4977 writel(val
, port
->base
+ MVPP22_XLG_CTRL1_REG
);
4980 /* Set defaults to the MVPP2 port */
4981 static void mvpp2_defaults_set(struct mvpp2_port
*port
)
4983 int tx_port_num
, val
, queue
, ptxq
, lrxq
;
4985 if (port
->priv
->hw_version
== MVPP21
) {
4986 /* Configure port to loopback if needed */
4987 if (port
->flags
& MVPP2_F_LOOPBACK
)
4988 mvpp2_port_loopback_set(port
);
4990 /* Update TX FIFO MIN Threshold */
4991 val
= readl(port
->base
+ MVPP2_GMAC_PORT_FIFO_CFG_1_REG
);
4992 val
&= ~MVPP2_GMAC_TX_FIFO_MIN_TH_ALL_MASK
;
4993 /* Min. TX threshold must be less than minimal packet length */
4994 val
|= MVPP2_GMAC_TX_FIFO_MIN_TH_MASK(64 - 4 - 2);
4995 writel(val
, port
->base
+ MVPP2_GMAC_PORT_FIFO_CFG_1_REG
);
4998 /* Disable Legacy WRR, Disable EJP, Release from reset */
4999 tx_port_num
= mvpp2_egress_port(port
);
5000 mvpp2_write(port
->priv
, MVPP2_TXP_SCHED_PORT_INDEX_REG
,
5002 mvpp2_write(port
->priv
, MVPP2_TXP_SCHED_CMD_1_REG
, 0);
5004 /* Close bandwidth for all queues */
5005 for (queue
= 0; queue
< MVPP2_MAX_TXQ
; queue
++) {
5006 ptxq
= mvpp2_txq_phys(port
->id
, queue
);
5007 mvpp2_write(port
->priv
,
5008 MVPP2_TXQ_SCHED_TOKEN_CNTR_REG(ptxq
), 0);
5011 /* Set refill period to 1 usec, refill tokens
5012 * and bucket size to maximum
5014 mvpp2_write(port
->priv
, MVPP2_TXP_SCHED_PERIOD_REG
,
5015 port
->priv
->tclk
/ USEC_PER_SEC
);
5016 val
= mvpp2_read(port
->priv
, MVPP2_TXP_SCHED_REFILL_REG
);
5017 val
&= ~MVPP2_TXP_REFILL_PERIOD_ALL_MASK
;
5018 val
|= MVPP2_TXP_REFILL_PERIOD_MASK(1);
5019 val
|= MVPP2_TXP_REFILL_TOKENS_ALL_MASK
;
5020 mvpp2_write(port
->priv
, MVPP2_TXP_SCHED_REFILL_REG
, val
);
5021 val
= MVPP2_TXP_TOKEN_SIZE_MAX
;
5022 mvpp2_write(port
->priv
, MVPP2_TXP_SCHED_TOKEN_SIZE_REG
, val
);
5024 /* Set MaximumLowLatencyPacketSize value to 256 */
5025 mvpp2_write(port
->priv
, MVPP2_RX_CTRL_REG(port
->id
),
5026 MVPP2_RX_USE_PSEUDO_FOR_CSUM_MASK
|
5027 MVPP2_RX_LOW_LATENCY_PKT_SIZE(256));
5029 /* Enable Rx cache snoop */
5030 for (lrxq
= 0; lrxq
< port
->nrxqs
; lrxq
++) {
5031 queue
= port
->rxqs
[lrxq
]->id
;
5032 val
= mvpp2_read(port
->priv
, MVPP2_RXQ_CONFIG_REG(queue
));
5033 val
|= MVPP2_SNOOP_PKT_SIZE_MASK
|
5034 MVPP2_SNOOP_BUF_HDR_MASK
;
5035 mvpp2_write(port
->priv
, MVPP2_RXQ_CONFIG_REG(queue
), val
);
5038 /* At default, mask all interrupts to all present cpus */
5039 mvpp2_interrupts_disable(port
);
5042 /* Enable/disable receiving packets */
5043 static void mvpp2_ingress_enable(struct mvpp2_port
*port
)
5048 for (lrxq
= 0; lrxq
< port
->nrxqs
; lrxq
++) {
5049 queue
= port
->rxqs
[lrxq
]->id
;
5050 val
= mvpp2_read(port
->priv
, MVPP2_RXQ_CONFIG_REG(queue
));
5051 val
&= ~MVPP2_RXQ_DISABLE_MASK
;
5052 mvpp2_write(port
->priv
, MVPP2_RXQ_CONFIG_REG(queue
), val
);
5056 static void mvpp2_ingress_disable(struct mvpp2_port
*port
)
5061 for (lrxq
= 0; lrxq
< port
->nrxqs
; lrxq
++) {
5062 queue
= port
->rxqs
[lrxq
]->id
;
5063 val
= mvpp2_read(port
->priv
, MVPP2_RXQ_CONFIG_REG(queue
));
5064 val
|= MVPP2_RXQ_DISABLE_MASK
;
5065 mvpp2_write(port
->priv
, MVPP2_RXQ_CONFIG_REG(queue
), val
);
5069 /* Enable transmit via physical egress queue
5070 * - HW starts take descriptors from DRAM
5072 static void mvpp2_egress_enable(struct mvpp2_port
*port
)
5076 int tx_port_num
= mvpp2_egress_port(port
);
5078 /* Enable all initialized TXs. */
5080 for (queue
= 0; queue
< port
->ntxqs
; queue
++) {
5081 struct mvpp2_tx_queue
*txq
= port
->txqs
[queue
];
5084 qmap
|= (1 << queue
);
5087 mvpp2_write(port
->priv
, MVPP2_TXP_SCHED_PORT_INDEX_REG
, tx_port_num
);
5088 mvpp2_write(port
->priv
, MVPP2_TXP_SCHED_Q_CMD_REG
, qmap
);
5091 /* Disable transmit via physical egress queue
5092 * - HW doesn't take descriptors from DRAM
5094 static void mvpp2_egress_disable(struct mvpp2_port
*port
)
5098 int tx_port_num
= mvpp2_egress_port(port
);
5100 /* Issue stop command for active channels only */
5101 mvpp2_write(port
->priv
, MVPP2_TXP_SCHED_PORT_INDEX_REG
, tx_port_num
);
5102 reg_data
= (mvpp2_read(port
->priv
, MVPP2_TXP_SCHED_Q_CMD_REG
)) &
5103 MVPP2_TXP_SCHED_ENQ_MASK
;
5105 mvpp2_write(port
->priv
, MVPP2_TXP_SCHED_Q_CMD_REG
,
5106 (reg_data
<< MVPP2_TXP_SCHED_DISQ_OFFSET
));
5108 /* Wait for all Tx activity to terminate. */
5111 if (delay
>= MVPP2_TX_DISABLE_TIMEOUT_MSEC
) {
5112 netdev_warn(port
->dev
,
5113 "Tx stop timed out, status=0x%08x\n",
5120 /* Check port TX Command register that all
5121 * Tx queues are stopped
5123 reg_data
= mvpp2_read(port
->priv
, MVPP2_TXP_SCHED_Q_CMD_REG
);
5124 } while (reg_data
& MVPP2_TXP_SCHED_ENQ_MASK
);
5127 /* Rx descriptors helper methods */
5129 /* Get number of Rx descriptors occupied by received packets */
5131 mvpp2_rxq_received(struct mvpp2_port
*port
, int rxq_id
)
5133 u32 val
= mvpp2_read(port
->priv
, MVPP2_RXQ_STATUS_REG(rxq_id
));
5135 return val
& MVPP2_RXQ_OCCUPIED_MASK
;
5138 /* Update Rx queue status with the number of occupied and available
5139 * Rx descriptor slots.
5142 mvpp2_rxq_status_update(struct mvpp2_port
*port
, int rxq_id
,
5143 int used_count
, int free_count
)
5145 /* Decrement the number of used descriptors and increment count
5146 * increment the number of free descriptors.
5148 u32 val
= used_count
| (free_count
<< MVPP2_RXQ_NUM_NEW_OFFSET
);
5150 mvpp2_write(port
->priv
, MVPP2_RXQ_STATUS_UPDATE_REG(rxq_id
), val
);
5153 /* Get pointer to next RX descriptor to be processed by SW */
5154 static inline struct mvpp2_rx_desc
*
5155 mvpp2_rxq_next_desc_get(struct mvpp2_rx_queue
*rxq
)
5157 int rx_desc
= rxq
->next_desc_to_proc
;
5159 rxq
->next_desc_to_proc
= MVPP2_QUEUE_NEXT_DESC(rxq
, rx_desc
);
5160 prefetch(rxq
->descs
+ rxq
->next_desc_to_proc
);
5161 return rxq
->descs
+ rx_desc
;
5164 /* Set rx queue offset */
5165 static void mvpp2_rxq_offset_set(struct mvpp2_port
*port
,
5166 int prxq
, int offset
)
5170 /* Convert offset from bytes to units of 32 bytes */
5171 offset
= offset
>> 5;
5173 val
= mvpp2_read(port
->priv
, MVPP2_RXQ_CONFIG_REG(prxq
));
5174 val
&= ~MVPP2_RXQ_PACKET_OFFSET_MASK
;
5177 val
|= ((offset
<< MVPP2_RXQ_PACKET_OFFSET_OFFS
) &
5178 MVPP2_RXQ_PACKET_OFFSET_MASK
);
5180 mvpp2_write(port
->priv
, MVPP2_RXQ_CONFIG_REG(prxq
), val
);
5183 /* Tx descriptors helper methods */
5185 /* Get pointer to next Tx descriptor to be processed (send) by HW */
5186 static struct mvpp2_tx_desc
*
5187 mvpp2_txq_next_desc_get(struct mvpp2_tx_queue
*txq
)
5189 int tx_desc
= txq
->next_desc_to_proc
;
5191 txq
->next_desc_to_proc
= MVPP2_QUEUE_NEXT_DESC(txq
, tx_desc
);
5192 return txq
->descs
+ tx_desc
;
5195 /* Update HW with number of aggregated Tx descriptors to be sent
5197 * Called only from mvpp2_tx(), so migration is disabled, using
5198 * smp_processor_id() is OK.
5200 static void mvpp2_aggr_txq_pend_desc_add(struct mvpp2_port
*port
, int pending
)
5202 /* aggregated access - relevant TXQ number is written in TX desc */
5203 mvpp2_percpu_write(port
->priv
, smp_processor_id(),
5204 MVPP2_AGGR_TXQ_UPDATE_REG
, pending
);
5208 /* Check if there are enough free descriptors in aggregated txq.
5209 * If not, update the number of occupied descriptors and repeat the check.
5211 * Called only from mvpp2_tx(), so migration is disabled, using
5212 * smp_processor_id() is OK.
5214 static int mvpp2_aggr_desc_num_check(struct mvpp2
*priv
,
5215 struct mvpp2_tx_queue
*aggr_txq
, int num
)
5217 if ((aggr_txq
->count
+ num
) > MVPP2_AGGR_TXQ_SIZE
) {
5218 /* Update number of occupied aggregated Tx descriptors */
5219 int cpu
= smp_processor_id();
5220 u32 val
= mvpp2_read(priv
, MVPP2_AGGR_TXQ_STATUS_REG(cpu
));
5222 aggr_txq
->count
= val
& MVPP2_AGGR_TXQ_PENDING_MASK
;
5225 if ((aggr_txq
->count
+ num
) > MVPP2_AGGR_TXQ_SIZE
)
5231 /* Reserved Tx descriptors allocation request
5233 * Called only from mvpp2_txq_reserved_desc_num_proc(), itself called
5234 * only by mvpp2_tx(), so migration is disabled, using
5235 * smp_processor_id() is OK.
5237 static int mvpp2_txq_alloc_reserved_desc(struct mvpp2
*priv
,
5238 struct mvpp2_tx_queue
*txq
, int num
)
5241 int cpu
= smp_processor_id();
5243 val
= (txq
->id
<< MVPP2_TXQ_RSVD_REQ_Q_OFFSET
) | num
;
5244 mvpp2_percpu_write(priv
, cpu
, MVPP2_TXQ_RSVD_REQ_REG
, val
);
5246 val
= mvpp2_percpu_read(priv
, cpu
, MVPP2_TXQ_RSVD_RSLT_REG
);
5248 return val
& MVPP2_TXQ_RSVD_RSLT_MASK
;
5251 /* Check if there are enough reserved descriptors for transmission.
5252 * If not, request chunk of reserved descriptors and check again.
5254 static int mvpp2_txq_reserved_desc_num_proc(struct mvpp2
*priv
,
5255 struct mvpp2_tx_queue
*txq
,
5256 struct mvpp2_txq_pcpu
*txq_pcpu
,
5259 int req
, cpu
, desc_count
;
5261 if (txq_pcpu
->reserved_num
>= num
)
5264 /* Not enough descriptors reserved! Update the reserved descriptor
5265 * count and check again.
5269 /* Compute total of used descriptors */
5270 for_each_present_cpu(cpu
) {
5271 struct mvpp2_txq_pcpu
*txq_pcpu_aux
;
5273 txq_pcpu_aux
= per_cpu_ptr(txq
->pcpu
, cpu
);
5274 desc_count
+= txq_pcpu_aux
->count
;
5275 desc_count
+= txq_pcpu_aux
->reserved_num
;
5278 req
= max(MVPP2_CPU_DESC_CHUNK
, num
- txq_pcpu
->reserved_num
);
5282 (txq
->size
- (num_present_cpus() * MVPP2_CPU_DESC_CHUNK
)))
5285 txq_pcpu
->reserved_num
+= mvpp2_txq_alloc_reserved_desc(priv
, txq
, req
);
5287 /* OK, the descriptor cound has been updated: check again. */
5288 if (txq_pcpu
->reserved_num
< num
)
5293 /* Release the last allocated Tx descriptor. Useful to handle DMA
5294 * mapping failures in the Tx path.
5296 static void mvpp2_txq_desc_put(struct mvpp2_tx_queue
*txq
)
5298 if (txq
->next_desc_to_proc
== 0)
5299 txq
->next_desc_to_proc
= txq
->last_desc
- 1;
5301 txq
->next_desc_to_proc
--;
5304 /* Set Tx descriptors fields relevant for CSUM calculation */
5305 static u32
mvpp2_txq_desc_csum(int l3_offs
, int l3_proto
,
5306 int ip_hdr_len
, int l4_proto
)
5310 /* fields: L3_offset, IP_hdrlen, L3_type, G_IPv4_chk,
5311 * G_L4_chk, L4_type required only for checksum calculation
5313 command
= (l3_offs
<< MVPP2_TXD_L3_OFF_SHIFT
);
5314 command
|= (ip_hdr_len
<< MVPP2_TXD_IP_HLEN_SHIFT
);
5315 command
|= MVPP2_TXD_IP_CSUM_DISABLE
;
5317 if (l3_proto
== swab16(ETH_P_IP
)) {
5318 command
&= ~MVPP2_TXD_IP_CSUM_DISABLE
; /* enable IPv4 csum */
5319 command
&= ~MVPP2_TXD_L3_IP6
; /* enable IPv4 */
5321 command
|= MVPP2_TXD_L3_IP6
; /* enable IPv6 */
5324 if (l4_proto
== IPPROTO_TCP
) {
5325 command
&= ~MVPP2_TXD_L4_UDP
; /* enable TCP */
5326 command
&= ~MVPP2_TXD_L4_CSUM_FRAG
; /* generate L4 csum */
5327 } else if (l4_proto
== IPPROTO_UDP
) {
5328 command
|= MVPP2_TXD_L4_UDP
; /* enable UDP */
5329 command
&= ~MVPP2_TXD_L4_CSUM_FRAG
; /* generate L4 csum */
5331 command
|= MVPP2_TXD_L4_CSUM_NOT
;
5337 /* Get number of sent descriptors and decrement counter.
5338 * The number of sent descriptors is returned.
5341 * Called only from mvpp2_txq_done(), called from mvpp2_tx()
5342 * (migration disabled) and from the TX completion tasklet (migration
5343 * disabled) so using smp_processor_id() is OK.
5345 static inline int mvpp2_txq_sent_desc_proc(struct mvpp2_port
*port
,
5346 struct mvpp2_tx_queue
*txq
)
5350 /* Reading status reg resets transmitted descriptor counter */
5351 val
= mvpp2_percpu_read(port
->priv
, smp_processor_id(),
5352 MVPP2_TXQ_SENT_REG(txq
->id
));
5354 return (val
& MVPP2_TRANSMITTED_COUNT_MASK
) >>
5355 MVPP2_TRANSMITTED_COUNT_OFFSET
;
5358 /* Called through on_each_cpu(), so runs on all CPUs, with migration
5359 * disabled, therefore using smp_processor_id() is OK.
5361 static void mvpp2_txq_sent_counter_clear(void *arg
)
5363 struct mvpp2_port
*port
= arg
;
5366 for (queue
= 0; queue
< port
->ntxqs
; queue
++) {
5367 int id
= port
->txqs
[queue
]->id
;
5369 mvpp2_percpu_read(port
->priv
, smp_processor_id(),
5370 MVPP2_TXQ_SENT_REG(id
));
5374 /* Set max sizes for Tx queues */
5375 static void mvpp2_txp_max_tx_size_set(struct mvpp2_port
*port
)
5378 int txq
, tx_port_num
;
5380 mtu
= port
->pkt_size
* 8;
5381 if (mtu
> MVPP2_TXP_MTU_MAX
)
5382 mtu
= MVPP2_TXP_MTU_MAX
;
5384 /* WA for wrong Token bucket update: Set MTU value = 3*real MTU value */
5387 /* Indirect access to registers */
5388 tx_port_num
= mvpp2_egress_port(port
);
5389 mvpp2_write(port
->priv
, MVPP2_TXP_SCHED_PORT_INDEX_REG
, tx_port_num
);
5392 val
= mvpp2_read(port
->priv
, MVPP2_TXP_SCHED_MTU_REG
);
5393 val
&= ~MVPP2_TXP_MTU_MAX
;
5395 mvpp2_write(port
->priv
, MVPP2_TXP_SCHED_MTU_REG
, val
);
5397 /* TXP token size and all TXQs token size must be larger that MTU */
5398 val
= mvpp2_read(port
->priv
, MVPP2_TXP_SCHED_TOKEN_SIZE_REG
);
5399 size
= val
& MVPP2_TXP_TOKEN_SIZE_MAX
;
5402 val
&= ~MVPP2_TXP_TOKEN_SIZE_MAX
;
5404 mvpp2_write(port
->priv
, MVPP2_TXP_SCHED_TOKEN_SIZE_REG
, val
);
5407 for (txq
= 0; txq
< port
->ntxqs
; txq
++) {
5408 val
= mvpp2_read(port
->priv
,
5409 MVPP2_TXQ_SCHED_TOKEN_SIZE_REG(txq
));
5410 size
= val
& MVPP2_TXQ_TOKEN_SIZE_MAX
;
5414 val
&= ~MVPP2_TXQ_TOKEN_SIZE_MAX
;
5416 mvpp2_write(port
->priv
,
5417 MVPP2_TXQ_SCHED_TOKEN_SIZE_REG(txq
),
5423 /* Set the number of packets that will be received before Rx interrupt
5424 * will be generated by HW.
5426 static void mvpp2_rx_pkts_coal_set(struct mvpp2_port
*port
,
5427 struct mvpp2_rx_queue
*rxq
)
5429 int cpu
= get_cpu();
5431 if (rxq
->pkts_coal
> MVPP2_OCCUPIED_THRESH_MASK
)
5432 rxq
->pkts_coal
= MVPP2_OCCUPIED_THRESH_MASK
;
5434 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_RXQ_NUM_REG
, rxq
->id
);
5435 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_RXQ_THRESH_REG
,
5441 /* For some reason in the LSP this is done on each CPU. Why ? */
5442 static void mvpp2_tx_pkts_coal_set(struct mvpp2_port
*port
,
5443 struct mvpp2_tx_queue
*txq
)
5445 int cpu
= get_cpu();
5448 if (txq
->done_pkts_coal
> MVPP2_TXQ_THRESH_MASK
)
5449 txq
->done_pkts_coal
= MVPP2_TXQ_THRESH_MASK
;
5451 val
= (txq
->done_pkts_coal
<< MVPP2_TXQ_THRESH_OFFSET
);
5452 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_TXQ_NUM_REG
, txq
->id
);
5453 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_TXQ_THRESH_REG
, val
);
5458 static u32
mvpp2_usec_to_cycles(u32 usec
, unsigned long clk_hz
)
5460 u64 tmp
= (u64
)clk_hz
* usec
;
5462 do_div(tmp
, USEC_PER_SEC
);
5464 return tmp
> U32_MAX
? U32_MAX
: tmp
;
5467 static u32
mvpp2_cycles_to_usec(u32 cycles
, unsigned long clk_hz
)
5469 u64 tmp
= (u64
)cycles
* USEC_PER_SEC
;
5471 do_div(tmp
, clk_hz
);
5473 return tmp
> U32_MAX
? U32_MAX
: tmp
;
5476 /* Set the time delay in usec before Rx interrupt */
5477 static void mvpp2_rx_time_coal_set(struct mvpp2_port
*port
,
5478 struct mvpp2_rx_queue
*rxq
)
5480 unsigned long freq
= port
->priv
->tclk
;
5481 u32 val
= mvpp2_usec_to_cycles(rxq
->time_coal
, freq
);
5483 if (val
> MVPP2_MAX_ISR_RX_THRESHOLD
) {
5485 mvpp2_cycles_to_usec(MVPP2_MAX_ISR_RX_THRESHOLD
, freq
);
5487 /* re-evaluate to get actual register value */
5488 val
= mvpp2_usec_to_cycles(rxq
->time_coal
, freq
);
5491 mvpp2_write(port
->priv
, MVPP2_ISR_RX_THRESHOLD_REG(rxq
->id
), val
);
5494 static void mvpp2_tx_time_coal_set(struct mvpp2_port
*port
)
5496 unsigned long freq
= port
->priv
->tclk
;
5497 u32 val
= mvpp2_usec_to_cycles(port
->tx_time_coal
, freq
);
5499 if (val
> MVPP2_MAX_ISR_TX_THRESHOLD
) {
5500 port
->tx_time_coal
=
5501 mvpp2_cycles_to_usec(MVPP2_MAX_ISR_TX_THRESHOLD
, freq
);
5503 /* re-evaluate to get actual register value */
5504 val
= mvpp2_usec_to_cycles(port
->tx_time_coal
, freq
);
5507 mvpp2_write(port
->priv
, MVPP2_ISR_TX_THRESHOLD_REG(port
->id
), val
);
5510 /* Free Tx queue skbuffs */
5511 static void mvpp2_txq_bufs_free(struct mvpp2_port
*port
,
5512 struct mvpp2_tx_queue
*txq
,
5513 struct mvpp2_txq_pcpu
*txq_pcpu
, int num
)
5517 for (i
= 0; i
< num
; i
++) {
5518 struct mvpp2_txq_pcpu_buf
*tx_buf
=
5519 txq_pcpu
->buffs
+ txq_pcpu
->txq_get_index
;
5521 if (!IS_TSO_HEADER(txq_pcpu
, tx_buf
->dma
))
5522 dma_unmap_single(port
->dev
->dev
.parent
, tx_buf
->dma
,
5523 tx_buf
->size
, DMA_TO_DEVICE
);
5525 dev_kfree_skb_any(tx_buf
->skb
);
5527 mvpp2_txq_inc_get(txq_pcpu
);
5531 static inline struct mvpp2_rx_queue
*mvpp2_get_rx_queue(struct mvpp2_port
*port
,
5534 int queue
= fls(cause
) - 1;
5536 return port
->rxqs
[queue
];
5539 static inline struct mvpp2_tx_queue
*mvpp2_get_tx_queue(struct mvpp2_port
*port
,
5542 int queue
= fls(cause
) - 1;
5544 return port
->txqs
[queue
];
5547 /* Handle end of transmission */
5548 static void mvpp2_txq_done(struct mvpp2_port
*port
, struct mvpp2_tx_queue
*txq
,
5549 struct mvpp2_txq_pcpu
*txq_pcpu
)
5551 struct netdev_queue
*nq
= netdev_get_tx_queue(port
->dev
, txq
->log_id
);
5554 if (txq_pcpu
->cpu
!= smp_processor_id())
5555 netdev_err(port
->dev
, "wrong cpu on the end of Tx processing\n");
5557 tx_done
= mvpp2_txq_sent_desc_proc(port
, txq
);
5560 mvpp2_txq_bufs_free(port
, txq
, txq_pcpu
, tx_done
);
5562 txq_pcpu
->count
-= tx_done
;
5564 if (netif_tx_queue_stopped(nq
))
5565 if (txq_pcpu
->count
<= txq_pcpu
->wake_threshold
)
5566 netif_tx_wake_queue(nq
);
5569 static unsigned int mvpp2_tx_done(struct mvpp2_port
*port
, u32 cause
,
5572 struct mvpp2_tx_queue
*txq
;
5573 struct mvpp2_txq_pcpu
*txq_pcpu
;
5574 unsigned int tx_todo
= 0;
5577 txq
= mvpp2_get_tx_queue(port
, cause
);
5581 txq_pcpu
= per_cpu_ptr(txq
->pcpu
, cpu
);
5583 if (txq_pcpu
->count
) {
5584 mvpp2_txq_done(port
, txq
, txq_pcpu
);
5585 tx_todo
+= txq_pcpu
->count
;
5588 cause
&= ~(1 << txq
->log_id
);
5593 /* Rx/Tx queue initialization/cleanup methods */
5595 /* Allocate and initialize descriptors for aggr TXQ */
5596 static int mvpp2_aggr_txq_init(struct platform_device
*pdev
,
5597 struct mvpp2_tx_queue
*aggr_txq
, int cpu
,
5602 /* Allocate memory for TX descriptors */
5603 aggr_txq
->descs
= dma_zalloc_coherent(&pdev
->dev
,
5604 MVPP2_AGGR_TXQ_SIZE
* MVPP2_DESC_ALIGNED_SIZE
,
5605 &aggr_txq
->descs_dma
, GFP_KERNEL
);
5606 if (!aggr_txq
->descs
)
5609 aggr_txq
->last_desc
= MVPP2_AGGR_TXQ_SIZE
- 1;
5611 /* Aggr TXQ no reset WA */
5612 aggr_txq
->next_desc_to_proc
= mvpp2_read(priv
,
5613 MVPP2_AGGR_TXQ_INDEX_REG(cpu
));
5615 /* Set Tx descriptors queue starting address indirect
5618 if (priv
->hw_version
== MVPP21
)
5619 txq_dma
= aggr_txq
->descs_dma
;
5621 txq_dma
= aggr_txq
->descs_dma
>>
5622 MVPP22_AGGR_TXQ_DESC_ADDR_OFFS
;
5624 mvpp2_write(priv
, MVPP2_AGGR_TXQ_DESC_ADDR_REG(cpu
), txq_dma
);
5625 mvpp2_write(priv
, MVPP2_AGGR_TXQ_DESC_SIZE_REG(cpu
),
5626 MVPP2_AGGR_TXQ_SIZE
);
5631 /* Create a specified Rx queue */
5632 static int mvpp2_rxq_init(struct mvpp2_port
*port
,
5633 struct mvpp2_rx_queue
*rxq
)
5639 rxq
->size
= port
->rx_ring_size
;
5641 /* Allocate memory for RX descriptors */
5642 rxq
->descs
= dma_alloc_coherent(port
->dev
->dev
.parent
,
5643 rxq
->size
* MVPP2_DESC_ALIGNED_SIZE
,
5644 &rxq
->descs_dma
, GFP_KERNEL
);
5648 rxq
->last_desc
= rxq
->size
- 1;
5650 /* Zero occupied and non-occupied counters - direct access */
5651 mvpp2_write(port
->priv
, MVPP2_RXQ_STATUS_REG(rxq
->id
), 0);
5653 /* Set Rx descriptors queue starting address - indirect access */
5655 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_RXQ_NUM_REG
, rxq
->id
);
5656 if (port
->priv
->hw_version
== MVPP21
)
5657 rxq_dma
= rxq
->descs_dma
;
5659 rxq_dma
= rxq
->descs_dma
>> MVPP22_DESC_ADDR_OFFS
;
5660 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_RXQ_DESC_ADDR_REG
, rxq_dma
);
5661 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_RXQ_DESC_SIZE_REG
, rxq
->size
);
5662 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_RXQ_INDEX_REG
, 0);
5666 mvpp2_rxq_offset_set(port
, rxq
->id
, NET_SKB_PAD
);
5668 /* Set coalescing pkts and time */
5669 mvpp2_rx_pkts_coal_set(port
, rxq
);
5670 mvpp2_rx_time_coal_set(port
, rxq
);
5672 /* Add number of descriptors ready for receiving packets */
5673 mvpp2_rxq_status_update(port
, rxq
->id
, 0, rxq
->size
);
5678 /* Push packets received by the RXQ to BM pool */
5679 static void mvpp2_rxq_drop_pkts(struct mvpp2_port
*port
,
5680 struct mvpp2_rx_queue
*rxq
)
5684 rx_received
= mvpp2_rxq_received(port
, rxq
->id
);
5688 for (i
= 0; i
< rx_received
; i
++) {
5689 struct mvpp2_rx_desc
*rx_desc
= mvpp2_rxq_next_desc_get(rxq
);
5690 u32 status
= mvpp2_rxdesc_status_get(port
, rx_desc
);
5693 pool
= (status
& MVPP2_RXD_BM_POOL_ID_MASK
) >>
5694 MVPP2_RXD_BM_POOL_ID_OFFS
;
5696 mvpp2_bm_pool_put(port
, pool
,
5697 mvpp2_rxdesc_dma_addr_get(port
, rx_desc
),
5698 mvpp2_rxdesc_cookie_get(port
, rx_desc
));
5700 mvpp2_rxq_status_update(port
, rxq
->id
, rx_received
, rx_received
);
5703 /* Cleanup Rx queue */
5704 static void mvpp2_rxq_deinit(struct mvpp2_port
*port
,
5705 struct mvpp2_rx_queue
*rxq
)
5709 mvpp2_rxq_drop_pkts(port
, rxq
);
5712 dma_free_coherent(port
->dev
->dev
.parent
,
5713 rxq
->size
* MVPP2_DESC_ALIGNED_SIZE
,
5719 rxq
->next_desc_to_proc
= 0;
5722 /* Clear Rx descriptors queue starting address and size;
5723 * free descriptor number
5725 mvpp2_write(port
->priv
, MVPP2_RXQ_STATUS_REG(rxq
->id
), 0);
5727 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_RXQ_NUM_REG
, rxq
->id
);
5728 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_RXQ_DESC_ADDR_REG
, 0);
5729 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_RXQ_DESC_SIZE_REG
, 0);
5733 /* Create and initialize a Tx queue */
5734 static int mvpp2_txq_init(struct mvpp2_port
*port
,
5735 struct mvpp2_tx_queue
*txq
)
5738 int cpu
, desc
, desc_per_txq
, tx_port_num
;
5739 struct mvpp2_txq_pcpu
*txq_pcpu
;
5741 txq
->size
= port
->tx_ring_size
;
5743 /* Allocate memory for Tx descriptors */
5744 txq
->descs
= dma_alloc_coherent(port
->dev
->dev
.parent
,
5745 txq
->size
* MVPP2_DESC_ALIGNED_SIZE
,
5746 &txq
->descs_dma
, GFP_KERNEL
);
5750 txq
->last_desc
= txq
->size
- 1;
5752 /* Set Tx descriptors queue starting address - indirect access */
5754 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_TXQ_NUM_REG
, txq
->id
);
5755 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_TXQ_DESC_ADDR_REG
,
5757 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_TXQ_DESC_SIZE_REG
,
5758 txq
->size
& MVPP2_TXQ_DESC_SIZE_MASK
);
5759 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_TXQ_INDEX_REG
, 0);
5760 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_TXQ_RSVD_CLR_REG
,
5761 txq
->id
<< MVPP2_TXQ_RSVD_CLR_OFFSET
);
5762 val
= mvpp2_percpu_read(port
->priv
, cpu
, MVPP2_TXQ_PENDING_REG
);
5763 val
&= ~MVPP2_TXQ_PENDING_MASK
;
5764 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_TXQ_PENDING_REG
, val
);
5766 /* Calculate base address in prefetch buffer. We reserve 16 descriptors
5767 * for each existing TXQ.
5768 * TCONTS for PON port must be continuous from 0 to MVPP2_MAX_TCONT
5769 * GBE ports assumed to be continious from 0 to MVPP2_MAX_PORTS
5772 desc
= (port
->id
* MVPP2_MAX_TXQ
* desc_per_txq
) +
5773 (txq
->log_id
* desc_per_txq
);
5775 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_TXQ_PREF_BUF_REG
,
5776 MVPP2_PREF_BUF_PTR(desc
) | MVPP2_PREF_BUF_SIZE_16
|
5777 MVPP2_PREF_BUF_THRESH(desc_per_txq
/ 2));
5780 /* WRR / EJP configuration - indirect access */
5781 tx_port_num
= mvpp2_egress_port(port
);
5782 mvpp2_write(port
->priv
, MVPP2_TXP_SCHED_PORT_INDEX_REG
, tx_port_num
);
5784 val
= mvpp2_read(port
->priv
, MVPP2_TXQ_SCHED_REFILL_REG(txq
->log_id
));
5785 val
&= ~MVPP2_TXQ_REFILL_PERIOD_ALL_MASK
;
5786 val
|= MVPP2_TXQ_REFILL_PERIOD_MASK(1);
5787 val
|= MVPP2_TXQ_REFILL_TOKENS_ALL_MASK
;
5788 mvpp2_write(port
->priv
, MVPP2_TXQ_SCHED_REFILL_REG(txq
->log_id
), val
);
5790 val
= MVPP2_TXQ_TOKEN_SIZE_MAX
;
5791 mvpp2_write(port
->priv
, MVPP2_TXQ_SCHED_TOKEN_SIZE_REG(txq
->log_id
),
5794 for_each_present_cpu(cpu
) {
5795 txq_pcpu
= per_cpu_ptr(txq
->pcpu
, cpu
);
5796 txq_pcpu
->size
= txq
->size
;
5797 txq_pcpu
->buffs
= kmalloc_array(txq_pcpu
->size
,
5798 sizeof(*txq_pcpu
->buffs
),
5800 if (!txq_pcpu
->buffs
)
5803 txq_pcpu
->count
= 0;
5804 txq_pcpu
->reserved_num
= 0;
5805 txq_pcpu
->txq_put_index
= 0;
5806 txq_pcpu
->txq_get_index
= 0;
5808 txq_pcpu
->stop_threshold
= txq
->size
- MVPP2_MAX_SKB_DESCS
;
5809 txq_pcpu
->wake_threshold
= txq_pcpu
->stop_threshold
/ 2;
5811 txq_pcpu
->tso_headers
=
5812 dma_alloc_coherent(port
->dev
->dev
.parent
,
5813 txq_pcpu
->size
* TSO_HEADER_SIZE
,
5814 &txq_pcpu
->tso_headers_dma
,
5816 if (!txq_pcpu
->tso_headers
)
5823 /* Free allocated TXQ resources */
5824 static void mvpp2_txq_deinit(struct mvpp2_port
*port
,
5825 struct mvpp2_tx_queue
*txq
)
5827 struct mvpp2_txq_pcpu
*txq_pcpu
;
5830 for_each_present_cpu(cpu
) {
5831 txq_pcpu
= per_cpu_ptr(txq
->pcpu
, cpu
);
5832 kfree(txq_pcpu
->buffs
);
5834 dma_free_coherent(port
->dev
->dev
.parent
,
5835 txq_pcpu
->size
* TSO_HEADER_SIZE
,
5836 txq_pcpu
->tso_headers
,
5837 txq_pcpu
->tso_headers_dma
);
5841 dma_free_coherent(port
->dev
->dev
.parent
,
5842 txq
->size
* MVPP2_DESC_ALIGNED_SIZE
,
5843 txq
->descs
, txq
->descs_dma
);
5847 txq
->next_desc_to_proc
= 0;
5850 /* Set minimum bandwidth for disabled TXQs */
5851 mvpp2_write(port
->priv
, MVPP2_TXQ_SCHED_TOKEN_CNTR_REG(txq
->id
), 0);
5853 /* Set Tx descriptors queue starting address and size */
5855 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_TXQ_NUM_REG
, txq
->id
);
5856 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_TXQ_DESC_ADDR_REG
, 0);
5857 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_TXQ_DESC_SIZE_REG
, 0);
5861 /* Cleanup Tx ports */
5862 static void mvpp2_txq_clean(struct mvpp2_port
*port
, struct mvpp2_tx_queue
*txq
)
5864 struct mvpp2_txq_pcpu
*txq_pcpu
;
5865 int delay
, pending
, cpu
;
5869 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_TXQ_NUM_REG
, txq
->id
);
5870 val
= mvpp2_percpu_read(port
->priv
, cpu
, MVPP2_TXQ_PREF_BUF_REG
);
5871 val
|= MVPP2_TXQ_DRAIN_EN_MASK
;
5872 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_TXQ_PREF_BUF_REG
, val
);
5874 /* The napi queue has been stopped so wait for all packets
5875 * to be transmitted.
5879 if (delay
>= MVPP2_TX_PENDING_TIMEOUT_MSEC
) {
5880 netdev_warn(port
->dev
,
5881 "port %d: cleaning queue %d timed out\n",
5882 port
->id
, txq
->log_id
);
5888 pending
= mvpp2_percpu_read(port
->priv
, cpu
,
5889 MVPP2_TXQ_PENDING_REG
);
5890 pending
&= MVPP2_TXQ_PENDING_MASK
;
5893 val
&= ~MVPP2_TXQ_DRAIN_EN_MASK
;
5894 mvpp2_percpu_write(port
->priv
, cpu
, MVPP2_TXQ_PREF_BUF_REG
, val
);
5897 for_each_present_cpu(cpu
) {
5898 txq_pcpu
= per_cpu_ptr(txq
->pcpu
, cpu
);
5900 /* Release all packets */
5901 mvpp2_txq_bufs_free(port
, txq
, txq_pcpu
, txq_pcpu
->count
);
5904 txq_pcpu
->count
= 0;
5905 txq_pcpu
->txq_put_index
= 0;
5906 txq_pcpu
->txq_get_index
= 0;
5910 /* Cleanup all Tx queues */
5911 static void mvpp2_cleanup_txqs(struct mvpp2_port
*port
)
5913 struct mvpp2_tx_queue
*txq
;
5917 val
= mvpp2_read(port
->priv
, MVPP2_TX_PORT_FLUSH_REG
);
5919 /* Reset Tx ports and delete Tx queues */
5920 val
|= MVPP2_TX_PORT_FLUSH_MASK(port
->id
);
5921 mvpp2_write(port
->priv
, MVPP2_TX_PORT_FLUSH_REG
, val
);
5923 for (queue
= 0; queue
< port
->ntxqs
; queue
++) {
5924 txq
= port
->txqs
[queue
];
5925 mvpp2_txq_clean(port
, txq
);
5926 mvpp2_txq_deinit(port
, txq
);
5929 on_each_cpu(mvpp2_txq_sent_counter_clear
, port
, 1);
5931 val
&= ~MVPP2_TX_PORT_FLUSH_MASK(port
->id
);
5932 mvpp2_write(port
->priv
, MVPP2_TX_PORT_FLUSH_REG
, val
);
5935 /* Cleanup all Rx queues */
5936 static void mvpp2_cleanup_rxqs(struct mvpp2_port
*port
)
5940 for (queue
= 0; queue
< port
->nrxqs
; queue
++)
5941 mvpp2_rxq_deinit(port
, port
->rxqs
[queue
]);
5944 /* Init all Rx queues for port */
5945 static int mvpp2_setup_rxqs(struct mvpp2_port
*port
)
5949 for (queue
= 0; queue
< port
->nrxqs
; queue
++) {
5950 err
= mvpp2_rxq_init(port
, port
->rxqs
[queue
]);
5957 mvpp2_cleanup_rxqs(port
);
5961 /* Init all tx queues for port */
5962 static int mvpp2_setup_txqs(struct mvpp2_port
*port
)
5964 struct mvpp2_tx_queue
*txq
;
5967 for (queue
= 0; queue
< port
->ntxqs
; queue
++) {
5968 txq
= port
->txqs
[queue
];
5969 err
= mvpp2_txq_init(port
, txq
);
5974 if (port
->has_tx_irqs
) {
5975 mvpp2_tx_time_coal_set(port
);
5976 for (queue
= 0; queue
< port
->ntxqs
; queue
++) {
5977 txq
= port
->txqs
[queue
];
5978 mvpp2_tx_pkts_coal_set(port
, txq
);
5982 on_each_cpu(mvpp2_txq_sent_counter_clear
, port
, 1);
5986 mvpp2_cleanup_txqs(port
);
5990 /* The callback for per-port interrupt */
5991 static irqreturn_t
mvpp2_isr(int irq
, void *dev_id
)
5993 struct mvpp2_queue_vector
*qv
= dev_id
;
5995 mvpp2_qvec_interrupt_disable(qv
);
5997 napi_schedule(&qv
->napi
);
6002 /* Per-port interrupt for link status changes */
6003 static irqreturn_t
mvpp2_link_status_isr(int irq
, void *dev_id
)
6005 struct mvpp2_port
*port
= (struct mvpp2_port
*)dev_id
;
6006 struct net_device
*dev
= port
->dev
;
6007 bool event
= false, link
= false;
6010 mvpp22_gop_mask_irq(port
);
6012 if (port
->gop_id
== 0 &&
6013 port
->phy_interface
== PHY_INTERFACE_MODE_10GKR
) {
6014 val
= readl(port
->base
+ MVPP22_XLG_INT_STAT
);
6015 if (val
& MVPP22_XLG_INT_STAT_LINK
) {
6017 val
= readl(port
->base
+ MVPP22_XLG_STATUS
);
6018 if (val
& MVPP22_XLG_STATUS_LINK_UP
)
6021 } else if (phy_interface_mode_is_rgmii(port
->phy_interface
) ||
6022 port
->phy_interface
== PHY_INTERFACE_MODE_SGMII
) {
6023 val
= readl(port
->base
+ MVPP22_GMAC_INT_STAT
);
6024 if (val
& MVPP22_GMAC_INT_STAT_LINK
) {
6026 val
= readl(port
->base
+ MVPP2_GMAC_STATUS0
);
6027 if (val
& MVPP2_GMAC_STATUS0_LINK_UP
)
6032 if (!netif_running(dev
) || !event
)
6036 mvpp2_interrupts_enable(port
);
6038 mvpp2_egress_enable(port
);
6039 mvpp2_ingress_enable(port
);
6040 netif_carrier_on(dev
);
6041 netif_tx_wake_all_queues(dev
);
6043 netif_tx_stop_all_queues(dev
);
6044 netif_carrier_off(dev
);
6045 mvpp2_ingress_disable(port
);
6046 mvpp2_egress_disable(port
);
6048 mvpp2_interrupts_disable(port
);
6052 mvpp22_gop_unmask_irq(port
);
6056 static void mvpp2_gmac_set_autoneg(struct mvpp2_port
*port
,
6057 struct phy_device
*phydev
)
6061 if (port
->phy_interface
!= PHY_INTERFACE_MODE_RGMII
&&
6062 port
->phy_interface
!= PHY_INTERFACE_MODE_RGMII_ID
&&
6063 port
->phy_interface
!= PHY_INTERFACE_MODE_RGMII_RXID
&&
6064 port
->phy_interface
!= PHY_INTERFACE_MODE_RGMII_TXID
&&
6065 port
->phy_interface
!= PHY_INTERFACE_MODE_SGMII
)
6068 val
= readl(port
->base
+ MVPP2_GMAC_AUTONEG_CONFIG
);
6069 val
&= ~(MVPP2_GMAC_CONFIG_MII_SPEED
|
6070 MVPP2_GMAC_CONFIG_GMII_SPEED
|
6071 MVPP2_GMAC_CONFIG_FULL_DUPLEX
|
6072 MVPP2_GMAC_AN_SPEED_EN
|
6073 MVPP2_GMAC_AN_DUPLEX_EN
);
6076 val
|= MVPP2_GMAC_CONFIG_FULL_DUPLEX
;
6078 if (phydev
->speed
== SPEED_1000
)
6079 val
|= MVPP2_GMAC_CONFIG_GMII_SPEED
;
6080 else if (phydev
->speed
== SPEED_100
)
6081 val
|= MVPP2_GMAC_CONFIG_MII_SPEED
;
6083 writel(val
, port
->base
+ MVPP2_GMAC_AUTONEG_CONFIG
);
6087 static void mvpp2_link_event(struct net_device
*dev
)
6089 struct mvpp2_port
*port
= netdev_priv(dev
);
6090 struct phy_device
*phydev
= dev
->phydev
;
6091 bool link_reconfigured
= false;
6095 if (port
->phy_interface
!= phydev
->interface
&& port
->comphy
) {
6096 /* disable current port for reconfiguration */
6097 mvpp2_interrupts_disable(port
);
6098 netif_carrier_off(port
->dev
);
6099 mvpp2_port_disable(port
);
6100 phy_power_off(port
->comphy
);
6102 /* comphy reconfiguration */
6103 port
->phy_interface
= phydev
->interface
;
6104 mvpp22_comphy_init(port
);
6106 /* gop/mac reconfiguration */
6107 mvpp22_gop_init(port
);
6108 mvpp2_port_mii_set(port
);
6110 link_reconfigured
= true;
6113 if ((port
->speed
!= phydev
->speed
) ||
6114 (port
->duplex
!= phydev
->duplex
)) {
6115 mvpp2_gmac_set_autoneg(port
, phydev
);
6117 port
->duplex
= phydev
->duplex
;
6118 port
->speed
= phydev
->speed
;
6122 if (phydev
->link
!= port
->link
|| link_reconfigured
) {
6123 port
->link
= phydev
->link
;
6126 if (port
->phy_interface
== PHY_INTERFACE_MODE_RGMII
||
6127 port
->phy_interface
== PHY_INTERFACE_MODE_RGMII_ID
||
6128 port
->phy_interface
== PHY_INTERFACE_MODE_RGMII_RXID
||
6129 port
->phy_interface
== PHY_INTERFACE_MODE_RGMII_TXID
||
6130 port
->phy_interface
== PHY_INTERFACE_MODE_SGMII
) {
6131 val
= readl(port
->base
+ MVPP2_GMAC_AUTONEG_CONFIG
);
6132 val
|= (MVPP2_GMAC_FORCE_LINK_PASS
|
6133 MVPP2_GMAC_FORCE_LINK_DOWN
);
6134 writel(val
, port
->base
+ MVPP2_GMAC_AUTONEG_CONFIG
);
6137 mvpp2_interrupts_enable(port
);
6138 mvpp2_port_enable(port
);
6140 mvpp2_egress_enable(port
);
6141 mvpp2_ingress_enable(port
);
6142 netif_carrier_on(dev
);
6143 netif_tx_wake_all_queues(dev
);
6148 netif_tx_stop_all_queues(dev
);
6149 netif_carrier_off(dev
);
6150 mvpp2_ingress_disable(port
);
6151 mvpp2_egress_disable(port
);
6153 mvpp2_port_disable(port
);
6154 mvpp2_interrupts_disable(port
);
6157 phy_print_status(phydev
);
6161 static void mvpp2_timer_set(struct mvpp2_port_pcpu
*port_pcpu
)
6165 if (!port_pcpu
->timer_scheduled
) {
6166 port_pcpu
->timer_scheduled
= true;
6167 interval
= MVPP2_TXDONE_HRTIMER_PERIOD_NS
;
6168 hrtimer_start(&port_pcpu
->tx_done_timer
, interval
,
6169 HRTIMER_MODE_REL_PINNED
);
6173 static void mvpp2_tx_proc_cb(unsigned long data
)
6175 struct net_device
*dev
= (struct net_device
*)data
;
6176 struct mvpp2_port
*port
= netdev_priv(dev
);
6177 struct mvpp2_port_pcpu
*port_pcpu
= this_cpu_ptr(port
->pcpu
);
6178 unsigned int tx_todo
, cause
;
6180 if (!netif_running(dev
))
6182 port_pcpu
->timer_scheduled
= false;
6184 /* Process all the Tx queues */
6185 cause
= (1 << port
->ntxqs
) - 1;
6186 tx_todo
= mvpp2_tx_done(port
, cause
, smp_processor_id());
6188 /* Set the timer in case not all the packets were processed */
6190 mvpp2_timer_set(port_pcpu
);
6193 static enum hrtimer_restart
mvpp2_hr_timer_cb(struct hrtimer
*timer
)
6195 struct mvpp2_port_pcpu
*port_pcpu
= container_of(timer
,
6196 struct mvpp2_port_pcpu
,
6199 tasklet_schedule(&port_pcpu
->tx_done_tasklet
);
6201 return HRTIMER_NORESTART
;
6204 /* Main RX/TX processing routines */
6206 /* Display more error info */
6207 static void mvpp2_rx_error(struct mvpp2_port
*port
,
6208 struct mvpp2_rx_desc
*rx_desc
)
6210 u32 status
= mvpp2_rxdesc_status_get(port
, rx_desc
);
6211 size_t sz
= mvpp2_rxdesc_size_get(port
, rx_desc
);
6213 switch (status
& MVPP2_RXD_ERR_CODE_MASK
) {
6214 case MVPP2_RXD_ERR_CRC
:
6215 netdev_err(port
->dev
, "bad rx status %08x (crc error), size=%zu\n",
6218 case MVPP2_RXD_ERR_OVERRUN
:
6219 netdev_err(port
->dev
, "bad rx status %08x (overrun error), size=%zu\n",
6222 case MVPP2_RXD_ERR_RESOURCE
:
6223 netdev_err(port
->dev
, "bad rx status %08x (resource error), size=%zu\n",
6229 /* Handle RX checksum offload */
6230 static void mvpp2_rx_csum(struct mvpp2_port
*port
, u32 status
,
6231 struct sk_buff
*skb
)
6233 if (((status
& MVPP2_RXD_L3_IP4
) &&
6234 !(status
& MVPP2_RXD_IP4_HEADER_ERR
)) ||
6235 (status
& MVPP2_RXD_L3_IP6
))
6236 if (((status
& MVPP2_RXD_L4_UDP
) ||
6237 (status
& MVPP2_RXD_L4_TCP
)) &&
6238 (status
& MVPP2_RXD_L4_CSUM_OK
)) {
6240 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
6244 skb
->ip_summed
= CHECKSUM_NONE
;
6247 /* Reuse skb if possible, or allocate a new skb and add it to BM pool */
6248 static int mvpp2_rx_refill(struct mvpp2_port
*port
,
6249 struct mvpp2_bm_pool
*bm_pool
, int pool
)
6251 dma_addr_t dma_addr
;
6252 phys_addr_t phys_addr
;
6255 /* No recycle or too many buffers are in use, so allocate a new skb */
6256 buf
= mvpp2_buf_alloc(port
, bm_pool
, &dma_addr
, &phys_addr
,
6261 mvpp2_bm_pool_put(port
, pool
, dma_addr
, phys_addr
);
6266 /* Handle tx checksum */
6267 static u32
mvpp2_skb_tx_csum(struct mvpp2_port
*port
, struct sk_buff
*skb
)
6269 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
6273 if (skb
->protocol
== htons(ETH_P_IP
)) {
6274 struct iphdr
*ip4h
= ip_hdr(skb
);
6276 /* Calculate IPv4 checksum and L4 checksum */
6277 ip_hdr_len
= ip4h
->ihl
;
6278 l4_proto
= ip4h
->protocol
;
6279 } else if (skb
->protocol
== htons(ETH_P_IPV6
)) {
6280 struct ipv6hdr
*ip6h
= ipv6_hdr(skb
);
6282 /* Read l4_protocol from one of IPv6 extra headers */
6283 if (skb_network_header_len(skb
) > 0)
6284 ip_hdr_len
= (skb_network_header_len(skb
) >> 2);
6285 l4_proto
= ip6h
->nexthdr
;
6287 return MVPP2_TXD_L4_CSUM_NOT
;
6290 return mvpp2_txq_desc_csum(skb_network_offset(skb
),
6291 skb
->protocol
, ip_hdr_len
, l4_proto
);
6294 return MVPP2_TXD_L4_CSUM_NOT
| MVPP2_TXD_IP_CSUM_DISABLE
;
6297 /* Main rx processing */
6298 static int mvpp2_rx(struct mvpp2_port
*port
, struct napi_struct
*napi
,
6299 int rx_todo
, struct mvpp2_rx_queue
*rxq
)
6301 struct net_device
*dev
= port
->dev
;
6307 /* Get number of received packets and clamp the to-do */
6308 rx_received
= mvpp2_rxq_received(port
, rxq
->id
);
6309 if (rx_todo
> rx_received
)
6310 rx_todo
= rx_received
;
6312 while (rx_done
< rx_todo
) {
6313 struct mvpp2_rx_desc
*rx_desc
= mvpp2_rxq_next_desc_get(rxq
);
6314 struct mvpp2_bm_pool
*bm_pool
;
6315 struct sk_buff
*skb
;
6316 unsigned int frag_size
;
6317 dma_addr_t dma_addr
;
6318 phys_addr_t phys_addr
;
6320 int pool
, rx_bytes
, err
;
6324 rx_status
= mvpp2_rxdesc_status_get(port
, rx_desc
);
6325 rx_bytes
= mvpp2_rxdesc_size_get(port
, rx_desc
);
6326 rx_bytes
-= MVPP2_MH_SIZE
;
6327 dma_addr
= mvpp2_rxdesc_dma_addr_get(port
, rx_desc
);
6328 phys_addr
= mvpp2_rxdesc_cookie_get(port
, rx_desc
);
6329 data
= (void *)phys_to_virt(phys_addr
);
6331 pool
= (rx_status
& MVPP2_RXD_BM_POOL_ID_MASK
) >>
6332 MVPP2_RXD_BM_POOL_ID_OFFS
;
6333 bm_pool
= &port
->priv
->bm_pools
[pool
];
6335 /* In case of an error, release the requested buffer pointer
6336 * to the Buffer Manager. This request process is controlled
6337 * by the hardware, and the information about the buffer is
6338 * comprised by the RX descriptor.
6340 if (rx_status
& MVPP2_RXD_ERR_SUMMARY
) {
6342 dev
->stats
.rx_errors
++;
6343 mvpp2_rx_error(port
, rx_desc
);
6344 /* Return the buffer to the pool */
6345 mvpp2_bm_pool_put(port
, pool
, dma_addr
, phys_addr
);
6349 if (bm_pool
->frag_size
> PAGE_SIZE
)
6352 frag_size
= bm_pool
->frag_size
;
6354 skb
= build_skb(data
, frag_size
);
6356 netdev_warn(port
->dev
, "skb build failed\n");
6357 goto err_drop_frame
;
6360 err
= mvpp2_rx_refill(port
, bm_pool
, pool
);
6362 netdev_err(port
->dev
, "failed to refill BM pools\n");
6363 goto err_drop_frame
;
6366 dma_unmap_single(dev
->dev
.parent
, dma_addr
,
6367 bm_pool
->buf_size
, DMA_FROM_DEVICE
);
6370 rcvd_bytes
+= rx_bytes
;
6372 skb_reserve(skb
, MVPP2_MH_SIZE
+ NET_SKB_PAD
);
6373 skb_put(skb
, rx_bytes
);
6374 skb
->protocol
= eth_type_trans(skb
, dev
);
6375 mvpp2_rx_csum(port
, rx_status
, skb
);
6377 napi_gro_receive(napi
, skb
);
6381 struct mvpp2_pcpu_stats
*stats
= this_cpu_ptr(port
->stats
);
6383 u64_stats_update_begin(&stats
->syncp
);
6384 stats
->rx_packets
+= rcvd_pkts
;
6385 stats
->rx_bytes
+= rcvd_bytes
;
6386 u64_stats_update_end(&stats
->syncp
);
6389 /* Update Rx queue management counters */
6391 mvpp2_rxq_status_update(port
, rxq
->id
, rx_done
, rx_done
);
6397 tx_desc_unmap_put(struct mvpp2_port
*port
, struct mvpp2_tx_queue
*txq
,
6398 struct mvpp2_tx_desc
*desc
)
6400 struct mvpp2_txq_pcpu
*txq_pcpu
= this_cpu_ptr(txq
->pcpu
);
6402 dma_addr_t buf_dma_addr
=
6403 mvpp2_txdesc_dma_addr_get(port
, desc
);
6405 mvpp2_txdesc_size_get(port
, desc
);
6406 if (!IS_TSO_HEADER(txq_pcpu
, buf_dma_addr
))
6407 dma_unmap_single(port
->dev
->dev
.parent
, buf_dma_addr
,
6408 buf_sz
, DMA_TO_DEVICE
);
6409 mvpp2_txq_desc_put(txq
);
6412 /* Handle tx fragmentation processing */
6413 static int mvpp2_tx_frag_process(struct mvpp2_port
*port
, struct sk_buff
*skb
,
6414 struct mvpp2_tx_queue
*aggr_txq
,
6415 struct mvpp2_tx_queue
*txq
)
6417 struct mvpp2_txq_pcpu
*txq_pcpu
= this_cpu_ptr(txq
->pcpu
);
6418 struct mvpp2_tx_desc
*tx_desc
;
6420 dma_addr_t buf_dma_addr
;
6422 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
6423 skb_frag_t
*frag
= &skb_shinfo(skb
)->frags
[i
];
6424 void *addr
= page_address(frag
->page
.p
) + frag
->page_offset
;
6426 tx_desc
= mvpp2_txq_next_desc_get(aggr_txq
);
6427 mvpp2_txdesc_txq_set(port
, tx_desc
, txq
->id
);
6428 mvpp2_txdesc_size_set(port
, tx_desc
, frag
->size
);
6430 buf_dma_addr
= dma_map_single(port
->dev
->dev
.parent
, addr
,
6433 if (dma_mapping_error(port
->dev
->dev
.parent
, buf_dma_addr
)) {
6434 mvpp2_txq_desc_put(txq
);
6438 mvpp2_txdesc_dma_addr_set(port
, tx_desc
, buf_dma_addr
);
6440 if (i
== (skb_shinfo(skb
)->nr_frags
- 1)) {
6441 /* Last descriptor */
6442 mvpp2_txdesc_cmd_set(port
, tx_desc
,
6444 mvpp2_txq_inc_put(port
, txq_pcpu
, skb
, tx_desc
);
6446 /* Descriptor in the middle: Not First, Not Last */
6447 mvpp2_txdesc_cmd_set(port
, tx_desc
, 0);
6448 mvpp2_txq_inc_put(port
, txq_pcpu
, NULL
, tx_desc
);
6454 /* Release all descriptors that were used to map fragments of
6455 * this packet, as well as the corresponding DMA mappings
6457 for (i
= i
- 1; i
>= 0; i
--) {
6458 tx_desc
= txq
->descs
+ i
;
6459 tx_desc_unmap_put(port
, txq
, tx_desc
);
6465 static inline void mvpp2_tso_put_hdr(struct sk_buff
*skb
,
6466 struct net_device
*dev
,
6467 struct mvpp2_tx_queue
*txq
,
6468 struct mvpp2_tx_queue
*aggr_txq
,
6469 struct mvpp2_txq_pcpu
*txq_pcpu
,
6472 struct mvpp2_port
*port
= netdev_priv(dev
);
6473 struct mvpp2_tx_desc
*tx_desc
= mvpp2_txq_next_desc_get(aggr_txq
);
6476 mvpp2_txdesc_txq_set(port
, tx_desc
, txq
->id
);
6477 mvpp2_txdesc_size_set(port
, tx_desc
, hdr_sz
);
6479 addr
= txq_pcpu
->tso_headers_dma
+
6480 txq_pcpu
->txq_put_index
* TSO_HEADER_SIZE
;
6481 mvpp2_txdesc_dma_addr_set(port
, tx_desc
, addr
);
6483 mvpp2_txdesc_cmd_set(port
, tx_desc
, mvpp2_skb_tx_csum(port
, skb
) |
6485 MVPP2_TXD_PADDING_DISABLE
);
6486 mvpp2_txq_inc_put(port
, txq_pcpu
, NULL
, tx_desc
);
6489 static inline int mvpp2_tso_put_data(struct sk_buff
*skb
,
6490 struct net_device
*dev
, struct tso_t
*tso
,
6491 struct mvpp2_tx_queue
*txq
,
6492 struct mvpp2_tx_queue
*aggr_txq
,
6493 struct mvpp2_txq_pcpu
*txq_pcpu
,
6494 int sz
, bool left
, bool last
)
6496 struct mvpp2_port
*port
= netdev_priv(dev
);
6497 struct mvpp2_tx_desc
*tx_desc
= mvpp2_txq_next_desc_get(aggr_txq
);
6498 dma_addr_t buf_dma_addr
;
6500 mvpp2_txdesc_txq_set(port
, tx_desc
, txq
->id
);
6501 mvpp2_txdesc_size_set(port
, tx_desc
, sz
);
6503 buf_dma_addr
= dma_map_single(dev
->dev
.parent
, tso
->data
, sz
,
6505 if (unlikely(dma_mapping_error(dev
->dev
.parent
, buf_dma_addr
))) {
6506 mvpp2_txq_desc_put(txq
);
6510 mvpp2_txdesc_dma_addr_set(port
, tx_desc
, buf_dma_addr
);
6513 mvpp2_txdesc_cmd_set(port
, tx_desc
, MVPP2_TXD_L_DESC
);
6515 mvpp2_txq_inc_put(port
, txq_pcpu
, skb
, tx_desc
);
6519 mvpp2_txdesc_cmd_set(port
, tx_desc
, 0);
6522 mvpp2_txq_inc_put(port
, txq_pcpu
, NULL
, tx_desc
);
6526 static int mvpp2_tx_tso(struct sk_buff
*skb
, struct net_device
*dev
,
6527 struct mvpp2_tx_queue
*txq
,
6528 struct mvpp2_tx_queue
*aggr_txq
,
6529 struct mvpp2_txq_pcpu
*txq_pcpu
)
6531 struct mvpp2_port
*port
= netdev_priv(dev
);
6533 int hdr_sz
= skb_transport_offset(skb
) + tcp_hdrlen(skb
);
6534 int i
, len
, descs
= 0;
6536 /* Check number of available descriptors */
6537 if (mvpp2_aggr_desc_num_check(port
->priv
, aggr_txq
,
6538 tso_count_descs(skb
)) ||
6539 mvpp2_txq_reserved_desc_num_proc(port
->priv
, txq
, txq_pcpu
,
6540 tso_count_descs(skb
)))
6543 tso_start(skb
, &tso
);
6544 len
= skb
->len
- hdr_sz
;
6546 int left
= min_t(int, skb_shinfo(skb
)->gso_size
, len
);
6547 char *hdr
= txq_pcpu
->tso_headers
+
6548 txq_pcpu
->txq_put_index
* TSO_HEADER_SIZE
;
6553 tso_build_hdr(skb
, hdr
, &tso
, left
, len
== 0);
6554 mvpp2_tso_put_hdr(skb
, dev
, txq
, aggr_txq
, txq_pcpu
, hdr_sz
);
6557 int sz
= min_t(int, tso
.size
, left
);
6561 if (mvpp2_tso_put_data(skb
, dev
, &tso
, txq
, aggr_txq
,
6562 txq_pcpu
, sz
, left
, len
== 0))
6564 tso_build_data(skb
, &tso
, sz
);
6571 for (i
= descs
- 1; i
>= 0; i
--) {
6572 struct mvpp2_tx_desc
*tx_desc
= txq
->descs
+ i
;
6573 tx_desc_unmap_put(port
, txq
, tx_desc
);
6578 /* Main tx processing */
6579 static int mvpp2_tx(struct sk_buff
*skb
, struct net_device
*dev
)
6581 struct mvpp2_port
*port
= netdev_priv(dev
);
6582 struct mvpp2_tx_queue
*txq
, *aggr_txq
;
6583 struct mvpp2_txq_pcpu
*txq_pcpu
;
6584 struct mvpp2_tx_desc
*tx_desc
;
6585 dma_addr_t buf_dma_addr
;
6590 txq_id
= skb_get_queue_mapping(skb
);
6591 txq
= port
->txqs
[txq_id
];
6592 txq_pcpu
= this_cpu_ptr(txq
->pcpu
);
6593 aggr_txq
= &port
->priv
->aggr_txqs
[smp_processor_id()];
6595 if (skb_is_gso(skb
)) {
6596 frags
= mvpp2_tx_tso(skb
, dev
, txq
, aggr_txq
, txq_pcpu
);
6599 frags
= skb_shinfo(skb
)->nr_frags
+ 1;
6601 /* Check number of available descriptors */
6602 if (mvpp2_aggr_desc_num_check(port
->priv
, aggr_txq
, frags
) ||
6603 mvpp2_txq_reserved_desc_num_proc(port
->priv
, txq
,
6609 /* Get a descriptor for the first part of the packet */
6610 tx_desc
= mvpp2_txq_next_desc_get(aggr_txq
);
6611 mvpp2_txdesc_txq_set(port
, tx_desc
, txq
->id
);
6612 mvpp2_txdesc_size_set(port
, tx_desc
, skb_headlen(skb
));
6614 buf_dma_addr
= dma_map_single(dev
->dev
.parent
, skb
->data
,
6615 skb_headlen(skb
), DMA_TO_DEVICE
);
6616 if (unlikely(dma_mapping_error(dev
->dev
.parent
, buf_dma_addr
))) {
6617 mvpp2_txq_desc_put(txq
);
6622 mvpp2_txdesc_dma_addr_set(port
, tx_desc
, buf_dma_addr
);
6624 tx_cmd
= mvpp2_skb_tx_csum(port
, skb
);
6627 /* First and Last descriptor */
6628 tx_cmd
|= MVPP2_TXD_F_DESC
| MVPP2_TXD_L_DESC
;
6629 mvpp2_txdesc_cmd_set(port
, tx_desc
, tx_cmd
);
6630 mvpp2_txq_inc_put(port
, txq_pcpu
, skb
, tx_desc
);
6632 /* First but not Last */
6633 tx_cmd
|= MVPP2_TXD_F_DESC
| MVPP2_TXD_PADDING_DISABLE
;
6634 mvpp2_txdesc_cmd_set(port
, tx_desc
, tx_cmd
);
6635 mvpp2_txq_inc_put(port
, txq_pcpu
, NULL
, tx_desc
);
6637 /* Continue with other skb fragments */
6638 if (mvpp2_tx_frag_process(port
, skb
, aggr_txq
, txq
)) {
6639 tx_desc_unmap_put(port
, txq
, tx_desc
);
6646 struct mvpp2_pcpu_stats
*stats
= this_cpu_ptr(port
->stats
);
6647 struct netdev_queue
*nq
= netdev_get_tx_queue(dev
, txq_id
);
6649 txq_pcpu
->reserved_num
-= frags
;
6650 txq_pcpu
->count
+= frags
;
6651 aggr_txq
->count
+= frags
;
6653 /* Enable transmit */
6655 mvpp2_aggr_txq_pend_desc_add(port
, frags
);
6657 if (txq_pcpu
->count
>= txq_pcpu
->stop_threshold
)
6658 netif_tx_stop_queue(nq
);
6660 u64_stats_update_begin(&stats
->syncp
);
6661 stats
->tx_packets
++;
6662 stats
->tx_bytes
+= skb
->len
;
6663 u64_stats_update_end(&stats
->syncp
);
6665 dev
->stats
.tx_dropped
++;
6666 dev_kfree_skb_any(skb
);
6669 /* Finalize TX processing */
6670 if (!port
->has_tx_irqs
&& txq_pcpu
->count
>= txq
->done_pkts_coal
)
6671 mvpp2_txq_done(port
, txq
, txq_pcpu
);
6673 /* Set the timer in case not all frags were processed */
6674 if (!port
->has_tx_irqs
&& txq_pcpu
->count
<= frags
&&
6675 txq_pcpu
->count
> 0) {
6676 struct mvpp2_port_pcpu
*port_pcpu
= this_cpu_ptr(port
->pcpu
);
6678 mvpp2_timer_set(port_pcpu
);
6681 return NETDEV_TX_OK
;
6684 static inline void mvpp2_cause_error(struct net_device
*dev
, int cause
)
6686 if (cause
& MVPP2_CAUSE_FCS_ERR_MASK
)
6687 netdev_err(dev
, "FCS error\n");
6688 if (cause
& MVPP2_CAUSE_RX_FIFO_OVERRUN_MASK
)
6689 netdev_err(dev
, "rx fifo overrun error\n");
6690 if (cause
& MVPP2_CAUSE_TX_FIFO_UNDERRUN_MASK
)
6691 netdev_err(dev
, "tx fifo underrun error\n");
6694 static int mvpp2_poll(struct napi_struct
*napi
, int budget
)
6696 u32 cause_rx_tx
, cause_rx
, cause_tx
, cause_misc
;
6698 struct mvpp2_port
*port
= netdev_priv(napi
->dev
);
6699 struct mvpp2_queue_vector
*qv
;
6700 int cpu
= smp_processor_id();
6702 qv
= container_of(napi
, struct mvpp2_queue_vector
, napi
);
6704 /* Rx/Tx cause register
6706 * Bits 0-15: each bit indicates received packets on the Rx queue
6707 * (bit 0 is for Rx queue 0).
6709 * Bits 16-23: each bit indicates transmitted packets on the Tx queue
6710 * (bit 16 is for Tx queue 0).
6712 * Each CPU has its own Rx/Tx cause register
6714 cause_rx_tx
= mvpp2_percpu_read(port
->priv
, qv
->sw_thread_id
,
6715 MVPP2_ISR_RX_TX_CAUSE_REG(port
->id
));
6717 cause_misc
= cause_rx_tx
& MVPP2_CAUSE_MISC_SUM_MASK
;
6719 mvpp2_cause_error(port
->dev
, cause_misc
);
6721 /* Clear the cause register */
6722 mvpp2_write(port
->priv
, MVPP2_ISR_MISC_CAUSE_REG
, 0);
6723 mvpp2_percpu_write(port
->priv
, cpu
,
6724 MVPP2_ISR_RX_TX_CAUSE_REG(port
->id
),
6725 cause_rx_tx
& ~MVPP2_CAUSE_MISC_SUM_MASK
);
6728 cause_tx
= cause_rx_tx
& MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_MASK
;
6730 cause_tx
>>= MVPP2_CAUSE_TXQ_OCCUP_DESC_ALL_OFFSET
;
6731 mvpp2_tx_done(port
, cause_tx
, qv
->sw_thread_id
);
6734 /* Process RX packets */
6735 cause_rx
= cause_rx_tx
& MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK
;
6736 cause_rx
<<= qv
->first_rxq
;
6737 cause_rx
|= qv
->pending_cause_rx
;
6738 while (cause_rx
&& budget
> 0) {
6740 struct mvpp2_rx_queue
*rxq
;
6742 rxq
= mvpp2_get_rx_queue(port
, cause_rx
);
6746 count
= mvpp2_rx(port
, napi
, budget
, rxq
);
6750 /* Clear the bit associated to this Rx queue
6751 * so that next iteration will continue from
6752 * the next Rx queue.
6754 cause_rx
&= ~(1 << rxq
->logic_rxq
);
6760 napi_complete_done(napi
, rx_done
);
6762 mvpp2_qvec_interrupt_enable(qv
);
6764 qv
->pending_cause_rx
= cause_rx
;
6768 /* Set hw internals when starting port */
6769 static void mvpp2_start_dev(struct mvpp2_port
*port
)
6771 struct net_device
*ndev
= port
->dev
;
6774 if (port
->gop_id
== 0 &&
6775 (port
->phy_interface
== PHY_INTERFACE_MODE_XAUI
||
6776 port
->phy_interface
== PHY_INTERFACE_MODE_10GKR
))
6777 mvpp2_xlg_max_rx_size_set(port
);
6779 mvpp2_gmac_max_rx_size_set(port
);
6781 mvpp2_txp_max_tx_size_set(port
);
6783 for (i
= 0; i
< port
->nqvecs
; i
++)
6784 napi_enable(&port
->qvecs
[i
].napi
);
6786 /* Enable interrupts on all CPUs */
6787 mvpp2_interrupts_enable(port
);
6789 if (port
->priv
->hw_version
== MVPP22
) {
6790 mvpp22_comphy_init(port
);
6791 mvpp22_gop_init(port
);
6794 mvpp2_port_mii_set(port
);
6795 mvpp2_port_enable(port
);
6797 phy_start(ndev
->phydev
);
6798 netif_tx_start_all_queues(port
->dev
);
6801 /* Set hw internals when stopping port */
6802 static void mvpp2_stop_dev(struct mvpp2_port
*port
)
6804 struct net_device
*ndev
= port
->dev
;
6807 /* Stop new packets from arriving to RXQs */
6808 mvpp2_ingress_disable(port
);
6812 /* Disable interrupts on all CPUs */
6813 mvpp2_interrupts_disable(port
);
6815 for (i
= 0; i
< port
->nqvecs
; i
++)
6816 napi_disable(&port
->qvecs
[i
].napi
);
6818 netif_carrier_off(port
->dev
);
6819 netif_tx_stop_all_queues(port
->dev
);
6821 mvpp2_egress_disable(port
);
6822 mvpp2_port_disable(port
);
6824 phy_stop(ndev
->phydev
);
6825 phy_power_off(port
->comphy
);
6828 static int mvpp2_check_ringparam_valid(struct net_device
*dev
,
6829 struct ethtool_ringparam
*ring
)
6831 u16 new_rx_pending
= ring
->rx_pending
;
6832 u16 new_tx_pending
= ring
->tx_pending
;
6834 if (ring
->rx_pending
== 0 || ring
->tx_pending
== 0)
6837 if (ring
->rx_pending
> MVPP2_MAX_RXD
)
6838 new_rx_pending
= MVPP2_MAX_RXD
;
6839 else if (!IS_ALIGNED(ring
->rx_pending
, 16))
6840 new_rx_pending
= ALIGN(ring
->rx_pending
, 16);
6842 if (ring
->tx_pending
> MVPP2_MAX_TXD
)
6843 new_tx_pending
= MVPP2_MAX_TXD
;
6844 else if (!IS_ALIGNED(ring
->tx_pending
, 32))
6845 new_tx_pending
= ALIGN(ring
->tx_pending
, 32);
6847 /* The Tx ring size cannot be smaller than the minimum number of
6848 * descriptors needed for TSO.
6850 if (new_tx_pending
< MVPP2_MAX_SKB_DESCS
)
6851 new_tx_pending
= ALIGN(MVPP2_MAX_SKB_DESCS
, 32);
6853 if (ring
->rx_pending
!= new_rx_pending
) {
6854 netdev_info(dev
, "illegal Rx ring size value %d, round to %d\n",
6855 ring
->rx_pending
, new_rx_pending
);
6856 ring
->rx_pending
= new_rx_pending
;
6859 if (ring
->tx_pending
!= new_tx_pending
) {
6860 netdev_info(dev
, "illegal Tx ring size value %d, round to %d\n",
6861 ring
->tx_pending
, new_tx_pending
);
6862 ring
->tx_pending
= new_tx_pending
;
6868 static void mvpp21_get_mac_address(struct mvpp2_port
*port
, unsigned char *addr
)
6870 u32 mac_addr_l
, mac_addr_m
, mac_addr_h
;
6872 mac_addr_l
= readl(port
->base
+ MVPP2_GMAC_CTRL_1_REG
);
6873 mac_addr_m
= readl(port
->priv
->lms_base
+ MVPP2_SRC_ADDR_MIDDLE
);
6874 mac_addr_h
= readl(port
->priv
->lms_base
+ MVPP2_SRC_ADDR_HIGH
);
6875 addr
[0] = (mac_addr_h
>> 24) & 0xFF;
6876 addr
[1] = (mac_addr_h
>> 16) & 0xFF;
6877 addr
[2] = (mac_addr_h
>> 8) & 0xFF;
6878 addr
[3] = mac_addr_h
& 0xFF;
6879 addr
[4] = mac_addr_m
& 0xFF;
6880 addr
[5] = (mac_addr_l
>> MVPP2_GMAC_SA_LOW_OFFS
) & 0xFF;
6883 static int mvpp2_phy_connect(struct mvpp2_port
*port
)
6885 struct phy_device
*phy_dev
;
6887 /* No PHY is attached */
6888 if (!port
->phy_node
)
6891 phy_dev
= of_phy_connect(port
->dev
, port
->phy_node
, mvpp2_link_event
, 0,
6892 port
->phy_interface
);
6894 netdev_err(port
->dev
, "cannot connect to phy\n");
6897 phy_dev
->supported
&= PHY_GBIT_FEATURES
;
6898 phy_dev
->advertising
= phy_dev
->supported
;
6907 static void mvpp2_phy_disconnect(struct mvpp2_port
*port
)
6909 struct net_device
*ndev
= port
->dev
;
6914 phy_disconnect(ndev
->phydev
);
6917 static int mvpp2_irqs_init(struct mvpp2_port
*port
)
6921 for (i
= 0; i
< port
->nqvecs
; i
++) {
6922 struct mvpp2_queue_vector
*qv
= port
->qvecs
+ i
;
6924 if (qv
->type
== MVPP2_QUEUE_VECTOR_PRIVATE
)
6925 irq_set_status_flags(qv
->irq
, IRQ_NO_BALANCING
);
6927 err
= request_irq(qv
->irq
, mvpp2_isr
, 0, port
->dev
->name
, qv
);
6931 if (qv
->type
== MVPP2_QUEUE_VECTOR_PRIVATE
)
6932 irq_set_affinity_hint(qv
->irq
,
6933 cpumask_of(qv
->sw_thread_id
));
6938 for (i
= 0; i
< port
->nqvecs
; i
++) {
6939 struct mvpp2_queue_vector
*qv
= port
->qvecs
+ i
;
6941 irq_set_affinity_hint(qv
->irq
, NULL
);
6942 free_irq(qv
->irq
, qv
);
6948 static void mvpp2_irqs_deinit(struct mvpp2_port
*port
)
6952 for (i
= 0; i
< port
->nqvecs
; i
++) {
6953 struct mvpp2_queue_vector
*qv
= port
->qvecs
+ i
;
6955 irq_set_affinity_hint(qv
->irq
, NULL
);
6956 irq_clear_status_flags(qv
->irq
, IRQ_NO_BALANCING
);
6957 free_irq(qv
->irq
, qv
);
6961 static void mvpp22_init_rss(struct mvpp2_port
*port
)
6963 struct mvpp2
*priv
= port
->priv
;
6966 /* Set the table width: replace the whole classifier Rx queue number
6967 * with the ones configured in RSS table entries.
6969 mvpp2_write(priv
, MVPP22_RSS_INDEX
, MVPP22_RSS_INDEX_TABLE(0));
6970 mvpp2_write(priv
, MVPP22_RSS_WIDTH
, 8);
6972 /* Loop through the classifier Rx Queues and map them to a RSS table.
6973 * Map them all to the first table (0) by default.
6975 for (i
= 0; i
< MVPP2_CLS_RX_QUEUES
; i
++) {
6976 mvpp2_write(priv
, MVPP22_RSS_INDEX
, MVPP22_RSS_INDEX_QUEUE(i
));
6977 mvpp2_write(priv
, MVPP22_RSS_TABLE
,
6978 MVPP22_RSS_TABLE_POINTER(0));
6981 /* Configure the first table to evenly distribute the packets across
6982 * real Rx Queues. The table entries map a hash to an port Rx Queue.
6984 for (i
= 0; i
< MVPP22_RSS_TABLE_ENTRIES
; i
++) {
6985 u32 sel
= MVPP22_RSS_INDEX_TABLE(0) |
6986 MVPP22_RSS_INDEX_TABLE_ENTRY(i
);
6987 mvpp2_write(priv
, MVPP22_RSS_INDEX
, sel
);
6989 mvpp2_write(priv
, MVPP22_RSS_TABLE_ENTRY
, i
% port
->nrxqs
);
6994 static int mvpp2_open(struct net_device
*dev
)
6996 struct mvpp2_port
*port
= netdev_priv(dev
);
6997 struct mvpp2
*priv
= port
->priv
;
6998 unsigned char mac_bcast
[ETH_ALEN
] = {
6999 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
7002 err
= mvpp2_prs_mac_da_accept(port
->priv
, port
->id
, mac_bcast
, true);
7004 netdev_err(dev
, "mvpp2_prs_mac_da_accept BC failed\n");
7007 err
= mvpp2_prs_mac_da_accept(port
->priv
, port
->id
,
7008 dev
->dev_addr
, true);
7010 netdev_err(dev
, "mvpp2_prs_mac_da_accept MC failed\n");
7013 err
= mvpp2_prs_tag_mode_set(port
->priv
, port
->id
, MVPP2_TAG_TYPE_MH
);
7015 netdev_err(dev
, "mvpp2_prs_tag_mode_set failed\n");
7018 err
= mvpp2_prs_def_flow(port
);
7020 netdev_err(dev
, "mvpp2_prs_def_flow failed\n");
7024 /* Allocate the Rx/Tx queues */
7025 err
= mvpp2_setup_rxqs(port
);
7027 netdev_err(port
->dev
, "cannot allocate Rx queues\n");
7031 err
= mvpp2_setup_txqs(port
);
7033 netdev_err(port
->dev
, "cannot allocate Tx queues\n");
7034 goto err_cleanup_rxqs
;
7037 err
= mvpp2_irqs_init(port
);
7039 netdev_err(port
->dev
, "cannot init IRQs\n");
7040 goto err_cleanup_txqs
;
7043 if (priv
->hw_version
== MVPP22
&& !port
->phy_node
&& port
->link_irq
) {
7044 err
= request_irq(port
->link_irq
, mvpp2_link_status_isr
, 0,
7047 netdev_err(port
->dev
, "cannot request link IRQ %d\n",
7052 mvpp22_gop_setup_irq(port
);
7055 /* In default link is down */
7056 netif_carrier_off(port
->dev
);
7058 err
= mvpp2_phy_connect(port
);
7060 goto err_free_link_irq
;
7062 /* Unmask interrupts on all CPUs */
7063 on_each_cpu(mvpp2_interrupts_unmask
, port
, 1);
7064 mvpp2_shared_interrupt_mask_unmask(port
, false);
7066 mvpp2_start_dev(port
);
7068 if (priv
->hw_version
== MVPP22
)
7069 mvpp22_init_rss(port
);
7071 /* Start hardware statistics gathering */
7072 queue_delayed_work(priv
->stats_queue
, &port
->stats_work
,
7073 MVPP2_MIB_COUNTERS_STATS_DELAY
);
7078 if (priv
->hw_version
== MVPP22
&& !port
->phy_node
&& port
->link_irq
)
7079 free_irq(port
->link_irq
, port
);
7081 mvpp2_irqs_deinit(port
);
7083 mvpp2_cleanup_txqs(port
);
7085 mvpp2_cleanup_rxqs(port
);
7089 static int mvpp2_stop(struct net_device
*dev
)
7091 struct mvpp2_port
*port
= netdev_priv(dev
);
7092 struct mvpp2_port_pcpu
*port_pcpu
;
7093 struct mvpp2
*priv
= port
->priv
;
7096 mvpp2_stop_dev(port
);
7097 mvpp2_phy_disconnect(port
);
7099 /* Mask interrupts on all CPUs */
7100 on_each_cpu(mvpp2_interrupts_mask
, port
, 1);
7101 mvpp2_shared_interrupt_mask_unmask(port
, true);
7103 if (priv
->hw_version
== MVPP22
&& !port
->phy_node
&& port
->link_irq
)
7104 free_irq(port
->link_irq
, port
);
7106 mvpp2_irqs_deinit(port
);
7107 if (!port
->has_tx_irqs
) {
7108 for_each_present_cpu(cpu
) {
7109 port_pcpu
= per_cpu_ptr(port
->pcpu
, cpu
);
7111 hrtimer_cancel(&port_pcpu
->tx_done_timer
);
7112 port_pcpu
->timer_scheduled
= false;
7113 tasklet_kill(&port_pcpu
->tx_done_tasklet
);
7116 mvpp2_cleanup_rxqs(port
);
7117 mvpp2_cleanup_txqs(port
);
7119 cancel_delayed_work_sync(&port
->stats_work
);
7124 static void mvpp2_set_rx_mode(struct net_device
*dev
)
7126 struct mvpp2_port
*port
= netdev_priv(dev
);
7127 struct mvpp2
*priv
= port
->priv
;
7128 struct netdev_hw_addr
*ha
;
7130 bool allmulti
= dev
->flags
& IFF_ALLMULTI
;
7133 mvpp2_prs_mac_promisc_set(priv
, id
, dev
->flags
& IFF_PROMISC
);
7134 mvpp2_prs_mac_multi_set(priv
, id
, MVPP2_PE_MAC_MC_ALL
, allmulti
);
7135 mvpp2_prs_mac_multi_set(priv
, id
, MVPP2_PE_MAC_MC_IP6
, allmulti
);
7137 /* Remove all port->id's mcast enries */
7138 mvpp2_prs_mcast_del_all(priv
, id
);
7141 netdev_for_each_mc_addr(ha
, dev
) {
7142 if (mvpp2_prs_mac_da_accept(priv
, id
, ha
->addr
, true)) {
7150 static int mvpp2_set_mac_address(struct net_device
*dev
, void *p
)
7152 struct mvpp2_port
*port
= netdev_priv(dev
);
7153 const struct sockaddr
*addr
= p
;
7156 if (!is_valid_ether_addr(addr
->sa_data
)) {
7157 err
= -EADDRNOTAVAIL
;
7161 if (!netif_running(dev
)) {
7162 err
= mvpp2_prs_update_mac_da(dev
, addr
->sa_data
);
7165 /* Reconfigure parser to accept the original MAC address */
7166 err
= mvpp2_prs_update_mac_da(dev
, dev
->dev_addr
);
7171 mvpp2_stop_dev(port
);
7173 err
= mvpp2_prs_update_mac_da(dev
, addr
->sa_data
);
7177 /* Reconfigure parser accept the original MAC address */
7178 err
= mvpp2_prs_update_mac_da(dev
, dev
->dev_addr
);
7182 mvpp2_start_dev(port
);
7183 mvpp2_egress_enable(port
);
7184 mvpp2_ingress_enable(port
);
7187 netdev_err(dev
, "failed to change MAC address\n");
7191 static int mvpp2_change_mtu(struct net_device
*dev
, int mtu
)
7193 struct mvpp2_port
*port
= netdev_priv(dev
);
7196 if (!IS_ALIGNED(MVPP2_RX_PKT_SIZE(mtu
), 8)) {
7197 netdev_info(dev
, "illegal MTU value %d, round to %d\n", mtu
,
7198 ALIGN(MVPP2_RX_PKT_SIZE(mtu
), 8));
7199 mtu
= ALIGN(MVPP2_RX_PKT_SIZE(mtu
), 8);
7202 if (!netif_running(dev
)) {
7203 err
= mvpp2_bm_update_mtu(dev
, mtu
);
7205 port
->pkt_size
= MVPP2_RX_PKT_SIZE(mtu
);
7209 /* Reconfigure BM to the original MTU */
7210 err
= mvpp2_bm_update_mtu(dev
, dev
->mtu
);
7215 mvpp2_stop_dev(port
);
7217 err
= mvpp2_bm_update_mtu(dev
, mtu
);
7219 port
->pkt_size
= MVPP2_RX_PKT_SIZE(mtu
);
7223 /* Reconfigure BM to the original MTU */
7224 err
= mvpp2_bm_update_mtu(dev
, dev
->mtu
);
7229 mvpp2_start_dev(port
);
7230 mvpp2_egress_enable(port
);
7231 mvpp2_ingress_enable(port
);
7235 netdev_err(dev
, "failed to change MTU\n");
7240 mvpp2_get_stats64(struct net_device
*dev
, struct rtnl_link_stats64
*stats
)
7242 struct mvpp2_port
*port
= netdev_priv(dev
);
7246 for_each_possible_cpu(cpu
) {
7247 struct mvpp2_pcpu_stats
*cpu_stats
;
7253 cpu_stats
= per_cpu_ptr(port
->stats
, cpu
);
7255 start
= u64_stats_fetch_begin_irq(&cpu_stats
->syncp
);
7256 rx_packets
= cpu_stats
->rx_packets
;
7257 rx_bytes
= cpu_stats
->rx_bytes
;
7258 tx_packets
= cpu_stats
->tx_packets
;
7259 tx_bytes
= cpu_stats
->tx_bytes
;
7260 } while (u64_stats_fetch_retry_irq(&cpu_stats
->syncp
, start
));
7262 stats
->rx_packets
+= rx_packets
;
7263 stats
->rx_bytes
+= rx_bytes
;
7264 stats
->tx_packets
+= tx_packets
;
7265 stats
->tx_bytes
+= tx_bytes
;
7268 stats
->rx_errors
= dev
->stats
.rx_errors
;
7269 stats
->rx_dropped
= dev
->stats
.rx_dropped
;
7270 stats
->tx_dropped
= dev
->stats
.tx_dropped
;
7273 static int mvpp2_ioctl(struct net_device
*dev
, struct ifreq
*ifr
, int cmd
)
7280 ret
= phy_mii_ioctl(dev
->phydev
, ifr
, cmd
);
7282 mvpp2_link_event(dev
);
7287 /* Ethtool methods */
7289 /* Set interrupt coalescing for ethtools */
7290 static int mvpp2_ethtool_set_coalesce(struct net_device
*dev
,
7291 struct ethtool_coalesce
*c
)
7293 struct mvpp2_port
*port
= netdev_priv(dev
);
7296 for (queue
= 0; queue
< port
->nrxqs
; queue
++) {
7297 struct mvpp2_rx_queue
*rxq
= port
->rxqs
[queue
];
7299 rxq
->time_coal
= c
->rx_coalesce_usecs
;
7300 rxq
->pkts_coal
= c
->rx_max_coalesced_frames
;
7301 mvpp2_rx_pkts_coal_set(port
, rxq
);
7302 mvpp2_rx_time_coal_set(port
, rxq
);
7305 if (port
->has_tx_irqs
) {
7306 port
->tx_time_coal
= c
->tx_coalesce_usecs
;
7307 mvpp2_tx_time_coal_set(port
);
7310 for (queue
= 0; queue
< port
->ntxqs
; queue
++) {
7311 struct mvpp2_tx_queue
*txq
= port
->txqs
[queue
];
7313 txq
->done_pkts_coal
= c
->tx_max_coalesced_frames
;
7315 if (port
->has_tx_irqs
)
7316 mvpp2_tx_pkts_coal_set(port
, txq
);
7322 /* get coalescing for ethtools */
7323 static int mvpp2_ethtool_get_coalesce(struct net_device
*dev
,
7324 struct ethtool_coalesce
*c
)
7326 struct mvpp2_port
*port
= netdev_priv(dev
);
7328 c
->rx_coalesce_usecs
= port
->rxqs
[0]->time_coal
;
7329 c
->rx_max_coalesced_frames
= port
->rxqs
[0]->pkts_coal
;
7330 c
->tx_max_coalesced_frames
= port
->txqs
[0]->done_pkts_coal
;
7334 static void mvpp2_ethtool_get_drvinfo(struct net_device
*dev
,
7335 struct ethtool_drvinfo
*drvinfo
)
7337 strlcpy(drvinfo
->driver
, MVPP2_DRIVER_NAME
,
7338 sizeof(drvinfo
->driver
));
7339 strlcpy(drvinfo
->version
, MVPP2_DRIVER_VERSION
,
7340 sizeof(drvinfo
->version
));
7341 strlcpy(drvinfo
->bus_info
, dev_name(&dev
->dev
),
7342 sizeof(drvinfo
->bus_info
));
7345 static void mvpp2_ethtool_get_ringparam(struct net_device
*dev
,
7346 struct ethtool_ringparam
*ring
)
7348 struct mvpp2_port
*port
= netdev_priv(dev
);
7350 ring
->rx_max_pending
= MVPP2_MAX_RXD
;
7351 ring
->tx_max_pending
= MVPP2_MAX_TXD
;
7352 ring
->rx_pending
= port
->rx_ring_size
;
7353 ring
->tx_pending
= port
->tx_ring_size
;
7356 static int mvpp2_ethtool_set_ringparam(struct net_device
*dev
,
7357 struct ethtool_ringparam
*ring
)
7359 struct mvpp2_port
*port
= netdev_priv(dev
);
7360 u16 prev_rx_ring_size
= port
->rx_ring_size
;
7361 u16 prev_tx_ring_size
= port
->tx_ring_size
;
7364 err
= mvpp2_check_ringparam_valid(dev
, ring
);
7368 if (!netif_running(dev
)) {
7369 port
->rx_ring_size
= ring
->rx_pending
;
7370 port
->tx_ring_size
= ring
->tx_pending
;
7374 /* The interface is running, so we have to force a
7375 * reallocation of the queues
7377 mvpp2_stop_dev(port
);
7378 mvpp2_cleanup_rxqs(port
);
7379 mvpp2_cleanup_txqs(port
);
7381 port
->rx_ring_size
= ring
->rx_pending
;
7382 port
->tx_ring_size
= ring
->tx_pending
;
7384 err
= mvpp2_setup_rxqs(port
);
7386 /* Reallocate Rx queues with the original ring size */
7387 port
->rx_ring_size
= prev_rx_ring_size
;
7388 ring
->rx_pending
= prev_rx_ring_size
;
7389 err
= mvpp2_setup_rxqs(port
);
7393 err
= mvpp2_setup_txqs(port
);
7395 /* Reallocate Tx queues with the original ring size */
7396 port
->tx_ring_size
= prev_tx_ring_size
;
7397 ring
->tx_pending
= prev_tx_ring_size
;
7398 err
= mvpp2_setup_txqs(port
);
7400 goto err_clean_rxqs
;
7403 mvpp2_start_dev(port
);
7404 mvpp2_egress_enable(port
);
7405 mvpp2_ingress_enable(port
);
7410 mvpp2_cleanup_rxqs(port
);
7412 netdev_err(dev
, "failed to change ring parameters");
7418 static const struct net_device_ops mvpp2_netdev_ops
= {
7419 .ndo_open
= mvpp2_open
,
7420 .ndo_stop
= mvpp2_stop
,
7421 .ndo_start_xmit
= mvpp2_tx
,
7422 .ndo_set_rx_mode
= mvpp2_set_rx_mode
,
7423 .ndo_set_mac_address
= mvpp2_set_mac_address
,
7424 .ndo_change_mtu
= mvpp2_change_mtu
,
7425 .ndo_get_stats64
= mvpp2_get_stats64
,
7426 .ndo_do_ioctl
= mvpp2_ioctl
,
7429 static const struct ethtool_ops mvpp2_eth_tool_ops
= {
7430 .nway_reset
= phy_ethtool_nway_reset
,
7431 .get_link
= ethtool_op_get_link
,
7432 .set_coalesce
= mvpp2_ethtool_set_coalesce
,
7433 .get_coalesce
= mvpp2_ethtool_get_coalesce
,
7434 .get_drvinfo
= mvpp2_ethtool_get_drvinfo
,
7435 .get_ringparam
= mvpp2_ethtool_get_ringparam
,
7436 .set_ringparam
= mvpp2_ethtool_set_ringparam
,
7437 .get_strings
= mvpp2_ethtool_get_strings
,
7438 .get_ethtool_stats
= mvpp2_ethtool_get_stats
,
7439 .get_sset_count
= mvpp2_ethtool_get_sset_count
,
7440 .get_link_ksettings
= phy_ethtool_get_link_ksettings
,
7441 .set_link_ksettings
= phy_ethtool_set_link_ksettings
,
7444 /* Used for PPv2.1, or PPv2.2 with the old Device Tree binding that
7445 * had a single IRQ defined per-port.
7447 static int mvpp2_simple_queue_vectors_init(struct mvpp2_port
*port
,
7448 struct device_node
*port_node
)
7450 struct mvpp2_queue_vector
*v
= &port
->qvecs
[0];
7453 v
->nrxqs
= port
->nrxqs
;
7454 v
->type
= MVPP2_QUEUE_VECTOR_SHARED
;
7455 v
->sw_thread_id
= 0;
7456 v
->sw_thread_mask
= *cpumask_bits(cpu_online_mask
);
7458 v
->irq
= irq_of_parse_and_map(port_node
, 0);
7461 netif_napi_add(port
->dev
, &v
->napi
, mvpp2_poll
,
7469 static int mvpp2_multi_queue_vectors_init(struct mvpp2_port
*port
,
7470 struct device_node
*port_node
)
7472 struct mvpp2_queue_vector
*v
;
7475 port
->nqvecs
= num_possible_cpus();
7476 if (queue_mode
== MVPP2_QDIST_SINGLE_MODE
)
7479 for (i
= 0; i
< port
->nqvecs
; i
++) {
7482 v
= port
->qvecs
+ i
;
7485 v
->type
= MVPP2_QUEUE_VECTOR_PRIVATE
;
7486 v
->sw_thread_id
= i
;
7487 v
->sw_thread_mask
= BIT(i
);
7489 snprintf(irqname
, sizeof(irqname
), "tx-cpu%d", i
);
7491 if (queue_mode
== MVPP2_QDIST_MULTI_MODE
) {
7492 v
->first_rxq
= i
* MVPP2_DEFAULT_RXQ
;
7493 v
->nrxqs
= MVPP2_DEFAULT_RXQ
;
7494 } else if (queue_mode
== MVPP2_QDIST_SINGLE_MODE
&&
7495 i
== (port
->nqvecs
- 1)) {
7497 v
->nrxqs
= port
->nrxqs
;
7498 v
->type
= MVPP2_QUEUE_VECTOR_SHARED
;
7499 strncpy(irqname
, "rx-shared", sizeof(irqname
));
7502 v
->irq
= of_irq_get_byname(port_node
, irqname
);
7508 netif_napi_add(port
->dev
, &v
->napi
, mvpp2_poll
,
7515 for (i
= 0; i
< port
->nqvecs
; i
++)
7516 irq_dispose_mapping(port
->qvecs
[i
].irq
);
7520 static int mvpp2_queue_vectors_init(struct mvpp2_port
*port
,
7521 struct device_node
*port_node
)
7523 if (port
->has_tx_irqs
)
7524 return mvpp2_multi_queue_vectors_init(port
, port_node
);
7526 return mvpp2_simple_queue_vectors_init(port
, port_node
);
7529 static void mvpp2_queue_vectors_deinit(struct mvpp2_port
*port
)
7533 for (i
= 0; i
< port
->nqvecs
; i
++)
7534 irq_dispose_mapping(port
->qvecs
[i
].irq
);
7537 /* Configure Rx queue group interrupt for this port */
7538 static void mvpp2_rx_irqs_setup(struct mvpp2_port
*port
)
7540 struct mvpp2
*priv
= port
->priv
;
7544 if (priv
->hw_version
== MVPP21
) {
7545 mvpp2_write(priv
, MVPP21_ISR_RXQ_GROUP_REG(port
->id
),
7550 /* Handle the more complicated PPv2.2 case */
7551 for (i
= 0; i
< port
->nqvecs
; i
++) {
7552 struct mvpp2_queue_vector
*qv
= port
->qvecs
+ i
;
7557 val
= qv
->sw_thread_id
;
7558 val
|= port
->id
<< MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_OFFSET
;
7559 mvpp2_write(priv
, MVPP22_ISR_RXQ_GROUP_INDEX_REG
, val
);
7561 val
= qv
->first_rxq
;
7562 val
|= qv
->nrxqs
<< MVPP22_ISR_RXQ_SUB_GROUP_SIZE_OFFSET
;
7563 mvpp2_write(priv
, MVPP22_ISR_RXQ_SUB_GROUP_CONFIG_REG
, val
);
7567 /* Initialize port HW */
7568 static int mvpp2_port_init(struct mvpp2_port
*port
)
7570 struct device
*dev
= port
->dev
->dev
.parent
;
7571 struct mvpp2
*priv
= port
->priv
;
7572 struct mvpp2_txq_pcpu
*txq_pcpu
;
7573 int queue
, cpu
, err
;
7575 /* Checks for hardware constraints */
7576 if (port
->first_rxq
+ port
->nrxqs
>
7577 MVPP2_MAX_PORTS
* priv
->max_port_rxqs
)
7580 if (port
->nrxqs
% 4 || (port
->nrxqs
> priv
->max_port_rxqs
) ||
7581 (port
->ntxqs
> MVPP2_MAX_TXQ
))
7585 mvpp2_egress_disable(port
);
7586 mvpp2_port_disable(port
);
7588 port
->tx_time_coal
= MVPP2_TXDONE_COAL_USEC
;
7590 port
->txqs
= devm_kcalloc(dev
, port
->ntxqs
, sizeof(*port
->txqs
),
7595 /* Associate physical Tx queues to this port and initialize.
7596 * The mapping is predefined.
7598 for (queue
= 0; queue
< port
->ntxqs
; queue
++) {
7599 int queue_phy_id
= mvpp2_txq_phys(port
->id
, queue
);
7600 struct mvpp2_tx_queue
*txq
;
7602 txq
= devm_kzalloc(dev
, sizeof(*txq
), GFP_KERNEL
);
7605 goto err_free_percpu
;
7608 txq
->pcpu
= alloc_percpu(struct mvpp2_txq_pcpu
);
7611 goto err_free_percpu
;
7614 txq
->id
= queue_phy_id
;
7615 txq
->log_id
= queue
;
7616 txq
->done_pkts_coal
= MVPP2_TXDONE_COAL_PKTS_THRESH
;
7617 for_each_present_cpu(cpu
) {
7618 txq_pcpu
= per_cpu_ptr(txq
->pcpu
, cpu
);
7619 txq_pcpu
->cpu
= cpu
;
7622 port
->txqs
[queue
] = txq
;
7625 port
->rxqs
= devm_kcalloc(dev
, port
->nrxqs
, sizeof(*port
->rxqs
),
7629 goto err_free_percpu
;
7632 /* Allocate and initialize Rx queue for this port */
7633 for (queue
= 0; queue
< port
->nrxqs
; queue
++) {
7634 struct mvpp2_rx_queue
*rxq
;
7636 /* Map physical Rx queue to port's logical Rx queue */
7637 rxq
= devm_kzalloc(dev
, sizeof(*rxq
), GFP_KERNEL
);
7640 goto err_free_percpu
;
7642 /* Map this Rx queue to a physical queue */
7643 rxq
->id
= port
->first_rxq
+ queue
;
7644 rxq
->port
= port
->id
;
7645 rxq
->logic_rxq
= queue
;
7647 port
->rxqs
[queue
] = rxq
;
7650 mvpp2_rx_irqs_setup(port
);
7652 /* Create Rx descriptor rings */
7653 for (queue
= 0; queue
< port
->nrxqs
; queue
++) {
7654 struct mvpp2_rx_queue
*rxq
= port
->rxqs
[queue
];
7656 rxq
->size
= port
->rx_ring_size
;
7657 rxq
->pkts_coal
= MVPP2_RX_COAL_PKTS
;
7658 rxq
->time_coal
= MVPP2_RX_COAL_USEC
;
7661 mvpp2_ingress_disable(port
);
7663 /* Port default configuration */
7664 mvpp2_defaults_set(port
);
7666 /* Port's classifier configuration */
7667 mvpp2_cls_oversize_rxq_set(port
);
7668 mvpp2_cls_port_config(port
);
7670 /* Provide an initial Rx packet size */
7671 port
->pkt_size
= MVPP2_RX_PKT_SIZE(port
->dev
->mtu
);
7673 /* Initialize pools for swf */
7674 err
= mvpp2_swf_bm_pool_init(port
);
7676 goto err_free_percpu
;
7681 for (queue
= 0; queue
< port
->ntxqs
; queue
++) {
7682 if (!port
->txqs
[queue
])
7684 free_percpu(port
->txqs
[queue
]->pcpu
);
7689 /* Checks if the port DT description has the TX interrupts
7690 * described. On PPv2.1, there are no such interrupts. On PPv2.2,
7691 * there are available, but we need to keep support for old DTs.
7693 static bool mvpp2_port_has_tx_irqs(struct mvpp2
*priv
,
7694 struct device_node
*port_node
)
7696 char *irqs
[5] = { "rx-shared", "tx-cpu0", "tx-cpu1",
7697 "tx-cpu2", "tx-cpu3" };
7700 if (priv
->hw_version
== MVPP21
)
7703 for (i
= 0; i
< 5; i
++) {
7704 ret
= of_property_match_string(port_node
, "interrupt-names",
7713 static void mvpp2_port_copy_mac_addr(struct net_device
*dev
, struct mvpp2
*priv
,
7714 struct device_node
*port_node
,
7717 struct mvpp2_port
*port
= netdev_priv(dev
);
7718 char hw_mac_addr
[ETH_ALEN
] = {0};
7719 const char *dt_mac_addr
;
7721 dt_mac_addr
= of_get_mac_address(port_node
);
7722 if (dt_mac_addr
&& is_valid_ether_addr(dt_mac_addr
)) {
7723 *mac_from
= "device tree";
7724 ether_addr_copy(dev
->dev_addr
, dt_mac_addr
);
7728 if (priv
->hw_version
== MVPP21
) {
7729 mvpp21_get_mac_address(port
, hw_mac_addr
);
7730 if (is_valid_ether_addr(hw_mac_addr
)) {
7731 *mac_from
= "hardware";
7732 ether_addr_copy(dev
->dev_addr
, hw_mac_addr
);
7737 *mac_from
= "random";
7738 eth_hw_addr_random(dev
);
7741 /* Ports initialization */
7742 static int mvpp2_port_probe(struct platform_device
*pdev
,
7743 struct device_node
*port_node
,
7744 struct mvpp2
*priv
, int index
)
7746 struct device_node
*phy_node
;
7748 struct mvpp2_port
*port
;
7749 struct mvpp2_port_pcpu
*port_pcpu
;
7750 struct net_device
*dev
;
7751 struct resource
*res
;
7752 char *mac_from
= "";
7753 unsigned int ntxqs
, nrxqs
;
7760 has_tx_irqs
= mvpp2_port_has_tx_irqs(priv
, port_node
);
7763 queue_mode
= MVPP2_QDIST_SINGLE_MODE
;
7765 ntxqs
= MVPP2_MAX_TXQ
;
7766 if (priv
->hw_version
== MVPP22
&& queue_mode
== MVPP2_QDIST_MULTI_MODE
)
7767 nrxqs
= MVPP2_DEFAULT_RXQ
* num_possible_cpus();
7769 nrxqs
= MVPP2_DEFAULT_RXQ
;
7771 dev
= alloc_etherdev_mqs(sizeof(*port
), ntxqs
, nrxqs
);
7775 phy_node
= of_parse_phandle(port_node
, "phy", 0);
7776 phy_mode
= of_get_phy_mode(port_node
);
7778 dev_err(&pdev
->dev
, "incorrect phy mode\n");
7780 goto err_free_netdev
;
7783 comphy
= devm_of_phy_get(&pdev
->dev
, port_node
, NULL
);
7784 if (IS_ERR(comphy
)) {
7785 if (PTR_ERR(comphy
) == -EPROBE_DEFER
) {
7786 err
= -EPROBE_DEFER
;
7787 goto err_free_netdev
;
7792 if (of_property_read_u32(port_node
, "port-id", &id
)) {
7794 dev_err(&pdev
->dev
, "missing port-id value\n");
7795 goto err_free_netdev
;
7798 dev
->tx_queue_len
= MVPP2_MAX_TXD
;
7799 dev
->watchdog_timeo
= 5 * HZ
;
7800 dev
->netdev_ops
= &mvpp2_netdev_ops
;
7801 dev
->ethtool_ops
= &mvpp2_eth_tool_ops
;
7803 port
= netdev_priv(dev
);
7805 port
->ntxqs
= ntxqs
;
7806 port
->nrxqs
= nrxqs
;
7808 port
->has_tx_irqs
= has_tx_irqs
;
7810 err
= mvpp2_queue_vectors_init(port
, port_node
);
7812 goto err_free_netdev
;
7814 port
->link_irq
= of_irq_get_byname(port_node
, "link");
7815 if (port
->link_irq
== -EPROBE_DEFER
) {
7816 err
= -EPROBE_DEFER
;
7817 goto err_deinit_qvecs
;
7819 if (port
->link_irq
<= 0)
7820 /* the link irq is optional */
7823 if (of_property_read_bool(port_node
, "marvell,loopback"))
7824 port
->flags
|= MVPP2_F_LOOPBACK
;
7827 if (priv
->hw_version
== MVPP21
)
7828 port
->first_rxq
= port
->id
* port
->nrxqs
;
7830 port
->first_rxq
= port
->id
* priv
->max_port_rxqs
;
7832 port
->phy_node
= phy_node
;
7833 port
->phy_interface
= phy_mode
;
7834 port
->comphy
= comphy
;
7836 if (priv
->hw_version
== MVPP21
) {
7837 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 2 + id
);
7838 port
->base
= devm_ioremap_resource(&pdev
->dev
, res
);
7839 if (IS_ERR(port
->base
)) {
7840 err
= PTR_ERR(port
->base
);
7844 port
->stats_base
= port
->priv
->lms_base
+
7845 MVPP21_MIB_COUNTERS_OFFSET
+
7846 port
->gop_id
* MVPP21_MIB_COUNTERS_PORT_SZ
;
7848 if (of_property_read_u32(port_node
, "gop-port-id",
7851 dev_err(&pdev
->dev
, "missing gop-port-id value\n");
7852 goto err_deinit_qvecs
;
7855 port
->base
= priv
->iface_base
+ MVPP22_GMAC_BASE(port
->gop_id
);
7856 port
->stats_base
= port
->priv
->iface_base
+
7857 MVPP22_MIB_COUNTERS_OFFSET
+
7858 port
->gop_id
* MVPP22_MIB_COUNTERS_PORT_SZ
;
7861 /* Alloc per-cpu and ethtool stats */
7862 port
->stats
= netdev_alloc_pcpu_stats(struct mvpp2_pcpu_stats
);
7868 port
->ethtool_stats
= devm_kcalloc(&pdev
->dev
,
7869 ARRAY_SIZE(mvpp2_ethtool_regs
),
7870 sizeof(u64
), GFP_KERNEL
);
7871 if (!port
->ethtool_stats
) {
7873 goto err_free_stats
;
7876 mutex_init(&port
->gather_stats_lock
);
7877 INIT_DELAYED_WORK(&port
->stats_work
, mvpp2_gather_hw_statistics
);
7879 mvpp2_port_copy_mac_addr(dev
, priv
, port_node
, &mac_from
);
7881 port
->tx_ring_size
= MVPP2_MAX_TXD
;
7882 port
->rx_ring_size
= MVPP2_MAX_RXD
;
7883 SET_NETDEV_DEV(dev
, &pdev
->dev
);
7885 err
= mvpp2_port_init(port
);
7887 dev_err(&pdev
->dev
, "failed to init port %d\n", id
);
7888 goto err_free_stats
;
7891 mvpp2_port_periodic_xon_disable(port
);
7893 if (priv
->hw_version
== MVPP21
)
7894 mvpp2_port_fc_adv_enable(port
);
7896 mvpp2_port_reset(port
);
7898 port
->pcpu
= alloc_percpu(struct mvpp2_port_pcpu
);
7901 goto err_free_txq_pcpu
;
7904 if (!port
->has_tx_irqs
) {
7905 for_each_present_cpu(cpu
) {
7906 port_pcpu
= per_cpu_ptr(port
->pcpu
, cpu
);
7908 hrtimer_init(&port_pcpu
->tx_done_timer
, CLOCK_MONOTONIC
,
7909 HRTIMER_MODE_REL_PINNED
);
7910 port_pcpu
->tx_done_timer
.function
= mvpp2_hr_timer_cb
;
7911 port_pcpu
->timer_scheduled
= false;
7913 tasklet_init(&port_pcpu
->tx_done_tasklet
,
7915 (unsigned long)dev
);
7919 features
= NETIF_F_SG
| NETIF_F_IP_CSUM
| NETIF_F_TSO
;
7920 dev
->features
= features
| NETIF_F_RXCSUM
;
7921 dev
->hw_features
|= features
| NETIF_F_RXCSUM
| NETIF_F_GRO
;
7922 dev
->vlan_features
|= features
;
7923 dev
->gso_max_segs
= MVPP2_MAX_TSO_SEGS
;
7925 /* MTU range: 68 - 9676 */
7926 dev
->min_mtu
= ETH_MIN_MTU
;
7927 /* 9676 == 9700 - 20 and rounding to 8 */
7928 dev
->max_mtu
= 9676;
7930 err
= register_netdev(dev
);
7932 dev_err(&pdev
->dev
, "failed to register netdev\n");
7933 goto err_free_port_pcpu
;
7935 netdev_info(dev
, "Using %s mac address %pM\n", mac_from
, dev
->dev_addr
);
7937 priv
->port_list
[index
] = port
;
7941 free_percpu(port
->pcpu
);
7943 for (i
= 0; i
< port
->ntxqs
; i
++)
7944 free_percpu(port
->txqs
[i
]->pcpu
);
7946 free_percpu(port
->stats
);
7949 irq_dispose_mapping(port
->link_irq
);
7951 mvpp2_queue_vectors_deinit(port
);
7953 of_node_put(phy_node
);
7958 /* Ports removal routine */
7959 static void mvpp2_port_remove(struct mvpp2_port
*port
)
7963 unregister_netdev(port
->dev
);
7964 of_node_put(port
->phy_node
);
7965 free_percpu(port
->pcpu
);
7966 free_percpu(port
->stats
);
7967 for (i
= 0; i
< port
->ntxqs
; i
++)
7968 free_percpu(port
->txqs
[i
]->pcpu
);
7969 mvpp2_queue_vectors_deinit(port
);
7971 irq_dispose_mapping(port
->link_irq
);
7972 free_netdev(port
->dev
);
7975 /* Initialize decoding windows */
7976 static void mvpp2_conf_mbus_windows(const struct mbus_dram_target_info
*dram
,
7982 for (i
= 0; i
< 6; i
++) {
7983 mvpp2_write(priv
, MVPP2_WIN_BASE(i
), 0);
7984 mvpp2_write(priv
, MVPP2_WIN_SIZE(i
), 0);
7987 mvpp2_write(priv
, MVPP2_WIN_REMAP(i
), 0);
7992 for (i
= 0; i
< dram
->num_cs
; i
++) {
7993 const struct mbus_dram_window
*cs
= dram
->cs
+ i
;
7995 mvpp2_write(priv
, MVPP2_WIN_BASE(i
),
7996 (cs
->base
& 0xffff0000) | (cs
->mbus_attr
<< 8) |
7997 dram
->mbus_dram_target_id
);
7999 mvpp2_write(priv
, MVPP2_WIN_SIZE(i
),
8000 (cs
->size
- 1) & 0xffff0000);
8002 win_enable
|= (1 << i
);
8005 mvpp2_write(priv
, MVPP2_BASE_ADDR_ENABLE
, win_enable
);
8008 /* Initialize Rx FIFO's */
8009 static void mvpp2_rx_fifo_init(struct mvpp2
*priv
)
8013 for (port
= 0; port
< MVPP2_MAX_PORTS
; port
++) {
8014 mvpp2_write(priv
, MVPP2_RX_DATA_FIFO_SIZE_REG(port
),
8015 MVPP2_RX_FIFO_PORT_DATA_SIZE_4KB
);
8016 mvpp2_write(priv
, MVPP2_RX_ATTR_FIFO_SIZE_REG(port
),
8017 MVPP2_RX_FIFO_PORT_ATTR_SIZE_4KB
);
8020 mvpp2_write(priv
, MVPP2_RX_MIN_PKT_SIZE_REG
,
8021 MVPP2_RX_FIFO_PORT_MIN_PKT
);
8022 mvpp2_write(priv
, MVPP2_RX_FIFO_INIT_REG
, 0x1);
8025 static void mvpp22_rx_fifo_init(struct mvpp2
*priv
)
8029 /* The FIFO size parameters are set depending on the maximum speed a
8030 * given port can handle:
8033 * - Ports 2 and 3: 1Gbps
8036 mvpp2_write(priv
, MVPP2_RX_DATA_FIFO_SIZE_REG(0),
8037 MVPP2_RX_FIFO_PORT_DATA_SIZE_32KB
);
8038 mvpp2_write(priv
, MVPP2_RX_ATTR_FIFO_SIZE_REG(0),
8039 MVPP2_RX_FIFO_PORT_ATTR_SIZE_32KB
);
8041 mvpp2_write(priv
, MVPP2_RX_DATA_FIFO_SIZE_REG(1),
8042 MVPP2_RX_FIFO_PORT_DATA_SIZE_8KB
);
8043 mvpp2_write(priv
, MVPP2_RX_ATTR_FIFO_SIZE_REG(1),
8044 MVPP2_RX_FIFO_PORT_ATTR_SIZE_8KB
);
8046 for (port
= 2; port
< MVPP2_MAX_PORTS
; port
++) {
8047 mvpp2_write(priv
, MVPP2_RX_DATA_FIFO_SIZE_REG(port
),
8048 MVPP2_RX_FIFO_PORT_DATA_SIZE_4KB
);
8049 mvpp2_write(priv
, MVPP2_RX_ATTR_FIFO_SIZE_REG(port
),
8050 MVPP2_RX_FIFO_PORT_ATTR_SIZE_4KB
);
8053 mvpp2_write(priv
, MVPP2_RX_MIN_PKT_SIZE_REG
,
8054 MVPP2_RX_FIFO_PORT_MIN_PKT
);
8055 mvpp2_write(priv
, MVPP2_RX_FIFO_INIT_REG
, 0x1);
8058 /* Initialize Tx FIFO's */
8059 static void mvpp22_tx_fifo_init(struct mvpp2
*priv
)
8063 for (port
= 0; port
< MVPP2_MAX_PORTS
; port
++)
8064 mvpp2_write(priv
, MVPP22_TX_FIFO_SIZE_REG(port
),
8065 MVPP22_TX_FIFO_DATA_SIZE_3KB
);
8068 static void mvpp2_axi_init(struct mvpp2
*priv
)
8070 u32 val
, rdval
, wrval
;
8072 mvpp2_write(priv
, MVPP22_BM_ADDR_HIGH_RLS_REG
, 0x0);
8074 /* AXI Bridge Configuration */
8076 rdval
= MVPP22_AXI_CODE_CACHE_RD_CACHE
8077 << MVPP22_AXI_ATTR_CACHE_OFFS
;
8078 rdval
|= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
8079 << MVPP22_AXI_ATTR_DOMAIN_OFFS
;
8081 wrval
= MVPP22_AXI_CODE_CACHE_WR_CACHE
8082 << MVPP22_AXI_ATTR_CACHE_OFFS
;
8083 wrval
|= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
8084 << MVPP22_AXI_ATTR_DOMAIN_OFFS
;
8087 mvpp2_write(priv
, MVPP22_AXI_BM_WR_ATTR_REG
, wrval
);
8088 mvpp2_write(priv
, MVPP22_AXI_BM_RD_ATTR_REG
, rdval
);
8091 mvpp2_write(priv
, MVPP22_AXI_AGGRQ_DESCR_RD_ATTR_REG
, rdval
);
8092 mvpp2_write(priv
, MVPP22_AXI_TXQ_DESCR_WR_ATTR_REG
, wrval
);
8093 mvpp2_write(priv
, MVPP22_AXI_TXQ_DESCR_RD_ATTR_REG
, rdval
);
8094 mvpp2_write(priv
, MVPP22_AXI_RXQ_DESCR_WR_ATTR_REG
, wrval
);
8097 mvpp2_write(priv
, MVPP22_AXI_TX_DATA_RD_ATTR_REG
, rdval
);
8098 mvpp2_write(priv
, MVPP22_AXI_RX_DATA_WR_ATTR_REG
, wrval
);
8100 val
= MVPP22_AXI_CODE_CACHE_NON_CACHE
8101 << MVPP22_AXI_CODE_CACHE_OFFS
;
8102 val
|= MVPP22_AXI_CODE_DOMAIN_SYSTEM
8103 << MVPP22_AXI_CODE_DOMAIN_OFFS
;
8104 mvpp2_write(priv
, MVPP22_AXI_RD_NORMAL_CODE_REG
, val
);
8105 mvpp2_write(priv
, MVPP22_AXI_WR_NORMAL_CODE_REG
, val
);
8107 val
= MVPP22_AXI_CODE_CACHE_RD_CACHE
8108 << MVPP22_AXI_CODE_CACHE_OFFS
;
8109 val
|= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
8110 << MVPP22_AXI_CODE_DOMAIN_OFFS
;
8112 mvpp2_write(priv
, MVPP22_AXI_RD_SNOOP_CODE_REG
, val
);
8114 val
= MVPP22_AXI_CODE_CACHE_WR_CACHE
8115 << MVPP22_AXI_CODE_CACHE_OFFS
;
8116 val
|= MVPP22_AXI_CODE_DOMAIN_OUTER_DOM
8117 << MVPP22_AXI_CODE_DOMAIN_OFFS
;
8119 mvpp2_write(priv
, MVPP22_AXI_WR_SNOOP_CODE_REG
, val
);
8122 /* Initialize network controller common part HW */
8123 static int mvpp2_init(struct platform_device
*pdev
, struct mvpp2
*priv
)
8125 const struct mbus_dram_target_info
*dram_target_info
;
8129 /* MBUS windows configuration */
8130 dram_target_info
= mv_mbus_dram_info();
8131 if (dram_target_info
)
8132 mvpp2_conf_mbus_windows(dram_target_info
, priv
);
8134 if (priv
->hw_version
== MVPP22
)
8135 mvpp2_axi_init(priv
);
8137 /* Disable HW PHY polling */
8138 if (priv
->hw_version
== MVPP21
) {
8139 val
= readl(priv
->lms_base
+ MVPP2_PHY_AN_CFG0_REG
);
8140 val
|= MVPP2_PHY_AN_STOP_SMI0_MASK
;
8141 writel(val
, priv
->lms_base
+ MVPP2_PHY_AN_CFG0_REG
);
8143 val
= readl(priv
->iface_base
+ MVPP22_SMI_MISC_CFG_REG
);
8144 val
&= ~MVPP22_SMI_POLLING_EN
;
8145 writel(val
, priv
->iface_base
+ MVPP22_SMI_MISC_CFG_REG
);
8148 /* Allocate and initialize aggregated TXQs */
8149 priv
->aggr_txqs
= devm_kcalloc(&pdev
->dev
, num_present_cpus(),
8150 sizeof(*priv
->aggr_txqs
),
8152 if (!priv
->aggr_txqs
)
8155 for_each_present_cpu(i
) {
8156 priv
->aggr_txqs
[i
].id
= i
;
8157 priv
->aggr_txqs
[i
].size
= MVPP2_AGGR_TXQ_SIZE
;
8158 err
= mvpp2_aggr_txq_init(pdev
, &priv
->aggr_txqs
[i
], i
, priv
);
8164 if (priv
->hw_version
== MVPP21
) {
8165 mvpp2_rx_fifo_init(priv
);
8167 mvpp22_rx_fifo_init(priv
);
8168 mvpp22_tx_fifo_init(priv
);
8171 if (priv
->hw_version
== MVPP21
)
8172 writel(MVPP2_EXT_GLOBAL_CTRL_DEFAULT
,
8173 priv
->lms_base
+ MVPP2_MNG_EXTENDED_GLOBAL_CTRL_REG
);
8175 /* Allow cache snoop when transmiting packets */
8176 mvpp2_write(priv
, MVPP2_TX_SNOOP_REG
, 0x1);
8178 /* Buffer Manager initialization */
8179 err
= mvpp2_bm_init(pdev
, priv
);
8183 /* Parser default initialization */
8184 err
= mvpp2_prs_default_init(pdev
, priv
);
8188 /* Classifier default initialization */
8189 mvpp2_cls_init(priv
);
8194 static int mvpp2_probe(struct platform_device
*pdev
)
8196 struct device_node
*dn
= pdev
->dev
.of_node
;
8197 struct device_node
*port_node
;
8199 struct resource
*res
;
8204 priv
= devm_kzalloc(&pdev
->dev
, sizeof(*priv
), GFP_KERNEL
);
8209 (unsigned long)of_device_get_match_data(&pdev
->dev
);
8211 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
8212 base
= devm_ioremap_resource(&pdev
->dev
, res
);
8214 return PTR_ERR(base
);
8216 if (priv
->hw_version
== MVPP21
) {
8217 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 1);
8218 priv
->lms_base
= devm_ioremap_resource(&pdev
->dev
, res
);
8219 if (IS_ERR(priv
->lms_base
))
8220 return PTR_ERR(priv
->lms_base
);
8222 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 1);
8223 priv
->iface_base
= devm_ioremap_resource(&pdev
->dev
, res
);
8224 if (IS_ERR(priv
->iface_base
))
8225 return PTR_ERR(priv
->iface_base
);
8227 priv
->sysctrl_base
=
8228 syscon_regmap_lookup_by_phandle(pdev
->dev
.of_node
,
8229 "marvell,system-controller");
8230 if (IS_ERR(priv
->sysctrl_base
))
8231 /* The system controller regmap is optional for dt
8232 * compatibility reasons. When not provided, the
8233 * configuration of the GoP relies on the
8234 * firmware/bootloader.
8236 priv
->sysctrl_base
= NULL
;
8239 for (i
= 0; i
< MVPP2_MAX_THREADS
; i
++) {
8242 addr_space_sz
= (priv
->hw_version
== MVPP21
?
8243 MVPP21_ADDR_SPACE_SZ
: MVPP22_ADDR_SPACE_SZ
);
8244 priv
->swth_base
[i
] = base
+ i
* addr_space_sz
;
8247 if (priv
->hw_version
== MVPP21
)
8248 priv
->max_port_rxqs
= 8;
8250 priv
->max_port_rxqs
= 32;
8252 priv
->pp_clk
= devm_clk_get(&pdev
->dev
, "pp_clk");
8253 if (IS_ERR(priv
->pp_clk
))
8254 return PTR_ERR(priv
->pp_clk
);
8255 err
= clk_prepare_enable(priv
->pp_clk
);
8259 priv
->gop_clk
= devm_clk_get(&pdev
->dev
, "gop_clk");
8260 if (IS_ERR(priv
->gop_clk
)) {
8261 err
= PTR_ERR(priv
->gop_clk
);
8264 err
= clk_prepare_enable(priv
->gop_clk
);
8268 if (priv
->hw_version
== MVPP22
) {
8269 priv
->mg_clk
= devm_clk_get(&pdev
->dev
, "mg_clk");
8270 if (IS_ERR(priv
->mg_clk
)) {
8271 err
= PTR_ERR(priv
->mg_clk
);
8275 err
= clk_prepare_enable(priv
->mg_clk
);
8279 priv
->axi_clk
= devm_clk_get(&pdev
->dev
, "axi_clk");
8280 if (IS_ERR(priv
->axi_clk
)) {
8281 err
= PTR_ERR(priv
->axi_clk
);
8282 if (err
== -EPROBE_DEFER
)
8284 priv
->axi_clk
= NULL
;
8286 err
= clk_prepare_enable(priv
->axi_clk
);
8292 /* Get system's tclk rate */
8293 priv
->tclk
= clk_get_rate(priv
->pp_clk
);
8295 if (priv
->hw_version
== MVPP22
) {
8296 err
= dma_set_mask(&pdev
->dev
, MVPP2_DESC_DMA_MASK
);
8299 /* Sadly, the BM pools all share the same register to
8300 * store the high 32 bits of their address. So they
8301 * must all have the same high 32 bits, which forces
8302 * us to restrict coherent memory to DMA_BIT_MASK(32).
8304 err
= dma_set_coherent_mask(&pdev
->dev
, DMA_BIT_MASK(32));
8309 /* Initialize network controller */
8310 err
= mvpp2_init(pdev
, priv
);
8312 dev_err(&pdev
->dev
, "failed to initialize controller\n");
8316 priv
->port_count
= of_get_available_child_count(dn
);
8317 if (priv
->port_count
== 0) {
8318 dev_err(&pdev
->dev
, "no ports enabled\n");
8323 priv
->port_list
= devm_kcalloc(&pdev
->dev
, priv
->port_count
,
8324 sizeof(*priv
->port_list
),
8326 if (!priv
->port_list
) {
8331 /* Initialize ports */
8333 for_each_available_child_of_node(dn
, port_node
) {
8334 err
= mvpp2_port_probe(pdev
, port_node
, priv
, i
);
8336 goto err_port_probe
;
8340 /* Statistics must be gathered regularly because some of them (like
8341 * packets counters) are 32-bit registers and could overflow quite
8342 * quickly. For instance, a 10Gb link used at full bandwidth with the
8343 * smallest packets (64B) will overflow a 32-bit counter in less than
8344 * 30 seconds. Then, use a workqueue to fill 64-bit counters.
8346 snprintf(priv
->queue_name
, sizeof(priv
->queue_name
),
8347 "stats-wq-%s%s", netdev_name(priv
->port_list
[0]->dev
),
8348 priv
->port_count
> 1 ? "+" : "");
8349 priv
->stats_queue
= create_singlethread_workqueue(priv
->queue_name
);
8350 if (!priv
->stats_queue
) {
8352 goto err_port_probe
;
8355 platform_set_drvdata(pdev
, priv
);
8360 for_each_available_child_of_node(dn
, port_node
) {
8361 if (priv
->port_list
[i
])
8362 mvpp2_port_remove(priv
->port_list
[i
]);
8366 clk_disable_unprepare(priv
->axi_clk
);
8367 if (priv
->hw_version
== MVPP22
)
8368 clk_disable_unprepare(priv
->mg_clk
);
8370 clk_disable_unprepare(priv
->gop_clk
);
8372 clk_disable_unprepare(priv
->pp_clk
);
8376 static int mvpp2_remove(struct platform_device
*pdev
)
8378 struct mvpp2
*priv
= platform_get_drvdata(pdev
);
8379 struct device_node
*dn
= pdev
->dev
.of_node
;
8380 struct device_node
*port_node
;
8383 flush_workqueue(priv
->stats_queue
);
8384 destroy_workqueue(priv
->stats_queue
);
8386 for_each_available_child_of_node(dn
, port_node
) {
8387 if (priv
->port_list
[i
]) {
8388 mutex_destroy(&priv
->port_list
[i
]->gather_stats_lock
);
8389 mvpp2_port_remove(priv
->port_list
[i
]);
8394 for (i
= 0; i
< MVPP2_BM_POOLS_NUM
; i
++) {
8395 struct mvpp2_bm_pool
*bm_pool
= &priv
->bm_pools
[i
];
8397 mvpp2_bm_pool_destroy(pdev
, priv
, bm_pool
);
8400 for_each_present_cpu(i
) {
8401 struct mvpp2_tx_queue
*aggr_txq
= &priv
->aggr_txqs
[i
];
8403 dma_free_coherent(&pdev
->dev
,
8404 MVPP2_AGGR_TXQ_SIZE
* MVPP2_DESC_ALIGNED_SIZE
,
8406 aggr_txq
->descs_dma
);
8409 clk_disable_unprepare(priv
->axi_clk
);
8410 clk_disable_unprepare(priv
->mg_clk
);
8411 clk_disable_unprepare(priv
->pp_clk
);
8412 clk_disable_unprepare(priv
->gop_clk
);
8417 static const struct of_device_id mvpp2_match
[] = {
8419 .compatible
= "marvell,armada-375-pp2",
8420 .data
= (void *)MVPP21
,
8423 .compatible
= "marvell,armada-7k-pp22",
8424 .data
= (void *)MVPP22
,
8428 MODULE_DEVICE_TABLE(of
, mvpp2_match
);
8430 static struct platform_driver mvpp2_driver
= {
8431 .probe
= mvpp2_probe
,
8432 .remove
= mvpp2_remove
,
8434 .name
= MVPP2_DRIVER_NAME
,
8435 .of_match_table
= mvpp2_match
,
8439 module_platform_driver(mvpp2_driver
);
8441 MODULE_DESCRIPTION("Marvell PPv2 Ethernet Driver - www.marvell.com");
8442 MODULE_AUTHOR("Marcin Wojtas <mw@semihalf.com>");
8443 MODULE_LICENSE("GPL v2");