]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/net/ethernet/sun/sunvnet_common.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
sunvnet: track port queues correctly
[mirror_ubuntu-artful-kernel.git] / drivers / net / ethernet / sun / sunvnet_common.c
CommitLineData
31762eaa
AY		 1/* sunvnet.c: Sun LDOM Virtual Network Driver.
2 *
3 * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
867fa150 4 * Copyright (C) 2016-2017 Oracle. All rights reserved.
31762eaa
AY		 5 */
6
7#include <linux/module.h>
8#include <linux/kernel.h>
9#include <linux/types.h>
10#include <linux/slab.h>
11#include <linux/delay.h>
12#include <linux/init.h>
13#include <linux/netdevice.h>
14#include <linux/ethtool.h>
15#include <linux/etherdevice.h>
16#include <linux/mutex.h>
17#include <linux/highmem.h>
18#include <linux/if_vlan.h>
19#define CREATE_TRACE_POINTS
20#include <trace/events/sunvnet.h>
21
22#if IS_ENABLED(CONFIG_IPV6)
23#include <linux/icmpv6.h>
24#endif
25
26#include <net/ip.h>
27#include <net/icmp.h>
28#include <net/route.h>
29
30#include <asm/vio.h>
31#include <asm/ldc.h>
32
33#include "sunvnet_common.h"
34
35/* Heuristic for the number of times to exponentially backoff and
36 * retry sending an LDC trigger when EAGAIN is encountered
37 */
38#define VNET_MAX_RETRIES 10
39
2493b842
SN		 40MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
41MODULE_DESCRIPTION("Sun LDOM virtual network support library");
42MODULE_LICENSE("GPL");
43MODULE_VERSION("1.1");
44
31762eaa 45static int __vnet_tx_trigger(struct vnet_port *port, u32 start);
31762eaa
AY		 46
47static inline u32 vnet_tx_dring_avail(struct vio_dring_state *dr)
48{
49 return vio_dring_avail(dr, VNET_TX_RING_SIZE);
50}
51
52static int vnet_handle_unknown(struct vnet_port *port, void *arg)
53{
54 struct vio_msg_tag *pkt = arg;
55
56 pr_err("Received unknown msg [%02x:%02x:%04x:%08x]\n",
57 pkt->type, pkt->stype, pkt->stype_env, pkt->sid);
58 pr_err("Resetting connection\n");
59
60 ldc_disconnect(port->vio.lp);
61
62 return -ECONNRESET;
63}
64
65static int vnet_port_alloc_tx_ring(struct vnet_port *port);
66
67int sunvnet_send_attr_common(struct vio_driver_state *vio)
68{
69 struct vnet_port *port = to_vnet_port(vio);
67d0719f 70 struct net_device *dev = VNET_PORT_TO_NET_DEVICE(port);
31762eaa
AY		 71 struct vio_net_attr_info pkt;
72 int framelen = ETH_FRAME_LEN;
73 int i, err;
74
75 err = vnet_port_alloc_tx_ring(to_vnet_port(vio));
76 if (err)
77 return err;
78
79 memset(&pkt, 0, sizeof(pkt));
80 pkt.tag.type = VIO_TYPE_CTRL;
81 pkt.tag.stype = VIO_SUBTYPE_INFO;
82 pkt.tag.stype_env = VIO_ATTR_INFO;
83 pkt.tag.sid = vio_send_sid(vio);
84 if (vio_version_before(vio, 1, 2))
85 pkt.xfer_mode = VIO_DRING_MODE;
86 else
87 pkt.xfer_mode = VIO_NEW_DRING_MODE;
88 pkt.addr_type = VNET_ADDR_ETHERMAC;
89 pkt.ack_freq = 0;
90 for (i = 0; i < 6; i++)
91 pkt.addr |= (u64)dev->dev_addr[i] << ((5 - i) * 8);
92 if (vio_version_after(vio, 1, 3)) {
93 if (port->rmtu) {
94 port->rmtu = min(VNET_MAXPACKET, port->rmtu);
95 pkt.mtu = port->rmtu;
96 } else {
97 port->rmtu = VNET_MAXPACKET;
98 pkt.mtu = port->rmtu;
99 }
100 if (vio_version_after_eq(vio, 1, 6))
101 pkt.options = VIO_TX_DRING;
102 } else if (vio_version_before(vio, 1, 3)) {
103 pkt.mtu = framelen;
104 } else { /* v1.3 */
105 pkt.mtu = framelen + VLAN_HLEN;
106 }
107
108 pkt.cflags = 0;
109 if (vio_version_after_eq(vio, 1, 7) && port->tso) {
110 pkt.cflags |= VNET_LSO_IPV4_CAPAB;
111 if (!port->tsolen)
112 port->tsolen = VNET_MAXTSO;
113 pkt.ipv4_lso_maxlen = port->tsolen;
114 }
115
116 pkt.plnk_updt = PHYSLINK_UPDATE_NONE;
117
118 viodbg(HS, "SEND NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] "
119 "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] "
120 "cflags[0x%04x] lso_max[%u]\n",
121 pkt.xfer_mode, pkt.addr_type,
122 (unsigned long long)pkt.addr,
123 pkt.ack_freq, pkt.plnk_updt, pkt.options,
124 (unsigned long long)pkt.mtu, pkt.cflags, pkt.ipv4_lso_maxlen);
125
31762eaa
AY		 126 return vio_ldc_send(vio, &pkt, sizeof(pkt));
127}
128EXPORT_SYMBOL_GPL(sunvnet_send_attr_common);
129
130static int handle_attr_info(struct vio_driver_state *vio,
131 struct vio_net_attr_info *pkt)
132{
133 struct vnet_port *port = to_vnet_port(vio);
134 u64 localmtu;
135 u8 xfer_mode;
136
137 viodbg(HS, "GOT NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] "
138 "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] "
139 " (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n",
140 pkt->xfer_mode, pkt->addr_type,
141 (unsigned long long)pkt->addr,
142 pkt->ack_freq, pkt->plnk_updt, pkt->options,
143 (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags,
144 pkt->ipv4_lso_maxlen);
145
146 pkt->tag.sid = vio_send_sid(vio);
147
148 xfer_mode = pkt->xfer_mode;
149 /* for version < 1.2, VIO_DRING_MODE = 0x3 and no bitmask */
150 if (vio_version_before(vio, 1, 2) && xfer_mode == VIO_DRING_MODE)
151 xfer_mode = VIO_NEW_DRING_MODE;
152
153 /* MTU negotiation:
154 * < v1.3 - ETH_FRAME_LEN exactly
155 * > v1.3 - MIN(pkt.mtu, VNET_MAXPACKET, port->rmtu) and change
156 * pkt->mtu for ACK
157 * = v1.3 - ETH_FRAME_LEN + VLAN_HLEN exactly
158 */
159 if (vio_version_before(vio, 1, 3)) {
160 localmtu = ETH_FRAME_LEN;
161 } else if (vio_version_after(vio, 1, 3)) {
162 localmtu = port->rmtu ? port->rmtu : VNET_MAXPACKET;
163 localmtu = min(pkt->mtu, localmtu);
164 pkt->mtu = localmtu;
165 } else { /* v1.3 */
166 localmtu = ETH_FRAME_LEN + VLAN_HLEN;
167 }
168 port->rmtu = localmtu;
169
170 /* LSO negotiation */
171 if (vio_version_after_eq(vio, 1, 7))
172 port->tso &= !!(pkt->cflags & VNET_LSO_IPV4_CAPAB);
173 else
174 port->tso = false;
175 if (port->tso) {
176 if (!port->tsolen)
177 port->tsolen = VNET_MAXTSO;
178 port->tsolen = min(port->tsolen, pkt->ipv4_lso_maxlen);
179 if (port->tsolen < VNET_MINTSO) {
180 port->tso = false;
181 port->tsolen = 0;
182 pkt->cflags &= ~VNET_LSO_IPV4_CAPAB;
183 }
184 pkt->ipv4_lso_maxlen = port->tsolen;
185 } else {
186 pkt->cflags &= ~VNET_LSO_IPV4_CAPAB;
187 pkt->ipv4_lso_maxlen = 0;
bc221a34 188 port->tsolen = 0;
31762eaa
AY		 189 }
190
191 /* for version >= 1.6, ACK packet mode we support */
192 if (vio_version_after_eq(vio, 1, 6)) {
193 pkt->xfer_mode = VIO_NEW_DRING_MODE;
194 pkt->options = VIO_TX_DRING;
195 }
196
197 if (!(xfer_mode | VIO_NEW_DRING_MODE) ||
198 pkt->addr_type != VNET_ADDR_ETHERMAC ||
199 pkt->mtu != localmtu) {
200 viodbg(HS, "SEND NET ATTR NACK\n");
201
202 pkt->tag.stype = VIO_SUBTYPE_NACK;
203
dc153f85 204 (void)vio_ldc_send(vio, pkt, sizeof(*pkt));
31762eaa
AY		 205
206 return -ECONNRESET;
31762eaa
AY		 207 }
208
dc153f85
AY		 209 viodbg(HS, "SEND NET ATTR ACK xmode[0x%x] atype[0x%x] "
210 "addr[%llx] ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] "
211 "mtu[%llu] (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n",
212 pkt->xfer_mode, pkt->addr_type,
213 (unsigned long long)pkt->addr,
214 pkt->ack_freq, pkt->plnk_updt, pkt->options,
215 (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags,
216 pkt->ipv4_lso_maxlen);
217
218 pkt->tag.stype = VIO_SUBTYPE_ACK;
219
220 return vio_ldc_send(vio, pkt, sizeof(*pkt));
31762eaa
AY		 221}
222
223static int handle_attr_ack(struct vio_driver_state *vio,
224 struct vio_net_attr_info *pkt)
225{
226 viodbg(HS, "GOT NET ATTR ACK\n");
227
228 return 0;
229}
230
231static int handle_attr_nack(struct vio_driver_state *vio,
232 struct vio_net_attr_info *pkt)
233{
234 viodbg(HS, "GOT NET ATTR NACK\n");
235
236 return -ECONNRESET;
237}
238
239int sunvnet_handle_attr_common(struct vio_driver_state *vio, void *arg)
240{
241 struct vio_net_attr_info *pkt = arg;
242
243 switch (pkt->tag.stype) {
244 case VIO_SUBTYPE_INFO:
245 return handle_attr_info(vio, pkt);
246
247 case VIO_SUBTYPE_ACK:
248 return handle_attr_ack(vio, pkt);
249
250 case VIO_SUBTYPE_NACK:
251 return handle_attr_nack(vio, pkt);
252
253 default:
254 return -ECONNRESET;
255 }
256}
257EXPORT_SYMBOL_GPL(sunvnet_handle_attr_common);
258
259void sunvnet_handshake_complete_common(struct vio_driver_state *vio)
260{
261 struct vio_dring_state *dr;
262
263 dr = &vio->drings[VIO_DRIVER_RX_RING];
dc153f85
AY		 264 dr->rcv_nxt = 1;
265 dr->snd_nxt = 1;
31762eaa
AY		 266
267 dr = &vio->drings[VIO_DRIVER_TX_RING];
dc153f85
AY		 268 dr->rcv_nxt = 1;
269 dr->snd_nxt = 1;
31762eaa
AY		 270}
271EXPORT_SYMBOL_GPL(sunvnet_handshake_complete_common);
272
273/* The hypervisor interface that implements copying to/from imported
274 * memory from another domain requires that copies are done to 8-byte
275 * aligned buffers, and that the lengths of such copies are also 8-byte
276 * multiples.
277 *
278 * So we align skb->data to an 8-byte multiple and pad-out the data
279 * area so we can round the copy length up to the next multiple of
280 * 8 for the copy.
281 *
282 * The transmitter puts the actual start of the packet 6 bytes into
283 * the buffer it sends over, so that the IP headers after the ethernet
284 * header are aligned properly. These 6 bytes are not in the descriptor
285 * length, they are simply implied. This offset is represented using
286 * the VNET_PACKET_SKIP macro.
287 */
288static struct sk_buff *alloc_and_align_skb(struct net_device *dev,
289 unsigned int len)
290{
dc153f85 291 struct sk_buff *skb;
31762eaa
AY		 292 unsigned long addr, off;
293
dc153f85 294 skb = netdev_alloc_skb(dev, len + VNET_PACKET_SKIP + 8 + 8);
31762eaa
AY		 295 if (unlikely(!skb))
296 return NULL;
297
dc153f85 298 addr = (unsigned long)skb->data;
31762eaa
AY		 299 off = ((addr + 7UL) & ~7UL) - addr;
300 if (off)
301 skb_reserve(skb, off);
302
303 return skb;
304}
305
306static inline void vnet_fullcsum(struct sk_buff *skb)
307{
308 struct iphdr *iph = ip_hdr(skb);
309 int offset = skb_transport_offset(skb);
310
311 if (skb->protocol != htons(ETH_P_IP))
312 return;
313 if (iph->protocol != IPPROTO_TCP &&
314 iph->protocol != IPPROTO_UDP)
315 return;
316 skb->ip_summed = CHECKSUM_NONE;
317 skb->csum_level = 1;
318 skb->csum = 0;
319 if (iph->protocol == IPPROTO_TCP) {
320 struct tcphdr *ptcp = tcp_hdr(skb);
321
322 ptcp->check = 0;
323 skb->csum = skb_checksum(skb, offset, skb->len - offset, 0);
324 ptcp->check = csum_tcpudp_magic(iph->saddr, iph->daddr,
325 skb->len - offset, IPPROTO_TCP,
326 skb->csum);
327 } else if (iph->protocol == IPPROTO_UDP) {
328 struct udphdr *pudp = udp_hdr(skb);
329
330 pudp->check = 0;
331 skb->csum = skb_checksum(skb, offset, skb->len - offset, 0);
332 pudp->check = csum_tcpudp_magic(iph->saddr, iph->daddr,
333 skb->len - offset, IPPROTO_UDP,
334 skb->csum);
335 }
336}
337
338static int vnet_rx_one(struct vnet_port *port, struct vio_net_desc *desc)
339{
67d0719f 340 struct net_device *dev = VNET_PORT_TO_NET_DEVICE(port);
31762eaa
AY		 341 unsigned int len = desc->size;
342 unsigned int copy_len;
343 struct sk_buff *skb;
344 int maxlen;
345 int err;
346
347 err = -EMSGSIZE;
348 if (port->tso && port->tsolen > port->rmtu)
349 maxlen = port->tsolen;
350 else
351 maxlen = port->rmtu;
352 if (unlikely(len < ETH_ZLEN || len > maxlen)) {
353 dev->stats.rx_length_errors++;
354 goto out_dropped;
355 }
356
357 skb = alloc_and_align_skb(dev, len);
358 err = -ENOMEM;
359 if (unlikely(!skb)) {
360 dev->stats.rx_missed_errors++;
361 goto out_dropped;
362 }
363
364 copy_len = (len + VNET_PACKET_SKIP + 7U) & ~7U;
365 skb_put(skb, copy_len);
366 err = ldc_copy(port->vio.lp, LDC_COPY_IN,
367 skb->data, copy_len, 0,
368 desc->cookies, desc->ncookies);
369 if (unlikely(err < 0)) {
370 dev->stats.rx_frame_errors++;
371 goto out_free_skb;
372 }
373
374 skb_pull(skb, VNET_PACKET_SKIP);
375 skb_trim(skb, len);
376 skb->protocol = eth_type_trans(skb, dev);
377
378 if (vio_version_after_eq(&port->vio, 1, 8)) {
379 struct vio_net_dext *dext = vio_net_ext(desc);
380
381 skb_reset_network_header(skb);
382
383 if (dext->flags & VNET_PKT_HCK_IPV4_HDRCKSUM) {
384 if (skb->protocol == ETH_P_IP) {
385 struct iphdr *iph = ip_hdr(skb);
386
387 iph->check = 0;
388 ip_send_check(iph);
389 }
390 }
391 if ((dext->flags & VNET_PKT_HCK_FULLCKSUM) &&
392 skb->ip_summed == CHECKSUM_NONE) {
393 if (skb->protocol == htons(ETH_P_IP)) {
394 struct iphdr *iph = ip_hdr(skb);
395 int ihl = iph->ihl * 4;
396
397 skb_reset_transport_header(skb);
398 skb_set_transport_header(skb, ihl);
399 vnet_fullcsum(skb);
400 }
401 }
402 if (dext->flags & VNET_PKT_HCK_IPV4_HDRCKSUM_OK) {
403 skb->ip_summed = CHECKSUM_PARTIAL;
404 skb->csum_level = 0;
405 if (dext->flags & VNET_PKT_HCK_FULLCKSUM_OK)
406 skb->csum_level = 1;
407 }
408 }
409
410 skb->ip_summed = port->switch_port ? CHECKSUM_NONE : CHECKSUM_PARTIAL;
411
412 dev->stats.rx_packets++;
413 dev->stats.rx_bytes += len;
0f512c84
SN		 414 port->stats.rx_packets++;
415 port->stats.rx_bytes += len;
31762eaa
AY		 416 napi_gro_receive(&port->napi, skb);
417 return 0;
418
419out_free_skb:
420 kfree_skb(skb);
421
422out_dropped:
423 dev->stats.rx_dropped++;
424 return err;
425}
426
427static int vnet_send_ack(struct vnet_port *port, struct vio_dring_state *dr,
428 u32 start, u32 end, u8 vio_dring_state)
429{
430 struct vio_dring_data hdr = {
431 .tag = {
432 .type = VIO_TYPE_DATA,
433 .stype = VIO_SUBTYPE_ACK,
434 .stype_env = VIO_DRING_DATA,
435 .sid = vio_send_sid(&port->vio),
436 },
437 .dring_ident = dr->ident,
438 .start_idx = start,
439 .end_idx = end,
440 .state = vio_dring_state,
441 };
442 int err, delay;
443 int retries = 0;
444
445 hdr.seq = dr->snd_nxt;
446 delay = 1;
447 do {
448 err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr));
449 if (err > 0) {
450 dr->snd_nxt++;
451 break;
452 }
453 udelay(delay);
454 if ((delay <<= 1) > 128)
455 delay = 128;
456 if (retries++ > VNET_MAX_RETRIES) {
457 pr_info("ECONNRESET %x:%x:%x:%x:%x:%x\n",
458 port->raddr[0], port->raddr[1],
459 port->raddr[2], port->raddr[3],
460 port->raddr[4], port->raddr[5]);
461 break;
462 }
463 } while (err == -EAGAIN);
464
465 if (err <= 0 && vio_dring_state == VIO_DRING_STOPPED) {
466 port->stop_rx_idx = end;
467 port->stop_rx = true;
468 } else {
469 port->stop_rx_idx = 0;
470 port->stop_rx = false;
471 }
472
473 return err;
474}
475
476static struct vio_net_desc *get_rx_desc(struct vnet_port *port,
477 struct vio_dring_state *dr,
478 u32 index)
479{
480 struct vio_net_desc *desc = port->vio.desc_buf;
481 int err;
482
483 err = ldc_get_dring_entry(port->vio.lp, desc, dr->entry_size,
484 (index * dr->entry_size),
485 dr->cookies, dr->ncookies);
486 if (err < 0)
487 return ERR_PTR(err);
488
489 return desc;
490}
491
492static int put_rx_desc(struct vnet_port *port,
493 struct vio_dring_state *dr,
494 struct vio_net_desc *desc,
495 u32 index)
496{
497 int err;
498
499 err = ldc_put_dring_entry(port->vio.lp, desc, dr->entry_size,
500 (index * dr->entry_size),
501 dr->cookies, dr->ncookies);
502 if (err < 0)
503 return err;
504
505 return 0;
506}
507
508static int vnet_walk_rx_one(struct vnet_port *port,
509 struct vio_dring_state *dr,
510 u32 index, int *needs_ack)
511{
512 struct vio_net_desc *desc = get_rx_desc(port, dr, index);
513 struct vio_driver_state *vio = &port->vio;
514 int err;
515
dc153f85 516 BUG_ON(!desc);
31762eaa
AY		 517 if (IS_ERR(desc))
518 return PTR_ERR(desc);
519
520 if (desc->hdr.state != VIO_DESC_READY)
521 return 1;
522
523 dma_rmb();
524
525 viodbg(DATA, "vio_walk_rx_one desc[%02x:%02x:%08x:%08x:%llx:%llx]\n",
526 desc->hdr.state, desc->hdr.ack,
527 desc->size, desc->ncookies,
528 desc->cookies[0].cookie_addr,
529 desc->cookies[0].cookie_size);
530
531 err = vnet_rx_one(port, desc);
532 if (err == -ECONNRESET)
533 return err;
534 trace_vnet_rx_one(port->vio._local_sid, port->vio._peer_sid,
535 index, desc->hdr.ack);
536 desc->hdr.state = VIO_DESC_DONE;
537 err = put_rx_desc(port, dr, desc, index);
538 if (err < 0)
539 return err;
540 *needs_ack = desc->hdr.ack;
541 return 0;
542}
543
544static int vnet_walk_rx(struct vnet_port *port, struct vio_dring_state *dr,
545 u32 start, u32 end, int *npkts, int budget)
546{
547 struct vio_driver_state *vio = &port->vio;
548 int ack_start = -1, ack_end = -1;
549 bool send_ack = true;
550
dc153f85
AY		 551 end = (end == (u32)-1) ? vio_dring_prev(dr, start)
552 : vio_dring_next(dr, end);
31762eaa
AY		 553
554 viodbg(DATA, "vnet_walk_rx start[%08x] end[%08x]\n", start, end);
555
556 while (start != end) {
557 int ack = 0, err = vnet_walk_rx_one(port, dr, start, &ack);
dc153f85 558
31762eaa
AY		 559 if (err == -ECONNRESET)
560 return err;
561 if (err != 0)
562 break;
563 (*npkts)++;
564 if (ack_start == -1)
565 ack_start = start;
566 ack_end = start;
567 start = vio_dring_next(dr, start);
568 if (ack && start != end) {
569 err = vnet_send_ack(port, dr, ack_start, ack_end,
570 VIO_DRING_ACTIVE);
571 if (err == -ECONNRESET)
572 return err;
573 ack_start = -1;
574 }
575 if ((*npkts) >= budget) {
576 send_ack = false;
577 break;
578 }
579 }
dc153f85
AY		 580 if (unlikely(ack_start == -1)) {
581 ack_end = vio_dring_prev(dr, start);
582 ack_start = ack_end;
583 }
31762eaa
AY		 584 if (send_ack) {
585 port->napi_resume = false;
586 trace_vnet_tx_send_stopped_ack(port->vio._local_sid,
587 port->vio._peer_sid,
588 ack_end, *npkts);
589 return vnet_send_ack(port, dr, ack_start, ack_end,
590 VIO_DRING_STOPPED);
591 } else {
592 trace_vnet_tx_defer_stopped_ack(port->vio._local_sid,
593 port->vio._peer_sid,
594 ack_end, *npkts);
595 port->napi_resume = true;
596 port->napi_stop_idx = ack_end;
597 return 1;
598 }
599}
600
601static int vnet_rx(struct vnet_port *port, void *msgbuf, int *npkts,
602 int budget)
603{
604 struct vio_dring_data *pkt = msgbuf;
605 struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_RX_RING];
606 struct vio_driver_state *vio = &port->vio;
607
608 viodbg(DATA, "vnet_rx stype_env[%04x] seq[%016llx] rcv_nxt[%016llx]\n",
609 pkt->tag.stype_env, pkt->seq, dr->rcv_nxt);
610
611 if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA))
612 return 0;
613 if (unlikely(pkt->seq != dr->rcv_nxt)) {
614 pr_err("RX out of sequence seq[0x%llx] rcv_nxt[0x%llx]\n",
615 pkt->seq, dr->rcv_nxt);
616 return 0;
617 }
618
619 if (!port->napi_resume)
620 dr->rcv_nxt++;
621
622 /* XXX Validate pkt->start_idx and pkt->end_idx XXX */
623
624 return vnet_walk_rx(port, dr, pkt->start_idx, pkt->end_idx,
625 npkts, budget);
626}
627
628static int idx_is_pending(struct vio_dring_state *dr, u32 end)
629{
630 u32 idx = dr->cons;
631 int found = 0;
632
633 while (idx != dr->prod) {
634 if (idx == end) {
635 found = 1;
636 break;
637 }
638 idx = vio_dring_next(dr, idx);
639 }
640 return found;
641}
642
643static int vnet_ack(struct vnet_port *port, void *msgbuf)
644{
645 struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING];
646 struct vio_dring_data *pkt = msgbuf;
647 struct net_device *dev;
31762eaa
AY		 648 u32 end;
649 struct vio_net_desc *desc;
650 struct netdev_queue *txq;
651
652 if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA))
653 return 0;
654
655 end = pkt->end_idx;
67d0719f 656 dev = VNET_PORT_TO_NET_DEVICE(port);
31762eaa
AY		 657 netif_tx_lock(dev);
658 if (unlikely(!idx_is_pending(dr, end))) {
659 netif_tx_unlock(dev);
660 return 0;
661 }
662
663 /* sync for race conditions with vnet_start_xmit() and tell xmit it
664 * is time to send a trigger.
665 */
666 trace_vnet_rx_stopped_ack(port->vio._local_sid,
667 port->vio._peer_sid, end);
668 dr->cons = vio_dring_next(dr, end);
669 desc = vio_dring_entry(dr, dr->cons);
670 if (desc->hdr.state == VIO_DESC_READY && !port->start_cons) {
671 /* vnet_start_xmit() just populated this dring but missed
672 * sending the "start" LDC message to the consumer.
673 * Send a "start" trigger on its behalf.
674 */
675 if (__vnet_tx_trigger(port, dr->cons) > 0)
676 port->start_cons = false;
677 else
678 port->start_cons = true;
679 } else {
680 port->start_cons = true;
681 }
682 netif_tx_unlock(dev);
683
684 txq = netdev_get_tx_queue(dev, port->q_index);
685 if (unlikely(netif_tx_queue_stopped(txq) &&
686 vnet_tx_dring_avail(dr) >= VNET_TX_WAKEUP_THRESH(dr)))
687 return 1;
688
689 return 0;
690}
691
692static int vnet_nack(struct vnet_port *port, void *msgbuf)
693{
694 /* XXX just reset or similar XXX */
695 return 0;
696}
697
698static int handle_mcast(struct vnet_port *port, void *msgbuf)
699{
700 struct vio_net_mcast_info *pkt = msgbuf;
67d0719f 701 struct net_device *dev = VNET_PORT_TO_NET_DEVICE(port);
31762eaa
AY		 702
703 if (pkt->tag.stype != VIO_SUBTYPE_ACK)
704 pr_err("%s: Got unexpected MCAST reply [%02x:%02x:%04x:%08x]\n",
67d0719f 705 dev->name,
31762eaa
AY		 706 pkt->tag.type,
707 pkt->tag.stype,
708 pkt->tag.stype_env,
709 pkt->tag.sid);
710
711 return 0;
712}
713
8778b276
AY		 714/* If the queue is stopped, wake it up so that we'll
715 * send out another START message at the next TX.
31762eaa
AY		 716 */
717static void maybe_tx_wakeup(struct vnet_port *port)
718{
719 struct netdev_queue *txq;
720
67d0719f
AY		 721 txq = netdev_get_tx_queue(VNET_PORT_TO_NET_DEVICE(port),
722 port->q_index);
31762eaa 723 __netif_tx_lock(txq, smp_processor_id());
d4aa89cc 724 if (likely(netif_tx_queue_stopped(txq)))
31762eaa 725 netif_tx_wake_queue(txq);
31762eaa
AY		 726 __netif_tx_unlock(txq);
727}
728
67d0719f 729bool sunvnet_port_is_up_common(struct vnet_port *vnet)
31762eaa
AY		 730{
731 struct vio_driver_state *vio = &vnet->vio;
732
733 return !!(vio->hs_state & VIO_HS_COMPLETE);
734}
67d0719f 735EXPORT_SYMBOL_GPL(sunvnet_port_is_up_common);
31762eaa
AY		 736
737static int vnet_event_napi(struct vnet_port *port, int budget)
738{
8778b276 739 struct net_device *dev = VNET_PORT_TO_NET_DEVICE(port);
31762eaa
AY		 740 struct vio_driver_state *vio = &port->vio;
741 int tx_wakeup, err;
742 int npkts = 0;
bf091f3f
SN		 743
744 /* we don't expect any other bits */
745 BUG_ON(port->rx_event & ~(LDC_EVENT_DATA_READY |
746 LDC_EVENT_RESET |
747 LDC_EVENT_UP));
748
749 /* RESET takes precedent over any other event */
750 if (port->rx_event & LDC_EVENT_RESET) {
867fa150
SN		 751 /* a link went down */
752
753 if (port->vsw == 1) {
754 netif_tx_stop_all_queues(dev);
755 netif_carrier_off(dev);
756 }
757
bf091f3f
SN		 758 vio_link_state_change(vio, LDC_EVENT_RESET);
759 vnet_port_reset(port);
760 vio_port_up(vio);
761
762 /* If the device is running but its tx queue was
763 * stopped (due to flow control), restart it.
764 * This is necessary since vnet_port_reset()
765 * clears the tx drings and thus we may never get
766 * back a VIO_TYPE_DATA ACK packet - which is
767 * the normal mechanism to restart the tx queue.
768 */
769 if (netif_running(dev))
770 maybe_tx_wakeup(port);
771
31762eaa 772 port->rx_event = 0;
0f512c84 773 port->stats.event_reset++;
31762eaa
AY		 774 return 0;
775 }
31762eaa 776
bf091f3f 777 if (port->rx_event & LDC_EVENT_UP) {
867fa150
SN		 778 /* a link came up */
779
780 if (port->vsw == 1) {
781 netif_carrier_on(port->dev);
782 netif_tx_start_all_queues(port->dev);
783 }
784
bf091f3f
SN		 785 vio_link_state_change(vio, LDC_EVENT_UP);
786 port->rx_event = 0;
0f512c84 787 port->stats.event_up++;
bf091f3f
SN		 788 return 0;
789 }
31762eaa 790
dc153f85
AY		 791 err = 0;
792 tx_wakeup = 0;
31762eaa
AY		 793 while (1) {
794 union {
795 struct vio_msg_tag tag;
796 u64 raw[8];
797 } msgbuf;
798
799 if (port->napi_resume) {
800 struct vio_dring_data *pkt =
801 (struct vio_dring_data *)&msgbuf;
802 struct vio_dring_state *dr =
803 &port->vio.drings[VIO_DRIVER_RX_RING];
804
805 pkt->tag.type = VIO_TYPE_DATA;
806 pkt->tag.stype = VIO_SUBTYPE_INFO;
807 pkt->tag.stype_env = VIO_DRING_DATA;
808 pkt->seq = dr->rcv_nxt;
dc153f85
AY		 809 pkt->start_idx = vio_dring_next(dr,
810 port->napi_stop_idx);
31762eaa 811 pkt->end_idx = -1;
bf091f3f
SN		 812 } else {
813 err = ldc_read(vio->lp, &msgbuf, sizeof(msgbuf));
814 if (unlikely(err < 0)) {
815 if (err == -ECONNRESET)
816 vio_conn_reset(vio);
817 break;
818 }
819 if (err == 0)
820 break;
821 viodbg(DATA, "TAG [%02x:%02x:%04x:%08x]\n",
822 msgbuf.tag.type,
823 msgbuf.tag.stype,
824 msgbuf.tag.stype_env,
825 msgbuf.tag.sid);
826 err = vio_validate_sid(vio, &msgbuf.tag);
827 if (err < 0)
828 break;
31762eaa 829 }
bf091f3f 830
31762eaa
AY		 831 if (likely(msgbuf.tag.type == VIO_TYPE_DATA)) {
832 if (msgbuf.tag.stype == VIO_SUBTYPE_INFO) {
67d0719f 833 if (!sunvnet_port_is_up_common(port)) {
31762eaa
AY		 834 /* failures like handshake_failure()
835 * may have cleaned up dring, but
836 * NAPI polling may bring us here.
837 */
838 err = -ECONNRESET;
839 break;
840 }
841 err = vnet_rx(port, &msgbuf, &npkts, budget);
842 if (npkts >= budget)
843 break;
844 if (npkts == 0)
845 break;
846 } else if (msgbuf.tag.stype == VIO_SUBTYPE_ACK) {
847 err = vnet_ack(port, &msgbuf);
848 if (err > 0)
849 tx_wakeup |= err;
850 } else if (msgbuf.tag.stype == VIO_SUBTYPE_NACK) {
851 err = vnet_nack(port, &msgbuf);
852 }
853 } else if (msgbuf.tag.type == VIO_TYPE_CTRL) {
854 if (msgbuf.tag.stype_env == VNET_MCAST_INFO)
855 err = handle_mcast(port, &msgbuf);
856 else
857 err = vio_control_pkt_engine(vio, &msgbuf);
858 if (err)
859 break;
860 } else {
861 err = vnet_handle_unknown(port, &msgbuf);
862 }
863 if (err == -ECONNRESET)
864 break;
865 }
866 if (unlikely(tx_wakeup && err != -ECONNRESET))
867 maybe_tx_wakeup(port);
868 return npkts;
869}
870
871int sunvnet_poll_common(struct napi_struct *napi, int budget)
872{
873 struct vnet_port *port = container_of(napi, struct vnet_port, napi);
874 struct vio_driver_state *vio = &port->vio;
875 int processed = vnet_event_napi(port, budget);
876
877 if (processed < budget) {
6ad20165 878 napi_complete_done(napi, processed);
31762eaa
AY		 879 port->rx_event &= ~LDC_EVENT_DATA_READY;
880 vio_set_intr(vio->vdev->rx_ino, HV_INTR_ENABLED);
881 }
882 return processed;
883}
884EXPORT_SYMBOL_GPL(sunvnet_poll_common);
885
886void sunvnet_event_common(void *arg, int event)
887{
888 struct vnet_port *port = arg;
889 struct vio_driver_state *vio = &port->vio;
890
891 port->rx_event |= event;
892 vio_set_intr(vio->vdev->rx_ino, HV_INTR_DISABLED);
893 napi_schedule(&port->napi);
31762eaa
AY		 894}
895EXPORT_SYMBOL_GPL(sunvnet_event_common);
896
897static int __vnet_tx_trigger(struct vnet_port *port, u32 start)
898{
899 struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING];
900 struct vio_dring_data hdr = {
901 .tag = {
902 .type = VIO_TYPE_DATA,
903 .stype = VIO_SUBTYPE_INFO,
904 .stype_env = VIO_DRING_DATA,
905 .sid = vio_send_sid(&port->vio),
906 },
907 .dring_ident = dr->ident,
908 .start_idx = start,
dc153f85 909 .end_idx = (u32)-1,
31762eaa
AY		 910 };
911 int err, delay;
912 int retries = 0;
913
914 if (port->stop_rx) {
915 trace_vnet_tx_pending_stopped_ack(port->vio._local_sid,
916 port->vio._peer_sid,
917 port->stop_rx_idx, -1);
918 err = vnet_send_ack(port,
919 &port->vio.drings[VIO_DRIVER_RX_RING],
920 port->stop_rx_idx, -1,
921 VIO_DRING_STOPPED);
922 if (err <= 0)
923 return err;
924 }
925
926 hdr.seq = dr->snd_nxt;
927 delay = 1;
928 do {
929 err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr));
930 if (err > 0) {
931 dr->snd_nxt++;
932 break;
933 }
934 udelay(delay);
935 if ((delay <<= 1) > 128)
936 delay = 128;
937 if (retries++ > VNET_MAX_RETRIES)
938 break;
939 } while (err == -EAGAIN);
940 trace_vnet_tx_trigger(port->vio._local_sid,
941 port->vio._peer_sid, start, err);
942
943 return err;
944}
945
31762eaa
AY		 946static struct sk_buff *vnet_clean_tx_ring(struct vnet_port *port,
947 unsigned *pending)
948{
949 struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING];
950 struct sk_buff *skb = NULL;
951 int i, txi;
952
953 *pending = 0;
954
955 txi = dr->prod;
956 for (i = 0; i < VNET_TX_RING_SIZE; ++i) {
957 struct vio_net_desc *d;
958
959 --txi;
960 if (txi < 0)
dc153f85 961 txi = VNET_TX_RING_SIZE - 1;
31762eaa
AY		 962
963 d = vio_dring_entry(dr, txi);
964
965 if (d->hdr.state == VIO_DESC_READY) {
966 (*pending)++;
967 continue;
968 }
969 if (port->tx_bufs[txi].skb) {
970 if (d->hdr.state != VIO_DESC_DONE)
971 pr_notice("invalid ring buffer state %d\n",
972 d->hdr.state);
973 BUG_ON(port->tx_bufs[txi].skb->next);
974
975 port->tx_bufs[txi].skb->next = skb;
976 skb = port->tx_bufs[txi].skb;
977 port->tx_bufs[txi].skb = NULL;
978
979 ldc_unmap(port->vio.lp,
980 port->tx_bufs[txi].cookies,
981 port->tx_bufs[txi].ncookies);
dc153f85 982 } else if (d->hdr.state == VIO_DESC_FREE) {
31762eaa 983 break;
dc153f85 984 }
31762eaa
AY		 985 d->hdr.state = VIO_DESC_FREE;
986 }
987 return skb;
988}
989
990static inline void vnet_free_skbs(struct sk_buff *skb)
991{
992 struct sk_buff *next;
993
994 while (skb) {
995 next = skb->next;
996 skb->next = NULL;
997 dev_kfree_skb(skb);
998 skb = next;
999 }
1000}
1001
1002void sunvnet_clean_timer_expire_common(unsigned long port0)
1003{
1004 struct vnet_port *port = (struct vnet_port *)port0;
1005 struct sk_buff *freeskbs;
1006 unsigned pending;
1007
67d0719f 1008 netif_tx_lock(VNET_PORT_TO_NET_DEVICE(port));
31762eaa 1009 freeskbs = vnet_clean_tx_ring(port, &pending);
67d0719f 1010 netif_tx_unlock(VNET_PORT_TO_NET_DEVICE(port));
31762eaa
AY		 1011
1012 vnet_free_skbs(freeskbs);
1013
1014 if (pending)
1015 (void)mod_timer(&port->clean_timer,
1016 jiffies + VNET_CLEAN_TIMEOUT);
1017 else
1018 del_timer(&port->clean_timer);
1019}
1020EXPORT_SYMBOL_GPL(sunvnet_clean_timer_expire_common);
1021
1022static inline int vnet_skb_map(struct ldc_channel *lp, struct sk_buff *skb,
1023 struct ldc_trans_cookie *cookies, int ncookies,
1024 unsigned int map_perm)
1025{
1026 int i, nc, err, blen;
1027
1028 /* header */
1029 blen = skb_headlen(skb);
1030 if (blen < ETH_ZLEN)
1031 blen = ETH_ZLEN;
1032 blen += VNET_PACKET_SKIP;
1033 blen += 8 - (blen & 7);
1034
dc153f85 1035 err = ldc_map_single(lp, skb->data - VNET_PACKET_SKIP, blen, cookies,
31762eaa
AY		 1036 ncookies, map_perm);
1037 if (err < 0)
1038 return err;
1039 nc = err;
1040
1041 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1042 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1043 u8 *vaddr;
1044
1045 if (nc < ncookies) {
1046 vaddr = kmap_atomic(skb_frag_page(f));
1047 blen = skb_frag_size(f);
1048 blen += 8 - (blen & 7);
1049 err = ldc_map_single(lp, vaddr + f->page_offset,
1050 blen, cookies + nc, ncookies - nc,
1051 map_perm);
1052 kunmap_atomic(vaddr);
1053 } else {
1054 err = -EMSGSIZE;
1055 }
1056
1057 if (err < 0) {
1058 ldc_unmap(lp, cookies, nc);
1059 return err;
1060 }
1061 nc += err;
1062 }
1063 return nc;
1064}
1065
1066static inline struct sk_buff *vnet_skb_shape(struct sk_buff *skb, int ncookies)
1067{
1068 struct sk_buff *nskb;
1069 int i, len, pad, docopy;
1070
1071 len = skb->len;
1072 pad = 0;
1073 if (len < ETH_ZLEN) {
1074 pad += ETH_ZLEN - skb->len;
1075 len += pad;
1076 }
1077 len += VNET_PACKET_SKIP;
1078 pad += 8 - (len & 7);
1079
1080 /* make sure we have enough cookies and alignment in every frag */
1081 docopy = skb_shinfo(skb)->nr_frags >= ncookies;
1082 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1083 skb_frag_t *f = &skb_shinfo(skb)->frags[i];
1084
1085 docopy |= f->page_offset & 7;
1086 }
1087 if (((unsigned long)skb->data & 7) != VNET_PACKET_SKIP ||
1088 skb_tailroom(skb) < pad ||
1089 skb_headroom(skb) < VNET_PACKET_SKIP || docopy) {
1090 int start = 0, offset;
1091 __wsum csum;
1092
1093 len = skb->len > ETH_ZLEN ? skb->len : ETH_ZLEN;
1094 nskb = alloc_and_align_skb(skb->dev, len);
dc153f85 1095 if (!nskb) {
31762eaa
AY		 1096 dev_kfree_skb(skb);
1097 return NULL;
1098 }
1099 skb_reserve(nskb, VNET_PACKET_SKIP);
1100
1101 nskb->protocol = skb->protocol;
1102 offset = skb_mac_header(skb) - skb->data;
1103 skb_set_mac_header(nskb, offset);
1104 offset = skb_network_header(skb) - skb->data;
1105 skb_set_network_header(nskb, offset);
1106 offset = skb_transport_header(skb) - skb->data;
1107 skb_set_transport_header(nskb, offset);
1108
1109 offset = 0;
1110 nskb->csum_offset = skb->csum_offset;
1111 nskb->ip_summed = skb->ip_summed;
1112
1113 if (skb->ip_summed == CHECKSUM_PARTIAL)
1114 start = skb_checksum_start_offset(skb);
1115 if (start) {
1116 struct iphdr *iph = ip_hdr(nskb);
1117 int offset = start + nskb->csum_offset;
1118
1119 if (skb_copy_bits(skb, 0, nskb->data, start)) {
1120 dev_kfree_skb(nskb);
1121 dev_kfree_skb(skb);
1122 return NULL;
1123 }
1124 *(__sum16 *)(skb->data + offset) = 0;
1125 csum = skb_copy_and_csum_bits(skb, start,
1126 nskb->data + start,
1127 skb->len - start, 0);
1128 if (iph->protocol == IPPROTO_TCP ||
1129 iph->protocol == IPPROTO_UDP) {
1130 csum = csum_tcpudp_magic(iph->saddr, iph->daddr,
1131 skb->len - start,
1132 iph->protocol, csum);
1133 }
1134 *(__sum16 *)(nskb->data + offset) = csum;
1135
1136 nskb->ip_summed = CHECKSUM_NONE;
1137 } else if (skb_copy_bits(skb, 0, nskb->data, skb->len)) {
1138 dev_kfree_skb(nskb);
1139 dev_kfree_skb(skb);
1140 return NULL;
1141 }
1142 (void)skb_put(nskb, skb->len);
1143 if (skb_is_gso(skb)) {
1144 skb_shinfo(nskb)->gso_size = skb_shinfo(skb)->gso_size;
1145 skb_shinfo(nskb)->gso_type = skb_shinfo(skb)->gso_type;
1146 }
1147 nskb->queue_mapping = skb->queue_mapping;
1148 dev_kfree_skb(skb);
1149 skb = nskb;
1150 }
1151 return skb;
1152}
1153
67d0719f
AY		 1154static int vnet_handle_offloads(struct vnet_port *port, struct sk_buff *skb,
1155 struct vnet_port *(*vnet_tx_port)
1156 (struct sk_buff *, struct net_device *))
31762eaa 1157{
67d0719f 1158 struct net_device *dev = VNET_PORT_TO_NET_DEVICE(port);
31762eaa
AY		 1159 struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING];
1160 struct sk_buff *segs;
1161 int maclen, datalen;
1162 int status;
1163 int gso_size, gso_type, gso_segs;
1164 int hlen = skb_transport_header(skb) - skb_mac_header(skb);
1165 int proto = IPPROTO_IP;
1166
1167 if (skb->protocol == htons(ETH_P_IP))
1168 proto = ip_hdr(skb)->protocol;
1169 else if (skb->protocol == htons(ETH_P_IPV6))
1170 proto = ipv6_hdr(skb)->nexthdr;
1171
dc153f85 1172 if (proto == IPPROTO_TCP) {
31762eaa 1173 hlen += tcp_hdr(skb)->doff * 4;
dc153f85 1174 } else if (proto == IPPROTO_UDP) {
31762eaa 1175 hlen += sizeof(struct udphdr);
dc153f85 1176 } else {
31762eaa
AY		 1177 pr_err("vnet_handle_offloads GSO with unknown transport "
1178 "protocol %d tproto %d\n", skb->protocol, proto);
1179 hlen = 128; /* XXX */
1180 }
1181 datalen = port->tsolen - hlen;
1182
1183 gso_size = skb_shinfo(skb)->gso_size;
1184 gso_type = skb_shinfo(skb)->gso_type;
1185 gso_segs = skb_shinfo(skb)->gso_segs;
1186
1187 if (port->tso && gso_size < datalen)
1188 gso_segs = DIV_ROUND_UP(skb->len - hlen, datalen);
1189
1190 if (unlikely(vnet_tx_dring_avail(dr) < gso_segs)) {
1191 struct netdev_queue *txq;
1192
1193 txq = netdev_get_tx_queue(dev, port->q_index);
1194 netif_tx_stop_queue(txq);
1195 if (vnet_tx_dring_avail(dr) < skb_shinfo(skb)->gso_segs)
1196 return NETDEV_TX_BUSY;
1197 netif_tx_wake_queue(txq);
1198 }
1199
1200 maclen = skb_network_header(skb) - skb_mac_header(skb);
1201 skb_pull(skb, maclen);
1202
1203 if (port->tso && gso_size < datalen) {
1204 if (skb_unclone(skb, GFP_ATOMIC))
1205 goto out_dropped;
1206
1207 /* segment to TSO size */
1208 skb_shinfo(skb)->gso_size = datalen;
1209 skb_shinfo(skb)->gso_segs = gso_segs;
1210 }
1211 segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO);
1212 if (IS_ERR(segs))
1213 goto out_dropped;
1214
1215 skb_push(skb, maclen);
1216 skb_reset_mac_header(skb);
1217
1218 status = 0;
1219 while (segs) {
1220 struct sk_buff *curr = segs;
1221
1222 segs = segs->next;
1223 curr->next = NULL;
1224 if (port->tso && curr->len > dev->mtu) {
1225 skb_shinfo(curr)->gso_size = gso_size;
1226 skb_shinfo(curr)->gso_type = gso_type;
1227 skb_shinfo(curr)->gso_segs =
1228 DIV_ROUND_UP(curr->len - hlen, gso_size);
dc153f85 1229 } else {
31762eaa 1230 skb_shinfo(curr)->gso_size = 0;
dc153f85 1231 }
31762eaa
AY		 1232
1233 skb_push(curr, maclen);
1234 skb_reset_mac_header(curr);
1235 memcpy(skb_mac_header(curr), skb_mac_header(skb),
1236 maclen);
1237 curr->csum_start = skb_transport_header(curr) - curr->head;
1238 if (ip_hdr(curr)->protocol == IPPROTO_TCP)
1239 curr->csum_offset = offsetof(struct tcphdr, check);
1240 else if (ip_hdr(curr)->protocol == IPPROTO_UDP)
1241 curr->csum_offset = offsetof(struct udphdr, check);
1242
1243 if (!(status & NETDEV_TX_MASK))
67d0719f
AY		 1244 status = sunvnet_start_xmit_common(curr, dev,
1245 vnet_tx_port);
31762eaa
AY		 1246 if (status & NETDEV_TX_MASK)
1247 dev_kfree_skb_any(curr);
1248 }
1249
1250 if (!(status & NETDEV_TX_MASK))
1251 dev_kfree_skb_any(skb);
1252 return status;
1253out_dropped:
1254 dev->stats.tx_dropped++;
1255 dev_kfree_skb_any(skb);
1256 return NETDEV_TX_OK;
1257}
1258
67d0719f
AY		 1259int sunvnet_start_xmit_common(struct sk_buff *skb, struct net_device *dev,
1260 struct vnet_port *(*vnet_tx_port)
1261 (struct sk_buff *, struct net_device *))
31762eaa 1262{
31762eaa
AY		 1263 struct vnet_port *port = NULL;
1264 struct vio_dring_state *dr;
1265 struct vio_net_desc *d;
1266 unsigned int len;
1267 struct sk_buff *freeskbs = NULL;
1268 int i, err, txi;
1269 unsigned pending = 0;
1270 struct netdev_queue *txq;
1271
1272 rcu_read_lock();
67d0719f 1273 port = vnet_tx_port(skb, dev);
daa86e50 1274 if (unlikely(!port))
31762eaa 1275 goto out_dropped;
31762eaa
AY		 1276
1277 if (skb_is_gso(skb) && skb->len > port->tsolen) {
67d0719f 1278 err = vnet_handle_offloads(port, skb, vnet_tx_port);
31762eaa
AY		 1279 rcu_read_unlock();
1280 return err;
1281 }
1282
1283 if (!skb_is_gso(skb) && skb->len > port->rmtu) {
1284 unsigned long localmtu = port->rmtu - ETH_HLEN;
1285
1286 if (vio_version_after_eq(&port->vio, 1, 3))
1287 localmtu -= VLAN_HLEN;
1288
1289 if (skb->protocol == htons(ETH_P_IP)) {
1290 struct flowi4 fl4;
1291 struct rtable *rt = NULL;
1292
1293 memset(&fl4, 0, sizeof(fl4));
1294 fl4.flowi4_oif = dev->ifindex;
1295 fl4.flowi4_tos = RT_TOS(ip_hdr(skb)->tos);
1296 fl4.daddr = ip_hdr(skb)->daddr;
1297 fl4.saddr = ip_hdr(skb)->saddr;
1298
1299 rt = ip_route_output_key(dev_net(dev), &fl4);
31762eaa
AY		 1300 if (!IS_ERR(rt)) {
1301 skb_dst_set(skb, &rt->dst);
1302 icmp_send(skb, ICMP_DEST_UNREACH,
1303 ICMP_FRAG_NEEDED,
1304 htonl(localmtu));
1305 }
1306 }
1307#if IS_ENABLED(CONFIG_IPV6)
1308 else if (skb->protocol == htons(ETH_P_IPV6))
1309 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, localmtu);
1310#endif
1311 goto out_dropped;
1312 }
1313
1314 skb = vnet_skb_shape(skb, 2);
1315
1316 if (unlikely(!skb))
1317 goto out_dropped;
1318
1319 if (skb->ip_summed == CHECKSUM_PARTIAL)
1320 vnet_fullcsum(skb);
1321
1322 dr = &port->vio.drings[VIO_DRIVER_TX_RING];
1323 i = skb_get_queue_mapping(skb);
1324 txq = netdev_get_tx_queue(dev, i);
1325 if (unlikely(vnet_tx_dring_avail(dr) < 1)) {
1326 if (!netif_tx_queue_stopped(txq)) {
1327 netif_tx_stop_queue(txq);
1328
1329 /* This is a hard error, log it. */
1330 netdev_err(dev, "BUG! Tx Ring full when queue awake!\n");
1331 dev->stats.tx_errors++;
1332 }
1333 rcu_read_unlock();
1334 return NETDEV_TX_BUSY;
1335 }
1336
1337 d = vio_dring_cur(dr);
1338
1339 txi = dr->prod;
1340
1341 freeskbs = vnet_clean_tx_ring(port, &pending);
1342
1343 BUG_ON(port->tx_bufs[txi].skb);
1344
1345 len = skb->len;
1346 if (len < ETH_ZLEN)
1347 len = ETH_ZLEN;
1348
1349 err = vnet_skb_map(port->vio.lp, skb, port->tx_bufs[txi].cookies, 2,
1350 (LDC_MAP_SHADOW | LDC_MAP_DIRECT | LDC_MAP_RW));
1351 if (err < 0) {
1352 netdev_info(dev, "tx buffer map error %d\n", err);
1353 goto out_dropped;
1354 }
1355
1356 port->tx_bufs[txi].skb = skb;
1357 skb = NULL;
1358 port->tx_bufs[txi].ncookies = err;
1359
1360 /* We don't rely on the ACKs to free the skb in vnet_start_xmit(),
1361 * thus it is safe to not set VIO_ACK_ENABLE for each transmission:
1362 * the protocol itself does not require it as long as the peer
1363 * sends a VIO_SUBTYPE_ACK for VIO_DRING_STOPPED.
1364 *
1365 * An ACK for every packet in the ring is expensive as the
1366 * sending of LDC messages is slow and affects performance.
1367 */
1368 d->hdr.ack = VIO_ACK_DISABLE;
1369 d->size = len;
1370 d->ncookies = port->tx_bufs[txi].ncookies;
1371 for (i = 0; i < d->ncookies; i++)
1372 d->cookies[i] = port->tx_bufs[txi].cookies[i];
1373 if (vio_version_after_eq(&port->vio, 1, 7)) {
1374 struct vio_net_dext *dext = vio_net_ext(d);
1375
1376 memset(dext, 0, sizeof(*dext));
1377 if (skb_is_gso(port->tx_bufs[txi].skb)) {
1378 dext->ipv4_lso_mss = skb_shinfo(port->tx_bufs[txi].skb)
1379 ->gso_size;
1380 dext->flags |= VNET_PKT_IPV4_LSO;
1381 }
1382 if (vio_version_after_eq(&port->vio, 1, 8) &&
1383 !port->switch_port) {
1384 dext->flags |= VNET_PKT_HCK_IPV4_HDRCKSUM_OK;
1385 dext->flags |= VNET_PKT_HCK_FULLCKSUM_OK;
1386 }
1387 }
1388
1389 /* This has to be a non-SMP write barrier because we are writing
1390 * to memory which is shared with the peer LDOM.
1391 */
1392 dma_wmb();
1393
1394 d->hdr.state = VIO_DESC_READY;
1395
1396 /* Exactly one ldc "start" trigger (for dr->cons) needs to be sent
1397 * to notify the consumer that some descriptors are READY.
1398 * After that "start" trigger, no additional triggers are needed until
1399 * a DRING_STOPPED is received from the consumer. The dr->cons field
1400 * (set up by vnet_ack()) has the value of the next dring index
1401 * that has not yet been ack-ed. We send a "start" trigger here
1402 * if, and only if, start_cons is true (reset it afterward). Conversely,
1403 * vnet_ack() should check if the dring corresponding to cons
1404 * is marked READY, but start_cons was false.
1405 * If so, vnet_ack() should send out the missed "start" trigger.
1406 *
1407 * Note that the dma_wmb() above makes sure the cookies et al. are
1408 * not globally visible before the VIO_DESC_READY, and that the
1409 * stores are ordered correctly by the compiler. The consumer will
1410 * not proceed until the VIO_DESC_READY is visible assuring that
1411 * the consumer does not observe anything related to descriptors
1412 * out of order. The HV trap from the LDC start trigger is the
1413 * producer to consumer announcement that work is available to the
1414 * consumer
1415 */
1416 if (!port->start_cons) { /* previous trigger suffices */
1417 trace_vnet_skip_tx_trigger(port->vio._local_sid,
1418 port->vio._peer_sid, dr->cons);
1419 goto ldc_start_done;
1420 }
1421
1422 err = __vnet_tx_trigger(port, dr->cons);
1423 if (unlikely(err < 0)) {
1424 netdev_info(dev, "TX trigger error %d\n", err);
1425 d->hdr.state = VIO_DESC_FREE;
1426 skb = port->tx_bufs[txi].skb;
1427 port->tx_bufs[txi].skb = NULL;
1428 dev->stats.tx_carrier_errors++;
1429 goto out_dropped;
1430 }
1431
1432ldc_start_done:
1433 port->start_cons = false;
1434
1435 dev->stats.tx_packets++;
1436 dev->stats.tx_bytes += port->tx_bufs[txi].skb->len;
0f512c84
SN		 1437 port->stats.tx_packets++;
1438 port->stats.tx_bytes += port->tx_bufs[txi].skb->len;
31762eaa
AY		 1439
1440 dr->prod = (dr->prod + 1) & (VNET_TX_RING_SIZE - 1);
1441 if (unlikely(vnet_tx_dring_avail(dr) < 1)) {
1442 netif_tx_stop_queue(txq);
fd263fb6 1443 smp_rmb();
31762eaa
AY		 1444 if (vnet_tx_dring_avail(dr) > VNET_TX_WAKEUP_THRESH(dr))
1445 netif_tx_wake_queue(txq);
1446 }
1447
1448 (void)mod_timer(&port->clean_timer, jiffies + VNET_CLEAN_TIMEOUT);
1449 rcu_read_unlock();
1450
1451 vnet_free_skbs(freeskbs);
1452
1453 return NETDEV_TX_OK;
1454
1455out_dropped:
1456 if (pending)
1457 (void)mod_timer(&port->clean_timer,
1458 jiffies + VNET_CLEAN_TIMEOUT);
1459 else if (port)
1460 del_timer(&port->clean_timer);
daa86e50 1461 rcu_read_unlock();
31762eaa
AY		 1462 if (skb)
1463 dev_kfree_skb(skb);
1464 vnet_free_skbs(freeskbs);
1465 dev->stats.tx_dropped++;
1466 return NETDEV_TX_OK;
1467}
1468EXPORT_SYMBOL_GPL(sunvnet_start_xmit_common);
1469
1470void sunvnet_tx_timeout_common(struct net_device *dev)
1471{
1472 /* XXX Implement me XXX */
1473}
1474EXPORT_SYMBOL_GPL(sunvnet_tx_timeout_common);
1475
1476int sunvnet_open_common(struct net_device *dev)
1477{
1478 netif_carrier_on(dev);
1479 netif_tx_start_all_queues(dev);
1480
1481 return 0;
1482}
1483EXPORT_SYMBOL_GPL(sunvnet_open_common);
1484
1485int sunvnet_close_common(struct net_device *dev)
1486{
1487 netif_tx_stop_all_queues(dev);
1488 netif_carrier_off(dev);
1489
1490 return 0;
1491}
1492EXPORT_SYMBOL_GPL(sunvnet_close_common);
1493
1494static struct vnet_mcast_entry *__vnet_mc_find(struct vnet *vp, u8 *addr)
1495{
1496 struct vnet_mcast_entry *m;
1497
1498 for (m = vp->mcast_list; m; m = m->next) {
1499 if (ether_addr_equal(m->addr, addr))
1500 return m;
1501 }
1502 return NULL;
1503}
1504
1505static void __update_mc_list(struct vnet *vp, struct net_device *dev)
1506{
1507 struct netdev_hw_addr *ha;
1508
1509 netdev_for_each_mc_addr(ha, dev) {
1510 struct vnet_mcast_entry *m;
1511
1512 m = __vnet_mc_find(vp, ha->addr);
1513 if (m) {
1514 m->hit = 1;
1515 continue;
1516 }
1517
1518 if (!m) {
1519 m = kzalloc(sizeof(*m), GFP_ATOMIC);
1520 if (!m)
1521 continue;
1522 memcpy(m->addr, ha->addr, ETH_ALEN);
1523 m->hit = 1;
1524
1525 m->next = vp->mcast_list;
1526 vp->mcast_list = m;
1527 }
1528 }
1529}
1530
1531static void __send_mc_list(struct vnet *vp, struct vnet_port *port)
1532{
1533 struct vio_net_mcast_info info;
1534 struct vnet_mcast_entry *m, **pp;
1535 int n_addrs;
1536
1537 memset(&info, 0, sizeof(info));
1538
1539 info.tag.type = VIO_TYPE_CTRL;
1540 info.tag.stype = VIO_SUBTYPE_INFO;
1541 info.tag.stype_env = VNET_MCAST_INFO;
1542 info.tag.sid = vio_send_sid(&port->vio);
1543 info.set = 1;
1544
1545 n_addrs = 0;
1546 for (m = vp->mcast_list; m; m = m->next) {
1547 if (m->sent)
1548 continue;
1549 m->sent = 1;
1550 memcpy(&info.mcast_addr[n_addrs * ETH_ALEN],
1551 m->addr, ETH_ALEN);
1552 if (++n_addrs == VNET_NUM_MCAST) {
1553 info.count = n_addrs;
1554
dc153f85
AY		 1555 (void)vio_ldc_send(&port->vio, &info,
1556 sizeof(info));
31762eaa
AY		 1557 n_addrs = 0;
1558 }
1559 }
1560 if (n_addrs) {
1561 info.count = n_addrs;
dc153f85 1562 (void)vio_ldc_send(&port->vio, &info, sizeof(info));
31762eaa
AY		 1563 }
1564
1565 info.set = 0;
1566
1567 n_addrs = 0;
1568 pp = &vp->mcast_list;
1569 while ((m = *pp) != NULL) {
1570 if (m->hit) {
1571 m->hit = 0;
1572 pp = &m->next;
1573 continue;
1574 }
1575
1576 memcpy(&info.mcast_addr[n_addrs * ETH_ALEN],
1577 m->addr, ETH_ALEN);
1578 if (++n_addrs == VNET_NUM_MCAST) {
1579 info.count = n_addrs;
dc153f85
AY		 1580 (void)vio_ldc_send(&port->vio, &info,
1581 sizeof(info));
31762eaa
AY		 1582 n_addrs = 0;
1583 }
1584
1585 *pp = m->next;
1586 kfree(m);
1587 }
1588 if (n_addrs) {
1589 info.count = n_addrs;
dc153f85 1590 (void)vio_ldc_send(&port->vio, &info, sizeof(info));
31762eaa
AY		 1591 }
1592}
1593
67d0719f 1594void sunvnet_set_rx_mode_common(struct net_device *dev, struct vnet *vp)
31762eaa 1595{
31762eaa
AY		 1596 struct vnet_port *port;
1597
1598 rcu_read_lock();
1599 list_for_each_entry_rcu(port, &vp->port_list, list) {
31762eaa
AY		 1600 if (port->switch_port) {
1601 __update_mc_list(vp, dev);
1602 __send_mc_list(vp, port);
1603 break;
1604 }
1605 }
1606 rcu_read_unlock();
1607}
1608EXPORT_SYMBOL_GPL(sunvnet_set_rx_mode_common);
1609
31762eaa
AY		 1610int sunvnet_set_mac_addr_common(struct net_device *dev, void *p)
1611{
1612 return -EINVAL;
1613}
1614EXPORT_SYMBOL_GPL(sunvnet_set_mac_addr_common);
1615
1616void sunvnet_port_free_tx_bufs_common(struct vnet_port *port)
1617{
1618 struct vio_dring_state *dr;
1619 int i;
1620
1621 dr = &port->vio.drings[VIO_DRIVER_TX_RING];
1622
dc153f85 1623 if (!dr->base)
31762eaa
AY		 1624 return;
1625
1626 for (i = 0; i < VNET_TX_RING_SIZE; i++) {
1627 struct vio_net_desc *d;
1628 void *skb = port->tx_bufs[i].skb;
1629
1630 if (!skb)
1631 continue;
1632
1633 d = vio_dring_entry(dr, i);
1634
1635 ldc_unmap(port->vio.lp,
1636 port->tx_bufs[i].cookies,
1637 port->tx_bufs[i].ncookies);
1638 dev_kfree_skb(skb);
1639 port->tx_bufs[i].skb = NULL;
1640 d->hdr.state = VIO_DESC_FREE;
1641 }
1642 ldc_free_exp_dring(port->vio.lp, dr->base,
1643 (dr->entry_size * dr->num_entries),
1644 dr->cookies, dr->ncookies);
1645 dr->base = NULL;
1646 dr->entry_size = 0;
1647 dr->num_entries = 0;
1648 dr->pending = 0;
1649 dr->ncookies = 0;
1650}
1651EXPORT_SYMBOL_GPL(sunvnet_port_free_tx_bufs_common);
1652
867fa150 1653void vnet_port_reset(struct vnet_port *port)
31762eaa
AY		 1654{
1655 del_timer(&port->clean_timer);
1656 sunvnet_port_free_tx_bufs_common(port);
1657 port->rmtu = 0;
bc221a34 1658 port->tso = (port->vsw == 0); /* no tso in vsw, misbehaves in bridge */
31762eaa
AY		 1659 port->tsolen = 0;
1660}
867fa150 1661EXPORT_SYMBOL_GPL(vnet_port_reset);
31762eaa
AY		 1662
1663static int vnet_port_alloc_tx_ring(struct vnet_port *port)
1664{
1665 struct vio_dring_state *dr;
1666 unsigned long len, elen;
1667 int i, err, ncookies;
1668 void *dring;
1669
1670 dr = &port->vio.drings[VIO_DRIVER_TX_RING];
1671
1672 elen = sizeof(struct vio_net_desc) +
1673 sizeof(struct ldc_trans_cookie) * 2;
1674 if (vio_version_after_eq(&port->vio, 1, 7))
1675 elen += sizeof(struct vio_net_dext);
1676 len = VNET_TX_RING_SIZE * elen;
1677
1678 ncookies = VIO_MAX_RING_COOKIES;
1679 dring = ldc_alloc_exp_dring(port->vio.lp, len,
1680 dr->cookies, &ncookies,
1681 (LDC_MAP_SHADOW |
1682 LDC_MAP_DIRECT |
1683 LDC_MAP_RW));
1684 if (IS_ERR(dring)) {
1685 err = PTR_ERR(dring);
1686 goto err_out;
1687 }
1688
1689 dr->base = dring;
1690 dr->entry_size = elen;
1691 dr->num_entries = VNET_TX_RING_SIZE;
dc153f85
AY		 1692 dr->prod = 0;
1693 dr->cons = 0;
31762eaa
AY		 1694 port->start_cons = true; /* need an initial trigger */
1695 dr->pending = VNET_TX_RING_SIZE;
1696 dr->ncookies = ncookies;
1697
1698 for (i = 0; i < VNET_TX_RING_SIZE; ++i) {
1699 struct vio_net_desc *d;
1700
1701 d = vio_dring_entry(dr, i);
1702 d->hdr.state = VIO_DESC_FREE;
1703 }
1704 return 0;
1705
1706err_out:
1707 sunvnet_port_free_tx_bufs_common(port);
1708
1709 return err;
1710}
1711
1712#ifdef CONFIG_NET_POLL_CONTROLLER
67d0719f 1713void sunvnet_poll_controller_common(struct net_device *dev, struct vnet *vp)
31762eaa 1714{
31762eaa
AY		 1715 struct vnet_port *port;
1716 unsigned long flags;
1717
1718 spin_lock_irqsave(&vp->lock, flags);
1719 if (!list_empty(&vp->port_list)) {
1720 port = list_entry(vp->port_list.next, struct vnet_port, list);
1721 napi_schedule(&port->napi);
1722 }
1723 spin_unlock_irqrestore(&vp->lock, flags);
1724}
1725EXPORT_SYMBOL_GPL(sunvnet_poll_controller_common);
1726#endif
1727
1728void sunvnet_port_add_txq_common(struct vnet_port *port)
1729{
1730 struct vnet *vp = port->vp;
e1f1e5f7
SN		 1731 int smallest = 0;
1732 int i;
1733
1734 /* find the first least-used q
1735 * When there are more ldoms than q's, we start to
1736 * double up on ports per queue.
1737 */
1738 for (i = 0; i < VNET_MAX_TXQS; i++) {
1739 if (vp->q_used[i] == 0) {
1740 smallest = i;
1741 break;
1742 }
1743 if (vp->q_used[i] < vp->q_used[smallest])
1744 smallest = i;
1745 }
31762eaa 1746
e1f1e5f7
SN		 1747 vp->nports++;
1748 vp->q_used[smallest]++;
1749 port->q_index = smallest;
67d0719f
AY		 1750 netif_tx_wake_queue(netdev_get_tx_queue(VNET_PORT_TO_NET_DEVICE(port),
1751 port->q_index));
31762eaa
AY		 1752}
1753EXPORT_SYMBOL_GPL(sunvnet_port_add_txq_common);
1754
1755void sunvnet_port_rm_txq_common(struct vnet_port *port)
1756{
1757 port->vp->nports--;
67d0719f
AY		 1758 netif_tx_stop_queue(netdev_get_tx_queue(VNET_PORT_TO_NET_DEVICE(port),
1759 port->q_index));
e1f1e5f7
SN		 1760 port->vp->q_used[port->q_index]--;
1761 port->q_index = 0;
31762eaa
AY		 1762}
1763EXPORT_SYMBOL_GPL(sunvnet_port_rm_txq_common);
Ubuntu Artful kernel mirror