]> git.proxmox.com Git - mirror_qemu.git/blame - hw/net/igb_core.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
e1000e: Add ICR clearing by corresponding IMS bit
[mirror_qemu.git] / hw / net / igb_core.c
CommitLineData
3a977dee
AO		 1/*
2 * Core code for QEMU igb emulation
3 *
4 * Datasheet:
5 * https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/82576eg-gbe-datasheet.pdf
6 *
7 * Copyright (c) 2020-2023 Red Hat, Inc.
8 * Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com)
9 * Developed by Daynix Computing LTD (http://www.daynix.com)
10 *
11 * Authors:
12 * Akihiko Odaki <akihiko.odaki@daynix.com>
13 * Gal Hammmer <gal.hammer@sap.com>
14 * Marcel Apfelbaum <marcel.apfelbaum@gmail.com>
15 * Dmitry Fleytman <dmitry@daynix.com>
16 * Leonid Bloch <leonid@daynix.com>
17 * Yan Vugenfirer <yan@daynix.com>
18 *
19 * Based on work done by:
20 * Nir Peleg, Tutis Systems Ltd. for Qumranet Inc.
21 * Copyright (c) 2008 Qumranet
22 * Based on work done by:
23 * Copyright (c) 2007 Dan Aloni
24 * Copyright (c) 2004 Antony T Curtis
25 *
26 * This library is free software; you can redistribute it and/or
27 * modify it under the terms of the GNU Lesser General Public
28 * License as published by the Free Software Foundation; either
29 * version 2.1 of the License, or (at your option) any later version.
30 *
31 * This library is distributed in the hope that it will be useful,
32 * but WITHOUT ANY WARRANTY; without even the implied warranty of
33 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
34 * Lesser General Public License for more details.
35 *
36 * You should have received a copy of the GNU Lesser General Public
37 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
38 */
39
40#include "qemu/osdep.h"
41#include "qemu/log.h"
42#include "net/net.h"
43#include "net/tap.h"
44#include "hw/net/mii.h"
45#include "hw/pci/msi.h"
46#include "hw/pci/msix.h"
47#include "sysemu/runstate.h"
48
49#include "net_tx_pkt.h"
50#include "net_rx_pkt.h"
51
52#include "igb_common.h"
53#include "e1000x_common.h"
54#include "igb_core.h"
55
56#include "trace.h"
57
58#define E1000E_MAX_TX_FRAGS (64)
59
60union e1000_rx_desc_union {
61 struct e1000_rx_desc legacy;
62 union e1000_adv_rx_desc adv;
63};
64
65typedef struct IGBTxPktVmdqCallbackContext {
66 IGBCore *core;
67 NetClientState *nc;
68} IGBTxPktVmdqCallbackContext;
69
dc9ef1bf
AO		 70typedef struct L2Header {
71 struct eth_header eth;
6aa262f8 72 struct vlan_header vlan[2];
dc9ef1bf
AO		 73} L2Header;
74
3a9926d9
AO		 75typedef struct PTP2 {
76 uint8_t message_id_transport_specific;
77 uint8_t version_ptp;
78 uint16_t message_length;
79 uint8_t subdomain_number;
80 uint8_t reserved0;
81 uint16_t flags;
82 uint64_t correction;
83 uint8_t reserved1[5];
84 uint8_t source_communication_technology;
85 uint32_t source_uuid_lo;
86 uint16_t source_uuid_hi;
87 uint16_t source_port_id;
88 uint16_t sequence_id;
89 uint8_t control;
90 uint8_t log_message_period;
91} PTP2;
92
3a977dee
AO		 93static ssize_t
94igb_receive_internal(IGBCore *core, const struct iovec *iov, int iovcnt,
95 bool has_vnet, bool *external_tx);
96
5844562b 97static void igb_raise_interrupts(IGBCore *core, size_t index, uint32_t causes);
3a977dee
AO		 98static void igb_reset(IGBCore *core, bool sw);
99
100static inline void
101igb_raise_legacy_irq(IGBCore *core)
102{
103 trace_e1000e_irq_legacy_notify(true);
104 e1000x_inc_reg_if_not_full(core->mac, IAC);
105 pci_set_irq(core->owner, 1);
106}
107
108static inline void
109igb_lower_legacy_irq(IGBCore *core)
110{
111 trace_e1000e_irq_legacy_notify(false);
112 pci_set_irq(core->owner, 0);
113}
114
fe619f20 115static void igb_msix_notify(IGBCore *core, unsigned int cause)
3a977dee
AO		 116{
117 PCIDevice *dev = core->owner;
118 uint16_t vfn;
fe619f20
AO		 119 uint32_t effective_eiac;
120 unsigned int vector;
3a977dee 121
fe619f20 122 vfn = 8 - (cause + 2) / IGBVF_MSIX_VEC_NUM;
3a977dee
AO		 123 if (vfn < pcie_sriov_num_vfs(core->owner)) {
124 dev = pcie_sriov_get_vf_at_index(core->owner, vfn);
125 assert(dev);
fe619f20
AO		 126 vector = (cause + 2) % IGBVF_MSIX_VEC_NUM;
127 } else if (cause >= IGB_MSIX_VEC_NUM) {
3a977dee
AO		 128 qemu_log_mask(LOG_GUEST_ERROR,
129 "igb: Tried to use vector unavailable for PF");
130 return;
fe619f20
AO		 131 } else {
132 vector = cause;
3a977dee
AO		 133 }
134
135 msix_notify(dev, vector);
fe619f20
AO		 136
137 trace_e1000e_irq_icr_clear_eiac(core->mac[EICR], core->mac[EIAC]);
138 effective_eiac = core->mac[EIAC] & BIT(cause);
139 core->mac[EICR] &= ~effective_eiac;
3a977dee
AO		 140}
141
142static inline void
143igb_intrmgr_rearm_timer(IGBIntrDelayTimer *timer)
144{
145 int64_t delay_ns = (int64_t) timer->core->mac[timer->delay_reg] *
146 timer->delay_resolution_ns;
147
148 trace_e1000e_irq_rearm_timer(timer->delay_reg << 2, delay_ns);
149
150 timer_mod(timer->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + delay_ns);
151
152 timer->running = true;
153}
154
155static void
156igb_intmgr_timer_resume(IGBIntrDelayTimer *timer)
157{
158 if (timer->running) {
159 igb_intrmgr_rearm_timer(timer);
160 }
161}
162
163static void
164igb_intmgr_timer_pause(IGBIntrDelayTimer *timer)
165{
166 if (timer->running) {
167 timer_del(timer->timer);
168 }
169}
170
171static void
172igb_intrmgr_on_msix_throttling_timer(void *opaque)
173{
174 IGBIntrDelayTimer *timer = opaque;
175 int idx = timer - &timer->core->eitr[0];
176
177 timer->running = false;
178
179 trace_e1000e_irq_msix_notify_postponed_vec(idx);
180 igb_msix_notify(timer->core, idx);
181}
182
183static void
184igb_intrmgr_initialize_all_timers(IGBCore *core, bool create)
185{
186 int i;
187
188 for (i = 0; i < IGB_INTR_NUM; i++) {
189 core->eitr[i].core = core;
190 core->eitr[i].delay_reg = EITR0 + i;
191 core->eitr[i].delay_resolution_ns = E1000_INTR_DELAY_NS_RES;
192 }
193
194 if (!create) {
195 return;
196 }
197
198 for (i = 0; i < IGB_INTR_NUM; i++) {
199 core->eitr[i].timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
200 igb_intrmgr_on_msix_throttling_timer,
201 &core->eitr[i]);
202 }
203}
204
205static void
206igb_intrmgr_resume(IGBCore *core)
207{
208 int i;
209
210 for (i = 0; i < IGB_INTR_NUM; i++) {
211 igb_intmgr_timer_resume(&core->eitr[i]);
212 }
213}
214
215static void
216igb_intrmgr_pause(IGBCore *core)
217{
218 int i;
219
220 for (i = 0; i < IGB_INTR_NUM; i++) {
221 igb_intmgr_timer_pause(&core->eitr[i]);
222 }
223}
224
225static void
226igb_intrmgr_reset(IGBCore *core)
227{
228 int i;
229
230 for (i = 0; i < IGB_INTR_NUM; i++) {
231 if (core->eitr[i].running) {
232 timer_del(core->eitr[i].timer);
233 igb_intrmgr_on_msix_throttling_timer(&core->eitr[i]);
234 }
235 }
236}
237
238static void
239igb_intrmgr_pci_unint(IGBCore *core)
240{
241 int i;
242
243 for (i = 0; i < IGB_INTR_NUM; i++) {
244 timer_free(core->eitr[i].timer);
245 }
246}
247
248static void
249igb_intrmgr_pci_realize(IGBCore *core)
250{
251 igb_intrmgr_initialize_all_timers(core, true);
252}
253
254static inline bool
255igb_rx_csum_enabled(IGBCore *core)
256{
257 return (core->mac[RXCSUM] & E1000_RXCSUM_PCSD) ? false : true;
258}
259
260static inline bool
261igb_rx_use_legacy_descriptor(IGBCore *core)
262{
263 /*
264 * TODO: If SRRCTL[n],DESCTYPE = 000b, the 82576 uses the legacy Rx
265 * descriptor.
266 */
267 return false;
268}
269
270static inline bool
271igb_rss_enabled(IGBCore *core)
272{
273 return (core->mac[MRQC] & 3) == E1000_MRQC_ENABLE_RSS_MQ &&
274 !igb_rx_csum_enabled(core) &&
275 !igb_rx_use_legacy_descriptor(core);
276}
277
278typedef struct E1000E_RSSInfo_st {
279 bool enabled;
280 uint32_t hash;
281 uint32_t queue;
282 uint32_t type;
283} E1000E_RSSInfo;
284
285static uint32_t
286igb_rss_get_hash_type(IGBCore *core, struct NetRxPkt *pkt)
287{
288 bool hasip4, hasip6;
289 EthL4HdrProto l4hdr_proto;
290
291 assert(igb_rss_enabled(core));
292
293 net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
294
295 if (hasip4) {
296 trace_e1000e_rx_rss_ip4(l4hdr_proto, core->mac[MRQC],
297 E1000_MRQC_EN_TCPIPV4(core->mac[MRQC]),
298 E1000_MRQC_EN_IPV4(core->mac[MRQC]));
299
300 if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP &&
301 E1000_MRQC_EN_TCPIPV4(core->mac[MRQC])) {
302 return E1000_MRQ_RSS_TYPE_IPV4TCP;
303 }
304
abc9a29d
AO		 305 if (l4hdr_proto == ETH_L4_HDR_PROTO_UDP &&
306 (core->mac[MRQC] & E1000_MRQC_RSS_FIELD_IPV4_UDP)) {
307 return E1000_MRQ_RSS_TYPE_IPV4UDP;
308 }
309
3a977dee
AO		 310 if (E1000_MRQC_EN_IPV4(core->mac[MRQC])) {
311 return E1000_MRQ_RSS_TYPE_IPV4;
312 }
313 } else if (hasip6) {
314 eth_ip6_hdr_info *ip6info = net_rx_pkt_get_ip6_info(pkt);
315
316 bool ex_dis = core->mac[RFCTL] & E1000_RFCTL_IPV6_EX_DIS;
317 bool new_ex_dis = core->mac[RFCTL] & E1000_RFCTL_NEW_IPV6_EXT_DIS;
318
319 /*
320 * Following two traces must not be combined because resulting
321 * event will have 11 arguments totally and some trace backends
322 * (at least "ust") have limitation of maximum 10 arguments per
323 * event. Events with more arguments fail to compile for
324 * backends like these.
325 */
326 trace_e1000e_rx_rss_ip6_rfctl(core->mac[RFCTL]);
327 trace_e1000e_rx_rss_ip6(ex_dis, new_ex_dis, l4hdr_proto,
328 ip6info->has_ext_hdrs,
329 ip6info->rss_ex_dst_valid,
330 ip6info->rss_ex_src_valid,
331 core->mac[MRQC],
5052fc9e 332 E1000_MRQC_EN_TCPIPV6EX(core->mac[MRQC]),
3a977dee
AO		 333 E1000_MRQC_EN_IPV6EX(core->mac[MRQC]),
334 E1000_MRQC_EN_IPV6(core->mac[MRQC]));
335
336 if ((!ex_dis || !ip6info->has_ext_hdrs) &&
337 (!new_ex_dis || !(ip6info->rss_ex_dst_valid ||
338 ip6info->rss_ex_src_valid))) {
339
340 if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP &&
5052fc9e
AO		 341 E1000_MRQC_EN_TCPIPV6EX(core->mac[MRQC])) {
342 return E1000_MRQ_RSS_TYPE_IPV6TCPEX;
3a977dee
AO		 343 }
344
abc9a29d
AO		 345 if (l4hdr_proto == ETH_L4_HDR_PROTO_UDP &&
346 (core->mac[MRQC] & E1000_MRQC_RSS_FIELD_IPV6_UDP)) {
347 return E1000_MRQ_RSS_TYPE_IPV6UDP;
348 }
349
3a977dee
AO		 350 if (E1000_MRQC_EN_IPV6EX(core->mac[MRQC])) {
351 return E1000_MRQ_RSS_TYPE_IPV6EX;
352 }
353
354 }
355
356 if (E1000_MRQC_EN_IPV6(core->mac[MRQC])) {
357 return E1000_MRQ_RSS_TYPE_IPV6;
358 }
359
360 }
361
362 return E1000_MRQ_RSS_TYPE_NONE;
363}
364
365static uint32_t
366igb_rss_calc_hash(IGBCore *core, struct NetRxPkt *pkt, E1000E_RSSInfo *info)
367{
368 NetRxPktRssType type;
369
370 assert(igb_rss_enabled(core));
371
372 switch (info->type) {
373 case E1000_MRQ_RSS_TYPE_IPV4:
374 type = NetPktRssIpV4;
375 break;
376 case E1000_MRQ_RSS_TYPE_IPV4TCP:
377 type = NetPktRssIpV4Tcp;
378 break;
5052fc9e 379 case E1000_MRQ_RSS_TYPE_IPV6TCPEX:
3a977dee
AO		 380 type = NetPktRssIpV6TcpEx;
381 break;
382 case E1000_MRQ_RSS_TYPE_IPV6:
383 type = NetPktRssIpV6;
384 break;
385 case E1000_MRQ_RSS_TYPE_IPV6EX:
386 type = NetPktRssIpV6Ex;
387 break;
abc9a29d
AO		 388 case E1000_MRQ_RSS_TYPE_IPV4UDP:
389 type = NetPktRssIpV4Udp;
390 break;
391 case E1000_MRQ_RSS_TYPE_IPV6UDP:
392 type = NetPktRssIpV6Udp;
393 break;
3a977dee
AO		 394 default:
395 assert(false);
396 return 0;
397 }
398
399 return net_rx_pkt_calc_rss_hash(pkt, type, (uint8_t *) &core->mac[RSSRK]);
400}
401
402static void
403igb_rss_parse_packet(IGBCore *core, struct NetRxPkt *pkt, bool tx,
404 E1000E_RSSInfo *info)
405{
406 trace_e1000e_rx_rss_started();
407
408 if (tx || !igb_rss_enabled(core)) {
409 info->enabled = false;
410 info->hash = 0;
411 info->queue = 0;
412 info->type = 0;
413 trace_e1000e_rx_rss_disabled();
414 return;
415 }
416
417 info->enabled = true;
418
419 info->type = igb_rss_get_hash_type(core, pkt);
420
421 trace_e1000e_rx_rss_type(info->type);
422
423 if (info->type == E1000_MRQ_RSS_TYPE_NONE) {
424 info->hash = 0;
425 info->queue = 0;
426 return;
427 }
428
429 info->hash = igb_rss_calc_hash(core, pkt, info);
430 info->queue = E1000_RSS_QUEUE(&core->mac[RETA], info->hash);
431}
432
fba7c3b7
SY		 433static void
434igb_tx_insert_vlan(IGBCore *core, uint16_t qn, struct igb_tx *tx,
435 uint16_t vlan, bool insert_vlan)
436{
437 if (core->mac[MRQC] & 1) {
438 uint16_t pool = qn % IGB_NUM_VM_POOLS;
439
440 if (core->mac[VMVIR0 + pool] & E1000_VMVIR_VLANA_DEFAULT) {
441 /* always insert default VLAN */
442 insert_vlan = true;
443 vlan = core->mac[VMVIR0 + pool] & 0xffff;
444 } else if (core->mac[VMVIR0 + pool] & E1000_VMVIR_VLANA_NEVER) {
445 insert_vlan = false;
446 }
447 }
448
e2097167 449 if (insert_vlan) {
fba7c3b7
SY		 450 net_tx_pkt_setup_vlan_header_ex(tx->tx_pkt, vlan,
451 core->mac[VET] & 0xffff);
452 }
453}
454
3a977dee
AO		 455static bool
456igb_setup_tx_offloads(IGBCore *core, struct igb_tx *tx)
457{
f199b13b
AO		 458 uint32_t idx = (tx->first_olinfo_status >> 4) & 1;
459
212f7b1d 460 if (tx->first_cmd_type_len & E1000_ADVTXD_DCMD_TSE) {
4847dabf 461 uint32_t mss = tx->ctx[idx].mss_l4len_idx >> E1000_ADVTXD_MSS_SHIFT;
212f7b1d 462 if (!net_tx_pkt_build_vheader(tx->tx_pkt, true, true, mss)) {
3a977dee
AO		 463 return false;
464 }
465
466 net_tx_pkt_update_ip_checksums(tx->tx_pkt);
467 e1000x_inc_reg_if_not_full(core->mac, TSCTC);
468 return true;
469 }
470
f199b13b
AO		 471 if ((tx->first_olinfo_status & E1000_ADVTXD_POTS_TXSM) &&
472 !((tx->ctx[idx].type_tucmd_mlhl & E1000_ADVTXD_TUCMD_L4T_SCTP) ?
473 net_tx_pkt_update_sctp_checksum(tx->tx_pkt) :
474 net_tx_pkt_build_vheader(tx->tx_pkt, false, true, 0))) {
475 return false;
3a977dee
AO		 476 }
477
212f7b1d 478 if (tx->first_olinfo_status & E1000_ADVTXD_POTS_IXSM) {
3a977dee
AO		 479 net_tx_pkt_update_ip_hdr_checksum(tx->tx_pkt);
480 }
481
482 return true;
483}
484
485static void igb_tx_pkt_mac_callback(void *core,
486 const struct iovec *iov,
487 int iovcnt,
488 const struct iovec *virt_iov,
489 int virt_iovcnt)
490{
491 igb_receive_internal(core, virt_iov, virt_iovcnt, true, NULL);
492}
493
494static void igb_tx_pkt_vmdq_callback(void *opaque,
495 const struct iovec *iov,
496 int iovcnt,
497 const struct iovec *virt_iov,
498 int virt_iovcnt)
499{
500 IGBTxPktVmdqCallbackContext *context = opaque;
501 bool external_tx;
502
503 igb_receive_internal(context->core, virt_iov, virt_iovcnt, true,
504 &external_tx);
505
506 if (external_tx) {
507 if (context->core->has_vnet) {
508 qemu_sendv_packet(context->nc, virt_iov, virt_iovcnt);
509 } else {
510 qemu_sendv_packet(context->nc, iov, iovcnt);
511 }
512 }
513}
514
515/* TX Packets Switching (7.10.3.6) */
516static bool igb_tx_pkt_switch(IGBCore *core, struct igb_tx *tx,
517 NetClientState *nc)
518{
519 IGBTxPktVmdqCallbackContext context;
520
521 /* TX switching is only used to serve VM to VM traffic. */
522 if (!(core->mac[MRQC] & 1)) {
523 goto send_out;
524 }
525
526 /* TX switching requires DTXSWC.Loopback_en bit enabled. */
527 if (!(core->mac[DTXSWC] & E1000_DTXSWC_VMDQ_LOOPBACK_EN)) {
528 goto send_out;
529 }
530
531 context.core = core;
532 context.nc = nc;
533
534 return net_tx_pkt_send_custom(tx->tx_pkt, false,
535 igb_tx_pkt_vmdq_callback, &context);
536
537send_out:
538 return net_tx_pkt_send(tx->tx_pkt, nc);
539}
540
541static bool
542igb_tx_pkt_send(IGBCore *core, struct igb_tx *tx, int queue_index)
543{
544 int target_queue = MIN(core->max_queue_num, queue_index);
545 NetClientState *queue = qemu_get_subqueue(core->owner_nic, target_queue);
546
547 if (!igb_setup_tx_offloads(core, tx)) {
548 return false;
549 }
550
551 net_tx_pkt_dump(tx->tx_pkt);
552
553 if ((core->phy[MII_BMCR] & MII_BMCR_LOOPBACK) ||
554 ((core->mac[RCTL] & E1000_RCTL_LBM_MAC) == E1000_RCTL_LBM_MAC)) {
555 return net_tx_pkt_send_custom(tx->tx_pkt, false,
556 igb_tx_pkt_mac_callback, core);
557 } else {
558 return igb_tx_pkt_switch(core, tx, queue);
559 }
560}
561
562static void
7581baed 563igb_on_tx_done_update_stats(IGBCore *core, struct NetTxPkt *tx_pkt, int qn)
3a977dee
AO		 564{
565 static const int PTCregs[6] = { PTC64, PTC127, PTC255, PTC511,
566 PTC1023, PTC1522 };
567
568 size_t tot_len = net_tx_pkt_get_total_len(tx_pkt) + 4;
569
570 e1000x_increase_size_stats(core->mac, PTCregs, tot_len);
571 e1000x_inc_reg_if_not_full(core->mac, TPT);
572 e1000x_grow_8reg_if_not_full(core->mac, TOTL, tot_len);
573
574 switch (net_tx_pkt_get_packet_type(tx_pkt)) {
575 case ETH_PKT_BCAST:
576 e1000x_inc_reg_if_not_full(core->mac, BPTC);
577 break;
578 case ETH_PKT_MCAST:
579 e1000x_inc_reg_if_not_full(core->mac, MPTC);
580 break;
581 case ETH_PKT_UCAST:
582 break;
583 default:
584 g_assert_not_reached();
585 }
586
8d689f6a
TC		 587 e1000x_inc_reg_if_not_full(core->mac, GPTC);
588 e1000x_grow_8reg_if_not_full(core->mac, GOTCL, tot_len);
7581baed
SY		 589
590 if (core->mac[MRQC] & 1) {
591 uint16_t pool = qn % IGB_NUM_VM_POOLS;
592
593 core->mac[PVFGOTC0 + (pool * 64)] += tot_len;
594 core->mac[PVFGPTC0 + (pool * 64)]++;
595 }
3a977dee
AO		 596}
597
598static void
599igb_process_tx_desc(IGBCore *core,
f4fdaf00 600 PCIDevice *dev,
3a977dee
AO		 601 struct igb_tx *tx,
602 union e1000_adv_tx_desc *tx_desc,
603 int queue_index)
604{
605 struct e1000_adv_tx_context_desc *tx_ctx_desc;
606 uint32_t cmd_type_len;
212f7b1d 607 uint32_t idx;
3a977dee
AO		 608 uint64_t buffer_addr;
609 uint16_t length;
610
611 cmd_type_len = le32_to_cpu(tx_desc->read.cmd_type_len);
612
613 if (cmd_type_len & E1000_ADVTXD_DCMD_DEXT) {
614 if ((cmd_type_len & E1000_ADVTXD_DTYP_DATA) ==
615 E1000_ADVTXD_DTYP_DATA) {
616 /* advanced transmit data descriptor */
617 if (tx->first) {
212f7b1d
AO		 618 tx->first_cmd_type_len = cmd_type_len;
619 tx->first_olinfo_status = le32_to_cpu(tx_desc->read.olinfo_status);
3a977dee
AO		 620 tx->first = false;
621 }
622 } else if ((cmd_type_len & E1000_ADVTXD_DTYP_CTXT) ==
623 E1000_ADVTXD_DTYP_CTXT) {
624 /* advanced transmit context descriptor */
625 tx_ctx_desc = (struct e1000_adv_tx_context_desc *)tx_desc;
212f7b1d
AO		 626 idx = (le32_to_cpu(tx_ctx_desc->mss_l4len_idx) >> 4) & 1;
627 tx->ctx[idx].vlan_macip_lens = le32_to_cpu(tx_ctx_desc->vlan_macip_lens);
628 tx->ctx[idx].seqnum_seed = le32_to_cpu(tx_ctx_desc->seqnum_seed);
629 tx->ctx[idx].type_tucmd_mlhl = le32_to_cpu(tx_ctx_desc->type_tucmd_mlhl);
630 tx->ctx[idx].mss_l4len_idx = le32_to_cpu(tx_ctx_desc->mss_l4len_idx);
3a977dee
AO		 631 return;
632 } else {
633 /* unknown descriptor type */
634 return;
635 }
636 } else {
637 /* legacy descriptor */
638
639 /* TODO: Implement a support for legacy descriptors (7.2.2.1). */
640 }
641
642 buffer_addr = le64_to_cpu(tx_desc->read.buffer_addr);
643 length = cmd_type_len & 0xFFFF;
644
645 if (!tx->skip_cp) {
a51db580
AO		 646 if (!net_tx_pkt_add_raw_fragment_pci(tx->tx_pkt, dev,
647 buffer_addr, length)) {
3a977dee
AO		 648 tx->skip_cp = true;
649 }
650 }
651
652 if (cmd_type_len & E1000_TXD_CMD_EOP) {
653 if (!tx->skip_cp && net_tx_pkt_parse(tx->tx_pkt)) {
fba7c3b7
SY		 654 idx = (tx->first_olinfo_status >> 4) & 1;
655 igb_tx_insert_vlan(core, queue_index, tx,
4847dabf 656 tx->ctx[idx].vlan_macip_lens >> IGB_TX_FLAGS_VLAN_SHIFT,
5d92e88a 657 !!(tx->first_cmd_type_len & E1000_TXD_CMD_VLE));
fba7c3b7 658
3dfc616e
AO		 659 if ((tx->first_cmd_type_len & E1000_ADVTXD_MAC_TSTAMP) &&
660 (core->mac[TSYNCTXCTL] & E1000_TSYNCTXCTL_ENABLED) &&
661 !(core->mac[TSYNCTXCTL] & E1000_TSYNCTXCTL_VALID)) {
662 core->mac[TSYNCTXCTL] |= E1000_TSYNCTXCTL_VALID;
663 e1000x_timestamp(core->mac, core->timadj, TXSTMPL, TXSTMPH);
664 }
665
3a977dee 666 if (igb_tx_pkt_send(core, tx, queue_index)) {
7581baed 667 igb_on_tx_done_update_stats(core, tx->tx_pkt, queue_index);
3a977dee
AO		 668 }
669 }
670
671 tx->first = true;
672 tx->skip_cp = false;
a51db580 673 net_tx_pkt_reset(tx->tx_pkt, net_tx_pkt_unmap_frag_pci, dev);
3a977dee
AO		 674 }
675}
676
677static uint32_t igb_tx_wb_eic(IGBCore *core, int queue_idx)
678{
679 uint32_t n, ent = 0;
680
681 n = igb_ivar_entry_tx(queue_idx);
682 ent = (core->mac[IVAR0 + n / 4] >> (8 * (n % 4))) & 0xff;
683
684 return (ent & E1000_IVAR_VALID) ? BIT(ent & 0x1f) : 0;
685}
686
687static uint32_t igb_rx_wb_eic(IGBCore *core, int queue_idx)
688{
689 uint32_t n, ent = 0;
690
691 n = igb_ivar_entry_rx(queue_idx);
692 ent = (core->mac[IVAR0 + n / 4] >> (8 * (n % 4))) & 0xff;
693
694 return (ent & E1000_IVAR_VALID) ? BIT(ent & 0x1f) : 0;
695}
696
697typedef struct E1000E_RingInfo_st {
698 int dbah;
699 int dbal;
700 int dlen;
701 int dh;
702 int dt;
703 int idx;
704} E1000E_RingInfo;
705
706static inline bool
707igb_ring_empty(IGBCore *core, const E1000E_RingInfo *r)
708{
709 return core->mac[r->dh] == core->mac[r->dt] ||
710 core->mac[r->dt] >= core->mac[r->dlen] / E1000_RING_DESC_LEN;
711}
712
713static inline uint64_t
714igb_ring_base(IGBCore *core, const E1000E_RingInfo *r)
715{
716 uint64_t bah = core->mac[r->dbah];
717 uint64_t bal = core->mac[r->dbal];
718
719 return (bah << 32) + bal;
720}
721
722static inline uint64_t
723igb_ring_head_descr(IGBCore *core, const E1000E_RingInfo *r)
724{
725 return igb_ring_base(core, r) + E1000_RING_DESC_LEN * core->mac[r->dh];
726}
727
728static inline void
729igb_ring_advance(IGBCore *core, const E1000E_RingInfo *r, uint32_t count)
730{
731 core->mac[r->dh] += count;
732
733 if (core->mac[r->dh] * E1000_RING_DESC_LEN >= core->mac[r->dlen]) {
734 core->mac[r->dh] = 0;
735 }
736}
737
738static inline uint32_t
739igb_ring_free_descr_num(IGBCore *core, const E1000E_RingInfo *r)
740{
741 trace_e1000e_ring_free_space(r->idx, core->mac[r->dlen],
742 core->mac[r->dh], core->mac[r->dt]);
743
744 if (core->mac[r->dh] <= core->mac[r->dt]) {
745 return core->mac[r->dt] - core->mac[r->dh];
746 }
747
748 if (core->mac[r->dh] > core->mac[r->dt]) {
749 return core->mac[r->dlen] / E1000_RING_DESC_LEN +
750 core->mac[r->dt] - core->mac[r->dh];
751 }
752
753 g_assert_not_reached();
754 return 0;
755}
756
757static inline bool
758igb_ring_enabled(IGBCore *core, const E1000E_RingInfo *r)
759{
760 return core->mac[r->dlen] > 0;
761}
762
763typedef struct IGB_TxRing_st {
764 const E1000E_RingInfo *i;
765 struct igb_tx *tx;
766} IGB_TxRing;
767
768static inline int
769igb_mq_queue_idx(int base_reg_idx, int reg_idx)
770{
771 return (reg_idx - base_reg_idx) / 16;
772}
773
774static inline void
775igb_tx_ring_init(IGBCore *core, IGB_TxRing *txr, int idx)
776{
777 static const E1000E_RingInfo i[IGB_NUM_QUEUES] = {
778 { TDBAH0, TDBAL0, TDLEN0, TDH0, TDT0, 0 },
779 { TDBAH1, TDBAL1, TDLEN1, TDH1, TDT1, 1 },
780 { TDBAH2, TDBAL2, TDLEN2, TDH2, TDT2, 2 },
781 { TDBAH3, TDBAL3, TDLEN3, TDH3, TDT3, 3 },
782 { TDBAH4, TDBAL4, TDLEN4, TDH4, TDT4, 4 },
783 { TDBAH5, TDBAL5, TDLEN5, TDH5, TDT5, 5 },
784 { TDBAH6, TDBAL6, TDLEN6, TDH6, TDT6, 6 },
785 { TDBAH7, TDBAL7, TDLEN7, TDH7, TDT7, 7 },
786 { TDBAH8, TDBAL8, TDLEN8, TDH8, TDT8, 8 },
787 { TDBAH9, TDBAL9, TDLEN9, TDH9, TDT9, 9 },
788 { TDBAH10, TDBAL10, TDLEN10, TDH10, TDT10, 10 },
789 { TDBAH11, TDBAL11, TDLEN11, TDH11, TDT11, 11 },
790 { TDBAH12, TDBAL12, TDLEN12, TDH12, TDT12, 12 },
791 { TDBAH13, TDBAL13, TDLEN13, TDH13, TDT13, 13 },
792 { TDBAH14, TDBAL14, TDLEN14, TDH14, TDT14, 14 },
793 { TDBAH15, TDBAL15, TDLEN15, TDH15, TDT15, 15 }
794 };
795
796 assert(idx < ARRAY_SIZE(i));
797
798 txr->i = &i[idx];
799 txr->tx = &core->tx[idx];
800}
801
802typedef struct E1000E_RxRing_st {
803 const E1000E_RingInfo *i;
804} E1000E_RxRing;
805
806static inline void
807igb_rx_ring_init(IGBCore *core, E1000E_RxRing *rxr, int idx)
808{
809 static const E1000E_RingInfo i[IGB_NUM_QUEUES] = {
810 { RDBAH0, RDBAL0, RDLEN0, RDH0, RDT0, 0 },
811 { RDBAH1, RDBAL1, RDLEN1, RDH1, RDT1, 1 },
812 { RDBAH2, RDBAL2, RDLEN2, RDH2, RDT2, 2 },
813 { RDBAH3, RDBAL3, RDLEN3, RDH3, RDT3, 3 },
814 { RDBAH4, RDBAL4, RDLEN4, RDH4, RDT4, 4 },
815 { RDBAH5, RDBAL5, RDLEN5, RDH5, RDT5, 5 },
816 { RDBAH6, RDBAL6, RDLEN6, RDH6, RDT6, 6 },
817 { RDBAH7, RDBAL7, RDLEN7, RDH7, RDT7, 7 },
818 { RDBAH8, RDBAL8, RDLEN8, RDH8, RDT8, 8 },
819 { RDBAH9, RDBAL9, RDLEN9, RDH9, RDT9, 9 },
820 { RDBAH10, RDBAL10, RDLEN10, RDH10, RDT10, 10 },
821 { RDBAH11, RDBAL11, RDLEN11, RDH11, RDT11, 11 },
822 { RDBAH12, RDBAL12, RDLEN12, RDH12, RDT12, 12 },
823 { RDBAH13, RDBAL13, RDLEN13, RDH13, RDT13, 13 },
824 { RDBAH14, RDBAL14, RDLEN14, RDH14, RDT14, 14 },
825 { RDBAH15, RDBAL15, RDLEN15, RDH15, RDT15, 15 }
826 };
827
828 assert(idx < ARRAY_SIZE(i));
829
830 rxr->i = &i[idx];
831}
832
833static uint32_t
834igb_txdesc_writeback(IGBCore *core, dma_addr_t base,
835 union e1000_adv_tx_desc *tx_desc,
836 const E1000E_RingInfo *txi)
837{
838 PCIDevice *d;
839 uint32_t cmd_type_len = le32_to_cpu(tx_desc->read.cmd_type_len);
840 uint64_t tdwba;
841
842 tdwba = core->mac[E1000_TDWBAL(txi->idx) >> 2];
843 tdwba |= (uint64_t)core->mac[E1000_TDWBAH(txi->idx) >> 2] << 32;
844
845 if (!(cmd_type_len & E1000_TXD_CMD_RS)) {
846 return 0;
847 }
848
849 d = pcie_sriov_get_vf_at_index(core->owner, txi->idx % 8);
850 if (!d) {
851 d = core->owner;
852 }
853
854 if (tdwba & 1) {
855 uint32_t buffer = cpu_to_le32(core->mac[txi->dh]);
856 pci_dma_write(d, tdwba & ~3, &buffer, sizeof(buffer));
857 } else {
858 uint32_t status = le32_to_cpu(tx_desc->wb.status) | E1000_TXD_STAT_DD;
859
860 tx_desc->wb.status = cpu_to_le32(status);
861 pci_dma_write(d, base + offsetof(union e1000_adv_tx_desc, wb),
862 &tx_desc->wb, sizeof(tx_desc->wb));
863 }
864
865 return igb_tx_wb_eic(core, txi->idx);
866}
867
3269ebb3
SY		 868static inline bool
869igb_tx_enabled(IGBCore *core, const E1000E_RingInfo *txi)
870{
871 bool vmdq = core->mac[MRQC] & 1;
872 uint16_t qn = txi->idx;
873 uint16_t pool = qn % IGB_NUM_VM_POOLS;
874
875 return (core->mac[TCTL] & E1000_TCTL_EN) &&
876 (!vmdq || core->mac[VFTE] & BIT(pool)) &&
877 (core->mac[TXDCTL0 + (qn * 16)] & E1000_TXDCTL_QUEUE_ENABLE);
878}
879
3a977dee
AO		 880static void
881igb_start_xmit(IGBCore *core, const IGB_TxRing *txr)
882{
883 PCIDevice *d;
884 dma_addr_t base;
885 union e1000_adv_tx_desc desc;
886 const E1000E_RingInfo *txi = txr->i;
887 uint32_t eic = 0;
888
3269ebb3 889 if (!igb_tx_enabled(core, txi)) {
3a977dee
AO		 890 trace_e1000e_tx_disabled();
891 return;
892 }
893
894 d = pcie_sriov_get_vf_at_index(core->owner, txi->idx % 8);
895 if (!d) {
896 d = core->owner;
897 }
898
899 while (!igb_ring_empty(core, txi)) {
900 base = igb_ring_head_descr(core, txi);
901
902 pci_dma_read(d, base, &desc, sizeof(desc));
903
904 trace_e1000e_tx_descr((void *)(intptr_t)desc.read.buffer_addr,
905 desc.read.cmd_type_len, desc.wb.status);
906
f4fdaf00 907 igb_process_tx_desc(core, d, txr->tx, &desc, txi->idx);
3a977dee
AO		 908 igb_ring_advance(core, txi, 1);
909 eic |= igb_txdesc_writeback(core, base, &desc, txi);
910 }
911
912 if (eic) {
5844562b
AO		 913 igb_raise_interrupts(core, EICR, eic);
914 igb_raise_interrupts(core, ICR, E1000_ICR_TXDW);
3a977dee 915 }
a51db580
AO		 916
917 net_tx_pkt_reset(txr->tx->tx_pkt, net_tx_pkt_unmap_frag_pci, d);
3a977dee
AO		 918}
919
920static uint32_t
921igb_rxbufsize(IGBCore *core, const E1000E_RingInfo *r)
922{
923 uint32_t srrctl = core->mac[E1000_SRRCTL(r->idx) >> 2];
924 uint32_t bsizepkt = srrctl & E1000_SRRCTL_BSIZEPKT_MASK;
925 if (bsizepkt) {
926 return bsizepkt << E1000_SRRCTL_BSIZEPKT_SHIFT;
927 }
928
929 return e1000x_rxbufsize(core->mac[RCTL]);
930}
931
932static bool
933igb_has_rxbufs(IGBCore *core, const E1000E_RingInfo *r, size_t total_size)
934{
935 uint32_t bufs = igb_ring_free_descr_num(core, r);
936 uint32_t bufsize = igb_rxbufsize(core, r);
937
938 trace_e1000e_rx_has_buffers(r->idx, bufs, total_size, bufsize);
939
940 return total_size <= bufs / (core->rx_desc_len / E1000_MIN_RX_DESC_LEN) *
941 bufsize;
942}
943
944void
945igb_start_recv(IGBCore *core)
946{
947 int i;
948
949 trace_e1000e_rx_start_recv();
950
951 for (i = 0; i <= core->max_queue_num; i++) {
952 qemu_flush_queued_packets(qemu_get_subqueue(core->owner_nic, i));
953 }
954}
955
956bool
957igb_can_receive(IGBCore *core)
958{
959 int i;
960
961 if (!e1000x_rx_ready(core->owner, core->mac)) {
962 return false;
963 }
964
965 for (i = 0; i < IGB_NUM_QUEUES; i++) {
966 E1000E_RxRing rxr;
3269ebb3
SY		 967 if (!(core->mac[RXDCTL0 + (i * 16)] & E1000_RXDCTL_QUEUE_ENABLE)) {
968 continue;
969 }
3a977dee
AO		 970
971 igb_rx_ring_init(core, &rxr, i);
972 if (igb_ring_enabled(core, rxr.i) && igb_has_rxbufs(core, rxr.i, 1)) {
973 trace_e1000e_rx_can_recv();
974 return true;
975 }
976 }
977
978 trace_e1000e_rx_can_recv_rings_full();
979 return false;
980}
981
982ssize_t
983igb_receive(IGBCore *core, const uint8_t *buf, size_t size)
984{
985 const struct iovec iov = {
986 .iov_base = (uint8_t *)buf,
987 .iov_len = size
988 };
989
990 return igb_receive_iov(core, &iov, 1);
991}
992
993static inline bool
994igb_rx_l3_cso_enabled(IGBCore *core)
995{
996 return !!(core->mac[RXCSUM] & E1000_RXCSUM_IPOFLD);
997}
998
999static inline bool
1000igb_rx_l4_cso_enabled(IGBCore *core)
1001{
1002 return !!(core->mac[RXCSUM] & E1000_RXCSUM_TUOFLD);
1003}
1004
bb97003e
AO		 1005static bool igb_rx_is_oversized(IGBCore *core, const struct eth_header *ehdr,
1006 size_t size, size_t vlan_num,
1007 bool lpe, uint16_t rlpml)
5f12d703 1008{
bb97003e
AO		 1009 size_t vlan_header_size = sizeof(struct vlan_header) * vlan_num;
1010 size_t header_size = sizeof(struct eth_header) + vlan_header_size;
1011 return lpe ? size + ETH_FCS_LEN > rlpml : size > header_size + ETH_MTU;
5f12d703
SY		 1012}
1013
3a9926d9
AO		 1014static uint16_t igb_receive_assign(IGBCore *core, const struct iovec *iov,
1015 size_t iovcnt, size_t iov_ofs,
1016 const L2Header *l2_header, size_t size,
1017 E1000E_RSSInfo *rss_info,
1018 uint16_t *etqf, bool *ts, bool *external_tx)
3a977dee
AO		 1019{
1020 static const int ta_shift[] = { 4, 3, 2, 0 };
dc9ef1bf 1021 const struct eth_header *ehdr = &l2_header->eth;
3a977dee
AO		 1022 uint32_t f, ra[2], *macp, rctl = core->mac[RCTL];
1023 uint16_t queues = 0;
5f12d703 1024 uint16_t oversized = 0;
6aa262f8 1025 size_t vlan_num = 0;
3a9926d9 1026 PTP2 ptp2;
bb97003e
AO		 1027 bool lpe;
1028 uint16_t rlpml;
3a977dee
AO		 1029 int i;
1030
1031 memset(rss_info, 0, sizeof(E1000E_RSSInfo));
3a9926d9 1032 *ts = false;
3a977dee
AO		 1033
1034 if (external_tx) {
1035 *external_tx = true;
1036 }
1037
6aa262f8
AO		 1038 if (core->mac[CTRL_EXT] & BIT(26)) {
1039 if (be16_to_cpu(ehdr->h_proto) == core->mac[VET] >> 16 &&
1040 be16_to_cpu(l2_header->vlan[0].h_proto) == (core->mac[VET] & 0xffff)) {
1041 vlan_num = 2;
1042 }
1043 } else {
1044 if (be16_to_cpu(ehdr->h_proto) == (core->mac[VET] & 0xffff)) {
1045 vlan_num = 1;
1046 }
1047 }
1048
bb97003e
AO		 1049 lpe = !!(core->mac[RCTL] & E1000_RCTL_LPE);
1050 rlpml = core->mac[RLPML];
1051 if (!(core->mac[RCTL] & E1000_RCTL_SBP) &&
1052 igb_rx_is_oversized(core, ehdr, size, vlan_num, lpe, rlpml)) {
1053 trace_e1000x_rx_oversized(size);
1054 return queues;
1055 }
1056
3a9926d9
AO		 1057 for (*etqf = 0; *etqf < 8; (*etqf)++) {
1058 if ((core->mac[ETQF0 + *etqf] & E1000_ETQF_FILTER_ENABLE) &&
1059 be16_to_cpu(ehdr->h_proto) == (core->mac[ETQF0 + *etqf] & E1000_ETQF_ETYPE_MASK)) {
1060 if ((core->mac[ETQF0 + *etqf] & E1000_ETQF_1588) &&
1061 (core->mac[TSYNCRXCTL] & E1000_TSYNCRXCTL_ENABLED) &&
1062 !(core->mac[TSYNCRXCTL] & E1000_TSYNCRXCTL_VALID) &&
1063 iov_to_buf(iov, iovcnt, iov_ofs + ETH_HLEN, &ptp2, sizeof(ptp2)) >= sizeof(ptp2) &&
1064 (ptp2.version_ptp & 15) == 2 &&
1065 ptp2.message_id_transport_specific == ((core->mac[TSYNCRXCFG] >> 8) & 255)) {
1066 e1000x_timestamp(core->mac, core->timadj, RXSTMPL, RXSTMPH);
1067 *ts = true;
1068 core->mac[TSYNCRXCTL] |= E1000_TSYNCRXCTL_VALID;
1069 core->mac[RXSATRL] = le32_to_cpu(ptp2.source_uuid_lo);
1070 core->mac[RXSATRH] = le16_to_cpu(ptp2.source_uuid_hi) |
1071 (le16_to_cpu(ptp2.sequence_id) << 16);
1072 }
1073 break;
1074 }
1075 }
1076
6aa262f8
AO		 1077 if (vlan_num &&
1078 !e1000x_rx_vlan_filter(core->mac, l2_header->vlan + vlan_num - 1)) {
e9e5b930 1079 return queues;
3a977dee
AO		 1080 }
1081
1082 if (core->mac[MRQC] & 1) {
1083 if (is_broadcast_ether_addr(ehdr->h_dest)) {
3269ebb3 1084 for (i = 0; i < IGB_NUM_VM_POOLS; i++) {
3a977dee
AO		 1085 if (core->mac[VMOLR0 + i] & E1000_VMOLR_BAM) {
1086 queues |= BIT(i);
1087 }
1088 }
1089 } else {
1090 for (macp = core->mac + RA; macp < core->mac + RA + 32; macp += 2) {
1091 if (!(macp[1] & E1000_RAH_AV)) {
1092 continue;
1093 }
1094 ra[0] = cpu_to_le32(macp[0]);
1095 ra[1] = cpu_to_le32(macp[1]);
1096 if (!memcmp(ehdr->h_dest, (uint8_t *)ra, ETH_ALEN)) {
1097 queues |= (macp[1] & E1000_RAH_POOL_MASK) / E1000_RAH_POOL_1;
1098 }
1099 }
1100
1101 for (macp = core->mac + RA2; macp < core->mac + RA2 + 16; macp += 2) {
1102 if (!(macp[1] & E1000_RAH_AV)) {
1103 continue;
1104 }
1105 ra[0] = cpu_to_le32(macp[0]);
1106 ra[1] = cpu_to_le32(macp[1]);
1107 if (!memcmp(ehdr->h_dest, (uint8_t *)ra, ETH_ALEN)) {
1108 queues |= (macp[1] & E1000_RAH_POOL_MASK) / E1000_RAH_POOL_1;
1109 }
1110 }
1111
1112 if (!queues) {
1113 macp = core->mac + (is_multicast_ether_addr(ehdr->h_dest) ? MTA : UTA);
1114
1115 f = ta_shift[(rctl >> E1000_RCTL_MO_SHIFT) & 3];
1116 f = (((ehdr->h_dest[5] << 8) | ehdr->h_dest[4]) >> f) & 0xfff;
1117 if (macp[f >> 5] & (1 << (f & 0x1f))) {
3269ebb3 1118 for (i = 0; i < IGB_NUM_VM_POOLS; i++) {
3a977dee
AO		 1119 if (core->mac[VMOLR0 + i] & E1000_VMOLR_ROMPE) {
1120 queues |= BIT(i);
1121 }
1122 }
1123 }
1124 } else if (is_unicast_ether_addr(ehdr->h_dest) && external_tx) {
1125 *external_tx = false;
1126 }
1127 }
1128
1129 if (e1000x_vlan_rx_filter_enabled(core->mac)) {
1130 uint16_t mask = 0;
1131
6aa262f8
AO		 1132 if (vlan_num) {
1133 uint16_t vid = be16_to_cpu(l2_header->vlan[vlan_num - 1].h_tci) & VLAN_VID_MASK;
1134
3a977dee
AO		 1135 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
1136 if ((core->mac[VLVF0 + i] & E1000_VLVF_VLANID_MASK) == vid &&
1137 (core->mac[VLVF0 + i] & E1000_VLVF_VLANID_ENABLE)) {
1138 uint32_t poolsel = core->mac[VLVF0 + i] & E1000_VLVF_POOLSEL_MASK;
1139 mask |= poolsel >> E1000_VLVF_POOLSEL_SHIFT;
1140 }
1141 }
1142 } else {
3269ebb3 1143 for (i = 0; i < IGB_NUM_VM_POOLS; i++) {
3a977dee
AO		 1144 if (core->mac[VMOLR0 + i] & E1000_VMOLR_AUPE) {
1145 mask |= BIT(i);
1146 }
1147 }
1148 }
1149
1150 queues &= mask;
1151 }
1152
1153 if (is_unicast_ether_addr(ehdr->h_dest) && !queues && !external_tx &&
1154 !(core->mac[VT_CTL] & E1000_VT_CTL_DISABLE_DEF_POOL)) {
1155 uint32_t def_pl = core->mac[VT_CTL] & E1000_VT_CTL_DEFAULT_POOL_MASK;
1156 queues = BIT(def_pl >> E1000_VT_CTL_DEFAULT_POOL_SHIFT);
1157 }
1158
3269ebb3 1159 queues &= core->mac[VFRE];
5f12d703
SY		 1160 if (queues) {
1161 for (i = 0; i < IGB_NUM_VM_POOLS; i++) {
bb97003e
AO		 1162 lpe = !!(core->mac[VMOLR0 + i] & E1000_VMOLR_LPE);
1163 rlpml = core->mac[VMOLR0 + i] & E1000_VMOLR_RLPML_MASK;
1164 if ((queues & BIT(i)) &&
1165 igb_rx_is_oversized(core, ehdr, size, vlan_num,
1166 lpe, rlpml)) {
5f12d703
SY		 1167 oversized |= BIT(i);
1168 }
1169 }
1170 /* 8.19.37 increment ROC if packet is oversized for all queues */
1171 if (oversized == queues) {
1172 trace_e1000x_rx_oversized(size);
1173 e1000x_inc_reg_if_not_full(core->mac, ROC);
1174 }
1175 queues &= ~oversized;
1176 }
1177
1178 if (queues) {
1179 igb_rss_parse_packet(core, core->rx_pkt,
1180 external_tx != NULL, rss_info);
3c2e0a68 1181 /* Sec 8.26.1: PQn = VFn + VQn*8 */
5f12d703 1182 if (rss_info->queue & 1) {
3c2e0a68
SY		 1183 for (i = 0; i < IGB_NUM_VM_POOLS; i++) {
1184 if ((queues & BIT(i)) &&
1185 (core->mac[VMOLR0 + i] & E1000_VMOLR_RSSE)) {
1186 queues |= BIT(i + IGB_NUM_VM_POOLS);
1187 queues &= ~BIT(i);
1188 }
1189 }
5f12d703 1190 }
3a977dee
AO		 1191 }
1192 } else {
e9e5b930 1193 bool accepted = e1000x_rx_group_filter(core->mac, ehdr);
3a977dee
AO		 1194 if (!accepted) {
1195 for (macp = core->mac + RA2; macp < core->mac + RA2 + 16; macp += 2) {
1196 if (!(macp[1] & E1000_RAH_AV)) {
1197 continue;
1198 }
1199 ra[0] = cpu_to_le32(macp[0]);
1200 ra[1] = cpu_to_le32(macp[1]);
1201 if (!memcmp(ehdr->h_dest, (uint8_t *)ra, ETH_ALEN)) {
1202 trace_e1000x_rx_flt_ucast_match((int)(macp - core->mac - RA2) / 2,
1203 MAC_ARG(ehdr->h_dest));
1204
1205 accepted = true;
1206 break;
1207 }
1208 }
1209 }
1210
1211 if (accepted) {
1212 igb_rss_parse_packet(core, core->rx_pkt, false, rss_info);
1213 queues = BIT(rss_info->queue);
1214 }
1215 }
1216
1217 return queues;
1218}
1219
1220static inline void
1221igb_read_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc,
1222 hwaddr *buff_addr)
1223{
1224 *buff_addr = le64_to_cpu(desc->buffer_addr);
1225}
1226
1227static inline void
1228igb_read_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
1229 hwaddr *buff_addr)
1230{
1231 *buff_addr = le64_to_cpu(desc->read.pkt_addr);
1232}
1233
1234static inline void
1235igb_read_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc,
1236 hwaddr *buff_addr)
1237{
1238 if (igb_rx_use_legacy_descriptor(core)) {
1239 igb_read_lgcy_rx_descr(core, &desc->legacy, buff_addr);
1240 } else {
1241 igb_read_adv_rx_descr(core, &desc->adv, buff_addr);
1242 }
1243}
1244
1245static void
1246igb_verify_csum_in_sw(IGBCore *core,
1247 struct NetRxPkt *pkt,
1248 uint32_t *status_flags,
1249 EthL4HdrProto l4hdr_proto)
1250{
1251 bool csum_valid;
1252 uint32_t csum_error;
1253
1254 if (igb_rx_l3_cso_enabled(core)) {
1255 if (!net_rx_pkt_validate_l3_csum(pkt, &csum_valid)) {
1256 trace_e1000e_rx_metadata_l3_csum_validation_failed();
1257 } else {
1258 csum_error = csum_valid ? 0 : E1000_RXDEXT_STATERR_IPE;
1259 *status_flags |= E1000_RXD_STAT_IPCS | csum_error;
1260 }
1261 } else {
1262 trace_e1000e_rx_metadata_l3_cso_disabled();
1263 }
1264
1265 if (!igb_rx_l4_cso_enabled(core)) {
1266 trace_e1000e_rx_metadata_l4_cso_disabled();
1267 return;
1268 }
1269
1270 if (!net_rx_pkt_validate_l4_csum(pkt, &csum_valid)) {
1271 trace_e1000e_rx_metadata_l4_csum_validation_failed();
1272 return;
1273 }
1274
1275 csum_error = csum_valid ? 0 : E1000_RXDEXT_STATERR_TCPE;
1276 *status_flags |= E1000_RXD_STAT_TCPCS | csum_error;
1277
1278 if (l4hdr_proto == ETH_L4_HDR_PROTO_UDP) {
1279 *status_flags |= E1000_RXD_STAT_UDPCS;
1280 }
1281}
1282
1283static void
1284igb_build_rx_metadata(IGBCore *core,
1285 struct NetRxPkt *pkt,
1286 bool is_eop,
3a9926d9 1287 const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts,
3a977dee
AO		 1288 uint16_t *pkt_info, uint16_t *hdr_info,
1289 uint32_t *rss,
1290 uint32_t *status_flags,
1291 uint16_t *ip_id,
1292 uint16_t *vlan_tag)
1293{
1294 struct virtio_net_hdr *vhdr;
907209e3 1295 bool hasip4, hasip6, csum_valid;
3a977dee 1296 EthL4HdrProto l4hdr_proto;
3a977dee
AO		 1297
1298 *status_flags = E1000_RXD_STAT_DD;
1299
1300 /* No additional metadata needed for non-EOP descriptors */
1301 /* TODO: EOP apply only to status so don't skip whole function. */
1302 if (!is_eop) {
1303 goto func_exit;
1304 }
1305
1306 *status_flags |= E1000_RXD_STAT_EOP;
1307
1308 net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
1309 trace_e1000e_rx_metadata_protocols(hasip4, hasip6, l4hdr_proto);
1310
1311 /* VLAN state */
1312 if (net_rx_pkt_is_vlan_stripped(pkt)) {
1313 *status_flags |= E1000_RXD_STAT_VP;
1314 *vlan_tag = cpu_to_le16(net_rx_pkt_get_vlan_tag(pkt));
1315 trace_e1000e_rx_metadata_vlan(*vlan_tag);
1316 }
1317
1318 /* Packet parsing results */
1319 if ((core->mac[RXCSUM] & E1000_RXCSUM_PCSD) != 0) {
1320 if (rss_info->enabled) {
1321 *rss = cpu_to_le32(rss_info->hash);
1322 trace_igb_rx_metadata_rss(*rss);
1323 }
1324 } else if (hasip4) {
1325 *status_flags |= E1000_RXD_STAT_IPIDV;
1326 *ip_id = cpu_to_le16(net_rx_pkt_get_ip_id(pkt));
1327 trace_e1000e_rx_metadata_ip_id(*ip_id);
1328 }
1329
1330 if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP && net_rx_pkt_is_tcp_ack(pkt)) {
1331 *status_flags |= E1000_RXD_STAT_ACK;
1332 trace_e1000e_rx_metadata_ack();
1333 }
1334
ed447c60
AO		 1335 if (pkt_info) {
1336 *pkt_info = rss_info->enabled ? rss_info->type : 0;
3a977dee 1337
3a9926d9
AO		 1338 if (etqf < 8) {
1339 *pkt_info |= (BIT(11) | etqf) << 4;
1340 } else {
1341 if (hasip4) {
1342 *pkt_info |= E1000_ADVRXD_PKT_IP4;
1343 }
3a977dee 1344
3a9926d9
AO		 1345 if (hasip6) {
1346 *pkt_info |= E1000_ADVRXD_PKT_IP6;
1347 }
3a977dee 1348
3a9926d9
AO		 1349 switch (l4hdr_proto) {
1350 case ETH_L4_HDR_PROTO_TCP:
1351 *pkt_info |= E1000_ADVRXD_PKT_TCP;
1352 break;
ed447c60 1353
3a9926d9
AO		 1354 case ETH_L4_HDR_PROTO_UDP:
1355 *pkt_info |= E1000_ADVRXD_PKT_UDP;
1356 break;
ed447c60 1357
3a9926d9
AO		 1358 case ETH_L4_HDR_PROTO_SCTP:
1359 *pkt_info |= E1000_ADVRXD_PKT_SCTP;
1360 break;
907209e3 1361
3a9926d9
AO		 1362 default:
1363 break;
1364 }
ed447c60 1365 }
3a977dee
AO		 1366 }
1367
1368 if (hdr_info) {
1369 *hdr_info = 0;
1370 }
1371
3a9926d9
AO		 1372 if (ts) {
1373 *status_flags |= BIT(16);
1374 }
1375
3a977dee
AO		 1376 /* RX CSO information */
1377 if (hasip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_XSUM_DIS)) {
1378 trace_e1000e_rx_metadata_ipv6_sum_disabled();
1379 goto func_exit;
1380 }
1381
1382 vhdr = net_rx_pkt_get_vhdr(pkt);
1383
1384 if (!(vhdr->flags & VIRTIO_NET_HDR_F_DATA_VALID) &&
1385 !(vhdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)) {
1386 trace_e1000e_rx_metadata_virthdr_no_csum_info();
1387 igb_verify_csum_in_sw(core, pkt, status_flags, l4hdr_proto);
1388 goto func_exit;
1389 }
1390
1391 if (igb_rx_l3_cso_enabled(core)) {
1392 *status_flags |= hasip4 ? E1000_RXD_STAT_IPCS : 0;
1393 } else {
1394 trace_e1000e_rx_metadata_l3_cso_disabled();
1395 }
1396
1397 if (igb_rx_l4_cso_enabled(core)) {
1398 switch (l4hdr_proto) {
907209e3
AO		 1399 case ETH_L4_HDR_PROTO_SCTP:
1400 if (!net_rx_pkt_validate_l4_csum(pkt, &csum_valid)) {
1401 trace_e1000e_rx_metadata_l4_csum_validation_failed();
1402 goto func_exit;
1403 }
1404 if (!csum_valid) {
1405 *status_flags |= E1000_RXDEXT_STATERR_TCPE;
1406 }
1407 /* fall through */
3a977dee
AO		 1408 case ETH_L4_HDR_PROTO_TCP:
1409 *status_flags |= E1000_RXD_STAT_TCPCS;
1410 break;
1411
1412 case ETH_L4_HDR_PROTO_UDP:
1413 *status_flags |= E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS;
1414 break;
1415
1416 default:
a09cc21e 1417 break;
3a977dee
AO		 1418 }
1419 } else {
1420 trace_e1000e_rx_metadata_l4_cso_disabled();
1421 }
1422
3a977dee 1423func_exit:
d5241351 1424 trace_e1000e_rx_metadata_status_flags(*status_flags);
3a977dee
AO		 1425 *status_flags = cpu_to_le32(*status_flags);
1426}
1427
1428static inline void
1429igb_write_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc,
1430 struct NetRxPkt *pkt,
3a9926d9 1431 const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts,
3a977dee
AO		 1432 uint16_t length)
1433{
1434 uint32_t status_flags, rss;
1435 uint16_t ip_id;
1436
1437 assert(!rss_info->enabled);
1438 desc->length = cpu_to_le16(length);
1439 desc->csum = 0;
1440
1441 igb_build_rx_metadata(core, pkt, pkt != NULL,
3a9926d9 1442 rss_info, etqf, ts,
3a977dee
AO		 1443 NULL, NULL, &rss,
1444 &status_flags, &ip_id,
1445 &desc->special);
1446 desc->errors = (uint8_t) (le32_to_cpu(status_flags) >> 24);
1447 desc->status = (uint8_t) le32_to_cpu(status_flags);
1448}
1449
1450static inline void
1451igb_write_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
1452 struct NetRxPkt *pkt,
3a9926d9 1453 const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts,
3a977dee
AO		 1454 uint16_t length)
1455{
1456 memset(&desc->wb, 0, sizeof(desc->wb));
1457
1458 desc->wb.upper.length = cpu_to_le16(length);
1459
1460 igb_build_rx_metadata(core, pkt, pkt != NULL,
3a9926d9 1461 rss_info, etqf, ts,
3a977dee
AO		 1462 &desc->wb.lower.lo_dword.pkt_info,
1463 &desc->wb.lower.lo_dword.hdr_info,
1464 &desc->wb.lower.hi_dword.rss,
1465 &desc->wb.upper.status_error,
1466 &desc->wb.lower.hi_dword.csum_ip.ip_id,
1467 &desc->wb.upper.vlan);
1468}
1469
1470static inline void
1471igb_write_rx_descr(IGBCore *core, union e1000_rx_desc_union *desc,
3a9926d9
AO		 1472 struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info,
1473 uint16_t etqf, bool ts, uint16_t length)
3a977dee
AO		 1474{
1475 if (igb_rx_use_legacy_descriptor(core)) {
3a9926d9
AO		 1476 igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info,
1477 etqf, ts, length);
3a977dee 1478 } else {
3a9926d9
AO		 1479 igb_write_adv_rx_descr(core, &desc->adv, pkt, rss_info,
1480 etqf, ts, length);
3a977dee
AO		 1481 }
1482}
1483
1484static inline void
1485igb_pci_dma_write_rx_desc(IGBCore *core, PCIDevice *dev, dma_addr_t addr,
1486 union e1000_rx_desc_union *desc, dma_addr_t len)
1487{
1488 if (igb_rx_use_legacy_descriptor(core)) {
1489 struct e1000_rx_desc *d = &desc->legacy;
1490 size_t offset = offsetof(struct e1000_rx_desc, status);
1491 uint8_t status = d->status;
1492
1493 d->status &= ~E1000_RXD_STAT_DD;
1494 pci_dma_write(dev, addr, desc, len);
1495
1496 if (status & E1000_RXD_STAT_DD) {
1497 d->status = status;
1498 pci_dma_write(dev, addr + offset, &status, sizeof(status));
1499 }
1500 } else {
1501 union e1000_adv_rx_desc *d = &desc->adv;
1502 size_t offset =
1503 offsetof(union e1000_adv_rx_desc, wb.upper.status_error);
1504 uint32_t status = d->wb.upper.status_error;
1505
1506 d->wb.upper.status_error &= ~E1000_RXD_STAT_DD;
1507 pci_dma_write(dev, addr, desc, len);
1508
1509 if (status & E1000_RXD_STAT_DD) {
1510 d->wb.upper.status_error = status;
1511 pci_dma_write(dev, addr + offset, &status, sizeof(status));
1512 }
1513 }
1514}
1515
1516static void
1517igb_write_to_rx_buffers(IGBCore *core,
1518 PCIDevice *d,
1519 hwaddr ba,
1520 uint16_t *written,
1521 const char *data,
1522 dma_addr_t data_len)
1523{
1524 trace_igb_rx_desc_buff_write(ba, *written, data, data_len);
1525 pci_dma_write(d, ba + *written, data, data_len);
1526 *written += data_len;
1527}
1528
1529static void
7581baed 1530igb_update_rx_stats(IGBCore *core, const E1000E_RingInfo *rxi,
f3f9b726 1531 size_t pkt_size, size_t pkt_fcs_size)
3a977dee 1532{
f3f9b726
AO		 1533 eth_pkt_types_e pkt_type = net_rx_pkt_get_packet_type(core->rx_pkt);
1534 e1000x_update_rx_total_stats(core->mac, pkt_type, pkt_size, pkt_fcs_size);
7581baed
SY		 1535
1536 if (core->mac[MRQC] & 1) {
1537 uint16_t pool = rxi->idx % IGB_NUM_VM_POOLS;
1538
f3f9b726 1539 core->mac[PVFGORC0 + (pool * 64)] += pkt_size + 4;
7581baed 1540 core->mac[PVFGPRC0 + (pool * 64)]++;
f3f9b726 1541 if (pkt_type == ETH_PKT_MCAST) {
7581baed
SY		 1542 core->mac[PVFMPRC0 + (pool * 64)]++;
1543 }
1544 }
3a977dee
AO		 1545}
1546
1547static inline bool
1548igb_rx_descr_threshold_hit(IGBCore *core, const E1000E_RingInfo *rxi)
1549{
1550 return igb_ring_free_descr_num(core, rxi) ==
1551 ((core->mac[E1000_SRRCTL(rxi->idx) >> 2] >> 20) & 31) * 16;
1552}
1553
1554static void
1555igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
1556 const E1000E_RxRing *rxr,
3a9926d9
AO		 1557 const E1000E_RSSInfo *rss_info,
1558 uint16_t etqf, bool ts)
3a977dee
AO		 1559{
1560 PCIDevice *d;
1561 dma_addr_t base;
1562 union e1000_rx_desc_union desc;
1563 size_t desc_size;
1564 size_t desc_offset = 0;
1565 size_t iov_ofs = 0;
1566
1567 struct iovec *iov = net_rx_pkt_get_iovec(pkt);
1568 size_t size = net_rx_pkt_get_total_len(pkt);
1569 size_t total_size = size + e1000x_fcs_len(core->mac);
1570 const E1000E_RingInfo *rxi = rxr->i;
1571 size_t bufsize = igb_rxbufsize(core, rxi);
1572
1573 d = pcie_sriov_get_vf_at_index(core->owner, rxi->idx % 8);
1574 if (!d) {
1575 d = core->owner;
1576 }
1577
1578 do {
1579 hwaddr ba;
1580 uint16_t written = 0;
1581 bool is_last = false;
1582
1583 desc_size = total_size - desc_offset;
1584
1585 if (desc_size > bufsize) {
1586 desc_size = bufsize;
1587 }
1588
1589 if (igb_ring_empty(core, rxi)) {
1590 return;
1591 }
1592
1593 base = igb_ring_head_descr(core, rxi);
1594
1595 pci_dma_read(d, base, &desc, core->rx_desc_len);
1596
1597 trace_e1000e_rx_descr(rxi->idx, base, core->rx_desc_len);
1598
1599 igb_read_rx_descr(core, &desc, &ba);
1600
1601 if (ba) {
1602 if (desc_offset < size) {
1603 static const uint32_t fcs_pad;
1604 size_t iov_copy;
1605 size_t copy_size = size - desc_offset;
1606 if (copy_size > bufsize) {
1607 copy_size = bufsize;
1608 }
1609
1610 /* Copy packet payload */
1611 while (copy_size) {
1612 iov_copy = MIN(copy_size, iov->iov_len - iov_ofs);
1613
1614 igb_write_to_rx_buffers(core, d, ba, &written,
1615 iov->iov_base + iov_ofs, iov_copy);
1616
1617 copy_size -= iov_copy;
1618 iov_ofs += iov_copy;
1619 if (iov_ofs == iov->iov_len) {
1620 iov++;
1621 iov_ofs = 0;
1622 }
1623 }
1624
1625 if (desc_offset + desc_size >= total_size) {
1626 /* Simulate FCS checksum presence in the last descriptor */
1627 igb_write_to_rx_buffers(core, d, ba, &written,
1628 (const char *) &fcs_pad, e1000x_fcs_len(core->mac));
1629 }
1630 }
1631 } else { /* as per intel docs; skip descriptors with null buf addr */
1632 trace_e1000e_rx_null_descriptor();
1633 }
1634 desc_offset += desc_size;
1635 if (desc_offset >= total_size) {
1636 is_last = true;
1637 }
1638
1639 igb_write_rx_descr(core, &desc, is_last ? core->rx_pkt : NULL,
3a9926d9 1640 rss_info, etqf, ts, written);
3a977dee
AO		 1641 igb_pci_dma_write_rx_desc(core, d, base, &desc, core->rx_desc_len);
1642
1643 igb_ring_advance(core, rxi, core->rx_desc_len / E1000_MIN_RX_DESC_LEN);
1644
1645 } while (desc_offset < total_size);
1646
7581baed 1647 igb_update_rx_stats(core, rxi, size, total_size);
3a977dee
AO		 1648}
1649
fba7c3b7
SY		 1650static bool
1651igb_rx_strip_vlan(IGBCore *core, const E1000E_RingInfo *rxi)
1652{
1653 if (core->mac[MRQC] & 1) {
1654 uint16_t pool = rxi->idx % IGB_NUM_VM_POOLS;
1655 /* Sec 7.10.3.8: CTRL.VME is ignored, only VMOLR/RPLOLR is used */
1656 return (net_rx_pkt_get_packet_type(core->rx_pkt) == ETH_PKT_MCAST) ?
1657 core->mac[RPLOLR] & E1000_RPLOLR_STRVLAN :
1658 core->mac[VMOLR0 + pool] & E1000_VMOLR_STRVLAN;
1659 }
1660
1661 return e1000x_vlan_enabled(core->mac);
1662}
1663
3a977dee
AO		 1664static inline void
1665igb_rx_fix_l4_csum(IGBCore *core, struct NetRxPkt *pkt)
1666{
1667 struct virtio_net_hdr *vhdr = net_rx_pkt_get_vhdr(pkt);
1668
1669 if (vhdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1670 net_rx_pkt_fix_l4_csum(pkt);
1671 }
1672}
1673
1674ssize_t
1675igb_receive_iov(IGBCore *core, const struct iovec *iov, int iovcnt)
1676{
1677 return igb_receive_internal(core, iov, iovcnt, core->has_vnet, NULL);
1678}
1679
1680static ssize_t
1681igb_receive_internal(IGBCore *core, const struct iovec *iov, int iovcnt,
1682 bool has_vnet, bool *external_tx)
1683{
3a977dee 1684 uint16_t queues = 0;
5c30aea4 1685 uint32_t causes = 0;
5844562b 1686 uint32_t ecauses = 0;
dc9ef1bf
AO		 1687 union {
1688 L2Header l2_header;
1689 uint8_t octets[ETH_ZLEN];
1690 } buf;
3a977dee 1691 struct iovec min_iov;
3a977dee
AO		 1692 size_t size, orig_size;
1693 size_t iov_ofs = 0;
1694 E1000E_RxRing rxr;
1695 E1000E_RSSInfo rss_info;
3a9926d9
AO		 1696 uint16_t etqf;
1697 bool ts;
3a977dee 1698 size_t total_size;
7e64a9ca 1699 int strip_vlan_index;
3a977dee
AO		 1700 int i;
1701
1702 trace_e1000e_rx_receive_iov(iovcnt);
1703
1704 if (external_tx) {
1705 *external_tx = true;
1706 }
1707
1708 if (!e1000x_hw_rx_enabled(core->mac)) {
1709 return -1;
1710 }
1711
1712 /* Pull virtio header in */
1713 if (has_vnet) {
1714 net_rx_pkt_set_vhdr_iovec(core->rx_pkt, iov, iovcnt);
1715 iov_ofs = sizeof(struct virtio_net_hdr);
1716 } else {
1717 net_rx_pkt_unset_vhdr(core->rx_pkt);
1718 }
1719
3a977dee
AO		 1720 orig_size = iov_size(iov, iovcnt);
1721 size = orig_size - iov_ofs;
1722
1723 /* Pad to minimum Ethernet frame length */
dc9ef1bf
AO		 1724 if (size < sizeof(buf)) {
1725 iov_to_buf(iov, iovcnt, iov_ofs, &buf, size);
1726 memset(&buf.octets[size], 0, sizeof(buf) - size);
3a977dee 1727 e1000x_inc_reg_if_not_full(core->mac, RUC);
dc9ef1bf
AO		 1728 min_iov.iov_base = &buf;
1729 min_iov.iov_len = size = sizeof(buf);
3a977dee
AO		 1730 iovcnt = 1;
1731 iov = &min_iov;
1732 iov_ofs = 0;
dc9ef1bf
AO		 1733 } else {
1734 iov_to_buf(iov, iovcnt, iov_ofs, &buf, sizeof(buf.l2_header));
3a977dee
AO		 1735 }
1736
dc9ef1bf
AO		 1737 net_rx_pkt_set_packet_type(core->rx_pkt,
1738 get_eth_packet_type(&buf.l2_header.eth));
2f0fa232 1739 net_rx_pkt_set_protocols(core->rx_pkt, iov, iovcnt, iov_ofs);
3a977dee 1740
3a9926d9
AO		 1741 queues = igb_receive_assign(core, iov, iovcnt, iov_ofs,
1742 &buf.l2_header, size,
1743 &rss_info, &etqf, &ts, external_tx);
3a977dee
AO		 1744 if (!queues) {
1745 trace_e1000e_rx_flt_dropped();
1746 return orig_size;
1747 }
1748
3a977dee 1749 for (i = 0; i < IGB_NUM_QUEUES; i++) {
3269ebb3
SY		 1750 if (!(queues & BIT(i)) ||
1751 !(core->mac[RXDCTL0 + (i * 16)] & E1000_RXDCTL_QUEUE_ENABLE)) {
3a977dee
AO		 1752 continue;
1753 }
1754
1755 igb_rx_ring_init(core, &rxr, i);
1756
7e64a9ca
AO		 1757 if (!igb_rx_strip_vlan(core, rxr.i)) {
1758 strip_vlan_index = -1;
1759 } else if (core->mac[CTRL_EXT] & BIT(26)) {
1760 strip_vlan_index = 1;
1761 } else {
1762 strip_vlan_index = 0;
1763 }
1764
fba7c3b7 1765 net_rx_pkt_attach_iovec_ex(core->rx_pkt, iov, iovcnt, iov_ofs,
7e64a9ca
AO		 1766 strip_vlan_index,
1767 core->mac[VET] & 0xffff,
1768 core->mac[VET] >> 16);
fba7c3b7
SY		 1769
1770 total_size = net_rx_pkt_get_total_len(core->rx_pkt) +
1771 e1000x_fcs_len(core->mac);
1772
3a977dee 1773 if (!igb_has_rxbufs(core, rxr.i, total_size)) {
5c30aea4 1774 causes |= E1000_ICS_RXO;
3a977dee
AO		 1775 trace_e1000e_rx_not_written_to_guest(rxr.i->idx);
1776 continue;
1777 }
1778
5c30aea4 1779 causes |= E1000_ICR_RXDW;
3a977dee
AO		 1780
1781 igb_rx_fix_l4_csum(core, core->rx_pkt);
3a9926d9 1782 igb_write_packet_to_guest(core, core->rx_pkt, &rxr, &rss_info, etqf, ts);
3a977dee
AO		 1783
1784 /* Check if receive descriptor minimum threshold hit */
1785 if (igb_rx_descr_threshold_hit(core, rxr.i)) {
5c30aea4 1786 causes |= E1000_ICS_RXDMT0;
3a977dee
AO		 1787 }
1788
5844562b 1789 ecauses |= igb_rx_wb_eic(core, rxr.i->idx);
3a977dee
AO		 1790
1791 trace_e1000e_rx_written_to_guest(rxr.i->idx);
1792 }
1793
5c30aea4 1794 trace_e1000e_rx_interrupt_set(causes);
5844562b
AO		 1795 igb_raise_interrupts(core, EICR, ecauses);
1796 igb_raise_interrupts(core, ICR, causes);
3a977dee
AO		 1797
1798 return orig_size;
1799}
1800
1801static inline bool
1802igb_have_autoneg(IGBCore *core)
1803{
1804 return core->phy[MII_BMCR] & MII_BMCR_AUTOEN;
1805}
1806
1807static void igb_update_flowctl_status(IGBCore *core)
1808{
1809 if (igb_have_autoneg(core) && core->phy[MII_BMSR] & MII_BMSR_AN_COMP) {
1810 trace_e1000e_link_autoneg_flowctl(true);
1811 core->mac[CTRL] |= E1000_CTRL_TFCE | E1000_CTRL_RFCE;
1812 } else {
1813 trace_e1000e_link_autoneg_flowctl(false);
1814 }
1815}
1816
1817static inline void
1818igb_link_down(IGBCore *core)
1819{
1820 e1000x_update_regs_on_link_down(core->mac, core->phy);
1821 igb_update_flowctl_status(core);
1822}
1823
1824static inline void
1825igb_set_phy_ctrl(IGBCore *core, uint16_t val)
1826{
1827 /* bits 0-5 reserved; MII_BMCR_[ANRESTART,RESET] are self clearing */
1828 core->phy[MII_BMCR] = val & ~(0x3f | MII_BMCR_RESET | MII_BMCR_ANRESTART);
1829
1830 if ((val & MII_BMCR_ANRESTART) && igb_have_autoneg(core)) {
1831 e1000x_restart_autoneg(core->mac, core->phy, core->autoneg_timer);
1832 }
1833}
1834
1835void igb_core_set_link_status(IGBCore *core)
1836{
1837 NetClientState *nc = qemu_get_queue(core->owner_nic);
1838 uint32_t old_status = core->mac[STATUS];
1839
1840 trace_e1000e_link_status_changed(nc->link_down ? false : true);
1841
1842 if (nc->link_down) {
1843 e1000x_update_regs_on_link_down(core->mac, core->phy);
1844 } else {
1845 if (igb_have_autoneg(core) &&
1846 !(core->phy[MII_BMSR] & MII_BMSR_AN_COMP)) {
1847 e1000x_restart_autoneg(core->mac, core->phy,
1848 core->autoneg_timer);
1849 } else {
1850 e1000x_update_regs_on_link_up(core->mac, core->phy);
1851 igb_start_recv(core);
1852 }
1853 }
1854
1855 if (core->mac[STATUS] != old_status) {
5844562b 1856 igb_raise_interrupts(core, ICR, E1000_ICR_LSC);
3a977dee
AO		 1857 }
1858}
1859
1860static void
1861igb_set_ctrl(IGBCore *core, int index, uint32_t val)
1862{
1863 trace_e1000e_core_ctrl_write(index, val);
1864
1865 /* RST is self clearing */
1866 core->mac[CTRL] = val & ~E1000_CTRL_RST;
1867 core->mac[CTRL_DUP] = core->mac[CTRL];
1868
1869 trace_e1000e_link_set_params(
1870 !!(val & E1000_CTRL_ASDE),
1871 (val & E1000_CTRL_SPD_SEL) >> E1000_CTRL_SPD_SHIFT,
1872 !!(val & E1000_CTRL_FRCSPD),
1873 !!(val & E1000_CTRL_FRCDPX),
1874 !!(val & E1000_CTRL_RFCE),
1875 !!(val & E1000_CTRL_TFCE));
1876
1877 if (val & E1000_CTRL_RST) {
1878 trace_e1000e_core_ctrl_sw_reset();
1879 igb_reset(core, true);
1880 }
1881
1882 if (val & E1000_CTRL_PHY_RST) {
1883 trace_e1000e_core_ctrl_phy_reset();
1884 core->mac[STATUS] |= E1000_STATUS_PHYRA;
1885 }
1886}
1887
1888static void
1889igb_set_rfctl(IGBCore *core, int index, uint32_t val)
1890{
1891 trace_e1000e_rx_set_rfctl(val);
1892
1893 if (!(val & E1000_RFCTL_ISCSI_DIS)) {
1894 trace_e1000e_wrn_iscsi_filtering_not_supported();
1895 }
1896
1897 if (!(val & E1000_RFCTL_NFSW_DIS)) {
1898 trace_e1000e_wrn_nfsw_filtering_not_supported();
1899 }
1900
1901 if (!(val & E1000_RFCTL_NFSR_DIS)) {
1902 trace_e1000e_wrn_nfsr_filtering_not_supported();
1903 }
1904
1905 core->mac[RFCTL] = val;
1906}
1907
1908static void
1909igb_calc_rxdesclen(IGBCore *core)
1910{
1911 if (igb_rx_use_legacy_descriptor(core)) {
1912 core->rx_desc_len = sizeof(struct e1000_rx_desc);
1913 } else {
1914 core->rx_desc_len = sizeof(union e1000_adv_rx_desc);
1915 }
1916 trace_e1000e_rx_desc_len(core->rx_desc_len);
1917}
1918
1919static void
1920igb_set_rx_control(IGBCore *core, int index, uint32_t val)
1921{
1922 core->mac[RCTL] = val;
1923 trace_e1000e_rx_set_rctl(core->mac[RCTL]);
1924
1925 if (val & E1000_RCTL_DTYP_MASK) {
1926 qemu_log_mask(LOG_GUEST_ERROR,
1927 "igb: RCTL.DTYP must be zero for compatibility");
1928 }
1929
1930 if (val & E1000_RCTL_EN) {
1931 igb_calc_rxdesclen(core);
1932 igb_start_recv(core);
1933 }
1934}
1935
3a977dee
AO		 1936static inline bool
1937igb_postpone_interrupt(IGBIntrDelayTimer *timer)
1938{
1939 if (timer->running) {
1940 trace_e1000e_irq_postponed_by_xitr(timer->delay_reg << 2);
1941
1942 return true;
1943 }
1944
1945 if (timer->core->mac[timer->delay_reg] != 0) {
1946 igb_intrmgr_rearm_timer(timer);
1947 }
1948
1949 return false;
1950}
1951
1952static inline bool
1953igb_eitr_should_postpone(IGBCore *core, int idx)
1954{
1955 return igb_postpone_interrupt(&core->eitr[idx]);
1956}
1957
5844562b 1958static void igb_send_msix(IGBCore *core, uint32_t causes)
3a977dee 1959{
3a977dee
AO		 1960 int vector;
1961
1962 for (vector = 0; vector < IGB_INTR_NUM; ++vector) {
1963 if ((causes & BIT(vector)) && !igb_eitr_should_postpone(core, vector)) {
1964
1965 trace_e1000e_irq_msix_notify_vec(vector);
1966 igb_msix_notify(core, vector);
3a977dee
AO		 1967 }
1968 }
1969}
1970
1971static inline void
1972igb_fix_icr_asserted(IGBCore *core)
1973{
1974 core->mac[ICR] &= ~E1000_ICR_ASSERTED;
1975 if (core->mac[ICR]) {
1976 core->mac[ICR] |= E1000_ICR_ASSERTED;
1977 }
1978
1979 trace_e1000e_irq_fix_icr_asserted(core->mac[ICR]);
1980}
1981
5844562b 1982static void igb_raise_interrupts(IGBCore *core, size_t index, uint32_t causes)
3a977dee 1983{
5844562b
AO		 1984 uint32_t old_causes = core->mac[ICR] & core->mac[IMS];
1985 uint32_t old_ecauses = core->mac[EICR] & core->mac[EIMS];
1986 uint32_t raised_causes;
1987 uint32_t raised_ecauses;
3a977dee
AO		 1988 uint32_t int_alloc;
1989
5844562b
AO		 1990 trace_e1000e_irq_set(index << 2,
1991 core->mac[index], core->mac[index] | causes);
1992
1993 core->mac[index] |= causes;
3a977dee 1994
191e8bde 1995 if (core->mac[GPIE] & E1000_GPIE_MSIX_MODE) {
5844562b
AO		 1996 raised_causes = core->mac[ICR] & core->mac[IMS] & ~old_causes;
1997
1998 if (raised_causes & E1000_ICR_DRSTA) {
1999 int_alloc = core->mac[IVAR_MISC] & 0xff;
2000 if (int_alloc & E1000_IVAR_VALID) {
2001 core->mac[EICR] |= BIT(int_alloc & 0x1f);
3a977dee 2002 }
5844562b
AO		 2003 }
2004 /* Check if other bits (excluding the TCP Timer) are enabled. */
2005 if (raised_causes & ~E1000_ICR_DRSTA) {
2006 int_alloc = (core->mac[IVAR_MISC] >> 8) & 0xff;
2007 if (int_alloc & E1000_IVAR_VALID) {
2008 core->mac[EICR] |= BIT(int_alloc & 0x1f);
3a977dee 2009 }
3a977dee
AO		 2010 }
2011
5844562b
AO		 2012 raised_ecauses = core->mac[EICR] & core->mac[EIMS] & ~old_ecauses;
2013 if (!raised_ecauses) {
2014 return;
3a977dee 2015 }
5844562b
AO		 2016
2017 igb_send_msix(core, raised_ecauses);
3a977dee
AO		 2018 } else {
2019 igb_fix_icr_asserted(core);
2020
5844562b
AO		 2021 raised_causes = core->mac[ICR] & core->mac[IMS] & ~old_causes;
2022 if (!raised_causes) {
2023 return;
3a977dee
AO		 2024 }
2025
5844562b 2026 core->mac[EICR] |= (raised_causes & E1000_ICR_DRSTA) | E1000_EICR_OTHER;
3a977dee 2027
191e8bde 2028 if (msix_enabled(core->owner)) {
5844562b
AO		 2029 trace_e1000e_irq_msix_notify_vec(0);
2030 msix_notify(core->owner, 0);
191e8bde 2031 } else if (msi_enabled(core->owner)) {
5844562b
AO		 2032 trace_e1000e_irq_msi_notify(raised_causes);
2033 msi_notify(core->owner, 0);
3a977dee 2034 } else {
5844562b 2035 igb_raise_legacy_irq(core);
3a977dee
AO		 2036 }
2037 }
2038}
2039
5844562b 2040static void igb_lower_interrupts(IGBCore *core, size_t index, uint32_t causes)
3a977dee 2041{
5844562b
AO		 2042 trace_e1000e_irq_clear(index << 2,
2043 core->mac[index], core->mac[index] & ~causes);
2044
2045 core->mac[index] &= ~causes;
3a977dee 2046
5844562b
AO		 2047 trace_e1000e_irq_pending_interrupts(core->mac[ICR] & core->mac[IMS],
2048 core->mac[ICR], core->mac[IMS]);
3a977dee 2049
5844562b
AO		 2050 if (!(core->mac[ICR] & core->mac[IMS]) &&
2051 !(core->mac[GPIE] & E1000_GPIE_MSIX_MODE)) {
2052 core->mac[EICR] &= ~E1000_EICR_OTHER;
3a977dee 2053
5844562b
AO		 2054 if (!msix_enabled(core->owner) && !msi_enabled(core->owner)) {
2055 igb_lower_legacy_irq(core);
2056 }
2057 }
3a977dee
AO		 2058}
2059
2060static void igb_set_eics(IGBCore *core, int index, uint32_t val)
2061{
2062 bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE);
5844562b 2063 uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK;
3a977dee
AO		 2064
2065 trace_igb_irq_write_eics(val, msix);
5844562b 2066 igb_raise_interrupts(core, EICR, val & mask);
3a977dee
AO		 2067}
2068
2069static void igb_set_eims(IGBCore *core, int index, uint32_t val)
2070{
2071 bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE);
5844562b 2072 uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK;
3a977dee
AO		 2073
2074 trace_igb_irq_write_eims(val, msix);
5844562b 2075 igb_raise_interrupts(core, EIMS, val & mask);
3a977dee
AO		 2076}
2077
3a977dee
AO		 2078static void mailbox_interrupt_to_vf(IGBCore *core, uint16_t vfn)
2079{
2080 uint32_t ent = core->mac[VTIVAR_MISC + vfn];
5844562b 2081 uint32_t causes;
3a977dee
AO		 2082
2083 if ((ent & E1000_IVAR_VALID)) {
5844562b
AO		 2084 causes = (ent & 0x3) << (22 - vfn * IGBVF_MSIX_VEC_NUM);
2085 igb_raise_interrupts(core, EICR, causes);
3a977dee
AO		 2086 }
2087}
2088
2089static void mailbox_interrupt_to_pf(IGBCore *core)
2090{
5844562b 2091 igb_raise_interrupts(core, ICR, E1000_ICR_VMMB);
3a977dee
AO		 2092}
2093
2094static void igb_set_pfmailbox(IGBCore *core, int index, uint32_t val)
2095{
2096 uint16_t vfn = index - P2VMAILBOX0;
2097
2098 trace_igb_set_pfmailbox(vfn, val);
2099
2100 if (val & E1000_P2VMAILBOX_STS) {
2101 core->mac[V2PMAILBOX0 + vfn] |= E1000_V2PMAILBOX_PFSTS;
2102 mailbox_interrupt_to_vf(core, vfn);
2103 }
2104
2105 if (val & E1000_P2VMAILBOX_ACK) {
2106 core->mac[V2PMAILBOX0 + vfn] |= E1000_V2PMAILBOX_PFACK;
2107 mailbox_interrupt_to_vf(core, vfn);
2108 }
2109
2110 /* Buffer Taken by PF (can be set only if the VFU is cleared). */
2111 if (val & E1000_P2VMAILBOX_PFU) {
2112 if (!(core->mac[index] & E1000_P2VMAILBOX_VFU)) {
2113 core->mac[index] |= E1000_P2VMAILBOX_PFU;
2114 core->mac[V2PMAILBOX0 + vfn] |= E1000_V2PMAILBOX_PFU;
2115 }
2116 } else {
2117 core->mac[index] &= ~E1000_P2VMAILBOX_PFU;
2118 core->mac[V2PMAILBOX0 + vfn] &= ~E1000_V2PMAILBOX_PFU;
2119 }
2120
2121 if (val & E1000_P2VMAILBOX_RVFU) {
2122 core->mac[V2PMAILBOX0 + vfn] &= ~E1000_V2PMAILBOX_VFU;
2123 core->mac[MBVFICR] &= ~((E1000_MBVFICR_VFACK_VF1 << vfn) |
2124 (E1000_MBVFICR_VFREQ_VF1 << vfn));
2125 }
2126}
2127
2128static void igb_set_vfmailbox(IGBCore *core, int index, uint32_t val)
2129{
2130 uint16_t vfn = index - V2PMAILBOX0;
2131
2132 trace_igb_set_vfmailbox(vfn, val);
2133
2134 if (val & E1000_V2PMAILBOX_REQ) {
2135 core->mac[MBVFICR] |= E1000_MBVFICR_VFREQ_VF1 << vfn;
2136 mailbox_interrupt_to_pf(core);
2137 }
2138
2139 if (val & E1000_V2PMAILBOX_ACK) {
2140 core->mac[MBVFICR] |= E1000_MBVFICR_VFACK_VF1 << vfn;
2141 mailbox_interrupt_to_pf(core);
2142 }
2143
2144 /* Buffer Taken by VF (can be set only if the PFU is cleared). */
2145 if (val & E1000_V2PMAILBOX_VFU) {
2146 if (!(core->mac[index] & E1000_V2PMAILBOX_PFU)) {
2147 core->mac[index] |= E1000_V2PMAILBOX_VFU;
2148 core->mac[P2VMAILBOX0 + vfn] |= E1000_P2VMAILBOX_VFU;
2149 }
2150 } else {
2151 core->mac[index] &= ~E1000_V2PMAILBOX_VFU;
2152 core->mac[P2VMAILBOX0 + vfn] &= ~E1000_P2VMAILBOX_VFU;
2153 }
2154}
2155
2e68546a
SY		 2156static void igb_vf_reset(IGBCore *core, uint16_t vfn)
2157{
3269ebb3
SY		 2158 uint16_t qn0 = vfn;
2159 uint16_t qn1 = vfn + IGB_NUM_VM_POOLS;
2160
2e68546a 2161 /* disable Rx and Tx for the VF*/
3269ebb3
SY		 2162 core->mac[RXDCTL0 + (qn0 * 16)] &= ~E1000_RXDCTL_QUEUE_ENABLE;
2163 core->mac[RXDCTL0 + (qn1 * 16)] &= ~E1000_RXDCTL_QUEUE_ENABLE;
2164 core->mac[TXDCTL0 + (qn0 * 16)] &= ~E1000_TXDCTL_QUEUE_ENABLE;
2165 core->mac[TXDCTL0 + (qn1 * 16)] &= ~E1000_TXDCTL_QUEUE_ENABLE;
2e68546a 2166 core->mac[VFRE] &= ~BIT(vfn);
3269ebb3 2167 core->mac[VFTE] &= ~BIT(vfn);
2e68546a
SY		 2168 /* indicate VF reset to PF */
2169 core->mac[VFLRE] |= BIT(vfn);
2170 /* VFLRE and mailbox use the same interrupt cause */
2171 mailbox_interrupt_to_pf(core);
2172}
2173
3a977dee
AO		 2174static void igb_w1c(IGBCore *core, int index, uint32_t val)
2175{
2176 core->mac[index] &= ~val;
2177}
2178
2179static void igb_set_eimc(IGBCore *core, int index, uint32_t val)
2180{
2181 bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE);
5844562b 2182 uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK;
3a977dee 2183
5844562b 2184 trace_igb_irq_write_eimc(val, msix);
3a977dee 2185
5844562b
AO		 2186 /* Interrupts are disabled via a write to EIMC and reflected in EIMS. */
2187 igb_lower_interrupts(core, EIMS, val & mask);
3a977dee
AO		 2188}
2189
2190static void igb_set_eiac(IGBCore *core, int index, uint32_t val)
2191{
2192 bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE);
2193
2194 if (msix) {
2195 trace_igb_irq_write_eiac(val);
2196
2197 /*
2198 * TODO: When using IOV, the bits that correspond to MSI-X vectors
2199 * that are assigned to a VF are read-only.
2200 */
2201 core->mac[EIAC] |= (val & E1000_EICR_MSIX_MASK);
2202 }
2203}
2204
2205static void igb_set_eiam(IGBCore *core, int index, uint32_t val)
2206{
2207 bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE);
2208
2209 /*
2210 * TODO: When using IOV, the bits that correspond to MSI-X vectors that
2211 * are assigned to a VF are read-only.
2212 */
2213 core->mac[EIAM] |=
2214 ~(val & (msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK));
2215
2216 trace_igb_irq_write_eiam(val, msix);
2217}
2218
2219static void igb_set_eicr(IGBCore *core, int index, uint32_t val)
2220{
2221 bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE);
2222
2223 /*
2224 * TODO: In IOV mode, only bit zero of this vector is available for the PF
2225 * function.
2226 */
5844562b 2227 uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK;
3a977dee
AO		 2228
2229 trace_igb_irq_write_eicr(val, msix);
5844562b 2230 igb_lower_interrupts(core, EICR, val & mask);
3a977dee
AO		 2231}
2232
2233static void igb_set_vtctrl(IGBCore *core, int index, uint32_t val)
2234{
2235 uint16_t vfn;
2236
2237 if (val & E1000_CTRL_RST) {
2238 vfn = (index - PVTCTRL0) / 0x40;
2239 igb_vf_reset(core, vfn);
2240 }
2241}
2242
2243static void igb_set_vteics(IGBCore *core, int index, uint32_t val)
2244{
2245 uint16_t vfn = (index - PVTEICS0) / 0x40;
2246
2247 core->mac[index] = val;
2248 igb_set_eics(core, EICS, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM));
2249}
2250
2251static void igb_set_vteims(IGBCore *core, int index, uint32_t val)
2252{
2253 uint16_t vfn = (index - PVTEIMS0) / 0x40;
2254
2255 core->mac[index] = val;
2256 igb_set_eims(core, EIMS, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM));
2257}
2258
2259static void igb_set_vteimc(IGBCore *core, int index, uint32_t val)
2260{
2261 uint16_t vfn = (index - PVTEIMC0) / 0x40;
2262
2263 core->mac[index] = val;
2264 igb_set_eimc(core, EIMC, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM));
2265}
2266
2267static void igb_set_vteiac(IGBCore *core, int index, uint32_t val)
2268{
2269 uint16_t vfn = (index - PVTEIAC0) / 0x40;
2270
2271 core->mac[index] = val;
2272 igb_set_eiac(core, EIAC, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM));
2273}
2274
2275static void igb_set_vteiam(IGBCore *core, int index, uint32_t val)
2276{
2277 uint16_t vfn = (index - PVTEIAM0) / 0x40;
2278
2279 core->mac[index] = val;
2280 igb_set_eiam(core, EIAM, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM));
2281}
2282
2283static void igb_set_vteicr(IGBCore *core, int index, uint32_t val)
2284{
2285 uint16_t vfn = (index - PVTEICR0) / 0x40;
2286
2287 core->mac[index] = val;
2288 igb_set_eicr(core, EICR, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM));
2289}
2290
2291static void igb_set_vtivar(IGBCore *core, int index, uint32_t val)
2292{
2293 uint16_t vfn = (index - VTIVAR);
2294 uint16_t qn = vfn;
2295 uint8_t ent;
2296 int n;
2297
2298 core->mac[index] = val;
2299
2300 /* Get assigned vector associated with queue Rx#0. */
2301 if ((val & E1000_IVAR_VALID)) {
2302 n = igb_ivar_entry_rx(qn);
2303 ent = E1000_IVAR_VALID | (24 - vfn * IGBVF_MSIX_VEC_NUM - (2 - (val & 0x7)));
2304 core->mac[IVAR0 + n / 4] |= ent << 8 * (n % 4);
2305 }
2306
2307 /* Get assigned vector associated with queue Tx#0 */
2308 ent = val >> 8;
2309 if ((ent & E1000_IVAR_VALID)) {
2310 n = igb_ivar_entry_tx(qn);
2311 ent = E1000_IVAR_VALID | (24 - vfn * IGBVF_MSIX_VEC_NUM - (2 - (ent & 0x7)));
2312 core->mac[IVAR0 + n / 4] |= ent << 8 * (n % 4);
2313 }
2314
2315 /*
2316 * Ignoring assigned vectors associated with queues Rx#1 and Tx#1 for now.
2317 */
2318}
2319
2320static inline void
2321igb_autoneg_timer(void *opaque)
2322{
2323 IGBCore *core = opaque;
2324 if (!qemu_get_queue(core->owner_nic)->link_down) {
2325 e1000x_update_regs_on_autoneg_done(core->mac, core->phy);
2326 igb_start_recv(core);
2327
2328 igb_update_flowctl_status(core);
2329 /* signal link status change to the guest */
5844562b 2330 igb_raise_interrupts(core, ICR, E1000_ICR_LSC);
3a977dee
AO		 2331 }
2332}
2333
2334static inline uint16_t
2335igb_get_reg_index_with_offset(const uint16_t *mac_reg_access, hwaddr addr)
2336{
2337 uint16_t index = (addr & 0x1ffff) >> 2;
2338 return index + (mac_reg_access[index] & 0xfffe);
2339}
2340
2341static const char igb_phy_regcap[MAX_PHY_REG_ADDRESS + 1] = {
2342 [MII_BMCR] = PHY_RW,
2343 [MII_BMSR] = PHY_R,
2344 [MII_PHYID1] = PHY_R,
2345 [MII_PHYID2] = PHY_R,
2346 [MII_ANAR] = PHY_RW,
2347 [MII_ANLPAR] = PHY_R,
2348 [MII_ANER] = PHY_R,
2349 [MII_ANNP] = PHY_RW,
2350 [MII_ANLPRNP] = PHY_R,
2351 [MII_CTRL1000] = PHY_RW,
2352 [MII_STAT1000] = PHY_R,
2353 [MII_EXTSTAT] = PHY_R,
2354
2355 [IGP01E1000_PHY_PORT_CONFIG] = PHY_RW,
2356 [IGP01E1000_PHY_PORT_STATUS] = PHY_R,
2357 [IGP01E1000_PHY_PORT_CTRL] = PHY_RW,
2358 [IGP01E1000_PHY_LINK_HEALTH] = PHY_R,
2359 [IGP02E1000_PHY_POWER_MGMT] = PHY_RW,
2360 [IGP01E1000_PHY_PAGE_SELECT] = PHY_W
2361};
2362
2363static void
2364igb_phy_reg_write(IGBCore *core, uint32_t addr, uint16_t data)
2365{
2366 assert(addr <= MAX_PHY_REG_ADDRESS);
2367
2368 if (addr == MII_BMCR) {
2369 igb_set_phy_ctrl(core, data);
2370 } else {
2371 core->phy[addr] = data;
2372 }
2373}
2374
2375static void
2376igb_set_mdic(IGBCore *core, int index, uint32_t val)
2377{
2378 uint32_t data = val & E1000_MDIC_DATA_MASK;
2379 uint32_t addr = ((val & E1000_MDIC_REG_MASK) >> E1000_MDIC_REG_SHIFT);
2380
2381 if ((val & E1000_MDIC_PHY_MASK) >> E1000_MDIC_PHY_SHIFT != 1) { /* phy # */
2382 val = core->mac[MDIC] | E1000_MDIC_ERROR;
2383 } else if (val & E1000_MDIC_OP_READ) {
2384 if (!(igb_phy_regcap[addr] & PHY_R)) {
2385 trace_igb_core_mdic_read_unhandled(addr);
2386 val |= E1000_MDIC_ERROR;
2387 } else {
2388 val = (val ^ data) | core->phy[addr];
2389 trace_igb_core_mdic_read(addr, val);
2390 }
2391 } else if (val & E1000_MDIC_OP_WRITE) {
2392 if (!(igb_phy_regcap[addr] & PHY_W)) {
2393 trace_igb_core_mdic_write_unhandled(addr);
2394 val |= E1000_MDIC_ERROR;
2395 } else {
2396 trace_igb_core_mdic_write(addr, data);
2397 igb_phy_reg_write(core, addr, data);
2398 }
2399 }
2400 core->mac[MDIC] = val | E1000_MDIC_READY;
2401
2402 if (val & E1000_MDIC_INT_EN) {
5844562b 2403 igb_raise_interrupts(core, ICR, E1000_ICR_MDAC);
3a977dee
AO		 2404 }
2405}
2406
2407static void
2408igb_set_rdt(IGBCore *core, int index, uint32_t val)
2409{
2410 core->mac[index] = val & 0xffff;
2411 trace_e1000e_rx_set_rdt(igb_mq_queue_idx(RDT0, index), val);
2412 igb_start_recv(core);
2413}
2414
2415static void
2416igb_set_status(IGBCore *core, int index, uint32_t val)
2417{
2418 if ((val & E1000_STATUS_PHYRA) == 0) {
2419 core->mac[index] &= ~E1000_STATUS_PHYRA;
2420 }
2421}
2422
2423static void
2424igb_set_ctrlext(IGBCore *core, int index, uint32_t val)
2425{
2e68546a
SY		 2426 trace_igb_link_set_ext_params(!!(val & E1000_CTRL_EXT_ASDCHK),
2427 !!(val & E1000_CTRL_EXT_SPD_BYPS),
2428 !!(val & E1000_CTRL_EXT_PFRSTD));
3a977dee
AO		 2429
2430 /* Zero self-clearing bits */
2431 val &= ~(E1000_CTRL_EXT_ASDCHK | E1000_CTRL_EXT_EE_RST);
2432 core->mac[CTRL_EXT] = val;
2e68546a
SY		 2433
2434 if (core->mac[CTRL_EXT] & E1000_CTRL_EXT_PFRSTD) {
2435 for (int vfn = 0; vfn < IGB_MAX_VF_FUNCTIONS; vfn++) {
2436 core->mac[V2PMAILBOX0 + vfn] &= ~E1000_V2PMAILBOX_RSTI;
2437 core->mac[V2PMAILBOX0 + vfn] |= E1000_V2PMAILBOX_RSTD;
2438 }
2439 }
3a977dee
AO		 2440}
2441
2442static void
2443igb_set_pbaclr(IGBCore *core, int index, uint32_t val)
2444{
2445 int i;
2446
2447 core->mac[PBACLR] = val & E1000_PBACLR_VALID_MASK;
2448
2449 if (!msix_enabled(core->owner)) {
2450 return;
2451 }
2452
2453 for (i = 0; i < IGB_INTR_NUM; i++) {
2454 if (core->mac[PBACLR] & BIT(i)) {
2455 msix_clr_pending(core->owner, i);
2456 }
2457 }
2458}
2459
2460static void
2461igb_set_fcrth(IGBCore *core, int index, uint32_t val)
2462{
2463 core->mac[FCRTH] = val & 0xFFF8;
2464}
2465
2466static void
2467igb_set_fcrtl(IGBCore *core, int index, uint32_t val)
2468{
2469 core->mac[FCRTL] = val & 0x8000FFF8;
2470}
2471
2472#define IGB_LOW_BITS_SET_FUNC(num) \
2473 static void \
2474 igb_set_##num##bit(IGBCore *core, int index, uint32_t val) \
2475 { \
2476 core->mac[index] = val & (BIT(num) - 1); \
2477 }
2478
2479IGB_LOW_BITS_SET_FUNC(4)
2480IGB_LOW_BITS_SET_FUNC(13)
2481IGB_LOW_BITS_SET_FUNC(16)
2482
2483static void
2484igb_set_dlen(IGBCore *core, int index, uint32_t val)
2485{
2486 core->mac[index] = val & 0xffff0;
2487}
2488
2489static void
2490igb_set_dbal(IGBCore *core, int index, uint32_t val)
2491{
2492 core->mac[index] = val & E1000_XDBAL_MASK;
2493}
2494
2495static void
2496igb_set_tdt(IGBCore *core, int index, uint32_t val)
2497{
2498 IGB_TxRing txr;
2499 int qn = igb_mq_queue_idx(TDT0, index);
2500
2501 core->mac[index] = val & 0xffff;
2502
2503 igb_tx_ring_init(core, &txr, qn);
2504 igb_start_xmit(core, &txr);
2505}
2506
2507static void
2508igb_set_ics(IGBCore *core, int index, uint32_t val)
2509{
2510 trace_e1000e_irq_write_ics(val);
5844562b 2511 igb_raise_interrupts(core, ICR, val);
3a977dee
AO		 2512}
2513
2514static void
2515igb_set_imc(IGBCore *core, int index, uint32_t val)
2516{
2517 trace_e1000e_irq_ims_clear_set_imc(val);
5844562b 2518 igb_lower_interrupts(core, IMS, val);
3a977dee
AO		 2519}
2520
2521static void
2522igb_set_ims(IGBCore *core, int index, uint32_t val)
2523{
5844562b 2524 igb_raise_interrupts(core, IMS, val & 0x77D4FBFD);
3a977dee
AO		 2525}
2526
5844562b 2527static void igb_nsicr(IGBCore *core)
3a977dee
AO		 2528{
2529 /*
f0b1df5c 2530 * If GPIE.NSICR = 0, then the clear of IMS will occur only if at
3a977dee
AO		 2531 * least one bit is set in the IMS and there is a true interrupt as
2532 * reflected in ICR.INTA.
2533 */
2534 if ((core->mac[GPIE] & E1000_GPIE_NSICR) ||
2535 (core->mac[IMS] && (core->mac[ICR] & E1000_ICR_INT_ASSERTED))) {
5844562b 2536 igb_lower_interrupts(core, IMS, core->mac[IAM]);
3a977dee
AO		 2537 }
2538}
2539
2540static void igb_set_icr(IGBCore *core, int index, uint32_t val)
2541{
5844562b
AO		 2542 igb_nsicr(core);
2543 igb_lower_interrupts(core, ICR, val);
3a977dee
AO		 2544}
2545
2546static uint32_t
2547igb_mac_readreg(IGBCore *core, int index)
2548{
2549 return core->mac[index];
2550}
2551
2552static uint32_t
2553igb_mac_ics_read(IGBCore *core, int index)
2554{
2555 trace_e1000e_irq_read_ics(core->mac[ICS]);
2556 return core->mac[ICS];
2557}
2558
2559static uint32_t
2560igb_mac_ims_read(IGBCore *core, int index)
2561{
2562 trace_e1000e_irq_read_ims(core->mac[IMS]);
2563 return core->mac[IMS];
2564}
2565
2566static uint32_t
2567igb_mac_swsm_read(IGBCore *core, int index)
2568{
2569 uint32_t val = core->mac[SWSM];
2570 core->mac[SWSM] = val | E1000_SWSM_SMBI;
2571 return val;
2572}
2573
2574static uint32_t
2575igb_mac_eitr_read(IGBCore *core, int index)
2576{
2577 return core->eitr_guest_value[index - EITR0];
2578}
2579
2580static uint32_t igb_mac_vfmailbox_read(IGBCore *core, int index)
2581{
2582 uint32_t val = core->mac[index];
2583
2584 core->mac[index] &= ~(E1000_V2PMAILBOX_PFSTS | E1000_V2PMAILBOX_PFACK |
2585 E1000_V2PMAILBOX_RSTD);
2586
2587 return val;
2588}
2589
2590static uint32_t
2591igb_mac_icr_read(IGBCore *core, int index)
2592{
2593 uint32_t ret = core->mac[ICR];
3a977dee
AO		 2594
2595 if (core->mac[GPIE] & E1000_GPIE_NSICR) {
2596 trace_igb_irq_icr_clear_gpie_nsicr();
5844562b 2597 igb_lower_interrupts(core, ICR, 0xffffffff);
3a977dee
AO		 2598 } else if (core->mac[IMS] == 0) {
2599 trace_e1000e_irq_icr_clear_zero_ims();
5844562b 2600 igb_lower_interrupts(core, ICR, 0xffffffff);
efb1fd7a
AO		 2601 } else if (core->mac[ICR] & E1000_ICR_INT_ASSERTED) {
2602 igb_lower_interrupts(core, ICR, 0xffffffff);
3a977dee
AO		 2603 } else if (!msix_enabled(core->owner)) {
2604 trace_e1000e_irq_icr_clear_nonmsix_icr_read();
5844562b 2605 igb_lower_interrupts(core, ICR, 0xffffffff);
3a977dee
AO		 2606 }
2607
5844562b 2608 igb_nsicr(core);
3a977dee
AO		 2609 return ret;
2610}
2611
2612static uint32_t
2613igb_mac_read_clr4(IGBCore *core, int index)
2614{
2615 uint32_t ret = core->mac[index];
2616
2617 core->mac[index] = 0;
2618 return ret;
2619}
2620
2621static uint32_t
2622igb_mac_read_clr8(IGBCore *core, int index)
2623{
2624 uint32_t ret = core->mac[index];
2625
2626 core->mac[index] = 0;
2627 core->mac[index - 1] = 0;
2628 return ret;
2629}
2630
2631static uint32_t
2632igb_get_ctrl(IGBCore *core, int index)
2633{
2634 uint32_t val = core->mac[CTRL];
2635
2636 trace_e1000e_link_read_params(
2637 !!(val & E1000_CTRL_ASDE),
2638 (val & E1000_CTRL_SPD_SEL) >> E1000_CTRL_SPD_SHIFT,
2639 !!(val & E1000_CTRL_FRCSPD),
2640 !!(val & E1000_CTRL_FRCDPX),
2641 !!(val & E1000_CTRL_RFCE),
2642 !!(val & E1000_CTRL_TFCE));
2643
2644 return val;
2645}
2646
2647static uint32_t igb_get_status(IGBCore *core, int index)
2648{
2649 uint32_t res = core->mac[STATUS];
2650 uint16_t num_vfs = pcie_sriov_num_vfs(core->owner);
2651
2652 if (core->mac[CTRL] & E1000_CTRL_FRCDPX) {
2653 res |= (core->mac[CTRL] & E1000_CTRL_FD) ? E1000_STATUS_FD : 0;
2654 } else {
2655 res |= E1000_STATUS_FD;
2656 }
2657
2658 if ((core->mac[CTRL] & E1000_CTRL_FRCSPD) ||
2659 (core->mac[CTRL_EXT] & E1000_CTRL_EXT_SPD_BYPS)) {
2660 switch (core->mac[CTRL] & E1000_CTRL_SPD_SEL) {
2661 case E1000_CTRL_SPD_10:
2662 res |= E1000_STATUS_SPEED_10;
2663 break;
2664 case E1000_CTRL_SPD_100:
2665 res |= E1000_STATUS_SPEED_100;
2666 break;
2667 case E1000_CTRL_SPD_1000:
2668 default:
2669 res |= E1000_STATUS_SPEED_1000;
2670 break;
2671 }
2672 } else {
2673 res |= E1000_STATUS_SPEED_1000;
2674 }
2675
2676 if (num_vfs) {
2677 res |= num_vfs << E1000_STATUS_NUM_VFS_SHIFT;
2678 res |= E1000_STATUS_IOV_MODE;
2679 }
2680
2681 /*
2682 * Windows driver 12.18.9.23 resets if E1000_STATUS_GIO_MASTER_ENABLE is
2683 * left set after E1000_CTRL_LRST is set.
2684 */
2685 if (!(core->mac[CTRL] & E1000_CTRL_GIO_MASTER_DISABLE) &&
2686 !(core->mac[CTRL] & E1000_CTRL_LRST)) {
2687 res |= E1000_STATUS_GIO_MASTER_ENABLE;
2688 }
2689
2690 return res;
2691}
2692
2693static void
2694igb_mac_writereg(IGBCore *core, int index, uint32_t val)
2695{
2696 core->mac[index] = val;
2697}
2698
2699static void
2700igb_mac_setmacaddr(IGBCore *core, int index, uint32_t val)
2701{
2702 uint32_t macaddr[2];
2703
2704 core->mac[index] = val;
2705
2706 macaddr[0] = cpu_to_le32(core->mac[RA]);
2707 macaddr[1] = cpu_to_le32(core->mac[RA + 1]);
2708 qemu_format_nic_info_str(qemu_get_queue(core->owner_nic),
2709 (uint8_t *) macaddr);
2710
2711 trace_e1000e_mac_set_sw(MAC_ARG(macaddr));
2712}
2713
2714static void
2715igb_set_eecd(IGBCore *core, int index, uint32_t val)
2716{
2717 static const uint32_t ro_bits = E1000_EECD_PRES |
2718 E1000_EECD_AUTO_RD |
2719 E1000_EECD_SIZE_EX_MASK;
2720
2721 core->mac[EECD] = (core->mac[EECD] & ro_bits) | (val & ~ro_bits);
2722}
2723
2724static void
2725igb_set_eerd(IGBCore *core, int index, uint32_t val)
2726{
2727 uint32_t addr = (val >> E1000_EERW_ADDR_SHIFT) & E1000_EERW_ADDR_MASK;
2728 uint32_t flags = 0;
2729 uint32_t data = 0;
2730
2731 if ((addr < IGB_EEPROM_SIZE) && (val & E1000_EERW_START)) {
2732 data = core->eeprom[addr];
2733 flags = E1000_EERW_DONE;
2734 }
2735
2736 core->mac[EERD] = flags |
2737 (addr << E1000_EERW_ADDR_SHIFT) |
2738 (data << E1000_EERW_DATA_SHIFT);
2739}
2740
2741static void
2742igb_set_eitr(IGBCore *core, int index, uint32_t val)
2743{
2744 uint32_t eitr_num = index - EITR0;
2745
2746 trace_igb_irq_eitr_set(eitr_num, val);
2747
2748 core->eitr_guest_value[eitr_num] = val & ~E1000_EITR_CNT_IGNR;
2749 core->mac[index] = val & 0x7FFE;
2750}
2751
2752static void
2753igb_update_rx_offloads(IGBCore *core)
2754{
2755 int cso_state = igb_rx_l4_cso_enabled(core);
2756
2757 trace_e1000e_rx_set_cso(cso_state);
2758
2759 if (core->has_vnet) {
2760 qemu_set_offload(qemu_get_queue(core->owner_nic)->peer,
2761 cso_state, 0, 0, 0, 0);
2762 }
2763}
2764
2765static void
2766igb_set_rxcsum(IGBCore *core, int index, uint32_t val)
2767{
2768 core->mac[RXCSUM] = val;
2769 igb_update_rx_offloads(core);
2770}
2771
2772static void
2773igb_set_gcr(IGBCore *core, int index, uint32_t val)
2774{
2775 uint32_t ro_bits = core->mac[GCR] & E1000_GCR_RO_BITS;
2776 core->mac[GCR] = (val & ~E1000_GCR_RO_BITS) | ro_bits;
2777}
2778
2779static uint32_t igb_get_systiml(IGBCore *core, int index)
2780{
2781 e1000x_timestamp(core->mac, core->timadj, SYSTIML, SYSTIMH);
2782 return core->mac[SYSTIML];
2783}
2784
2785static uint32_t igb_get_rxsatrh(IGBCore *core, int index)
2786{
2787 core->mac[TSYNCRXCTL] &= ~E1000_TSYNCRXCTL_VALID;
2788 return core->mac[RXSATRH];
2789}
2790
2791static uint32_t igb_get_txstmph(IGBCore *core, int index)
2792{
2793 core->mac[TSYNCTXCTL] &= ~E1000_TSYNCTXCTL_VALID;
2794 return core->mac[TXSTMPH];
2795}
2796
2797static void igb_set_timinca(IGBCore *core, int index, uint32_t val)
2798{
2799 e1000x_set_timinca(core->mac, &core->timadj, val);
2800}
2801
2802static void igb_set_timadjh(IGBCore *core, int index, uint32_t val)
2803{
2804 core->mac[TIMADJH] = val;
2805 core->timadj += core->mac[TIMADJL] | ((int64_t)core->mac[TIMADJH] << 32);
2806}
2807
2808#define igb_getreg(x) [x] = igb_mac_readreg
2809typedef uint32_t (*readops)(IGBCore *, int);
2810static const readops igb_macreg_readops[] = {
2811 igb_getreg(WUFC),
2812 igb_getreg(MANC),
2813 igb_getreg(TOTL),
2814 igb_getreg(RDT0),
2815 igb_getreg(RDT1),
2816 igb_getreg(RDT2),
2817 igb_getreg(RDT3),
2818 igb_getreg(RDT4),
2819 igb_getreg(RDT5),
2820 igb_getreg(RDT6),
2821 igb_getreg(RDT7),
2822 igb_getreg(RDT8),
2823 igb_getreg(RDT9),
2824 igb_getreg(RDT10),
2825 igb_getreg(RDT11),
2826 igb_getreg(RDT12),
2827 igb_getreg(RDT13),
2828 igb_getreg(RDT14),
2829 igb_getreg(RDT15),
2830 igb_getreg(RDBAH0),
2831 igb_getreg(RDBAH1),
2832 igb_getreg(RDBAH2),
2833 igb_getreg(RDBAH3),
2834 igb_getreg(RDBAH4),
2835 igb_getreg(RDBAH5),
2836 igb_getreg(RDBAH6),
2837 igb_getreg(RDBAH7),
2838 igb_getreg(RDBAH8),
2839 igb_getreg(RDBAH9),
2840 igb_getreg(RDBAH10),
2841 igb_getreg(RDBAH11),
2842 igb_getreg(RDBAH12),
2843 igb_getreg(RDBAH13),
2844 igb_getreg(RDBAH14),
2845 igb_getreg(RDBAH15),
2846 igb_getreg(TDBAL0),
2847 igb_getreg(TDBAL1),
2848 igb_getreg(TDBAL2),
2849 igb_getreg(TDBAL3),
2850 igb_getreg(TDBAL4),
2851 igb_getreg(TDBAL5),
2852 igb_getreg(TDBAL6),
2853 igb_getreg(TDBAL7),
2854 igb_getreg(TDBAL8),
2855 igb_getreg(TDBAL9),
2856 igb_getreg(TDBAL10),
2857 igb_getreg(TDBAL11),
2858 igb_getreg(TDBAL12),
2859 igb_getreg(TDBAL13),
2860 igb_getreg(TDBAL14),
2861 igb_getreg(TDBAL15),
2862 igb_getreg(RDLEN0),
2863 igb_getreg(RDLEN1),
2864 igb_getreg(RDLEN2),
2865 igb_getreg(RDLEN3),
2866 igb_getreg(RDLEN4),
2867 igb_getreg(RDLEN5),
2868 igb_getreg(RDLEN6),
2869 igb_getreg(RDLEN7),
2870 igb_getreg(RDLEN8),
2871 igb_getreg(RDLEN9),
2872 igb_getreg(RDLEN10),
2873 igb_getreg(RDLEN11),
2874 igb_getreg(RDLEN12),
2875 igb_getreg(RDLEN13),
2876 igb_getreg(RDLEN14),
2877 igb_getreg(RDLEN15),
2878 igb_getreg(SRRCTL0),
2879 igb_getreg(SRRCTL1),
2880 igb_getreg(SRRCTL2),
2881 igb_getreg(SRRCTL3),
2882 igb_getreg(SRRCTL4),
2883 igb_getreg(SRRCTL5),
2884 igb_getreg(SRRCTL6),
2885 igb_getreg(SRRCTL7),
2886 igb_getreg(SRRCTL8),
2887 igb_getreg(SRRCTL9),
2888 igb_getreg(SRRCTL10),
2889 igb_getreg(SRRCTL11),
2890 igb_getreg(SRRCTL12),
2891 igb_getreg(SRRCTL13),
2892 igb_getreg(SRRCTL14),
2893 igb_getreg(SRRCTL15),
2894 igb_getreg(LATECOL),
2895 igb_getreg(XONTXC),
2896 igb_getreg(TDFH),
2897 igb_getreg(TDFT),
2898 igb_getreg(TDFHS),
2899 igb_getreg(TDFTS),
2900 igb_getreg(TDFPC),
2901 igb_getreg(WUS),
2902 igb_getreg(RDFH),
2903 igb_getreg(RDFT),
2904 igb_getreg(RDFHS),
2905 igb_getreg(RDFTS),
2906 igb_getreg(RDFPC),
2907 igb_getreg(GORCL),
2908 igb_getreg(MGTPRC),
2909 igb_getreg(EERD),
2910 igb_getreg(EIAC),
2911 igb_getreg(MANC2H),
2912 igb_getreg(RXCSUM),
2913 igb_getreg(GSCL_3),
2914 igb_getreg(GSCN_2),
2915 igb_getreg(FCAH),
2916 igb_getreg(FCRTH),
2917 igb_getreg(FLOP),
2918 igb_getreg(RXSTMPH),
2919 igb_getreg(TXSTMPL),
2920 igb_getreg(TIMADJL),
2921 igb_getreg(RDH0),
2922 igb_getreg(RDH1),
2923 igb_getreg(RDH2),
2924 igb_getreg(RDH3),
2925 igb_getreg(RDH4),
2926 igb_getreg(RDH5),
2927 igb_getreg(RDH6),
2928 igb_getreg(RDH7),
2929 igb_getreg(RDH8),
2930 igb_getreg(RDH9),
2931 igb_getreg(RDH10),
2932 igb_getreg(RDH11),
2933 igb_getreg(RDH12),
2934 igb_getreg(RDH13),
2935 igb_getreg(RDH14),
2936 igb_getreg(RDH15),
2937 igb_getreg(TDT0),
2938 igb_getreg(TDT1),
2939 igb_getreg(TDT2),
2940 igb_getreg(TDT3),
2941 igb_getreg(TDT4),
2942 igb_getreg(TDT5),
2943 igb_getreg(TDT6),
2944 igb_getreg(TDT7),
2945 igb_getreg(TDT8),
2946 igb_getreg(TDT9),
2947 igb_getreg(TDT10),
2948 igb_getreg(TDT11),
2949 igb_getreg(TDT12),
2950 igb_getreg(TDT13),
2951 igb_getreg(TDT14),
2952 igb_getreg(TDT15),
2953 igb_getreg(TNCRS),
2954 igb_getreg(RJC),
2955 igb_getreg(IAM),
2956 igb_getreg(GSCL_2),
2957 igb_getreg(TIPG),
2958 igb_getreg(FLMNGCTL),
2959 igb_getreg(FLMNGCNT),
2960 igb_getreg(TSYNCTXCTL),
2961 igb_getreg(EEMNGDATA),
2962 igb_getreg(CTRL_EXT),
2963 igb_getreg(SYSTIMH),
2964 igb_getreg(EEMNGCTL),
2965 igb_getreg(FLMNGDATA),
2966 igb_getreg(TSYNCRXCTL),
2967 igb_getreg(LEDCTL),
2968 igb_getreg(TCTL),
2969 igb_getreg(TCTL_EXT),
2970 igb_getreg(DTXCTL),
2971 igb_getreg(RXPBS),
2972 igb_getreg(TDH0),
2973 igb_getreg(TDH1),
2974 igb_getreg(TDH2),
2975 igb_getreg(TDH3),
2976 igb_getreg(TDH4),
2977 igb_getreg(TDH5),
2978 igb_getreg(TDH6),
2979 igb_getreg(TDH7),
2980 igb_getreg(TDH8),
2981 igb_getreg(TDH9),
2982 igb_getreg(TDH10),
2983 igb_getreg(TDH11),
2984 igb_getreg(TDH12),
2985 igb_getreg(TDH13),
2986 igb_getreg(TDH14),
2987 igb_getreg(TDH15),
2988 igb_getreg(ECOL),
2989 igb_getreg(DC),
2990 igb_getreg(RLEC),
2991 igb_getreg(XOFFTXC),
2992 igb_getreg(RFC),
2993 igb_getreg(RNBC),
2994 igb_getreg(MGTPTC),
2995 igb_getreg(TIMINCA),
2996 igb_getreg(FACTPS),
2997 igb_getreg(GSCL_1),
2998 igb_getreg(GSCN_0),
2999 igb_getreg(PBACLR),
3000 igb_getreg(FCTTV),
3001 igb_getreg(RXSATRL),
3002 igb_getreg(TORL),
3003 igb_getreg(TDLEN0),
3004 igb_getreg(TDLEN1),
3005 igb_getreg(TDLEN2),
3006 igb_getreg(TDLEN3),
3007 igb_getreg(TDLEN4),
3008 igb_getreg(TDLEN5),
3009 igb_getreg(TDLEN6),
3010 igb_getreg(TDLEN7),
3011 igb_getreg(TDLEN8),
3012 igb_getreg(TDLEN9),
3013 igb_getreg(TDLEN10),
3014 igb_getreg(TDLEN11),
3015 igb_getreg(TDLEN12),
3016 igb_getreg(TDLEN13),
3017 igb_getreg(TDLEN14),
3018 igb_getreg(TDLEN15),
3019 igb_getreg(MCC),
3020 igb_getreg(WUC),
3021 igb_getreg(EECD),
3022 igb_getreg(FCRTV),
3023 igb_getreg(TXDCTL0),
3024 igb_getreg(TXDCTL1),
3025 igb_getreg(TXDCTL2),
3026 igb_getreg(TXDCTL3),
3027 igb_getreg(TXDCTL4),
3028 igb_getreg(TXDCTL5),
3029 igb_getreg(TXDCTL6),
3030 igb_getreg(TXDCTL7),
3031 igb_getreg(TXDCTL8),
3032 igb_getreg(TXDCTL9),
3033 igb_getreg(TXDCTL10),
3034 igb_getreg(TXDCTL11),
3035 igb_getreg(TXDCTL12),
3036 igb_getreg(TXDCTL13),
3037 igb_getreg(TXDCTL14),
3038 igb_getreg(TXDCTL15),
3039 igb_getreg(TXCTL0),
3040 igb_getreg(TXCTL1),
3041 igb_getreg(TXCTL2),
3042 igb_getreg(TXCTL3),
3043 igb_getreg(TXCTL4),
3044 igb_getreg(TXCTL5),
3045 igb_getreg(TXCTL6),
3046 igb_getreg(TXCTL7),
3047 igb_getreg(TXCTL8),
3048 igb_getreg(TXCTL9),
3049 igb_getreg(TXCTL10),
3050 igb_getreg(TXCTL11),
3051 igb_getreg(TXCTL12),
3052 igb_getreg(TXCTL13),
3053 igb_getreg(TXCTL14),
3054 igb_getreg(TXCTL15),
3055 igb_getreg(TDWBAL0),
3056 igb_getreg(TDWBAL1),
3057 igb_getreg(TDWBAL2),
3058 igb_getreg(TDWBAL3),
3059 igb_getreg(TDWBAL4),
3060 igb_getreg(TDWBAL5),
3061 igb_getreg(TDWBAL6),
3062 igb_getreg(TDWBAL7),
3063 igb_getreg(TDWBAL8),
3064 igb_getreg(TDWBAL9),
3065 igb_getreg(TDWBAL10),
3066 igb_getreg(TDWBAL11),
3067 igb_getreg(TDWBAL12),
3068 igb_getreg(TDWBAL13),
3069 igb_getreg(TDWBAL14),
3070 igb_getreg(TDWBAL15),
3071 igb_getreg(TDWBAH0),
3072 igb_getreg(TDWBAH1),
3073 igb_getreg(TDWBAH2),
3074 igb_getreg(TDWBAH3),
3075 igb_getreg(TDWBAH4),
3076 igb_getreg(TDWBAH5),
3077 igb_getreg(TDWBAH6),
3078 igb_getreg(TDWBAH7),
3079 igb_getreg(TDWBAH8),
3080 igb_getreg(TDWBAH9),
3081 igb_getreg(TDWBAH10),
3082 igb_getreg(TDWBAH11),
3083 igb_getreg(TDWBAH12),
3084 igb_getreg(TDWBAH13),
3085 igb_getreg(TDWBAH14),
3086 igb_getreg(TDWBAH15),
3087 igb_getreg(PVTCTRL0),
3088 igb_getreg(PVTCTRL1),
3089 igb_getreg(PVTCTRL2),
3090 igb_getreg(PVTCTRL3),
3091 igb_getreg(PVTCTRL4),
3092 igb_getreg(PVTCTRL5),
3093 igb_getreg(PVTCTRL6),
3094 igb_getreg(PVTCTRL7),
3095 igb_getreg(PVTEIMS0),
3096 igb_getreg(PVTEIMS1),
3097 igb_getreg(PVTEIMS2),
3098 igb_getreg(PVTEIMS3),
3099 igb_getreg(PVTEIMS4),
3100 igb_getreg(PVTEIMS5),
3101 igb_getreg(PVTEIMS6),
3102 igb_getreg(PVTEIMS7),
3103 igb_getreg(PVTEIAC0),
3104 igb_getreg(PVTEIAC1),
3105 igb_getreg(PVTEIAC2),
3106 igb_getreg(PVTEIAC3),
3107 igb_getreg(PVTEIAC4),
3108 igb_getreg(PVTEIAC5),
3109 igb_getreg(PVTEIAC6),
3110 igb_getreg(PVTEIAC7),
3111 igb_getreg(PVTEIAM0),
3112 igb_getreg(PVTEIAM1),
3113 igb_getreg(PVTEIAM2),
3114 igb_getreg(PVTEIAM3),
3115 igb_getreg(PVTEIAM4),
3116 igb_getreg(PVTEIAM5),
3117 igb_getreg(PVTEIAM6),
3118 igb_getreg(PVTEIAM7),
3119 igb_getreg(PVFGPRC0),
3120 igb_getreg(PVFGPRC1),
3121 igb_getreg(PVFGPRC2),
3122 igb_getreg(PVFGPRC3),
3123 igb_getreg(PVFGPRC4),
3124 igb_getreg(PVFGPRC5),
3125 igb_getreg(PVFGPRC6),
3126 igb_getreg(PVFGPRC7),
3127 igb_getreg(PVFGPTC0),
3128 igb_getreg(PVFGPTC1),
3129 igb_getreg(PVFGPTC2),
3130 igb_getreg(PVFGPTC3),
3131 igb_getreg(PVFGPTC4),
3132 igb_getreg(PVFGPTC5),
3133 igb_getreg(PVFGPTC6),
3134 igb_getreg(PVFGPTC7),
3135 igb_getreg(PVFGORC0),
3136 igb_getreg(PVFGORC1),
3137 igb_getreg(PVFGORC2),
3138 igb_getreg(PVFGORC3),
3139 igb_getreg(PVFGORC4),
3140 igb_getreg(PVFGORC5),
3141 igb_getreg(PVFGORC6),
3142 igb_getreg(PVFGORC7),
3143 igb_getreg(PVFGOTC0),
3144 igb_getreg(PVFGOTC1),
3145 igb_getreg(PVFGOTC2),
3146 igb_getreg(PVFGOTC3),
3147 igb_getreg(PVFGOTC4),
3148 igb_getreg(PVFGOTC5),
3149 igb_getreg(PVFGOTC6),
3150 igb_getreg(PVFGOTC7),
3151 igb_getreg(PVFMPRC0),
3152 igb_getreg(PVFMPRC1),
3153 igb_getreg(PVFMPRC2),
3154 igb_getreg(PVFMPRC3),
3155 igb_getreg(PVFMPRC4),
3156 igb_getreg(PVFMPRC5),
3157 igb_getreg(PVFMPRC6),
3158 igb_getreg(PVFMPRC7),
3159 igb_getreg(PVFGPRLBC0),
3160 igb_getreg(PVFGPRLBC1),
3161 igb_getreg(PVFGPRLBC2),
3162 igb_getreg(PVFGPRLBC3),
3163 igb_getreg(PVFGPRLBC4),
3164 igb_getreg(PVFGPRLBC5),
3165 igb_getreg(PVFGPRLBC6),
3166 igb_getreg(PVFGPRLBC7),
3167 igb_getreg(PVFGPTLBC0),
3168 igb_getreg(PVFGPTLBC1),
3169 igb_getreg(PVFGPTLBC2),
3170 igb_getreg(PVFGPTLBC3),
3171 igb_getreg(PVFGPTLBC4),
3172 igb_getreg(PVFGPTLBC5),
3173 igb_getreg(PVFGPTLBC6),
3174 igb_getreg(PVFGPTLBC7),
3175 igb_getreg(PVFGORLBC0),
3176 igb_getreg(PVFGORLBC1),
3177 igb_getreg(PVFGORLBC2),
3178 igb_getreg(PVFGORLBC3),
3179 igb_getreg(PVFGORLBC4),
3180 igb_getreg(PVFGORLBC5),
3181 igb_getreg(PVFGORLBC6),
3182 igb_getreg(PVFGORLBC7),
3183 igb_getreg(PVFGOTLBC0),
3184 igb_getreg(PVFGOTLBC1),
3185 igb_getreg(PVFGOTLBC2),
3186 igb_getreg(PVFGOTLBC3),
3187 igb_getreg(PVFGOTLBC4),
3188 igb_getreg(PVFGOTLBC5),
3189 igb_getreg(PVFGOTLBC6),
3190 igb_getreg(PVFGOTLBC7),
3191 igb_getreg(RCTL),
3192 igb_getreg(MDIC),
3193 igb_getreg(FCRUC),
3194 igb_getreg(VET),
3195 igb_getreg(RDBAL0),
3196 igb_getreg(RDBAL1),
3197 igb_getreg(RDBAL2),
3198 igb_getreg(RDBAL3),
3199 igb_getreg(RDBAL4),
3200 igb_getreg(RDBAL5),
3201 igb_getreg(RDBAL6),
3202 igb_getreg(RDBAL7),
3203 igb_getreg(RDBAL8),
3204 igb_getreg(RDBAL9),
3205 igb_getreg(RDBAL10),
3206 igb_getreg(RDBAL11),
3207 igb_getreg(RDBAL12),
3208 igb_getreg(RDBAL13),
3209 igb_getreg(RDBAL14),
3210 igb_getreg(RDBAL15),
3211 igb_getreg(TDBAH0),
3212 igb_getreg(TDBAH1),
3213 igb_getreg(TDBAH2),
3214 igb_getreg(TDBAH3),
3215 igb_getreg(TDBAH4),
3216 igb_getreg(TDBAH5),
3217 igb_getreg(TDBAH6),
3218 igb_getreg(TDBAH7),
3219 igb_getreg(TDBAH8),
3220 igb_getreg(TDBAH9),
3221 igb_getreg(TDBAH10),
3222 igb_getreg(TDBAH11),
3223 igb_getreg(TDBAH12),
3224 igb_getreg(TDBAH13),
3225 igb_getreg(TDBAH14),
3226 igb_getreg(TDBAH15),
3227 igb_getreg(SCC),
3228 igb_getreg(COLC),
3229 igb_getreg(XOFFRXC),
3230 igb_getreg(IPAV),
3231 igb_getreg(GOTCL),
3232 igb_getreg(MGTPDC),
3233 igb_getreg(GCR),
3234 igb_getreg(MFVAL),
3235 igb_getreg(FUNCTAG),
3236 igb_getreg(GSCL_4),
3237 igb_getreg(GSCN_3),
3238 igb_getreg(MRQC),
3239 igb_getreg(FCT),
3240 igb_getreg(FLA),
3241 igb_getreg(RXDCTL0),
3242 igb_getreg(RXDCTL1),
3243 igb_getreg(RXDCTL2),
3244 igb_getreg(RXDCTL3),
3245 igb_getreg(RXDCTL4),
3246 igb_getreg(RXDCTL5),
3247 igb_getreg(RXDCTL6),
3248 igb_getreg(RXDCTL7),
3249 igb_getreg(RXDCTL8),
3250 igb_getreg(RXDCTL9),
3251 igb_getreg(RXDCTL10),
3252 igb_getreg(RXDCTL11),
3253 igb_getreg(RXDCTL12),
3254 igb_getreg(RXDCTL13),
3255 igb_getreg(RXDCTL14),
3256 igb_getreg(RXDCTL15),
3257 igb_getreg(RXSTMPL),
3258 igb_getreg(TIMADJH),
3259 igb_getreg(FCRTL),
3260 igb_getreg(XONRXC),
3261 igb_getreg(RFCTL),
3262 igb_getreg(GSCN_1),
3263 igb_getreg(FCAL),
3264 igb_getreg(GPIE),
3265 igb_getreg(TXPBS),
3266 igb_getreg(RLPML),
3267
3268 [TOTH] = igb_mac_read_clr8,
3269 [GOTCH] = igb_mac_read_clr8,
3270 [PRC64] = igb_mac_read_clr4,
3271 [PRC255] = igb_mac_read_clr4,
3272 [PRC1023] = igb_mac_read_clr4,
3273 [PTC64] = igb_mac_read_clr4,
3274 [PTC255] = igb_mac_read_clr4,
3275 [PTC1023] = igb_mac_read_clr4,
3276 [GPRC] = igb_mac_read_clr4,
3277 [TPT] = igb_mac_read_clr4,
3278 [RUC] = igb_mac_read_clr4,
3279 [BPRC] = igb_mac_read_clr4,
3280 [MPTC] = igb_mac_read_clr4,
3281 [IAC] = igb_mac_read_clr4,
3282 [ICR] = igb_mac_icr_read,
3283 [STATUS] = igb_get_status,
3284 [ICS] = igb_mac_ics_read,
3285 /*
3286 * 8.8.10: Reading the IMC register returns the value of the IMS register.
3287 */
3288 [IMC] = igb_mac_ims_read,
3289 [TORH] = igb_mac_read_clr8,
3290 [GORCH] = igb_mac_read_clr8,
3291 [PRC127] = igb_mac_read_clr4,
3292 [PRC511] = igb_mac_read_clr4,
3293 [PRC1522] = igb_mac_read_clr4,
3294 [PTC127] = igb_mac_read_clr4,
3295 [PTC511] = igb_mac_read_clr4,
3296 [PTC1522] = igb_mac_read_clr4,
3297 [GPTC] = igb_mac_read_clr4,
3298 [TPR] = igb_mac_read_clr4,
3299 [ROC] = igb_mac_read_clr4,
3300 [MPRC] = igb_mac_read_clr4,
3301 [BPTC] = igb_mac_read_clr4,
3302 [TSCTC] = igb_mac_read_clr4,
3303 [CTRL] = igb_get_ctrl,
3304 [SWSM] = igb_mac_swsm_read,
3305 [IMS] = igb_mac_ims_read,
3306 [SYSTIML] = igb_get_systiml,
3307 [RXSATRH] = igb_get_rxsatrh,
3308 [TXSTMPH] = igb_get_txstmph,
3309
3310 [CRCERRS ... MPC] = igb_mac_readreg,
3311 [IP6AT ... IP6AT + 3] = igb_mac_readreg,
3312 [IP4AT ... IP4AT + 6] = igb_mac_readreg,
3313 [RA ... RA + 31] = igb_mac_readreg,
3314 [RA2 ... RA2 + 31] = igb_mac_readreg,
3315 [WUPM ... WUPM + 31] = igb_mac_readreg,
3316 [MTA ... MTA + E1000_MC_TBL_SIZE - 1] = igb_mac_readreg,
3317 [VFTA ... VFTA + E1000_VLAN_FILTER_TBL_SIZE - 1] = igb_mac_readreg,
3318 [FFMT ... FFMT + 254] = igb_mac_readreg,
3319 [MDEF ... MDEF + 7] = igb_mac_readreg,
3320 [FTFT ... FTFT + 254] = igb_mac_readreg,
3321 [RETA ... RETA + 31] = igb_mac_readreg,
3322 [RSSRK ... RSSRK + 9] = igb_mac_readreg,
3323 [MAVTV0 ... MAVTV3] = igb_mac_readreg,
3324 [EITR0 ... EITR0 + IGB_INTR_NUM - 1] = igb_mac_eitr_read,
3325 [PVTEICR0] = igb_mac_read_clr4,
3326 [PVTEICR1] = igb_mac_read_clr4,
3327 [PVTEICR2] = igb_mac_read_clr4,
3328 [PVTEICR3] = igb_mac_read_clr4,
3329 [PVTEICR4] = igb_mac_read_clr4,
3330 [PVTEICR5] = igb_mac_read_clr4,
3331 [PVTEICR6] = igb_mac_read_clr4,
3332 [PVTEICR7] = igb_mac_read_clr4,
3333
3334 /* IGB specific: */
3335 [FWSM] = igb_mac_readreg,
3336 [SW_FW_SYNC] = igb_mac_readreg,
3337 [HTCBDPC] = igb_mac_read_clr4,
3338 [EICR] = igb_mac_read_clr4,
3339 [EIMS] = igb_mac_readreg,
3340 [EIAM] = igb_mac_readreg,
3341 [IVAR0 ... IVAR0 + 7] = igb_mac_readreg,
3342 igb_getreg(IVAR_MISC),
3a9926d9
AO		 3343 igb_getreg(TSYNCRXCFG),
3344 [ETQF0 ... ETQF0 + 7] = igb_mac_readreg,
3a977dee
AO		 3345 igb_getreg(VT_CTL),
3346 [P2VMAILBOX0 ... P2VMAILBOX7] = igb_mac_readreg,
3347 [V2PMAILBOX0 ... V2PMAILBOX7] = igb_mac_vfmailbox_read,
3348 igb_getreg(MBVFICR),
3349 [VMBMEM0 ... VMBMEM0 + 127] = igb_mac_readreg,
3350 igb_getreg(MBVFIMR),
3351 igb_getreg(VFLRE),
3352 igb_getreg(VFRE),
3353 igb_getreg(VFTE),
3354 igb_getreg(QDE),
3355 igb_getreg(DTXSWC),
3356 igb_getreg(RPLOLR),
3357 [VLVF0 ... VLVF0 + E1000_VLVF_ARRAY_SIZE - 1] = igb_mac_readreg,
3358 [VMVIR0 ... VMVIR7] = igb_mac_readreg,
3359 [VMOLR0 ... VMOLR7] = igb_mac_readreg,
3360 [WVBR] = igb_mac_read_clr4,
3361 [RQDPC0] = igb_mac_read_clr4,
3362 [RQDPC1] = igb_mac_read_clr4,
3363 [RQDPC2] = igb_mac_read_clr4,
3364 [RQDPC3] = igb_mac_read_clr4,
3365 [RQDPC4] = igb_mac_read_clr4,
3366 [RQDPC5] = igb_mac_read_clr4,
3367 [RQDPC6] = igb_mac_read_clr4,
3368 [RQDPC7] = igb_mac_read_clr4,
3369 [RQDPC8] = igb_mac_read_clr4,
3370 [RQDPC9] = igb_mac_read_clr4,
3371 [RQDPC10] = igb_mac_read_clr4,
3372 [RQDPC11] = igb_mac_read_clr4,
3373 [RQDPC12] = igb_mac_read_clr4,
3374 [RQDPC13] = igb_mac_read_clr4,
3375 [RQDPC14] = igb_mac_read_clr4,
3376 [RQDPC15] = igb_mac_read_clr4,
3377 [VTIVAR ... VTIVAR + 7] = igb_mac_readreg,
3378 [VTIVAR_MISC ... VTIVAR_MISC + 7] = igb_mac_readreg,
3379};
3380enum { IGB_NREADOPS = ARRAY_SIZE(igb_macreg_readops) };
3381
3382#define igb_putreg(x) [x] = igb_mac_writereg
3383typedef void (*writeops)(IGBCore *, int, uint32_t);
3384static const writeops igb_macreg_writeops[] = {
3385 igb_putreg(SWSM),
3386 igb_putreg(WUFC),
3387 igb_putreg(RDBAH0),
3388 igb_putreg(RDBAH1),
3389 igb_putreg(RDBAH2),
3390 igb_putreg(RDBAH3),
3391 igb_putreg(RDBAH4),
3392 igb_putreg(RDBAH5),
3393 igb_putreg(RDBAH6),
3394 igb_putreg(RDBAH7),
3395 igb_putreg(RDBAH8),
3396 igb_putreg(RDBAH9),
3397 igb_putreg(RDBAH10),
3398 igb_putreg(RDBAH11),
3399 igb_putreg(RDBAH12),
3400 igb_putreg(RDBAH13),
3401 igb_putreg(RDBAH14),
3402 igb_putreg(RDBAH15),
3403 igb_putreg(SRRCTL0),
3404 igb_putreg(SRRCTL1),
3405 igb_putreg(SRRCTL2),
3406 igb_putreg(SRRCTL3),
3407 igb_putreg(SRRCTL4),
3408 igb_putreg(SRRCTL5),
3409 igb_putreg(SRRCTL6),
3410 igb_putreg(SRRCTL7),
3411 igb_putreg(SRRCTL8),
3412 igb_putreg(SRRCTL9),
3413 igb_putreg(SRRCTL10),
3414 igb_putreg(SRRCTL11),
3415 igb_putreg(SRRCTL12),
3416 igb_putreg(SRRCTL13),
3417 igb_putreg(SRRCTL14),
3418 igb_putreg(SRRCTL15),
3419 igb_putreg(RXDCTL0),
3420 igb_putreg(RXDCTL1),
3421 igb_putreg(RXDCTL2),
3422 igb_putreg(RXDCTL3),
3423 igb_putreg(RXDCTL4),
3424 igb_putreg(RXDCTL5),
3425 igb_putreg(RXDCTL6),
3426 igb_putreg(RXDCTL7),
3427 igb_putreg(RXDCTL8),
3428 igb_putreg(RXDCTL9),
3429 igb_putreg(RXDCTL10),
3430 igb_putreg(RXDCTL11),
3431 igb_putreg(RXDCTL12),
3432 igb_putreg(RXDCTL13),
3433 igb_putreg(RXDCTL14),
3434 igb_putreg(RXDCTL15),
3435 igb_putreg(LEDCTL),
3436 igb_putreg(TCTL),
3437 igb_putreg(TCTL_EXT),
3438 igb_putreg(DTXCTL),
3439 igb_putreg(RXPBS),
3440 igb_putreg(RQDPC0),
3441 igb_putreg(FCAL),
3442 igb_putreg(FCRUC),
3443 igb_putreg(WUC),
3444 igb_putreg(WUS),
3445 igb_putreg(IPAV),
3446 igb_putreg(TDBAH0),
3447 igb_putreg(TDBAH1),
3448 igb_putreg(TDBAH2),
3449 igb_putreg(TDBAH3),
3450 igb_putreg(TDBAH4),
3451 igb_putreg(TDBAH5),
3452 igb_putreg(TDBAH6),
3453 igb_putreg(TDBAH7),
3454 igb_putreg(TDBAH8),
3455 igb_putreg(TDBAH9),
3456 igb_putreg(TDBAH10),
3457 igb_putreg(TDBAH11),
3458 igb_putreg(TDBAH12),
3459 igb_putreg(TDBAH13),
3460 igb_putreg(TDBAH14),
3461 igb_putreg(TDBAH15),
3462 igb_putreg(IAM),
3463 igb_putreg(MANC),
3464 igb_putreg(MANC2H),
3465 igb_putreg(MFVAL),
3466 igb_putreg(FACTPS),
3467 igb_putreg(FUNCTAG),
3468 igb_putreg(GSCL_1),
3469 igb_putreg(GSCL_2),
3470 igb_putreg(GSCL_3),
3471 igb_putreg(GSCL_4),
3472 igb_putreg(GSCN_0),
3473 igb_putreg(GSCN_1),
3474 igb_putreg(GSCN_2),
3475 igb_putreg(GSCN_3),
3476 igb_putreg(MRQC),
3477 igb_putreg(FLOP),
3478 igb_putreg(FLA),
3479 igb_putreg(TXDCTL0),
3480 igb_putreg(TXDCTL1),
3481 igb_putreg(TXDCTL2),
3482 igb_putreg(TXDCTL3),
3483 igb_putreg(TXDCTL4),
3484 igb_putreg(TXDCTL5),
3485 igb_putreg(TXDCTL6),
3486 igb_putreg(TXDCTL7),
3487 igb_putreg(TXDCTL8),
3488 igb_putreg(TXDCTL9),
3489 igb_putreg(TXDCTL10),
3490 igb_putreg(TXDCTL11),
3491 igb_putreg(TXDCTL12),
3492 igb_putreg(TXDCTL13),
3493 igb_putreg(TXDCTL14),
3494 igb_putreg(TXDCTL15),
3495 igb_putreg(TXCTL0),
3496 igb_putreg(TXCTL1),
3497 igb_putreg(TXCTL2),
3498 igb_putreg(TXCTL3),
3499 igb_putreg(TXCTL4),
3500 igb_putreg(TXCTL5),
3501 igb_putreg(TXCTL6),
3502 igb_putreg(TXCTL7),
3503 igb_putreg(TXCTL8),
3504 igb_putreg(TXCTL9),
3505 igb_putreg(TXCTL10),
3506 igb_putreg(TXCTL11),
3507 igb_putreg(TXCTL12),
3508 igb_putreg(TXCTL13),
3509 igb_putreg(TXCTL14),
3510 igb_putreg(TXCTL15),
3511 igb_putreg(TDWBAL0),
3512 igb_putreg(TDWBAL1),
3513 igb_putreg(TDWBAL2),
3514 igb_putreg(TDWBAL3),
3515 igb_putreg(TDWBAL4),
3516 igb_putreg(TDWBAL5),
3517 igb_putreg(TDWBAL6),
3518 igb_putreg(TDWBAL7),
3519 igb_putreg(TDWBAL8),
3520 igb_putreg(TDWBAL9),
3521 igb_putreg(TDWBAL10),
3522 igb_putreg(TDWBAL11),
3523 igb_putreg(TDWBAL12),
3524 igb_putreg(TDWBAL13),
3525 igb_putreg(TDWBAL14),
3526 igb_putreg(TDWBAL15),
3527 igb_putreg(TDWBAH0),
3528 igb_putreg(TDWBAH1),
3529 igb_putreg(TDWBAH2),
3530 igb_putreg(TDWBAH3),
3531 igb_putreg(TDWBAH4),
3532 igb_putreg(TDWBAH5),
3533 igb_putreg(TDWBAH6),
3534 igb_putreg(TDWBAH7),
3535 igb_putreg(TDWBAH8),
3536 igb_putreg(TDWBAH9),
3537 igb_putreg(TDWBAH10),
3538 igb_putreg(TDWBAH11),
3539 igb_putreg(TDWBAH12),
3540 igb_putreg(TDWBAH13),
3541 igb_putreg(TDWBAH14),
3542 igb_putreg(TDWBAH15),
3543 igb_putreg(TIPG),
3544 igb_putreg(RXSTMPH),
3545 igb_putreg(RXSTMPL),
3546 igb_putreg(RXSATRL),
3547 igb_putreg(RXSATRH),
3548 igb_putreg(TXSTMPL),
3549 igb_putreg(TXSTMPH),
3550 igb_putreg(SYSTIML),
3551 igb_putreg(SYSTIMH),
3552 igb_putreg(TIMADJL),
3553 igb_putreg(TSYNCRXCTL),
3554 igb_putreg(TSYNCTXCTL),
3555 igb_putreg(EEMNGCTL),
3556 igb_putreg(GPIE),
3557 igb_putreg(TXPBS),
3558 igb_putreg(RLPML),
3559 igb_putreg(VET),
3560
3561 [TDH0] = igb_set_16bit,
3562 [TDH1] = igb_set_16bit,
3563 [TDH2] = igb_set_16bit,
3564 [TDH3] = igb_set_16bit,
3565 [TDH4] = igb_set_16bit,
3566 [TDH5] = igb_set_16bit,
3567 [TDH6] = igb_set_16bit,
3568 [TDH7] = igb_set_16bit,
3569 [TDH8] = igb_set_16bit,
3570 [TDH9] = igb_set_16bit,
3571 [TDH10] = igb_set_16bit,
3572 [TDH11] = igb_set_16bit,
3573 [TDH12] = igb_set_16bit,
3574 [TDH13] = igb_set_16bit,
3575 [TDH14] = igb_set_16bit,
3576 [TDH15] = igb_set_16bit,
3577 [TDT0] = igb_set_tdt,
3578 [TDT1] = igb_set_tdt,
3579 [TDT2] = igb_set_tdt,
3580 [TDT3] = igb_set_tdt,
3581 [TDT4] = igb_set_tdt,
3582 [TDT5] = igb_set_tdt,
3583 [TDT6] = igb_set_tdt,
3584 [TDT7] = igb_set_tdt,
3585 [TDT8] = igb_set_tdt,
3586 [TDT9] = igb_set_tdt,
3587 [TDT10] = igb_set_tdt,
3588 [TDT11] = igb_set_tdt,
3589 [TDT12] = igb_set_tdt,
3590 [TDT13] = igb_set_tdt,
3591 [TDT14] = igb_set_tdt,
3592 [TDT15] = igb_set_tdt,
3593 [MDIC] = igb_set_mdic,
3594 [ICS] = igb_set_ics,
3595 [RDH0] = igb_set_16bit,
3596 [RDH1] = igb_set_16bit,
3597 [RDH2] = igb_set_16bit,
3598 [RDH3] = igb_set_16bit,
3599 [RDH4] = igb_set_16bit,
3600 [RDH5] = igb_set_16bit,
3601 [RDH6] = igb_set_16bit,
3602 [RDH7] = igb_set_16bit,
3603 [RDH8] = igb_set_16bit,
3604 [RDH9] = igb_set_16bit,
3605 [RDH10] = igb_set_16bit,
3606 [RDH11] = igb_set_16bit,
3607 [RDH12] = igb_set_16bit,
3608 [RDH13] = igb_set_16bit,
3609 [RDH14] = igb_set_16bit,
3610 [RDH15] = igb_set_16bit,
3611 [RDT0] = igb_set_rdt,
3612 [RDT1] = igb_set_rdt,
3613 [RDT2] = igb_set_rdt,
3614 [RDT3] = igb_set_rdt,
3615 [RDT4] = igb_set_rdt,
3616 [RDT5] = igb_set_rdt,
3617 [RDT6] = igb_set_rdt,
3618 [RDT7] = igb_set_rdt,
3619 [RDT8] = igb_set_rdt,
3620 [RDT9] = igb_set_rdt,
3621 [RDT10] = igb_set_rdt,
3622 [RDT11] = igb_set_rdt,
3623 [RDT12] = igb_set_rdt,
3624 [RDT13] = igb_set_rdt,
3625 [RDT14] = igb_set_rdt,
3626 [RDT15] = igb_set_rdt,
3627 [IMC] = igb_set_imc,
3628 [IMS] = igb_set_ims,
3629 [ICR] = igb_set_icr,
3630 [EECD] = igb_set_eecd,
3631 [RCTL] = igb_set_rx_control,
3632 [CTRL] = igb_set_ctrl,
3633 [EERD] = igb_set_eerd,
3634 [TDFH] = igb_set_13bit,
3635 [TDFT] = igb_set_13bit,
3636 [TDFHS] = igb_set_13bit,
3637 [TDFTS] = igb_set_13bit,
3638 [TDFPC] = igb_set_13bit,
3639 [RDFH] = igb_set_13bit,
3640 [RDFT] = igb_set_13bit,
3641 [RDFHS] = igb_set_13bit,
3642 [RDFTS] = igb_set_13bit,
3643 [RDFPC] = igb_set_13bit,
3644 [GCR] = igb_set_gcr,
3645 [RXCSUM] = igb_set_rxcsum,
3646 [TDLEN0] = igb_set_dlen,
3647 [TDLEN1] = igb_set_dlen,
3648 [TDLEN2] = igb_set_dlen,
3649 [TDLEN3] = igb_set_dlen,
3650 [TDLEN4] = igb_set_dlen,
3651 [TDLEN5] = igb_set_dlen,
3652 [TDLEN6] = igb_set_dlen,
3653 [TDLEN7] = igb_set_dlen,
3654 [TDLEN8] = igb_set_dlen,
3655 [TDLEN9] = igb_set_dlen,
3656 [TDLEN10] = igb_set_dlen,
3657 [TDLEN11] = igb_set_dlen,
3658 [TDLEN12] = igb_set_dlen,
3659 [TDLEN13] = igb_set_dlen,
3660 [TDLEN14] = igb_set_dlen,
3661 [TDLEN15] = igb_set_dlen,
3662 [RDLEN0] = igb_set_dlen,
3663 [RDLEN1] = igb_set_dlen,
3664 [RDLEN2] = igb_set_dlen,
3665 [RDLEN3] = igb_set_dlen,
3666 [RDLEN4] = igb_set_dlen,
3667 [RDLEN5] = igb_set_dlen,
3668 [RDLEN6] = igb_set_dlen,
3669 [RDLEN7] = igb_set_dlen,
3670 [RDLEN8] = igb_set_dlen,
3671 [RDLEN9] = igb_set_dlen,
3672 [RDLEN10] = igb_set_dlen,
3673 [RDLEN11] = igb_set_dlen,
3674 [RDLEN12] = igb_set_dlen,
3675 [RDLEN13] = igb_set_dlen,
3676 [RDLEN14] = igb_set_dlen,
3677 [RDLEN15] = igb_set_dlen,
3678 [TDBAL0] = igb_set_dbal,
3679 [TDBAL1] = igb_set_dbal,
3680 [TDBAL2] = igb_set_dbal,
3681 [TDBAL3] = igb_set_dbal,
3682 [TDBAL4] = igb_set_dbal,
3683 [TDBAL5] = igb_set_dbal,
3684 [TDBAL6] = igb_set_dbal,
3685 [TDBAL7] = igb_set_dbal,
3686 [TDBAL8] = igb_set_dbal,
3687 [TDBAL9] = igb_set_dbal,
3688 [TDBAL10] = igb_set_dbal,
3689 [TDBAL11] = igb_set_dbal,
3690 [TDBAL12] = igb_set_dbal,
3691 [TDBAL13] = igb_set_dbal,
3692 [TDBAL14] = igb_set_dbal,
3693 [TDBAL15] = igb_set_dbal,
3694 [RDBAL0] = igb_set_dbal,
3695 [RDBAL1] = igb_set_dbal,
3696 [RDBAL2] = igb_set_dbal,
3697 [RDBAL3] = igb_set_dbal,
3698 [RDBAL4] = igb_set_dbal,
3699 [RDBAL5] = igb_set_dbal,
3700 [RDBAL6] = igb_set_dbal,
3701 [RDBAL7] = igb_set_dbal,
3702 [RDBAL8] = igb_set_dbal,
3703 [RDBAL9] = igb_set_dbal,
3704 [RDBAL10] = igb_set_dbal,
3705 [RDBAL11] = igb_set_dbal,
3706 [RDBAL12] = igb_set_dbal,
3707 [RDBAL13] = igb_set_dbal,
3708 [RDBAL14] = igb_set_dbal,
3709 [RDBAL15] = igb_set_dbal,
3710 [STATUS] = igb_set_status,
3711 [PBACLR] = igb_set_pbaclr,
3712 [CTRL_EXT] = igb_set_ctrlext,
3713 [FCAH] = igb_set_16bit,
3714 [FCT] = igb_set_16bit,
3715 [FCTTV] = igb_set_16bit,
3716 [FCRTV] = igb_set_16bit,
3717 [FCRTH] = igb_set_fcrth,
3718 [FCRTL] = igb_set_fcrtl,
3719 [CTRL_DUP] = igb_set_ctrl,
3720 [RFCTL] = igb_set_rfctl,
3721 [TIMINCA] = igb_set_timinca,
3722 [TIMADJH] = igb_set_timadjh,
3723
3724 [IP6AT ... IP6AT + 3] = igb_mac_writereg,
3725 [IP4AT ... IP4AT + 6] = igb_mac_writereg,
3726 [RA] = igb_mac_writereg,
3727 [RA + 1] = igb_mac_setmacaddr,
3728 [RA + 2 ... RA + 31] = igb_mac_writereg,
3729 [RA2 ... RA2 + 31] = igb_mac_writereg,
3730 [WUPM ... WUPM + 31] = igb_mac_writereg,
3731 [MTA ... MTA + E1000_MC_TBL_SIZE - 1] = igb_mac_writereg,
3732 [VFTA ... VFTA + E1000_VLAN_FILTER_TBL_SIZE - 1] = igb_mac_writereg,
3733 [FFMT ... FFMT + 254] = igb_set_4bit,
3734 [MDEF ... MDEF + 7] = igb_mac_writereg,
3735 [FTFT ... FTFT + 254] = igb_mac_writereg,
3736 [RETA ... RETA + 31] = igb_mac_writereg,
3737 [RSSRK ... RSSRK + 9] = igb_mac_writereg,
3738 [MAVTV0 ... MAVTV3] = igb_mac_writereg,
3739 [EITR0 ... EITR0 + IGB_INTR_NUM - 1] = igb_set_eitr,
3740
3741 /* IGB specific: */
3742 [FWSM] = igb_mac_writereg,
3743 [SW_FW_SYNC] = igb_mac_writereg,
3744 [EICR] = igb_set_eicr,
3745 [EICS] = igb_set_eics,
3746 [EIAC] = igb_set_eiac,
3747 [EIAM] = igb_set_eiam,
3748 [EIMC] = igb_set_eimc,
3749 [EIMS] = igb_set_eims,
3750 [IVAR0 ... IVAR0 + 7] = igb_mac_writereg,
3751 igb_putreg(IVAR_MISC),
3a9926d9
AO		 3752 igb_putreg(TSYNCRXCFG),
3753 [ETQF0 ... ETQF0 + 7] = igb_mac_writereg,
3a977dee
AO		 3754 igb_putreg(VT_CTL),
3755 [P2VMAILBOX0 ... P2VMAILBOX7] = igb_set_pfmailbox,
3756 [V2PMAILBOX0 ... V2PMAILBOX7] = igb_set_vfmailbox,
3757 [MBVFICR] = igb_w1c,
3758 [VMBMEM0 ... VMBMEM0 + 127] = igb_mac_writereg,
3759 igb_putreg(MBVFIMR),
3760 [VFLRE] = igb_w1c,
3761 igb_putreg(VFRE),
3762 igb_putreg(VFTE),
3763 igb_putreg(QDE),
3764 igb_putreg(DTXSWC),
3765 igb_putreg(RPLOLR),
3766 [VLVF0 ... VLVF0 + E1000_VLVF_ARRAY_SIZE - 1] = igb_mac_writereg,
3767 [VMVIR0 ... VMVIR7] = igb_mac_writereg,
3768 [VMOLR0 ... VMOLR7] = igb_mac_writereg,
3769 [UTA ... UTA + E1000_MC_TBL_SIZE - 1] = igb_mac_writereg,
3770 [PVTCTRL0] = igb_set_vtctrl,
3771 [PVTCTRL1] = igb_set_vtctrl,
3772 [PVTCTRL2] = igb_set_vtctrl,
3773 [PVTCTRL3] = igb_set_vtctrl,
3774 [PVTCTRL4] = igb_set_vtctrl,
3775 [PVTCTRL5] = igb_set_vtctrl,
3776 [PVTCTRL6] = igb_set_vtctrl,
3777 [PVTCTRL7] = igb_set_vtctrl,
3778 [PVTEICS0] = igb_set_vteics,
3779 [PVTEICS1] = igb_set_vteics,
3780 [PVTEICS2] = igb_set_vteics,
3781 [PVTEICS3] = igb_set_vteics,
3782 [PVTEICS4] = igb_set_vteics,
3783 [PVTEICS5] = igb_set_vteics,
3784 [PVTEICS6] = igb_set_vteics,
3785 [PVTEICS7] = igb_set_vteics,
3786 [PVTEIMS0] = igb_set_vteims,
3787 [PVTEIMS1] = igb_set_vteims,
3788 [PVTEIMS2] = igb_set_vteims,
3789 [PVTEIMS3] = igb_set_vteims,
3790 [PVTEIMS4] = igb_set_vteims,
3791 [PVTEIMS5] = igb_set_vteims,
3792 [PVTEIMS6] = igb_set_vteims,
3793 [PVTEIMS7] = igb_set_vteims,
3794 [PVTEIMC0] = igb_set_vteimc,
3795 [PVTEIMC1] = igb_set_vteimc,
3796 [PVTEIMC2] = igb_set_vteimc,
3797 [PVTEIMC3] = igb_set_vteimc,
3798 [PVTEIMC4] = igb_set_vteimc,
3799 [PVTEIMC5] = igb_set_vteimc,
3800 [PVTEIMC6] = igb_set_vteimc,
3801 [PVTEIMC7] = igb_set_vteimc,
3802 [PVTEIAC0] = igb_set_vteiac,
3803 [PVTEIAC1] = igb_set_vteiac,
3804 [PVTEIAC2] = igb_set_vteiac,
3805 [PVTEIAC3] = igb_set_vteiac,
3806 [PVTEIAC4] = igb_set_vteiac,
3807 [PVTEIAC5] = igb_set_vteiac,
3808 [PVTEIAC6] = igb_set_vteiac,
3809 [PVTEIAC7] = igb_set_vteiac,
3810 [PVTEIAM0] = igb_set_vteiam,
3811 [PVTEIAM1] = igb_set_vteiam,
3812 [PVTEIAM2] = igb_set_vteiam,
3813 [PVTEIAM3] = igb_set_vteiam,
3814 [PVTEIAM4] = igb_set_vteiam,
3815 [PVTEIAM5] = igb_set_vteiam,
3816 [PVTEIAM6] = igb_set_vteiam,
3817 [PVTEIAM7] = igb_set_vteiam,
3818 [PVTEICR0] = igb_set_vteicr,
3819 [PVTEICR1] = igb_set_vteicr,
3820 [PVTEICR2] = igb_set_vteicr,
3821 [PVTEICR3] = igb_set_vteicr,
3822 [PVTEICR4] = igb_set_vteicr,
3823 [PVTEICR5] = igb_set_vteicr,
3824 [PVTEICR6] = igb_set_vteicr,
3825 [PVTEICR7] = igb_set_vteicr,
3826 [VTIVAR ... VTIVAR + 7] = igb_set_vtivar,
3827 [VTIVAR_MISC ... VTIVAR_MISC + 7] = igb_mac_writereg
3828};
3829enum { IGB_NWRITEOPS = ARRAY_SIZE(igb_macreg_writeops) };
3830
3831enum { MAC_ACCESS_PARTIAL = 1 };
3832
3833/*
3834 * The array below combines alias offsets of the index values for the
3835 * MAC registers that have aliases, with the indication of not fully
3836 * implemented registers (lowest bit). This combination is possible
3837 * because all of the offsets are even.
3838 */
3839static const uint16_t mac_reg_access[E1000E_MAC_SIZE] = {
3840 /* Alias index offsets */
3841 [FCRTL_A] = 0x07fe,
3842 [RDFH_A] = 0xe904, [RDFT_A] = 0xe904,
3843 [TDFH_A] = 0xed00, [TDFT_A] = 0xed00,
3844 [RA_A ... RA_A + 31] = 0x14f0,
3845 [VFTA_A ... VFTA_A + E1000_VLAN_FILTER_TBL_SIZE - 1] = 0x1400,
3846
3847 [RDBAL0_A] = 0x2600,
3848 [RDBAH0_A] = 0x2600,
3849 [RDLEN0_A] = 0x2600,
3850 [SRRCTL0_A] = 0x2600,
3851 [RDH0_A] = 0x2600,
3852 [RDT0_A] = 0x2600,
3853 [RXDCTL0_A] = 0x2600,
3854 [RXCTL0_A] = 0x2600,
3855 [RQDPC0_A] = 0x2600,
3856 [RDBAL1_A] = 0x25D0,
3857 [RDBAL2_A] = 0x25A0,
3858 [RDBAL3_A] = 0x2570,
3859 [RDBAH1_A] = 0x25D0,
3860 [RDBAH2_A] = 0x25A0,
3861 [RDBAH3_A] = 0x2570,
3862 [RDLEN1_A] = 0x25D0,
3863 [RDLEN2_A] = 0x25A0,
3864 [RDLEN3_A] = 0x2570,
3865 [SRRCTL1_A] = 0x25D0,
3866 [SRRCTL2_A] = 0x25A0,
3867 [SRRCTL3_A] = 0x2570,
3868 [RDH1_A] = 0x25D0,
3869 [RDH2_A] = 0x25A0,
3870 [RDH3_A] = 0x2570,
3871 [RDT1_A] = 0x25D0,
3872 [RDT2_A] = 0x25A0,
3873 [RDT3_A] = 0x2570,
3874 [RXDCTL1_A] = 0x25D0,
3875 [RXDCTL2_A] = 0x25A0,
3876 [RXDCTL3_A] = 0x2570,
3877 [RXCTL1_A] = 0x25D0,
3878 [RXCTL2_A] = 0x25A0,
3879 [RXCTL3_A] = 0x2570,
3880 [RQDPC1_A] = 0x25D0,
3881 [RQDPC2_A] = 0x25A0,
3882 [RQDPC3_A] = 0x2570,
3883 [TDBAL0_A] = 0x2A00,
3884 [TDBAH0_A] = 0x2A00,
3885 [TDLEN0_A] = 0x2A00,
3886 [TDH0_A] = 0x2A00,
3887 [TDT0_A] = 0x2A00,
3888 [TXCTL0_A] = 0x2A00,
3889 [TDWBAL0_A] = 0x2A00,
3890 [TDWBAH0_A] = 0x2A00,
3891 [TDBAL1_A] = 0x29D0,
3892 [TDBAL2_A] = 0x29A0,
3893 [TDBAL3_A] = 0x2970,
3894 [TDBAH1_A] = 0x29D0,
3895 [TDBAH2_A] = 0x29A0,
3896 [TDBAH3_A] = 0x2970,
3897 [TDLEN1_A] = 0x29D0,
3898 [TDLEN2_A] = 0x29A0,
3899 [TDLEN3_A] = 0x2970,
3900 [TDH1_A] = 0x29D0,
3901 [TDH2_A] = 0x29A0,
3902 [TDH3_A] = 0x2970,
3903 [TDT1_A] = 0x29D0,
3904 [TDT2_A] = 0x29A0,
3905 [TDT3_A] = 0x2970,
3906 [TXDCTL0_A] = 0x2A00,
3907 [TXDCTL1_A] = 0x29D0,
3908 [TXDCTL2_A] = 0x29A0,
3909 [TXDCTL3_A] = 0x2970,
3910 [TXCTL1_A] = 0x29D0,
3911 [TXCTL2_A] = 0x29A0,
3912 [TXCTL3_A] = 0x29D0,
3913 [TDWBAL1_A] = 0x29D0,
3914 [TDWBAL2_A] = 0x29A0,
3915 [TDWBAL3_A] = 0x2970,
3916 [TDWBAH1_A] = 0x29D0,
3917 [TDWBAH2_A] = 0x29A0,
3918 [TDWBAH3_A] = 0x2970,
3919
3920 /* Access options */
3921 [RDFH] = MAC_ACCESS_PARTIAL, [RDFT] = MAC_ACCESS_PARTIAL,
3922 [RDFHS] = MAC_ACCESS_PARTIAL, [RDFTS] = MAC_ACCESS_PARTIAL,
3923 [RDFPC] = MAC_ACCESS_PARTIAL,
3924 [TDFH] = MAC_ACCESS_PARTIAL, [TDFT] = MAC_ACCESS_PARTIAL,
3925 [TDFHS] = MAC_ACCESS_PARTIAL, [TDFTS] = MAC_ACCESS_PARTIAL,
3926 [TDFPC] = MAC_ACCESS_PARTIAL, [EECD] = MAC_ACCESS_PARTIAL,
3927 [FLA] = MAC_ACCESS_PARTIAL,
3928 [FCAL] = MAC_ACCESS_PARTIAL, [FCAH] = MAC_ACCESS_PARTIAL,
3929 [FCT] = MAC_ACCESS_PARTIAL, [FCTTV] = MAC_ACCESS_PARTIAL,
3930 [FCRTV] = MAC_ACCESS_PARTIAL, [FCRTL] = MAC_ACCESS_PARTIAL,
3931 [FCRTH] = MAC_ACCESS_PARTIAL,
3932 [MAVTV0 ... MAVTV3] = MAC_ACCESS_PARTIAL
3933};
3934
3935void
3936igb_core_write(IGBCore *core, hwaddr addr, uint64_t val, unsigned size)
3937{
3938 uint16_t index = igb_get_reg_index_with_offset(mac_reg_access, addr);
3939
3940 if (index < IGB_NWRITEOPS && igb_macreg_writeops[index]) {
3941 if (mac_reg_access[index] & MAC_ACCESS_PARTIAL) {
3942 trace_e1000e_wrn_regs_write_trivial(index << 2);
3943 }
3944 trace_e1000e_core_write(index << 2, size, val);
3945 igb_macreg_writeops[index](core, index, val);
3946 } else if (index < IGB_NREADOPS && igb_macreg_readops[index]) {
3947 trace_e1000e_wrn_regs_write_ro(index << 2, size, val);
3948 } else {
3949 trace_e1000e_wrn_regs_write_unknown(index << 2, size, val);
3950 }
3951}
3952
3953uint64_t
3954igb_core_read(IGBCore *core, hwaddr addr, unsigned size)
3955{
3956 uint64_t val;
3957 uint16_t index = igb_get_reg_index_with_offset(mac_reg_access, addr);
3958
3959 if (index < IGB_NREADOPS && igb_macreg_readops[index]) {
3960 if (mac_reg_access[index] & MAC_ACCESS_PARTIAL) {
3961 trace_e1000e_wrn_regs_read_trivial(index << 2);
3962 }
3963 val = igb_macreg_readops[index](core, index);
3964 trace_e1000e_core_read(index << 2, size, val);
3965 return val;
3966 } else {
3967 trace_e1000e_wrn_regs_read_unknown(index << 2, size);
3968 }
3969 return 0;
3970}
3971
3972static inline void
3973igb_autoneg_pause(IGBCore *core)
3974{
3975 timer_del(core->autoneg_timer);
3976}
3977
3978static void
3979igb_autoneg_resume(IGBCore *core)
3980{
3981 if (igb_have_autoneg(core) &&
3982 !(core->phy[MII_BMSR] & MII_BMSR_AN_COMP)) {
3983 qemu_get_queue(core->owner_nic)->link_down = false;
3984 timer_mod(core->autoneg_timer,
3985 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 500);
3986 }
3987}
3988
3989static void
3990igb_vm_state_change(void *opaque, bool running, RunState state)
3991{
3992 IGBCore *core = opaque;
3993
3994 if (running) {
3995 trace_e1000e_vm_state_running();
3996 igb_intrmgr_resume(core);
3997 igb_autoneg_resume(core);
3998 } else {
3999 trace_e1000e_vm_state_stopped();
4000 igb_autoneg_pause(core);
4001 igb_intrmgr_pause(core);
4002 }
4003}
4004
4005void
4006igb_core_pci_realize(IGBCore *core,
4007 const uint16_t *eeprom_templ,
4008 uint32_t eeprom_size,
4009 const uint8_t *macaddr)
4010{
4011 int i;
4012
4013 core->autoneg_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
4014 igb_autoneg_timer, core);
4015 igb_intrmgr_pci_realize(core);
4016
4017 core->vmstate = qemu_add_vm_change_state_handler(igb_vm_state_change, core);
4018
4019 for (i = 0; i < IGB_NUM_QUEUES; i++) {
a51db580 4020 net_tx_pkt_init(&core->tx[i].tx_pkt, E1000E_MAX_TX_FRAGS);
3a977dee
AO		 4021 }
4022
4023 net_rx_pkt_init(&core->rx_pkt);
4024
4025 e1000x_core_prepare_eeprom(core->eeprom,
4026 eeprom_templ,
4027 eeprom_size,
4028 PCI_DEVICE_GET_CLASS(core->owner)->device_id,
4029 macaddr);
4030 igb_update_rx_offloads(core);
4031}
4032
4033void
4034igb_core_pci_uninit(IGBCore *core)
4035{
4036 int i;
4037
4038 timer_free(core->autoneg_timer);
4039
4040 igb_intrmgr_pci_unint(core);
4041
4042 qemu_del_vm_change_state_handler(core->vmstate);
4043
4044 for (i = 0; i < IGB_NUM_QUEUES; i++) {
3a977dee
AO		 4045 net_tx_pkt_uninit(core->tx[i].tx_pkt);
4046 }
4047
4048 net_rx_pkt_uninit(core->rx_pkt);
4049}
4050
4051static const uint16_t
4052igb_phy_reg_init[] = {
4053 [MII_BMCR] = MII_BMCR_SPEED1000 |
4054 MII_BMCR_FD |
4055 MII_BMCR_AUTOEN,
4056
4057 [MII_BMSR] = MII_BMSR_EXTCAP |
4058 MII_BMSR_LINK_ST |
4059 MII_BMSR_AUTONEG |
4060 MII_BMSR_MFPS |
4061 MII_BMSR_EXTSTAT |
4062 MII_BMSR_10T_HD |
4063 MII_BMSR_10T_FD |
4064 MII_BMSR_100TX_HD |
4065 MII_BMSR_100TX_FD,
4066
4067 [MII_PHYID1] = IGP03E1000_E_PHY_ID >> 16,
4068 [MII_PHYID2] = (IGP03E1000_E_PHY_ID & 0xfff0) | 1,
4069 [MII_ANAR] = MII_ANAR_CSMACD | MII_ANAR_10 |
4070 MII_ANAR_10FD | MII_ANAR_TX |
4071 MII_ANAR_TXFD | MII_ANAR_PAUSE |
4072 MII_ANAR_PAUSE_ASYM,
4073 [MII_ANLPAR] = MII_ANLPAR_10 | MII_ANLPAR_10FD |
4074 MII_ANLPAR_TX | MII_ANLPAR_TXFD |
4075 MII_ANLPAR_T4 | MII_ANLPAR_PAUSE,
4076 [MII_ANER] = MII_ANER_NP | MII_ANER_NWAY,
4077 [MII_ANNP] = 0x1 | MII_ANNP_MP,
4078 [MII_CTRL1000] = MII_CTRL1000_HALF | MII_CTRL1000_FULL |
4079 MII_CTRL1000_PORT | MII_CTRL1000_MASTER,
4080 [MII_STAT1000] = MII_STAT1000_HALF | MII_STAT1000_FULL |
4081 MII_STAT1000_ROK | MII_STAT1000_LOK,
4082 [MII_EXTSTAT] = MII_EXTSTAT_1000T_HD | MII_EXTSTAT_1000T_FD,
4083
4084 [IGP01E1000_PHY_PORT_CONFIG] = BIT(5) | BIT(8),
4085 [IGP01E1000_PHY_PORT_STATUS] = IGP01E1000_PSSR_SPEED_1000MBPS,
4086 [IGP02E1000_PHY_POWER_MGMT] = BIT(0) | BIT(3) | IGP02E1000_PM_D3_LPLU |
4087 IGP01E1000_PSCFR_SMART_SPEED
4088};
4089
4090static const uint32_t igb_mac_reg_init[] = {
4091 [LEDCTL] = 2 | (3 << 8) | BIT(15) | (6 << 16) | (7 << 24),
4092 [EEMNGCTL] = BIT(31),
3269ebb3 4093 [TXDCTL0] = E1000_TXDCTL_QUEUE_ENABLE,
3a977dee
AO		 4094 [RXDCTL0] = E1000_RXDCTL_QUEUE_ENABLE | (1 << 16),
4095 [RXDCTL1] = 1 << 16,
4096 [RXDCTL2] = 1 << 16,
4097 [RXDCTL3] = 1 << 16,
4098 [RXDCTL4] = 1 << 16,
4099 [RXDCTL5] = 1 << 16,
4100 [RXDCTL6] = 1 << 16,
4101 [RXDCTL7] = 1 << 16,
4102 [RXDCTL8] = 1 << 16,
4103 [RXDCTL9] = 1 << 16,
4104 [RXDCTL10] = 1 << 16,
4105 [RXDCTL11] = 1 << 16,
4106 [RXDCTL12] = 1 << 16,
4107 [RXDCTL13] = 1 << 16,
4108 [RXDCTL14] = 1 << 16,
4109 [RXDCTL15] = 1 << 16,
4110 [TIPG] = 0x08 | (0x04 << 10) | (0x06 << 20),
4111 [CTRL] = E1000_CTRL_FD | E1000_CTRL_LRST | E1000_CTRL_SPD_1000 |
4112 E1000_CTRL_ADVD3WUC,
4113 [STATUS] = E1000_STATUS_PHYRA | BIT(31),
4114 [EECD] = E1000_EECD_FWE_DIS | E1000_EECD_PRES |
4115 (2 << E1000_EECD_SIZE_EX_SHIFT),
4116 [GCR] = E1000_L0S_ADJUST |
4117 E1000_GCR_CMPL_TMOUT_RESEND |
4118 E1000_GCR_CAP_VER2 |
4119 E1000_L1_ENTRY_LATENCY_MSB |
4120 E1000_L1_ENTRY_LATENCY_LSB,
4121 [RXCSUM] = E1000_RXCSUM_IPOFLD | E1000_RXCSUM_TUOFLD,
4122 [TXPBS] = 0x28,
4123 [RXPBS] = 0x40,
4124 [TCTL] = E1000_TCTL_PSP | (0xF << E1000_CT_SHIFT) |
4125 (0x40 << E1000_COLD_SHIFT) | (0x1 << 26) | (0xA << 28),
4126 [TCTL_EXT] = 0x40 | (0x42 << 10),
4127 [DTXCTL] = E1000_DTXCTL_8023LL | E1000_DTXCTL_SPOOF_INT,
4128 [VET] = ETH_P_VLAN | (ETH_P_VLAN << 16),
4129
4130 [V2PMAILBOX0 ... V2PMAILBOX0 + IGB_MAX_VF_FUNCTIONS - 1] = E1000_V2PMAILBOX_RSTI,
4131 [MBVFIMR] = 0xFF,
4132 [VFRE] = 0xFF,
4133 [VFTE] = 0xFF,
4134 [VMOLR0 ... VMOLR0 + 7] = 0x2600 | E1000_VMOLR_STRCRC,
4135 [RPLOLR] = E1000_RPLOLR_STRCRC,
4136 [RLPML] = 0x2600,
882e54da
AO		 4137 [TXCTL0] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4138 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4139 E1000_DCA_TXCTRL_DESC_RRO_EN,
4140 [TXCTL1] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4141 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4142 E1000_DCA_TXCTRL_DESC_RRO_EN,
4143 [TXCTL2] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4144 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4145 E1000_DCA_TXCTRL_DESC_RRO_EN,
4146 [TXCTL3] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4147 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4148 E1000_DCA_TXCTRL_DESC_RRO_EN,
4149 [TXCTL4] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4150 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4151 E1000_DCA_TXCTRL_DESC_RRO_EN,
4152 [TXCTL5] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4153 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4154 E1000_DCA_TXCTRL_DESC_RRO_EN,
4155 [TXCTL6] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4156 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4157 E1000_DCA_TXCTRL_DESC_RRO_EN,
4158 [TXCTL7] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4159 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4160 E1000_DCA_TXCTRL_DESC_RRO_EN,
4161 [TXCTL8] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4162 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4163 E1000_DCA_TXCTRL_DESC_RRO_EN,
4164 [TXCTL9] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4165 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4166 E1000_DCA_TXCTRL_DESC_RRO_EN,
4167 [TXCTL10] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4168 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4169 E1000_DCA_TXCTRL_DESC_RRO_EN,
4170 [TXCTL11] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4171 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4172 E1000_DCA_TXCTRL_DESC_RRO_EN,
4173 [TXCTL12] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4174 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4175 E1000_DCA_TXCTRL_DESC_RRO_EN,
4176 [TXCTL13] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4177 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4178 E1000_DCA_TXCTRL_DESC_RRO_EN,
4179 [TXCTL14] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4180 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4181 E1000_DCA_TXCTRL_DESC_RRO_EN,
4182 [TXCTL15] = E1000_DCA_TXCTRL_DATA_RRO_EN |
4183 E1000_DCA_TXCTRL_TX_WB_RO_EN |
4184 E1000_DCA_TXCTRL_DESC_RRO_EN,
3a977dee
AO		 4185};
4186
4187static void igb_reset(IGBCore *core, bool sw)
4188{
4189 struct igb_tx *tx;
4190 int i;
4191
4192 timer_del(core->autoneg_timer);
4193
4194 igb_intrmgr_reset(core);
4195
4196 memset(core->phy, 0, sizeof core->phy);
4197 memcpy(core->phy, igb_phy_reg_init, sizeof igb_phy_reg_init);
4198
4199 for (i = 0; i < E1000E_MAC_SIZE; i++) {
4200 if (sw &&
4201 (i == RXPBS || i == TXPBS ||
4202 (i >= EITR0 && i < EITR0 + IGB_INTR_NUM))) {
4203 continue;
4204 }
4205
4206 core->mac[i] = i < ARRAY_SIZE(igb_mac_reg_init) ?
4207 igb_mac_reg_init[i] : 0;
4208 }
4209
4210 if (qemu_get_queue(core->owner_nic)->link_down) {
4211 igb_link_down(core);
4212 }
4213
4214 e1000x_reset_mac_addr(core->owner_nic, core->mac, core->permanent_mac);
4215
2e68546a
SY		 4216 for (int vfn = 0; vfn < IGB_MAX_VF_FUNCTIONS; vfn++) {
4217 /* Set RSTI, so VF can identify a PF reset is in progress */
4218 core->mac[V2PMAILBOX0 + vfn] |= E1000_V2PMAILBOX_RSTI;
4219 }
4220
3a977dee
AO		 4221 for (i = 0; i < ARRAY_SIZE(core->tx); i++) {
4222 tx = &core->tx[i];
212f7b1d 4223 memset(tx->ctx, 0, sizeof(tx->ctx));
3a977dee
AO		 4224 tx->first = true;
4225 tx->skip_cp = false;
4226 }
4227}
4228
4229void
4230igb_core_reset(IGBCore *core)
4231{
4232 igb_reset(core, false);
4233}
4234
4235void igb_core_pre_save(IGBCore *core)
4236{
4237 int i;
4238 NetClientState *nc = qemu_get_queue(core->owner_nic);
4239
4240 /*
4241 * If link is down and auto-negotiation is supported and ongoing,
4242 * complete auto-negotiation immediately. This allows us to look
4243 * at MII_BMSR_AN_COMP to infer link status on load.
4244 */
4245 if (nc->link_down && igb_have_autoneg(core)) {
4246 core->phy[MII_BMSR] |= MII_BMSR_AN_COMP;
4247 igb_update_flowctl_status(core);
4248 }
4249
4250 for (i = 0; i < ARRAY_SIZE(core->tx); i++) {
4251 if (net_tx_pkt_has_fragments(core->tx[i].tx_pkt)) {
4252 core->tx[i].skip_cp = true;
4253 }
4254 }
4255}
4256
4257int
4258igb_core_post_load(IGBCore *core)
4259{
4260 NetClientState *nc = qemu_get_queue(core->owner_nic);
4261
4262 /*
4263 * nc.link_down can't be migrated, so infer link_down according
4264 * to link status bit in core.mac[STATUS].
4265 */
4266 nc->link_down = (core->mac[STATUS] & E1000_STATUS_LU) == 0;
4267
4268 return 0;
4269}
QEMU mirror