datapath-windows/ovsext/OvsUser.c

   1 /*
   2  * Copyright (c) 2014 VMware, Inc.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at:
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /*
  18  * OvsUser.c
  19  *      Manage packet queue for packet miss for userAction.
  20  */
  21
  22
  23 #include "precomp.h"
  24
  25 #include "Datapath.h"
  26 #include "OvsSwitch.h"
  27 #include "OvsVport.h"
  28 #include "OvsEvent.h"
  29 #include "OvsUser.h"
  30 #include "OvsPacketIO.h"
  31 #include "OvsChecksum.h"
  32 #include "OvsNetProto.h"
  33 #include "OvsFlow.h"
  34 #include "OvsTunnelIntf.h"
  35
  36 #ifdef OVS_DBG_MOD
  37 #undef OVS_DBG_MOD
  38 #endif
  39 #define OVS_DBG_MOD OVS_DBG_USER
  40 #include "OvsDebug.h"
  41
  42 OVS_USER_PACKET_QUEUE ovsPacketQueues[OVS_MAX_NUM_PACKET_QUEUES];
  43
  44 POVS_PACKET_QUEUE_ELEM OvsGetNextPacket(POVS_OPEN_INSTANCE instance);
  45 extern PNDIS_SPIN_LOCK gOvsCtrlLock;
  46 extern POVS_SWITCH_CONTEXT gOvsSwitchContext;
  47 OVS_USER_STATS ovsUserStats;
  48
  49
  50 NTSTATUS
  51 OvsUserInit()
  52 {
  53     UINT32 i;
  54     POVS_USER_PACKET_QUEUE queue;
  55     for (i = 0; i < OVS_MAX_NUM_PACKET_QUEUES; i++) {
  56         queue = &ovsPacketQueues[i];
  57         RtlZeroMemory(queue, sizeof (*queue));
  58         InitializeListHead(&queue->packetList);
  59         NdisAllocateSpinLock(&queue->queueLock);
  60     }
  61     return STATUS_SUCCESS;
  62 }
  63
  64 VOID
  65 OvsUserCleanup()
  66 {
  67     UINT32 i;
  68     POVS_USER_PACKET_QUEUE queue;
  69     for (i = 0; i < OVS_MAX_NUM_PACKET_QUEUES; i++) {
  70         queue = &ovsPacketQueues[i];
  71         ASSERT(IsListEmpty(&queue->packetList));
  72         ASSERT(queue->instance == NULL);
  73         ASSERT(queue->pendingIrp == NULL);
  74         NdisFreeSpinLock(&queue->queueLock);
  75     }
  76 }
  77
  78 static VOID
  79 OvsPurgePacketQueue(POVS_USER_PACKET_QUEUE queue,
  80                     POVS_OPEN_INSTANCE instance)
  81 {
  82     PLIST_ENTRY link, next;
  83     LIST_ENTRY tmp;
  84     POVS_PACKET_QUEUE_ELEM elem;
  85
  86     InitializeListHead(&tmp);
  87     NdisAcquireSpinLock(&queue->queueLock);
  88     if (queue->instance != instance) {
  89         NdisReleaseSpinLock(&queue->queueLock);
  90         return;
  91     }
  92
  93     if (queue->numPackets) {
  94         OvsAppendList(&tmp, &queue->packetList);
  95         queue->numPackets = 0;
  96     }
  97     NdisReleaseSpinLock(&queue->queueLock);
  98     LIST_FORALL_SAFE(&tmp, link, next) {
  99         RemoveEntryList(link);
 100         elem = CONTAINING_RECORD(link, OVS_PACKET_QUEUE_ELEM, link);
 101         OvsFreeMemory(elem);
 102     }
 103 }
 104
 105
 106 VOID
 107 OvsCleanupPacketQueue(POVS_OPEN_INSTANCE instance)
 108 {
 109     POVS_USER_PACKET_QUEUE queue;
 110     POVS_PACKET_QUEUE_ELEM elem;
 111     PLIST_ENTRY link, next;
 112     LIST_ENTRY tmp;
 113     PIRP irp = NULL;
 114
 115     InitializeListHead(&tmp);
 116     queue = (POVS_USER_PACKET_QUEUE)instance->packetQueue;
 117     if (queue) {
 118         PDRIVER_CANCEL cancelRoutine;
 119         NdisAcquireSpinLock(&queue->queueLock);
 120         if (queue->instance != instance) {
 121             NdisReleaseSpinLock(&queue->queueLock);
 122             return;
 123         }
 124
 125         if (queue->numPackets) {
 126             OvsAppendList(&tmp, &queue->packetList);
 127             queue->numPackets = 0;
 128         }
 129         queue->instance = NULL;
 130         queue->queueId = OVS_MAX_NUM_PACKET_QUEUES;
 131         instance->packetQueue = NULL;
 132         irp = queue->pendingIrp;
 133         queue->pendingIrp = NULL;
 134         if (irp) {
 135             cancelRoutine = IoSetCancelRoutine(irp, NULL);
 136             if (cancelRoutine == NULL) {
 137                 irp = NULL;
 138             }
 139         }
 140         NdisReleaseSpinLock(&queue->queueLock);
 141     }
 142     LIST_FORALL_SAFE(&tmp, link, next) {
 143         RemoveEntryList(link);
 144         elem = CONTAINING_RECORD(link, OVS_PACKET_QUEUE_ELEM, link);
 145         OvsFreeMemory(elem);
 146     }
 147     if (irp) {
 148         OvsCompleteIrpRequest(irp, 0, STATUS_SUCCESS);
 149     }
 150 }
 151
 152 NTSTATUS
 153 OvsSubscribeDpIoctl(PFILE_OBJECT fileObject,
 154                     PVOID inputBuffer,
 155                     UINT32 inputLength)
 156 {
 157     POVS_OPEN_INSTANCE instance = (POVS_OPEN_INSTANCE)fileObject->FsContext;
 158     UINT32 queueId;
 159     POVS_USER_PACKET_QUEUE queue;
 160     if (inputLength < sizeof (UINT32)) {
 161         return STATUS_INVALID_PARAMETER;
 162     }
 163     queueId = *(UINT32 *)inputBuffer;
 164     if (instance->packetQueue && queueId >= OVS_MAX_NUM_PACKET_QUEUES) {
 165         /*
 166          * unsubscribe
 167          */
 168         OvsCleanupPacketQueue(instance);
 169     } else if (instance->packetQueue == NULL &&
 170                queueId < OVS_MAX_NUM_PACKET_QUEUES) {
 171         queue = &ovsPacketQueues[queueId];
 172         NdisAcquireSpinLock(&queue->queueLock);
 173         if (ovsPacketQueues[queueId].instance) {
 174              if (ovsPacketQueues[queueId].instance != instance) {
 175                  NdisReleaseSpinLock(&queue->queueLock);
 176                  return STATUS_INSUFFICIENT_RESOURCES;
 177              } else {
 178                  NdisReleaseSpinLock(&queue->queueLock);
 179                  return STATUS_SUCCESS;
 180              }
 181         }
 182         queue->queueId = queueId;
 183         queue->instance = instance;
 184         instance->packetQueue = queue;
 185         ASSERT(IsListEmpty(&queue->packetList));
 186         NdisReleaseSpinLock(&queue->queueLock);
 187     } else {
 188         return STATUS_INVALID_PARAMETER;
 189     }
 190     return STATUS_SUCCESS;
 191 }
 192
 193
 194 NTSTATUS
 195 OvsReadDpIoctl(PFILE_OBJECT fileObject,
 196                PVOID outputBuffer,
 197                UINT32 outputLength,
 198                UINT32 *replyLen)
 199 {
 200     POVS_OPEN_INSTANCE instance = (POVS_OPEN_INSTANCE)fileObject->FsContext;
 201     POVS_PACKET_QUEUE_ELEM elem;
 202     UINT32 len;
 203
 204 #define TCP_CSUM_OFFSET  16
 205 #define UDP_CSUM_OFFSET  6
 206     ASSERT(instance);
 207
 208     if (instance->packetQueue == NULL) {
 209         return STATUS_INVALID_PARAMETER;
 210     }
 211     if (outputLength < (sizeof (OVS_PACKET_INFO) + OVS_MIN_PACKET_SIZE)) {
 212         return STATUS_BUFFER_TOO_SMALL;
 213     }
 214
 215     elem = OvsGetNextPacket(instance);
 216     if (elem) {
 217         /*
 218          * XXX revisit this later
 219          */
 220         len = elem->packet.totalLen > outputLength ? outputLength :
 221                  elem->packet.totalLen;
 222
 223         if ((elem->hdrInfo.tcpCsumNeeded || elem->hdrInfo.udpCsumNeeded) &&
 224             len == elem->packet.totalLen) {
 225             UINT16 sum, *ptr;
 226             UINT16 size = (UINT16)(elem->packet.userDataLen +
 227                                    elem->hdrInfo.l4Offset +
 228                                    (UINT16)sizeof (OVS_PACKET_INFO));
 229             RtlCopyMemory(outputBuffer, &elem->packet, size);
 230             ASSERT(len - size >=  elem->hdrInfo.l4PayLoad);
 231             sum = CopyAndCalculateChecksum((UINT8 *)outputBuffer + size,
 232                                            (UINT8 *)&elem->packet + size,
 233                                            elem->hdrInfo.l4PayLoad, 0);
 234             ptr =(UINT16 *)((UINT8 *)outputBuffer + size +
 235                             (elem->hdrInfo.tcpCsumNeeded ?
 236                              TCP_CSUM_OFFSET : UDP_CSUM_OFFSET));
 237             *ptr = sum;
 238             ovsUserStats.l4Csum++;
 239         } else {
 240             RtlCopyMemory(outputBuffer, &elem->packet, len);
 241         }
 242
 243         *replyLen = len;
 244         OvsFreeMemory(elem);
 245     }
 246     return STATUS_SUCCESS;
 247 }
 248
 249 /* Helper function to allocate a Forwarding Context for an NBL */
 250 NTSTATUS
 251 OvsAllocateForwardingContextForNBL(POVS_SWITCH_CONTEXT switchContext,
 252                                    PNET_BUFFER_LIST nbl)
 253 {
 254     return switchContext->NdisSwitchHandlers.
 255         AllocateNetBufferListForwardingContext(
 256             switchContext->NdisSwitchContext, nbl);
 257 }
 258
 259 /*
 260  * --------------------------------------------------------------------------
 261  * This function allocates all the stuff necessary for creating an NBL from the
 262  * input buffer of specified length, namely, a nonpaged data buffer of size
 263  * length, an MDL from it, and a NB and NBL from it. It does not allocate an NBL
 264  * context yet. It also copies data from the specified buffer to the NBL.
 265  * --------------------------------------------------------------------------
 266  */
 267 PNET_BUFFER_LIST
 268 OvsAllocateNBLForUserBuffer(POVS_SWITCH_CONTEXT switchContext,
 269                             PVOID userBuffer,
 270                             ULONG length)
 271 {
 272     UINT8 *data = NULL;
 273     PNET_BUFFER_LIST nbl = NULL;
 274     PNET_BUFFER nb;
 275     PMDL mdl;
 276
 277     if (length > OVS_DEFAULT_DATA_SIZE) {
 278         nbl = OvsAllocateVariableSizeNBL(switchContext, length,
 279                                          OVS_DEFAULT_HEADROOM_SIZE);
 280
 281     } else {
 282         nbl = OvsAllocateFixSizeNBL(switchContext, length,
 283                                     OVS_DEFAULT_HEADROOM_SIZE);
 284     }
 285     if (nbl == NULL) {
 286         return NULL;
 287     }
 288
 289     nb = NET_BUFFER_LIST_FIRST_NB(nbl);
 290     mdl = NET_BUFFER_CURRENT_MDL(nb);
 291     data = (PUINT8)MmGetSystemAddressForMdlSafe(mdl, LowPagePriority) +
 292                     NET_BUFFER_CURRENT_MDL_OFFSET(nb);
 293     if (!data) {
 294         OvsCompleteNBL(switchContext, nbl, TRUE);
 295         return NULL;
 296     }
 297
 298     NdisMoveMemory(data, userBuffer, length);
 299
 300     return nbl;
 301 }
 302
 303 NTSTATUS
 304 OvsExecuteDpIoctl(PVOID inputBuffer,
 305                   UINT32 inputLength,
 306                   UINT32 outputLength)
 307 {
 308     NTSTATUS                    status = STATUS_SUCCESS;
 309     NTSTATUS                    ndisStatus;
 310     OvsPacketExecute            *execute;
 311     LOCK_STATE_EX               lockState;
 312     PNET_BUFFER_LIST pNbl;
 313     PNL_ATTR actions;
 314     PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO fwdDetail;
 315     OvsFlowKey key;
 316     OVS_PACKET_HDR_INFO layers;
 317     POVS_VPORT_ENTRY vport;
 318
 319     if (inputLength < sizeof(*execute) || outputLength != 0) {
 320         return STATUS_INFO_LENGTH_MISMATCH;
 321     }
 322
 323     NdisAcquireSpinLock(gOvsCtrlLock);
 324     if (gOvsSwitchContext == NULL) {
 325         status = STATUS_INVALID_PARAMETER;
 326         goto unlock;
 327     }
 328
 329     execute = (struct OvsPacketExecute *) inputBuffer;
 330
 331     if (execute->packetLen == 0) {
 332         status = STATUS_INVALID_PARAMETER;
 333         goto unlock;
 334     }
 335
 336     if (inputLength != sizeof (*execute) +
 337                        execute->actionsLen + execute->packetLen) {
 338         status = STATUS_INFO_LENGTH_MISMATCH;
 339         goto unlock;
 340     }
 341     actions = (PNL_ATTR)((PCHAR)&execute->actions + execute->packetLen);
 342
 343     /*
 344      * Allocate the NBL, copy the data from the userspace buffer. Allocate
 345      * also, the forwarding context for the packet.
 346      */
 347     pNbl = OvsAllocateNBLForUserBuffer(gOvsSwitchContext, &execute->packetBuf,
 348                                        execute->packetLen);
 349     if (pNbl == NULL) {
 350         status = STATUS_NO_MEMORY;
 351         goto unlock;
 352     }
 353
 354     fwdDetail = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(pNbl);
 355     vport = OvsFindVportByPortNo(gOvsSwitchContext, execute->inPort);
 356     if (vport) {
 357         fwdDetail->SourcePortId = vport->portId;
 358         fwdDetail->SourceNicIndex = vport->nicIndex;
 359     } else {
 360         fwdDetail->SourcePortId = NDIS_SWITCH_DEFAULT_PORT_ID;
 361         fwdDetail->SourceNicIndex = 0;
 362     }
 363     // XXX: Figure out if any of the other members of fwdDetail need to be set.
 364
 365     ndisStatus = OvsExtractFlow(pNbl, fwdDetail->SourcePortId, &key, &layers,
 366                               NULL);
 367     if (ndisStatus == NDIS_STATUS_SUCCESS) {
 368         ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 369         NdisAcquireRWLockRead(gOvsSwitchContext->dispatchLock, &lockState,
 370                               NDIS_RWL_AT_DISPATCH_LEVEL);
 371         ndisStatus = OvsActionsExecute(gOvsSwitchContext, NULL, pNbl,
 372                                        vport ? vport->portNo :
 373                                                OVS_DEFAULT_PORT_NO,
 374                                        NDIS_SEND_FLAGS_SWITCH_DESTINATION_GROUP,
 375                                        &key, NULL, &layers, actions,
 376                                        execute->actionsLen);
 377         pNbl = NULL;
 378         NdisReleaseRWLock(gOvsSwitchContext->dispatchLock, &lockState);
 379     }
 380     if (ndisStatus != NDIS_STATUS_SUCCESS) {
 381         status = STATUS_UNSUCCESSFUL;
 382     }
 383
 384     if (pNbl) {
 385         OvsCompleteNBL(gOvsSwitchContext, pNbl, TRUE);
 386     }
 387 unlock:
 388     NdisReleaseSpinLock(gOvsCtrlLock);
 389     return status;
 390 }
 391
 392
 393 NTSTATUS
 394 OvsPurgeDpIoctl(PFILE_OBJECT fileObject)
 395 {
 396     POVS_OPEN_INSTANCE instance = (POVS_OPEN_INSTANCE)fileObject->FsContext;
 397     POVS_USER_PACKET_QUEUE queue = (POVS_USER_PACKET_QUEUE)instance->packetQueue;
 398
 399     if (queue == NULL) {
 400         return STATUS_INVALID_PARAMETER;
 401     }
 402     OvsPurgePacketQueue(queue, instance);
 403     return STATUS_SUCCESS;
 404 }
 405
 406 VOID
 407 OvsCancelIrpDatapath(PDEVICE_OBJECT deviceObject,
 408                      PIRP irp)
 409 {
 410     PIO_STACK_LOCATION irpSp;
 411     PFILE_OBJECT fileObject;
 412     POVS_OPEN_INSTANCE instance;
 413     POVS_USER_PACKET_QUEUE queue = NULL;
 414
 415     UNREFERENCED_PARAMETER(deviceObject);
 416
 417     IoReleaseCancelSpinLock(irp->CancelIrql);
 418     irpSp = IoGetCurrentIrpStackLocation(irp);
 419     fileObject = irpSp->FileObject;
 420
 421     if (fileObject == NULL) {
 422         goto done;
 423     }
 424     NdisAcquireSpinLock(gOvsCtrlLock);
 425     instance = (POVS_OPEN_INSTANCE)fileObject->FsContext;
 426     if (instance) {
 427         queue = instance->packetQueue;
 428     }
 429     if (instance == NULL || queue == NULL) {
 430         NdisReleaseSpinLock(gOvsCtrlLock);
 431         goto done;
 432     }
 433     NdisReleaseSpinLock(gOvsCtrlLock);
 434     NdisAcquireSpinLock(&queue->queueLock);
 435     if (queue->pendingIrp == irp) {
 436         queue->pendingIrp = NULL;
 437     }
 438     NdisReleaseSpinLock(&queue->queueLock);
 439 done:
 440     OvsCompleteIrpRequest(irp, 0, STATUS_CANCELLED);
 441 }
 442
 443
 444 NTSTATUS
 445 OvsWaitDpIoctl(PIRP irp, PFILE_OBJECT fileObject)
 446 {
 447     POVS_OPEN_INSTANCE instance = (POVS_OPEN_INSTANCE)fileObject->FsContext;
 448     POVS_USER_PACKET_QUEUE queue =
 449                (POVS_USER_PACKET_QUEUE)instance->packetQueue;
 450     NTSTATUS status = STATUS_SUCCESS;
 451     BOOLEAN cancelled = FALSE;
 452
 453     if (queue == NULL) {
 454         return STATUS_INVALID_PARAMETER;
 455     }
 456     NdisAcquireSpinLock(&queue->queueLock);
 457     if (queue->instance != instance) {
 458         NdisReleaseSpinLock(&queue->queueLock);
 459         return STATUS_INVALID_PARAMETER;
 460     }
 461     if (queue->pendingIrp) {
 462         NdisReleaseSpinLock(&queue->queueLock);
 463         return STATUS_DEVICE_BUSY;
 464     }
 465     if (queue->numPackets == 0) {
 466         PDRIVER_CANCEL cancelRoutine;
 467         IoMarkIrpPending(irp);
 468         IoSetCancelRoutine(irp, OvsCancelIrpDatapath);
 469         if (irp->Cancel) {
 470             cancelRoutine = IoSetCancelRoutine(irp, NULL);
 471             if (cancelRoutine) {
 472                 cancelled = TRUE;
 473             }
 474         } else {
 475             queue->pendingIrp = irp;
 476         }
 477         status = STATUS_PENDING;
 478     }
 479     NdisReleaseSpinLock(&queue->queueLock);
 480     if (cancelled) {
 481         OvsCompleteIrpRequest(irp, 0, STATUS_CANCELLED);
 482         OVS_LOG_INFO("Datapath IRP cancelled: %p", irp);
 483     }
 484     return status;
 485 }
 486
 487
 488 POVS_PACKET_QUEUE_ELEM
 489 OvsGetNextPacket(POVS_OPEN_INSTANCE instance)
 490 {
 491     POVS_USER_PACKET_QUEUE queue;
 492     PLIST_ENTRY link;
 493     queue = (POVS_USER_PACKET_QUEUE)instance->packetQueue;
 494     if (queue == NULL) {
 495         return NULL;
 496     }
 497     NdisAcquireSpinLock(&queue->queueLock);
 498     if (queue->instance != instance || queue->numPackets == 0) {
 499         NdisReleaseSpinLock(&queue->queueLock);
 500         return NULL;
 501     }
 502     link = RemoveHeadList(&queue->packetList);
 503     queue->numPackets--;
 504     NdisReleaseSpinLock(&queue->queueLock);
 505     return CONTAINING_RECORD(link, OVS_PACKET_QUEUE_ELEM, link);
 506 }
 507
 508
 509 POVS_USER_PACKET_QUEUE
 510 OvsGetQueue(UINT32 queueId)
 511 {
 512     POVS_USER_PACKET_QUEUE queue;
 513     if (queueId >= OVS_MAX_NUM_PACKET_QUEUES) {
 514         return NULL;
 515     }
 516     queue = &ovsPacketQueues[queueId];
 517     return queue->instance != NULL ? queue : NULL;
 518 }
 519
 520 /*
 521  *----------------------------------------------------------------------------
 522  * OvsCreateQueuePacket --
 523  *
 524  *  Create a packet which will be forwarded to user space.
 525  *
 526  * InputParameter:
 527  *   queueId Identify the queue the packet to be inserted
 528  *      This will be used when multiple queues is supported
 529  *      in userspace
 530  *   userData: when cmd is user action, this field contain
 531  *      user action data.
 532  *   userDataLen: as name indicated
 533  *   cmd: either miss or user action
 534  *   inPort: datapath port id from which the packet is received.
 535  *   tunnelKey: tunnelKey for tunneled packet
 536  *   nbl:  the NET_BUFFER_LIST which contain the packet
 537  *   nb: the packet
 538  *   isRecv: This is used to decide how to interprete the csum info
 539  *   hdrInfo: include hdr info initialized during flow extraction.
 540  *
 541  * Results:
 542  *    NULL if fail to create the packet
 543  *    The packet element otherwise
 544  *----------------------------------------------------------------------------
 545  */
 546 POVS_PACKET_QUEUE_ELEM
 547 OvsCreateQueuePacket(UINT32 queueId,
 548                      PVOID userData,
 549                      UINT32 userDataLen,
 550                      UINT32 cmd,
 551                      UINT32 inPort,
 552                      OvsIPv4TunnelKey *tunnelKey,
 553                      PNET_BUFFER_LIST nbl,
 554                      PNET_BUFFER nb,
 555                      BOOLEAN isRecv,
 556                      POVS_PACKET_HDR_INFO hdrInfo)
 557 {
 558 #define VLAN_TAG_SIZE 4
 559     UINT32 allocLen, dataLen, extraLen = 0;
 560     POVS_PACKET_QUEUE_ELEM elem;
 561     PMDL mdl;
 562     UINT8 *src, *dst;
 563     NDIS_TCP_IP_CHECKSUM_NET_BUFFER_LIST_INFO csumInfo;
 564     NDIS_NET_BUFFER_LIST_8021Q_INFO vlanInfo;
 565
 566     csumInfo.Value = NET_BUFFER_LIST_INFO(nbl, TcpIpChecksumNetBufferListInfo);
 567
 568     if (isRecv && (csumInfo.Receive.TcpChecksumFailed ||
 569                    (csumInfo.Receive.UdpChecksumFailed &&
 570                     !hdrInfo->udpCsumZero) ||
 571                    csumInfo.Receive.IpChecksumFailed)) {
 572         OVS_LOG_INFO("Packet dropped due to checksum failure.");
 573         ovsUserStats.dropDuetoChecksum++;
 574         return NULL;
 575     }
 576
 577     vlanInfo.Value = NET_BUFFER_LIST_INFO(nbl, Ieee8021QNetBufferListInfo);
 578     if (vlanInfo.TagHeader.VlanId) {
 579         /*
 580          * We may also need to check priority XXX
 581          */
 582         extraLen = VLAN_TAG_SIZE;
 583     }
 584
 585     dataLen = NET_BUFFER_DATA_LENGTH(nb);
 586     allocLen = sizeof (OVS_PACKET_QUEUE_ELEM) + userDataLen + dataLen +
 587            extraLen;
 588
 589     elem = (POVS_PACKET_QUEUE_ELEM)OvsAllocateMemory(allocLen);
 590     if (elem == NULL) {
 591         ovsUserStats.dropDuetoResource++;
 592         return NULL;
 593     }
 594     elem->hdrInfo.value = hdrInfo->value;
 595     elem->packet.totalLen = sizeof (OVS_PACKET_INFO) + userDataLen + dataLen +
 596        extraLen;
 597     elem->packet.queue = queueId;
 598     elem->packet.userDataLen = userDataLen;
 599     elem->packet.inPort = inPort;
 600     elem->packet.cmd = cmd;
 601     if (cmd == (UINT32)OVS_PACKET_CMD_MISS) {
 602         ovsUserStats.miss++;
 603     } else {
 604         ovsUserStats.action++;
 605     }
 606     elem->packet.packetLen = dataLen + extraLen;
 607     if (tunnelKey) {
 608         RtlCopyMemory(&elem->packet.tunnelKey, tunnelKey,
 609                       sizeof (*tunnelKey));
 610     } else {
 611         RtlZeroMemory(&elem->packet.tunnelKey,
 612                       sizeof (elem->packet.tunnelKey));
 613     }
 614
 615     dst = elem->packet.data;
 616     if (userDataLen) {
 617         RtlCopyMemory(dst, userData, userDataLen);
 618         dst = dst + userDataLen;
 619     }
 620     dst += extraLen;
 621
 622     mdl = NET_BUFFER_CURRENT_MDL(nb);
 623     src = NdisGetDataBuffer(nb, dataLen, dst, 1, 0);
 624     if (src == NULL) {
 625         OvsFreeMemory(elem);
 626         ovsUserStats.dropDuetoResource++;
 627         return NULL;
 628     } else if (src != dst) {
 629         /* Copy the data from the NDIS buffer to dst. */
 630         RtlCopyMemory(dst, src, dataLen);
 631     }
 632
 633     dst =  elem->packet.data + userDataLen + extraLen;
 634     /*
 635      * Fix IP hdr if necessary
 636      */
 637     if ((isRecv && csumInfo.Receive.IpChecksumValueInvalid) ||
 638         (!isRecv && csumInfo.Transmit.IsIPv4 &&
 639          csumInfo.Transmit.IpHeaderChecksum)) {
 640         PIPV4_HEADER ipHdr = (PIPV4_HEADER)(dst + hdrInfo->l3Offset);
 641         ASSERT(elem->hdrInfo.isIPv4);
 642         ASSERT(ipHdr->Version == 4);
 643         ipHdr->HeaderChecksum = IPChecksum((UINT8 *)ipHdr,
 644                                            ipHdr->HeaderLength << 2,
 645                                            (UINT16)~ipHdr->HeaderChecksum);
 646         ovsUserStats.ipCsum++;
 647     }
 648     ASSERT(elem->hdrInfo.tcpCsumNeeded == 0 &&
 649            elem->hdrInfo.udpCsumNeeded == 0);
 650     /*
 651      * Fow now, we will not do verification
 652      * There is no correctness issue here.
 653      * XXX
 654      */
 655     /*
 656      * calculate TCP/UDP pseudo checksum
 657      */
 658     if (isRecv && csumInfo.Receive.TcpChecksumValueInvalid) {
 659         /*
 660          * Only this case, we need to reclaculate pseudo checksum
 661          * all other cases, it is assumed the pseudo checksum is
 662          * filled already.
 663          *
 664          */
 665         PTCP_HDR tcpHdr = (PTCP_HDR)(dst + hdrInfo->l4Offset);
 666         if (hdrInfo->isIPv4) {
 667             PIPV4_HEADER ipHdr = (PIPV4_HEADER)(dst + hdrInfo->l3Offset);
 668             elem->hdrInfo.l4PayLoad = (UINT16)(ntohs(ipHdr->TotalLength) -
 669                                                (ipHdr->HeaderLength << 2));
 670             tcpHdr->th_sum =
 671                  IPPseudoChecksum((UINT32 *)&ipHdr->SourceAddress,
 672                                   (UINT32 *)&ipHdr->DestinationAddress,
 673                                   IPPROTO_TCP, elem->hdrInfo.l4PayLoad);
 674         } else {
 675             PIPV6_HEADER ipv6Hdr = (PIPV6_HEADER)(dst + hdrInfo->l3Offset);
 676             elem->hdrInfo.l4PayLoad =
 677                   (UINT16)(ntohs(ipv6Hdr->PayloadLength) +
 678                            hdrInfo->l3Offset + sizeof(IPV6_HEADER) -
 679                            hdrInfo->l4Offset);
 680             ASSERT(hdrInfo->isIPv6);
 681             tcpHdr->th_sum =
 682                 IPv6PseudoChecksum((UINT32 *)&ipv6Hdr->SourceAddress,
 683                                    (UINT32 *)&ipv6Hdr->DestinationAddress,
 684                                    IPPROTO_TCP, elem->hdrInfo.l4PayLoad);
 685         }
 686         elem->hdrInfo.tcpCsumNeeded = 1;
 687         ovsUserStats.recalTcpCsum++;
 688     } else if (!isRecv) {
 689         if (csumInfo.Transmit.TcpChecksum) {
 690             elem->hdrInfo.tcpCsumNeeded = 1;
 691         } else if (csumInfo.Transmit.UdpChecksum) {
 692             elem->hdrInfo.udpCsumNeeded = 1;
 693         }
 694         if (elem->hdrInfo.tcpCsumNeeded || elem->hdrInfo.udpCsumNeeded) {
 695 #ifdef DBG
 696             UINT16 sum, *ptr;
 697             UINT8 proto =
 698                elem->hdrInfo.tcpCsumNeeded ? IPPROTO_TCP : IPPROTO_UDP;
 699 #endif
 700             if (hdrInfo->isIPv4) {
 701                 PIPV4_HEADER ipHdr = (PIPV4_HEADER)(dst + hdrInfo->l3Offset);
 702                 elem->hdrInfo.l4PayLoad = (UINT16)(ntohs(ipHdr->TotalLength) -
 703                                                    (ipHdr->HeaderLength << 2));
 704 #ifdef DBG
 705                 sum = IPPseudoChecksum((UINT32 *)&ipHdr->SourceAddress,
 706                                        (UINT32 *)&ipHdr->DestinationAddress,
 707                                        proto, elem->hdrInfo.l4PayLoad);
 708 #endif
 709             } else {
 710                 PIPV6_HEADER ipv6Hdr = (PIPV6_HEADER)(dst +
 711                                                       hdrInfo->l3Offset);
 712                 elem->hdrInfo.l4PayLoad =
 713                        (UINT16)(ntohs(ipv6Hdr->PayloadLength) +
 714                                 hdrInfo->l3Offset + sizeof(IPV6_HEADER) -
 715                                 hdrInfo->l4Offset);
 716                 ASSERT(hdrInfo->isIPv6);
 717 #ifdef DBG
 718                 sum = IPv6PseudoChecksum((UINT32 *)&ipv6Hdr->SourceAddress,
 719                                          (UINT32 *)&ipv6Hdr->DestinationAddress,
 720                                          proto, elem->hdrInfo.l4PayLoad);
 721 #endif
 722             }
 723 #ifdef DBG
 724             ptr = (UINT16 *)(dst + hdrInfo->l4Offset +
 725                              (elem->hdrInfo.tcpCsumNeeded ?
 726                               TCP_CSUM_OFFSET : UDP_CSUM_OFFSET));
 727             ASSERT(*ptr == sum);
 728 #endif
 729         }
 730     }
 731     /*
 732      * Finally insert VLAN tag
 733      */
 734     if (extraLen) {
 735         dst = elem->packet.data + userDataLen;
 736         src = dst + extraLen;
 737         ((UINT32 *)dst)[0] = ((UINT32 *)src)[0];
 738         ((UINT32 *)dst)[1] = ((UINT32 *)src)[1];
 739         ((UINT32 *)dst)[2] = ((UINT32 *)src)[2];
 740         dst += 12;
 741         ((UINT16 *)dst)[0] = htons(0x8100);
 742         ((UINT16 *)dst)[1] = htons(vlanInfo.TagHeader.VlanId |
 743                                    (vlanInfo.TagHeader.UserPriority << 13));
 744         elem->hdrInfo.l3Offset += VLAN_TAG_SIZE;
 745         elem->hdrInfo.l4Offset += VLAN_TAG_SIZE;
 746         ovsUserStats.vlanInsert++;
 747     }
 748
 749     return elem;
 750 }
 751
 752
 753 VOID
 754 OvsQueuePackets(UINT32 queueId,
 755                 PLIST_ENTRY packetList,
 756                 UINT32 numElems)
 757 {
 758     POVS_USER_PACKET_QUEUE queue = OvsGetQueue(queueId);
 759     POVS_PACKET_QUEUE_ELEM elem;
 760     PIRP irp = NULL;
 761     PLIST_ENTRY  link;
 762     UINT32 num = 0;
 763
 764     OVS_LOG_LOUD("Enter: queueId %u, numELems: %u",
 765                   queueId, numElems);
 766     if (queue == NULL) {
 767         goto cleanup;
 768     }
 769
 770     NdisAcquireSpinLock(&queue->queueLock);
 771     if (queue->instance == NULL) {
 772         NdisReleaseSpinLock(&queue->queueLock);
 773         goto cleanup;
 774     } else {
 775         OvsAppendList(&queue->packetList, packetList);
 776         queue->numPackets += numElems;
 777     }
 778     if (queue->pendingIrp) {
 779         PDRIVER_CANCEL cancelRoutine;
 780         irp = queue->pendingIrp;
 781         queue->pendingIrp = NULL;
 782         cancelRoutine = IoSetCancelRoutine(irp, NULL);
 783         if (cancelRoutine == NULL) {
 784             irp = NULL;
 785         }
 786     }
 787     NdisReleaseSpinLock(&queue->queueLock);
 788     if (irp) {
 789         OvsCompleteIrpRequest(irp, 0, STATUS_SUCCESS);
 790     }
 791
 792 cleanup:
 793     while (!IsListEmpty(packetList)) {
 794         link = RemoveHeadList(packetList);
 795         elem = CONTAINING_RECORD(link, OVS_PACKET_QUEUE_ELEM, link);
 796         OvsFreeMemory(elem);
 797         num++;
 798     }
 799     OVS_LOG_LOUD("Exit: drop %u packets", num);
 800 }
 801
 802
 803 /*
 804  *----------------------------------------------------------------------------
 805  * OvsCreateAndAddPackets --
 806  *
 807  *  Create a packet and forwarded to user space.
 808  *
 809  *  This function would fragment packet if needed, and queue
 810  *  each segment to user space.
 811  *----------------------------------------------------------------------------
 812  */
 813 NTSTATUS
 814 OvsCreateAndAddPackets(UINT32 queueId,
 815                        PVOID userData,
 816                        UINT32 userDataLen,
 817                        UINT32 cmd,
 818                        UINT32 inPort,
 819                        OvsIPv4TunnelKey *tunnelKey,
 820                        PNET_BUFFER_LIST nbl,
 821                        BOOLEAN isRecv,
 822                        POVS_PACKET_HDR_INFO hdrInfo,
 823                        POVS_SWITCH_CONTEXT switchContext,
 824                        LIST_ENTRY *list,
 825                        UINT32 *num)
 826 {
 827     POVS_PACKET_QUEUE_ELEM elem;
 828     PNET_BUFFER_LIST newNbl = NULL;
 829     PNET_BUFFER nb;
 830
 831     if (hdrInfo->isTcp) {
 832         NDIS_TCP_LARGE_SEND_OFFLOAD_NET_BUFFER_LIST_INFO tsoInfo;
 833         UINT32 packetLength;
 834
 835         tsoInfo.Value = NET_BUFFER_LIST_INFO(nbl, TcpLargeSendNetBufferListInfo);
 836         nb = NET_BUFFER_LIST_FIRST_NB(nbl);
 837         packetLength = NET_BUFFER_DATA_LENGTH(nb);
 838
 839         OVS_LOG_TRACE("MSS %u packet len %u",
 840                 tsoInfo.LsoV1Transmit.MSS, packetLength);
 841         if (tsoInfo.LsoV1Transmit.MSS) {
 842             OVS_LOG_TRACE("l4Offset %d", hdrInfo->l4Offset);
 843             newNbl = OvsTcpSegmentNBL(switchContext, nbl, hdrInfo,
 844                     tsoInfo.LsoV1Transmit.MSS , 0);
 845             if (newNbl == NULL) {
 846                 return NDIS_STATUS_FAILURE;
 847             }
 848             nbl = newNbl;
 849         }
 850     }
 851
 852     nb = NET_BUFFER_LIST_FIRST_NB(nbl);
 853     while (nb) {
 854         elem = OvsCreateQueuePacket(queueId, userData, userDataLen,
 855                                     cmd, inPort, tunnelKey, nbl, nb,
 856                                     isRecv, hdrInfo);
 857         if (elem) {
 858             InsertTailList(list, &elem->link);
 859             (*num)++;
 860         }
 861         nb = NET_BUFFER_NEXT_NB(nb);
 862     }
 863     if (newNbl) {
 864         OvsCompleteNBL(switchContext, newNbl, TRUE);
 865     }
 866     return NDIS_STATUS_SUCCESS;
 867 }