datapath-windows/ovsext/PacketIO.c

   1 /*
   2  * Copyright (c) 2014 VMware, Inc.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at:
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /*
  18  * This file contains the implementation of the datapath/forwarding
  19  * functionality of the OVS.
  20  */
  21
  22 #include "precomp.h"
  23
  24 #include "Actions.h"
  25 #include "Switch.h"
  26 #include "Vport.h"
  27 #include "NetProto.h"
  28 #include "User.h"
  29 #include "PacketIO.h"
  30 #include "Flow.h"
  31 #include "Event.h"
  32 #include "User.h"
  33
  34 /* Due to an imported header file */
  35 #pragma warning( disable:4505 )
  36
  37 #ifdef OVS_DBG_MOD
  38 #undef OVS_DBG_MOD
  39 #endif
  40 #define OVS_DBG_MOD OVS_DBG_DISPATCH
  41 #include "Debug.h"
  42
  43 extern NDIS_STRING ovsExtGuidUC;
  44 extern NDIS_STRING ovsExtFriendlyNameUC;
  45
  46 static VOID OvsFinalizeCompletionList(OvsCompletionList *completionList);
  47 static VOID OvsCompleteNBLIngress(POVS_SWITCH_CONTEXT switchContext,
  48                     PNET_BUFFER_LIST netBufferLists, ULONG sendCompleteFlags);
  49 static NTSTATUS OvsCreateNewNBLsFromMultipleNBs(
  50                     POVS_SWITCH_CONTEXT switchContext,
  51                     PNET_BUFFER_LIST *curNbl,
  52                     PNET_BUFFER_LIST *nextNbl);
  53
  54 VOID
  55 OvsInitCompletionList(OvsCompletionList *completionList,
  56                       POVS_SWITCH_CONTEXT switchContext,
  57                       ULONG sendCompleteFlags)
  58 {
  59     ASSERT(completionList);
  60     completionList->dropNbl = NULL;
  61     completionList->dropNblNext = &completionList->dropNbl;
  62     completionList->switchContext = switchContext;
  63     completionList->sendCompleteFlags = sendCompleteFlags;
  64 }
  65
  66 /* Utility function used to complete an NBL. */
  67 VOID
  68 OvsAddPktCompletionList(OvsCompletionList *completionList,
  69                         BOOLEAN incoming,
  70                         NDIS_SWITCH_PORT_ID sourcePort,
  71                         PNET_BUFFER_LIST netBufferList,
  72                         UINT32 netBufferListCount,
  73                         PNDIS_STRING filterReason)
  74 {
  75     POVS_BUFFER_CONTEXT ctx;
  76
  77     /* XXX: We handle one NBL at a time. */
  78     ASSERT(netBufferList->Next == NULL);
  79
  80     /* Make sure it has a context. */
  81     ctx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(netBufferList);
  82     ASSERT(ctx && ctx->magic == OVS_CTX_MAGIC);
  83
  84     completionList->switchContext->NdisSwitchHandlers.ReportFilteredNetBufferLists(
  85         completionList->switchContext->NdisSwitchContext, &ovsExtGuidUC,
  86         &ovsExtFriendlyNameUC, sourcePort,
  87         incoming ? NDIS_SWITCH_REPORT_FILTERED_NBL_FLAGS_IS_INCOMING : 0,
  88         netBufferListCount, netBufferList, filterReason);
  89
  90     *completionList->dropNblNext = netBufferList;
  91     completionList->dropNblNext = &netBufferList->Next;
  92     ASSERT(completionList->dropNbl);
  93 }
  94
  95 static __inline VOID
  96 OvsReportNBLIngressError(POVS_SWITCH_CONTEXT switchContext,
  97                          PNET_BUFFER_LIST nblList,
  98                          PNDIS_STRING filterReason,
  99                          NDIS_STATUS error)
 100 {
 101     PNET_BUFFER_LIST nbl = nblList;
 102     while (nbl) {
 103         PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO fwdDetail;
 104         fwdDetail = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(nbl);
 105
 106         nbl->Status = error;
 107
 108         /* This can be optimized by batching NBL's from the same
 109          * SourcePortId. */
 110         switchContext->NdisSwitchHandlers.ReportFilteredNetBufferLists(
 111             switchContext->NdisSwitchContext, &ovsExtGuidUC,
 112             &ovsExtFriendlyNameUC, fwdDetail->SourcePortId,
 113             NDIS_SWITCH_REPORT_FILTERED_NBL_FLAGS_IS_INCOMING,
 114             1 /*Nbl count.*/, nbl, filterReason);
 115
 116         nbl = NET_BUFFER_LIST_NEXT_NBL(nbl);
 117     }
 118 }
 119
 120 static __inline ULONG
 121 OvsGetSendCompleteFlags(ULONG sendFlags)
 122 {
 123     BOOLEAN dispatch, sameSource;
 124     ULONG sendCompleteFlags;
 125
 126     dispatch = NDIS_TEST_SEND_AT_DISPATCH_LEVEL(sendFlags);
 127     sendCompleteFlags = (dispatch ?
 128                         NDIS_SEND_COMPLETE_FLAGS_DISPATCH_LEVEL : 0);
 129     sameSource = NDIS_TEST_SEND_FLAG(sendFlags,
 130                                         NDIS_SEND_FLAGS_SWITCH_SINGLE_SOURCE);
 131     sendCompleteFlags |= (sameSource ?
 132                         NDIS_SEND_COMPLETE_FLAGS_SWITCH_SINGLE_SOURCE : 0);
 133
 134     return sendCompleteFlags;
 135 }
 136
 137 VOID
 138 OvsSendNBLIngress(POVS_SWITCH_CONTEXT switchContext,
 139                   PNET_BUFFER_LIST netBufferLists,
 140                   ULONG sendFlags)
 141 {
 142     if (switchContext->dataFlowState == OvsSwitchPaused) {
 143         /* If a filter module is in the Paused state, the filter driver must not
 144          * originate any send requests for that filter module. If NDIS calls
 145          * FilterSendNetBufferLists, the driver must not call
 146          * NdisFSendNetBufferLists to pass on the data until the driver is
 147          * restarted. The driver should call NdisFSendNetBufferListsComplete
 148          * immediately to complete the send operation. It should set the
 149          * complete status in each NET_BUFFER_LIST structure to
 150          * NDIS_STATUS_PAUSED.
 151          *
 152          * http://msdn.microsoft.com/en-us/library/windows/hardware/
 153          * ff549966(v=vs.85).aspx */
 154         NDIS_STRING filterReason;
 155         ULONG sendCompleteFlags = OvsGetSendCompleteFlags(sendFlags);
 156
 157         RtlInitUnicodeString(&filterReason,
 158                              L"Switch state PAUSED, drop before FSendNBL.");
 159         OvsReportNBLIngressError(switchContext, netBufferLists, &filterReason,
 160                                  NDIS_STATUS_PAUSED);
 161         OvsCompleteNBLIngress(switchContext, netBufferLists,
 162                               sendCompleteFlags);
 163         return;
 164     }
 165
 166     ASSERT(switchContext->dataFlowState == OvsSwitchRunning);
 167
 168     NdisFSendNetBufferLists(switchContext->NdisFilterHandle, netBufferLists,
 169                             NDIS_DEFAULT_PORT_NUMBER, sendFlags);
 170 }
 171
 172 static __inline VOID
 173 OvsStartNBLIngressError(POVS_SWITCH_CONTEXT switchContext,
 174                         PNET_BUFFER_LIST nblList,
 175                         ULONG sendCompleteFlags,
 176                         PNDIS_STRING filterReason,
 177                         NDIS_STATUS error)
 178 {
 179     ASSERT(error);
 180     OvsReportNBLIngressError(switchContext, nblList, filterReason, error);
 181     NdisFSendNetBufferListsComplete(switchContext->NdisFilterHandle, nblList,
 182                                     sendCompleteFlags);
 183 }
 184
 185 static VOID
 186 OvsAppendNativeForwardedPacket(POVS_SWITCH_CONTEXT switchContext,
 187                                PNET_BUFFER_LIST curNbl,
 188                                PNET_BUFFER_LIST *nativeNbls,
 189                                ULONG flags,
 190                                BOOLEAN isRecv)
 191 {
 192     POVS_BUFFER_CONTEXT ctx = { 0 };
 193     NDIS_STRING filterReason;
 194
 195     *nativeNbls = curNbl;
 196
 197     ctx = OvsInitExternalNBLContext(switchContext, curNbl, isRecv);
 198     if (ctx == NULL) {
 199         RtlInitUnicodeString(&filterReason,
 200                              L"Cannot allocate native NBL context.");
 201
 202         OvsStartNBLIngressError(switchContext, curNbl, flags, &filterReason,
 203                                 NDIS_STATUS_RESOURCES);
 204     }
 205 }
 206
 207 static VOID
 208 OvsStartNBLIngress(POVS_SWITCH_CONTEXT switchContext,
 209                    PNET_BUFFER_LIST netBufferLists,
 210                    ULONG SendFlags)
 211 {
 212     NDIS_SWITCH_PORT_ID sourcePort = 0;
 213     NDIS_SWITCH_NIC_INDEX sourceIndex = 0;
 214     PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO fwdDetail;
 215     PNET_BUFFER_LIST curNbl = NULL, nextNbl = NULL;
 216     ULONG sendCompleteFlags;
 217     UCHAR dispatch;
 218     LOCK_STATE_EX lockState, dpLockState;
 219     NDIS_STATUS status;
 220     NDIS_STRING filterReason;
 221     LIST_ENTRY missedPackets;
 222     UINT32 num = 0;
 223     OvsCompletionList completionList;
 224 #if (NDIS_SUPPORT_NDIS640)
 225     PNET_BUFFER_LIST nativeForwardedNbls = NULL;
 226     PNET_BUFFER_LIST *nextNativeForwardedNbl = &nativeForwardedNbls;
 227 #endif
 228
 229     dispatch = NDIS_TEST_SEND_AT_DISPATCH_LEVEL(SendFlags)?
 230                                             NDIS_RWL_AT_DISPATCH_LEVEL : 0;
 231     sendCompleteFlags = OvsGetSendCompleteFlags(SendFlags);
 232     SendFlags |= NDIS_SEND_FLAGS_SWITCH_DESTINATION_GROUP;
 233
 234     InitializeListHead(&missedPackets);
 235     OvsInitCompletionList(&completionList, switchContext, sendCompleteFlags);
 236
 237     for (curNbl = netBufferLists; curNbl != NULL; curNbl = nextNbl) {
 238         POVS_VPORT_ENTRY vport = NULL;
 239         UINT32 portNo = 0;
 240         OVS_DATAPATH *datapath = &switchContext->datapath;
 241         OVS_PACKET_HDR_INFO layers = { 0 };
 242         OvsFlowKey key = { 0 };
 243         UINT64 hash = 0;
 244         PNET_BUFFER curNb = NULL;
 245         POVS_BUFFER_CONTEXT ctx = NULL;
 246
 247         nextNbl = curNbl->Next;
 248         curNbl->Next = NULL;
 249
 250         fwdDetail = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(curNbl);
 251         sourcePort = fwdDetail->SourcePortId;
 252         sourceIndex = (NDIS_SWITCH_NIC_INDEX)fwdDetail->SourceNicIndex;
 253
 254 #if (NDIS_SUPPORT_NDIS640)
 255         if (fwdDetail->NativeForwardingRequired) {
 256             /* Add current NBL to those that require native forwarding. */
 257             OvsAppendNativeForwardedPacket(
 258                 switchContext,
 259                 curNbl,
 260                 nextNativeForwardedNbl,
 261                 sendCompleteFlags,
 262                 sourcePort == switchContext->virtualExternalPortId);
 263             continue;
 264         }
 265 #endif /* NDIS_SUPPORT_NDIS640 */
 266
 267         ctx = OvsInitExternalNBLContext(switchContext, curNbl,
 268                   sourcePort == switchContext->virtualExternalPortId);
 269         if (ctx == NULL) {
 270             RtlInitUnicodeString(&filterReason,
 271                 L"Cannot allocate external NBL context.");
 272
 273             OvsStartNBLIngressError(switchContext, curNbl,
 274                                     sendCompleteFlags, &filterReason,
 275                                     NDIS_STATUS_RESOURCES);
 276             continue;
 277         }
 278
 279         /* Ethernet Header is a guaranteed safe access. */
 280         curNb = NET_BUFFER_LIST_FIRST_NB(curNbl);
 281         if (curNb->Next != NULL) {
 282             /* Create a NET_BUFFER_LIST for each NET_BUFFER. */
 283             status = OvsCreateNewNBLsFromMultipleNBs(switchContext,
 284                                                      &curNbl,
 285                                                      &nextNbl);
 286             if (!NT_SUCCESS(status)) {
 287                 RtlInitUnicodeString(&filterReason,
 288                                      L"Cannot allocate NBLs with single NB.");
 289
 290                 OvsStartNBLIngressError(switchContext, curNbl,
 291                                         sendCompleteFlags, &filterReason,
 292                                         NDIS_STATUS_RESOURCES);
 293                 continue;
 294             }
 295         }
 296         {
 297             OvsFlow *flow;
 298
 299             /* Take the DispatchLock so none of the VPORTs disconnect while
 300              * we are setting destination ports.
 301              *
 302              * XXX: acquire/release the dispatch lock for a "batch" of packets
 303              * rather than for each packet. */
 304             NdisAcquireRWLockRead(switchContext->dispatchLock, &lockState,
 305                                   dispatch);
 306
 307             vport = OvsFindVportByPortIdAndNicIndex(switchContext, sourcePort,
 308                                                     sourceIndex);
 309             if (vport == NULL || vport->ovsState != OVS_STATE_CONNECTED) {
 310                 RtlInitUnicodeString(&filterReason,
 311                     L"OVS-Cannot forward packet from unknown source port");
 312                 goto dropit;
 313             } else {
 314                 portNo = vport->portNo;
 315             }
 316
 317             vport->stats.rxPackets++;
 318             vport->stats.rxBytes += NET_BUFFER_DATA_LENGTH(curNb);
 319
 320             status = OvsExtractFlow(curNbl, vport->portNo, &key, &layers, NULL);
 321             if (status != NDIS_STATUS_SUCCESS) {
 322                 RtlInitUnicodeString(&filterReason, L"OVS-Flow extract failed");
 323                 goto dropit;
 324             }
 325
 326             ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 327             OvsAcquireDatapathRead(datapath, &dpLockState, TRUE);
 328
 329             flow = OvsLookupFlow(datapath, &key, &hash, FALSE);
 330             if (flow) {
 331                 OvsFlowUsed(flow, curNbl, &layers);
 332                 datapath->hits++;
 333                 /* If successful, OvsActionsExecute() consumes the NBL.
 334                  * Otherwise, it adds it to the completionList. No need to
 335                  * check the return value. */
 336                 OvsActionsExecute(switchContext, &completionList, curNbl,
 337                                   portNo, SendFlags, &key, &hash, &layers,
 338                                   flow->actions, flow->actionsLen);
 339                 OvsReleaseDatapath(datapath, &dpLockState);
 340                 NdisReleaseRWLock(switchContext->dispatchLock, &lockState);
 341                 continue;
 342             } else {
 343                 OvsReleaseDatapath(datapath, &dpLockState);
 344
 345                 datapath->misses++;
 346                 status = OvsCreateAndAddPackets(NULL, 0, OVS_PACKET_CMD_MISS,
 347                              vport, &key, curNbl,
 348                              sourcePort == switchContext->virtualExternalPortId,
 349                              &layers, switchContext, &missedPackets, &num);
 350                 if (status == NDIS_STATUS_SUCCESS) {
 351                     /* Complete the packet since it was copied to user
 352                      * buffer. */
 353                     RtlInitUnicodeString(&filterReason,
 354                         L"OVS-Dropped since packet was copied to userspace");
 355                 } else {
 356                     RtlInitUnicodeString(&filterReason,
 357                         L"OVS-Dropped due to failure to queue to userspace");
 358                 }
 359                 goto dropit;
 360             }
 361
 362 dropit:
 363             OvsAddPktCompletionList(&completionList, TRUE, sourcePort, curNbl, 0,
 364                                     &filterReason);
 365             NdisReleaseRWLock(switchContext->dispatchLock, &lockState);
 366         }
 367     }
 368
 369 #if (NDIS_SUPPORT_NDIS640)
 370     if (nativeForwardedNbls) {
 371         /* This is NVGRE encapsulated traffic and is forwarded to NDIS
 372          * in order to be handled by the HNV module. */
 373         OvsSendNBLIngress(switchContext, nativeForwardedNbls, SendFlags);
 374     }
 375 #endif /* NDIS_SUPPORT_NDIS640 */
 376
 377     /* Queue the missed packets. */
 378     OvsQueuePackets(&missedPackets, num);
 379     OvsFinalizeCompletionList(&completionList);
 380 }
 381
 382
 383 /*
 384  * --------------------------------------------------------------------------
 385  * Implements filter driver's FilterSendNetBufferLists Function.
 386  * --------------------------------------------------------------------------
 387  */
 388 VOID
 389 OvsExtSendNBL(NDIS_HANDLE filterModuleContext,
 390               PNET_BUFFER_LIST netBufferLists,
 391               NDIS_PORT_NUMBER portNumber,
 392               ULONG sendFlags)
 393 {
 394     UNREFERENCED_PARAMETER(portNumber);
 395
 396     /* 'filterModuleContext' is the switch context that gets created in the
 397      * AttachHandler. */
 398     POVS_SWITCH_CONTEXT switchContext;
 399     switchContext = (POVS_SWITCH_CONTEXT) filterModuleContext;
 400
 401     if (switchContext->dataFlowState == OvsSwitchPaused) {
 402         NDIS_STRING filterReason;
 403         ULONG sendCompleteFlags = OvsGetSendCompleteFlags(sendFlags);
 404
 405         RtlInitUnicodeString(&filterReason,
 406                              L"Switch state PAUSED, drop on ingress.");
 407         OvsStartNBLIngressError(switchContext, netBufferLists,
 408                                 sendCompleteFlags, &filterReason,
 409                                 NDIS_STATUS_PAUSED);
 410         return;
 411     }
 412
 413     ASSERT(switchContext->dataFlowState == OvsSwitchRunning);
 414
 415     OvsStartNBLIngress(switchContext, netBufferLists, sendFlags);
 416 }
 417
 418 static VOID
 419 OvsCompleteNBLIngress(POVS_SWITCH_CONTEXT switchContext,
 420                       PNET_BUFFER_LIST netBufferLists,
 421                       ULONG sendCompleteFlags)
 422 {
 423     PNET_BUFFER_LIST curNbl = NULL, nextNbl = NULL;
 424     OvsCompletionList newList;
 425
 426     newList.dropNbl = NULL;
 427     newList.dropNblNext = &newList.dropNbl;
 428
 429     for (curNbl = netBufferLists; curNbl != NULL; curNbl = nextNbl) {
 430         nextNbl = curNbl->Next;
 431         curNbl->Next = NULL;
 432
 433         curNbl = OvsCompleteNBL(switchContext, curNbl, TRUE);
 434         if (curNbl != NULL) {
 435             /* NBL originated from the upper layer. */
 436             *newList.dropNblNext = curNbl;
 437             newList.dropNblNext = &curNbl->Next;
 438         }
 439     }
 440
 441     /* Complete the NBL's that were sent by the upper layer. */
 442     if (newList.dropNbl != NULL) {
 443         NdisFSendNetBufferListsComplete(switchContext->NdisFilterHandle, newList.dropNbl,
 444                                         sendCompleteFlags);
 445     }
 446 }
 447
 448
 449 /*
 450  * --------------------------------------------------------------------------
 451  * Implements filter driver's FilterSendNetBufferListsComplete function.
 452  * --------------------------------------------------------------------------
 453  */
 454 VOID
 455 OvsExtSendNBLComplete(NDIS_HANDLE filterModuleContext,
 456                       PNET_BUFFER_LIST netBufferLists,
 457                       ULONG sendCompleteFlags)
 458 {
 459     OvsCompleteNBLIngress((POVS_SWITCH_CONTEXT)filterModuleContext,
 460                           netBufferLists, sendCompleteFlags);
 461 }
 462
 463
 464 VOID
 465 OvsFinalizeCompletionList(OvsCompletionList *completionList)
 466 {
 467     if (completionList->dropNbl != NULL) {
 468         OvsCompleteNBLIngress(completionList->switchContext,
 469                               completionList->dropNbl,
 470                               completionList->sendCompleteFlags);
 471
 472         completionList->dropNbl = NULL;
 473         completionList->dropNblNext = &completionList->dropNbl;
 474     }
 475 }
 476
 477 /*
 478  * --------------------------------------------------------------------------
 479  * Implements filter driver's FilterCancelSendNetBufferLists function.
 480  *
 481  * "If a filter driver specifies a FilterSendNetBufferLists function and it
 482  * queues send requests, it must also specify a
 483  * FilterCancelSendNetBufferLists function."
 484  *
 485  * http://msdn.microsoft.com/en-us/library/windows/hardware/
 486  * ff549966(v=vs.85).aspx
 487  * --------------------------------------------------------------------------
 488  */
 489 VOID
 490 OvsExtCancelSendNBL(NDIS_HANDLE filterModuleContext,
 491                     PVOID CancelId)
 492 {
 493     UNREFERENCED_PARAMETER(filterModuleContext);
 494     UNREFERENCED_PARAMETER(CancelId);
 495
 496     /* All send requests get completed synchronously, so there is no need to
 497      * implement this callback. */
 498 }
 499
 500 static NTSTATUS
 501 OvsCreateNewNBLsFromMultipleNBs(POVS_SWITCH_CONTEXT switchContext,
 502                                 PNET_BUFFER_LIST *curNbl,
 503                                 PNET_BUFFER_LIST *nextNbl)
 504 {
 505     NTSTATUS status = STATUS_SUCCESS;
 506     PNET_BUFFER_LIST newNbls = NULL;
 507     PNET_BUFFER_LIST lastNbl = NULL;
 508     PNET_BUFFER_LIST nbl = NULL;
 509     BOOLEAN error = TRUE;
 510
 511     do {
 512         /* Create new NBLs from curNbl with multiple net buffers. */
 513         newNbls = OvsPartialCopyToMultipleNBLs(switchContext,
 514                                                *curNbl, 0, 0, TRUE);
 515         if (NULL == newNbls) {
 516             OVS_LOG_ERROR("Failed to allocate NBLs with single NB.");
 517             status = NDIS_STATUS_RESOURCES;
 518             break;
 519         }
 520
 521         nbl = newNbls;
 522         while (nbl) {
 523             lastNbl = nbl;
 524             nbl = NET_BUFFER_LIST_NEXT_NBL(nbl);
 525         }
 526         lastNbl->Next = *nextNbl;
 527         *nextNbl = newNbls->Next;
 528
 529         OvsCompleteNBL(switchContext, *curNbl, TRUE);
 530
 531         *curNbl = newNbls;
 532         (*curNbl)->Next = NULL;
 533
 534         error = FALSE;
 535     } while (error);
 536
 537     return status;
 538 }