datapath-windows/ovsext/BufferMgmt.c

   1 /*
   2  * Copyright (c) 2014, 2016 VMware, Inc.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at:
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 /*
  18  * ****************************************************************************
  19  *
  20  *       Simple Buffer Management framework for OVS
  21  *
  22  *  It introduces four NDIS buffer pools
  23  *     **Fix size net buffer list pool--this is used for small buffer
  24  *     One allocation will include NBL + NB + MDL + Data + CONTEXT.
  25  *
  26  *     **Variable size net buffer list pool--this is used for variable size
  27  *     buffer. The allocation of net buffer list will include NBL + NB +
  28  *     CONTEXT, a separate allocation of MDL + data buffer is required.
  29  *
  30  *     **NBL only net buffer list pool-- this is used for partial copy
  31  *     (or clone). In this case we can not allocate net buffer list and
  32  *     net buffer at the same time.
  33  *
  34  *     **Net buffer pool-- this is required when net buffer need to be
  35  *     allocated separately.
  36  *
  37  *  A Buffer context is defined to track the buffer specific information
  38  *  so that during NBL completion, proper action can be taken. Please see
  39  *  code for details.
  40  *
  41  *  Here is the usage of the management API
  42  *  All external NBL should be initialized its NBL context by calling
  43  *     OvsInitExternalNBLContext()
  44  *
  45  *  After the external NBL context is initialized, it can call the following
  46  *  API to allocate, copy or partial copy NBL.
  47  *
  48  *     OvsAllocateFixSizeNBL()
  49  *     OvsAllocateVariableSizeNBL()
  50  *
  51  *     OvsPartialCopyNBL()
  52  *     OvsPartialCopyToMultipleNBLs()
  53  *
  54  *     OvsFullCopyNBL()
  55  *     OvsFullCopyToMultipleNBLs()
  56  *
  57  *  See code comments for detail description of the functions.
  58  *
  59  *  All NBLs is completed through
  60  *       OvsCompleteNBL()
  61  *     If this API return non NULL value, then the returned NBL should be
  62  *     returned to upper layer by calling
  63  *     NdisFSendNetBufferListsComplete() if the buffer is from upper
  64  *     layer. In case of WFP, it can call the corresponding completion routine
  65  *     to return the NBL to the framework.
  66  *
  67  *  NOTE:
  68  *     1. Copy or partial copy will not copy destination port array
  69  *     2. Copy or partial copy will copy src port id and index
  70  *     3. New Allocated NBL will have src port set to default port id
  71  *     4. If original packet has direction flag set, the copied or partial
  72  *        copied NBL will still be in same direction.
  73  *     5. When you advance or retreate the buffer, you may need to update
  74  *        relevant meta data to keep it consistent.
  75  *
  76  * ****************************************************************************
  77  */
  78
  79 #include "precomp.h"
  80 #include "Debug.h"
  81 #include "Flow.h"
  82 #include "Offload.h"
  83 #include "NetProto.h"
  84 #include "PacketParser.h"
  85 #include "Switch.h"
  86 #include "Vport.h"
  87
  88 #ifdef OVS_DBG_MOD
  89 #undef OVS_DBG_MOD
  90 #endif
  91 #define OVS_DBG_MOD OVS_DBG_BUFMGMT
  92
  93
  94 /*
  95  * --------------------------------------------------------------------------
  96  * OvsInitBufferPool --
  97  *
  98  *    Allocate NBL and NB pool
  99  *
 100  * XXX: more optimization may be done for buffer management include local cache
 101  * of NBL, NB, data, context, MDL.
 102  * --------------------------------------------------------------------------
 103  */
 104 NDIS_STATUS
 105 OvsInitBufferPool(PVOID ovsContext)
 106 {
 107     POVS_NBL_POOL ovsPool;
 108     POVS_SWITCH_CONTEXT context = (POVS_SWITCH_CONTEXT)ovsContext;
 109     NET_BUFFER_LIST_POOL_PARAMETERS  nblParam;
 110     NET_BUFFER_POOL_PARAMETERS nbParam;
 111
 112     C_ASSERT(MEMORY_ALLOCATION_ALIGNMENT >= 8);
 113
 114     OVS_LOG_TRACE("Enter: context: %p", context);
 115
 116     ovsPool = &context->ovsPool;
 117     RtlZeroMemory(ovsPool, sizeof (OVS_NBL_POOL));
 118     ovsPool->ndisHandle = context->NdisFilterHandle;
 119     ovsPool->ndisContext = context->NdisSwitchContext;
 120     /*
 121      * fix size NBL pool includes
 122      *    NBL + NB + MDL + DATA + Context
 123      *    This is mainly used for Packet execute or slow path when copy is
 124      *    required and size is less than OVS_DEFAULT_DATA_SIZE. We expect
 125      *    Most of packet from user space will use this Pool. (This is
 126      *    true for all bfd and cfm packet.
 127      */
 128     RtlZeroMemory(&nblParam, sizeof (nblParam));
 129     OVS_INIT_OBJECT_HEADER(&nblParam.Header,
 130                            NDIS_OBJECT_TYPE_DEFAULT,
 131                            NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1,
 132                            NDIS_SIZEOF_NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1);
 133     nblParam.ContextSize = OVS_DEFAULT_NBL_CONTEXT_SIZE;
 134     nblParam.PoolTag = OVS_FIX_SIZE_NBL_POOL_TAG;
 135     nblParam.fAllocateNetBuffer = TRUE;
 136     nblParam.DataSize = OVS_DEFAULT_DATA_SIZE + OVS_DEFAULT_HEADROOM_SIZE;
 137
 138     ovsPool->fixSizePool =
 139         NdisAllocateNetBufferListPool(context->NdisSwitchContext, &nblParam);
 140     if (ovsPool->fixSizePool == NULL) {
 141         goto pool_cleanup;
 142     }
 143
 144     /*
 145      * Zero Size NBL Pool includes
 146      *    NBL + NB + Context
 147      *    This is mainly for packet with large data Size, in this case MDL and
 148      *    Data will be allocate separately.
 149      */
 150     RtlZeroMemory(&nblParam, sizeof (nblParam));
 151     OVS_INIT_OBJECT_HEADER(&nblParam.Header,
 152                            NDIS_OBJECT_TYPE_DEFAULT,
 153                            NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1,
 154                            NDIS_SIZEOF_NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1);
 155
 156     nblParam.ContextSize = OVS_DEFAULT_NBL_CONTEXT_SIZE;
 157     nblParam.PoolTag = OVS_VARIABLE_SIZE_NBL_POOL_TAG;
 158     nblParam.fAllocateNetBuffer = TRUE;
 159     nblParam.DataSize = 0;
 160
 161     ovsPool->zeroSizePool =
 162         NdisAllocateNetBufferListPool(context->NdisSwitchContext, &nblParam);
 163     if (ovsPool->zeroSizePool == NULL) {
 164         goto pool_cleanup;
 165     }
 166
 167     /*
 168      * NBL only pool just includes
 169      *    NBL (+ context)
 170      *    This is mainly used for clone and partial copy
 171      */
 172     RtlZeroMemory(&nblParam, sizeof (nblParam));
 173     OVS_INIT_OBJECT_HEADER(&nblParam.Header,
 174                            NDIS_OBJECT_TYPE_DEFAULT,
 175                            NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1,
 176                            NDIS_SIZEOF_NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1);
 177
 178     nblParam.ContextSize = OVS_DEFAULT_NBL_CONTEXT_SIZE;
 179     nblParam.PoolTag = OVS_NBL_ONLY_POOL_TAG;
 180     nblParam.fAllocateNetBuffer = FALSE;
 181     nblParam.DataSize = 0;
 182
 183     ovsPool->nblOnlyPool =
 184         NdisAllocateNetBufferListPool(context->NdisSwitchContext, &nblParam);
 185     if (ovsPool->nblOnlyPool == NULL) {
 186         goto pool_cleanup;
 187     }
 188
 189     /* nb Pool
 190      *    NB only pool, used for copy
 191      */
 192
 193     OVS_INIT_OBJECT_HEADER(&nbParam.Header,
 194                            NDIS_OBJECT_TYPE_DEFAULT,
 195                            NET_BUFFER_POOL_PARAMETERS_REVISION_1,
 196                            NDIS_SIZEOF_NET_BUFFER_POOL_PARAMETERS_REVISION_1);
 197     nbParam.PoolTag = OVS_NET_BUFFER_POOL_TAG;
 198     nbParam.DataSize = 0;
 199     ovsPool->nbPool =
 200         NdisAllocateNetBufferPool(context->NdisSwitchContext, &nbParam);
 201     if (ovsPool->nbPool == NULL) {
 202         goto pool_cleanup;
 203     }
 204     OVS_LOG_TRACE("Exit: fixSizePool: %p zeroSizePool: %p nblOnlyPool: %p"
 205                   "nbPool: %p", ovsPool->fixSizePool, ovsPool->zeroSizePool,
 206                   ovsPool->nblOnlyPool, ovsPool->nbPool);
 207     return NDIS_STATUS_SUCCESS;
 208
 209 pool_cleanup:
 210     OvsCleanupBufferPool(context);
 211     OVS_LOG_TRACE("Exit: Fail to initialize ovs buffer pool");
 212     return NDIS_STATUS_RESOURCES;
 213 }
 214
 215
 216 /*
 217  * --------------------------------------------------------------------------
 218  * OvsCleanupBufferPool --
 219  *  Free Buffer pool for NBL and NB.
 220  * --------------------------------------------------------------------------
 221  */
 222 VOID
 223 OvsCleanupBufferPool(PVOID ovsContext)
 224 {
 225     POVS_NBL_POOL ovsPool;
 226     POVS_SWITCH_CONTEXT context = (POVS_SWITCH_CONTEXT)ovsContext;
 227     ovsPool = &context->ovsPool;
 228     OVS_LOG_TRACE("Enter: context: %p", context);
 229 #ifdef DBG
 230     ASSERT(ovsPool->fixNBLCount == 0);
 231     ASSERT(ovsPool->zeroNBLCount == 0);
 232     ASSERT(ovsPool->nblOnlyCount == 0);
 233     ASSERT(ovsPool->nbCount == 0);
 234     ASSERT(ovsPool->sysNBLCount == 0);
 235     ASSERT(ovsPool->fragNBLCount == 0);
 236 #endif
 237
 238     if (ovsPool->fixSizePool) {
 239         NdisFreeNetBufferListPool(ovsPool->fixSizePool);
 240         ovsPool->fixSizePool = NULL;
 241     }
 242     if (ovsPool->zeroSizePool) {
 243         NdisFreeNetBufferListPool(ovsPool->zeroSizePool);
 244         ovsPool->zeroSizePool = NULL;
 245     }
 246     if (ovsPool->nblOnlyPool) {
 247         NdisFreeNetBufferListPool(ovsPool->nblOnlyPool);
 248         ovsPool->nblOnlyPool = NULL;
 249     }
 250     if (ovsPool->nbPool) {
 251         NdisFreeNetBufferPool(ovsPool->nbPool);
 252         ovsPool->nbPool = NULL;
 253     }
 254     OVS_LOG_TRACE("Exit: cleanup OVS Buffer pool");
 255 }
 256
 257
 258 static VOID
 259 OvsInitNBLContext(POVS_BUFFER_CONTEXT ctx,
 260                   UINT16 flags,
 261                   UINT32 origDataLength,
 262                   UINT32 srcPortNo)
 263 {
 264     ctx->magic = OVS_CTX_MAGIC;
 265     ctx->refCount = 1;
 266     ctx->flags = flags;
 267     ctx->srcPortNo = srcPortNo;
 268     ctx->origDataLength = origDataLength;
 269     ctx->mru = 0;
 270 }
 271
 272
 273 static VOID
 274 OvsDumpForwardingDetails(PNET_BUFFER_LIST nbl)
 275 {
 276     PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO info;
 277     info = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(nbl);
 278     if (info == NULL) {
 279         return;
 280     }
 281     OVS_LOG_INFO("nbl: %p, numAvailableDest: %d, srcId:%d, srcIndex: %d "
 282                  "isDataSafe: %s, safeDataSize: %d",
 283                  nbl, info->NumAvailableDestinations, info->SourcePortId,
 284                  info->SourceNicIndex,
 285                  info->IsPacketDataSafe ? "TRUE" : "FALSE",
 286                  info->IsPacketDataSafe ? 0 : info->SafePacketDataSize);
 287
 288 }
 289
 290 static VOID
 291 OvsDumpNBLContext(PNET_BUFFER_LIST nbl)
 292 {
 293     PNET_BUFFER_LIST_CONTEXT ctx = nbl->Context;
 294     if (ctx == NULL) {
 295         OVS_LOG_INFO("No Net Buffer List context");
 296         return;
 297     }
 298     while (ctx) {
 299         OVS_LOG_INFO("nbl: %p, ctx: %p, TotalSize: %d, Offset: %d",
 300                      nbl, ctx, ctx->Size, ctx->Offset);
 301         ctx = ctx->Next;
 302     }
 303 }
 304
 305
 306 static VOID
 307 OvsDumpMDLChain(PMDL mdl)
 308 {
 309     PMDL tmp;
 310     tmp = mdl;
 311     while (tmp) {
 312         OVS_LOG_INFO("MDL: %p, Size: %d, MappedSystemVa: %p, StartVa: %p"
 313                      " ByteCount: %d, ByteOffset: %d",
 314                      tmp, tmp->Size, tmp->MappedSystemVa,
 315                      tmp->StartVa, tmp->ByteCount, tmp->ByteOffset);
 316         tmp = tmp->Next;
 317     }
 318 }
 319
 320
 321 static VOID
 322 OvsDumpNetBuffer(PNET_BUFFER nb)
 323 {
 324     OVS_LOG_INFO("NET_BUFFER: %p, ChecksumBias: %d Handle: %p, MDLChain: %p "
 325                  "CurrMDL: %p, CurrOffset: %d, DataLen: %d, Offset: %d",
 326                  nb,
 327                  NET_BUFFER_CHECKSUM_BIAS(nb), nb->NdisPoolHandle,
 328                  NET_BUFFER_FIRST_MDL(nb),
 329                  NET_BUFFER_CURRENT_MDL(nb),
 330                  NET_BUFFER_CURRENT_MDL_OFFSET(nb),
 331                  NET_BUFFER_DATA_LENGTH(nb),
 332                  NET_BUFFER_DATA_OFFSET(nb));
 333     OvsDumpMDLChain(NET_BUFFER_FIRST_MDL(nb));
 334 }
 335
 336
 337 static VOID
 338 OvsDumpNetBufferList(PNET_BUFFER_LIST nbl)
 339 {
 340     PNET_BUFFER nb;
 341     OVS_LOG_INFO("NBL: %p, parent: %p, SrcHandle: %p, ChildCount:%d "
 342                  "poolHandle: %p",
 343                  nbl, nbl->ParentNetBufferList,
 344                  nbl->SourceHandle, nbl->ChildRefCount,
 345                  nbl->NdisPoolHandle);
 346     OvsDumpNBLContext(nbl);
 347     nb = NET_BUFFER_LIST_FIRST_NB(nbl);
 348     while (nb) {
 349         OvsDumpNetBuffer(nb);
 350         nb = NET_BUFFER_NEXT_NB(nb);
 351     }
 352 }
 353
 354 /*
 355  * --------------------------------------------------------------------------
 356  * OvsAllocateFixSizeNBL --
 357  *
 358  *    Allocate fix size NBL which include
 359  *       NBL + NB + MBL + Data + Context
 360  *    Please note:
 361  *       * Forwarding Context is allocated, but forwarding detail information
 362  *       is not initailized.
 363  *       * The headroom can not be larger than OVS_DEFAULT_HEADROOM_SIZE(128
 364  *       byte).
 365  * --------------------------------------------------------------------------
 366  */
 367 PNET_BUFFER_LIST
 368 OvsAllocateFixSizeNBL(PVOID ovsContext,
 369                       UINT32 size,
 370                       UINT32 headRoom)
 371 {
 372     PNET_BUFFER_LIST nbl = NULL;
 373     POVS_SWITCH_CONTEXT context = (POVS_SWITCH_CONTEXT)ovsContext;
 374     POVS_BUFFER_CONTEXT ctx;
 375     POVS_NBL_POOL ovsPool = &context->ovsPool;
 376     NDIS_STATUS status;
 377     UINT32 line;
 378     PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO info;
 379
 380     if ((headRoom + size) > OVS_FIX_NBL_DATA_SIZE || size == 0) {
 381         line = __LINE__;
 382         goto allocate_done;
 383     }
 384
 385     nbl = NdisAllocateNetBufferList(ovsPool->fixSizePool,
 386                                     (UINT16)sizeof (OVS_BUFFER_CONTEXT),
 387                                     (UINT16)OVS_DEFAULT_NBL_CONTEXT_FILL);
 388
 389     if (nbl == NULL) {
 390         line = __LINE__;
 391         goto allocate_done;
 392     }
 393
 394     nbl->SourceHandle = ovsPool->ndisHandle;
 395     status = context->NdisSwitchHandlers.
 396              AllocateNetBufferListForwardingContext(ovsPool->ndisContext, nbl);
 397
 398     if (status != NDIS_STATUS_SUCCESS) {
 399         NdisFreeNetBufferList(nbl);
 400         nbl = NULL;
 401         line = __LINE__;
 402         goto allocate_done;
 403     }
 404     info = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(nbl);
 405     ASSERT(info);
 406     info->IsPacketDataSafe = TRUE;
 407     info->SourcePortId = NDIS_SWITCH_DEFAULT_PORT_ID;
 408
 409     status = NdisRetreatNetBufferDataStart(NET_BUFFER_LIST_FIRST_NB(nbl),
 410                                            size, 0, NULL);
 411     ASSERT(status == NDIS_STATUS_SUCCESS);
 412
 413 #ifdef DBG
 414     InterlockedIncrement((LONG volatile *)&ovsPool->fixNBLCount);
 415     OvsDumpNetBufferList(nbl);
 416     OvsDumpForwardingDetails(nbl);
 417 #endif
 418
 419     ctx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(nbl);
 420     ASSERT(ctx);
 421
 422     OvsInitNBLContext(ctx, OVS_BUFFER_FROM_FIX_SIZE_POOL |
 423                       OVS_BUFFER_PRIVATE_FORWARD_CONTEXT, size,
 424                       OVS_DPPORT_NUMBER_INVALID);
 425     line = __LINE__;
 426 allocate_done:
 427     OVS_LOG_LOUD("Allocate Fix NBL: %p, line: %d", nbl, line);
 428     return nbl;
 429 }
 430
 431
 432 static PMDL
 433 OvsAllocateMDLAndData(NDIS_HANDLE ndisHandle,
 434                       UINT32 dataSize)
 435 {
 436     PMDL mdl;
 437     PVOID data;
 438
 439     data = OvsAllocateMemoryWithTag(dataSize, OVS_MDL_POOL_TAG);
 440     if (data == NULL) {
 441         return NULL;
 442     }
 443
 444     mdl = NdisAllocateMdl(ndisHandle, data, dataSize);
 445     if (mdl == NULL) {
 446         OvsFreeMemoryWithTag(data, OVS_MDL_POOL_TAG);
 447     }
 448
 449     return mdl;
 450 }
 451
 452
 453 static VOID
 454 OvsFreeMDLAndData(PMDL mdl)
 455 {
 456     PVOID data;
 457
 458     data = MmGetMdlVirtualAddress(mdl);
 459     NdisFreeMdl(mdl);
 460     OvsFreeMemoryWithTag(data, OVS_MDL_POOL_TAG);
 461 }
 462
 463
 464 /*
 465  * --------------------------------------------------------------------------
 466  * OvsAllocateVariableSizeNBL --
 467  *
 468  *    Allocate variable size NBL, the NBL looks like
 469  *      NBL + NB + Context
 470  *      MDL + Data
 471  * --------------------------------------------------------------------------
 472  */
 473 PNET_BUFFER_LIST
 474 OvsAllocateVariableSizeNBL(PVOID ovsContext,
 475                            UINT32 size,
 476                            UINT32 headRoom)
 477 {
 478     PNET_BUFFER_LIST nbl = NULL;
 479     POVS_SWITCH_CONTEXT context = (POVS_SWITCH_CONTEXT)ovsContext;
 480     POVS_NBL_POOL ovsPool = &context->ovsPool;
 481     POVS_BUFFER_CONTEXT ctx;
 482     UINT32 realSize;
 483     PMDL mdl;
 484     NDIS_STATUS status;
 485     PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO info;
 486     if (size == 0) {
 487         return NULL;
 488     }
 489     realSize = MEM_ALIGN_SIZE(size + headRoom);
 490
 491     mdl = OvsAllocateMDLAndData(ovsPool->ndisHandle, realSize);
 492     if (mdl == NULL) {
 493         return NULL;
 494     }
 495
 496     nbl = NdisAllocateNetBufferAndNetBufferList(ovsPool->zeroSizePool,
 497                                          (UINT16)sizeof (OVS_BUFFER_CONTEXT),
 498                                          (UINT16)OVS_DEFAULT_NBL_CONTEXT_FILL,
 499                                                 mdl, realSize, 0);
 500     if (nbl == NULL) {
 501         OvsFreeMDLAndData(mdl);
 502         return NULL;
 503     }
 504
 505     nbl->SourceHandle = ovsPool->ndisHandle;
 506     status = context->NdisSwitchHandlers.
 507              AllocateNetBufferListForwardingContext(ovsPool->ndisContext, nbl);
 508
 509     if (status != NDIS_STATUS_SUCCESS) {
 510        /*
 511         * do we need to remove mdl from nbl XXX
 512         */
 513         OvsFreeMDLAndData(mdl);
 514         NdisFreeNetBufferList(nbl);
 515         return NULL;
 516     }
 517
 518     info = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(nbl);
 519     ASSERT(info);
 520     info->IsPacketDataSafe = TRUE;
 521     info->SourcePortId = NDIS_SWITCH_DEFAULT_PORT_ID;
 522     status = NdisRetreatNetBufferDataStart(NET_BUFFER_LIST_FIRST_NB(nbl),
 523                                            size, 0, NULL);
 524     ASSERT(status == NDIS_STATUS_SUCCESS);
 525
 526 #ifdef DBG
 527     InterlockedIncrement((LONG volatile *)&ovsPool->zeroNBLCount);
 528     OvsDumpNetBufferList(nbl);
 529     OvsDumpForwardingDetails(nbl);
 530 #endif
 531
 532     ctx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(nbl);
 533
 534     OvsInitNBLContext(ctx, OVS_BUFFER_PRIVATE_MDL | OVS_BUFFER_PRIVATE_DATA |
 535                            OVS_BUFFER_PRIVATE_FORWARD_CONTEXT |
 536                            OVS_BUFFER_FROM_ZERO_SIZE_POOL,
 537                            size, OVS_DPPORT_NUMBER_INVALID);
 538
 539     OVS_LOG_LOUD("Allocate variable size NBL: %p", nbl);
 540     return nbl;
 541 }
 542
 543
 544 /*
 545  * --------------------------------------------------------------------------
 546  * OvsInitExternalNBLContext --
 547  *
 548  *     For NBL not allocated by OVS, it will allocate and initialize
 549  *     the NBL context.
 550  * --------------------------------------------------------------------------
 551  */
 552 POVS_BUFFER_CONTEXT
 553 OvsInitExternalNBLContext(PVOID ovsContext,
 554                           PNET_BUFFER_LIST nbl,
 555                           BOOLEAN isRecv)
 556 {
 557     NDIS_HANDLE poolHandle;
 558     POVS_SWITCH_CONTEXT context = (POVS_SWITCH_CONTEXT)ovsContext;
 559     POVS_BUFFER_CONTEXT ctx;
 560     PNET_BUFFER nb;
 561     NDIS_STATUS status;
 562     UINT16 flags;
 563
 564     poolHandle = NdisGetPoolFromNetBufferList(nbl);
 565
 566     if (poolHandle == context->ovsPool.ndisHandle ||
 567         nbl->SourceHandle == context->ovsPool.ndisHandle) {
 568         return (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(nbl);
 569     }
 570     status = NdisAllocateNetBufferListContext(nbl, sizeof (OVS_BUFFER_CONTEXT),
 571                                               OVS_DEFAULT_NBL_CONTEXT_FILL,
 572                                               OVS_OTHER_POOL_TAG);
 573     if (status != NDIS_STATUS_SUCCESS) {
 574         return NULL;
 575     }
 576 #ifdef DBG
 577     OvsDumpNBLContext(nbl);
 578     InterlockedIncrement((LONG volatile *)&context->ovsPool.sysNBLCount);
 579 #endif
 580     flags = isRecv ? OVS_BUFFER_RECV_BUFFER : OVS_BUFFER_SEND_BUFFER;
 581     flags |= OVS_BUFFER_NEED_COMPLETE | OVS_BUFFER_PRIVATE_CONTEXT;
 582     ctx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(nbl);
 583
 584     nb = NET_BUFFER_LIST_FIRST_NB(nbl);
 585     /*
 586      * we use first nb to decide whether we need advance or retreat during
 587      * complete.
 588      */
 589     OvsInitNBLContext(ctx, flags, NET_BUFFER_DATA_LENGTH(nb),
 590                       OVS_DPPORT_NUMBER_INVALID);
 591     return ctx;
 592 }
 593
 594 /*
 595  * --------------------------------------------------------------------------
 596  * OvsAllocateNBLContext
 597  *
 598  *    Create NBL buffer context and forwarding context.
 599  * --------------------------------------------------------------------------
 600  */
 601 NDIS_STATUS
 602 OvsAllocateNBLContext(POVS_SWITCH_CONTEXT context,
 603                       PNET_BUFFER_LIST nbl)
 604 {
 605     POVS_NBL_POOL ovsPool = &context->ovsPool;
 606     NDIS_STATUS status;
 607
 608     status = NdisAllocateNetBufferListContext(nbl,
 609                                               sizeof (OVS_BUFFER_CONTEXT),
 610                                               OVS_DEFAULT_NBL_CONTEXT_FILL,
 611                                               OVS_OTHER_POOL_TAG);
 612     if (status != NDIS_STATUS_SUCCESS) {
 613         return NDIS_STATUS_FAILURE;
 614     }
 615
 616     nbl->SourceHandle = ovsPool->ndisHandle;
 617     status = context->NdisSwitchHandlers.
 618         AllocateNetBufferListForwardingContext(ovsPool->ndisContext, nbl);
 619
 620     if (status != NDIS_STATUS_SUCCESS) {
 621         NdisFreeNetBufferListContext(nbl, sizeof (OVS_BUFFER_CONTEXT));
 622         return NDIS_STATUS_FAILURE;
 623     }
 624     return status;
 625 }
 626
 627 /*
 628  * --------------------------------------------------------------------------
 629  * OvsFreeNBLContext
 630  *
 631  *    Free the NBL buffer context and forwarding context.
 632  * --------------------------------------------------------------------------
 633  */
 634 NDIS_STATUS
 635 OvsFreeNBLContext(POVS_SWITCH_CONTEXT context,
 636                   PNET_BUFFER_LIST nbl)
 637 {
 638     POVS_NBL_POOL ovsPool = &context->ovsPool;
 639
 640     context->NdisSwitchHandlers.
 641          FreeNetBufferListForwardingContext(ovsPool->ndisContext, nbl);
 642     NdisFreeNetBufferListContext(nbl, sizeof (OVS_BUFFER_CONTEXT));
 643
 644     return NDIS_STATUS_SUCCESS;
 645 }
 646
 647 /*
 648  * --------------------------------------------------------------------------
 649  * OvsCopyNBLInfo
 650  *
 651  *    Copy NBL info from src to dst
 652  * --------------------------------------------------------------------------
 653  */
 654 NDIS_STATUS
 655 OvsCopyNBLInfo(PNET_BUFFER_LIST srcNbl, PNET_BUFFER_LIST dstNbl,
 656                POVS_BUFFER_CONTEXT srcCtx, UINT32 copySize,
 657                BOOLEAN copyNblInfo)
 658 {
 659     PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO srcInfo, dstInfo;
 660     NDIS_STATUS status = NDIS_STATUS_SUCCESS;
 661
 662     srcInfo = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(srcNbl);
 663     dstInfo = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(dstNbl);
 664     if (srcInfo) {
 665 #ifdef OVS_USE_COPY_NET_BUFFER_LIST_INFO
 666         status = context->NdisSwitchHandlers.
 667             CopyNetBufferListInfo(ovsPool->ndisContext, dstNbl, srcNbl, 0);
 668
 669         if (status != NDIS_STATUS_SUCCESS) {
 670             return status;
 671         }
 672 #else
 673         dstInfo->SourcePortId = srcInfo->SourcePortId;
 674         dstInfo->SourceNicIndex = srcInfo->SourceNicIndex;
 675         if (copyNblInfo) {
 676             if (srcCtx->flags & OVS_BUFFER_RECV_BUFFER) {
 677                 NdisCopyReceiveNetBufferListInfo(dstNbl, srcNbl);
 678             } else if (srcCtx->flags & OVS_BUFFER_SEND_BUFFER) {
 679                 NdisCopySendNetBufferListInfo(dstNbl, srcNbl);
 680             }
 681         }
 682 #endif
 683         dstInfo->IsPacketDataSafe = srcInfo->IsPacketDataSafe;
 684         if (!srcInfo->IsPacketDataSafe && copySize >
 685             srcInfo->SafePacketDataSize) {
 686             srcInfo->SafePacketDataSize = copySize;
 687         }
 688     } else {
 689         /*
 690          * Assume all data are safe
 691          */
 692         dstInfo->IsPacketDataSafe = TRUE;
 693         dstInfo->SourcePortId = NDIS_SWITCH_DEFAULT_PORT_ID;
 694     }
 695     return status;
 696 }
 697
 698 /*
 699  * --------------------------------------------------------------------------
 700  * OvsPartialCopyNBL --
 701  *
 702  *    Partial copy NBL, if there is multiple NB in NBL, each one will be
 703  *    copied. We also reserve headroom for the new NBL.
 704  *
 705  *    Please note,
 706  *       NBL should have OVS_BUFFER_CONTEXT setup before calling
 707  *       this function.
 708  *       The NBL should already have ref to itself so that during copy
 709  *       it will not be freed.
 710  * --------------------------------------------------------------------------
 711  */
 712 PNET_BUFFER_LIST
 713 OvsPartialCopyNBL(PVOID ovsContext,
 714                   PNET_BUFFER_LIST nbl,
 715                   UINT32 copySize,
 716                   UINT32 headRoom,
 717                   BOOLEAN copyNblInfo)
 718 {
 719     PNET_BUFFER_LIST newNbl;
 720     POVS_SWITCH_CONTEXT context = (POVS_SWITCH_CONTEXT)ovsContext;
 721     NDIS_STATUS status;
 722     PNET_BUFFER srcNb, dstNb;
 723     ULONG byteCopied;
 724     POVS_NBL_POOL ovsPool = &context->ovsPool;
 725     POVS_BUFFER_CONTEXT srcCtx, dstCtx;
 726     UINT16 flags;
 727
 728     srcCtx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(nbl);
 729     if (srcCtx == NULL || srcCtx->magic != OVS_CTX_MAGIC) {
 730         OVS_LOG_INFO("src nbl must have ctx initialized");
 731         ASSERT(srcCtx && srcCtx->magic == OVS_CTX_MAGIC);
 732         return NULL;
 733     }
 734
 735     if (copySize) {
 736         NdisAdvanceNetBufferListDataStart(nbl, copySize, FALSE, NULL);
 737     }
 738     newNbl = NdisAllocateCloneNetBufferList(nbl, ovsPool->nblOnlyPool,
 739                                             NULL, 0);
 740     if (copySize) {
 741         status = NdisRetreatNetBufferListDataStart(nbl, copySize, 0,
 742                                                    NULL, NULL);
 743         ASSERT(status == NDIS_STATUS_SUCCESS);
 744     }
 745
 746     if (newNbl == NULL) {
 747         return NULL;
 748     }
 749
 750     /*
 751      * Allocate private memory for copy
 752      */
 753     if (copySize + headRoom) {
 754         status = NdisRetreatNetBufferListDataStart(newNbl, copySize + headRoom,
 755                                                    0, NULL, NULL);
 756         if (status != NDIS_STATUS_SUCCESS) {
 757             goto retreat_error;
 758         }
 759
 760         if (headRoom) {
 761             NdisAdvanceNetBufferListDataStart(newNbl, headRoom, FALSE, NULL);
 762         }
 763         if (copySize) {
 764             srcNb = NET_BUFFER_LIST_FIRST_NB(nbl);
 765             dstNb = NET_BUFFER_LIST_FIRST_NB(newNbl);
 766
 767             while (srcNb) {
 768                 status = NdisCopyFromNetBufferToNetBuffer(dstNb, 0, copySize,
 769                                                           srcNb, 0,
 770                                                           &byteCopied);
 771                 if (status != NDIS_STATUS_SUCCESS || copySize != byteCopied) {
 772                     goto nbl_context_error;
 773                 }
 774                 srcNb = NET_BUFFER_NEXT_NB(srcNb);
 775                 dstNb = NET_BUFFER_NEXT_NB(dstNb);
 776             }
 777         }
 778     }
 779
 780     status = OvsAllocateNBLContext(context, newNbl);
 781     if (status != NDIS_STATUS_SUCCESS) {
 782         goto nbl_context_error;
 783     }
 784
 785     status = OvsCopyNBLInfo(nbl, newNbl, srcCtx, copySize, copyNblInfo);
 786     if (status != NDIS_STATUS_SUCCESS) {
 787         goto copy_list_info_error;
 788     }
 789
 790 #ifdef DBG
 791     InterlockedIncrement((LONG volatile *)&ovsPool->nblOnlyCount);
 792 #endif
 793
 794     newNbl->ParentNetBufferList = nbl;
 795
 796     dstCtx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(newNbl);
 797     ASSERT(dstCtx != NULL);
 798
 799     flags = srcCtx->flags & (OVS_BUFFER_RECV_BUFFER | OVS_BUFFER_SEND_BUFFER);
 800
 801     flags |= OVS_BUFFER_FROM_NBL_ONLY_POOL | OVS_BUFFER_PRIVATE_CONTEXT |
 802              OVS_BUFFER_PRIVATE_FORWARD_CONTEXT;
 803
 804     srcNb = NET_BUFFER_LIST_FIRST_NB(nbl);
 805     ASSERT(srcNb);
 806     OvsInitNBLContext(dstCtx, flags, NET_BUFFER_DATA_LENGTH(srcNb) - copySize,
 807                       OVS_DPPORT_NUMBER_INVALID);
 808
 809     InterlockedIncrement((LONG volatile *)&srcCtx->refCount);
 810
 811 #ifdef DBG
 812     OvsDumpNetBufferList(nbl);
 813     OvsDumpForwardingDetails(nbl);
 814
 815     OvsDumpNetBufferList(newNbl);
 816     OvsDumpForwardingDetails(newNbl);
 817 #endif
 818
 819     OVS_LOG_LOUD("Partial Copy new NBL: %p", newNbl);
 820     return newNbl;
 821
 822 copy_list_info_error:
 823     OvsFreeNBLContext(context, newNbl);
 824 nbl_context_error:
 825     if (copySize) {
 826         NdisAdvanceNetBufferListDataStart(newNbl, copySize, TRUE, NULL);
 827     }
 828 retreat_error:
 829     NdisFreeCloneNetBufferList(newNbl, 0);
 830     return NULL;
 831 }
 832
 833 /*
 834  * --------------------------------------------------------------------------
 835  * OvsPartialCopyToMultipleNBLs --
 836  *
 837  *     This is similar to OvsPartialCopyNBL() except that each NB will
 838  *     have its own NBL.
 839  * --------------------------------------------------------------------------
 840  */
 841 PNET_BUFFER_LIST
 842 OvsPartialCopyToMultipleNBLs(PVOID ovsContext,
 843                              PNET_BUFFER_LIST nbl,
 844                              UINT32 copySize,
 845                              UINT32 headRoom,
 846                              BOOLEAN copyNblInfo)
 847 {
 848     PNET_BUFFER nb, nextNb = NULL, firstNb, prevNb;
 849     POVS_SWITCH_CONTEXT context = (POVS_SWITCH_CONTEXT)ovsContext;
 850     PNET_BUFFER_LIST firstNbl = NULL, newNbl, prevNbl = NULL;
 851
 852     nb = NET_BUFFER_LIST_FIRST_NB(nbl);
 853     if (NET_BUFFER_NEXT_NB(nb) == NULL) {
 854         return OvsPartialCopyNBL(context, nbl, copySize, headRoom, copyNblInfo);
 855     }
 856
 857     firstNb = nb;
 858     prevNb = nb;
 859
 860     while (nb) {
 861         nextNb = NET_BUFFER_NEXT_NB(nb);
 862         NET_BUFFER_NEXT_NB(nb) = NULL;
 863
 864         NET_BUFFER_LIST_FIRST_NB(nbl) = nb;
 865
 866         newNbl = OvsPartialCopyNBL(context, nbl, copySize, headRoom,
 867                                    copyNblInfo);
 868         if (newNbl == NULL) {
 869             goto cleanup;
 870         }
 871         if (prevNbl == NULL) {
 872             firstNbl = newNbl;
 873         } else {
 874             NET_BUFFER_LIST_NEXT_NBL(prevNbl) = newNbl;
 875             NET_BUFFER_NEXT_NB(prevNb) = nb;
 876         }
 877         prevNbl = newNbl;
 878         prevNb = nb;
 879         nb = nextNb;
 880     }
 881     NET_BUFFER_LIST_FIRST_NB(nbl) = firstNb;
 882     return firstNbl;
 883
 884 cleanup:
 885     NET_BUFFER_NEXT_NB(prevNb) = nb;
 886     NET_BUFFER_NEXT_NB(nb) = nextNb;
 887     NET_BUFFER_LIST_FIRST_NB(nbl) = firstNb;
 888
 889     newNbl = firstNbl;
 890     while (newNbl) {
 891         firstNbl = NET_BUFFER_LIST_NEXT_NBL(newNbl);
 892         NET_BUFFER_LIST_NEXT_NBL(newNbl) = NULL;
 893         OvsCompleteNBL(context, newNbl, TRUE);
 894         newNbl = firstNbl;
 895     }
 896     return NULL;
 897 }
 898
 899
 900 static PNET_BUFFER_LIST
 901 OvsCopySinglePacketNBL(PVOID ovsContext,
 902                        PNET_BUFFER_LIST nbl,
 903                        PNET_BUFFER nb,
 904                        UINT32 headRoom,
 905                        BOOLEAN copyNblInfo)
 906 {
 907     UINT32 size;
 908     ULONG copiedSize;
 909     POVS_SWITCH_CONTEXT context = (POVS_SWITCH_CONTEXT)ovsContext;
 910     PNET_BUFFER_LIST newNbl;
 911     PNET_BUFFER newNb;
 912     NDIS_STATUS status;
 913     POVS_BUFFER_CONTEXT srcCtx, dstCtx;
 914
 915     size = NET_BUFFER_DATA_LENGTH(nb);
 916     if ((size + headRoom) <= OVS_FIX_NBL_DATA_SIZE) {
 917         newNbl = OvsAllocateFixSizeNBL(context, size, headRoom);
 918     } else {
 919         newNbl = OvsAllocateVariableSizeNBL(context, size, headRoom);
 920     }
 921     if (newNbl == NULL) {
 922         return NULL;
 923     }
 924     newNb = NET_BUFFER_LIST_FIRST_NB(newNbl);
 925     status = NdisCopyFromNetBufferToNetBuffer(newNb, 0, size, nb, 0,
 926                                               &copiedSize);
 927
 928     srcCtx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(nbl);
 929     if (status == NDIS_STATUS_SUCCESS) {
 930         status = OvsCopyNBLInfo(nbl, newNbl, srcCtx, copiedSize, copyNblInfo);
 931     }
 932
 933     if (status != NDIS_STATUS_SUCCESS || copiedSize != size) {
 934         OvsCompleteNBL(context, newNbl, TRUE);
 935         return NULL;
 936     }
 937
 938     dstCtx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(newNbl);
 939     ASSERT(dstCtx && srcCtx);
 940     ASSERT(srcCtx->magic == OVS_CTX_MAGIC && dstCtx->magic == OVS_CTX_MAGIC);
 941
 942     dstCtx->flags |= srcCtx->flags & (OVS_BUFFER_RECV_BUFFER |
 943                                       OVS_BUFFER_SEND_BUFFER);
 944 #ifdef DBG
 945     OvsDumpNetBufferList(newNbl);
 946     OvsDumpForwardingDetails(newNbl);
 947 #endif
 948     OVS_LOG_LOUD("Copy single nb to new NBL: %p", newNbl);
 949     return newNbl;
 950 }
 951
 952 /*
 953  * --------------------------------------------------------------------------
 954  * OvsFullCopyNBL --
 955  *
 956  *    Copy the NBL to a new NBL including data.
 957  *
 958  * Notes:
 959  *     The NBL can have multiple NBs, but the final result is one NBL.
 960  * --------------------------------------------------------------------------
 961  */
 962 PNET_BUFFER_LIST
 963 OvsFullCopyNBL(PVOID ovsContext,
 964                PNET_BUFFER_LIST nbl,
 965                UINT32 headRoom,
 966                BOOLEAN copyNblInfo)
 967 {
 968     POVS_SWITCH_CONTEXT context = (POVS_SWITCH_CONTEXT)ovsContext;
 969     POVS_NBL_POOL ovsPool = &context->ovsPool;
 970     PNET_BUFFER_LIST newNbl;
 971     PNET_BUFFER nb, newNb, firstNb = NULL, prevNb = NULL;
 972     POVS_BUFFER_CONTEXT dstCtx, srcCtx;
 973     PMDL mdl;
 974     NDIS_STATUS status;
 975     UINT32 size, totalSize;
 976     ULONG copiedSize;
 977     UINT16 flags;
 978     PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO dstInfo;
 979
 980     srcCtx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(nbl);
 981     if (srcCtx == NULL || srcCtx->magic != OVS_CTX_MAGIC) {
 982         OVS_LOG_INFO("src nbl must have ctx initialized");
 983         ASSERT(srcCtx && srcCtx->magic == OVS_CTX_MAGIC);
 984         return NULL;
 985     }
 986
 987     nb = NET_BUFFER_LIST_FIRST_NB(nbl);
 988     if (nb == NULL) {
 989         return NULL;
 990     }
 991
 992     if (NET_BUFFER_NEXT_NB(nb) == NULL) {
 993         return OvsCopySinglePacketNBL(context, nbl, nb, headRoom, copyNblInfo);
 994     }
 995
 996     newNbl = NdisAllocateNetBufferList(ovsPool->nblOnlyPool,
 997                                        (UINT16)sizeof (OVS_BUFFER_CONTEXT),
 998                                        (UINT16)OVS_DEFAULT_NBL_CONTEXT_FILL);
 999     if (newNbl == NULL) {
1000         return NULL;
1001     }
1002
1003     while (nb) {
1004         size = NET_BUFFER_DATA_LENGTH(nb);
1005         totalSize = MEM_ALIGN_SIZE(size + headRoom);
1006         mdl = OvsAllocateMDLAndData(ovsPool->ndisHandle, totalSize);
1007
1008         if (mdl == NULL) {
1009             goto nblcopy_error;
1010         }
1011         newNb = NdisAllocateNetBuffer(ovsPool->nbPool, mdl, totalSize, 0);
1012         if (newNb == NULL) {
1013             OvsFreeMDLAndData(mdl);
1014             goto nblcopy_error;
1015         }
1016         if (firstNb == NULL) {
1017             firstNb = newNb;
1018         } else {
1019             NET_BUFFER_NEXT_NB(prevNb) = newNb;
1020         }
1021         prevNb = newNb;
1022 #ifdef DBG
1023         InterlockedIncrement((LONG volatile *)&ovsPool->nbCount);
1024 #endif
1025         status = NdisRetreatNetBufferDataStart(newNb, size, 0, NULL);
1026         ASSERT(status == NDIS_STATUS_SUCCESS);
1027
1028         status = NdisCopyFromNetBufferToNetBuffer(newNb, 0, size, nb, 0,
1029                                                   &copiedSize);
1030         if (status != NDIS_STATUS_SUCCESS || size != copiedSize) {
1031             goto nblcopy_error;
1032         }
1033
1034         nb = NET_BUFFER_NEXT_NB(nb);
1035     }
1036
1037     NET_BUFFER_LIST_FIRST_NB(newNbl) = firstNb;
1038
1039     newNbl->SourceHandle = ovsPool->ndisHandle;
1040     status = context->NdisSwitchHandlers.
1041          AllocateNetBufferListForwardingContext(ovsPool->ndisContext, newNbl);
1042
1043     if (status != NDIS_STATUS_SUCCESS) {
1044         goto nblcopy_error;
1045     }
1046
1047     status = OvsCopyNBLInfo(nbl, newNbl, srcCtx, 0, copyNblInfo);
1048     if (status != NDIS_STATUS_SUCCESS) {
1049         goto nblcopy_error;
1050     }
1051
1052     dstInfo = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(newNbl);
1053     dstInfo->IsPacketDataSafe = TRUE;
1054
1055     dstCtx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(newNbl);
1056
1057     flags = srcCtx->flags & (OVS_BUFFER_RECV_BUFFER | OVS_BUFFER_SEND_BUFFER);
1058
1059     flags |= OVS_BUFFER_PRIVATE_MDL | OVS_BUFFER_PRIVATE_DATA |
1060              OVS_BUFFER_PRIVATE_NET_BUFFER | OVS_BUFFER_FROM_NBL_ONLY_POOL |
1061              OVS_BUFFER_PRIVATE_FORWARD_CONTEXT;
1062
1063     OvsInitNBLContext(dstCtx, flags, NET_BUFFER_DATA_LENGTH(firstNb),
1064                       OVS_DPPORT_NUMBER_INVALID);
1065
1066 #ifdef DBG
1067     OvsDumpNetBufferList(nbl);
1068     OvsDumpForwardingDetails(nbl);
1069     InterlockedIncrement((LONG volatile *)&ovsPool->nblOnlyCount);
1070 #endif
1071     OVS_LOG_LOUD("newNbl: %p", newNbl);
1072     return newNbl;
1073
1074 nblcopy_error:
1075     while (firstNb) {
1076 #ifdef DBG
1077         InterlockedDecrement((LONG volatile *)&ovsPool->nbCount);
1078 #endif
1079         prevNb = firstNb;
1080         firstNb = NET_BUFFER_NEXT_NB(prevNb);
1081         mdl = NET_BUFFER_FIRST_MDL(prevNb);
1082         NET_BUFFER_FIRST_MDL(prevNb) = NULL;
1083         NdisFreeNetBuffer(prevNb);
1084         OvsFreeMDLAndData(mdl);
1085     }
1086     NdisFreeNetBufferList(newNbl);
1087     OVS_LOG_ERROR("OvsFullCopyNBL failed");
1088     return NULL;
1089 }
1090
1091 NDIS_STATUS
1092 GetIpHeaderInfo(PNET_BUFFER_LIST curNbl,
1093                 UINT32 *hdrSize)
1094 {
1095     CHAR *ethBuf[sizeof(EthHdr)];
1096     EthHdr *eth;
1097     IPHdr *ipHdr;
1098     PNET_BUFFER curNb;
1099
1100     curNb = NET_BUFFER_LIST_FIRST_NB(curNbl);
1101     ASSERT(NET_BUFFER_NEXT_NB(curNb) == NULL);
1102
1103     eth = (EthHdr *)NdisGetDataBuffer(curNb, ETH_HEADER_LENGTH,
1104                                       (PVOID)&ethBuf, 1, 0);
1105     if (eth == NULL) {
1106         return NDIS_STATUS_INVALID_PACKET;
1107     }
1108     ipHdr = (IPHdr *)((PCHAR)eth + ETH_HEADER_LENGTH);
1109     if (ipHdr == NULL) {
1110         return NDIS_STATUS_INVALID_PACKET;
1111     }
1112     *hdrSize = (UINT32)(ETH_HEADER_LENGTH + (ipHdr->ihl * 4));
1113     return NDIS_STATUS_SUCCESS;
1114 }
1115
1116 /*
1117  * --------------------------------------------------------------------------
1118  * GetSegmentHeaderInfo
1119  *
1120  *    Extract header size and sequence number for the segment.
1121  * --------------------------------------------------------------------------
1122  */
1123 static NDIS_STATUS
1124 GetSegmentHeaderInfo(PNET_BUFFER_LIST nbl,
1125                      const POVS_PACKET_HDR_INFO hdrInfo,
1126                      UINT32 *hdrSize, UINT32 *seqNumber)
1127 {
1128     TCPHdr tcpStorage;
1129     const TCPHdr *tcp;
1130
1131     /* Parse the orginal Eth/IP/TCP header */
1132     tcp = OvsGetPacketBytes(nbl, sizeof *tcp, hdrInfo->l4Offset, &tcpStorage);
1133     if (tcp == NULL) {
1134         return NDIS_STATUS_FAILURE;
1135     }
1136     *seqNumber = ntohl(tcp->seq);
1137     *hdrSize = hdrInfo->l4Offset + TCP_HDR_LEN(tcp);
1138
1139     return NDIS_STATUS_SUCCESS;
1140 }
1141
1142 /*
1143  * --------------------------------------------------------------------------
1144  * FixFragmentHeader
1145  *
1146  *    Fix IP length, Offset, IP checksum.
1147  *    XXX - Support IpV6 Fragments
1148  * --------------------------------------------------------------------------
1149  */
1150 static NDIS_STATUS
1151 FixFragmentHeader(PNET_BUFFER nb, UINT16 fragmentSize,
1152                   BOOLEAN lastPacket, UINT16 offset)
1153 {
1154     EthHdr *dstEth = NULL;
1155     PMDL mdl = NULL;
1156     PUINT8 bufferStart = NULL;
1157
1158     mdl = NET_BUFFER_FIRST_MDL(nb);
1159
1160     bufferStart = (PUINT8)MmGetSystemAddressForMdlSafe(mdl, LowPagePriority);
1161     if (!bufferStart) {
1162         return NDIS_STATUS_RESOURCES;
1163     }
1164     dstEth = (EthHdr *)(bufferStart + NET_BUFFER_CURRENT_MDL_OFFSET(nb));
1165
1166     switch (dstEth->Type) {
1167     case ETH_TYPE_IPV4_NBO:
1168     {
1169         IPHdr *dstIP = NULL;
1170         ASSERT((INT)MmGetMdlByteCount(mdl) - NET_BUFFER_CURRENT_MDL_OFFSET(nb)
1171             >= sizeof(EthHdr) + sizeof(IPHdr));
1172
1173         dstIP = (IPHdr *)((PCHAR)dstEth + sizeof(*dstEth));
1174         ASSERT((INT)MmGetMdlByteCount(mdl) - NET_BUFFER_CURRENT_MDL_OFFSET(nb)
1175             >= sizeof(EthHdr) + dstIP->ihl * 4);
1176         dstIP->tot_len = htons(fragmentSize + dstIP->ihl * 4);
1177         if (lastPacket) {
1178             dstIP->frag_off = htons(offset & IP_OFFSET);
1179         } else {
1180             dstIP->frag_off = htons((offset & IP_OFFSET) | IP_MF);
1181         }
1182
1183         dstIP->check = 0;
1184         dstIP->check = IPChecksum((UINT8 *)dstIP, dstIP->ihl * 4, 0);
1185         break;
1186     }
1187     case ETH_TYPE_IPV6_NBO:
1188     {
1189         return NDIS_STATUS_NOT_SUPPORTED;
1190     }
1191     default:
1192         OVS_LOG_ERROR("Invalid eth type: %d\n", dstEth->Type);
1193         ASSERT(! "Invalid eth type");
1194     }
1195
1196     return STATUS_SUCCESS;
1197 }
1198
1199 /*
1200  * --------------------------------------------------------------------------
1201  * FixSegmentHeader
1202  *
1203  *    Fix IP length, IP checksum, TCP sequence number and TCP checksum
1204  *    in the segment.
1205  * --------------------------------------------------------------------------
1206  */
1207 static NDIS_STATUS
1208 FixSegmentHeader(PNET_BUFFER nb, UINT16 segmentSize, UINT32 seqNumber,
1209                  BOOLEAN lastPacket, UINT16 packetCounter)
1210 {
1211     EthHdr *dstEth = NULL;
1212     TCPHdr *dstTCP = NULL;
1213     PMDL mdl = NULL;
1214     PUINT8 bufferStart = NULL;
1215
1216     mdl = NET_BUFFER_FIRST_MDL(nb);
1217
1218     bufferStart = (PUINT8)MmGetSystemAddressForMdlSafe(mdl, LowPagePriority);
1219     if (!bufferStart) {
1220         return NDIS_STATUS_RESOURCES;
1221     }
1222     dstEth = (EthHdr *)(bufferStart + NET_BUFFER_CURRENT_MDL_OFFSET(nb));
1223
1224     switch (dstEth->Type) {
1225     case ETH_TYPE_IPV4_NBO:
1226     {
1227         IPHdr *dstIP = NULL;
1228
1229         ASSERT((INT)MmGetMdlByteCount(mdl) - NET_BUFFER_CURRENT_MDL_OFFSET(nb)
1230                 >= sizeof(EthHdr) + sizeof(IPHdr) + sizeof(TCPHdr));
1231         dstIP = (IPHdr *)((PCHAR)dstEth + sizeof(*dstEth));
1232         dstTCP = (TCPHdr *)((PCHAR)dstIP + dstIP->ihl * 4);
1233         ASSERT((INT)MmGetMdlByteCount(mdl) - NET_BUFFER_CURRENT_MDL_OFFSET(nb)
1234                 >= sizeof(EthHdr) + dstIP->ihl * 4 + TCP_HDR_LEN(dstTCP));
1235
1236         /* Fix IP length and checksum */
1237         ASSERT(dstIP->protocol == IPPROTO_TCP);
1238         dstIP->tot_len = htons(segmentSize + dstIP->ihl * 4 + TCP_HDR_LEN(dstTCP));
1239         dstIP->id += packetCounter;
1240         dstIP->check = 0;
1241         dstIP->check = IPChecksum((UINT8 *)dstIP, dstIP->ihl * 4, 0);
1242         dstTCP->seq = htonl(seqNumber);
1243
1244         /*
1245          * Set the TCP FIN and PSH bit only for the last packet
1246          * More information can be found under:
1247          * https://msdn.microsoft.com/en-us/library/windows/hardware/ff568840%28v=vs.85%29.aspx
1248          */
1249         if (dstTCP->fin) {
1250             dstTCP->fin = lastPacket;
1251         }
1252         if (dstTCP->psh) {
1253             dstTCP->psh = lastPacket;
1254         }
1255         UINT16 csumLength = segmentSize + TCP_HDR_LEN(dstTCP);
1256         dstTCP->check = IPPseudoChecksum(&dstIP->saddr,
1257                                          &dstIP->daddr,
1258                                          IPPROTO_TCP,
1259                                          csumLength);
1260         dstTCP->check = CalculateChecksumNB(nb,
1261                                             csumLength,
1262                                             sizeof(*dstEth) + dstIP->ihl * 4);
1263         break;
1264     }
1265     case ETH_TYPE_IPV6_NBO:
1266     {
1267         IPv6Hdr *dstIP = NULL;
1268
1269         ASSERT((INT)MmGetMdlByteCount(mdl) - NET_BUFFER_CURRENT_MDL_OFFSET(nb)
1270             >= sizeof(EthHdr) + sizeof(IPv6Hdr) + sizeof(TCPHdr));
1271         dstIP = (IPv6Hdr *)((PCHAR)dstEth + sizeof(*dstEth));
1272         dstTCP = (TCPHdr *)((PCHAR)dstIP + sizeof(IPv6Hdr));
1273         ASSERT((INT)MmGetMdlByteCount(mdl) - NET_BUFFER_CURRENT_MDL_OFFSET(nb)
1274             >= sizeof(EthHdr) + sizeof(IPv6Hdr) + TCP_HDR_LEN(dstTCP));
1275
1276         /* Fix IP length */
1277         ASSERT(dstIP->nexthdr == IPPROTO_TCP);
1278         dstIP->payload_len = htons(segmentSize + sizeof(IPv6Hdr) + TCP_HDR_LEN(dstTCP));
1279
1280         dstTCP->seq = htonl(seqNumber);
1281         if (dstTCP->fin) {
1282             dstTCP->fin = lastPacket;
1283         }
1284         if (dstTCP->psh) {
1285             dstTCP->psh = lastPacket;
1286         }
1287
1288         UINT16 csumLength = segmentSize + TCP_HDR_LEN(dstTCP);
1289         dstTCP->check = IPv6PseudoChecksum((UINT32*)&dstIP->saddr,
1290                                            (UINT32*)&dstIP->daddr,
1291                                            IPPROTO_TCP,
1292                                            csumLength);
1293         dstTCP->check = CalculateChecksumNB(nb,
1294                                             csumLength,
1295                                             sizeof(*dstEth) + sizeof(IPv6Hdr));
1296         break;
1297     }
1298     default:
1299         OVS_LOG_ERROR("Invalid eth type: %d\n", dstEth->Type);
1300         ASSERT(! "Invalid eth type");
1301     }
1302
1303     return STATUS_SUCCESS;
1304 }
1305  /*
1306   * --------------------------------------------------------------------------
1307  * OvsTcpSegmentNBL --
1308  *    Wrapper function to Fragment a given NBL based on MSS
1309  * --------------------------------------------------------------------------
1310  */
1311 PNET_BUFFER_LIST
1312 OvsTcpSegmentNBL(PVOID ovsContext,
1313                  PNET_BUFFER_LIST nbl,
1314                  POVS_PACKET_HDR_INFO hdrInfo,
1315                  UINT32 mss,
1316                  UINT32 headRoom,
1317                  BOOLEAN isIpFragment)
1318 {
1319     return OvsFragmentNBL(ovsContext, nbl, hdrInfo, mss, headRoom, isIpFragment);
1320 }
1321
1322
1323 /*
1324  * --------------------------------------------------------------------------
1325  * OvsFragmentNBL --
1326  *
1327  *    Fragment NBL payload, and prepend each segment with Ether/IP/TCP header.
1328  *    Leave headRoom for additional encap.
1329  *
1330  *    Please note,
1331  *       NBL should have OVS_BUFFER_CONTEXT setup before calling
1332  *       this function.
1333  *       The NBL should already have ref to itself so that during copy
1334  *       it will not be freed.
1335  *       Currently this API assert there is only one NB in an NBL, it needs
1336  *       to be fixed if we receive multiple NBs in an NBL.
1337  * --------------------------------------------------------------------------
1338  */
1339 PNET_BUFFER_LIST
1340 OvsFragmentNBL(PVOID ovsContext,
1341                PNET_BUFFER_LIST nbl,
1342                POVS_PACKET_HDR_INFO hdrInfo,
1343                UINT32 fragmentSize,
1344                UINT32 headRoom,
1345                BOOLEAN isIpFragment)
1346 {
1347     POVS_SWITCH_CONTEXT context = (POVS_SWITCH_CONTEXT)ovsContext;
1348 #ifdef DBG
1349     POVS_NBL_POOL ovsPool = &context->ovsPool;
1350 #endif
1351     POVS_BUFFER_CONTEXT dstCtx, srcCtx;
1352     UINT32 size, hdrSize, nblSize, seqNumber = 0;
1353     PNET_BUFFER_LIST newNbl;
1354     PNET_BUFFER nb, newNb;
1355     NDIS_STATUS status;
1356     UINT16 segmentSize;
1357     ULONG copiedSize;
1358     UINT16 offset = 0, packetCounter = 0;
1359
1360     srcCtx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(nbl);
1361     if (srcCtx == NULL || srcCtx->magic != OVS_CTX_MAGIC) {
1362         OVS_LOG_INFO("src nbl must have ctx initialized");
1363         ASSERT(srcCtx && srcCtx->magic == OVS_CTX_MAGIC);
1364         return NULL;
1365     }
1366
1367     nb = NET_BUFFER_LIST_FIRST_NB(nbl);
1368     ASSERT(NET_BUFFER_NEXT_NB(nb) == NULL);
1369
1370     /* Figure out the header size */
1371     if (isIpFragment) {
1372         status = GetIpHeaderInfo(nbl, &hdrSize);
1373     } else {
1374         status = GetSegmentHeaderInfo(nbl, hdrInfo, &hdrSize, &seqNumber);
1375     }
1376     if (status != NDIS_STATUS_SUCCESS) {
1377         OVS_LOG_INFO("Cannot parse NBL header");
1378         return NULL;
1379     }
1380     /* Get the NBL size. */
1381     if (isIpFragment) {
1382         nblSize = fragmentSize - hdrSize;
1383     } else {
1384         nblSize = fragmentSize;
1385     }
1386     size = NET_BUFFER_DATA_LENGTH(nb) - hdrSize;
1387
1388     /* XXX add to ovsPool counters? */
1389     newNbl = NdisAllocateFragmentNetBufferList(nbl, NULL, NULL, hdrSize,
1390                                                nblSize, hdrSize + headRoom ,
1391                                                0, 0);
1392     if (newNbl == NULL) {
1393         return NULL;
1394     }
1395
1396     /* Now deal with TCP payload */
1397     for (newNb = NET_BUFFER_LIST_FIRST_NB(newNbl); newNb != NULL;
1398             newNb = NET_BUFFER_NEXT_NB(newNb)) {
1399         segmentSize = (size > nblSize ? nblSize : size) & 0xffff;
1400         if (headRoom) {
1401             NdisAdvanceNetBufferDataStart(newNb, headRoom, FALSE, NULL);
1402         }
1403
1404         /* Now copy the eth/IP/TCP header and fix up */
1405         status = NdisCopyFromNetBufferToNetBuffer(newNb, 0, hdrSize, nb, 0,
1406                                                   &copiedSize);
1407         if (status != NDIS_STATUS_SUCCESS || hdrSize != copiedSize) {
1408             goto nblcopy_error;
1409         }
1410
1411         if (isIpFragment) {
1412             status = FixFragmentHeader(newNb, segmentSize,
1413                                        NET_BUFFER_NEXT_NB(newNb) == NULL,
1414                                        offset);
1415         } else {
1416             status = FixSegmentHeader(newNb, segmentSize, seqNumber,
1417                                       NET_BUFFER_NEXT_NB(newNb) == NULL,
1418                                       packetCounter);
1419         }
1420
1421         if (status != NDIS_STATUS_SUCCESS) {
1422             goto nblcopy_error;
1423         }
1424
1425         /* Move on to the next segment */
1426         if (isIpFragment) {
1427             offset += (segmentSize) / 8;
1428         } else {
1429             seqNumber += segmentSize;
1430         }
1431         size -= segmentSize;
1432         packetCounter++;
1433     }
1434
1435     status = OvsAllocateNBLContext(context, newNbl);
1436     if (status != NDIS_STATUS_SUCCESS) {
1437         goto nblcopy_error;
1438     }
1439
1440     status = OvsCopyNBLInfo(nbl, newNbl, srcCtx, hdrSize + headRoom, FALSE);
1441     if (status != NDIS_STATUS_SUCCESS) {
1442         goto nbl_context_error;
1443     }
1444
1445     if (isIpFragment) {
1446     /*Copy with Flag - NDIS_SWITCH_COPY_NBL_INFO_FLAGS_PRESERVE_DESTINATIONS.*/
1447         status = context->NdisSwitchHandlers.
1448             CopyNetBufferListInfo(context->ovsPool.ndisContext, newNbl, nbl, 1);
1449
1450         if (status != NDIS_STATUS_SUCCESS) {
1451             goto nbl_context_error;
1452         }
1453     }
1454     newNbl->ParentNetBufferList = nbl;
1455
1456     /* Remember it's a fragment NBL so we can free it properly */
1457     dstCtx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(newNbl);
1458     ASSERT(dstCtx != NULL);
1459     dstCtx->flags = OVS_BUFFER_FRAGMENT | OVS_BUFFER_PRIVATE_CONTEXT |
1460         OVS_BUFFER_PRIVATE_FORWARD_CONTEXT | OVS_BUFFER_SEND_BUFFER;
1461     dstCtx->refCount = 1;
1462     dstCtx->magic = OVS_CTX_MAGIC;
1463     dstCtx->dataOffsetDelta = hdrSize + headRoom;
1464     dstCtx->mru = 0;
1465
1466     InterlockedIncrement((LONG volatile *)&srcCtx->refCount);
1467 #ifdef DBG
1468     InterlockedIncrement((LONG volatile *)&ovsPool->fragNBLCount);
1469
1470     OvsDumpNetBufferList(nbl);
1471     OvsDumpForwardingDetails(nbl);
1472
1473     OvsDumpNetBufferList(newNbl);
1474     OvsDumpForwardingDetails(newNbl);
1475 #endif
1476     OVS_LOG_TRACE("Fragment nbl %p to newNbl: %p", nbl, newNbl);
1477     return newNbl;
1478
1479 nbl_context_error:
1480     OvsFreeNBLContext(context, newNbl);
1481 nblcopy_error:
1482 #ifdef DBG
1483     InterlockedDecrement((LONG volatile *)&ovsPool->fragNBLCount);
1484 #endif
1485     NdisFreeFragmentNetBufferList(newNbl, hdrSize + headRoom, 0);
1486     return NULL;
1487 }
1488
1489 /*
1490  * --------------------------------------------------------------------------
1491  * OvsAllocateNBLFromBuffer --
1492  *
1493  * This function allocates all the stuff necessary for creating an NBL from the
1494  * input buffer of specified length, namely, a nonpaged data buffer of size
1495  * length, an MDL from it, and a NB and NBL from it. It does not allocate an NBL
1496  * context yet. It also copies data from the specified buffer to the NBL.
1497  * --------------------------------------------------------------------------
1498  */
1499 PNET_BUFFER_LIST
1500 OvsAllocateNBLFromBuffer(PVOID context,
1501                          PVOID buffer,
1502                          ULONG length)
1503 {
1504     POVS_SWITCH_CONTEXT switchContext = (POVS_SWITCH_CONTEXT)context;
1505     UINT8 *data = NULL;
1506     PNET_BUFFER_LIST nbl = NULL;
1507     PNET_BUFFER nb;
1508     PMDL mdl;
1509
1510     if (length > OVS_DEFAULT_DATA_SIZE) {
1511         nbl = OvsAllocateVariableSizeNBL(switchContext, length,
1512                                          OVS_DEFAULT_HEADROOM_SIZE);
1513
1514     } else {
1515         nbl = OvsAllocateFixSizeNBL(switchContext, length,
1516                                     OVS_DEFAULT_HEADROOM_SIZE);
1517     }
1518     if (nbl == NULL) {
1519         return NULL;
1520     }
1521
1522     nb = NET_BUFFER_LIST_FIRST_NB(nbl);
1523     mdl = NET_BUFFER_CURRENT_MDL(nb);
1524     data = (PUINT8)MmGetSystemAddressForMdlSafe(mdl, LowPagePriority) +
1525                     NET_BUFFER_CURRENT_MDL_OFFSET(nb);
1526     if (!data) {
1527         OvsCompleteNBL(switchContext, nbl, TRUE);
1528         return NULL;
1529     }
1530
1531     NdisMoveMemory(data, buffer, length);
1532
1533     return nbl;
1534 }
1535
1536 /*
1537  * --------------------------------------------------------------------------
1538  * OvsFullCopyToMultipleNBLs --
1539  *
1540  *    Copy NBL to multiple NBLs, each NB will have its own NBL
1541  * --------------------------------------------------------------------------
1542  */
1543 PNET_BUFFER_LIST
1544 OvsFullCopyToMultipleNBLs(PVOID ovsContext,
1545                           PNET_BUFFER_LIST nbl,
1546                           UINT32 headRoom,
1547                           BOOLEAN copyNblInfo)
1548 {
1549
1550     POVS_SWITCH_CONTEXT context = (POVS_SWITCH_CONTEXT)ovsContext;
1551     PNET_BUFFER_LIST firstNbl, currNbl, newNbl;
1552     PNET_BUFFER nb;
1553     POVS_BUFFER_CONTEXT srcCtx;
1554
1555     srcCtx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(nbl);
1556     if (srcCtx == NULL || srcCtx->magic != OVS_CTX_MAGIC) {
1557         OVS_LOG_INFO("src nbl must have ctx initialized");
1558         ASSERT(srcCtx && srcCtx->magic == OVS_CTX_MAGIC);
1559         return NULL;
1560     }
1561
1562     nb =  NET_BUFFER_LIST_FIRST_NB(nbl);
1563     newNbl = OvsCopySinglePacketNBL(context, nbl, nb, headRoom, copyNblInfo);
1564
1565     if (newNbl == NULL || NET_BUFFER_NEXT_NB(nb) == NULL) {
1566         return newNbl;
1567     } else {
1568         firstNbl = newNbl;
1569         currNbl = newNbl;
1570     }
1571
1572     while (nb) {
1573         newNbl = OvsCopySinglePacketNBL(context, nbl, nb, headRoom,
1574                                         copyNblInfo);
1575         if (newNbl == NULL) {
1576             goto copymultiple_error;
1577         }
1578         NET_BUFFER_LIST_NEXT_NBL(currNbl) = newNbl;
1579         currNbl = newNbl;
1580         nb = NET_BUFFER_NEXT_NB(nb);
1581     }
1582     return firstNbl;
1583
1584 copymultiple_error:
1585     while (firstNbl) {
1586         currNbl = firstNbl;
1587         firstNbl = NET_BUFFER_LIST_NEXT_NBL(firstNbl);
1588         NET_BUFFER_LIST_NEXT_NBL(currNbl) = NULL;
1589         OvsCompleteNBL(context, currNbl, TRUE);
1590     }
1591     return NULL;
1592
1593 }
1594
1595
1596 /*
1597  * --------------------------------------------------------------------------
1598  * OvsCompleteNBL --
1599  *
1600  *     This function tries to free the NBL allocated by OVS buffer
1601  *     management module. If it trigger the completion of the parent
1602  *     NBL, it will recursively call itself. If it trigger the completion
1603  *     of external NBL, it will be returned to the caller. The caller
1604  *     is responsible to call API to return to upper layer.
1605  * --------------------------------------------------------------------------
1606  */
1607 PNET_BUFFER_LIST
1608 OvsCompleteNBL(POVS_SWITCH_CONTEXT context,
1609                PNET_BUFFER_LIST nbl,
1610                BOOLEAN updateRef)
1611 {
1612     POVS_BUFFER_CONTEXT ctx;
1613     UINT16 flags;
1614     PNET_BUFFER_LIST parent;
1615     NDIS_STATUS status;
1616     NDIS_HANDLE poolHandle;
1617     LONG value;
1618     POVS_NBL_POOL ovsPool = &context->ovsPool;
1619     PNET_BUFFER nb;
1620
1621
1622     ctx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(nbl);
1623
1624     ASSERT(ctx && ctx->magic == OVS_CTX_MAGIC);
1625
1626     OVS_LOG_TRACE("Enter: nbl: %p, ctx: %p, refCount: %d, updateRef:%d",
1627                  nbl, ctx, ctx->refCount, updateRef);
1628
1629     if (updateRef) {
1630         value = InterlockedDecrement((LONG volatile *)&ctx->refCount);
1631         if (value != 0) {
1632             return NULL;
1633         }
1634     } else {
1635         /*
1636          * This is a special case, the refCount must be zero
1637          */
1638         ASSERT(ctx->refCount == 0);
1639     }
1640
1641     nb = NET_BUFFER_LIST_FIRST_NB(nbl);
1642
1643     flags = ctx->flags;
1644     if (!(flags & OVS_BUFFER_FRAGMENT) &&
1645         NET_BUFFER_DATA_LENGTH(nb) != ctx->origDataLength) {
1646         UINT32 diff;
1647         if (NET_BUFFER_DATA_LENGTH(nb) < ctx->origDataLength) {
1648             diff = ctx->origDataLength -NET_BUFFER_DATA_LENGTH(nb);
1649             status = NdisRetreatNetBufferListDataStart(nbl, diff, 0,
1650                                                        NULL, NULL);
1651             ASSERT(status == NDIS_STATUS_SUCCESS);
1652         } else {
1653             diff = NET_BUFFER_DATA_LENGTH(nb) - ctx->origDataLength;
1654             NdisAdvanceNetBufferListDataStart(nbl, diff, TRUE, NULL);
1655         }
1656     }
1657
1658     if (ctx->flags & OVS_BUFFER_PRIVATE_CONTEXT) {
1659         NdisFreeNetBufferListContext(nbl, sizeof (OVS_BUFFER_CONTEXT));
1660     }
1661
1662     if (flags & OVS_BUFFER_NEED_COMPLETE) {
1663         /*
1664          * return to caller for completion
1665          */
1666 #ifdef DBG
1667         InterlockedDecrement((LONG volatile *)&ovsPool->sysNBLCount);
1668 #endif
1669         return nbl;
1670     }
1671
1672     if (flags & OVS_BUFFER_PRIVATE_FORWARD_CONTEXT) {
1673         context->NdisSwitchHandlers.
1674               FreeNetBufferListForwardingContext(ovsPool->ndisContext, nbl);
1675     }
1676
1677     if (flags & (OVS_BUFFER_PRIVATE_MDL | OVS_BUFFER_PRIVATE_DATA)) {
1678         PNET_BUFFER nbTemp = NET_BUFFER_LIST_FIRST_NB(nbl);
1679         while (nbTemp) {
1680             PMDL mdl = NET_BUFFER_FIRST_MDL(nbTemp);
1681             if (mdl) {
1682                 ASSERT(mdl->Next == NULL);
1683                 OvsFreeMDLAndData(mdl);
1684             }
1685             NET_BUFFER_FIRST_MDL(nbTemp) = NULL;
1686             nbTemp = NET_BUFFER_NEXT_NB(nbTemp);
1687         }
1688     }
1689
1690     if (flags & OVS_BUFFER_PRIVATE_NET_BUFFER) {
1691         PNET_BUFFER nbTemp, nextNb;
1692
1693         nbTemp = NET_BUFFER_LIST_FIRST_NB(nbl);
1694         while (nbTemp) {
1695             nextNb = NET_BUFFER_NEXT_NB(nbTemp);
1696             NdisFreeNetBuffer(nbTemp);
1697 #ifdef DBG
1698             InterlockedDecrement((LONG volatile *)&ovsPool->nbCount);
1699 #endif
1700             nbTemp = nextNb;
1701         }
1702         NET_BUFFER_LIST_FIRST_NB(nbl) = NULL;
1703     }
1704
1705     parent = nbl->ParentNetBufferList;
1706
1707     poolHandle = NdisGetPoolFromNetBufferList(nbl);
1708     if (flags & OVS_BUFFER_FROM_FIX_SIZE_POOL) {
1709         ASSERT(poolHandle == ovsPool->fixSizePool);
1710 #ifdef DBG
1711         InterlockedDecrement((LONG volatile *)&ovsPool->fixNBLCount);
1712 #endif
1713         NdisFreeNetBufferList(nbl);
1714     } else if (flags & OVS_BUFFER_FROM_ZERO_SIZE_POOL) {
1715         ASSERT(poolHandle == ovsPool->zeroSizePool);
1716 #ifdef DBG
1717         InterlockedDecrement((LONG volatile *)&ovsPool->zeroNBLCount);
1718 #endif
1719         NdisFreeNetBufferList(nbl);
1720     } else if (flags & OVS_BUFFER_FROM_NBL_ONLY_POOL) {
1721         ASSERT(poolHandle == ovsPool->nblOnlyPool);
1722 #ifdef DBG
1723         InterlockedDecrement((LONG volatile *)&ovsPool->nblOnlyCount);
1724 #endif
1725         NdisFreeCloneNetBufferList(nbl, 0);
1726     } else if (flags & OVS_BUFFER_FRAGMENT) {
1727         OVS_LOG_TRACE("Free fragment %p parent %p", nbl, parent);
1728 #ifdef DBG
1729         InterlockedDecrement((LONG volatile *)&ovsPool->fragNBLCount);
1730 #endif
1731         NdisFreeFragmentNetBufferList(nbl, ctx->dataOffsetDelta, 0);
1732     }
1733
1734     if (parent != NULL) {
1735         ctx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(parent);
1736         ASSERT(ctx && ctx->magic == OVS_CTX_MAGIC);
1737         value = InterlockedDecrement((LONG volatile *)&ctx->refCount);
1738         if (value == 0) {
1739             return OvsCompleteNBL(context, parent, FALSE);
1740         }
1741     }
1742     return NULL;
1743 }
1744
1745 /*
1746  * --------------------------------------------------------------------------
1747  * OvsSetCtxSourcePortNo --
1748  *      Setter function which stores the source port of an NBL in the NBL
1749  * Context Info.
1750  * --------------------------------------------------------------------------
1751  */
1752 NDIS_STATUS
1753 OvsSetCtxSourcePortNo(PNET_BUFFER_LIST nbl,
1754                       UINT32 portNo)
1755 {
1756     POVS_BUFFER_CONTEXT ctx;
1757     ctx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(nbl);
1758     if (ctx == NULL) {
1759         ASSERT(ctx && ctx->magic == OVS_CTX_MAGIC);
1760         return STATUS_INVALID_PARAMETER;
1761     }
1762
1763     ctx->srcPortNo = portNo;
1764     return NDIS_STATUS_SUCCESS;
1765 }
1766
1767 /*
1768  * --------------------------------------------------------------------------
1769  * OvsGetCtxSourcePortNo --
1770  *      Get source port of an NBL from its Context Info.
1771  * --------------------------------------------------------------------------
1772  */
1773 NDIS_STATUS
1774 OvsGetCtxSourcePortNo(PNET_BUFFER_LIST nbl,
1775                       UINT32 *portNo)
1776 {
1777     POVS_BUFFER_CONTEXT ctx;
1778     ctx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(nbl);
1779     if (ctx == NULL || portNo == NULL) {
1780         ASSERT(ctx && ctx->magic == OVS_CTX_MAGIC);
1781         return STATUS_INVALID_PARAMETER;
1782     }
1783     *portNo = ctx->srcPortNo;
1784     return NDIS_STATUS_SUCCESS;
1785 }