[mirror_ovs.git] / datapath-windows / ovsext / PacketIO.c

/*
 * Copyright (c) 2014 VMware, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * This file contains the implementation of the datapath/forwarding
 * functionality of the OVS.
 */

#include "precomp.h"

#include "Actions.h"
#include "Switch.h"
#include "Vport.h"
#include "NetProto.h"
#include "User.h"
#include "PacketIO.h"
#include "Flow.h"
#include "Event.h"
#include "User.h"

/* Due to an imported header file */
#pragma warning( disable:4505 )

#ifdef OVS_DBG_MOD
#undef OVS_DBG_MOD
#endif
#define OVS_DBG_MOD OVS_DBG_DISPATCH
#include "Debug.h"

extern NDIS_STRING ovsExtGuidUC;
extern NDIS_STRING ovsExtFriendlyNameUC;

static VOID OvsFinalizeCompletionList(OvsCompletionList *completionList);
static VOID OvsCompleteNBLIngress(POVS_SWITCH_CONTEXT switchContext,
                    PNET_BUFFER_LIST netBufferLists, ULONG sendCompleteFlags);
static NTSTATUS OvsCreateNewNBLsFromMultipleNBs(
                    POVS_SWITCH_CONTEXT switchContext,
                    PNET_BUFFER_LIST *curNbl,
                    PNET_BUFFER_LIST *nextNbl);

__inline VOID
OvsInitCompletionList(OvsCompletionList *completionList,
                      POVS_SWITCH_CONTEXT switchContext,
                      ULONG sendCompleteFlags)
{
    ASSERT(completionList);
    completionList->dropNbl = NULL;
    completionList->dropNblNext = &completionList->dropNbl;
    completionList->switchContext = switchContext;
    completionList->sendCompleteFlags = sendCompleteFlags;
}

/* Utility function used to complete an NBL. */
__inline VOID
OvsAddPktCompletionList(OvsCompletionList *completionList,
                        BOOLEAN incoming,
                        NDIS_SWITCH_PORT_ID sourcePort,
                        PNET_BUFFER_LIST netBufferList,
                        UINT32 netBufferListCount,
                        PNDIS_STRING filterReason)
{
    POVS_BUFFER_CONTEXT ctx;

    /* XXX: We handle one NBL at a time. */
    ASSERT(netBufferList->Next == NULL);

    /* Make sure it has a context. */
    ctx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(netBufferList);
    ASSERT(ctx && ctx->magic == OVS_CTX_MAGIC);

    completionList->switchContext->NdisSwitchHandlers.ReportFilteredNetBufferLists(
        completionList->switchContext->NdisSwitchContext, &ovsExtGuidUC,
        &ovsExtFriendlyNameUC, sourcePort,
        incoming ? NDIS_SWITCH_REPORT_FILTERED_NBL_FLAGS_IS_INCOMING : 0,
        netBufferListCount, netBufferList, filterReason);

    *completionList->dropNblNext = netBufferList;
    completionList->dropNblNext = &netBufferList->Next;
    ASSERT(completionList->dropNbl);
}

static __inline VOID
OvsReportNBLIngressError(POVS_SWITCH_CONTEXT switchContext,
                         PNET_BUFFER_LIST nblList,
                         PNDIS_STRING filterReason,
                         NDIS_STATUS error)
{
    PNET_BUFFER_LIST nbl = nblList;
    while (nbl) {
        PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO fwdDetail;
        fwdDetail = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(nbl);

        nbl->Status = error;

        /* This can be optimized by batching NBL's from the same
         * SourcePortId. */
        switchContext->NdisSwitchHandlers.ReportFilteredNetBufferLists(
            switchContext->NdisSwitchContext, &ovsExtGuidUC,
            &ovsExtFriendlyNameUC, fwdDetail->SourcePortId,
            NDIS_SWITCH_REPORT_FILTERED_NBL_FLAGS_IS_INCOMING,
            1 /*Nbl count.*/, nbl, filterReason);

        nbl = NET_BUFFER_LIST_NEXT_NBL(nbl);
    }
}

static __inline ULONG
OvsGetSendCompleteFlags(ULONG sendFlags)
{
    BOOLEAN dispatch, sameSource;
    ULONG sendCompleteFlags;

    dispatch = NDIS_TEST_SEND_AT_DISPATCH_LEVEL(sendFlags);
    sendCompleteFlags = (dispatch ?
                        NDIS_SEND_COMPLETE_FLAGS_DISPATCH_LEVEL : 0);
    sameSource = NDIS_TEST_SEND_FLAG(sendFlags,
                                        NDIS_SEND_FLAGS_SWITCH_SINGLE_SOURCE);
    sendCompleteFlags |= (sameSource ?
                        NDIS_SEND_COMPLETE_FLAGS_SWITCH_SINGLE_SOURCE : 0);

    return sendCompleteFlags;
}

VOID
OvsSendNBLIngress(POVS_SWITCH_CONTEXT switchContext,
                  PNET_BUFFER_LIST netBufferLists,
                  ULONG sendFlags)
{
    if (switchContext->dataFlowState == OvsSwitchPaused) {
        /* If a filter module is in the Paused state, the filter driver must not
         * originate any send requests for that filter module. If NDIS calls
         * FilterSendNetBufferLists, the driver must not call
         * NdisFSendNetBufferLists to pass on the data until the driver is
         * restarted. The driver should call NdisFSendNetBufferListsComplete
         * immediately to complete the send operation. It should set the
         * complete status in each NET_BUFFER_LIST structure to
         * NDIS_STATUS_PAUSED.
         *
         * http://msdn.microsoft.com/en-us/library/windows/hardware/
         * ff549966(v=vs.85).aspx */
        NDIS_STRING filterReason;
        ULONG sendCompleteFlags = OvsGetSendCompleteFlags(sendFlags);

        RtlInitUnicodeString(&filterReason,
                             L"Switch state PAUSED, drop before FSendNBL.");
        OvsReportNBLIngressError(switchContext, netBufferLists, &filterReason,
                                 NDIS_STATUS_PAUSED);
        OvsCompleteNBLIngress(switchContext, netBufferLists,
                              sendCompleteFlags);
        return;
    }

    ASSERT(switchContext->dataFlowState == OvsSwitchRunning);

    NdisFSendNetBufferLists(switchContext->NdisFilterHandle, netBufferLists,
                            NDIS_DEFAULT_PORT_NUMBER, sendFlags);
}

static __inline VOID
OvsStartNBLIngressError(POVS_SWITCH_CONTEXT switchContext,
                        PNET_BUFFER_LIST nblList,
                        ULONG sendCompleteFlags,
                        PNDIS_STRING filterReason,
                        NDIS_STATUS error)
{
    ASSERT(error);
    OvsReportNBLIngressError(switchContext, nblList, filterReason, error);
    NdisFSendNetBufferListsComplete(switchContext->NdisFilterHandle, nblList,
                                    sendCompleteFlags);
}

static VOID
OvsAppendNativeForwardedPacket(POVS_SWITCH_CONTEXT switchContext,
                               PNET_BUFFER_LIST curNbl,
                               PNET_BUFFER_LIST *nativeNbls,
                               ULONG flags,
                               BOOLEAN isRecv)
{
    POVS_BUFFER_CONTEXT ctx = { 0 };
    NDIS_STRING filterReason;

    *nativeNbls = curNbl;
    nativeNbls = &(curNbl->Next);

    ctx = OvsInitExternalNBLContext(switchContext, curNbl, isRecv);
    if (ctx == NULL) {
        RtlInitUnicodeString(&filterReason,
                             L"Cannot allocate native NBL context.");

        OvsStartNBLIngressError(switchContext, curNbl, flags, &filterReason,
                                NDIS_STATUS_RESOURCES);
    }
}

static VOID
OvsStartNBLIngress(POVS_SWITCH_CONTEXT switchContext,
                   PNET_BUFFER_LIST netBufferLists,
                   ULONG SendFlags)
{
    NDIS_SWITCH_PORT_ID sourcePort = 0;
    NDIS_SWITCH_NIC_INDEX sourceIndex = 0;
    PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO fwdDetail;
    PNET_BUFFER_LIST curNbl = NULL, nextNbl = NULL;
    ULONG sendCompleteFlags;
    UCHAR dispatch;
    LOCK_STATE_EX lockState, dpLockState;
    NDIS_STATUS status;
    NDIS_STRING filterReason;
    LIST_ENTRY missedPackets;
    UINT32 num = 0;
    OvsCompletionList completionList;
#if (NDIS_SUPPORT_NDIS640)
    PNET_BUFFER_LIST nativeForwardedNbls = NULL;
    PNET_BUFFER_LIST *nextNativeForwardedNbl = &nativeForwardedNbls;
#endif

    dispatch = NDIS_TEST_SEND_AT_DISPATCH_LEVEL(SendFlags)?
                                            NDIS_RWL_AT_DISPATCH_LEVEL : 0;
    sendCompleteFlags = OvsGetSendCompleteFlags(SendFlags);
    SendFlags |= NDIS_SEND_FLAGS_SWITCH_DESTINATION_GROUP;

    InitializeListHead(&missedPackets);
    OvsInitCompletionList(&completionList, switchContext, sendCompleteFlags);

    for (curNbl = netBufferLists; curNbl != NULL; curNbl = nextNbl) {
        POVS_VPORT_ENTRY vport = NULL;
        UINT32 portNo = 0;
        OVS_DATAPATH *datapath = &switchContext->datapath;
        OVS_PACKET_HDR_INFO layers = { 0 };
        OvsFlowKey key = { 0 };
        UINT64 hash = 0;
        PNET_BUFFER curNb = NULL;
        POVS_BUFFER_CONTEXT ctx = NULL;

        nextNbl = curNbl->Next;
        curNbl->Next = NULL;

        fwdDetail = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(curNbl);
        sourcePort = fwdDetail->SourcePortId;
        sourceIndex = (NDIS_SWITCH_NIC_INDEX)fwdDetail->SourceNicIndex;

#if (NDIS_SUPPORT_NDIS640)
        if (fwdDetail->NativeForwardingRequired) {
            /* Add current NBL to those that require native forwarding. */
            OvsAppendNativeForwardedPacket(
                switchContext,
                curNbl,
                nextNativeForwardedNbl,
                sendCompleteFlags,
                sourcePort == switchContext->virtualExternalPortId);
            continue;
        }
#endif /* NDIS_SUPPORT_NDIS640 */

        ctx = OvsInitExternalNBLContext(switchContext, curNbl,
                  sourcePort == switchContext->virtualExternalPortId);
        if (ctx == NULL) {
            RtlInitUnicodeString(&filterReason,
                L"Cannot allocate external NBL context.");

            OvsStartNBLIngressError(switchContext, curNbl,
                                    sendCompleteFlags, &filterReason,
                                    NDIS_STATUS_RESOURCES);
            continue;
        }

        /* Ethernet Header is a guaranteed safe access. */
        curNb = NET_BUFFER_LIST_FIRST_NB(curNbl);
        if (curNb->Next != NULL) {
            /* Create a NET_BUFFER_LIST for each NET_BUFFER. */
            status = OvsCreateNewNBLsFromMultipleNBs(switchContext,
                                                     &curNbl,
                                                     &nextNbl);
            if (!NT_SUCCESS(status)) {
                RtlInitUnicodeString(&filterReason,
                                     L"Cannot allocate NBLs with single NB.");

                OvsStartNBLIngressError(switchContext, curNbl,
                                        sendCompleteFlags, &filterReason,
                                        NDIS_STATUS_RESOURCES);
                continue;
            }
        }
        {
            OvsFlow *flow;

            /* Take the DispatchLock so none of the VPORTs disconnect while
             * we are setting destination ports.
             *
             * XXX: acquire/release the dispatch lock for a "batch" of packets
             * rather than for each packet. */
            NdisAcquireRWLockRead(switchContext->dispatchLock, &lockState,
                                  dispatch);

            vport = OvsFindVportByPortIdAndNicIndex(switchContext, sourcePort,
                                                    sourceIndex);
            if (vport == NULL || vport->ovsState != OVS_STATE_CONNECTED) {
                RtlInitUnicodeString(&filterReason,
                    L"OVS-Cannot forward packet from unknown source port");
                goto dropit;
            } else {
                portNo = vport->portNo;
            }

            vport->stats.rxPackets++;
            vport->stats.rxBytes += NET_BUFFER_DATA_LENGTH(curNb);

            status = OvsExtractFlow(curNbl, vport->portNo, &key, &layers, NULL);
            if (status != NDIS_STATUS_SUCCESS) {
                RtlInitUnicodeString(&filterReason, L"OVS-Flow extract failed");
                goto dropit;
            }

            ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
            OvsAcquireDatapathRead(datapath, &dpLockState, TRUE);

            flow = OvsLookupFlow(datapath, &key, &hash, FALSE);
            if (flow) {
                OvsFlowUsed(flow, curNbl, &layers);
                datapath->hits++;
                /* If successful, OvsActionsExecute() consumes the NBL.
                 * Otherwise, it adds it to the completionList. No need to
                 * check the return value. */
                OvsActionsExecute(switchContext, &completionList, curNbl,
                                  portNo, SendFlags, &key, &hash, &layers,
                                  flow->actions, flow->actionsLen);
                OvsReleaseDatapath(datapath, &dpLockState);
                NdisReleaseRWLock(switchContext->dispatchLock, &lockState);
                continue;
            } else {
                OvsReleaseDatapath(datapath, &dpLockState);

                datapath->misses++;
                status = OvsCreateAndAddPackets(NULL, 0, OVS_PACKET_CMD_MISS,
                             vport, &key, curNbl,
                             sourcePort == switchContext->virtualExternalPortId,
                             &layers, switchContext, &missedPackets, &num);
                if (status == NDIS_STATUS_SUCCESS) {
                    /* Complete the packet since it was copied to user
                     * buffer. */
                    RtlInitUnicodeString(&filterReason,
                        L"OVS-Dropped since packet was copied to userspace");
                } else {
                    RtlInitUnicodeString(&filterReason,
                        L"OVS-Dropped due to failure to queue to userspace");
                }
                goto dropit;
            }

dropit:
            OvsAddPktCompletionList(&completionList, TRUE, sourcePort, curNbl, 0,
                                    &filterReason);
            NdisReleaseRWLock(switchContext->dispatchLock, &lockState);
        }
    }

#if (NDIS_SUPPORT_NDIS640)
    if (nativeForwardedNbls) {
        /* This is NVGRE encapsulated traffic and is forwarded to NDIS
         * in order to be handled by the HNV module. */
        OvsSendNBLIngress(switchContext, nativeForwardedNbls, SendFlags);
    }
#endif /* NDIS_SUPPORT_NDIS640 */

    /* Queue the missed packets. */
    OvsQueuePackets(&missedPackets, num);
    OvsFinalizeCompletionList(&completionList);
}


/*
 * --------------------------------------------------------------------------
 * Implements filter driver's FilterSendNetBufferLists Function.
 * --------------------------------------------------------------------------
 */
VOID
OvsExtSendNBL(NDIS_HANDLE filterModuleContext,
              PNET_BUFFER_LIST netBufferLists,
              NDIS_PORT_NUMBER portNumber,
              ULONG sendFlags)
{
    UNREFERENCED_PARAMETER(portNumber);

    /* 'filterModuleContext' is the switch context that gets created in the
     * AttachHandler. */
    POVS_SWITCH_CONTEXT switchContext;
    switchContext = (POVS_SWITCH_CONTEXT) filterModuleContext;

    if (switchContext->dataFlowState == OvsSwitchPaused) {
        NDIS_STRING filterReason;
        ULONG sendCompleteFlags = OvsGetSendCompleteFlags(sendFlags);

        RtlInitUnicodeString(&filterReason,
                             L"Switch state PAUSED, drop on ingress.");
        OvsStartNBLIngressError(switchContext, netBufferLists,
                                sendCompleteFlags, &filterReason,
                                NDIS_STATUS_PAUSED);
        return;
    }

    ASSERT(switchContext->dataFlowState == OvsSwitchRunning);

    OvsStartNBLIngress(switchContext, netBufferLists, sendFlags);
}

static VOID
OvsCompleteNBLIngress(POVS_SWITCH_CONTEXT switchContext,
                      PNET_BUFFER_LIST netBufferLists,
                      ULONG sendCompleteFlags)
{
    PNET_BUFFER_LIST curNbl = NULL, nextNbl = NULL;
    OvsCompletionList newList;

    newList.dropNbl = NULL;
    newList.dropNblNext = &newList.dropNbl;

    for (curNbl = netBufferLists; curNbl != NULL; curNbl = nextNbl) {
        nextNbl = curNbl->Next;
        curNbl->Next = NULL;

        curNbl = OvsCompleteNBL(switchContext, curNbl, TRUE);
        if (curNbl != NULL) {
            /* NBL originated from the upper layer. */
            *newList.dropNblNext = curNbl;
            newList.dropNblNext = &curNbl->Next;
        }
    }

    /* Complete the NBL's that were sent by the upper layer. */
    if (newList.dropNbl != NULL) {
        NdisFSendNetBufferListsComplete(switchContext->NdisFilterHandle, newList.dropNbl,
                                        sendCompleteFlags);
    }
}


/*
 * --------------------------------------------------------------------------
 * Implements filter driver's FilterSendNetBufferListsComplete function.
 * --------------------------------------------------------------------------
 */
VOID
OvsExtSendNBLComplete(NDIS_HANDLE filterModuleContext,
                      PNET_BUFFER_LIST netBufferLists,
                      ULONG sendCompleteFlags)
{
    OvsCompleteNBLIngress((POVS_SWITCH_CONTEXT)filterModuleContext,
                          netBufferLists, sendCompleteFlags);
}


VOID
OvsFinalizeCompletionList(OvsCompletionList *completionList)
{
    if (completionList->dropNbl != NULL) {
        OvsCompleteNBLIngress(completionList->switchContext,
                              completionList->dropNbl,
                              completionList->sendCompleteFlags);

        completionList->dropNbl = NULL;
        completionList->dropNblNext = &completionList->dropNbl;
    }
}

/*
 * --------------------------------------------------------------------------
 * Implements filter driver's FilterCancelSendNetBufferLists function.
 *
 * "If a filter driver specifies a FilterSendNetBufferLists function and it
 * queues send requests, it must also specify a
 * FilterCancelSendNetBufferLists function."
 *
 * http://msdn.microsoft.com/en-us/library/windows/hardware/
 * ff549966(v=vs.85).aspx
 * --------------------------------------------------------------------------
 */
VOID
OvsExtCancelSendNBL(NDIS_HANDLE filterModuleContext,
                    PVOID CancelId)
{
    UNREFERENCED_PARAMETER(filterModuleContext);
    UNREFERENCED_PARAMETER(CancelId);

    /* All send requests get completed synchronously, so there is no need to
     * implement this callback. */
}

static NTSTATUS
OvsCreateNewNBLsFromMultipleNBs(POVS_SWITCH_CONTEXT switchContext,
                                PNET_BUFFER_LIST *curNbl,
                                PNET_BUFFER_LIST *nextNbl)
{
    NTSTATUS status = STATUS_SUCCESS;
    PNET_BUFFER_LIST newNbls = NULL;
    PNET_BUFFER_LIST lastNbl = NULL;
    PNET_BUFFER_LIST nbl = NULL;
    BOOLEAN error = TRUE;

    do {
        /* Create new NBLs from curNbl with multiple net buffers. */
        newNbls = OvsPartialCopyToMultipleNBLs(switchContext,
                                               *curNbl, 0, 0, TRUE);
        if (NULL == newNbls) {
            OVS_LOG_ERROR("Failed to allocate NBLs with single NB.");
            status = NDIS_STATUS_RESOURCES;
            break;
        }

        nbl = newNbls;
        while (nbl) {
            lastNbl = nbl;
            nbl = NET_BUFFER_LIST_NEXT_NBL(nbl);
        }
        lastNbl->Next = *nextNbl;
        *nextNbl = newNbls->Next;

        OvsCompleteNBL(switchContext, *curNbl, TRUE);

        *curNbl = newNbls;
        (*curNbl)->Next = NULL;

        error = FALSE;
    } while (error);

    return status;
}
Commit	Line	Data
c803536e SS	1	/*
	2	* Copyright (c) 2014 VMware, Inc.
	3	*
	4	* Licensed under the Apache License, Version 2.0 (the "License");
	5	* you may not use this file except in compliance with the License.
	6	* You may obtain a copy of the License at:
	7	*
	8	* http://www.apache.org/licenses/LICENSE-2.0
	9	*
	10	* Unless required by applicable law or agreed to in writing, software
	11	* distributed under the License is distributed on an "AS IS" BASIS,
	12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	13	* See the License for the specific language governing permissions and
	14	* limitations under the License.
	15	*/
	16
	17	/*
	18	* This file contains the implementation of the datapath/forwarding
	19	* functionality of the OVS.
	20	*/
	21
	22	#include "precomp.h"
ee25964a SV	23
ee25964a SV	24	#include "Actions.h"
fa1324c9 SG	25	#include "Switch.h"
	26	#include "Vport.h"
	27	#include "NetProto.h"
	28	#include "User.h"
	29	#include "PacketIO.h"
	30	#include "Flow.h"
	31	#include "Event.h"
	32	#include "User.h"
c803536e SS	33
	34	/* Due to an imported header file */
	35	#pragma warning( disable:4505 )
	36
	37	#ifdef OVS_DBG_MOD
	38	#undef OVS_DBG_MOD
	39	#endif
	40	#define OVS_DBG_MOD OVS_DBG_DISPATCH
fa1324c9	41	#include "Debug.h"
c803536e SS	42
	43	extern NDIS_STRING ovsExtGuidUC;
	44	extern NDIS_STRING ovsExtFriendlyNameUC;
	45
	46	static VOID OvsFinalizeCompletionList(OvsCompletionList *completionList);
	47	static VOID OvsCompleteNBLIngress(POVS_SWITCH_CONTEXT switchContext,
	48	PNET_BUFFER_LIST netBufferLists, ULONG sendCompleteFlags);
58b11928 SV	49	static NTSTATUS OvsCreateNewNBLsFromMultipleNBs(
	50	POVS_SWITCH_CONTEXT switchContext,
	51	PNET_BUFFER_LIST *curNbl,
	52	PNET_BUFFER_LIST *nextNbl);
c803536e SS	53
	54	__inline VOID
	55	OvsInitCompletionList(OvsCompletionList *completionList,
	56	POVS_SWITCH_CONTEXT switchContext,
	57	ULONG sendCompleteFlags)
	58	{
	59	ASSERT(completionList);
	60	completionList->dropNbl = NULL;
	61	completionList->dropNblNext = &completionList->dropNbl;
	62	completionList->switchContext = switchContext;
	63	completionList->sendCompleteFlags = sendCompleteFlags;
	64	}
	65
	66	/* Utility function used to complete an NBL. */
	67	__inline VOID
	68	OvsAddPktCompletionList(OvsCompletionList *completionList,
	69	BOOLEAN incoming,
	70	NDIS_SWITCH_PORT_ID sourcePort,
	71	PNET_BUFFER_LIST netBufferList,
	72	UINT32 netBufferListCount,
	73	PNDIS_STRING filterReason)
	74	{
	75	POVS_BUFFER_CONTEXT ctx;
	76
	77	/* XXX: We handle one NBL at a time. */
	78	ASSERT(netBufferList->Next == NULL);
	79
	80	/* Make sure it has a context. */
	81	ctx = (POVS_BUFFER_CONTEXT)NET_BUFFER_LIST_CONTEXT_DATA_START(netBufferList);
	82	ASSERT(ctx && ctx->magic == OVS_CTX_MAGIC);
	83
	84	completionList->switchContext->NdisSwitchHandlers.ReportFilteredNetBufferLists(
	85	completionList->switchContext->NdisSwitchContext, &ovsExtGuidUC,
	86	&ovsExtFriendlyNameUC, sourcePort,
	87	incoming ? NDIS_SWITCH_REPORT_FILTERED_NBL_FLAGS_IS_INCOMING : 0,
	88	netBufferListCount, netBufferList, filterReason);
	89
	90	*completionList->dropNblNext = netBufferList;
	91	completionList->dropNblNext = &netBufferList->Next;
	92	ASSERT(completionList->dropNbl);
	93	}
	94
	95	static __inline VOID
	96	OvsReportNBLIngressError(POVS_SWITCH_CONTEXT switchContext,
	97	PNET_BUFFER_LIST nblList,
	98	PNDIS_STRING filterReason,
	99	NDIS_STATUS error)
	100	{
	101	PNET_BUFFER_LIST nbl = nblList;
	102	while (nbl) {
	103	PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO fwdDetail;
	104	fwdDetail = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(nbl);
	105
	106	nbl->Status = error;
	107
	108	/* This can be optimized by batching NBL's from the same
	109	* SourcePortId. */
	110	switchContext->NdisSwitchHandlers.ReportFilteredNetBufferLists(
	111	switchContext->NdisSwitchContext, &ovsExtGuidUC,
	112	&ovsExtFriendlyNameUC, fwdDetail->SourcePortId,
	113	NDIS_SWITCH_REPORT_FILTERED_NBL_FLAGS_IS_INCOMING,
	114	1 /Nbl count./, nbl, filterReason);
	115
	116	nbl = NET_BUFFER_LIST_NEXT_NBL(nbl);
117	}
118	}
119
120	static __inline ULONG
121	OvsGetSendCompleteFlags(ULONG sendFlags)
122	{
123	BOOLEAN dispatch, sameSource;
124	ULONG sendCompleteFlags;
125
126	dispatch = NDIS_TEST_SEND_AT_DISPATCH_LEVEL(sendFlags);
127	sendCompleteFlags = (dispatch ?
128	NDIS_SEND_COMPLETE_FLAGS_DISPATCH_LEVEL : 0);
129	sameSource = NDIS_TEST_SEND_FLAG(sendFlags,
130	NDIS_SEND_FLAGS_SWITCH_SINGLE_SOURCE);
131	sendCompleteFlags \|= (sameSource ?
132	NDIS_SEND_COMPLETE_FLAGS_SWITCH_SINGLE_SOURCE : 0);
133
134	return sendCompleteFlags;
135	}
136
137	VOID
138	OvsSendNBLIngress(POVS_SWITCH_CONTEXT switchContext,
139	PNET_BUFFER_LIST netBufferLists,
140	ULONG sendFlags)
141	{
142	if (switchContext->dataFlowState == OvsSwitchPaused) {
143	/* If a filter module is in the Paused state, the filter driver must not
144	* originate any send requests for that filter module. If NDIS calls
145	* FilterSendNetBufferLists, the driver must not call
146	* NdisFSendNetBufferLists to pass on the data until the driver is
147	* restarted. The driver should call NdisFSendNetBufferListsComplete
148	* immediately to complete the send operation. It should set the
149	* complete status in each NET_BUFFER_LIST structure to
150	* NDIS_STATUS_PAUSED.
151	*
152	* http://msdn.microsoft.com/en-us/library/windows/hardware/
153	* ff549966(v=vs.85).aspx */
154	NDIS_STRING filterReason;
155	ULONG sendCompleteFlags = OvsGetSendCompleteFlags(sendFlags);
156
157	RtlInitUnicodeString(&filterReason,
158	L"Switch state PAUSED, drop before FSendNBL.");
159	OvsReportNBLIngressError(switchContext, netBufferLists, &filterReason,
160	NDIS_STATUS_PAUSED);
161	OvsCompleteNBLIngress(switchContext, netBufferLists,
162	sendCompleteFlags);
163	return;
164	}
165
166	ASSERT(switchContext->dataFlowState == OvsSwitchRunning);
167
168	NdisFSendNetBufferLists(switchContext->NdisFilterHandle, netBufferLists,
169	NDIS_DEFAULT_PORT_NUMBER, sendFlags);
170	}
171
172	static __inline VOID
173	OvsStartNBLIngressError(POVS_SWITCH_CONTEXT switchContext,
174	PNET_BUFFER_LIST nblList,
175	ULONG sendCompleteFlags,
176	PNDIS_STRING filterReason,
177	NDIS_STATUS error)
178	{
179	ASSERT(error);
180	OvsReportNBLIngressError(switchContext, nblList, filterReason, error);
181	NdisFSendNetBufferListsComplete(switchContext->NdisFilterHandle, nblList,
182	sendCompleteFlags);
183	}
184
d0000b68 SV	185	static VOID
	186	OvsAppendNativeForwardedPacket(POVS_SWITCH_CONTEXT switchContext,
	187	PNET_BUFFER_LIST curNbl,
	188	PNET_BUFFER_LIST *nativeNbls,
	189	ULONG flags,
	190	BOOLEAN isRecv)
	191	{
	192	POVS_BUFFER_CONTEXT ctx = { 0 };
	193	NDIS_STRING filterReason;
	194
	195	*nativeNbls = curNbl;
	196	nativeNbls = &(curNbl->Next);
	197
	198	ctx = OvsInitExternalNBLContext(switchContext, curNbl, isRecv);
	199	if (ctx == NULL) {
	200	RtlInitUnicodeString(&filterReason,
	201	L"Cannot allocate native NBL context.");
	202
	203	OvsStartNBLIngressError(switchContext, curNbl, flags, &filterReason,
	204	NDIS_STATUS_RESOURCES);
	205	}
	206	}
	207
c803536e SS	208	static VOID
	209	OvsStartNBLIngress(POVS_SWITCH_CONTEXT switchContext,
	210	PNET_BUFFER_LIST netBufferLists,
	211	ULONG SendFlags)
	212	{
	213	NDIS_SWITCH_PORT_ID sourcePort = 0;
	214	NDIS_SWITCH_NIC_INDEX sourceIndex = 0;
	215	PNDIS_SWITCH_FORWARDING_DETAIL_NET_BUFFER_LIST_INFO fwdDetail;
	216	PNET_BUFFER_LIST curNbl = NULL, nextNbl = NULL;
	217	ULONG sendCompleteFlags;
	218	UCHAR dispatch;
	219	LOCK_STATE_EX lockState, dpLockState;
	220	NDIS_STATUS status;
	221	NDIS_STRING filterReason;
	222	LIST_ENTRY missedPackets;
	223	UINT32 num = 0;
	224	OvsCompletionList completionList;
8217bc65	225	#if (NDIS_SUPPORT_NDIS640)
d6b7cf62	226	PNET_BUFFER_LIST nativeForwardedNbls = NULL;
d6b7cf62	227	PNET_BUFFER_LIST *nextNativeForwardedNbl = &nativeForwardedNbls;
8217bc65	228	#endif
c803536e SS	229
	230	dispatch = NDIS_TEST_SEND_AT_DISPATCH_LEVEL(SendFlags)?
	231	NDIS_RWL_AT_DISPATCH_LEVEL : 0;
	232	sendCompleteFlags = OvsGetSendCompleteFlags(SendFlags);
	233	SendFlags \|= NDIS_SEND_FLAGS_SWITCH_DESTINATION_GROUP;
	234
	235	InitializeListHead(&missedPackets);
	236	OvsInitCompletionList(&completionList, switchContext, sendCompleteFlags);
	237
	238	for (curNbl = netBufferLists; curNbl != NULL; curNbl = nextNbl) {
ee25964a SV	239	POVS_VPORT_ENTRY vport = NULL;
ee25964a SV	240	UINT32 portNo = 0;
c803536e	241	OVS_DATAPATH *datapath = &switchContext->datapath;
ee25964a SV	242	OVS_PACKET_HDR_INFO layers = { 0 };
	243	OvsFlowKey key = { 0 };
	244	UINT64 hash = 0;
	245	PNET_BUFFER curNb = NULL;
	246	POVS_BUFFER_CONTEXT ctx = NULL;
c803536e SS	247
	248	nextNbl = curNbl->Next;
	249	curNbl->Next = NULL;
	250
d0000b68 SV	251	fwdDetail = NET_BUFFER_LIST_SWITCH_FORWARDING_DETAIL(curNbl);
	252	sourcePort = fwdDetail->SourcePortId;
	253	sourceIndex = (NDIS_SWITCH_NIC_INDEX)fwdDetail->SourceNicIndex;
	254
	255	#if (NDIS_SUPPORT_NDIS640)
	256	if (fwdDetail->NativeForwardingRequired) {
	257	/* Add current NBL to those that require native forwarding. */
	258	OvsAppendNativeForwardedPacket(
	259	switchContext,
	260	curNbl,
	261	nextNativeForwardedNbl,
	262	sendCompleteFlags,
	263	sourcePort == switchContext->virtualExternalPortId);
	264	continue;
	265	}
8217bc65	266	#endif /* NDIS_SUPPORT_NDIS640 */
d0000b68	267
58b11928 SV	268	ctx = OvsInitExternalNBLContext(switchContext, curNbl,
	269	sourcePort == switchContext->virtualExternalPortId);
	270	if (ctx == NULL) {
3bfe44a4	271	RtlInitUnicodeString(&filterReason,
58b11928 SV	272	L"Cannot allocate external NBL context.");
58b11928 SV	273
3bfe44a4 NR	274	OvsStartNBLIngressError(switchContext, curNbl,
	275	sendCompleteFlags, &filterReason,
	276	NDIS_STATUS_RESOURCES);
	277	continue;
58b11928 SV	278	}
	279
	280	/* Ethernet Header is a guaranteed safe access. */
	281	curNb = NET_BUFFER_LIST_FIRST_NB(curNbl);
	282	if (curNb->Next != NULL) {
	283	/* Create a NET_BUFFER_LIST for each NET_BUFFER. */
	284	status = OvsCreateNewNBLsFromMultipleNBs(switchContext,
	285	&curNbl,
	286	&nextNbl);
	287	if (!NT_SUCCESS(status)) {
	288	RtlInitUnicodeString(&filterReason,
	289	L"Cannot allocate NBLs with single NB.");
	290
	291	OvsStartNBLIngressError(switchContext, curNbl,
	292	sendCompleteFlags, &filterReason,
	293	NDIS_STATUS_RESOURCES);
	294	continue;
	295	}
	296	}
	297	{
c803536e SS	298	OvsFlow *flow;
c803536e SS	299
c803536e SS	300	/* Take the DispatchLock so none of the VPORTs disconnect while
	301	* we are setting destination ports.
	302	*
	303	* XXX: acquire/release the dispatch lock for a "batch" of packets
	304	* rather than for each packet. */
	305	NdisAcquireRWLockRead(switchContext->dispatchLock, &lockState,
	306	dispatch);
	307
c803536e SS	308	vport = OvsFindVportByPortIdAndNicIndex(switchContext, sourcePort,
	309	sourceIndex);
	310	if (vport == NULL \|\| vport->ovsState != OVS_STATE_CONNECTED) {
	311	RtlInitUnicodeString(&filterReason,
	312	L"OVS-Cannot forward packet from unknown source port");
	313	goto dropit;
	314	} else {
	315	portNo = vport->portNo;
	316	}
	317
	318	vport->stats.rxPackets++;
	319	vport->stats.rxBytes += NET_BUFFER_DATA_LENGTH(curNb);
	320
	321	status = OvsExtractFlow(curNbl, vport->portNo, &key, &layers, NULL);
	322	if (status != NDIS_STATUS_SUCCESS) {
	323	RtlInitUnicodeString(&filterReason, L"OVS-Flow extract failed");
	324	goto dropit;
	325	}
	326
	327	ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
1ce4551c	328	OvsAcquireDatapathRead(datapath, &dpLockState, TRUE);
c803536e SS	329
	330	flow = OvsLookupFlow(datapath, &key, &hash, FALSE);
	331	if (flow) {
	332	OvsFlowUsed(flow, curNbl, &layers);
	333	datapath->hits++;
	334	/* If successful, OvsActionsExecute() consumes the NBL.
	335	* Otherwise, it adds it to the completionList. No need to
	336	* check the return value. */
	337	OvsActionsExecute(switchContext, &completionList, curNbl,
4c470e88 SV	338	portNo, SendFlags, &key, &hash, &layers,
4c470e88 SV	339	flow->actions, flow->actionsLen);
c803536e SS	340	OvsReleaseDatapath(datapath, &dpLockState);
	341	NdisReleaseRWLock(switchContext->dispatchLock, &lockState);
	342	continue;
	343	} else {
	344	OvsReleaseDatapath(datapath, &dpLockState);
	345
	346	datapath->misses++;
640ebde7	347	status = OvsCreateAndAddPackets(NULL, 0, OVS_PACKET_CMD_MISS,
4c470e88	348	vport, &key, curNbl,
7434992b NR	349	sourcePort == switchContext->virtualExternalPortId,
7434992b NR	350	&layers, switchContext, &missedPackets, &num);
c803536e SS	351	if (status == NDIS_STATUS_SUCCESS) {
	352	/* Complete the packet since it was copied to user
	353	* buffer. */
	354	RtlInitUnicodeString(&filterReason,
	355	L"OVS-Dropped since packet was copied to userspace");
	356	} else {
	357	RtlInitUnicodeString(&filterReason,
	358	L"OVS-Dropped due to failure to queue to userspace");
	359	}
	360	goto dropit;
	361	}
	362
	363	dropit:
	364	OvsAddPktCompletionList(&completionList, TRUE, sourcePort, curNbl, 0,
	365	&filterReason);
	366	NdisReleaseRWLock(switchContext->dispatchLock, &lockState);
	367	}
	368	}
	369
8217bc65	370	#if (NDIS_SUPPORT_NDIS640)
d6b7cf62 SV	371	if (nativeForwardedNbls) {
	372	/* This is NVGRE encapsulated traffic and is forwarded to NDIS
	373	* in order to be handled by the HNV module. */
	374	OvsSendNBLIngress(switchContext, nativeForwardedNbls, SendFlags);
	375	}
8217bc65	376	#endif /* NDIS_SUPPORT_NDIS640 */
d6b7cf62	377
c803536e	378	/* Queue the missed packets. */
4a3c9b70	379	OvsQueuePackets(&missedPackets, num);
c803536e SS	380	OvsFinalizeCompletionList(&completionList);
	381	}
	382
	383
	384	/*
	385	* --------------------------------------------------------------------------
	386	* Implements filter driver's FilterSendNetBufferLists Function.
	387	* --------------------------------------------------------------------------
	388	*/
	389	VOID
	390	OvsExtSendNBL(NDIS_HANDLE filterModuleContext,
	391	PNET_BUFFER_LIST netBufferLists,
	392	NDIS_PORT_NUMBER portNumber,
	393	ULONG sendFlags)
	394	{
	395	UNREFERENCED_PARAMETER(portNumber);
	396
	397	/* 'filterModuleContext' is the switch context that gets created in the
	398	* AttachHandler. */
	399	POVS_SWITCH_CONTEXT switchContext;
	400	switchContext = (POVS_SWITCH_CONTEXT) filterModuleContext;
	401
	402	if (switchContext->dataFlowState == OvsSwitchPaused) {
	403	NDIS_STRING filterReason;
	404	ULONG sendCompleteFlags = OvsGetSendCompleteFlags(sendFlags);
	405
	406	RtlInitUnicodeString(&filterReason,
	407	L"Switch state PAUSED, drop on ingress.");
	408	OvsStartNBLIngressError(switchContext, netBufferLists,
	409	sendCompleteFlags, &filterReason,
	410	NDIS_STATUS_PAUSED);
	411	return;
	412	}
	413
	414	ASSERT(switchContext->dataFlowState == OvsSwitchRunning);
	415
	416	OvsStartNBLIngress(switchContext, netBufferLists, sendFlags);
	417	}
	418
	419	static VOID
	420	OvsCompleteNBLIngress(POVS_SWITCH_CONTEXT switchContext,
	421	PNET_BUFFER_LIST netBufferLists,
	422	ULONG sendCompleteFlags)
	423	{
	424	PNET_BUFFER_LIST curNbl = NULL, nextNbl = NULL;
	425	OvsCompletionList newList;
	426
	427	newList.dropNbl = NULL;
	428	newList.dropNblNext = &newList.dropNbl;
	429
	430	for (curNbl = netBufferLists; curNbl != NULL; curNbl = nextNbl) {
	431	nextNbl = curNbl->Next;
	432	curNbl->Next = NULL;
	433
	434	curNbl = OvsCompleteNBL(switchContext, curNbl, TRUE);
	435	if (curNbl != NULL) {
	436	/* NBL originated from the upper layer. */
	437	*newList.dropNblNext = curNbl;
	438	newList.dropNblNext = &curNbl->Next;
	439	}
	440	}
	441
	442	/* Complete the NBL's that were sent by the upper layer. */
	443	if (newList.dropNbl != NULL) {
444	NdisFSendNetBufferListsComplete(switchContext->NdisFilterHandle, newList.dropNbl,
445	sendCompleteFlags);
446	}
447	}
448
449
450	/*
451	* --------------------------------------------------------------------------
452	* Implements filter driver's FilterSendNetBufferListsComplete function.
453	* --------------------------------------------------------------------------
454	*/
455	VOID
456	OvsExtSendNBLComplete(NDIS_HANDLE filterModuleContext,
457	PNET_BUFFER_LIST netBufferLists,
458	ULONG sendCompleteFlags)
459	{
460	OvsCompleteNBLIngress((POVS_SWITCH_CONTEXT)filterModuleContext,
461	netBufferLists, sendCompleteFlags);
462	}
463
464
465	VOID
466	OvsFinalizeCompletionList(OvsCompletionList *completionList)
467	{
468	if (completionList->dropNbl != NULL) {
469	OvsCompleteNBLIngress(completionList->switchContext,
470	completionList->dropNbl,
471	completionList->sendCompleteFlags);
472
473	completionList->dropNbl = NULL;
474	completionList->dropNblNext = &completionList->dropNbl;
475	}
476	}
477
478	/*
479	* --------------------------------------------------------------------------
480	* Implements filter driver's FilterCancelSendNetBufferLists function.
481	*
482	* "If a filter driver specifies a FilterSendNetBufferLists function and it
483	* queues send requests, it must also specify a
484	* FilterCancelSendNetBufferLists function."
485	*
486	* http://msdn.microsoft.com/en-us/library/windows/hardware/
487	* ff549966(v=vs.85).aspx
488	* --------------------------------------------------------------------------
489	*/
490	VOID
491	OvsExtCancelSendNBL(NDIS_HANDLE filterModuleContext,
492	PVOID CancelId)
493	{
494	UNREFERENCED_PARAMETER(filterModuleContext);
495	UNREFERENCED_PARAMETER(CancelId);
496
497	/* All send requests get completed synchronously, so there is no need to
498	* implement this callback. */
499	}
58b11928 SV	500
	501	static NTSTATUS
	502	OvsCreateNewNBLsFromMultipleNBs(POVS_SWITCH_CONTEXT switchContext,
	503	PNET_BUFFER_LIST *curNbl,
	504	PNET_BUFFER_LIST *nextNbl)
	505	{
	506	NTSTATUS status = STATUS_SUCCESS;
	507	PNET_BUFFER_LIST newNbls = NULL;
	508	PNET_BUFFER_LIST lastNbl = NULL;
	509	PNET_BUFFER_LIST nbl = NULL;
58b11928 SV	510	BOOLEAN error = TRUE;
	511
	512	do {
	513	/* Create new NBLs from curNbl with multiple net buffers. */
	514	newNbls = OvsPartialCopyToMultipleNBLs(switchContext,
	515	*curNbl, 0, 0, TRUE);
	516	if (NULL == newNbls) {
	517	OVS_LOG_ERROR("Failed to allocate NBLs with single NB.");
	518	status = NDIS_STATUS_RESOURCES;
	519	break;
	520	}
	521
	522	nbl = newNbls;
	523	while (nbl) {
	524	lastNbl = nbl;
	525	nbl = NET_BUFFER_LIST_NEXT_NBL(nbl);
	526	}
	527	lastNbl->Next = *nextNbl;
	528	*nextNbl = newNbls->Next;
58b11928 SV	529
	530	OvsCompleteNBL(switchContext, *curNbl, TRUE);
	531
21f21788 SV	532	*curNbl = newNbls;
	533	(*curNbl)->Next = NULL;
	534
58b11928 SV	535	error = FALSE;
	536	} while (error);
	537
	538	return status;
	539	}