diff mbox

[ovs-dev] datapath-windows: Hot add CPU support.

Message ID 508339EC0242094682895ED3EC4EBA315014E9@CBSEX1.cloudbase.local
State Not Applicable
Headers show

Commit Message

Paul Boca April 1, 2016, 7:10 a.m. UTC
Hi Sorin!

The only issue I see here is in OvsCpuChange. This function is called twice for each processor added to the system, once with KeProcessorAddStartNotify and second with KeProcessorAddCompleteNotify or KeProcessorAddFailureNotify.
You make reallocation on StartNotify and AddFailureNotify, but it's possible that for current processor the callback OvsCpuChange will be called with both states, sequentially.
And KeQueryActiveProcessorCountEx will return you the old processor count on KeProcessorAddStartNotify - the current added processor is not completely active yet.

If you change KeProcessorAddStartNotify with KeProcessorAddCompleteNotify, all should be fine.

Thanks,
Paul

-----Original Message-----
From: dev [mailto:dev-bounces@openvswitch.org] On Behalf Of Sorin Vinturis

Sent: Thursday, March 31, 2016 3:03 PM
To: dev@openvswitch.org
Subject: [ovs-dev] [PATCH] datapath-windows: Hot add CPU support.

Hot add CPU is the ability to dynamically add CPUs to a running system. Adding CPUs can occur physically by adding new hardware, logically by online hardware partitioning, or virtually through a virtualization layer.

This patch add support to reallocate any per-cpu resources, in case a new processor is added.

Signed-off-by: Sorin Vinturis <svinturis@cloudbasesolutions.com>

Reported-by: Sorin Vinturis <svinturis@cloudbasesolutions.com>
Reported-at: https://github.com/openvswitch/ovs-issues/issues/112
---
 datapath-windows/ovsext/Actions.c  |  11 +-
 datapath-windows/ovsext/Datapath.c |   4 -
 datapath-windows/ovsext/Driver.c   |   9 ++
 datapath-windows/ovsext/Recirc.c   | 245 +++++++++++++++++++++++++++++++------
 datapath-windows/ovsext/Recirc.h   |  87 ++++++-------
 datapath-windows/ovsext/Util.c     | 122 +++++++++++++++++-
 datapath-windows/ovsext/Util.h     |  27 +++-
 7 files changed, 410 insertions(+), 95 deletions(-)

+/*
+ * 
+-----------------------------------------------------------------------
+---
+ * OvsPerCpuDataInit --
+ *     The function allocates necessary per-processor resources and registers
+ *     processor change callback.
+ * 
+-----------------------------------------------------------------------
+---
+ */
+NTSTATUS
+OvsPerCpuDataInit(NDIS_HANDLE NdisFilterHandle);
+
+/*
+ * 
+-----------------------------------------------------------------------
+---
+ * OvsPerCpuDataCleanup --
+ *     The function frees all per-processor resources and deregisters
+ *     processor change callback.
+ * 
+-----------------------------------------------------------------------
+---
+ */
+VOID
+OvsPerCpuDataCleanup();
+
 #endif /* __UTIL_H_ */
--
1.9.0.msysgit.0
_______________________________________________
dev mailing list
dev@openvswitch.org
http://openvswitch.org/mailman/listinfo/dev
diff mbox

Patch

diff --git a/datapath-windows/ovsext/Actions.c b/datapath-windows/ovsext/Actions.c
index a91454d..513a1a4 100644
--- a/datapath-windows/ovsext/Actions.c
+++ b/datapath-windows/ovsext/Actions.c
@@ -40,6 +40,8 @@ 
 
 #define OVS_DEST_PORTS_ARRAY_MIN_SIZE 2
 
+extern PNDIS_RW_LOCK_EX ovsDeferredActionLevelLock;
+
 typedef struct _OVS_ACTION_STATS {
     UINT64 rxGre;
     UINT64 txGre;
@@ -1973,14 +1975,19 @@  OvsDoRecirc(POVS_SWITCH_CONTEXT switchContext,
 
     flow = OvsLookupFlow(&ovsFwdCtx.switchContext->datapath, key, &hash, FALSE);
     if (flow) {
-        UINT32 level = OvsDeferredActionsLevelGet();
+        UINT32 level = 0;
+        LOCK_STATE_EX lockState;
+
+        NdisAcquireRWLockRead(ovsDeferredActionLevelLock, &lockState, 
+ 0);
 
+        level = OvsDeferredActionsLevelGet();
         if (level > DEFERRED_ACTION_EXEC_LEVEL) {
             OvsCompleteNBLForwardingCtx(&ovsFwdCtx,
                 L"OVS-Dropped due to deferred actions execution level limit \
                   reached");
             ovsActionStats.deferredActionsExecLimit++;
             ovsFwdCtx.curNbl = NULL;
+            NdisReleaseRWLock(ovsDeferredActionLevelLock, &lockState);
             return NDIS_STATUS_FAILURE;
         }
 
@@ -1999,6 +2006,8 @@  OvsDoRecirc(POVS_SWITCH_CONTEXT switchContext,
         ovsFwdCtx.curNbl = NULL;
 
         OvsDeferredActionsLevelDec();
+
+        NdisReleaseRWLock(ovsDeferredActionLevelLock, &lockState);
     } else {
         POVS_VPORT_ENTRY vport = NULL;
         LIST_ENTRY missedPackets;
diff --git a/datapath-windows/ovsext/Datapath.c b/datapath-windows/ovsext/Datapath.c
index 464fa97..5725114 100644
--- a/datapath-windows/ovsext/Datapath.c
+++ b/datapath-windows/ovsext/Datapath.c
@@ -385,8 +385,6 @@  OvsInit()
     gOvsCtrlLock = &ovsCtrlLockObj;
     NdisAllocateSpinLock(gOvsCtrlLock);
     OvsInitEventQueue();
-    OvsDeferredActionsQueueAlloc();
-    OvsDeferredActionsLevelAlloc();
 }
 
 VOID
@@ -397,8 +395,6 @@  OvsCleanup()
         NdisFreeSpinLock(gOvsCtrlLock);
         gOvsCtrlLock = NULL;
     }
-    OvsDeferredActionsQueueFree();
-    OvsDeferredActionsLevelFree();
 }
 
 VOID
diff --git a/datapath-windows/ovsext/Driver.c b/datapath-windows/ovsext/Driver.c
index 853886e..f5d3f9c 100644
--- a/datapath-windows/ovsext/Driver.c
+++ b/datapath-windows/ovsext/Driver.c
@@ -152,6 +152,12 @@  DriverEntry(PDRIVER_OBJECT driverObject,
         goto cleanup;
     }
 
+    /* Allocate per-cpu structures and register processor change callback. */
+    status = OvsPerCpuDataInit(gOvsExtDriverHandle);
+    if (!NT_SUCCESS(status)) {
+        goto cleanup;
+    }
+
 cleanup:
     if (status != NDIS_STATUS_SUCCESS){
         OvsCleanup();
@@ -180,6 +186,9 @@  OvsExtUnload(struct _DRIVER_OBJECT *driverObject)
 
     OvsDeleteDeviceObject();
 
+    /* Release per-cpu structures and deregister processor change callback. */
+    OvsPerCpuDataCleanup();
+
     NdisFDeregisterFilterDriver(gOvsExtDriverHandle);
 }
 
diff --git a/datapath-windows/ovsext/Recirc.c b/datapath-windows/ovsext/Recirc.c
index 86e6f51..267e051 100644
--- a/datapath-windows/ovsext/Recirc.c
+++ b/datapath-windows/ovsext/Recirc.c
@@ -18,71 +18,214 @@ 
 #include "Flow.h"
 #include "Jhash.h"
 
-static POVS_DEFERRED_ACTION_QUEUE ovsDeferredActionQueue = NULL; -static UINT32* ovsDeferredActionLevel = NULL;
+/*
+ * 
+-----------------------------------------------------------------------
+---
+ * '_OVS_DEFERRED_ACTION_QUEUE' structure is responsible for keeping 
+track of
+ * all deferred actions. The maximum number of deferred actions should 
+not
+ * exceed 'DEFERRED_ACTION_QUEUE_SIZE'.
+ * 
+-----------------------------------------------------------------------
+---
+ */
+typedef struct _OVS_DEFERRED_ACTION_QUEUE {
+    UINT32  head;
+    UINT32  tail;
+    OVS_DEFERRED_ACTION queue[DEFERRED_ACTION_QUEUE_SIZE];
+} OVS_DEFERRED_ACTION_QUEUE, *POVS_DEFERRED_ACTION_QUEUE;
+
+static POVS_DEFERRED_ACTION_QUEUE   ovsDeferredActionQueue = NULL;
+static UINT32*                      ovsDeferredActionLevel = NULL;
+static PNDIS_RW_LOCK_EX             ovsDeferredActionQueueLock = NULL;
+PNDIS_RW_LOCK_EX                    ovsDeferredActionLevelLock = NULL;
+static ULONG                        ovsActiveProcessorCount = 0;
 
 /*
  * --------------------------------------------------------------------------
- * OvsDeferredActionsQueueAlloc --
- *     The function allocates per-cpu deferred actions queue.
+ * OvsDeferredActionsLevelRealloc --
+ *     The function allocates per-processor deferred actions level.
  * --------------------------------------------------------------------------
  */
-BOOLEAN
-OvsDeferredActionsQueueAlloc()
+NTSTATUS
+OvsDeferredActionsLevelRealloc(PULONG count)
 {
-    ovsDeferredActionQueue =
-        OvsAllocateMemoryPerCpu(sizeof(*ovsDeferredActionQueue),
-                                OVS_RECIRC_POOL_TAG);
-    if (!ovsDeferredActionQueue) {
-        return FALSE;
+    NTSTATUS status = STATUS_SUCCESS;
+    PUINT32 level = NULL;
+    LOCK_STATE_EX lockState;
+
+    NdisAcquireRWLockWrite(ovsDeferredActionLevelLock, &lockState, 0);
+
+    level = OvsReallocateMemoryPerCpu(ovsDeferredActionLevel,
+                                      sizeof(*ovsDeferredActionLevel),
+                                      OVS_RECIRC_POOL_TAG,
+                                      ovsActiveProcessorCount,
+                                      count);
+    if (!level) {
+        NdisReleaseRWLock(ovsDeferredActionLevelLock, &lockState);
+        status = NDIS_STATUS_RESOURCES;
+        goto exit;
     }
-    return TRUE;
+
+    ovsDeferredActionLevel = level;
+
+    NdisReleaseRWLock(ovsDeferredActionLevelLock, &lockState);
+
+exit:
+    return status;
 }
 
 /*
  * --------------------------------------------------------------------------
- * OvsDeferredActionsQueueFree --
- *     The function frees per-cpu deferred actions queue.
+ * OvsDeferredActionsQueueRealloc --
+ *     The function allocates per-processor deferred actions queue.
  * --------------------------------------------------------------------------
  */
-VOID
-OvsDeferredActionsQueueFree()
+NTSTATUS
+OvsDeferredActionsQueueRealloc(PULONG count)
 {
-    OvsFreeMemoryWithTag(ovsDeferredActionQueue,
-                         OVS_RECIRC_POOL_TAG);
-    ovsDeferredActionQueue = NULL;
+    NTSTATUS status = STATUS_SUCCESS;
+    POVS_DEFERRED_ACTION_QUEUE queue = NULL;
+    LOCK_STATE_EX lockState;
+
+    NdisAcquireRWLockWrite(ovsDeferredActionQueueLock, &lockState, 0);
+
+    queue = OvsReallocateMemoryPerCpu(ovsDeferredActionQueue,
+                                      sizeof(*ovsDeferredActionQueue),
+                                      OVS_RECIRC_POOL_TAG,
+                                      ovsActiveProcessorCount,
+                                      count);
+    if (!queue) {
+        NdisReleaseRWLock(ovsDeferredActionQueueLock, &lockState);
+        status = NDIS_STATUS_RESOURCES;
+        goto exit;
+    }
+
+    ovsDeferredActionQueue = queue;
+
+    NdisReleaseRWLock(ovsDeferredActionQueueLock, &lockState);
+
+exit:
+    return status;
 }
 
 /*
  * --------------------------------------------------------------------------
- * OvsDeferredActionsLevelAlloc --
- *     The function allocates per-cpu deferred actions execution level.
+ * OvsDeferredActionsRealloc --
+ *     The function reallocates per-cpu necessary resources and is triggered
+ *     by the 'OvsCpuChange' callback.
+ *
+ *     If the function was triggered by a 'KeProcessorAddStartNotify'
+ *       notification, it means that the operating system is about to add a
+ *       new processor and the function reallocates new space to accomodate
+ *       new per-processor data requirements.
+ *
+ *     If the function was triggered by a 'KeProcessorAddFailureNotify'
+ *       notification, it means that the operating system failed to add the
+ *       new processor and the function frees all per-processor resources
+ *       that were allocated for the new processor.
  * --------------------------------------------------------------------------
  */
-BOOLEAN
-OvsDeferredActionsLevelAlloc()
+NTSTATUS
+OvsDeferredActionsRealloc()
+{
+    NTSTATUS status = STATUS_SUCCESS;
+    ULONG count = 0;
+
+    status = OvsDeferredActionsQueueRealloc(&count);
+    if (!NT_SUCCESS(status)) {
+        goto exit;
+    }
+
+    status = OvsDeferredActionsLevelRealloc(NULL);
+    if (!NT_SUCCESS(status)) {
+        goto exit;
+    }
+
+    ovsActiveProcessorCount = count;
+
+exit:
+    return status;
+}
+
+/*
+ * 
+-----------------------------------------------------------------------
+---
+ * OvsDeferredActionsInit --
+ *     The function allocates all necessary per-processor deferred actions
+ *     resources.
+ * 
+-----------------------------------------------------------------------
+---
+ */
+NTSTATUS
+OvsDeferredActionsInit(NDIS_HANDLE NdisFilterHandle)
 {
+    NTSTATUS status = STATUS_SUCCESS;
+
+    ovsDeferredActionQueueLock = NdisAllocateRWLock(NdisFilterHandle);
+    if (!ovsDeferredActionQueueLock) {
+        status = NDIS_STATUS_RESOURCES;
+        goto cleanup;
+    }
+
+    ovsDeferredActionLevelLock = NdisAllocateRWLock(NdisFilterHandle);
+    if (!ovsDeferredActionLevelLock) {
+        status = NDIS_STATUS_RESOURCES;
+        goto cleanup;
+    }
+
+    ovsDeferredActionQueue =
+        OvsAllocateMemoryPerCpu(sizeof(*ovsDeferredActionQueue),
+                                OVS_RECIRC_POOL_TAG,
+                                &ovsActiveProcessorCount);
+    if (!ovsDeferredActionQueue) {
+        status = NDIS_STATUS_RESOURCES;
+        goto cleanup;
+    }
+
     ovsDeferredActionLevel =
         OvsAllocateMemoryPerCpu(sizeof(*ovsDeferredActionLevel),
-                                OVS_RECIRC_POOL_TAG);
+                                OVS_RECIRC_POOL_TAG,
+                                NULL);
     if (!ovsDeferredActionLevel) {
-        return FALSE;
+        status = NDIS_STATUS_RESOURCES;
+        goto cleanup;
     }
-    return TRUE;
+
+cleanup:
+    if (!NT_SUCCESS(status)) {
+        OvsDeferredActionsCleanup();
+    }
+
+    return status;
 }
 
 /*
  * --------------------------------------------------------------------------
- * OvsDeferredActionsLevelFree --
- *     The function frees per-cpu deferred actions execution level.
+ * OvsDeferredActionsCleanup --
+ *     The function frees all per-processor deferred actions resources.
  * --------------------------------------------------------------------------
  */
 VOID
-OvsDeferredActionsLevelFree()
+OvsDeferredActionsCleanup()
 {
-    OvsFreeMemoryWithTag(ovsDeferredActionLevel,
-                         OVS_RECIRC_POOL_TAG);
-    ovsDeferredActionLevel = NULL;
+    if (ovsDeferredActionLevel) {
+        OvsFreeMemoryWithTag(ovsDeferredActionLevel,
+                             OVS_RECIRC_POOL_TAG);
+        ovsDeferredActionLevel = NULL;
+    }
+
+    if (ovsDeferredActionQueue) {
+        OvsFreeMemoryWithTag(ovsDeferredActionQueue,
+                             OVS_RECIRC_POOL_TAG);
+        ovsDeferredActionQueue = NULL;
+    }
+
+    if (ovsDeferredActionLevelLock) {
+        NdisFreeRWLock(ovsDeferredActionLevelLock);
+        ovsDeferredActionLevelLock = NULL;
+    }
+
+    if (ovsDeferredActionQueueLock) {
+        NdisFreeRWLock(ovsDeferredActionQueueLock);
+        ovsDeferredActionQueueLock = NULL;
+    }
+
+    ovsActiveProcessorCount = 0;
 }
 
 /*
@@ -104,7 +247,9 @@  OvsDeferredActionsQueueGet()
     }
 
     index = KeGetCurrentProcessorNumberEx(NULL);
-    queue = &ovsDeferredActionQueue[index];
+    if (index < ovsActiveProcessorCount) {
+        queue = &ovsDeferredActionQueue[index];
+    }
 
     if (oldIrql < DISPATCH_LEVEL) {
         KeLowerIrql(oldIrql);
@@ -132,7 +277,9 @@  OvsDeferredActionsLevelGet()
     }
 
     index = KeGetCurrentProcessorNumberEx(NULL);
-    level = &ovsDeferredActionLevel[index];
+    if (index < ovsActiveProcessorCount) {
+        level = &ovsDeferredActionLevel[index];
+    }
 
     if (oldIrql < DISPATCH_LEVEL) {
         KeLowerIrql(oldIrql);
@@ -160,8 +307,10 @@  OvsDeferredActionsLevelInc()
     }
 
     index = KeGetCurrentProcessorNumberEx(NULL);
-    level = &ovsDeferredActionLevel[index];
-    (*level)++;
+    if (index < ovsActiveProcessorCount) {
+        level = &ovsDeferredActionLevel[index];
+        (*level)++;
+    }
 
     if (oldIrql < DISPATCH_LEVEL) {
         KeLowerIrql(oldIrql);
@@ -187,8 +336,10 @@  OvsDeferredActionsLevelDec()
     }
 
     index = KeGetCurrentProcessorNumberEx(NULL);
-    level = &ovsDeferredActionLevel[index];
-    (*level)--;
+    if (index < ovsActiveProcessorCount) {
+        level = &ovsDeferredActionLevel[index];
+        (*level)--;
+    }
 
     if (oldIrql < DISPATCH_LEVEL) {
         KeLowerIrql(oldIrql);
@@ -293,8 +444,13 @@  OvsAddDeferredActions(PNET_BUFFER_LIST nbl,
                       OvsFlowKey *key,
                       const PNL_ATTR actions)  {
-    POVS_DEFERRED_ACTION_QUEUE queue = OvsDeferredActionsQueueGet();
+    POVS_DEFERRED_ACTION_QUEUE queue = NULL;
     POVS_DEFERRED_ACTION deferredAction = NULL;
+    LOCK_STATE_EX lockState;
+
+    NdisAcquireRWLockRead(ovsDeferredActionQueueLock, &lockState, 0);
+
+    queue =  OvsDeferredActionsQueueGet();
 
     deferredAction = OvsDeferredActionsQueuePush(queue);
     if (deferredAction) {
@@ -303,6 +459,8 @@  OvsAddDeferredActions(PNET_BUFFER_LIST nbl,
         deferredAction->key = *key;
     }
 
+    NdisReleaseRWLock(ovsDeferredActionQueueLock, &lockState);
+
     return deferredAction;
 }
 
@@ -321,8 +479,13 @@  OvsProcessDeferredActions(POVS_SWITCH_CONTEXT switchContext,
                           OVS_PACKET_HDR_INFO *layers)  {
     NDIS_STATUS status = NDIS_STATUS_SUCCESS;
-    POVS_DEFERRED_ACTION_QUEUE queue = OvsDeferredActionsQueueGet();
+    POVS_DEFERRED_ACTION_QUEUE queue = NULL;
     POVS_DEFERRED_ACTION deferredAction = NULL;
+    LOCK_STATE_EX lockState;
+
+    NdisAcquireRWLockRead(ovsDeferredActionQueueLock, &lockState, 0);
+
+    queue = OvsDeferredActionsQueueGet();
 
     /* Process all deferred actions. */
     while ((deferredAction = OvsDeferredActionsQueuePop(queue)) != NULL) { @@ -345,5 +508,7 @@ OvsProcessDeferredActions(POVS_SWITCH_CONTEXT switchContext,
         }
     }
 
+    NdisReleaseRWLock(ovsDeferredActionQueueLock, &lockState);
+
     return status;
 }
diff --git a/datapath-windows/ovsext/Recirc.h b/datapath-windows/ovsext/Recirc.h
index ee05763..7d89ad5 100644
--- a/datapath-windows/ovsext/Recirc.h
+++ b/datapath-windows/ovsext/Recirc.h
@@ -30,19 +30,6 @@  typedef struct _OVS_DEFERRED_ACTION {
 
 /*
  * --------------------------------------------------------------------------
- * '_OVS_DEFERRED_ACTION_QUEUE' structure is responsible for keeping track of
- * all deferred actions. The maximum number of deferred actions should not
- * exceed 'DEFERRED_ACTION_QUEUE_SIZE'.
- * --------------------------------------------------------------------------
- */
-typedef struct _OVS_DEFERRED_ACTION_QUEUE {
-    UINT32  head;
-    UINT32  tail;
-    OVS_DEFERRED_ACTION queue[DEFERRED_ACTION_QUEUE_SIZE];
-} OVS_DEFERRED_ACTION_QUEUE, *POVS_DEFERRED_ACTION_QUEUE;
-
-/*
- * --------------------------------------------------------------------------
  * OvsProcessDeferredActions --
  *     This function processes all deferred actions contained in the queue
  *     corresponding to the current CPU.
@@ -69,68 +56,72 @@  OvsAddDeferredActions(PNET_BUFFER_LIST packet,
 
 /*
  * --------------------------------------------------------------------------
- * OvsDeferredActionsQueueAlloc --
- *     The function allocates per-cpu deferred actions queue.
+ * OvsDeferredActionsLevelGet --
+ *     The function returns the deferred action execution level corresponding
+ *     to the current processor.
  * --------------------------------------------------------------------------
  */
-BOOLEAN
-OvsDeferredActionsQueueAlloc();
+UINT32
+OvsDeferredActionsLevelGet();
 
 /*
  * --------------------------------------------------------------------------
- * OvsDeferredActionsQueueFree --
- *     The function frees per-cpu deferred actions queue.
+ * OvsDeferredActionsLevelInc --
+ *     The function increments the deferred action execution level
+ *     corresponding to the current processor.
  * --------------------------------------------------------------------------
  */
 VOID
-OvsDeferredActionsQueueFree();
-
-/*
- * --------------------------------------------------------------------------
- * OvsDeferredActionsLevelAlloc --
- *     The function allocates per-cpu deferred actions execution level.
- * --------------------------------------------------------------------------
- */
-BOOLEAN
-OvsDeferredActionsLevelAlloc();
+OvsDeferredActionsLevelInc();
 
 /*
  * --------------------------------------------------------------------------
- * OvsDeferredActionsLevelFree --
- *     The function frees per-cpu deferred actions execution level.
+ * OvsDeferredActionsLevelDec --
+ *     The function decrements the deferred action execution level
+ *     corresponding to the current processor.
  * --------------------------------------------------------------------------
- */
+*/
 VOID
-OvsDeferredActionsLevelFree();
+OvsDeferredActionsLevelDec();
 
 /*
  * --------------------------------------------------------------------------
- * OvsDeferredActionsLevelGet --
- *     The function returns the deferred action execution level corresponding
- *     to the current processor.
+ * OvsDeferredActionsRealloc --
+ *     The function reallocates per-cpu necessary resources and is triggered
+ *     by the 'OvsCpuChange' callback.
+ *
+ *     If the function was triggered due to a 'KeProcessorAddStartNotify'
+ *       notification, it means that the operating system is about to add a
+ *       new processor and the function reallocates new space to accomodate
+ *       new per-processor data requirements.
+ *
+ *     If the function was triggered due to a 'KeProcessorAddFailureNotify'
+ *       notification, it means that the operating system failed to add the
+ *       new processor and the function frees all per-processor resources
+ *       that were allocated for the new processor.
  * --------------------------------------------------------------------------
  */
-UINT32
-OvsDeferredActionsLevelGet();
+NTSTATUS
+OvsDeferredActionsRealloc();
 
 /*
  * --------------------------------------------------------------------------
- * OvsDeferredActionsLevelInc --
- *     The function increments the deferred action execution level
- *     corresponding to the current processor.
+ * OvsDeferredActionsInit --
+ *     The function allocates necessary per-processor resources and registers
+ *     processor change callback.
  * --------------------------------------------------------------------------
  */
-VOID
-OvsDeferredActionsLevelInc();
+NTSTATUS
+OvsDeferredActionsInit(NDIS_HANDLE NdisFilterHandle);
 
 /*
  * --------------------------------------------------------------------------
- * OvsDeferredActionsLevelDec --
- *     The function decrements the deferred action execution level
- *     corresponding to the current processor.
+ * OvsDeferredActionsCleanup --
+ *     The function frees all per-processor resources and deregisters
+ *     processor change callback.
  * --------------------------------------------------------------------------
-*/
+ */
 VOID
-OvsDeferredActionsLevelDec();
+OvsDeferredActionsCleanup();
 
 #endif /* __RECIRC_H_ */
diff --git a/datapath-windows/ovsext/Util.c b/datapath-windows/ovsext/Util.c index 14c4493..28229e9 100644
--- a/datapath-windows/ovsext/Util.c
+++ b/datapath-windows/ovsext/Util.c
@@ -15,6 +15,7 @@ 
  */
 
 #include "precomp.h"
+#include "Recirc.h"
 #ifdef OVS_DBG_MOD
 #undef OVS_DBG_MOD
 #endif
@@ -23,6 +24,7 @@ 
 #include "Debug.h"
 
 extern NDIS_HANDLE gOvsExtDriverHandle;
+static VOID* ovsRegistrationHandle = NULL;
 
 VOID*
 OvsAllocateMemoryWithTag(size_t size, ULONG tag) @@ -118,7 +120,7 @@ OvsCompareString(PVOID string1, PVOID string2)  }
 
 VOID *
-OvsAllocateMemoryPerCpu(size_t size, ULONG tag)
+OvsAllocateMemoryPerCpu(size_t size, ULONG tag, PULONG cpuCount)
 {
     VOID *ptr = NULL;
     ULONG count = KeQueryActiveProcessorCountEx(ALL_PROCESSOR_GROUPS);
@@ -128,7 +130,125 @@  OvsAllocateMemoryPerCpu(size_t size, ULONG tag)
     ptr = OvsAllocateMemoryWithTag(count * size, tag);
     if (ptr) {
         RtlZeroMemory(ptr, count * size);
+        if (cpuCount) {
+            *cpuCount = count;
+        }
     }
 
     return ptr;
 }
+
+VOID *
+OvsReallocateMemoryPerCpu(VOID *buffer,
+                          size_t bufferSize,
+                          ULONG tag,
+                          ULONG oldCpuCount,
+                          ULONG *newCpuCount) {
+    VOID *ptr = NULL;
+    ULONG cpuCount = 0;
+
+    ptr = OvsAllocateMemoryPerCpu(bufferSize, tag, &cpuCount);
+    if (ptr) {
+        ULONG count = 0;
+        count = (cpuCount > oldCpuCount) ? oldCpuCount : cpuCount;
+        RtlCopyMemory(ptr, buffer, count * bufferSize);
+        if (newCpuCount) {
+            *newCpuCount = cpuCount;
+        }
+
+        if (buffer) {
+            OvsFreeMemoryWithTag(buffer, tag);
+        }
+    }
+
+    return ptr;
+}
+
+/*
+ * 
+-----------------------------------------------------------------------
+---
+ * OvsCpuChange --
+ *     This is the processor change callback function that is to be called by
+ *     the operating system whenever a new processor is added to the hardware
+ *     partition.
+ *
+ *     'KeProcessorAddStartNotify' notification is received when the OS is
+ *     about to add the processor; at this state, any per-processor data
+ *     structures must be allocated for the new processor to prepare the
+ *     driver for execution on the new processor.
+ *
+ *     'KeProcessorAddFailureNotify' notification is received when the OS
+ *     failed to add the processor and any per-processor data structures that
+ *     were allocated for the new processor should be freed.
+ * 
+-----------------------------------------------------------------------
+---
+ */
+VOID
+OvsCpuChange(__in PVOID CallbackContext,
+             __in PKE_PROCESSOR_CHANGE_NOTIFY_CONTEXT ChangeContext,
+             __inout PNTSTATUS OperationStatus) {
+    UNREFERENCED_PARAMETER(CallbackContext);
+
+    switch (ChangeContext->State) {
+    case KeProcessorAddFailureNotify:
+    case KeProcessorAddStartNotify:
+    {
+        NTSTATUS status = STATUS_SUCCESS;
+
+        status = OvsDeferredActionsRealloc();
+        if (!NT_SUCCESS(status) &&
+            ChangeContext->State == KeProcessorAddStartNotify) {
+            *OperationStatus = status;
+        }
+        break;
+    }
+    default:
+        break;
+    }
+}
+
+/*
+ * 
+-----------------------------------------------------------------------
+---
+ * OvsPerCpuDataInit --
+ *     The function allocates necessary per-processor resources and registers
+ *     processor change callback.
+ * 
+-----------------------------------------------------------------------
+---
+ */
+NTSTATUS
+OvsPerCpuDataInit(NDIS_HANDLE ndisFilterHandle) {
+    NTSTATUS status = STATUS_SUCCESS;
+
+    status = OvsDeferredActionsInit(ndisFilterHandle);
+    if (!NT_SUCCESS(status)) {
+        goto exit;
+    }
+
+    ovsRegistrationHandle =
+        KeRegisterProcessorChangeCallback(OvsCpuChange, NULL, 0);
+    if (!ovsRegistrationHandle) {
+        status = NDIS_STATUS_FAILURE;
+        goto exit;
+    }
+
+exit:
+    return status;
+}
+
+/*
+ * 
+-----------------------------------------------------------------------
+---
+ * OvsPerCpuDataCleanup --
+ *     The function frees all per-processor resources and deregisters
+ *     processor change callback.
+ * 
+-----------------------------------------------------------------------
+---
+ */
+VOID
+OvsPerCpuDataCleanup()
+{
+    if (ovsRegistrationHandle) {
+        KeDeregisterProcessorChangeCallback(ovsRegistrationHandle);
+        ovsRegistrationHandle = NULL;
+    }
+
+    OvsDeferredActionsCleanup();
+}
diff --git a/datapath-windows/ovsext/Util.h b/datapath-windows/ovsext/Util.h index 038754d..5747278 100644
--- a/datapath-windows/ovsext/Util.h
+++ b/datapath-windows/ovsext/Util.h
@@ -41,7 +41,12 @@ 
 VOID *OvsAllocateMemory(size_t size);
 VOID *OvsAllocateMemoryWithTag(size_t size, ULONG tag);  VOID *OvsAllocateAlignedMemory(size_t size, UINT16 align); -VOID *OvsAllocateMemoryPerCpu(size_t size, ULONG tag);
+VOID *OvsAllocateMemoryPerCpu(size_t size, ULONG tag, PULONG cpuCount); 
+VOID *OvsReallocateMemoryPerCpu(VOID *ptr,
+                                size_t size,
+                                ULONG tag,
+                                ULONG oldCpuCount,
+                                ULONG *newCpuCount);
 VOID OvsFreeMemory(VOID *ptr);
 VOID OvsFreeMemoryWithTag(VOID *ptr, ULONG tag);  VOID OvsFreeAlignedMemory(VOID *ptr); @@ -94,4 +99,24 @@ VOID OvsAppendList(PLIST_ENTRY dst, PLIST_ENTRY src);
 
 BOOLEAN OvsCompareString(PVOID string1, PVOID string2);