internal static void TestPrepareUpdate(ActorStateTable stateTable, ReplicationUnit replicationUnit) { foreach (var actorStateDataWrapper in replicationUnit.ActorStateDataWrapperList) { TestLog( "PrepareUpdate(SequenceNumber: {0}, Type: {1} Key: {2}, IsDelete: {3})", replicationUnit.SequenceNumber, actorStateDataWrapper.Type.ToString(), actorStateDataWrapper.Key, actorStateDataWrapper.IsDelete); } stateTable.PrepareUpdate(replicationUnit.ActorStateDataWrapperList, replicationUnit.SequenceNumber); }
internal static void TryReadAndVerify( ActorStateTable stateTable, ReplicationUnit replicationUnit, bool expectedResult, TimeSpan expectedTimestamp) { foreach (var actorStateDataWrapper in replicationUnit.ActorStateDataWrapperList) { TryReadAndVerify( stateTable, actorStateDataWrapper.Key, expectedResult, expectedTimestamp); } }
internal static void TryReadAndVerify( ActorStateTable stateTable, ReplicationUnit replicationUnit, bool expectedResult, string expectedReminderName) { foreach (var actorStateDataWrapper in replicationUnit.ActorStateDataWrapperList) { TryReadAndVerify( stateTable, actorStateDataWrapper.Key, expectedResult, expectedReminderName); } }
internal static void TestApply(ActorStateTable stateTable, ReplicationUnit replicationUnit, bool testLog = true) { if (testLog) { foreach (var actorStateDataWrapper in replicationUnit.ActorStateDataWrapperList) { TestLog( "Apply(SequenceNumber: {0}, Key: {1}, IsDelete: {2})", replicationUnit.SequenceNumber, actorStateDataWrapper.Key, actorStateDataWrapper.IsDelete); } } replicationUnit.UpdateSequenceNumber(); stateTable.ApplyUpdates(replicationUnit.ActorStateDataWrapperList); }
internal static void VerifyStateTableSnapshot( ActorStateTable stateTable, Dictionary <ActorStateType, int> statesPerReplication, long maxSequenceNumber, long expectedCommittedSequenceNumber, long expectedKnownSequenceNumber, long expectedCount) { var enumerator = stateTable.GetShallowCopiesEnumerator(maxSequenceNumber); VerifyStateTableSnapshot( stateTable, statesPerReplication, enumerator, expectedCommittedSequenceNumber, expectedKnownSequenceNumber, expectedCount); }
private static bool TryGetActorStateData( ActorStateTable stateTable, ActorStateType type, string key, bool expectedResult, out ActorStateData data) { data = null; bool actualResult = stateTable.TryGetValue(type, key, out data); FailTestIf(expectedResult != actualResult, "TryGetValue({0}, {1}): expected={2} actual={3}", type, key, expectedResult, actualResult); return(actualResult); }
internal static void TryReadAndVerify( ActorStateTable stateTable, string key, bool expectedResult, string expectedReminderName) { ActorStateData data; var actualResult = TryGetActorStateData(stateTable, ActorStateType.Reminder, key, expectedResult, out data); if (actualResult) { var resultReminder = data.ActorReminderData; FailTestIf(expectedReminderName != resultReminder.Name, "Data({0}): expected={1} actual={2}", key, expectedReminderName, resultReminder.Name); } }
internal static void TryReadAndVerify( ActorStateTable stateTable, string key, bool expectedResult, TimeSpan expectedTimestamp) { ActorStateData data; var actualResult = TryGetActorStateData(stateTable, ActorStateType.LogicalTimestamp, key, expectedResult, out data); if (actualResult) { var resultTimestamp = data.LogicalTimestamp.Value; FailTestIf(expectedTimestamp != resultTimestamp, "Data({0}): expected={1} actual={2}", key, expectedTimestamp, resultTimestamp); } }
internal static void TryReadAndVerify( ActorStateTable stateTable, string key, bool expectedResult, long expectedLength) { ActorStateData data; var actualResult = TryGetActorStateData(stateTable, ActorStateType.Actor, key, expectedResult, out data); if (actualResult) { var resultBuffer = data.ActorState; FailTestIf(expectedLength != resultBuffer.Length, "DataLength({0}): expected={1} actual={2}", key, expectedLength, resultBuffer.Length); } }
internal static void VerifyReads( ActorStateTable stateTable, ReplicationUnit replicationUnit, Dictionary <ActorStateType, int> statesPerReplication, bool expectedResult, long expectedLength, long expectedCommittedSequenceNumber, long expectedKnownSequenceNumber, long expectedCount) { VerifyStateTableSnapshot( stateTable, statesPerReplication, long.MaxValue, expectedCommittedSequenceNumber, expectedKnownSequenceNumber, expectedCount); TryReadAndVerify(stateTable, replicationUnit, expectedResult, expectedLength); }
/// <summary> /// The replicationUnitList should enumerate the replicationUnit in increasing order of sequence number. /// </summary> internal static List <List <ActorStateDataWrapper> > TestApplyBatch( ActorStateTable stateTable, IEnumerable <ReplicationUnit> replicationUnitList) { var replicationData = new List <List <ActorStateDataWrapper> >(); var batch = new List <ActorStateDataWrapper>(); foreach (var replicationUnit in replicationUnitList) { replicationUnit.UpdateSequenceNumber(); replicationData.Add(replicationUnit.ActorStateDataWrapperList); batch.AddRange(replicationUnit.ActorStateDataWrapperList); TestLog("ApplyBatch({0}, {1}, {2})", replicationUnit.SequenceNumber, replicationUnit.StateType, replicationUnit.GetKeyString()); } stateTable.ApplyUpdates(batch); return(replicationData); }
internal static void TestCommitUpdate(ActorStateTable stateTable, long sequenceNumber) { TestLog("Commiting Sequence Number ({0})...", sequenceNumber); stateTable.CommitUpdateAsync(sequenceNumber).Wait(); TestLog("Commited Sequence Number ({0})", sequenceNumber); }
/// <summary> /// This function verifies snapshot when the replication units are uniform /// i.e. they either contain update entries or delete entries. /// </summary internal static void VerifyStateTableSnapshot( ActorStateTable stateTable, Dictionary <ActorStateType, int> statesPerReplication, ActorStateTable.ActorStateEnumerator enumerator, long expectedCommittedSequenceNumber, long expectedKnownSequenceNumber, long expectedCount) { var actualCommittedSequenceNumber = stateTable.GetHighestCommittedSequenceNumber(); var actualKnownSequenceNumber = stateTable.GetHighestKnownSequenceNumber(); TestLog( "ActorStateTable committed:{0} known:{1}", stateTable.GetHighestCommittedSequenceNumber(), stateTable.GetHighestKnownSequenceNumber()); FailTestIf( expectedCommittedSequenceNumber != actualCommittedSequenceNumber, "Committed sequence number: expected:{0} actual:{1}", expectedCommittedSequenceNumber, actualCommittedSequenceNumber); FailTestIf( expectedKnownSequenceNumber != actualKnownSequenceNumber, "Known sequence number: expected:{0} actual:{1}", expectedKnownSequenceNumber, actualKnownSequenceNumber); // Group the actor state by replication unit var replicationUnits = GroupByRelicationUnit(enumerator); long actualCount = 0; long lastSequenceNumber = 0; foreach (var replicationUnit in replicationUnits) { TestLog( "[{0}] '{1}' {2}", replicationUnit.SequenceNumber, replicationUnit.StateType, replicationUnit.GetKeyString()); FailTestIf( replicationUnit.SequenceNumber <= lastSequenceNumber, "Sequence number: previous:{0} current:{1}", lastSequenceNumber, replicationUnit.SequenceNumber); if ((replicationUnit.SequenceNumber == actualCommittedSequenceNumber) && replicationUnit.IsLastActorStateDelete) { FailTestIf( replicationUnit.ActorStateCount != 1, "States in replication unit at end of committed entry list: IsDelete:{0} ActualStateCount:{1} ExpectedStateCount:1", replicationUnit.IsLastActorStateDelete, replicationUnit.ActorStateCount); } else if (replicationUnit.SequenceNumber <= actualCommittedSequenceNumber) { // The replication unit should only contain update entries FailTestIf( replicationUnit.ActorStateCount != statesPerReplication[replicationUnit.StateType], "States in replication unit: ActualCount:{0} ExpectedCount:{1}", replicationUnit.ActorStateCount, statesPerReplication[replicationUnit.StateType]); FailTestIf( replicationUnit.ActorStateUpdateCount != statesPerReplication[replicationUnit.StateType], "Update states in replication unit: ActualUpdateCount:{0} ExpectedUpdateCount:{1}", replicationUnit.ActorStateUpdateCount, statesPerReplication[replicationUnit.StateType]); } else { // The replication unit should only contain either update entries or delete entries if (replicationUnit.IsLastActorStateDelete) { FailTestIf( replicationUnit.ActorStateUpdateCount != 0, "States in replication unit: IsDelete:{0} ActualUpdateStateCount:{1} ExpectedUpdateStateCount:0", replicationUnit.IsLastActorStateDelete, replicationUnit.ActorStateUpdateCount); FailTestIf( replicationUnit.ActorStateCount != statesPerReplication[replicationUnit.StateType], "States in replication unit: IsDelete:{0} ActualStateCount:{1} ExpectedStateCount:{2}", replicationUnit.IsLastActorStateDelete, replicationUnit.ActorStateCount, statesPerReplication[replicationUnit.StateType]); } else { FailTestIf( replicationUnit.ActorStateUpdateCount != statesPerReplication[replicationUnit.StateType], "States in replication unit: IsDelete:{0} ActualUpdateStateCount:{1} ExpectedUpdateStateCount:{2}", replicationUnit.IsLastActorStateDelete, replicationUnit.ActorStateUpdateCount, statesPerReplication[replicationUnit.StateType]); FailTestIf( replicationUnit.ActorStateCount != statesPerReplication[replicationUnit.StateType], "States in replication unit: IsDelete:{0} ActualStateCount:{1} ExpectedStateCount:{2}", replicationUnit.IsLastActorStateDelete, replicationUnit.ActorStateCount, statesPerReplication[replicationUnit.StateType]); } } foreach (var actorStateDataWrapper in replicationUnit.ActorStateDataWrapperList) { string dataString = GetDataString(actorStateDataWrapper); TestLog( "[{0}] '{1}' {2} {3}", actorStateDataWrapper.SequenceNumber, actorStateDataWrapper.Type, actorStateDataWrapper.Key, dataString); FailTestIf( actorStateDataWrapper.SequenceNumber != replicationUnit.SequenceNumber, "Sequence number same as replication unit: RU ={0} ActorState={1}", replicationUnit.SequenceNumber, actorStateDataWrapper.SequenceNumber); FailTestIf( actorStateDataWrapper.SequenceNumber != replicationUnit.SequenceNumber, "State type same as replication unit: RU ={0} ActorState={1}", replicationUnit.StateType, actorStateDataWrapper.Type); FailTestIf(string.IsNullOrEmpty(dataString), "Data not null or empty"); } lastSequenceNumber = replicationUnit.SequenceNumber; actualCount += replicationUnit.ActorStateDataWrapperList.Count; } FailTestIf( expectedCount != actualCount, "Total: expected:{0} actual:{1}\n", expectedCount, actualCount); }
private void TestReplication(Dictionary <ActorStateType, int> statesPerReplication) { TestCase("### TestReplication ###"); TestCase("### StatesPerReplication (Actor:{0}, TimeStamp:{1}, Reminder:{2}) ###", statesPerReplication[ActorStateType.Actor], statesPerReplication[ActorStateType.LogicalTimestamp], statesPerReplication[ActorStateType.Reminder]); var maxReplicationMessageSize = 2048; var maxCopyMessageSize = maxReplicationMessageSize / 2; var copyOrReplicationSerializer = VolatileActorStateProvider.CreateCopyOrReplicationOperationSerializer(); var dataLength = (maxCopyMessageSize / (3 * statesPerReplication[ActorStateType.Actor])) + 1; var primaryStateTable = new ActorStateTable(); var copyKeyPrefixs = new string[] { "a", "b", "c", "d", "e" }; var primarySequenceNumber = 0; var replicationUnitBatch = new List <ReplicationUnit>(); var replicationUnitDict = new Dictionary <string, ReplicationUnit>(); foreach (var keyPrefix in copyKeyPrefixs) { var replicationUnit = ReplicationUnit.CreateForUpdateActor( ++primarySequenceNumber, keyPrefix, statesPerReplication[ActorStateType.Actor], dataLength); replicationUnitBatch.Add(replicationUnit); replicationUnitDict[keyPrefix] = replicationUnit; } TestApplyBatch(primaryStateTable, replicationUnitBatch); foreach (var keyPrefix in copyKeyPrefixs) { VerifyReads( primaryStateTable, replicationUnitDict[keyPrefix], statesPerReplication, true, dataLength, primarySequenceNumber, primarySequenceNumber, primarySequenceNumber * statesPerReplication[ActorStateType.Actor]); } var copyStateEnumerator = new CopyStateEnumerator( primaryStateTable.GetShallowCopiesEnumerator(primarySequenceNumber), copyOrReplicationSerializer, primarySequenceNumber, maxCopyMessageSize); var replicationKeyPrefixes = new string[] { "w", "x", "y", "z" }; replicationUnitBatch = new List <ReplicationUnit>(); foreach (var keyPrefix in replicationKeyPrefixes) { var replicationUnit = ReplicationUnit.CreateForUpdateActor( ++primarySequenceNumber, keyPrefix, statesPerReplication[ActorStateType.Actor], dataLength); replicationUnitBatch.Add(replicationUnit); replicationUnitDict[keyPrefix] = replicationUnit; } var replicationData = TestApplyBatch(primaryStateTable, replicationUnitBatch); foreach (var keyPrefix in replicationKeyPrefixes) { VerifyReads( primaryStateTable, replicationUnitDict[keyPrefix], statesPerReplication, true, dataLength, primarySequenceNumber, primarySequenceNumber, primarySequenceNumber * statesPerReplication[ActorStateType.Actor]); } var secondaryStateTable = new ActorStateTable(); var stateReplicator = new MockStateReplicator( copyOrReplicationSerializer, copyStateEnumerator, replicationData, Task.FromResult(true)); var partition = new Mock <IStatefulServicePartition>(); var secondaryPump = new SecondaryPump( partition.Object, secondaryStateTable, stateReplicator, copyOrReplicationSerializer, new VolatileLogicalTimeManager(new MockSnapshotHandler(), TimeSpan.MaxValue), "SecondaryPumpUnitTest"); TestCase("# TestReplication: Testcase 1: Pump replication operations"); secondaryPump.StartReplicationPump(); secondaryPump.WaitForPumpCompletionAsync().Wait(); foreach (var keyPrefix in replicationKeyPrefixes) { VerifyReads( secondaryStateTable, replicationUnitDict[keyPrefix], statesPerReplication, true, dataLength, primarySequenceNumber, primarySequenceNumber, replicationKeyPrefixes.Length * statesPerReplication[ActorStateType.Actor]); } TestCase("# Passed"); }
private void TestCopyAndReplicationDelete(Dictionary <ActorStateType, int> statesPerReplication) { TestCase("### TestCopyAndReplicationDelete ###"); TestCase("### StatesPerReplication (Actor:{0}, TimeStamp:{1}, Reminder:{2}) ###", statesPerReplication[ActorStateType.Actor], statesPerReplication[ActorStateType.LogicalTimestamp], statesPerReplication[ActorStateType.Reminder]); var maxReplicationMessageSize = 2048; var maxCopyMessageSize = maxReplicationMessageSize / 2; var copyOrReplicationSerializer = VolatileActorStateProvider.CreateCopyOrReplicationOperationSerializer(); var dataLength = (maxCopyMessageSize / (3 * statesPerReplication[ActorStateType.Actor])) + 1; var primaryStateTable = new ActorStateTable(); var copyKeyPrefixes = new string[] { "a", "b", "c", "d", "e" }; var primarySequenceNumber = 0; var replicationUnitBatch = new List <ReplicationUnit>(); var replicationUnitDict = new Dictionary <string, ReplicationUnit>(); foreach (var keyPrefix in copyKeyPrefixes) { var replicationUnit = ReplicationUnit.CreateForUpdateActor( ++primarySequenceNumber, keyPrefix, statesPerReplication[ActorStateType.Actor], dataLength); replicationUnitBatch.Add(replicationUnit); replicationUnitDict[keyPrefix] = replicationUnit; } TestApplyBatch(primaryStateTable, replicationUnitBatch); foreach (var keyPrefix in copyKeyPrefixes) { VerifyReads( primaryStateTable, replicationUnitDict[keyPrefix], statesPerReplication, true, dataLength, primarySequenceNumber, primarySequenceNumber, primarySequenceNumber * statesPerReplication[ActorStateType.Actor]); } var deletedCopyKeyPrefixes = new string[] { "a", "c", "e" }; foreach (var keyPrefix in deletedCopyKeyPrefixes) { var replicationUnit = ReplicationUnit.CreateForDeleteActor( ++primarySequenceNumber, keyPrefix, statesPerReplication[ActorStateType.Actor]); TestPrepareUpdate(primaryStateTable, replicationUnit); TestCommitUpdate(primaryStateTable, primarySequenceNumber); } var copyStateEnumerator = new CopyStateEnumerator( primaryStateTable.GetShallowCopiesEnumerator(primarySequenceNumber), copyOrReplicationSerializer, primarySequenceNumber, maxCopyMessageSize); var replicationKeyPrefixes = new string[] { "v", "w", "x", "y", "z" }; replicationUnitBatch = new List <ReplicationUnit>(); foreach (var keyPrefix in replicationKeyPrefixes) { var replicationUnit = ReplicationUnit.CreateForUpdateActor( ++primarySequenceNumber, keyPrefix, statesPerReplication[ActorStateType.Actor], dataLength); replicationUnitBatch.Add(replicationUnit); replicationUnitDict[keyPrefix] = replicationUnit; } var replicationData = TestApplyBatch(primaryStateTable, replicationUnitBatch); long expectedCount = (copyKeyPrefixes.Length - deletedCopyKeyPrefixes.Length + replicationKeyPrefixes.Length) * statesPerReplication[ActorStateType.Actor]; foreach (var keyPrefix in replicationKeyPrefixes) { VerifyReads( primaryStateTable, replicationUnitDict[keyPrefix], statesPerReplication, true, dataLength, primarySequenceNumber, primarySequenceNumber, expectedCount); } var deletedReplicationKeyPrefixes = new string[] { "v", "x", "z" }; foreach (var keyPrefix in deletedReplicationKeyPrefixes) { var replicationUnit = ReplicationUnit.CreateForDeleteActor( ++primarySequenceNumber, keyPrefix, statesPerReplication[ActorStateType.Actor]); TestPrepareUpdate(primaryStateTable, replicationUnit); TestCommitUpdate(primaryStateTable, primarySequenceNumber); replicationUnit.UpdateSequenceNumber(); replicationData.Add(replicationUnit.ActorStateDataWrapperList); } var secondaryStateTable = new ActorStateTable(); var replicationStreamReadySignal = new TaskCompletionSource <object>(); var stateReplicator = new MockStateReplicator( copyOrReplicationSerializer, copyStateEnumerator, replicationData, replicationStreamReadySignal.Task); var partition = new Mock <IStatefulServicePartition>(); var secondaryPump = new SecondaryPump( partition.Object, secondaryStateTable, stateReplicator, copyOrReplicationSerializer, new VolatileLogicalTimeManager(new MockSnapshotHandler(), TimeSpan.MaxValue), "SecondaryPumpUnitTest"); TestCase("# TestCopyAndReplicationDelete: Testcase 1: Pump copy and replication operations"); secondaryPump.StartCopyAndReplicationPump(); Task pumpCompletionTask = secondaryPump.WaitForPumpCompletionAsync(); // Wait for copy pump to drain copy stream Thread.Sleep(TimeSpan.FromSeconds(5)); FailTestIf( pumpCompletionTask.IsCompleted, "pumpCompletionTask.IsCompleted. Expected: false Actual: {0}.", pumpCompletionTask.IsCompleted); foreach (var keyPrefix in copyKeyPrefixes) { bool expectedExists = !deletedCopyKeyPrefixes.ToList().Exists(o => (o == keyPrefix)); VerifyReads( secondaryStateTable, replicationUnitDict[keyPrefix], statesPerReplication, expectedExists, dataLength, copyKeyPrefixes.Length + deletedCopyKeyPrefixes.Length, copyKeyPrefixes.Length + deletedCopyKeyPrefixes.Length, (copyKeyPrefixes.Length - deletedCopyKeyPrefixes.Length) * statesPerReplication[ActorStateType.Actor] + 1); } TestCase("# Signal replication stream to be ready."); replicationStreamReadySignal.SetResult(null); pumpCompletionTask.Wait(); expectedCount = (copyKeyPrefixes.Length + replicationKeyPrefixes.Length - deletedCopyKeyPrefixes.Length - deletedReplicationKeyPrefixes.Length) * statesPerReplication[ActorStateType.Actor] + 1; foreach (var keyPrefix in replicationKeyPrefixes) { bool expectedExists = !deletedReplicationKeyPrefixes.ToList().Exists(o => (o == keyPrefix)); VerifyReads( secondaryStateTable, replicationUnitDict[keyPrefix], statesPerReplication, expectedExists, dataLength, primarySequenceNumber, primarySequenceNumber, expectedCount); } TestCase("# TestCopyAndReplicationDelete: Testcase 2: Pump replication operations"); secondaryPump.StartReplicationPump(); secondaryPump.WaitForPumpCompletionAsync().Wait(); foreach (var keyPrefix in replicationKeyPrefixes) { bool expectedExists = !deletedReplicationKeyPrefixes.ToList().Exists(o => (o == keyPrefix)); VerifyReads( secondaryStateTable, replicationUnitDict[keyPrefix], statesPerReplication, expectedExists, dataLength, primarySequenceNumber, primarySequenceNumber, expectedCount); } TestCase("# Passed"); }
private void TestCopyUpToSequenceNumber(Dictionary <ActorStateType, int> statesPerReplication) { TestCase("### TestCopyUpToSequenceNumber ###"); TestCase("### StatesPerReplication (Actor:{0}, TimeStamp:{1}, Reminder:{2}) ###", statesPerReplication[ActorStateType.Actor], statesPerReplication[ActorStateType.LogicalTimestamp], statesPerReplication[ActorStateType.Reminder]); var maxReplicationMessageSize = 2048; var maxCopyMessageSize = maxReplicationMessageSize / 2; var copyOrReplicationSerializer = VolatileActorStateProvider.CreateCopyOrReplicationOperationSerializer(); var dataLength = (maxCopyMessageSize / (3 * statesPerReplication[ActorStateType.Actor])) + 1; var primaryStateTable = new ActorStateTable(); var copyKeyPrefixes = new string[] { "a", "b", "c", "d", "e" }; var primarySequenceNumber = 0; var replicationUnitBatch = new List <ReplicationUnit>(); var replicationUnitDict = new Dictionary <string, ReplicationUnit>(); foreach (var keyPrefix in copyKeyPrefixes) { var replicationUnit = ReplicationUnit.CreateForUpdateActor( ++primarySequenceNumber, keyPrefix, statesPerReplication[ActorStateType.Actor], dataLength); replicationUnitBatch.Add(replicationUnit); replicationUnitDict[keyPrefix] = replicationUnit; } TestApplyBatch(primaryStateTable, replicationUnitBatch); foreach (var keyPrefix in copyKeyPrefixes) { VerifyReads( primaryStateTable, replicationUnitDict[keyPrefix], statesPerReplication, true, dataLength, primarySequenceNumber, primarySequenceNumber, primarySequenceNumber * statesPerReplication[ActorStateType.Actor]); } var replicationKeyPrefixes = new string[] { "w", "x", "y", "z" }; replicationUnitBatch = new List <ReplicationUnit>(); foreach (var keyPrefix in replicationKeyPrefixes) { var replicationUnit = ReplicationUnit.CreateForUpdateActor( ++primarySequenceNumber, keyPrefix, statesPerReplication[ActorStateType.Actor], dataLength); replicationUnitBatch.Add(replicationUnit); replicationUnitDict[keyPrefix] = replicationUnit; } TestApplyBatch(primaryStateTable, replicationUnitBatch); foreach (var keyPrefix in replicationKeyPrefixes) { VerifyReads( primaryStateTable, replicationUnitDict[keyPrefix], statesPerReplication, true, dataLength, primarySequenceNumber, primarySequenceNumber, primarySequenceNumber * statesPerReplication[ActorStateType.Actor]); } var upToSequenceNumber = copyKeyPrefixes.Length + replicationKeyPrefixes.Length / 2; var copyStateEnumerator = new CopyStateEnumerator( primaryStateTable.GetShallowCopiesEnumerator(upToSequenceNumber), copyOrReplicationSerializer, upToSequenceNumber, maxCopyMessageSize); var secondaryStateTable = new ActorStateTable(); var replicationStreamReadySignal = new TaskCompletionSource <object>(); var stateReplicator = new MockStateReplicator( copyOrReplicationSerializer, copyStateEnumerator, new List <List <ActorStateDataWrapper> >(), replicationStreamReadySignal.Task); var partition = new Mock <IStatefulServicePartition>(); var secondaryPump = new SecondaryPump( partition.Object, secondaryStateTable, stateReplicator, copyOrReplicationSerializer, new VolatileLogicalTimeManager(new MockSnapshotHandler(), TimeSpan.MaxValue), "SecondaryPumpUnitTest"); TestCase("# TestCopyUpToSequenceNumber: Testcase 1: Pump copy operations"); secondaryPump.StartCopyAndReplicationPump(); Task pumpCompletionTask = secondaryPump.WaitForPumpCompletionAsync(); Thread.Sleep(TimeSpan.FromSeconds(5)); FailTestIf( pumpCompletionTask.IsCompleted, "Pump CopyAndReplicationTask completed before replication stream is ready."); foreach (var keyPrefix in copyKeyPrefixes) { VerifyReads( secondaryStateTable, replicationUnitDict[keyPrefix], statesPerReplication, true, dataLength, upToSequenceNumber, upToSequenceNumber, upToSequenceNumber * statesPerReplication[ActorStateType.Actor]); } TestCase("# Signal replication stream to be ready."); replicationStreamReadySignal.SetResult(null); pumpCompletionTask.Wait(); foreach (var keyPrefix in replicationKeyPrefixes) { VerifyReads( secondaryStateTable, replicationUnitDict[keyPrefix], statesPerReplication, (keyPrefix == "w" || keyPrefix == "x") ? true : false, dataLength, upToSequenceNumber, upToSequenceNumber, upToSequenceNumber * statesPerReplication[ActorStateType.Actor]); } TestCase("# TestCopyUpToSequenceNumber: Testcase 2: Pump replication operations (none)"); secondaryPump.StartReplicationPump(); secondaryPump.WaitForPumpCompletionAsync().Wait(); foreach (var keyPrefix in replicationKeyPrefixes) { VerifyReads( secondaryStateTable, replicationUnitDict[keyPrefix], statesPerReplication, (keyPrefix == "w" || keyPrefix == "x") ? true : false, dataLength, upToSequenceNumber, upToSequenceNumber, upToSequenceNumber * statesPerReplication[ActorStateType.Actor]); } TestCase("# Passed"); }