/// <summary> /// Adds a version (insert or delete) to the store component for a given key. /// </summary> /// <param name="isIdempotent">Specifies if the add operation is idempotent.</param> /// <param name="key">Key for which the version is added.</param> /// <param name="item">Version item to add.</param> public void Add(bool isIdempotent, TKey key, TVersionedItem <TValue> item) { // Checks. Diagnostics.Assert( item.VersionSequenceNumber >= 0, this.traceType, "unexpected replication sequence number {0}", item.VersionSequenceNumber); TWriteSetItemContext context; if (this.writeSet.TryGetValue(key, out context)) { if (isIdempotent) { // Apply only if the version sequence number is higher. if (item.VersionSequenceNumber > context.LatestValue.VersionSequenceNumber) { this.Update(key, context, item); } } else { // Update must succeed this.Update(key, context, item); } } else { // First time this key is modified - add it. this.writeSet.Add(key, new TWriteSetItemContext(item)); } }
private void UpdateAddList(TKey key, TVersionedItem <TValue> value, IReadableStoreComponent <TKey, TVersionedItem <TValue> > consolidationManager) { // Note: Temporary Assert that should not be required. bool isInReadOnlyMode = Volatile.Read(ref this.isReadonly); Diagnostics.Assert(isInReadOnlyMode == false, this.traceType, "Write operation cannot come after the differential has become readonly."); var snapAddList = this.addList; Diagnostics.Assert( snapAddList != null, this.traceType, "If Apply's are coming, perform checkpoint could not have been called (hence Sort). Version: {0} IsReadOnly: {1}", value.VersionSequenceNumber, Volatile.Read(ref this.isReadonly)); // If the verion being added is a delete then it does not need to be added to the add list. // Either // a) Insert is in LC - 1 and hence in addList already // b) Insert is in the same txn and hence it is not needed in addlist // c) Insert/Update is in Consolidated and hence not needed in addlist. if (value.Kind == RecordKind.DeletedVersion) { return; } // If consolidated component does not have the key or the latest version it has is a delete, we need to add it to addList. // Reminder: AddList is a list of all keys that are in the differential but not in consolidated. var valueInConsolidationManager = consolidationManager.Read(key); if (valueInConsolidationManager == null || valueInConsolidationManager.Kind == RecordKind.DeletedVersion) { snapAddList.TryAdd(key); } }
/// <summary> /// Reads the value for a given key. /// </summary> /// <param name="key">Key to read.</param> /// <param name="visibilityLsn"></param> /// <returns>True and value if the key exists.</returns> public TVersionedItem <TValue> Read(TKey key, long visibilityLsn) { TVersionedItem <TValue> versionedItem = null; TVersionedItem <TValue> potentialVersionedItem = null; var versionedItemsOldDifferential = this.ReadVersions(key); if (versionedItemsOldDifferential != null) { potentialVersionedItem = versionedItemsOldDifferential.CurrentVersion; if (potentialVersionedItem != null && potentialVersionedItem.VersionSequenceNumber <= visibilityLsn) { versionedItem = potentialVersionedItem; } else { potentialVersionedItem = versionedItemsOldDifferential.PreviousVersion; if (potentialVersionedItem != null && potentialVersionedItem.VersionSequenceNumber <= visibilityLsn) { versionedItem = potentialVersionedItem; } } } return(versionedItem); }
private void WriteValue <TValue>( InMemoryBinaryWriter memoryBuffer, long basePosition, TVersionedItem <TValue> item, IStateSerializer <TValue> valueSerializer) { // Deleted items don't have values. Only serialize valid items. if (item.Kind != RecordKind.DeletedVersion) { // WriteItemAsync valueSerializer followed by checksum. // Serialize the value. var valueStartPosition = memoryBuffer.BaseStream.Position; valueSerializer.Write(item.Value, memoryBuffer); var valueEndPosition = memoryBuffer.BaseStream.Position; Diagnostics.Assert( valueEndPosition >= valueStartPosition, DifferentialStoreConstants.TraceType, "User's value IStateSerializer moved the stream position backwards unexpectedly!"); // Write the checksum of just that value's bytes. var valueSize = checked ((int)(valueEndPosition - valueStartPosition)); var checksum = CRC64.ToCRC64(memoryBuffer.BaseStream.GetBuffer(), checked ((int)valueStartPosition), valueSize); // Update the in-memory offset and size for this item. item.Offset = basePosition + valueStartPosition; item.ValueSize = valueSize; item.ValueChecksum = checksum; // Update checkpoint file in-memory metadata. this.Properties.ValueCount++; } // Update the in-memory metadata about which file this key-value exists in on disk. item.FileId = this.FileId; }
/// <summary> /// Read key from file for merge. /// </summary> /// <remarks> /// The data is written is 8 bytes aligned. /// /// Name Type Size /// /// KeySize int 4 /// Kind byte 1 /// RESERVED 3 /// VersionSequenceNumber long 8 /// /// (DeletedVersion) /// TimeStamp long 8 /// /// (Inserted || Updated) /// Offset long 8 /// ValueChecksum ulong 8 /// ValueSize int 4 /// RESERVED 4 /// /// Key TKey N /// PADDING (N % 8 ==0) ? 0 : 8 - (N % 8) /// /// RESERVED: Fixed padding that is usable to add fields in future. /// PADDING: Due to dynamic size, cannot be used for adding fields. /// /// Note: Larges Key size supported is int.MaxValue in bytes. /// </remarks> public KeyData <TKey, TValue> ReadKey <TKey, TValue>(InMemoryBinaryReader memoryBuffer, IStateSerializer <TKey> keySerializer) { memoryBuffer.ThrowIfNotAligned(); // This mirrors WriteKey(). var keySize = memoryBuffer.ReadInt32(); var kind = (RecordKind)memoryBuffer.ReadByte(); memoryBuffer.ReadPaddingUntilAligned(true); var lsn = memoryBuffer.ReadInt64(); long valueOffset = 0; var valueSize = 0; ulong valueChecksum = 0; long TimeStamp = 0; if (kind == RecordKind.DeletedVersion) { TimeStamp = memoryBuffer.ReadInt64(); } else { valueOffset = memoryBuffer.ReadInt64(); valueChecksum = memoryBuffer.ReadUInt64(); valueSize = memoryBuffer.ReadInt32(); memoryBuffer.ReadPaddingUntilAligned(true); } // Protection in case the user's key serializer doesn't leave the stream at the correct end point. var keyPosition = memoryBuffer.BaseStream.Position; var key = keySerializer.Read(memoryBuffer); memoryBuffer.BaseStream.Position = keyPosition + keySize; memoryBuffer.ReadPaddingUntilAligned(false); TVersionedItem <TValue> value = null; switch (kind) { case RecordKind.DeletedVersion: value = new TDeletedItem <TValue>(lsn, this.FileId); break; case RecordKind.InsertedVersion: value = new TInsertedItem <TValue>(lsn, this.FileId, valueOffset, valueSize, valueChecksum); break; case RecordKind.UpdatedVersion: value = new TUpdatedItem <TValue>(lsn, this.FileId, valueOffset, valueSize, valueChecksum); break; default: throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_KeyCheckpointFile_RecordKind, (byte)kind)); } return(new KeyData <TKey, TValue>(key, value, TimeStamp)); }
public static Task <TValue> ReadValueAsync <TValue>( FileMetadata fileMetadata, TVersionedItem <TValue> item, IStateSerializer <TValue> valueSerializer, CancellationToken cancellationToken, string traceType) { // Consistency checks. Diagnostics.Assert(fileMetadata.CheckpointFile != null, traceType, "Checkpoint file with id '{0}' does not exist in memory.", item.FileId); // Read from disk. return(fileMetadata.CheckpointFile.ReadValueAsync <TValue>(item, valueSerializer)); }
/// <summary> /// Update used for copy-on write for onees created on merge. /// </summary> /// <param name="key"></param> /// <param name="value"></param> public void Update(TKey key, TVersionedItem <TValue> value) { var existingValue = this.Read(key); Diagnostics.Assert(value != null, this.traceType, "value cannot be null"); Diagnostics.Assert( existingValue.VersionSequenceNumber == value.VersionSequenceNumber, this.traceType, "Update must replace the item with the same version."); this.component[key] = value; }
/// <summary> /// Adds a new entry to this store component. /// </summary> /// <param name="key">Key to add.</param> /// <param name="value">Value to add.</param> public void Add(TKey key, TVersionedItem <TValue> value) { // When items are moved to snapshot component, there can be cases where an lsn comes from the differential and subsequently a // smaller lsn comes from the consolidated component, when this happens the smaller value should not replace the larger lsn. bool isAdded = this.component.TryAdd(key, value); if (isAdded == false) { this.component.Update( key, (currentKey, currentValue) => { return((currentValue.VersionSequenceNumber <= value.VersionSequenceNumber) ? value : currentValue); }); } }
public static Task <byte[]> ReadValueAsync <TValue>( Dictionary <uint, FileMetadata> metadataTable, TVersionedItem <TValue> item, CancellationToken cancellationToken, string traceType) { FileMetadata fileMetadata = null; if (!metadataTable.TryGetValue(item.FileId, out fileMetadata)) { throw new InvalidOperationException(string.Format(CultureInfo.CurrentCulture, SR.Error_MetadataManager_FileDoesNotExist, item.FileId)); } // Consistency checks. Diagnostics.Assert(fileMetadata.CheckpointFile != null, traceType, "Checkpoint file with id '{0}' does not exist in memory.", item.FileId); // Read from disk. return(fileMetadata.CheckpointFile.ReadValueAsync <TValue>(item)); }
private void Update(TKey key, TWriteSetItemContext currentContext, TVersionedItem <TValue> newItem) { var lastVersionedItem = currentContext.LatestValue; // Further checks. if (lastVersionedItem.VersionSequenceNumber > 0) { Diagnostics.Assert( lastVersionedItem.VersionSequenceNumber.CompareTo(newItem.VersionSequenceNumber) < 0, this.traceType, "unexpected version sequence number ({0},{1})", lastVersionedItem.VersionSequenceNumber, newItem.VersionSequenceNumber); } if (newItem is TDeletedItem <TValue> ) { Diagnostics.Assert( lastVersionedItem is TInsertedItem <TValue> || lastVersionedItem is TUpdatedItem <TValue>, this.traceType, "unexpected deleted/deleted version sequence ({0},{1})", lastVersionedItem.VersionSequenceNumber, newItem.VersionSequenceNumber); } else if (newItem is TInsertedItem <TValue> ) { Diagnostics.Assert( lastVersionedItem is TDeletedItem <TValue>, this.traceType, "unexpected inserted/inserted version sequence ({0},{1})", lastVersionedItem.VersionSequenceNumber, newItem.VersionSequenceNumber); } else if (newItem is TUpdatedItem <TValue> ) { Diagnostics.Assert( lastVersionedItem is TInsertedItem <TValue> || lastVersionedItem is TUpdatedItem <TValue>, this.traceType, "unexpected deleted/updated version sequence ({0},{1})", lastVersionedItem.VersionSequenceNumber, newItem.VersionSequenceNumber); } // Update last versioned item. this.writeSet[key] = new TWriteSetItemContext(currentContext, newItem); }
public int CompareTo(KeyCheckpointFileAsyncEnumerator <TKey, TValue> other) { // Compare Keys var compare = this.keyComparer.Compare(this.Current.Key, other.Current.Key); if (compare != 0) { return(compare); } // Compare LSNs var currentLsn = this.Current.Value.VersionSequenceNumber; var otherLsn = other.Current.Value.VersionSequenceNumber; if (currentLsn > otherLsn) { return(-1); } else if (currentLsn < otherLsn) { return(1); } // If they are both deleted items, compare timestamps. TVersionedItem <TValue> currentItem = this.Current.Value; TVersionedItem <TValue> otherItem = other.Current.Value; if (currentItem.Kind == RecordKind.DeletedVersion && otherItem.Kind == RecordKind.DeletedVersion) { Diagnostics.Assert(currentItem.FileId != otherItem.FileId, this.traceType, "Current item file id should not be same as other item"); if (this.Current.TimeStamp > other.Current.TimeStamp) { return(-1); } else if (this.Current.TimeStamp < other.Current.TimeStamp) { return(1); } return(0); } return(0); }
/// <summary> /// Reads the value for a given key. /// </summary> /// <param name="key">Key to read.</param> /// <returns>True and value if the key exists.</returns> public TVersionedItem <TValue> Read(TKey key) { TVersionedItem <TValue> versionedItem = null; foreach (var deltaDifferentialState in this.GetInReverseOrder()) { versionedItem = deltaDifferentialState.Read(key); if (versionedItem != null) { break; } } if (versionedItem == null) { versionedItem = this.consolidatedState.Read(key); } return(versionedItem); }
/// <summary> /// Adds a new entry to this store component. /// </summary> /// <param name="key">Key to add.</param> /// <param name="value">Value to add.</param> public void Add(TKey key, TVersionedItem <TValue> value) { // There are no concurrent checkpoint, so it is okay to do it without a lock. Diagnostics.Assert(value != null, this.traceType, "value cannot be null"); Diagnostics.Assert(value.Kind != RecordKind.DeletedVersion, this.traceType, "Deleted items are not added from consolidation."); var existingValue = this.Read(key); Diagnostics.Assert(existingValue == null, this.traceType, "existing item is being added."); try { this.component.Add(key, value); } catch (ArgumentException) { Diagnostics.Assert(false, this.traceType, "failed to add to consolidated component"); } }
public async Task <byte[]> ReadValueAsync <TValue>(TVersionedItem <TValue> item) { Stream fileStream = null; try { // Acquire a re-usable file stream for exclusive use during this read. fileStream = this.ReaderPool.AcquireStream(); var snapFileStream = fileStream as FileStream; Diagnostics.Assert(snapFileStream != null, this.traceType, "fileStream must be a FileStream"); Microsoft.ServiceFabric.Replicator.Utility.SetIoPriorityHint(snapFileStream.SafeFileHandle, this.priorityHint); // TODO: use memory stream pool here. using (var stream = new MemoryStream(capacity: item.ValueSize)) { // Read the value bytes and the checksum into memory. fileStream.Position = item.Offset; stream.SetLength(item.ValueSize); await fileStream.ReadAsync(stream.GetBuffer(), 0, item.ValueSize).ConfigureAwait(false); // Read the checksum from memory. stream.Position = item.ValueSize; var checksum = item.ValueChecksum; // Re-compute the checksum. var expectedChecksum = CRC64.ToCRC64(stream.GetBuffer(), 0, item.ValueSize); if (checksum != expectedChecksum) { throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FailedReadValue_ChecksumMismatch_TwoArgs, checksum, expectedChecksum)); } return(stream.GetBuffer()); } } finally { // Return the file stream to the pool for re-use. this.ReaderPool.ReleaseStream(fileStream); } }
/// <summary> /// Reads the value for a given key. /// </summary> /// <param name="key">Key to read.</param> /// <param name="visbilityLsn"></param> /// <returns>True and value if the key exists.</returns> public TVersionedItem <TValue> Read(TKey key, long visbilityLsn) { TVersionedItem <TValue> versionedItem = null; foreach (var deltaDifferentialState in this.GetInReverseOrder()) { versionedItem = deltaDifferentialState.Read(key, visbilityLsn); if (versionedItem != null) { return(versionedItem); } } versionedItem = this.consolidatedState.Read(key, visbilityLsn); if (versionedItem != null) { return(versionedItem); } return(null); }
/// <summary> /// Called during false progress processing for multi-versioned and historical stores. /// </summary> /// <param name="keyLockResourceNameHash"></param> /// <param name="sequenceNumber">Sequence number at which the false progress occurred.</param> /// <param name="storeModificationType">Specifies the type of item to be removed.</param> /// <param name="key"></param> /// <returns>True and value if the key exists.</returns> /// <devnote>AddList update not required since it is ok to have more keys then necessary.</devnote> internal bool UndoFalseProgress(TKey key, ulong keyLockResourceNameHash, long sequenceNumber, StoreModificationType storeModificationType) { // In the current model, checkpoint cannot have false progress. Diagnostics.Assert(this.isReadonly == false, this.traceType, "Write operation cannot come after the differential has become readonly."); // Find the item sequence number. var differentialStateVersions = this.ReadVersions(key); // Remove item. TVersionedItem <TValue> undoItem = null; if (differentialStateVersions != null) { var previousVersion = differentialStateVersions.PreviousVersion; if (previousVersion != null && previousVersion.VersionSequenceNumber == sequenceNumber) { undoItem = previousVersion; differentialStateVersions.PreviousVersion = null; } else { var currentVersion = differentialStateVersions.CurrentVersion; Diagnostics.Assert(currentVersion != null, this.traceType, "current version cannot be null"); if (currentVersion.VersionSequenceNumber == sequenceNumber) { undoItem = currentVersion; // Update current version and make prev version null. if (differentialStateVersions.PreviousVersion != null) { differentialStateVersions.CurrentVersion = differentialStateVersions.PreviousVersion; differentialStateVersions.PreviousVersion = null; } else { // Remove key this.RemoveKey(key); } } } if (undoItem != null) { switch (storeModificationType) { case StoreModificationType.Add: // Remove implies undoing insert. Diagnostics.Assert(undoItem is TInsertedItem <TValue>, this.traceType, "unexpected deleted version"); break; case StoreModificationType.Remove: // Add implies undoing delete. Diagnostics.Assert(undoItem is TDeletedItem <TValue>, this.traceType, "unexpected inserted version"); break; case StoreModificationType.Update: Diagnostics.Assert(undoItem is TUpdatedItem <TValue> || undoItem is TInsertedItem <TValue>, this.traceType, "unexpected updated version"); break; } return(false); } } return(true); }
/// <summary> /// Add a value to the given file stream, using the memory buffer to stage writes before issuing bulk disk IOs. /// </summary> /// <typeparam name="TValue"></typeparam> /// <param name="fileStream"></param> /// <param name="memoryBuffer"></param> /// <param name="item"></param> /// <param name="value"></param> /// <param name="traceType"></param> /// <returns></returns> public void WriteItem <TValue>(Stream fileStream, InMemoryBinaryWriter memoryBuffer, TVersionedItem <TValue> item, byte[] value, string traceType) { // Write the value into the memory buffer. this.WriteValue(memoryBuffer, fileStream.Position, item, value); }
/// <summary> /// Read the given value from disk. /// </summary> /// <typeparam name="TValue"></typeparam> /// <param name="item"></param> /// <param name="valueSerializer"></param> /// <returns></returns> public async Task <TValue> ReadValueAsync <TValue>(TVersionedItem <TValue> item, IStateSerializer <TValue> valueSerializer) { // Validate the item has a value that can be read from disk. if (item == null) { throw new ArgumentNullException(SR.Error_Item); } if (item.Kind == RecordKind.DeletedVersion) { throw new ArgumentException(SR.Error_ValueCheckpoint_DeletedItemValue, SR.Error_Item); } // Validate that the item's disk properties are valid. if (item.Offset < this.Properties.ValuesHandle.Offset) { throw new ArgumentOutOfRangeException(SR.Error_Item, SR.Error_ValueCheckpoint_TVersionedItem_Offset_Negative); } if (item.ValueSize < 0) { throw new ArgumentOutOfRangeException(string.Format(CultureInfo.CurrentCulture, SR.Error_Item, SR.Error_ValueCheckpoint_TVersionedItem_ValueSize_Negative, item.ValueSize)); } if (item.Offset + item.ValueSize > this.Properties.ValuesHandle.EndOffset) { throw new ArgumentOutOfRangeException(SR.Error_Item, SR.Error_ValueCheckpoint_StreamRangeExceeded); } Stream fileStream = null; try { // Acquire a re-usable file stream for exclusive use during this read. fileStream = this.ReaderPool.AcquireStream(); var snapFileStream = fileStream as FileStream; Diagnostics.Assert(snapFileStream != null, this.traceType, "fileStream must be a FileStream"); Microsoft.ServiceFabric.Replicator.Utility.SetIoPriorityHint(snapFileStream.SafeFileHandle, this.priorityHint); // TODO: use memory stream pool here. using (var stream = new MemoryStream(capacity: item.ValueSize)) using (var reader = new BinaryReader(stream)) { // Read the value bytes and the checksum into memory. fileStream.Position = item.Offset; stream.SetLength(item.ValueSize); await fileStream.ReadAsync(stream.GetBuffer(), 0, item.ValueSize).ConfigureAwait(false); // Read the checksum from memory. stream.Position = item.ValueSize; var checksum = item.ValueChecksum; // Re-compute the checksum. var expectedChecksum = CRC64.ToCRC64(stream.GetBuffer(), 0, item.ValueSize); if (checksum != expectedChecksum) { throw new InvalidDataException(string.Format(CultureInfo.CurrentCulture, SR.Error_FailedReadValue_ChecksumMismatch_TwoArgs, checksum, expectedChecksum)); } // Deserialize the value into memory. stream.Position = 0; return(valueSerializer.Read(reader)); } } finally { // Return the file stream to the pool for re-use. this.ReaderPool.ReleaseStream(fileStream); } }
public TWriteSetItemContext(TVersionedItem <TValue> value) { this.LatestValue = value; this.CreateSequenceNumber = value.VersionSequenceNumber; this.FirstVersionKind = value.Kind; }
public TWriteSetItemContext(TWriteSetItemContext previous, TVersionedItem <TValue> latestValue) { this.LatestValue = latestValue; this.CreateSequenceNumber = previous.CreateSequenceNumber; this.FirstVersionKind = previous.FirstVersionKind; }
/// <summary> /// Read the given value from disk. /// </summary> /// <typeparam name="TValue"></typeparam> /// <param name="item"></param> /// <param name="valueSerializer"></param> /// <returns></returns> public Task <TValue> ReadValueAsync <TValue>(TVersionedItem <TValue> item, IStateSerializer <TValue> valueSerializer) { return(this.ValueCheckpointFile.ReadValueAsync(item, valueSerializer)); }
/// <summary> /// Read the given value from disk. /// </summary> /// <typeparam name="TValue"></typeparam> /// <param name="item"></param> /// <returns></returns> public Task <byte[]> ReadValueAsync <TValue>(TVersionedItem <TValue> item) { return(this.ValueCheckpointFile.ReadValueAsync(item)); }
/// <summary> /// Adds a new entry to this store component. /// </summary> /// <devnote> /// Consolidated Manager is only required for the AddList feature. /// </devnote> public void Add(TKey key, TVersionedItem <TValue> value, IReadableStoreComponent <TKey, TVersionedItem <TValue> > consolidationManager) { bool isInReadOnlyMode = Volatile.Read(ref this.isReadonly); Diagnostics.Assert(isInReadOnlyMode == false, this.traceType, "Write operation cannot come after the differential has become readonly."); var concurrentDictionary = this.component as ConcurrentDictionary <TKey, DifferentialStateVersions <TValue> >; Diagnostics.Assert(concurrentDictionary != null, this.traceType, "write operation came after becoming readonly."); var differentialVersions = concurrentDictionary.GetOrAdd(key, k => new DifferentialStateVersions <TValue>()); if (differentialVersions.CurrentVersion == null) { Diagnostics.Assert(differentialVersions.PreviousVersion == null, this.traceType, "previous version should be null"); differentialVersions.CurrentVersion = value; if (value.Kind == RecordKind.InsertedVersion) { Diagnostics.Assert( (differentialVersions.flags & DifferentialStateFlags.NewKey) == DifferentialStateFlags.None, this.traceType, "DifferentialStateFlags.NewKey is already set"); differentialVersions.flags |= DifferentialStateFlags.NewKey; } } else { // Assert that version sequence number is lesser or equal compared to the new one. // Sequence number can be equal if the same key gets updated multiple times in the same transaction. if (differentialVersions.CurrentVersion.VersionSequenceNumber > value.VersionSequenceNumber) { var message = string.Format(@"New lsn {0} should be greater than or equal the existing current lsn {1}. This error could be caused by an incorrect implementation of key comparer or key serializer. Please check your implementation of IEquatable, IComparable, and custom serializer (if applicable).", value.VersionSequenceNumber, differentialVersions.CurrentVersion.VersionSequenceNumber); FabricEvents.Events.StoreDiagnosticError(this.traceType, message); Diagnostics.Assert( false, this.traceType, message); } if (differentialVersions.CurrentVersion.VersionSequenceNumber == value.VersionSequenceNumber) { // Set the latest value in case there are multiple updates in the same transaction(same lsn). differentialVersions.CurrentVersion = value; this.UpdateAddList(key, value, consolidationManager); return; } // Check if previous version is null if (differentialVersions.PreviousVersion == null) { differentialVersions.PreviousVersion = differentialVersions.CurrentVersion; differentialVersions.CurrentVersion = value; } else { // Move to snapshot container, if needed. var removeVersionResult = this.transactionalReplicator.TryRemoveVersion( this.stateProviderId, differentialVersions.PreviousVersion.VersionSequenceNumber, differentialVersions.CurrentVersion.VersionSequenceNumber); if (!removeVersionResult.CanBeRemoved) { var enumerationSet = removeVersionResult.EnumerationSet; foreach (var snapshotVisibilityLsn in enumerationSet) { var snapshotComponent = this.snapshotContainer.Read(snapshotVisibilityLsn); if (snapshotComponent == null) { snapshotComponent = new SnapshotComponent <TKey, TValue, TKeyComparer, TKeyEqualityComparer>(this.snapshotContainer, this.loadValueCounter, this.isValueAReferenceType, this.traceType); snapshotComponent = this.snapshotContainer.GetOrAdd(snapshotVisibilityLsn, snapshotComponent); } snapshotComponent.Add(key, differentialVersions.PreviousVersion); } // Wait on notifications and remove the entry from the container if (removeVersionResult.EnumerationCompletionNotifications != null) { foreach (var enumerationResult in removeVersionResult.EnumerationCompletionNotifications) { enumerationResult.Notification.ContinueWith(x => { this.snapshotContainer.Remove(enumerationResult.VisibilitySequenceNumber); }) .IgnoreExceptionVoid(); } } } // Remove from differential state differentialVersions.PreviousVersion = differentialVersions.CurrentVersion; differentialVersions.CurrentVersion = value; } } this.UpdateAddList(key, value, consolidationManager); }