/// <summary> /// Parallel task that checks if the transaction has been aborted internally /// If so, cause the waiting dequeuer to throw InvalidOperationException /// </summary> private async Task PollTransactionStateAsync( Transaction txn, TaskCompletionSource <ConditionalValue <IListElement <T> > > tcs, CancellationToken cancellationToken) { while (true) { // Ignore the exception thrown by this call // As this polls every 4 secs, The cancellationTokenSource could have been disposed and set to null // causing Task.Delay to throw ObjectDisposedException. // The token would be honoured by the StartTimerAsync task and the tcs would be set to the TaskCanceledException. await Task.Delay(this.DefaultTransactionStatePollingTimeout, cancellationToken).ConfigureAwait(false); if (tcs.Task.IsCompleted) { // If the tcs was already set, no need to wait check the transaction state. // End transaction polling task return; } switch (txn.State) { case TransactionState.Active: case TransactionState.Reading: // Do nothing, the transaction is still active, go back to sleep break; case TransactionState.Aborted: case TransactionState.Aborting: case TransactionState.Faulted: tcs.TrySetException( new InvalidOperationException( string.Format("DataStore.PollTransactionStateAsync : The transaction was aborted or faulted, dispose the transaction and try again, Txn : {0}", txn))); return; case TransactionState.Committed: case TransactionState.Committing: var exc = new InvalidOperationException( string.Format( "DataStore.PollTransactionStateAsync : The transaction was commited but DequeueAsync did not finish. Make sure DequeueAsync was awaited. Txn : {0}", txn)); if (tcs.TrySetException(exc)) { // If we were able to set the result, then DequeueAsync was not awaited. // Do not Assert as this is something that the user is doing incorrectly. // Do not throw as the exception would be thrown by the tcs. FabricEvents.Events.ReliableConcurrentQueue_ExceptionWarning(this.traceType, exc.ToString()); } return; default: // Unknown transaction state : Assert TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.PollTransactionStateAsync", "Unknown transaction state"); return; } } }
/// <summary> /// Drain all the dequeuers. /// Would be called during ChangeRole (to S, None), Abort, Close. /// </summary> /// <param name="cause">The DrainCause</param> private void DrainDequeuers(DequeuerState cause) { TaskCompletionSource <ConditionalValue <IListElement <T> > > tcs; Exception exc = null; switch (cause) { case DequeuerState.Aborting: case DequeuerState.Closing: exc = new FabricObjectClosedException(string.Format("Dequeue was canceled as the queue is {0}. Please dispose the current transaction and retry the operation with a new transaction.", cause)); break; case DequeuerState.ChangeRoleFromPrimary: exc = new FabricNotPrimaryException("Dequeue was canceled as the role is no longer primary. Please dispose the current transaction and retry the operation with a new transaction."); break; default: TestableAssertHelper.FailInvalidData( this.traceType, "DataStore.DrainDequeuers", "DequeueScheduler.DrainDequeuers : Invalid DrainQueue cause : {0}", cause); break; // unreachable } lock (this.updateLatch) { this.dequeuersState = cause; } while (this.dequeuerQueue.TryDequeue(out tcs)) { tcs.TrySetException(exc); } }
public static void SafeFileReplace(string currentFilePath, string newFilePath, string backupFilePath, string traceType) { if (FabricFile.Exists(backupFilePath)) { FabricFile.Delete(backupFilePath); } if (FabricFile.Exists(backupFilePath)) { TestableAssertHelper.FailInvalidData(traceType, "CheckpointFileHelper.SafeFileReplace", "!FabricFile.Exists(backupFilePath) : {0}", backupFilePath); } // Previous replace could have failed in the middle before the next metadata table file got renamed to current. if (!FabricFile.Exists(currentFilePath)) { FabricFile.Move(newFilePath, currentFilePath); } else { FabricFile.Replace(newFilePath, currentFilePath, backupFilePath, ignoreMetadataErrors: false); } if (FabricFile.Exists(backupFilePath)) { FabricFile.Delete(backupFilePath); } if (FabricFile.Exists(backupFilePath)) { TestableAssertHelper.FailInvalidData(traceType, "CheckpointFileHelper.SafeFileReplace", "!FabricFile.Exists(backupFilePath) : {0}", backupFilePath); } }
public IListElement <T> GetListElement(long listElementId) { if (!this.HasListElement(listElementId)) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.GetListElement", "this.HasListElement(listElementId)"); } return(this.ListElements[listElementId]); }
public void AddRef() { if (this.refCount <= 0) { TestableAssertHelper.FailInvalidOperation(this.traceType, "Checkpoint.AddRef", "refCount > 0. refCount: {0}", this.refCount); } Interlocked.Increment(ref this.refCount); }
public async Task ReadAsync() { if (this.ListElements != null) { TestableAssertHelper.FailInvalidData(this.traceType, "Checkpoint.ReadAsync", "this.ListElements == null"); } this.ListElements = await this.ReadListElementsAsync().ConfigureAwait(false); }
public void AddListElement(IListElement <T> listElement) { if (this.HasListElement(listElement)) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.AddListElement", "!this.HasListElement(listElement)"); } this.ListElements[listElement.Id] = (ListElement)listElement; }
/// <summary> /// Try and dequeue from the DataStore. /// If there are dequeuers waiting or there is no element to dequeue, this dequeueur would be added to the waiting queue. /// Else, the result would be set for the <paramref name="tcs"/> /// Caller must await on tcs.Task /// </summary> public void TryDequeue( Transaction txn, TaskCompletionSource <ConditionalValue <IListElement <T> > > tcs, TimeSpan timeout, CancellationToken cancellationToken) { lock (this.updateLatch) { switch (this.dequeuersState) { case DequeuerState.Aborting: case DequeuerState.Closing: tcs.TrySetException( new FabricObjectClosedException( string.Format("Queue is closed. Please dispose the current transaction and retry the operation with a new transaction, Cause : {0}", this.dequeuersState))); return; case DequeuerState.None: case DequeuerState.ChangeRoleFromPrimary: tcs.TrySetException( new FabricNotPrimaryException( string.Format("Current role not Primary. Please dispose the current transaction and retry the operation with a new transaction, Cause : {0}", this.dequeuersState))); return; case DequeuerState.OnPrimary: // In the expected state, continue DataStore.TryDequeue break; default: TestableAssertHelper.FailInvalidOperation( this.traceType, "TryDequeue", "DataStore.TryDequeue : Trying to add a dequeuer in an invalid state. Current DequeuerState = {0}", this.dequeuersState); break; // unreachable } var res = this.TryUnlinkHead(); if (res.HasValue) { // There is something to unlink and no waiting dequeuer if (!tcs.TrySetResult(res)) { TestableAssertHelper.FailInvalidOperation( this.traceType, "TryDequeue", "Could not set the tcs result after unlinking from the DataStore. TCS {0}", res); } } else { this.AddDequeuer(txn, tcs, timeout, cancellationToken); } } }
/// <summary> /// Assert that there are no dequeuers waiting currently /// </summary> private void AssertNoDequeuer() { if (!this.dequeuerQueue.IsEmpty) { TestableAssertHelper.FailInvalidOperation( this.traceType, "DataStore.AssertNoDequeuer", "DataStore.AssertNoDequeuers : Waiting Dequeuer Queue is not empty. Count = {0}", this.dequeuerQueue.Count); } }
public CopyStream <T> GetCopyStream() { if (this.refCount <= 0) { TestableAssertHelper.FailInvalidOperation( this.traceType, "Checkpoint.GetCopyStream", "refCount > 0. refCount: {0}", this.refCount); } return(new CopyStream <T>(this, this.traceType)); }
public void ReleaseRef() { var newRefCount = Interlocked.Decrement(ref this.refCount); if (newRefCount < 0) { TestableAssertHelper.FailInvalidOperation(this.traceType, "Checkpoint.ReleaseRef", "newRefCount >= 0. newRefCount: {0}", newRefCount); } if (newRefCount == 0) { FabricFile.Delete(this.FilePath); } }
public void RemoveListElement(IListElement <T> listElement) { this.AssertUnlinked(listElement); ListElement removed; var res = this.ListElements.TryRemove(listElement.Id, out removed); if (!res) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.RemoveListElement", "res"); } if (removed != (ListElement)listElement) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.RemoveListElement", "removed == (ReliableQueueListElement)listElement"); } }
public static bool TryDeleteCopyFile(string directory, string traceType) { var filePath = GetCopyFilePath(directory); if (!FabricFile.Exists(filePath)) { return(false); } FabricFile.Delete(filePath); if (FabricFile.Exists(filePath)) { TestableAssertHelper.FailInvalidData(traceType, "CheckpointFileHelper.TryDeleteCopyFile", "!FabricFile.Exists(filePath). filePath: {0}", filePath); } return(true); }
public ListElement(long id, T value, string traceType, ListElementState state = ListElementState.Invalid) { this.traceType = traceType; if (id == ForbiddenId) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.ListElement", "id == ForbiddenId: {0}", ForbiddenId); } this.id = id; this.value = value; this.State = state; this.Next = null; this.Previous = null; this.DequeueCommittingTransaction = null; this.stateMachineLock = new object(); }
public void LinkTail(IListElement <T> listElement) { this.AssertUnlinked(listElement); lock (this.updateLatch) { switch (this.queueMode) { // IReliableConcurrentQueue case RCQMode.NonBlocking: { this.LinkTailUnsafe(listElement); break; } // IReliableConcurrentBlockingQueue case RCQMode.Blocking: { bool waitingDeq = false; if (this.dequeuersState == DequeuerState.OnPrimary) { waitingDeq = this.TryNotifyDequeuer(listElement); } else { this.AssertNoDequeuer(); } if (!waitingDeq) { // If there was no dequeuer waiting, link the element this.LinkTailUnsafe(listElement); } break; } default: TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.LinkTail", "Invalid QueueMode : {0}", this.queueMode); break; } } }
public void AssertLinked(IListElement <T> listElement) { if (listElement == null) { TestableAssertHelper.FailArgumentNull(this.traceType, "DataStore.AssertLinked", "listElement"); } if (!this.IsLinked(listElement)) { TestableAssertHelper.FailInvalidOperation( this.traceType, "DataStore.AssertLinked", "listElement <id: {0}, next: {1}, previous: {2}, state: {3}> is not linked.", listElement.Id, listElement.Next != null ? listElement.Next.Id : ListElement.ForbiddenId, listElement.Previous != null ? listElement.Previous.Id : ListElement.ForbiddenId, listElement.State); } }
private bool IsTail(IListElement <T> listElement) { if (this.Tail != listElement) { return(false); } if (listElement.Next != null) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.IsTail", "Tail must have null Next pointer"); } if ((this.Head == listElement) != (listElement.Previous == null)) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.IsTail", "If the listElement is also the head, it must not have a valid previous pointer. Otherwise, it must have a valid previous pointer."); } return(true); }
private bool IsHead(IListElement <T> listElement) { if (this.Head != listElement) { return(false); } if (listElement.Previous != null) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.IsHead", "Head must have null Previous pointer"); } if ((listElement == this.Tail) != (listElement.Next == null)) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.IsHead", "If the listElement is also the tail, it must not have a valid next pointer. Otherwise, it must have a valid next pointer."); } return(true); }
public async Task ReplaceCurrentAsync() { if (this.FileName != NewCheckpointManagerFileName) { TestableAssertHelper.FailInvalidOperation( this.traceType, "CheckpointManager.ReplaceCurrentAsync", "this.FileName == NewCheckpointManagerFileName; expected: {0} actual: {1}", NewCheckpointManagerFileName, this.FileName); } await this.VerifyAsync().ConfigureAwait(false); var currentFilePath = Path.Combine(this.directory, CurrentCheckpointManagerFileName); var backupFilePath = Path.Combine(this.directory, BackupCheckpointManagerFileName); var newFilePath = Path.Combine(this.directory, NewCheckpointManagerFileName); CheckpointFileHelper.SafeFileReplace(currentFilePath, newFilePath, backupFilePath, this.traceType); this.FileName = CurrentCheckpointManagerFileName; }
// returns false if this.ListElements was not set (no checkpoint), true if this.ListElements was set. public async Task <bool> TryReadAsync() { if (this.ListElements != null) { TestableAssertHelper.FailInvalidOperation( this.traceType, "Checkpoint.TryReadAsync", "this.ListElements == null. <listElementsCount: {0}, filePath: {1}, refCount: {2}>", this.ListElements.Count, this.FilePath, this.refCount); } if (!FabricFile.Exists(this.FilePath)) { FabricEvents.Events.ReliableConcurrentQueue_NoCheckpoint("Checkpoint.TryReadAsync@", this.FilePath); return(false); } await this.ReadAsync().ConfigureAwait(false); return(true); }
private async Task VerifyAsync() { // Read from disk to verify var res = await TryReadCheckpointFile(this.directory, this.FileName, this.valueSerializer, this.traceType).ConfigureAwait(false); if (!res.HasValue) { throw new InvalidDataException(string.Format("Checkpoint file '{0}' not found.", Path.Combine(this.directory, this.FileName))); } if ((this.CurrentCheckpoint == null) != (res.Value.CurrentCheckpoint == null)) { TestableAssertHelper.FailInvalidData( this.traceType, "CheckpointManager.VerifyAsync", "Disagreement on checkpoint existing. In-Memory: {0} On-Disk: {1}", this.CurrentCheckpoint != null ? this.CurrentCheckpoint.FilePath : "null", res.Value.CurrentCheckpoint != null ? res.Value.CurrentCheckpoint.FilePath : "null"); } await this.CurrentCheckpoint.VerifyAsync().ConfigureAwait(false); }
/// <summary> /// Should be called under the update latch /// <paramref name="listElement"/> must be unlinked, Caller must Assert on this /// </summary> /// <param name="listElement"></param> private void LinkTailUnsafe(IListElement <T> listElement) { var queueListElement = (ListElement)listElement; if (this.Tail == null) { if (this.Head != null) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.LinkTailUnsafe", "this.Head == null"); } if (this.LinkedCount != 0) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.LinkTailUnsafe", "this.LinkedCount == 0"); } this.head = queueListElement; this.tail = queueListElement; } else { if (this.Head == null) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.LinkTailUnsafe", "this.Head != null"); } if (this.LinkedCount == 0) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.LinkTailUnsafe", "this.LinkedCount != 0"); } this.tail.Next = queueListElement; queueListElement.Previous = this.tail; this.tail = queueListElement; } this.LinkedCount++; }
protected override bool IsEqual(object stateOrKey) { // Should never be called. TestableAssertHelper.FailInvalidOperation("OperationLockContext", "IsEqual", "should never be called"); return(false); }
public async Task WriteAsync() { if (this.ListElements == null) { TestableAssertHelper.FailArgumentNull(this.traceType, "Checkpoint.WriteAsync", "this.ListElements == null"); } if (this.valueSerializer == null) { TestableAssertHelper.FailArgumentNull(this.traceType, "Checkpoint.WriteAsync", "this.valueSerializer == null"); } using (var stream = FabricFile.Open(this.FilePath, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.SequentialScan)) { await stream.WriteAsync( CheckpointFileHelper.MetadataMarkerSegment.Array, CheckpointFileHelper.MetadataMarkerSegment.Offset, CheckpointFileHelper.MetadataMarkerSegment.Count).ConfigureAwait(false); var versionBytes = BitConverter.GetBytes(FileVersion); await stream.WriteAsync(versionBytes, 0, versionBytes.Length).ConfigureAwait(false); var frame = new CheckpointFrame <T>(this.valueSerializer); foreach (var listElement in this.ListElements) { if (frame.Size >= CheckpointFileHelper.ChunkWriteThreshold) { await stream.WriteAsync( CheckpointFileHelper.BeginChunkMarkerSegment.Array, CheckpointFileHelper.BeginChunkMarkerSegment.Offset, CheckpointFileHelper.BeginChunkMarkerSegment.Count).ConfigureAwait(false); await frame.WriteAsync(stream).ConfigureAwait(false); frame.Reset(); } frame.AddListElement(listElement); } if (frame.ListElementsCount > 0) { await stream.WriteAsync( CheckpointFileHelper.BeginChunkMarkerSegment.Array, CheckpointFileHelper.BeginChunkMarkerSegment.Offset, CheckpointFileHelper.BeginChunkMarkerSegment.Count).ConfigureAwait(false); await frame.WriteAsync(stream).ConfigureAwait(false); frame.Reset(); } await stream.WriteAsync( CheckpointFileHelper.EndFramesMarkerSegment.Array, CheckpointFileHelper.EndFramesMarkerSegment.Offset, CheckpointFileHelper.EndFramesMarkerSegment.Count).ConfigureAwait(false); } }
/// <summary> /// Unlink a listElement from the list. Must be called under the update lock. /// </summary> /// <param name="listElement"></param> private void UnlinkUnsafe(IListElement <T> listElement) { if (listElement == null) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.UnlinkUnsafe", "listElement != null"); } if (this.LinkedCount == 0) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.UnlinkUnsafe", "this.LinkedCount != 0"); } if (this.Head == null) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.UnlinkUnsafe", "this.Head != null"); } if (this.Head.Previous != null) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.UnlinkUnsafe", "this.Head.Previous == null"); } if (this.Tail == null) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.UnlinkUnsafe", "this.Tail != null"); } if (this.Tail.Next != null) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.UnlinkUnsafe", "this.Tail.Next == null"); } if (this.LinkedCount != 1 && listElement.Next == null && listElement.Previous == null) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.UnlinkUnsafe", "!(this.LinkedCount != 1 && listElement.Next == null && listElement.Previous == null)"); } var queueListElement = (ListElement)listElement; if (listElement.Previous == null) { if (listElement != this.Head) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.UnlinkUnsafe", "listElement == this.Head"); } this.head = this.head.Next; } else { queueListElement.Previous.Next = queueListElement.Next; } if (listElement.Next == null) { if (listElement != this.Tail) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.UnlinkUnsafe", "listElement == this.Tail"); } this.tail = queueListElement.Previous; } else { queueListElement.Next.Previous = queueListElement.Previous; } queueListElement.Previous = null; queueListElement.Next = null; this.LinkedCount--; }
private bool IsLinkedUnsafe(IListElement <T> listElement) { // The order of the following checks is significant // is this listElement in the dictionary at all if (!this.HasListElement(listElement)) { if (listElement.Next != null) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.IsLinkedUnsafe", "sanity: If this listElement is not in the dictionary, it must not have a valid next pointer."); } if (listElement.Previous != null) { TestableAssertHelper.FailInvalidOperation( this.traceType, "DataStore.IsLinkedUnsafe", "sanity: If this listElement is not in the dictionary, it must not have a valid previous pointer."); } return(false); } // is it linked at the head or the tail if (this.IsHead(listElement) || this.IsTail(listElement)) { return(true); } // is it not linked at all if (listElement.Next == null && listElement.Previous == null) { return(false); } // is it linked in the middle if (listElement.Next != null && listElement.Previous != null) { if (!(this.HasListElement(listElement.Next))) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.IsLinkedUnsafe", "sanity: listElement.Next must be in the dictionary."); } if (listElement.Next.Previous != listElement) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.IsLinkedUnsafe", "sanity: listElement.Next.Previous must be listElement."); } if (!(this.HasListElement(listElement.Previous))) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.IsLinkedUnsafe", "sanity: listElement.Previous must be in the dictionary."); } if (listElement.Previous.Next != listElement) { TestableAssertHelper.FailInvalidOperation(this.traceType, "DataStore.IsLinkedUnsafe", "sanity: listElement.Previous.Next must be listElement."); } return(true); } // it is partially linked and thus incorrect TestableAssertHelper.FailInvalidOperation( this.traceType, "DataStore.IsLinkedUnsafe", "Invalid listElement state: <id: {0}, next: {1}, previous: {2}, state: {3}>", listElement.Id, listElement.Next != null ? listElement.Next.Id : ListElement.ForbiddenId, listElement.Previous != null ? listElement.Previous.Id : ListElement.ForbiddenId, listElement.State); throw new InvalidOperationException("Invalid listElement state: " + listElement); // unreachable }