public Task Schedule(
SubDirectoryTransfer subDirectoryTransfer,
CancellationToken cancellationToken,
Action persistDirTransfer,
int timeOut)
{
if (this.semaphore.Wait(timeOut, cancellationToken))
{
try
{
if (null != persistDirTransfer)
{
persistDirTransfer();
}
}
catch
{
this.semaphore.Release();
throw;
}
Task task = subDirectoryTransfer.ExecuteAsync(cancellationToken);
task.ContinueWith((sourceTask) =>
{
this.semaphore.Release();
return(sourceTask);
});
return(task);
}
else
{
return(null);
}
}
private bool ScheduleSubDirectoryTransfer(
SubDirectoryTransfer subDirectoryTransfer,
CancellationToken cancellationToken,
Action persistDirTransfer,
int timeOut)
{
Interlocked.Increment(ref this.outstandingTasks);
Task directoryListTask = null;
try
{
directoryListTask = this.directoryListingScheduler.Schedule(
subDirectoryTransfer,
cancellationToken,
persistDirTransfer,
timeOut);
}
catch (OperationCanceledException)
{
// Ignore exception
}
catch (ObjectDisposedException)
{
// Ignore exception
}
this.WaitOnSubDirectoryListTask(directoryListTask, subDirectoryTransfer);
return(null != directoryListTask);
}
protected HierarchyDirectoryTransfer(HierarchyDirectoryTransfer other)
: base(other)
{
other.progressUpdateLock?.EnterWriteLock();
this.ProgressTracker = other.ProgressTracker.Copy();
lock (other.continuationTokenLock)
{
foreach (var subDirTransfer in other.subDirectories)
{
this.subDirectories.Enqueue(subDirTransfer);
}
foreach (var subDirTransfer in other.ongoingSubDirTransfers)
{
var newSubDirTransfer = new SubDirectoryTransfer(subDirTransfer.Key);
this.ongoingSubDirTransfers.TryAdd(newSubDirTransfer, new object());
}
// copy transfers
this.subTransfers = other.subTransfers.Copy();
this.enumerationStarted = other.enumerationStarted;
}
this.subTransfers.OverallProgressTracker.Parent = this.ProgressTracker;
other.progressUpdateLock?.ExitWriteLock();
}
internal void AddOngoingSubDirTransfer(SubDirectoryTransfer directoryTransfer)
{
lock (this.journalLock)
{
long offset = this.SearchFreeOffset();
directoryTransfer.Journal = this;
directoryTransfer.StreamJournalOffset = offset + 2 * sizeof(long);
this.stream.Position = offset + 2 * sizeof(long);
#if BINARY_SERIALIZATION
this.formatter.Serialize(this.stream, directoryTransfer);
#else
this.WriteObject(this.subDirTransferSerializer, directoryTransfer);
#endif
long continuationTokenOffset = offset + 2 * sizeof(long) + 4096;
this.stream.Position = continuationTokenOffset;
directoryTransfer.ListContinuationToken.Journal = this;
directoryTransfer.ListContinuationToken.StreamJournalOffset = continuationTokenOffset;
#if BINARY_SERIALIZATION
this.formatter.Serialize(this.stream, directoryTransfer.ListContinuationToken);
#else
this.WriteObject(this.continuationTokenSerializer, directoryTransfer.ListContinuationToken);
#endif
if (this.OngoingSubDirTransferChunkHead == 0)
{
if (this.OngoingSubDirTransferChunkTail != 0)
{
#if !NO_FILEFORMAT_EX
throw new FileFormatException(Resources.RestartableLogCorrupted);
#else
throw new InvalidOperationException(Resources.RestartableLogCorrupted);
#endif
}
this.OngoingSubDirTransferChunkHead = offset;
this.OngoingSubDirTransferChunkTail = offset;
this.stream.Position = this.OngoingSubDirTransferChunkHead;
this.stream.Write(BitConverter.GetBytes(0L), 0, sizeof(long));
this.stream.Write(BitConverter.GetBytes(0L), 0, sizeof(long));
}
else
{
this.stream.Position = this.OngoingSubDirTransferChunkTail + sizeof(long);
this.stream.Write(BitConverter.GetBytes(offset), 0, sizeof(long));
this.stream.Position = offset;
this.stream.Write(BitConverter.GetBytes(this.OngoingSubDirTransferChunkTail), 0, sizeof(long));
this.stream.Write(BitConverter.GetBytes(0L), 0, sizeof(long));
this.OngoingSubDirTransferChunkTail = offset;
}
this.WriteJournalHead();
this.stream.Flush();
}
}
internal void RemoveSubDirTransfer(SubDirectoryTransfer directoryTransfer)
{
lock (this.journalLock)
{
long offset = directoryTransfer.StreamJournalOffset - 2 * sizeof(long);
this.FreeChunk(offset, ref this.OngoingSubDirTransferChunkHead, ref this.OngoingSubDirTransferChunkTail);
this.stream.Flush();
}
}
public SubDirectoryTransfer(SubDirectoryTransfer other)
{
this.relativePath = other.relativePath;
this.enumerateContinuationToken = other.enumerateContinuationToken;
}
private async void WaitOnSubDirectoryListTask(Task directoryListTask, SubDirectoryTransfer subDirTransfer)
{
if (null == directoryListTask)
{
this.SignalSubDirTaskDecrement();
}
else
{
bool shouldStopOthers = false;
bool errorHappened = false;
try
{
await directoryListTask;
}
catch (OperationCanceledException)
{
// Ingore this exception, there's other place reporting such kind of exception when cancellation is triggered.
errorHappened = true;
}
catch (Exception ex)
{
if (ex is TransferException)
{
this.enumerateException = ex;
}
else if ((null != ex.InnerException) &&
(ex.InnerException is OperationCanceledException))
{
// Ingore this exception, there's other place reporting such kind of exception when cancellation is triggered.
errorHappened = true;
}
else
{
this.enumerateException = new TransferException(
TransferErrorCode.FailToEnumerateDirectory,
string.Format(CultureInfo.CurrentCulture,
Resources.EnumerateDirectoryException,
subDirTransfer.Source.Instance.ConvertToString()),
ex);
}
shouldStopOthers = true;
errorHappened = true;
}
finally
{
if (!errorHappened)
{
if (null != this.Journal)
{
this.Journal.RemoveSubDirTransfer(subDirTransfer);
}
else
{
object subDirTransferValue = null;
this.ongoingSubDirTransfers.TryRemove(subDirTransfer, out subDirTransferValue);
}
}
this.SignalSubDirTaskDecrement();
}
if (shouldStopOthers)
{
this.cancellationTokenSource?.Cancel();
}
}
}
private async Task DoEnumerationAndTransferAsync(TransferScheduler scheduler, CancellationToken cancellationToken)
{
await Task.Yield();
this.Resume(scheduler, cancellationToken);
if (0 == this.enumerationStarted)
{
// Got nothing from checkpoint, start directory transfer from the very beginning.
var subDirTransfer = new SubDirectoryTransfer(this, "");
lock (this.continuationTokenLock)
{
if (null == this.Journal)
{
this.ongoingSubDirTransfers.TryAdd(subDirTransfer, new object());
}
this.enumerationStarted = 1;
}
if (null != this.Journal)
{
this.Journal.AddOngoingSubDirTransfer(subDirTransfer);
this.Journal?.UpdateJournalItem(this);
}
this.ScheduleSubDirectoryTransfer(
subDirTransfer,
this.cancellationTokenSource.Token,
null,
-1);
}
bool gotDirectory = true;
while (true)
{
Utils.CheckCancellation(this.cancellationTokenSource.Token);
if (!gotDirectory)
{
newAddSubDirResetEventSlim.Wait(cancellationToken);
newAddSubDirResetEventSlim.Reset();
}
// Check whether theres ongoing subdirectory listing thread.
bool listCompleted = (1 == Interlocked.Read(ref this.outstandingTasks));
string subDirRelativePath = null;
if (null != this.Journal)
{
subDirRelativePath = this.Journal.PeekSubDirTransfer();
}
else
{
this.subDirectories.TryPeek(out subDirRelativePath);
}
if (string.IsNullOrEmpty(subDirRelativePath))
{
if (listCompleted)
{
// There's no ongoing subdirectory listing thread,
// and no subdirectory pending on listing
// This means that the whole listing is completed.
break;
}
gotDirectory = false;
continue;
}
else
{
gotDirectory = true;
Utils.CheckCancellation(this.cancellationTokenSource.Token);
SubDirectoryTransfer subDirTransfer = new SubDirectoryTransfer(this, subDirRelativePath);
this.ScheduleSubDirectoryTransfer(
subDirTransfer,
this.cancellationTokenSource.Token,
() =>
{
if (null != this.Journal)
{
this.Journal.AddOngoingSubDirTransfer(subDirTransfer);
this.Journal.RemoveFirstSubDirTransfer();
}
else
{
this.ongoingSubDirTransfers.TryAdd(subDirTransfer, new object());
this.subDirectories.TryDequeue(out subDirRelativePath);
}
},
-1);
}
}
}
public IEnumerable <SubDirectoryTransfer> ListSubDirTransfers()
{
long currentOffset = this.OngoingSubDirTransferChunkHead;
bool shouldBreak = false;
while (true)
{
SubDirectoryTransfer transfer = null;
lock (this.journalLock)
{
if (0 == this.OngoingSubDirTransferChunkHead)
{
shouldBreak = true;
}
else
{
this.stream.Position = currentOffset;
long previousUsedChunk = this.ReadLong();
long nextUsedChunk = this.ReadLong();
if (0 == previousUsedChunk)
{
if (this.OngoingSubDirTransferChunkHead != currentOffset)
{
#if !NO_FILEFORMAT_EX
throw new FileFormatException(Resources.RestartableLogCorrupted);
#else
throw new InvalidOperationException(Resources.RestartableLogCorrupted);
#endif
}
}
else
{
if (this.OngoingSubDirTransferChunkHead == currentOffset)
{
#if !NO_FILEFORMAT_EX
throw new FileFormatException(Resources.RestartableLogCorrupted);
#else
throw new InvalidOperationException(Resources.RestartableLogCorrupted);
#endif
}
}
try
{
#if BINARY_SERIALIZATION
transfer = this.formatter.Deserialize(this.stream) as SubDirectoryTransfer;
#else
transfer = this.ReadObject(this.subDirTransferSerializer) as SubDirectoryTransfer;
#endif
}
catch
{
#if !NO_FILEFORMAT_EX
throw new FileFormatException(Resources.RestartableLogCorrupted);
#else
throw new InvalidOperationException(Resources.RestartableLogCorrupted);
#endif
}
if (null == transfer)
{
#if !NO_FILEFORMAT_EX
throw new FileFormatException(Resources.RestartableLogCorrupted);
#else
throw new InvalidOperationException(Resources.RestartableLogCorrupted);
#endif
}
transfer.StreamJournalOffset = currentOffset + 2 * sizeof(long);
transfer.Journal = this;
this.stream.Position = transfer.StreamJournalOffset + 4096;
#if BINARY_SERIALIZATION
transfer.ListContinuationToken = this.formatter.Deserialize(this.stream) as SerializableListContinuationToken;
#else
transfer.ListContinuationToken = this.ReadObject(this.continuationTokenSerializer) as SerializableListContinuationToken;
#endif
transfer.ListContinuationToken.Journal = this;
transfer.ListContinuationToken.StreamJournalOffset = transfer.StreamJournalOffset + 4096;
if (0 == nextUsedChunk)
{
if (this.OngoingSubDirTransferChunkTail != currentOffset)
{
#if !NO_FILEFORMAT_EX
throw new FileFormatException(Resources.RestartableLogCorrupted);
#else
throw new InvalidOperationException(Resources.RestartableLogCorrupted);
#endif
}
shouldBreak = true;
}
else
{
if (this.OngoingSubDirTransferChunkTail == currentOffset)
{
#if !NO_FILEFORMAT_EX
throw new FileFormatException(Resources.RestartableLogCorrupted);
#else
throw new InvalidOperationException(Resources.RestartableLogCorrupted);
#endif
}
}
currentOffset = nextUsedChunk;
}
}
if (null != transfer)
{
yield return(transfer);
}
if (shouldBreak)
{
yield break;
}
}
}