private async Task <ReadDeltaResult> ReadDelta(ISyncContext context, IDocumentStorageProvider storage, IClient dropbox, Func <Task> progressAction)
{
Entries delta;
// we will load local state lazily upon first use
var localState = new Lazy <LocalState>(() => new LocalState(storage.StateSnapshot));
var numChangesProcessed = 0;
var cursor = context.Cursor;
do
{
delta = await dropbox.Core.Metadata.DeltaAsync(cursor, path_prefix : _options.Path, include_media_info : true);
Debug.Assert(delta != null);
Debug.Assert(delta.entries != null);
// if this is a reset notification (or we start with an empty cursor),
// we should reset local state and then restore it by iterating through delta
if (delta.reset || String.IsNullOrEmpty(cursor))
{
localState.Value.Reset();
numChangesProcessed++;
}
foreach (var entry in delta.entries.Where(IsDeleteOrAddFileDelta))
{
numChangesProcessed++;
var path = entry.Item1.ToLowerInvariant();
if (entry.Item2 == null)
{
localState.Value.RemoveAtPath(path);
}
else if (!entry.Item2.is_dir)
{
localState.Value.AddAtPath(path, entry.Item2);
}
}
cursor = delta.cursor;
if (progressAction != null)
{
await progressAction();
}
} while (delta.has_more);
context.Cursor = cursor;
return(new ReadDeltaResult
{
ChangesProcessed = numChangesProcessed,
PendingChanges = localState.IsValueCreated
? localState.Value.ChangeList.Values
: null // no changes to process
});
}
public async Task <SyncResult> Sync(
ISyncContext syncContext,
IDocumentStorageProvider storage,
Func <Task> progressAction = null)
{
if (syncContext == null)
{
throw new ArgumentNullException("syncContext");
}
if (syncContext.AccessToken == null)
{
throw new ArgumentException("An access token must be provided");
}
if (String.IsNullOrEmpty(syncContext.AccessToken))
{
throw new InvalidOperationException("Specified data context does not have valid token.");
}
var dropbox = _clientFactory(syncContext.AccessToken);
// read delta using last indexing state
// NB: this may update sync context, which should be persisted by the caller to enable reuse of the cursor
var readResult = await ReadDelta(syncContext, storage, dropbox, progressAction);
Debug.Assert(readResult != null);
if (readResult.ChangesProcessed <= 0)
{
return(new SyncResult());
}
Debug.Assert(readResult.PendingChanges != null);
// apply accumulated changes
var applyResult = await ApplyPendingChanges(storage, dropbox, readResult.PendingChanges, progressAction);
return(applyResult.ToIndexingResult(readResult.ChangesProcessed));
}
/// <summary>
/// Process pending changes - download thumbnails and update destination with batch document operations.
/// </summary>
private async Task <ApplyChangesResult> ApplyPendingChanges(
IDocumentStorageProvider storage,
IClient dropbox,
ICollection <PendingChange> pendingChanges,
Func <Task> batchProgressAction)
{
var result = new ApplyChangesResult();
// filter changes according to specified options
var filteredChanges = pendingChanges
.Where(c =>
c.Change == DocumentAction.Delete ||
!_options.WithThumbnailsOnly || c.Meta.thumb_exists ||
_options.IncludeSharedFolders || String.IsNullOrEmpty(c.Meta.parent_shared_folder_id)
);
var itemsProcessed = 0;
// split accumulated changes to smaller batches to enable progress reporting and status checks
foreach (var batch in filteredChanges.Chunkify(_options.ItemsPerBatch))
{
Debug.WriteLine("Processing next batch of changes [{0} / {1}]", itemsProcessed, pendingChanges.Count);
// these will buffer metadata updates
var documentsToDelete = new List <Document>();
var documentsToUpdate = new List <Document>();
var documentsToAdd = new List <Document>();
// create a list for thumbnail download tasks
var thumbsToDownload = new List <PendingChange>();
// sort through updates and adds
foreach (var change in batch)
{
if (change.Change != DocumentAction.Delete &&
_options.DownloadThumbnails && change.Meta.thumb_exists)
{
thumbsToDownload.Add(change);
}
else
{
var document = change.ToDocument();
switch (change.Change)
{
case DocumentAction.Add:
documentsToAdd.Add(document);
break;
case DocumentAction.Update:
documentsToUpdate.Add(document);
break;
case DocumentAction.Delete:
document.FilePath = change.DeletedFilePath;
documentsToDelete.Add(document);
break;
}
}
}
var downloadCount = await DownloadThumbnails(dropbox, thumbsToDownload, documentsToAdd, documentsToUpdate);
result.Skipped += thumbsToDownload.Count - downloadCount;
// now flush accumulated changes
if (documentsToDelete.Count > 0)
{
Debug.WriteLine("Deleting documents [{0}]", documentsToDelete.Count);
result.Deleted += await storage.DeleteAsync(documentsToDelete);
}
if (documentsToAdd.Count > 0)
{
Debug.WriteLine("Adding documents [{0}]", documentsToUpdate.Count + documentsToAdd.Count);
result.Added += await storage.AddAsync(documentsToAdd);
}
if (documentsToUpdate.Count > 0)
{
Debug.WriteLine("Updating documents [{0}]", documentsToUpdate.Count + documentsToAdd.Count);
result.Updated += await storage.UpdateAsync(documentsToUpdate);
}
if (batchProgressAction != null)
{
await batchProgressAction();
}
itemsProcessed += _options.ItemsPerBatch;
}
return(result);
}
