private async Task ApplyChangesParallel(OneDriveModel.OneDriveChangeCollection changes, string accessToken, int maxParallelCount)
{
List <Task> tasks = new List <Task>();
string wwwroot = _environment.WebRootPath;
using (var sem = new SemaphoreSlim(initialCount: maxParallelCount, maxCount: maxParallelCount))
using (var filesPerSecLimiter = new RateLimiter(MaxFilesPerSeconds, TimeSpan.FromSeconds(1)))
{
foreach (OneDriveModel.OneDriveChange change in changes)
{
tasks.Add(Task.Run(async() =>
{
try
{
await sem.WaitAsync();
await filesPerSecLimiter.ThrottleAsync();
bool applied = await ProcessChangeWithRetry(change, accessToken, wwwroot);
if (applied)
{
Interlocked.Increment(ref _successCount);
}
else
{
Interlocked.Increment(ref _failedCount);
}
}
finally
{
sem.Release();
}
}));
}
await Task.WhenAll(tasks);
}
}
internal async Task ApplyChangesCore(DropboxDeployInfo info, bool useOAuth20)
{
string parent = info.Path.TrimEnd('/') + '/';
DateTime updateMessageTime = DateTime.UtcNow;
var sem = new SemaphoreSlim(MaxConcurrentRequests, MaxConcurrentRequests);
var rateLimiter = new RateLimiter(MaxFilesPerSecs * 10, TimeSpan.FromSeconds(10));
var tasks = new List<Task>();
foreach (DropboxEntryInfo entry in info.Deltas)
{
if (CanIgnoreEntry(parent, entry))
{
continue;
}
var path = entry.Path;
if (entry.IsDeleted)
{
SafeDelete(parent, path);
Interlocked.Increment(ref _successCount);
}
else if (entry.IsDirectory)
{
SafeCreateDir(parent, path);
Interlocked.Increment(ref _successCount);
}
else
{
DateTime modified;
if (DateTime.TryParse(entry.Modified, out modified))
{
modified = modified.ToUniversalTime();
}
else
{
modified = DateTime.UtcNow;
}
if (!IsFileChanged(parent, path, modified))
{
LogInfo("file unchanged {0}", path);
Interlocked.Increment(ref _successCount);
continue;
}
try
{
var task = Task.Run(async () =>
{
await sem.WaitAsync();
try
{
await rateLimiter.ThrottleAsync();
using (HttpClient client = useOAuth20 ? CreateDropboxV2HttpClient(DropboxApiContentUri, info.Token) :
CreateDropboxHttpClient(info, entry))
{
await ProcessFileAsync(client, entry.Path, parent, modified);
}
}
finally
{
Interlocked.Increment(ref _fileCount);
sem.Release();
}
});
tasks.Add(task);
}
catch (Exception ex)
{
LogError("{0}", ex);
throw;
}
}
}
await Task.WhenAll(tasks);
}
public async Task TestDropboxRateLimiter(int limit, int total)
{
var interval = 1;
var rateLimiter = new RateLimiter(limit, TimeSpan.FromSeconds(interval));
var duration = TimeSpan.FromSeconds(total / limit - interval - 0.5);
var start = DateTime.Now;
while (--total > 0)
{
await rateLimiter.ThrottleAsync();
}
// Assert
var end = DateTime.Now;
Assert.True((end - start) >= duration, (end - start) + "<" + duration);
}