public DapSync(DapSource dapSource)
{
dap = dapSource;
sync_limiter = new RateLimiter (RateLimitedSync);
BuildPreferences ();
BuildSyncLists ();
UpdateSensitivities ();
}
public void KeepsRateLimitersSeparate()
{
var rateLimiter = new RateLimiter <int>(new[]
{
new KeyValuePair <int, RateLimit>(1, new RateLimit(1, TimeSpan.FromSeconds(1))),
new KeyValuePair <int, RateLimit>(2, new RateLimit(1, TimeSpan.FromSeconds(1)))
});
rateLimiter.CanExecute(1).Should().BeTrue();
rateLimiter.CanExecute(1).Should().BeFalse();
rateLimiter.CanExecute(2).Should().BeTrue();
rateLimiter.CanExecute(2).Should().BeFalse();
}
private readonly IRateLimiter <int> _rateLimiter3; // CanExecute will always return false and tokens will not be updated on any iteration
public Multi_RateLimiter()
{
_rateLimiter1 = RateLimiter
.WithRateLimits(new[] { new KeyValuePair <int, RateLimit>(1, new RateLimit(Int32.MaxValue, TimeSpan.FromMinutes(1))) })
.Build();
_rateLimiter2 = RateLimiter
.WithRateLimits(new[] { new KeyValuePair <int, RateLimit>(1, new RateLimit(Int32.MaxValue, TimeSpan.MaxValue)) })
.Build();
_rateLimiter3 = RateLimiter
.WithRateLimits(new[] { new KeyValuePair <int, RateLimit>(1, new RateLimit(0, TimeSpan.FromMinutes(1))) })
.Build();
}
private void InitializeTyrant()
{
_haveMessagesReceived = 0;
_startTime = Stopwatch.GetTimestamp();
RateLimiter = new RateLimiter();
_uploadRateForRecip = MarketRate;
_lastRateReductionTime = DateTime.Now;
_lastMeasuredDownloadRate = 0;
_maxObservedDownloadSpeed = 0;
RoundsChoked = 0;
RoundsUnchoked = 0;
}
public void ThrowsIfDisposed()
{
var updates = new Subject <IEnumerable <KeyValuePair <int, RateLimit> > >();
var rateLimiter = RateLimiter
.WithRateLimits(updates)
.Build();
rateLimiter.Dispose();
Action action = () => rateLimiter.CanExecute(1);
action.Should().Throw <ObjectDisposedException>();
}
public ErrorReportingTest(LogValidatingFixture fixture)
{
_testId = IdGenerator.FromDateTime();
_server = GetTestServer <ErrorReportingTestApplication>();
_client = _server.CreateClient();
_fixture = fixture;
// The rate limiter instance is static and only set once. If we do not reset it at the
// beginning of each test the qps will not change. This is dependent on the tests not
// running in parallel, which they don't.
RateLimiter.Reset();
}
public void ThrowsIfDisposed()
{
var updates = new Subject <RateLimit>();
var rateLimiter = RateLimiter
.WithRateLimit(updates)
.Build();
rateLimiter.Dispose();
Action action = () => rateLimiter.CanExecute();
action.Should().Throw <ObjectDisposedException>();
}
public void WaitToProceed_ZeroWait_DoesntWait()
{
_unitUnderTest = new RateLimiter(1, TimeSpan.FromSeconds(0));
Stopwatch timer = Stopwatch.StartNew();
_unitUnderTest.WaitToProceed();
_unitUnderTest.WaitToProceed();
_unitUnderTest.WaitToProceed();
timer.Stop();
Assert.IsTrue(timer.Elapsed.TotalSeconds < 1);
}
/// <summary>
/// Initializes a new instance of the <see cref="TcpTransportProtocol"/> class that will listen on the specified port.
/// </summary>
/// <param name="port">Port to listen on for incoming connections.</param>
/// <param name="bindToNextAvailablePort">If the specified port is in use, attempts to bind to the next available port.</param>
/// <param name="localBindAddress">The local address to use for connections.</param>
/// <param name="acceptConnectionHandler">Invoked to accept the connection.</param>
public TcpTransportProtocol(
int port,
bool bindToNextAvailablePort,
IPAddress localBindAddress,
Action <AcceptConnectionEventArgs> acceptConnectionHandler)
{
streams = new ConcurrentBag <TcpTransportStream>();
AcceptConnectionHandler = acceptConnectionHandler;
this.bindToNextAvailablePort = bindToNextAvailablePort;
Port = port;
LocalBindAddress = localBindAddress;
LocalConection = new LocalTcpConnectionDetails(port, null, localBindAddress);
RateLimiter = new RateLimiter();
}
public void Properties()
{
const int count = 3;
var duration = TimeSpan.FromSeconds(3);
var rateLimiter = new RateLimiter
{
Count = count,
Duration = duration
};
Assert.Equal(count, rateLimiter.Count);
Assert.Equal(duration, rateLimiter.Duration);
}
public void TestSimpleRateUpdate()
{
var limiter = RateLimiter.Create(5.0, TimeSpan.FromSeconds(5));
limiter.GetRate().Should().Be(5.0);
limiter.SetRate(10.0);
limiter.GetRate().Should().Be(10.0);
limiter.Invoking(x => x.SetRate(0.0))
.ShouldThrow <ArgumentException>();
limiter.Invoking(x => x.SetRate(-10.0))
.ShouldThrow <ArgumentException>();
}
public void RequestingHigherCountUsesUpMoreTokens(int count)
{
var rateLimiter = new RateLimiter <int>(new[]
{
new KeyValuePair <int, RateLimit>(1, new RateLimit(1000, TimeSpan.MaxValue))
});
for (var i = 0; i < 1000 / count; i++)
{
rateLimiter.CanExecute(1, count).Should().BeTrue();
}
rateLimiter.CanExecute(1).Should().BeFalse();
}
public void InstanceCountIsSetCorrectly(int instanceCount)
{
var rateLimit = new RateLimit(1, TimeSpan.FromSeconds(1));
var rateLimiter = RateLimiter
.WithRateLimits(new[]
{
new KeyValuePair <int, RateLimit>(1, rateLimit)
})
.WithInstanceCount(instanceCount)
.Build();
rateLimiter.InstanceCount.Should().Be(instanceCount);
}
private void CreateRateLimiters()
{
var downloader = new RateLimiter();
DownloadLimiter = new RateLimiterGroup();
DownloadLimiter.Add(new PauseLimiter(this));
DownloadLimiter.Add(downloader);
var uploader = new RateLimiter();
UploadLimiter = new RateLimiterGroup();
UploadLimiter.Add(new PauseLimiter(this));
UploadLimiter.Add(uploader);
}
public void CanSetRateLimit()
{
var rateLimit = new RateLimit(1, TimeSpan.FromSeconds(1));
var rateLimiter = new RateLimiter(rateLimit);
rateLimiter.GetRateLimit().Should().Be(rateLimit);
var newRateLimit = new RateLimit(2, TimeSpan.FromMinutes(1));
rateLimiter.SetRateLimit(newRateLimit);
rateLimiter.GetRateLimit().Should().Be(newRateLimit);
}
public void WaitToProceed_PartialSecWait_Waits()
{
_unitUnderTest = new RateLimiter(1, TimeSpan.FromSeconds(1.5));
Stopwatch timer = Stopwatch.StartNew();
_unitUnderTest.WaitToProceed();//will not wait for first call
_unitUnderTest.WaitToProceed();
_unitUnderTest.WaitToProceed();
timer.Stop();
Assert.IsTrue(timer.Elapsed.TotalSeconds > 1.5);
}
public void WaitToProceed_ValidWaitTime_Waits()
{
_unitUnderTest = new RateLimiter(1, TimeSpan.FromMilliseconds(700));
var timer = Stopwatch.StartNew();
_unitUnderTest.WaitToProceed();//will not wait for first call
_unitUnderTest.WaitToProceed();
_unitUnderTest.WaitToProceed();
timer.Stop();
Assert.IsTrue(timer.Elapsed.TotalSeconds > 1);
}
public TraceSnippetsTests()
{
_testId = IdGenerator.FromDateTime();
_server = GetTestServer <TraceTestApplication.Startup>();
_client = _server.CreateClient();
_startTime = Timestamp.FromDateTime(DateTime.UtcNow);
// The rate limiter instance is static and only set once. If we do not reset it at the
// beginning of each tests the qps will not change. This is dependent on the tests not
// running in parallel.
RateLimiter.Reset();
}
public async Task TestSimpleWithWait()
{
var limiter = RateLimiter.Create(5.0, stopwatch);
await limiter.Acquire(); // R0.00
stopwatch.SleepMillis(200); // U0.20, we are read for the next request
await limiter.Acquire(); // R0.00, granted immediately
await limiter.Acquire(); // R0.20
stopwatch.Events.Should()
.ContainInOrder("R0.00", "U0.20", "R0.00", "R0.20");
}
static void Main(string[] args)
{
NLogLogger.Use();
ILog log = Logger.Get <Program>();
var actor = new RpcActor();
actor.Bootup();
var rateLimiter = new RateLimiter();
var helloService = new HelloService(actor, rateLimiter);
var calcService = new CalcService(actor);
var orderService = new OrderService(actor, rateLimiter);
actor.RegisterRpcService(helloService);
actor.RegisterRpcService(calcService);
actor.RegisterRpcService(orderService);
while (true)
{
try
{
string text = Console.ReadLine().ToLowerInvariant();
if (text == "quit" || text == "exit")
{
break;
}
else
{
int times = 0;
if (int.TryParse(text, out times))
{
for (int i = 0; i < times; i++)
{
NotifyOrderChanged(log, orderService);
}
}
}
}
catch (Exception ex)
{
log.Error(ex.Message, ex);
}
}
actor.Shutdown();
}
public async Task CanTrace_StressTest()
{
var source = new CancellationTokenSource();
try
{
// Create a rate limiter that allows 1 QPS
var rateLimiter = RateLimiter.GetInstance(1);
int canTraceCounter = 0;
int threads = 10;
// Start a timer to get the total time from tasks starting to ending.
Stopwatch timer = Stopwatch.StartNew();
// Start 10 threads to hit the rate limiter
Task[] tasks = new Task[threads];
for (int i = 0; i < threads; i++)
{
tasks[i] = Task.Run(async() =>
{
while (!source.IsCancellationRequested)
{
if (rateLimiter.CanTrace())
{
Interlocked.Increment(ref canTraceCounter);
}
await Task.Yield();
}
});
}
// Set a timeout to 2.1 seconds which should allow 2 traces unless control
// of the tasks does not return right away. To ensure a proper trace count
// use the number of seconds to check the trace count.
await Task.Delay(2100);
var elapsedSeconds = Math.Floor(timer.Elapsed.TotalSeconds);
// Allow for the trace count to be one lower than expected to account
// for the elapsed time being on the second mark or close and the
// limiter not getting a final request.
Assert.InRange(canTraceCounter, elapsedSeconds - 1, elapsedSeconds);
}
finally
{
source.Cancel();
source.Dispose();
}
}
internal RateLimitedIndexOutput(RateLimiter rateLimiter, IndexOutput @delegate)
{
// TODO should we make buffer size configurable
if (@delegate is BufferedIndexOutput)
{
BufferedDelegate = (BufferedIndexOutput)@delegate;
this.@delegate = @delegate;
}
else
{
this.@delegate = @delegate;
BufferedDelegate = null;
}
this.RateLimiter = rateLimiter;
}
internal RateLimitedIndexOutput(RateLimiter rateLimiter, IndexOutput @delegate)
{
// TODO should we make buffer size configurable
if (@delegate is BufferedIndexOutput)
{
BufferedDelegate = (BufferedIndexOutput)@delegate;
this.@delegate = @delegate;
}
else
{
this.@delegate = @delegate;
BufferedDelegate = null;
}
this.RateLimiter = rateLimiter;
}
public void OnInstanceCountChangedIsSetCorrectly()
{
var instanceCountChanges = new List <InstanceCountChangedNotification>();
Action <InstanceCountChangedNotification> onInstanceCountChanged = instanceCountChanges.Add;
var rateLimiter = RateLimiter
.WithRateLimit(new RateLimit(10, TimeSpan.FromSeconds(1)))
.OnInstanceCountChanged(onInstanceCountChanged)
.Build();
rateLimiter.InstanceCount = 2;
instanceCountChanges.Should().ContainSingle();
}
protected BaseWindow()
{
this.PreventFocusLoss();
HandleDisposableViewModel();
// Unsubscribe not needed - same class
DataContextChanged += BaseWindow_DataContextChanged;
Closing += BaseWindow_Closing;
Closed += BaseWindow_Closed;
Loaded += BaseWindow_Loaded;
// KeyDown += BaseWindow_KeyDown;
ContentRendered += BaseWindow_ContentRenderedAsync;
_sizeChangedRateLimiter = new RateLimiter(SynchronizationContext.Current);
}
public async Task TestInfinityWarmUpTimeElapsed()
{
var limiter = RateLimiter.Create(double.PositiveInfinity, TimeSpan.FromSeconds(10), stopwatch);
stopwatch.Instant += 1000000;
limiter.SetRate(1.0);
for (var i = 0; i < 5; i++)
{
await limiter.Acquire();
}
stopwatch.Events.Should()
.ContainInOrder("R0.00", "R1.00", "R1.00", "R1.00", "R1.00");
}
public void OnFailureIsSetCorrectly()
{
var failureCount = 0;
Action onfailure = () => failureCount++;
var rateLimiter = RateLimiter
.WithRateLimit(new RateLimit(0, TimeSpan.FromSeconds(1)))
.OnFailure(onfailure)
.Build();
rateLimiter.CanExecute().Should().BeFalse();
failureCount.Should().Be(1);
}
public void OnSuccessIsSetCorrectly()
{
var successCount = 0;
Action onSuccess = () => successCount++;
var rateLimiter = RateLimiter
.WithRateLimit(new RateLimit(1, TimeSpan.FromSeconds(1)))
.OnSuccess(onSuccess)
.Build();
rateLimiter.CanExecute().Should().BeTrue();
successCount.Should().Be(1);
}
public void CanSetRateLimit()
{
var rateLimit = new RateLimit(1, TimeSpan.FromSeconds(1));
var rateLimiter = new RateLimiter <int>(new[]
{
new KeyValuePair <int, RateLimit>(1, rateLimit)
});
var newRateLimit = new RateLimit(2, TimeSpan.FromMinutes(1));
rateLimiter.SetRateLimit(1, newRateLimit);
rateLimiter.GetRateLimit(1).Should().Be(newRateLimit);
}
public void Example()
{
var rateLimiter = new RateLimiter(new RateLimitStore(new MemoryDistributedCache(options)));
string identity = "identity";
var rnd = new Random(Guid.NewGuid().GetHashCode());
RateLimitResult result;
var rateQuota = new RateQuota(100, 50, RateLimitPeriod.Min);
for (int i = 0; i < 50; i++)
{
result = rateLimiter.RateLimit(rateQuota, identity, 30);
Console.WriteLine(result);
Thread.Sleep(rnd.Next(1000));
}
}
public void If_hard_limit_is_exceeded_Then_do_not_rate_limit_and_do_not_run()
{
var rateLimiter = new RateLimiter(1, 2);
rateLimiter.RateLimit(() => Task.Delay(100));
rateLimiter.RateLimit(() => Task.Delay(0));
var sw = Stopwatch.StartNew();
Task.WaitAll(Enumerable.Range(0, 100).Select(x => rateLimiter.RateLimit(_func)).ToArray());
sw.Stop();
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(120));
Assert.That(_taskHasRun, Is.False);
}
public void If_soft_limit_is_exhausted_Then_rate_limit_and_run()
{
var rateLimiter = new RateLimiter(1, 2);
rateLimiter.RateLimit(() => Task.Delay(20));
var sw = Stopwatch.StartNew();
var result = rateLimiter.RateLimit(_func).Result;
sw.Stop();
Assert.That(sw.ElapsedMilliseconds, Is.GreaterThan(10));
Assert.That(result, Is.True);
Assert.That(_taskHasRun, Is.True);
}
private void InitializeTyrant()
{
_haveMessagesReceived = 0;
_startTime = Stopwatch.GetTimestamp();
RateLimiter = new RateLimiter();
_uploadRateForRecip = MarketRate;
_lastRateReductionTime = DateTime.Now;
_lastMeasuredDownloadRate = 0;
_maxObservedDownloadSpeed = 0;
RoundsChoked = 0;
RoundsUnchoked = 0;
}
internal async Task ApplyChangesCore(DropboxDeployInfo info)
{
string parent = info.Path.TrimEnd('/') + '/';
DateTime updateMessageTime = DateTime.UtcNow;
var sem = new SemaphoreSlim(MaxConcurrentRequests, MaxConcurrentRequests);
var rateLimiter = new RateLimiter(MaxFilesPerSecs * 10, TimeSpan.FromSeconds(10));
var tasks = new List<Task>();
foreach (DropboxDeltaInfo delta in info.Deltas)
{
if (!delta.Path.StartsWith(parent, StringComparison.OrdinalIgnoreCase))
{
continue;
}
// Ignore .git and .hg files and folders
string pathWithSlash = delta.Path + "/";
if (pathWithSlash.IndexOf("/.git/", StringComparison.OrdinalIgnoreCase) != -1 ||
pathWithSlash.IndexOf("/.hg/", StringComparison.OrdinalIgnoreCase) != -1)
{
continue;
}
var path = delta.Path;
if (delta.IsDeleted)
{
SafeDelete(parent, path);
Interlocked.Increment(ref _successCount);
}
else if (delta.IsDirectory)
{
SafeCreateDir(parent, path);
Interlocked.Increment(ref _successCount);
}
else
{
DateTime modified;
if (DateTime.TryParse(delta.Modified, out modified))
{
modified = modified.ToUniversalTime();
}
else
{
modified = DateTime.UtcNow;
}
if (!IsFileChanged(parent, path, modified))
{
LogInfo("file unchanged {0}", path);
Interlocked.Increment(ref _successCount);
continue;
}
try
{
var task = Task.Run(async () =>
{
await sem.WaitAsync();
try
{
await rateLimiter.ThrottleAsync();
await ProcessFileAsync(info, delta, parent, path, modified);
}
finally
{
Interlocked.Increment(ref _fileCount);
sem.Release();
}
});
tasks.Add(task);
}
catch (Exception ex)
{
LogError("{0}", ex);
throw;
}
}
}
await Task.WhenAll(tasks);
}
private IRateLimiter GetRateLimter(Uri uri, long minCrawlDelayInMillisecs)
{
IRateLimiter rateLimiter;
_rateLimiterLookup.TryGetValue(uri.Authority, out rateLimiter);
if (rateLimiter == null && minCrawlDelayInMillisecs > 0)
{
rateLimiter = new RateLimiter(1, TimeSpan.FromMilliseconds(minCrawlDelayInMillisecs));
if (_rateLimiterLookup.TryAdd(uri.Authority, rateLimiter))
_logger.DebugFormat("Added new domain [{0}] with minCrawlDelayInMillisecs of [{1}] milliseconds", uri.Authority, minCrawlDelayInMillisecs);
else
_logger.WarnFormat("Unable to add new domain [{0}] with minCrawlDelayInMillisecs of [{1}] milliseconds", uri.Authority, minCrawlDelayInMillisecs);
}
return rateLimiter;
}
public void AddOrUpdateDomain(Uri uri, long minCrawlDelayInMillisecs)
{
if (uri == null)
throw new ArgumentNullException("uri");
if (minCrawlDelayInMillisecs < 1)
throw new ArgumentException("minCrawlDelayInMillisecs");
var delayToUse = Math.Max(minCrawlDelayInMillisecs, _defaultMinCrawlDelayInMillisecs);
if (delayToUse > 0)
{
var rateLimiter = new RateLimiter(1, TimeSpan.FromMilliseconds(delayToUse));
_rateLimiterLookup.AddOrUpdate(uri.Authority, rateLimiter, (key, oldValue) => rateLimiter);
_logger.DebugFormat("Added/updated domain [{0}] with minCrawlDelayInMillisecs of [{1}] milliseconds", uri.Authority, delayToUse);
}
}
private void CreateRateLimiters()
{
var downloader = new RateLimiter();
_downloadLimiter = new RateLimiterGroup();
_downloadLimiter.Add(new DiskWriterLimiter(DiskManager));
_downloadLimiter.Add(downloader);
var uploader = new RateLimiter();
_uploadLimiter = new RateLimiterGroup();
_downloadLimiter.Add(new DiskWriterLimiter(DiskManager));
_uploadLimiter.Add(uploader);
MainLoop.QueueTimeout(TimeSpan.FromSeconds(1), delegate
{
downloader.UpdateChunks(Settings.GlobalMaxDownloadSpeed, TotalDownloadSpeed);
uploader.UpdateChunks(Settings.GlobalMaxUploadSpeed, TotalUploadSpeed);
return !Disposed;
});
}
private void DatabaseSourceInitialize ()
{
InitializeTrackModel ();
current_filters_schema = CreateSchema<string[]> ("current_filters");
DatabaseSource filter_src = Parent as DatabaseSource ?? this;
foreach (IFilterListModel filter in filter_src.CreateFiltersFor (this)) {
AvailableFilters.Add (filter);
DefaultFilters.Add (filter);
}
reload_limiter = new RateLimiter (RateLimitedReload);
}
internal DiskManager(ClientEngine engine, PieceWriter.PieceWriter writer)
{
_bufferedReads = new Queue<BufferedIO>();
_bufferedWrites = new Queue<BufferedIO>();
_cache = new Cache<BufferedIO>(true).Synchronize();
_engine = engine;
ReadLimiter = new RateLimiter();
_readMonitor = new SpeedMonitor();
_writeMonitor = new SpeedMonitor();
WriteLimiter = new RateLimiter();
Writer = writer;
_loopTask = delegate
{
if (Disposed)
return;
while (this._bufferedWrites.Count > 0 &&
WriteLimiter.TryProcess(_bufferedWrites.Peek().InternalBuffer.Length/2048))
{
BufferedIO write;
lock (_bufferLock)
write = this._bufferedWrites.Dequeue();
try
{
PerformWrite(write);
_cache.Enqueue(write);
}
catch (Exception ex)
{
if (write.Manager != null)
SetError(write.Manager, Reason.WriteFailure, ex);
}
}
while (this._bufferedReads.Count > 0 && ReadLimiter.TryProcess(_bufferedReads.Peek().Count/2048))
{
BufferedIO read;
lock (_bufferLock)
read = this._bufferedReads.Dequeue();
try
{
PerformRead(read);
_cache.Enqueue(read);
}
catch (Exception ex)
{
if (read.Manager != null)
SetError(read.Manager, Reason.ReadFailure, ex);
}
}
};
IoLoop.QueueTimeout(TimeSpan.FromSeconds(1), delegate
{
if (Disposed)
return false;
_readMonitor.Tick();
_writeMonitor.Tick();
_loopTask();
return true;
});
}
public bool ShouldLimit(RateLimiter limiter)
{
return limiter?.ShouldLimit(this.workItem) ?? false;
}
internal async Task ApplyChangesCore(DropboxDeployInfo info, bool useOAuth20)
{
string parent = info.Path.TrimEnd('/') + '/';
DateTime updateMessageTime = DateTime.UtcNow;
var sem = new SemaphoreSlim(MaxConcurrentRequests, MaxConcurrentRequests);
var rateLimiter = new RateLimiter(MaxFilesPerSecs * 10, TimeSpan.FromSeconds(10));
var tasks = new List<Task>();
foreach (DropboxEntryInfo entry in info.Deltas)
{
if (CanIgnoreEntry(parent, entry))
{
continue;
}
var path = entry.Path;
if (entry.IsDeleted)
{
SafeDelete(parent, path);
Interlocked.Increment(ref _successCount);
}
else if (entry.IsDirectory)
{
SafeCreateDir(parent, path);
Interlocked.Increment(ref _successCount);
}
else
{
DateTime modified;
if (DateTime.TryParse(entry.Modified, out modified))
{
modified = modified.ToUniversalTime();
}
else
{
modified = DateTime.UtcNow;
}
if (!IsFileChanged(parent, path, modified))
{
LogInfo("file unchanged {0}", path);
Interlocked.Increment(ref _successCount);
continue;
}
try
{
var task = Task.Run(async () =>
{
await sem.WaitAsync();
try
{
await rateLimiter.ThrottleAsync();
using (HttpClient client = useOAuth20 ? CreateDropboxV2HttpClient(DropboxApiContentUri, info.Token) :
CreateDropboxHttpClient(info, entry))
{
await ProcessFileAsync(client, entry.Path, parent, modified);
}
}
finally
{
Interlocked.Increment(ref _fileCount);
sem.Release();
}
});
tasks.Add(task);
}
catch (Exception ex)
{
LogError("{0}", ex);
throw;
}
}
}
await Task.WhenAll(tasks);
}
private void ApplyChanges(DropboxHandler.DropboxInfo deploymentInfo)
{
DropboxDeployInfo info = deploymentInfo.DeployInfo;
Semaphore sem = new Semaphore(MaxConcurrentRequests, MaxConcurrentRequests);
List<Task> tasks = new List<Task>();
string parent = info.Path.TrimEnd('/') + '/';
DateTime updateMessageTime = DateTime.UtcNow;
int totals = info.Deltas.Count();
var rateLimiter = new RateLimiter(MaxFilesPerSecs * 10, TimeSpan.FromSeconds(10));
foreach (DropboxDeltaInfo delta in info.Deltas)
{
if (!delta.Path.StartsWith(parent, StringComparison.OrdinalIgnoreCase))
{
Interlocked.Increment(ref _successCount);
continue;
}
// Ignore .git and .hg files and folders
string pathWithSlash = delta.Path + "/";
if (pathWithSlash.StartsWith(parent + ".git/", StringComparison.OrdinalIgnoreCase) ||
pathWithSlash.StartsWith(parent + ".hg/", StringComparison.OrdinalIgnoreCase))
{
continue;
}
var path = delta.Path;
if (delta.IsDeleted)
{
SafeDelete(parent, path);
Interlocked.Increment(ref _successCount);
}
else if (delta.IsDirectory)
{
SafeCreateDir(parent, path);
Interlocked.Increment(ref _successCount);
}
else
{
DateTime modified = DateTime.Parse(delta.Modified).ToUniversalTime();
if (!IsFileChanged(parent, path, modified))
{
LogInfo("file unchanged {0}", path);
Interlocked.Increment(ref _successCount);
continue;
}
// throttle concurrent get file dropbox
sem.WaitOne();
Task task;
try
{
// Using ContinueWith instead of Then to avoid SyncContext deadlock in 4.5
task = GetFileAsync(info, delta).ContinueWith(t =>
{
rateLimiter.Throtte();
sem.Release();
if (!t.IsFaulted && !t.IsCanceled)
{
using (StreamInfo streamInfo = t.Result)
{
SafeWriteFile(parent, path, streamInfo.Stream, modified);
}
Interlocked.Increment(ref _successCount);
Interlocked.Increment(ref _fileCount);
}
else
{
Interlocked.Increment(ref _failedCount);
}
if (DateTime.UtcNow.Subtract(updateMessageTime) > TimeSpan.FromSeconds(5))
{
lock (_status)
{
_status.Open(deploymentInfo.TargetChangeset.Id).UpdateProgress(
String.Format(CultureInfo.CurrentUICulture,
_failedCount == 0 ? Resources.Dropbox_SynchronizingProgress : Resources.Dropbox_SynchronizingProgressWithFailure,
((_successCount + _failedCount) * 100) / totals,
totals,
_failedCount));
}
updateMessageTime = DateTime.UtcNow;
}
return t;
}).FastUnwrap();
}
catch (Exception ex)
{
sem.Release();
LogError("{0}", ex);
Task.WaitAll(tasks.ToArray(), _timeout);
throw;
}
tasks.Add(task);
}
}
if (!Task.WaitAll(tasks.ToArray(), _timeout))
{
throw new TimeoutException(RS.Format(Resources.Error_SyncDropboxTimeout, (int)_timeout.TotalSeconds));
}
}