private Task<StreamInfo> RetryGetFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, int retries)
{
// First retry is 1s, second retry is 20s assuming rate-limit
Thread.Sleep(retries == 1 ? 20000 : 1000);
return GetFileAsync(info, delta, retries - 1);
}
private Task<Stream> GetFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta)
{
var parameters = new Dictionary<string, string>
{
{ "oauth_consumer_key", info.ConsumerKey },
{ "oauth_signature_method", info.SignatureMethod },
{ "oauth_timestamp", info.TimeStamp },
{ "oauth_nonce", delta.Nonce },
{ "oauth_version", info.OAuthVersion },
{ "oauth_token", info.Token },
{ "oauth_signature", delta.Signature }
};
var sb = new StringBuilder();
foreach (var key in parameters.Keys)
{
if (sb.Length != 0)
{
sb.Append(',');
}
sb.AppendFormat("{0}=\"{1}\"", key, parameters[key]);
}
var client = new HttpClient();
client.BaseAddress = new Uri(DropboxApiContentUri);
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", sb.ToString());
return client.GetAsync(SandboxFilePath + delta.Path).Then(response =>
{
return response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync();
}).Finally(() => client.Dispose());
}
private DropboxDeployInfo GetDeployInfo(OAuthInfo oauth, string cursor = null)
{
List<DropboxDeltaInfo> deltas = new List<DropboxDeltaInfo>();
string timeStamp = GetUtcTimeStamp();
string oldCursor = cursor;
string newCursor = "";
while (true)
{
DeltaInfo delta = GetDeltaInfo(oauth, cursor);
newCursor = delta.cursor;
if (newCursor == oldCursor)
{
break;
}
foreach (EntryInfo info in delta.entries)
{
DropboxDeltaInfo item = new DropboxDeltaInfo { Path = info.path };
if (info.metadata == null || info.metadata.is_deleted || string.IsNullOrEmpty(info.metadata.path))
{
item.IsDeleted = true;
}
else
{
item.IsDirectory = info.metadata.is_dir;
if (!item.IsDirectory)
{
item.Modified = info.metadata.modified;
item.Nonce = GetNonce();
item.Signature = GetSignature(oauth, info.path, timeStamp, item.Nonce);
}
}
deltas.Add(item);
}
if (!delta.has_more)
{
break;
}
}
if (deltas.Count == 0)
{
throw new InvalidOperationException("the repo is up-to-date.");
}
return new DropboxDeployInfo
{
TimeStamp = timeStamp,
Token = oauth.Token,
ConsumerKey = oauth.ConsumerKey,
OAuthVersion = "1.0",
SignatureMethod = "HMAC-SHA1",
OldCursor = oldCursor,
NewCursor = newCursor,
Path = "/",
Deltas = deltas
};
}
private Task<StreamInfo> GetFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, int retries = MaxRetries)
{
var parameters = new Dictionary<string, string>
{
{ "oauth_consumer_key", info.ConsumerKey },
{ "oauth_signature_method", info.SignatureMethod },
{ "oauth_timestamp", info.TimeStamp },
{ "oauth_nonce", delta.Nonce },
{ "oauth_version", info.OAuthVersion },
{ "oauth_token", info.Token },
{ "oauth_signature", delta.Signature }
};
var sb = new StringBuilder();
foreach (var key in parameters.Keys)
{
if (sb.Length != 0)
{
sb.Append(',');
}
sb.AppendFormat("{0}=\"{1}\"", key, parameters[key]);
}
var client = new HttpClient();
client.BaseAddress = new Uri(DropboxApiContentUri);
client.Timeout = _timeout;
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", sb.ToString());
// Using ContinueWith instead of Then to avoid SyncContext deadlock in 4.5
var tcs = new TaskCompletionSource<Task<StreamInfo>>();
client.GetAsync(SandboxFilePath + DropboxPathEncode(delta.Path.ToLower(CultureInfo.CurrentCulture))).ContinueWith(t =>
{
if (t.IsFaulted)
{
using (client)
{
if (retries <= 0)
{
LogError("Get(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + t.Exception);
tcs.TrySetException(t.Exception.InnerExceptions);
}
else
{
LogError("Retry(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + t.Exception);
tcs.TrySetResult(RetryGetFileAsync(info, delta, retries));
}
}
}
else if (t.IsCanceled)
{
using (client)
{
tcs.TrySetCanceled();
}
}
else
{
try
{
HttpResponseMessage response = t.Result;
if ((int)response.StatusCode < 500 || retries <= 0)
{
response.EnsureSuccessStatusCode();
tcs.TrySetResult(GetStreamInfo(client, response, delta.Path));
if (retries < MaxRetries)
{
// Success due to retried
Interlocked.Increment(ref _retriedCount);
}
}
else
{
using (client)
{
using (response)
{
LogError("Retry(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + (int)response.StatusCode);
tcs.TrySetResult(RetryGetFileAsync(info, delta, retries));
}
}
}
}
catch (Exception ex)
{
LogError("Get(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + ex);
tcs.TrySetException(ex);
}
}
});
return tcs.Task.FastUnwrap();
}
private static string GetOAuthHeader(DropboxDeployInfo info, DropboxDeltaInfo delta)
{
var parameters = new Dictionary<string, string>
{
{ "oauth_consumer_key", info.ConsumerKey },
{ "oauth_signature_method", info.SignatureMethod },
{ "oauth_timestamp", info.TimeStamp },
{ "oauth_nonce", delta.Nonce },
{ "oauth_version", info.OAuthVersion },
{ "oauth_token", info.Token },
{ "oauth_signature", delta.Signature }
};
var sb = new StringBuilder();
foreach (var item in parameters)
{
if (sb.Length != 0)
{
sb.Append(',');
}
sb.AppendFormat("{0}=\"{1}\"", item.Key, item.Value);
}
return sb.ToString();
}
private Task<Stream> GetFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta)
{
var parameters = new Dictionary<string, string>
{
{ "oauth_consumer_key", info.ConsumerKey },
{ "oauth_signature_method", info.SignatureMethod },
{ "oauth_timestamp", info.TimeStamp },
{ "oauth_nonce", delta.Nonce },
{ "oauth_version", info.OAuthVersion },
{ "oauth_token", info.Token },
{ "oauth_signature", delta.Signature }
};
var sb = new StringBuilder();
foreach (var key in parameters.Keys)
{
if (sb.Length != 0)
{
sb.Append(',');
}
sb.AppendFormat("{0}=\"{1}\"", key, parameters[key]);
}
var client = new HttpClient();
client.BaseAddress = new Uri(DropboxApiContentUri);
client.Timeout = _timeout;
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", sb.ToString());
// Using ContinueWith instead of Then to avoid SyncContext deadlock in 4.5
TaskCompletionSource<Task<Stream>> tcs = new TaskCompletionSource<Task<Stream>>();
client.GetAsync(SandboxFilePath + delta.Path.ToLower()).ContinueWith(t =>
{
if (t.IsFaulted)
{
_tracer.TraceError("Get '" + SandboxFilePath + delta.Path + "' failed with " + t.Exception);
tcs.TrySetException(t.Exception.InnerExceptions);
}
else if (t.IsCanceled)
{
tcs.TrySetCanceled();
}
else
{
try
{
tcs.TrySetResult(t.Result.EnsureSuccessStatusCode().Content.ReadAsStreamAsync());
}
catch (Exception ex)
{
_tracer.TraceError("Get '" + SandboxFilePath + delta.Path + "' failed with " + ex);
tcs.TrySetException(ex);
}
}
});
return tcs.Task.FastUnwrap();
}
public virtual async Task ProcessFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc)
{
var oauthHeader = GetOAuthHeader(info, delta);
int retries = 0;
while (retries <= MaxRetries)
{
using (var client = CreateDropboxHttpClient())
{
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", oauthHeader);
try
{
string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant());
using (HttpResponseMessage response = await client.GetAsync(requestPath))
{
using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync())
{
await SafeWriteFile(parent, path, lastModifiedUtc, stream);
}
}
if (retries > 0)
{
// Increment the successful retry count
Interlocked.Increment(ref _retriedCount);
}
// Increment the total success counter
Interlocked.Increment(ref _successCount);
break;
}
catch (Exception ex)
{
if (retries == MaxRetries)
{
Interlocked.Increment(ref _failedCount);
LogError("Get({0}) '{1}' failed with {2}", retries, SandboxFilePath + delta.Path, ex.Message);
throw;
}
}
// First retry is 1s, second retry is 20s assuming rate-limit
await Task.Delay(retries == MaxRetries - 1 ? TimeSpan.FromSeconds(1) : RetryWaitToAvoidRateLimit);
retries++;
}
}
}
private async Task ProcessFileAsyncCore(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc)
{
var parameters = new Dictionary<string, string>
{
{ "oauth_consumer_key", info.ConsumerKey },
{ "oauth_signature_method", info.SignatureMethod },
{ "oauth_timestamp", info.TimeStamp },
{ "oauth_nonce", delta.Nonce },
{ "oauth_version", info.OAuthVersion },
{ "oauth_token", info.Token },
{ "oauth_signature", delta.Signature }
};
var sb = new StringBuilder();
foreach (var key in parameters.Keys)
{
if (sb.Length != 0)
{
sb.Append(',');
}
sb.AppendFormat("{0}=\"{1}\"", key, parameters[key]);
}
int retries = MaxRetries;
while (retries >= 0)
{
retries--;
using (var client = new HttpClient { BaseAddress = new Uri(DropboxApiContentUri), Timeout = _timeout })
{
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", sb.ToString());
try
{
string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant());
using (HttpResponseMessage response = await client.GetAsync(requestPath))
{
using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync())
{
await SafeWriteFile(parent, path, lastModifiedUtc, stream);
}
}
if (retries < MaxRetries - 1)
{
Interlocked.Increment(ref _retriedCount);
}
Interlocked.Increment(ref _successCount);
break;
}
catch (Exception ex)
{
if (retries <= 0)
{
Interlocked.Increment(ref _failedCount);
LogError("Get({0}) '{1}'failed with {2}", MaxRetries - retries - 1, SandboxFilePath + delta.Path, ex.Message);
break;
}
else
{
// First retry is 1s, second retry is 20s assuming rate-limit
Thread.Sleep(retries == 1 ? TimeSpan.FromSeconds(20) : TimeSpan.FromSeconds(1));
continue;
}
}
finally
{
Interlocked.Increment(ref _fileCount);
}
}
}
}
private async Task ProcessFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc)
{
var oauthHeader = GetOAuthHeader(info, delta);
int retries = MaxRetries;
while (retries >= 0)
{
retries--;
using (var client = new HttpClient { BaseAddress = new Uri(DropboxApiContentUri), Timeout = _timeout })
{
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", oauthHeader);
try
{
string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant());
using (HttpResponseMessage response = await client.GetAsync(requestPath))
{
using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync())
{
await SafeWriteFile(parent, path, lastModifiedUtc, stream);
}
}
if (retries < MaxRetries - 1)
{
Interlocked.Increment(ref _retriedCount);
}
Interlocked.Increment(ref _successCount);
break;
}
catch (Exception ex)
{
if (retries <= 0)
{
Interlocked.Increment(ref _failedCount);
LogError("Get({0}) '{1}'failed with {2}", MaxRetries - retries - 1, SandboxFilePath + delta.Path, ex.Message);
break;
}
}
// First retry is 1s, second retry is 20s assuming rate-limit
await Task.Delay(retries == 1 ? TimeSpan.FromSeconds(20) : TimeSpan.FromSeconds(1));
}
}
}
public async Task ApplyChangesCoreDeletesFilesForDeltasThatHaveBeenDeleted()
{
// Arrange
var helper = CreateDropboxHelper();
var fileDeltaInfo = new DropboxDeltaInfo { Path = "foo/bar.txt", IsDeleted = true };
var dirDeltaInfo = new DropboxDeltaInfo { Path = "foo/baz/", IsDeleted = true, IsDirectory = true };
var deployInfo = new DropboxDeployInfo { Path = "foo" };
deployInfo.Deltas = new[] { fileDeltaInfo, dirDeltaInfo };
string filePath = Path.Combine(helper.Environment.RepositoryPath, "bar.txt"),
dirPath = Path.Combine(helper.Environment.RepositoryPath, "baz");
File.WriteAllBytes(filePath, new byte[0]);
Directory.CreateDirectory(dirPath);
// Act
await helper.ApplyChangesCore(deployInfo);
// Assert
Assert.False(File.Exists(filePath));
Assert.False(Directory.Exists(dirPath));
}
public async Task ApplyChangesCoreCreatesDirectoriesForDirectoryDeltas()
{
// Arrange
var helper = CreateDropboxHelper();
var dirDeltaInfo = new DropboxDeltaInfo { Path = "foo/qux/", IsDirectory = true };
var deployInfo = new DropboxDeployInfo { Path = "foo" };
deployInfo.Deltas = new[] { dirDeltaInfo };
string dirPath = Path.Combine(helper.Environment.RepositoryPath, "qux");
// Act
await helper.ApplyChangesCore(deployInfo);
// Assert
Assert.True(Directory.Exists(dirPath));
}