private static string GetOAuthHeader(DropboxDeployInfo info, DropboxDeltaInfo delta)
{
var parameters = new Dictionary <string, string>
{
{ "oauth_consumer_key", info.ConsumerKey },
{ "oauth_signature_method", info.SignatureMethod },
{ "oauth_timestamp", info.TimeStamp },
{ "oauth_nonce", delta.Nonce },
{ "oauth_version", info.OAuthVersion },
{ "oauth_token", info.Token },
{ "oauth_signature", delta.Signature }
};
var sb = new StringBuilder();
foreach (var item in parameters)
{
if (sb.Length != 0)
{
sb.Append(',');
}
sb.AppendFormat("{0}=\"{1}\"", item.Key, item.Value);
}
return(sb.ToString());
}
public async Task ApplyChangesCoreDeletesFilesForDeltasThatHaveBeenDeleted()
{
// Arrange
var helper = CreateDropboxHelper();
var fileDeltaInfo = new DropboxDeltaInfo {
Path = "foo/bar.txt", IsDeleted = true
};
var dirDeltaInfo = new DropboxDeltaInfo {
Path = "foo/baz/", IsDeleted = true, IsDirectory = true
};
var deployInfo = new DropboxDeployInfo {
Path = "foo"
};
deployInfo.Deltas = new[] { fileDeltaInfo, dirDeltaInfo };
string filePath = Path.Combine(helper.Environment.RepositoryPath, "bar.txt"),
dirPath = Path.Combine(helper.Environment.RepositoryPath, "baz");
File.WriteAllBytes(filePath, new byte[0]);
Directory.CreateDirectory(dirPath);
// Act
await helper.ApplyChangesCore(deployInfo);
// Assert
Assert.False(File.Exists(filePath));
Assert.False(Directory.Exists(dirPath));
}
private async Task ProcessFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc)
{
var oauthHeader = GetOAuthHeader(info, delta);
int retries = MaxRetries;
while (retries >= 0)
{
retries--;
using (var client = new HttpClient {
BaseAddress = new Uri(DropboxApiContentUri), Timeout = _timeout
})
{
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", oauthHeader);
try
{
string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant());
using (HttpResponseMessage response = await client.GetAsync(requestPath))
{
using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync())
{
await SafeWriteFile(parent, path, lastModifiedUtc, stream);
}
}
if (retries < MaxRetries - 1)
{
Interlocked.Increment(ref _retriedCount);
}
Interlocked.Increment(ref _successCount);
break;
}
catch (Exception ex)
{
if (retries <= 0)
{
Interlocked.Increment(ref _failedCount);
LogError("Get({0}) '{1}'failed with {2}", MaxRetries - retries - 1, SandboxFilePath + delta.Path, ex.Message);
break;
}
}
// First retry is 1s, second retry is 20s assuming rate-limit
await Task.Delay(retries == 1?TimeSpan.FromSeconds(20) : TimeSpan.FromSeconds(1));
}
}
}
public virtual async Task ProcessFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc)
{
var oauthHeader = GetOAuthHeader(info, delta);
int retries = 0;
while (retries <= MaxRetries)
{
using (var client = CreateDropboxHttpClient())
{
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", oauthHeader);
try
{
string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant());
using (HttpResponseMessage response = await client.GetAsync(requestPath))
{
using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync())
{
await SafeWriteFile(parent, path, lastModifiedUtc, stream);
}
}
if (retries > 0)
{
// Increment the successful retry count
Interlocked.Increment(ref _retriedCount);
}
// Increment the total success counter
Interlocked.Increment(ref _successCount);
break;
}
catch (Exception ex)
{
if (retries == MaxRetries)
{
Interlocked.Increment(ref _failedCount);
LogError("Get({0}) '{1}' failed with {2}", retries, SandboxFilePath + delta.Path, ex.Message);
throw;
}
}
// First retry is 1s, second retry is 20s assuming rate-limit
await Task.Delay(retries == MaxRetries - 1?TimeSpan.FromSeconds(1) : RetryWaitToAvoidRateLimit);
retries++;
}
}
}
public async Task ApplyChangesCoreCreatesDirectoriesForDirectoryDeltas()
{
// Arrange
var helper = CreateDropboxHelper();
var dirDeltaInfo = new DropboxDeltaInfo {
Path = "foo/qux/", IsDirectory = true
};
var deployInfo = new DropboxDeployInfo {
Path = "foo"
};
deployInfo.Deltas = new[] { dirDeltaInfo };
string dirPath = Path.Combine(helper.Environment.RepositoryPath, "qux");
// Act
await helper.ApplyChangesCore(deployInfo);
// Assert
Assert.True(Directory.Exists(dirPath));
}
private async Task ProcessFileAsyncCore(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc)
{
var parameters = new Dictionary <string, string>
{
{ "oauth_consumer_key", info.ConsumerKey },
{ "oauth_signature_method", info.SignatureMethod },
{ "oauth_timestamp", info.TimeStamp },
{ "oauth_nonce", delta.Nonce },
{ "oauth_version", info.OAuthVersion },
{ "oauth_token", info.Token },
{ "oauth_signature", delta.Signature }
};
var sb = new StringBuilder();
foreach (var key in parameters.Keys)
{
if (sb.Length != 0)
{
sb.Append(',');
}
sb.AppendFormat("{0}=\"{1}\"", key, parameters[key]);
}
int retries = MaxRetries;
while (retries >= 0)
{
retries--;
using (var client = new HttpClient {
BaseAddress = new Uri(DropboxApiContentUri), Timeout = _timeout
})
{
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", sb.ToString());
try
{
string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant());
using (HttpResponseMessage response = await client.GetAsync(requestPath))
{
using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync())
{
await SafeWriteFile(parent, path, lastModifiedUtc, stream);
}
}
if (retries < MaxRetries - 1)
{
Interlocked.Increment(ref _retriedCount);
}
Interlocked.Increment(ref _successCount);
break;
}
catch (Exception ex)
{
if (retries <= 0)
{
Interlocked.Increment(ref _failedCount);
LogError("Get({0}) '{1}'failed with {2}", MaxRetries - retries - 1, SandboxFilePath + delta.Path, ex.Message);
break;
}
else
{
// First retry is 1s, second retry is 20s assuming rate-limit
Thread.Sleep(retries == 1 ? TimeSpan.FromSeconds(20) : TimeSpan.FromSeconds(1));
continue;
}
}
finally
{
Interlocked.Increment(ref _fileCount);
}
}
}
}
private DropboxDeployInfo GetDeployInfo(string path, OAuthInfo oauth, AccountInfo account, string cursor = null)
{
List <DropboxDeltaInfo> deltas = new List <DropboxDeltaInfo>();
string timeStamp = GetUtcTimeStamp();
string oldCursor = cursor;
string newCursor = "";
while (true)
{
DeltaInfo delta = GetDeltaInfo(oauth, cursor);
newCursor = delta.cursor;
if (newCursor == oldCursor)
{
break;
}
foreach (EntryInfo info in delta.entries)
{
DropboxDeltaInfo item = new DropboxDeltaInfo();
if (!info.metadata.path.StartsWith(path))
{
continue;
}
if (info.metadata == null || info.metadata.is_deleted || string.IsNullOrEmpty(info.metadata.path))
{
item.Path = info.path;
item.IsDeleted = true;
}
else
{
item.Path = info.metadata.path;
item.IsDirectory = info.metadata.is_dir;
if (!item.IsDirectory)
{
item.Modified = info.metadata.modified;
item.Nonce = GetNonce();
item.Signature = GetSignature(oauth, info.path, timeStamp, item.Nonce);
}
}
deltas.Add(item);
}
if (!delta.has_more)
{
break;
}
cursor = newCursor;
}
if (deltas.Count == 0)
{
throw new InvalidOperationException("the repo is up-to-date.");
}
return(new DropboxDeployInfo
{
TimeStamp = timeStamp,
Token = oauth.Token,
ConsumerKey = oauth.ConsumerKey,
OAuthVersion = "1.0",
SignatureMethod = "HMAC-SHA1",
OldCursor = oldCursor,
NewCursor = newCursor,
Path = path,
UserName = account.display_name,
Email = account.email,
Deltas = deltas
});
}
