public async Task GetDeltasPopulatesEntriesNodeWhenHasMoreIsSet()
{
// Arrange
const string DeltaPayload1 = @"{""reset"": true, ""cursor"": ""cursor1"", ""has_more"": true, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/bar.txt"", ""is_dir"": false}]]}";
const string DeltaPayload2 = @"{""reset"": false, ""cursor"": ""cursor2"", ""has_more"": false, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/qux.txt"", ""is_dir"": false}]]}";
var dropboxInfo = new DropboxDeployInfo
{
Token = "Some-token",
Path = "/foo"
};
int i = 0;
var handler = new TestMessageHandler(_ =>
{
var content = new StringContent(i++ == 0 ? DeltaPayload1 : DeltaPayload2, Encoding.UTF8, "application/json");
return(new HttpResponseMessage {
Content = content
});
});
var helper = CreateDropboxHelper(handler);
// Act
await helper.UpdateDropboxDeployInfo(dropboxInfo);
// Assert
Assert.Equal("cursor2", dropboxInfo.NewCursor);
Assert.Equal(2, dropboxInfo.Deltas.Count);
Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path);
Assert.Equal("/foo/qux.txt", dropboxInfo.Deltas[1].Path);
}
public async Task ApplyChangesCoreDeletesFilesForDeltasThatHaveBeenDeleted()
{
// Arrange
var helper = CreateDropboxHelper();
var fileDeltaInfo = new DropboxEntryInfo {
Path = "foo/bar.txt", IsDeleted = true
};
var dirDeltaInfo = new DropboxEntryInfo {
Path = "foo/baz/", IsDeleted = true, IsDirectory = true
};
var deployInfo = new DropboxDeployInfo {
Path = "foo"
};
deployInfo.Deltas.AddRange(new [] { fileDeltaInfo, dirDeltaInfo });
string filePath = Path.Combine(helper.Environment.RepositoryPath, "bar.txt"),
dirPath = Path.Combine(helper.Environment.RepositoryPath, "baz");
File.WriteAllBytes(filePath, new byte[0]);
Directory.CreateDirectory(dirPath);
// Act
await helper.ApplyChangesCore(deployInfo, useOAuth20 : false);
// Assert
Assert.False(File.Exists(filePath));
Assert.False(Directory.Exists(dirPath));
}
public async Task GetDeltasPopulatesEntriesNodeWhenHasMoreIsSet()
{
// Arrange
const string DeltaPayload1 = @"{""cursor"": ""cursor1"", ""has_more"": true, ""entries"": [{"".tag"": ""file"", ""path_display"": ""/foo/bar.txt"", ""server_modified"": ""2017-08-30T19:20:02Z""}]}";
const string DeltaPayload2 = @"{""cursor"": ""cursor2"", ""has_more"": false, ""entries"": [{"".tag"": ""file"", ""path_display"": ""/foo/qux.txt"", ""server_modified"": ""2017-08-30T21:40:35Z""}]}";
var dropboxInfo = new DropboxDeployInfo
{
Token = "Some-token",
Path = "/foo"
};
int i = 0;
var handler = new TestMessageHandler(_ =>
{
var content = new StringContent(i++ == 0 ? DeltaPayload1 : DeltaPayload2, Encoding.UTF8, "application/json");
return(new HttpResponseMessage {
Content = content
});
});
var helper = CreateDropboxHelper(handler);
// Act
await helper.UpdateDropboxDeployInfo(dropboxInfo);
// Assert
Assert.Equal("cursor2", dropboxInfo.NewCursor);
Assert.Equal(2, dropboxInfo.Deltas.Count);
Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path);
Assert.Equal("/foo/qux.txt", dropboxInfo.Deltas[1].Path);
}
public async Task TestDropboxBasic()
{
OAuthInfo oauth = GetOAuthInfo();
if (oauth == null)
{
// only run in private kudu
return;
}
AccountInfo account = GetAccountInfo(oauth);
DropboxDeployInfo deploy = GetDeployInfo("/BasicTest", oauth, account);
string appName = "DropboxTest";
await ApplicationManager.RunAsync(appName, async appManager =>
{
HttpClient client = HttpClientHelper.CreateClient(appManager.ServiceUrl, appManager.DeploymentManager.Credentials);
var result = await client.PostAsJsonAsync("deploy?scmType=Dropbox", deploy);
result.EnsureSuccessful();
await Task.WhenAll(
KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/default.html", "Hello Default!"),
KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/temp/temp.html", "Hello Temp!"),
KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/New Folder/New File.html", "Hello New File!")
);
});
}
public async Task ApplyChangesCoreThrowsIfAnyFileTaskFails()
{
// Arrange
var helper = CreateDropboxHelper();
int processedFiles = 0;
Mock.Get(helper).Setup(h => h.ProcessFileAsync(It.IsAny <HttpClient>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <DateTime>()))
.Callback(() => Interlocked.Increment(ref processedFiles))
.Returns(Task.FromResult(0));
Mock.Get(helper).Setup(h => h.ProcessFileAsync(It.IsAny <HttpClient>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <DateTime>()))
.Callback(() => Interlocked.Increment(ref processedFiles))
.Returns(GetFailedTask());
var deployInfo = new DropboxDeployInfo {
Path = "foo"
};
deployInfo.Deltas.AddRange(new []
{
new DropboxEntryInfo {
Path = "foo/test.txt"
},
new DropboxEntryInfo {
Path = "foo/bar.txt"
},
new DropboxEntryInfo {
Path = "foo/qux.txt"
},
new DropboxEntryInfo {
Path = "foo/buzz.png"
},
new DropboxEntryInfo {
Path = "foo/baz.php"
},
new DropboxEntryInfo {
Path = "foo/file0.php"
},
new DropboxEntryInfo {
Path = "foo/file1.php"
},
new DropboxEntryInfo {
Path = "foo/file2.php"
},
new DropboxEntryInfo {
Path = "foo/file3.php"
},
});
// Act
await Assert.ThrowsAsync <HttpRequestException>(async() => await helper.ApplyChangesCore(deployInfo, useOAuth20: false));
// Assert
// Ensure we processed other files
Assert.Equal(deployInfo.Deltas.Count, processedFiles);
}
public async Task TestDropboxBasicForBasicScenario(Scenario scenario)
{
OAuthInfo oauth = GetOAuthInfo();
if (oauth == null)
{
// only run in private kudu
return;
}
AccountInfo account = GetAccountInfo(oauth);
DropboxDeployInfo deploy = GetDeployInfo("/BasicTest", oauth, account);
string appName = "DropboxTest";
await ApplicationManager.RunAsync(appName, async appManager =>
{
if (scenario == Scenario.NoRepository)
{
await appManager.SettingsManager.SetValue(SettingsKeys.NoRepository, "1");
}
else if (scenario == Scenario.InPlace)
{
await appManager.SettingsManager.SetValue(SettingsKeys.RepositoryPath, "wwwroot");
}
HttpClient client = HttpClientHelper.CreateClient(appManager.ServiceUrl, appManager.DeploymentManager.Credentials);
var result = await client.PostAsJsonAsync("deploy?scmType=Dropbox", deploy);
result.EnsureSuccessful();
await Task.WhenAll(
KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/default.html", "Hello Default!"),
KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/temp/temp.html", "Hello Temp!"),
KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/New Folder/New File.html", "Hello New File!")
);
var repositoryGit = appManager.VfsManager.Exists(@"site\repository\.git");
var wwwrootGit = appManager.VfsManager.Exists(@"site\wwwroot\.git");
if (scenario == Scenario.NoRepository)
{
Assert.False(repositoryGit, @"site\repository\.git should not exist for " + scenario);
Assert.False(wwwrootGit, @"site\wwwroot\.git should not exist for " + scenario);
}
else if (scenario == Scenario.InPlace)
{
Assert.False(repositoryGit, @"site\repository\.git should not exist for " + scenario);
Assert.True(wwwrootGit, @"site\wwwroot\.git should exist for " + scenario);
}
else if (scenario == Scenario.Default)
{
Assert.True(repositoryGit, @"site\repository\.git should exist for " + scenario);
Assert.False(wwwrootGit, @"site\wwwroot\.git should not exist for " + scenario);
}
});
}
public void TestDropboxConcurrent()
{
OAuthInfo oauth = GetOAuthInfo();
if (oauth == null)
{
// only run in private kudu
return;
}
AccountInfo account = GetAccountInfo(oauth);
DropboxDeployInfo deploy = GetDeployInfo("/BasicTest", oauth, account);
string appName = "DropboxTest";
ApplicationManager.Run(appName, appManager =>
{
HttpClient client = HttpClientHelper.CreateClient(appManager.ServiceUrl, appManager.DeploymentManager.Credentials);
var tasks = new Task <HttpResponseMessage>[]
{
client.PostAsJsonAsync("deploy", deploy),
client.PostAsJsonAsync("deploy", deploy)
};
Task.WaitAll(tasks);
// both 200 and 202 is success
tasks[0].Result.EnsureSuccessful();
tasks[1].Result.EnsureSuccessful();
var success = (tasks[0].Result.StatusCode == HttpStatusCode.OK) ? tasks[0].Result : tasks[1].Result;
var failure = !Object.ReferenceEquals(success, tasks[0].Result) ? tasks[0].Result : tasks[1].Result;
success.EnsureSuccessful();
Assert.Equal(HttpStatusCode.Accepted, failure.StatusCode);
var results = appManager.DeploymentManager.GetResultsAsync().Result.ToList();
Assert.Equal(1, results.Count);
Assert.Equal(DeployStatus.Success, results[0].Status);
Assert.Equal("Dropbox", results[0].Deployer);
});
}
public async Task ApplyChangesCoreCreatesDirectoriesForDirectoryDeltas()
{
// Arrange
var helper = CreateDropboxHelper();
var dirDeltaInfo = new DropboxEntryInfo {
Path = "foo/qux/", IsDirectory = true
};
var deployInfo = new DropboxDeployInfo {
Path = "foo"
};
deployInfo.Deltas.Add(dirDeltaInfo);
string dirPath = Path.Combine(helper.Environment.RepositoryPath, "qux");
// Act
await helper.ApplyChangesCore(deployInfo, useOAuth20 : false);
// Assert
Assert.True(Directory.Exists(dirPath));
}
public async Task GetDeltasPopulatesEntriesNode()
{
// Arrange
const string DeltaPayload = @"{""reset"": true, ""cursor"": ""AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA"", ""has_more"": false, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""thumb_exists"": false, ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/bar.txt"", ""is_dir"": false, ""icon"": ""page_white"", ""root"": ""dropbox"", ""mime_type"": ""application/epub+zip"", ""size"": ""120.7 KB""}]]}";
var dropboxInfo = new DropboxDeployInfo
{
Token = "Some-token",
Path = "/foo"
};
var handler = new TestMessageHandler(DeltaPayload, isJson: true);
var helper = CreateDropboxHelper(handler);
// Act
await helper.UpdateDropboxDeployInfo(dropboxInfo);
// Assert
Assert.Null(dropboxInfo.OldCursor);
Assert.Equal("AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA", dropboxInfo.NewCursor);
Assert.Equal(1, dropboxInfo.Deltas.Count);
Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path);
}
public async Task GetDeltasPopulatesEntriesNode()
{
// Arrange
const string DeltaPayload = @"{""cursor"": ""AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA"", ""has_more"": false, ""entries"": [{"".tag"": ""file"", ""path_display"": ""/foo/bar.txt"", ""server_modified"": ""2017-08-30T18:50:02Z""}]}";
var dropboxInfo = new DropboxDeployInfo
{
Token = "Some-token",
Path = "/foo"
};
var handler = new TestMessageHandler(DeltaPayload, isJson: true);
var helper = CreateDropboxHelper(handler);
// Act
await helper.UpdateDropboxDeployInfo(dropboxInfo);
// Assert
Assert.Null(dropboxInfo.OldCursor);
Assert.Equal("AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA", dropboxInfo.NewCursor);
Assert.Equal(1, dropboxInfo.Deltas.Count);
Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path);
}
public void TestDropboxSpecialFolderChars()
{
OAuthInfo oauth = GetOAuthInfo();
if (oauth == null)
{
// only run in private kudu
return;
}
AccountInfo account = GetAccountInfo(oauth);
DropboxDeployInfo deploy = GetDeployInfo("/!@#$faiztest$#!@", oauth, account);
string appName = "SpecialCharsTest";
ApplicationManager.Run(appName, appManager =>
{
HttpClient client = HttpClientHelper.CreateClient(appManager.ServiceUrl, appManager.DeploymentManager.Credentials);
client.PostAsJsonAsync("deploy?scmType=Dropbox", deploy).Result.EnsureSuccessful();
});
}
public ChangeSet Sync(DropboxDeployInfo info, string branch)
{
if (_settings.GetValue(CursorKey) != info.OldCursor)
{
throw new InvalidOperationException(Resources.Error_MismatchDropboxCursor);
}
if (!IsEmptyRepo())
{
// git checkout --force <branch>
_repository.Update(branch);
}
ChangeSet changeSet;
string prefix = "Partially";
try
{
using (_tracer.Step("Synch with Dropbox"))
{
// Sync dropbox => repository directory
ApplyChanges(info);
}
prefix = "Successfully";
}
finally
{
// Commit anyway even partial change
changeSet = _repository.Commit(prefix + " sync with dropbox at " + DateTime.UtcNow.ToString("g"), String.Format("{0} <{1}>", info.UserName, info.Email));
}
// Save new dropboc cursor
_tracer.Trace("Update dropbox cursor");
_settings.SetValue(CursorKey, info.NewCursor);
return changeSet;
}
internal DropboxDeployInfo GetDeployInfo(string path, OAuthInfo oauth, AccountInfo account, string cursor = null)
{
List <DropboxEntryInfo> deltas = new List <DropboxEntryInfo>();
string timeStamp = GetUtcTimeStamp();
string oldCursor = cursor;
string newCursor = "";
while (true)
{
DeltaInfo delta = GetDeltaInfo(oauth, cursor);
newCursor = delta.cursor;
if (newCursor == oldCursor)
{
break;
}
foreach (EntryInfo info in delta.entries)
{
DropboxEntryInfo item = new DropboxEntryInfo();
if (info.metadata != null && !info.metadata.path.StartsWith(path))
{
continue;
}
if (info.metadata == null || info.metadata.is_deleted || string.IsNullOrEmpty(info.metadata.path))
{
item.Path = info.path;
item.IsDeleted = true;
}
else
{
item.Path = info.metadata.path;
item.IsDirectory = info.metadata.is_dir;
if (!item.IsDirectory)
{
item.Modified = info.metadata.modified;
item.Nonce = GetNonce();
item.Signature = GetSignature(oauth, info.path, timeStamp, item.Nonce);
}
}
deltas.Add(item);
}
if (!delta.has_more)
{
break;
}
cursor = newCursor;
}
if (deltas.Count == 0)
{
throw new InvalidOperationException("the repo is up-to-date.");
}
var deployInfo = new DropboxDeployInfo
{
TimeStamp = timeStamp,
Token = oauth.Token,
ConsumerKey = oauth.ConsumerKey,
OAuthVersion = "1.0",
SignatureMethod = "HMAC-SHA1",
OldCursor = oldCursor,
NewCursor = newCursor,
Path = path,
UserName = account.display_name,
Email = account.email,
};
deployInfo.Deltas.AddRange(deltas);
return(deployInfo);
}
private async Task ProcessFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc)
{
var oauthHeader = GetOAuthHeader(info, delta);
int retries = MaxRetries;
while (retries >= 0)
{
retries--;
using (var client = new HttpClient { BaseAddress = new Uri(DropboxApiContentUri), Timeout = _timeout })
{
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", oauthHeader);
try
{
string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant());
using (HttpResponseMessage response = await client.GetAsync(requestPath))
{
using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync())
{
await SafeWriteFile(parent, path, lastModifiedUtc, stream);
}
}
if (retries < MaxRetries - 1)
{
Interlocked.Increment(ref _retriedCount);
}
Interlocked.Increment(ref _successCount);
break;
}
catch (Exception ex)
{
if (retries <= 0)
{
Interlocked.Increment(ref _failedCount);
LogError("Get({0}) '{1}'failed with {2}", MaxRetries - retries - 1, SandboxFilePath + delta.Path, ex.Message);
break;
}
}
// First retry is 1s, second retry is 20s assuming rate-limit
await Task.Delay(retries == 1 ? TimeSpan.FromSeconds(20) : TimeSpan.FromSeconds(1));
}
}
}
public virtual async Task ProcessFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc)
{
var oauthHeader = GetOAuthHeader(info, delta);
int retries = 0;
while (retries <= MaxRetries)
{
using (var client = CreateDropboxHttpClient())
{
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", oauthHeader);
try
{
string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant());
using (HttpResponseMessage response = await client.GetAsync(requestPath))
{
using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync())
{
await SafeWriteFile(parent, path, lastModifiedUtc, stream);
}
}
if (retries > 0)
{
// Increment the successful retry count
Interlocked.Increment(ref _retriedCount);
}
// Increment the total success counter
Interlocked.Increment(ref _successCount);
break;
}
catch (Exception ex)
{
if (retries == MaxRetries)
{
Interlocked.Increment(ref _failedCount);
LogError("Get({0}) '{1}' failed with {2}", retries, SandboxFilePath + delta.Path, ex.Message);
throw;
}
}
// First retry is 1s, second retry is 20s assuming rate-limit
await Task.Delay(retries == MaxRetries - 1 ? TimeSpan.FromSeconds(1) : RetryWaitToAvoidRateLimit);
retries++;
}
}
}
internal async Task ApplyChangesCore(DropboxDeployInfo info)
{
string parent = info.Path.TrimEnd('/') + '/';
DateTime updateMessageTime = DateTime.UtcNow;
var sem = new SemaphoreSlim(MaxConcurrentRequests, MaxConcurrentRequests);
var rateLimiter = new RateLimiter(MaxFilesPerSecs * 10, TimeSpan.FromSeconds(10));
var tasks = new List<Task>();
foreach (DropboxDeltaInfo delta in info.Deltas)
{
if (!delta.Path.StartsWith(parent, StringComparison.OrdinalIgnoreCase))
{
continue;
}
// Ignore .git and .hg files and folders
string pathWithSlash = delta.Path + "/";
if (pathWithSlash.IndexOf("/.git/", StringComparison.OrdinalIgnoreCase) != -1 ||
pathWithSlash.IndexOf("/.hg/", StringComparison.OrdinalIgnoreCase) != -1)
{
continue;
}
var path = delta.Path;
if (delta.IsDeleted)
{
SafeDelete(parent, path);
Interlocked.Increment(ref _successCount);
}
else if (delta.IsDirectory)
{
SafeCreateDir(parent, path);
Interlocked.Increment(ref _successCount);
}
else
{
DateTime modified;
if (DateTime.TryParse(delta.Modified, out modified))
{
modified = modified.ToUniversalTime();
}
else
{
modified = DateTime.UtcNow;
}
if (!IsFileChanged(parent, path, modified))
{
LogInfo("file unchanged {0}", path);
Interlocked.Increment(ref _successCount);
continue;
}
try
{
var task = Task.Run(async () =>
{
await sem.WaitAsync();
try
{
await rateLimiter.ThrottleAsync();
await ProcessFileAsync(info, delta, parent, path, modified);
}
finally
{
Interlocked.Increment(ref _fileCount);
sem.Release();
}
});
tasks.Add(task);
}
catch (Exception ex)
{
LogError("{0}", ex);
throw;
}
}
}
await Task.WhenAll(tasks);
}
private Task<StreamInfo> GetFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, int retries = MaxRetries)
{
var parameters = new Dictionary<string, string>
{
{ "oauth_consumer_key", info.ConsumerKey },
{ "oauth_signature_method", info.SignatureMethod },
{ "oauth_timestamp", info.TimeStamp },
{ "oauth_nonce", delta.Nonce },
{ "oauth_version", info.OAuthVersion },
{ "oauth_token", info.Token },
{ "oauth_signature", delta.Signature }
};
var sb = new StringBuilder();
foreach (var key in parameters.Keys)
{
if (sb.Length != 0)
{
sb.Append(',');
}
sb.AppendFormat("{0}=\"{1}\"", key, parameters[key]);
}
var client = new HttpClient();
client.BaseAddress = new Uri(DropboxApiContentUri);
client.Timeout = _timeout;
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", sb.ToString());
// Using ContinueWith instead of Then to avoid SyncContext deadlock in 4.5
var tcs = new TaskCompletionSource<Task<StreamInfo>>();
client.GetAsync(SandboxFilePath + DropboxPathEncode(delta.Path.ToLower(CultureInfo.CurrentCulture))).ContinueWith(t =>
{
if (t.IsFaulted)
{
using (client)
{
if (retries <= 0)
{
LogError("Get(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + t.Exception);
tcs.TrySetException(t.Exception.InnerExceptions);
}
else
{
LogError("Retry(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + t.Exception);
tcs.TrySetResult(RetryGetFileAsync(info, delta, retries));
}
}
}
else if (t.IsCanceled)
{
using (client)
{
tcs.TrySetCanceled();
}
}
else
{
try
{
HttpResponseMessage response = t.Result;
if ((int)response.StatusCode < 500 || retries <= 0)
{
response.EnsureSuccessStatusCode();
tcs.TrySetResult(GetStreamInfo(client, response, delta.Path));
if (retries < MaxRetries)
{
// Success due to retried
Interlocked.Increment(ref _retriedCount);
}
}
else
{
using (client)
{
using (response)
{
LogError("Retry(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + (int)response.StatusCode);
tcs.TrySetResult(RetryGetFileAsync(info, delta, retries));
}
}
}
}
catch (Exception ex)
{
LogError("Get(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + ex);
tcs.TrySetException(ex);
}
}
});
return tcs.Task.FastUnwrap();
}
private Task<StreamInfo> RetryGetFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, int retries)
{
// First retry is 1s, second retry is 20s assuming rate-limit
Thread.Sleep(retries == 1 ? 20000 : 1000);
return GetFileAsync(info, delta, retries - 1);
}
public async Task ApplyChangesCoreCreatesDirectoriesForDirectoryDeltas()
{
// Arrange
var helper = CreateDropboxHelper();
var dirDeltaInfo = new DropboxEntryInfo { Path = "foo/qux/", IsDirectory = true };
var deployInfo = new DropboxDeployInfo { Path = "foo" };
deployInfo.Deltas.Add(dirDeltaInfo);
string dirPath = Path.Combine(helper.Environment.RepositoryPath, "qux");
// Act
await helper.ApplyChangesCore(deployInfo, useOAuth20: false);
// Assert
Assert.True(Directory.Exists(dirPath));
}
private static string GetOAuthHeader(DropboxDeployInfo info, DropboxDeltaInfo delta)
{
var parameters = new Dictionary<string, string>
{
{ "oauth_consumer_key", info.ConsumerKey },
{ "oauth_signature_method", info.SignatureMethod },
{ "oauth_timestamp", info.TimeStamp },
{ "oauth_nonce", delta.Nonce },
{ "oauth_version", info.OAuthVersion },
{ "oauth_token", info.Token },
{ "oauth_signature", delta.Signature }
};
var sb = new StringBuilder();
foreach (var item in parameters)
{
if (sb.Length != 0)
{
sb.Append(',');
}
sb.AppendFormat("{0}=\"{1}\"", item.Key, item.Value);
}
return sb.ToString();
}
public async Task GetDeltasPopulatesEntriesNode()
{
// Arrange
const string DeltaPayload = @"{""reset"": true, ""cursor"": ""AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA"", ""has_more"": false, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""thumb_exists"": false, ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/bar.txt"", ""is_dir"": false, ""icon"": ""page_white"", ""root"": ""dropbox"", ""mime_type"": ""application/epub+zip"", ""size"": ""120.7 KB""}]]}";
var dropboxInfo = new DropboxDeployInfo
{
Token = "Some-token",
Path = "/foo"
};
var handler = new TestMessageHandler(DeltaPayload, isJson: true);
var helper = CreateDropboxHelper(handler);
// Act
await helper.UpdateDropboxDeployInfo(dropboxInfo);
// Assert
Assert.Null(dropboxInfo.OldCursor);
Assert.Equal("AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA", dropboxInfo.NewCursor);
Assert.Equal(1, dropboxInfo.Deltas.Count);
Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path);
}
public async Task ApplyChangesCoreThrowsIfAnyFileTaskFails()
{
// Arrange
var helper = CreateDropboxHelper();
int processedFiles = 0;
Mock.Get(helper).Setup(h => h.ProcessFileAsync(It.IsAny<HttpClient>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<DateTime>()))
.Callback(() => Interlocked.Increment(ref processedFiles))
.Returns(Task.FromResult(0));
Mock.Get(helper).Setup(h => h.ProcessFileAsync(It.IsAny<HttpClient>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<DateTime>()))
.Callback(() => Interlocked.Increment(ref processedFiles))
.Returns(GetFailedTask());
var deployInfo = new DropboxDeployInfo { Path = "foo" };
deployInfo.Deltas.AddRange(new []
{
new DropboxEntryInfo { Path = "foo/test.txt" },
new DropboxEntryInfo { Path = "foo/bar.txt" },
new DropboxEntryInfo { Path = "foo/qux.txt" },
new DropboxEntryInfo { Path = "foo/buzz.png" },
new DropboxEntryInfo { Path = "foo/baz.php"},
new DropboxEntryInfo { Path = "foo/file0.php"},
new DropboxEntryInfo { Path = "foo/file1.php"},
new DropboxEntryInfo { Path = "foo/file2.php"},
new DropboxEntryInfo { Path = "foo/file3.php"},
});
// Act
await ExceptionAssert.ThrowsAsync<HttpRequestException>(async() => await helper.ApplyChangesCore(deployInfo, useOAuth20: false));
// Assert
// Ensure we processed other files
Assert.Equal(deployInfo.Deltas.Count, processedFiles);
}
public async Task GetDeltasPopulatesEntriesNodeWhenHasMoreIsSet()
{
// Arrange
const string DeltaPayload1 = @"{""reset"": true, ""cursor"": ""cursor1"", ""has_more"": true, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/bar.txt"", ""is_dir"": false}]]}";
const string DeltaPayload2 = @"{""reset"": false, ""cursor"": ""cursor2"", ""has_more"": false, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/qux.txt"", ""is_dir"": false}]]}";
var dropboxInfo = new DropboxDeployInfo
{
Token = "Some-token",
Path = "/foo"
};
int i = 0;
var handler = new TestMessageHandler(_ =>
{
var content = new StringContent(i++ == 0 ? DeltaPayload1 : DeltaPayload2, Encoding.UTF8, "application/json");
return new HttpResponseMessage { Content = content };
});
var helper = CreateDropboxHelper(handler);
// Act
await helper.UpdateDropboxDeployInfo(dropboxInfo);
// Assert
Assert.Equal("cursor2", dropboxInfo.NewCursor);
Assert.Equal(2, dropboxInfo.Deltas.Count);
Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path);
Assert.Equal("/foo/qux.txt", dropboxInfo.Deltas[1].Path);
}
public async Task ApplyChangesCoreDeletesFilesForDeltasThatHaveBeenDeleted()
{
// Arrange
var helper = CreateDropboxHelper();
var fileDeltaInfo = new DropboxEntryInfo { Path = "foo/bar.txt", IsDeleted = true };
var dirDeltaInfo = new DropboxEntryInfo { Path = "foo/baz/", IsDeleted = true, IsDirectory = true };
var deployInfo = new DropboxDeployInfo { Path = "foo" };
deployInfo.Deltas.AddRange(new [] { fileDeltaInfo, dirDeltaInfo });
string filePath = Path.Combine(helper.Environment.RepositoryPath, "bar.txt"),
dirPath = Path.Combine(helper.Environment.RepositoryPath, "baz");
File.WriteAllBytes(filePath, new byte[0]);
Directory.CreateDirectory(dirPath);
// Act
await helper.ApplyChangesCore(deployInfo, useOAuth20: false);
// Assert
Assert.False(File.Exists(filePath));
Assert.False(Directory.Exists(dirPath));
}
private void ApplyChanges(DropboxDeployInfo info)
{
Semaphore sem = new Semaphore(MaxConcurrentRequests, MaxConcurrentRequests);
List<Task> tasks = new List<Task>();
string parent = info.Path.TrimEnd('/') + '/';
foreach (DropboxDeltaInfo delta in info.Deltas)
{
if (!delta.Path.StartsWith(parent, StringComparison.OrdinalIgnoreCase))
{
continue;
}
var path = delta.Path;
if (delta.IsDeleted)
{
SafeDelete(parent, path);
}
else if (delta.IsDirectory)
{
SafeCreateDir(parent, path);
}
else
{
DateTime modified = DateTime.Parse(delta.Modified).ToUniversalTime();
if (!IsFileChanged(parent, path, modified))
{
_tracer.Trace("file unchanged {0}", path);
continue;
}
// throttle concurrent get file dropbox
sem.WaitOne();
Task task;
try
{
// Using ContinueWith instead of Then to avoid SyncContext deadlock in 4.5
task = GetFileAsync(info, delta).ContinueWith(t =>
{
sem.Release();
if (!t.IsFaulted && !t.IsCanceled)
{
using (Stream stream = t.Result)
{
SafeWriteFile(parent, path, stream, modified);
}
}
return t;
}).FastUnwrap();
}
catch (Exception ex)
{
sem.Release();
_tracer.TraceError(ex);
Task.WaitAll(tasks.ToArray(), _timeout);
throw;
}
tasks.Add(task);
}
}
if (!Task.WaitAll(tasks.ToArray(), _timeout))
{
throw new TimeoutException(RS.Format(Resources.Error_SyncDropboxTimeout, (int)_timeout.TotalSeconds));
}
}
private async Task ProcessFileAsyncCore(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc)
{
var parameters = new Dictionary<string, string>
{
{ "oauth_consumer_key", info.ConsumerKey },
{ "oauth_signature_method", info.SignatureMethod },
{ "oauth_timestamp", info.TimeStamp },
{ "oauth_nonce", delta.Nonce },
{ "oauth_version", info.OAuthVersion },
{ "oauth_token", info.Token },
{ "oauth_signature", delta.Signature }
};
var sb = new StringBuilder();
foreach (var key in parameters.Keys)
{
if (sb.Length != 0)
{
sb.Append(',');
}
sb.AppendFormat("{0}=\"{1}\"", key, parameters[key]);
}
int retries = MaxRetries;
while (retries >= 0)
{
retries--;
using (var client = new HttpClient { BaseAddress = new Uri(DropboxApiContentUri), Timeout = _timeout })
{
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", sb.ToString());
try
{
string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant());
using (HttpResponseMessage response = await client.GetAsync(requestPath))
{
using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync())
{
await SafeWriteFile(parent, path, lastModifiedUtc, stream);
}
}
if (retries < MaxRetries - 1)
{
Interlocked.Increment(ref _retriedCount);
}
Interlocked.Increment(ref _successCount);
break;
}
catch (Exception ex)
{
if (retries <= 0)
{
Interlocked.Increment(ref _failedCount);
LogError("Get({0}) '{1}'failed with {2}", MaxRetries - retries - 1, SandboxFilePath + delta.Path, ex.Message);
break;
}
else
{
// First retry is 1s, second retry is 20s assuming rate-limit
Thread.Sleep(retries == 1 ? TimeSpan.FromSeconds(20) : TimeSpan.FromSeconds(1));
continue;
}
}
finally
{
Interlocked.Increment(ref _fileCount);
}
}
}
}
private Task<Stream> GetFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta)
{
var parameters = new Dictionary<string, string>
{
{ "oauth_consumer_key", info.ConsumerKey },
{ "oauth_signature_method", info.SignatureMethod },
{ "oauth_timestamp", info.TimeStamp },
{ "oauth_nonce", delta.Nonce },
{ "oauth_version", info.OAuthVersion },
{ "oauth_token", info.Token },
{ "oauth_signature", delta.Signature }
};
var sb = new StringBuilder();
foreach (var key in parameters.Keys)
{
if (sb.Length != 0)
{
sb.Append(',');
}
sb.AppendFormat("{0}=\"{1}\"", key, parameters[key]);
}
var client = new HttpClient();
client.BaseAddress = new Uri(DropboxApiContentUri);
client.Timeout = _timeout;
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", sb.ToString());
// Using ContinueWith instead of Then to avoid SyncContext deadlock in 4.5
TaskCompletionSource<Task<Stream>> tcs = new TaskCompletionSource<Task<Stream>>();
client.GetAsync(SandboxFilePath + delta.Path.ToLower()).ContinueWith(t =>
{
if (t.IsFaulted)
{
_tracer.TraceError("Get '" + SandboxFilePath + delta.Path + "' failed with " + t.Exception);
tcs.TrySetException(t.Exception.InnerExceptions);
}
else if (t.IsCanceled)
{
tcs.TrySetCanceled();
}
else
{
try
{
tcs.TrySetResult(t.Result.EnsureSuccessStatusCode().Content.ReadAsStreamAsync());
}
catch (Exception ex)
{
_tracer.TraceError("Get '" + SandboxFilePath + delta.Path + "' failed with " + ex);
tcs.TrySetException(ex);
}
}
});
return tcs.Task.FastUnwrap();
}