public async Task GetDeltasPopulatesEntriesNodeWhenHasMoreIsSet() { // Arrange const string DeltaPayload1 = @"{""reset"": true, ""cursor"": ""cursor1"", ""has_more"": true, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/bar.txt"", ""is_dir"": false}]]}"; const string DeltaPayload2 = @"{""reset"": false, ""cursor"": ""cursor2"", ""has_more"": false, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/qux.txt"", ""is_dir"": false}]]}"; var dropboxInfo = new DropboxDeployInfo { Token = "Some-token", Path = "/foo" }; int i = 0; var handler = new TestMessageHandler(_ => { var content = new StringContent(i++ == 0 ? DeltaPayload1 : DeltaPayload2, Encoding.UTF8, "application/json"); return(new HttpResponseMessage { Content = content }); }); var helper = CreateDropboxHelper(handler); // Act await helper.UpdateDropboxDeployInfo(dropboxInfo); // Assert Assert.Equal("cursor2", dropboxInfo.NewCursor); Assert.Equal(2, dropboxInfo.Deltas.Count); Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path); Assert.Equal("/foo/qux.txt", dropboxInfo.Deltas[1].Path); }
public async Task ApplyChangesCoreDeletesFilesForDeltasThatHaveBeenDeleted() { // Arrange var helper = CreateDropboxHelper(); var fileDeltaInfo = new DropboxEntryInfo { Path = "foo/bar.txt", IsDeleted = true }; var dirDeltaInfo = new DropboxEntryInfo { Path = "foo/baz/", IsDeleted = true, IsDirectory = true }; var deployInfo = new DropboxDeployInfo { Path = "foo" }; deployInfo.Deltas.AddRange(new [] { fileDeltaInfo, dirDeltaInfo }); string filePath = Path.Combine(helper.Environment.RepositoryPath, "bar.txt"), dirPath = Path.Combine(helper.Environment.RepositoryPath, "baz"); File.WriteAllBytes(filePath, new byte[0]); Directory.CreateDirectory(dirPath); // Act await helper.ApplyChangesCore(deployInfo, useOAuth20 : false); // Assert Assert.False(File.Exists(filePath)); Assert.False(Directory.Exists(dirPath)); }
public async Task GetDeltasPopulatesEntriesNodeWhenHasMoreIsSet() { // Arrange const string DeltaPayload1 = @"{""cursor"": ""cursor1"", ""has_more"": true, ""entries"": [{"".tag"": ""file"", ""path_display"": ""/foo/bar.txt"", ""server_modified"": ""2017-08-30T19:20:02Z""}]}"; const string DeltaPayload2 = @"{""cursor"": ""cursor2"", ""has_more"": false, ""entries"": [{"".tag"": ""file"", ""path_display"": ""/foo/qux.txt"", ""server_modified"": ""2017-08-30T21:40:35Z""}]}"; var dropboxInfo = new DropboxDeployInfo { Token = "Some-token", Path = "/foo" }; int i = 0; var handler = new TestMessageHandler(_ => { var content = new StringContent(i++ == 0 ? DeltaPayload1 : DeltaPayload2, Encoding.UTF8, "application/json"); return(new HttpResponseMessage { Content = content }); }); var helper = CreateDropboxHelper(handler); // Act await helper.UpdateDropboxDeployInfo(dropboxInfo); // Assert Assert.Equal("cursor2", dropboxInfo.NewCursor); Assert.Equal(2, dropboxInfo.Deltas.Count); Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path); Assert.Equal("/foo/qux.txt", dropboxInfo.Deltas[1].Path); }
public async Task TestDropboxBasic() { OAuthInfo oauth = GetOAuthInfo(); if (oauth == null) { // only run in private kudu return; } AccountInfo account = GetAccountInfo(oauth); DropboxDeployInfo deploy = GetDeployInfo("/BasicTest", oauth, account); string appName = "DropboxTest"; await ApplicationManager.RunAsync(appName, async appManager => { HttpClient client = HttpClientHelper.CreateClient(appManager.ServiceUrl, appManager.DeploymentManager.Credentials); var result = await client.PostAsJsonAsync("deploy?scmType=Dropbox", deploy); result.EnsureSuccessful(); await Task.WhenAll( KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/default.html", "Hello Default!"), KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/temp/temp.html", "Hello Temp!"), KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/New Folder/New File.html", "Hello New File!") ); }); }
public async Task ApplyChangesCoreThrowsIfAnyFileTaskFails() { // Arrange var helper = CreateDropboxHelper(); int processedFiles = 0; Mock.Get(helper).Setup(h => h.ProcessFileAsync(It.IsAny <HttpClient>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <DateTime>())) .Callback(() => Interlocked.Increment(ref processedFiles)) .Returns(Task.FromResult(0)); Mock.Get(helper).Setup(h => h.ProcessFileAsync(It.IsAny <HttpClient>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <DateTime>())) .Callback(() => Interlocked.Increment(ref processedFiles)) .Returns(GetFailedTask()); var deployInfo = new DropboxDeployInfo { Path = "foo" }; deployInfo.Deltas.AddRange(new [] { new DropboxEntryInfo { Path = "foo/test.txt" }, new DropboxEntryInfo { Path = "foo/bar.txt" }, new DropboxEntryInfo { Path = "foo/qux.txt" }, new DropboxEntryInfo { Path = "foo/buzz.png" }, new DropboxEntryInfo { Path = "foo/baz.php" }, new DropboxEntryInfo { Path = "foo/file0.php" }, new DropboxEntryInfo { Path = "foo/file1.php" }, new DropboxEntryInfo { Path = "foo/file2.php" }, new DropboxEntryInfo { Path = "foo/file3.php" }, }); // Act await Assert.ThrowsAsync <HttpRequestException>(async() => await helper.ApplyChangesCore(deployInfo, useOAuth20: false)); // Assert // Ensure we processed other files Assert.Equal(deployInfo.Deltas.Count, processedFiles); }
public async Task TestDropboxBasicForBasicScenario(Scenario scenario) { OAuthInfo oauth = GetOAuthInfo(); if (oauth == null) { // only run in private kudu return; } AccountInfo account = GetAccountInfo(oauth); DropboxDeployInfo deploy = GetDeployInfo("/BasicTest", oauth, account); string appName = "DropboxTest"; await ApplicationManager.RunAsync(appName, async appManager => { if (scenario == Scenario.NoRepository) { await appManager.SettingsManager.SetValue(SettingsKeys.NoRepository, "1"); } else if (scenario == Scenario.InPlace) { await appManager.SettingsManager.SetValue(SettingsKeys.RepositoryPath, "wwwroot"); } HttpClient client = HttpClientHelper.CreateClient(appManager.ServiceUrl, appManager.DeploymentManager.Credentials); var result = await client.PostAsJsonAsync("deploy?scmType=Dropbox", deploy); result.EnsureSuccessful(); await Task.WhenAll( KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/default.html", "Hello Default!"), KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/temp/temp.html", "Hello Temp!"), KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/New Folder/New File.html", "Hello New File!") ); var repositoryGit = appManager.VfsManager.Exists(@"site\repository\.git"); var wwwrootGit = appManager.VfsManager.Exists(@"site\wwwroot\.git"); if (scenario == Scenario.NoRepository) { Assert.False(repositoryGit, @"site\repository\.git should not exist for " + scenario); Assert.False(wwwrootGit, @"site\wwwroot\.git should not exist for " + scenario); } else if (scenario == Scenario.InPlace) { Assert.False(repositoryGit, @"site\repository\.git should not exist for " + scenario); Assert.True(wwwrootGit, @"site\wwwroot\.git should exist for " + scenario); } else if (scenario == Scenario.Default) { Assert.True(repositoryGit, @"site\repository\.git should exist for " + scenario); Assert.False(wwwrootGit, @"site\wwwroot\.git should not exist for " + scenario); } }); }
public void TestDropboxConcurrent() { OAuthInfo oauth = GetOAuthInfo(); if (oauth == null) { // only run in private kudu return; } AccountInfo account = GetAccountInfo(oauth); DropboxDeployInfo deploy = GetDeployInfo("/BasicTest", oauth, account); string appName = "DropboxTest"; ApplicationManager.Run(appName, appManager => { HttpClient client = HttpClientHelper.CreateClient(appManager.ServiceUrl, appManager.DeploymentManager.Credentials); var tasks = new Task <HttpResponseMessage>[] { client.PostAsJsonAsync("deploy", deploy), client.PostAsJsonAsync("deploy", deploy) }; Task.WaitAll(tasks); // both 200 and 202 is success tasks[0].Result.EnsureSuccessful(); tasks[1].Result.EnsureSuccessful(); var success = (tasks[0].Result.StatusCode == HttpStatusCode.OK) ? tasks[0].Result : tasks[1].Result; var failure = !Object.ReferenceEquals(success, tasks[0].Result) ? tasks[0].Result : tasks[1].Result; success.EnsureSuccessful(); Assert.Equal(HttpStatusCode.Accepted, failure.StatusCode); var results = appManager.DeploymentManager.GetResultsAsync().Result.ToList(); Assert.Equal(1, results.Count); Assert.Equal(DeployStatus.Success, results[0].Status); Assert.Equal("Dropbox", results[0].Deployer); }); }
public async Task ApplyChangesCoreCreatesDirectoriesForDirectoryDeltas() { // Arrange var helper = CreateDropboxHelper(); var dirDeltaInfo = new DropboxEntryInfo { Path = "foo/qux/", IsDirectory = true }; var deployInfo = new DropboxDeployInfo { Path = "foo" }; deployInfo.Deltas.Add(dirDeltaInfo); string dirPath = Path.Combine(helper.Environment.RepositoryPath, "qux"); // Act await helper.ApplyChangesCore(deployInfo, useOAuth20 : false); // Assert Assert.True(Directory.Exists(dirPath)); }
public async Task GetDeltasPopulatesEntriesNode() { // Arrange const string DeltaPayload = @"{""reset"": true, ""cursor"": ""AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA"", ""has_more"": false, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""thumb_exists"": false, ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/bar.txt"", ""is_dir"": false, ""icon"": ""page_white"", ""root"": ""dropbox"", ""mime_type"": ""application/epub+zip"", ""size"": ""120.7 KB""}]]}"; var dropboxInfo = new DropboxDeployInfo { Token = "Some-token", Path = "/foo" }; var handler = new TestMessageHandler(DeltaPayload, isJson: true); var helper = CreateDropboxHelper(handler); // Act await helper.UpdateDropboxDeployInfo(dropboxInfo); // Assert Assert.Null(dropboxInfo.OldCursor); Assert.Equal("AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA", dropboxInfo.NewCursor); Assert.Equal(1, dropboxInfo.Deltas.Count); Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path); }
public async Task GetDeltasPopulatesEntriesNode() { // Arrange const string DeltaPayload = @"{""cursor"": ""AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA"", ""has_more"": false, ""entries"": [{"".tag"": ""file"", ""path_display"": ""/foo/bar.txt"", ""server_modified"": ""2017-08-30T18:50:02Z""}]}"; var dropboxInfo = new DropboxDeployInfo { Token = "Some-token", Path = "/foo" }; var handler = new TestMessageHandler(DeltaPayload, isJson: true); var helper = CreateDropboxHelper(handler); // Act await helper.UpdateDropboxDeployInfo(dropboxInfo); // Assert Assert.Null(dropboxInfo.OldCursor); Assert.Equal("AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA", dropboxInfo.NewCursor); Assert.Equal(1, dropboxInfo.Deltas.Count); Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path); }
public void TestDropboxSpecialFolderChars() { OAuthInfo oauth = GetOAuthInfo(); if (oauth == null) { // only run in private kudu return; } AccountInfo account = GetAccountInfo(oauth); DropboxDeployInfo deploy = GetDeployInfo("/!@#$faiztest$#!@", oauth, account); string appName = "SpecialCharsTest"; ApplicationManager.Run(appName, appManager => { HttpClient client = HttpClientHelper.CreateClient(appManager.ServiceUrl, appManager.DeploymentManager.Credentials); client.PostAsJsonAsync("deploy?scmType=Dropbox", deploy).Result.EnsureSuccessful(); }); }
public ChangeSet Sync(DropboxDeployInfo info, string branch) { if (_settings.GetValue(CursorKey) != info.OldCursor) { throw new InvalidOperationException(Resources.Error_MismatchDropboxCursor); } if (!IsEmptyRepo()) { // git checkout --force <branch> _repository.Update(branch); } ChangeSet changeSet; string prefix = "Partially"; try { using (_tracer.Step("Synch with Dropbox")) { // Sync dropbox => repository directory ApplyChanges(info); } prefix = "Successfully"; } finally { // Commit anyway even partial change changeSet = _repository.Commit(prefix + " sync with dropbox at " + DateTime.UtcNow.ToString("g"), String.Format("{0} <{1}>", info.UserName, info.Email)); } // Save new dropboc cursor _tracer.Trace("Update dropbox cursor"); _settings.SetValue(CursorKey, info.NewCursor); return changeSet; }
internal DropboxDeployInfo GetDeployInfo(string path, OAuthInfo oauth, AccountInfo account, string cursor = null) { List <DropboxEntryInfo> deltas = new List <DropboxEntryInfo>(); string timeStamp = GetUtcTimeStamp(); string oldCursor = cursor; string newCursor = ""; while (true) { DeltaInfo delta = GetDeltaInfo(oauth, cursor); newCursor = delta.cursor; if (newCursor == oldCursor) { break; } foreach (EntryInfo info in delta.entries) { DropboxEntryInfo item = new DropboxEntryInfo(); if (info.metadata != null && !info.metadata.path.StartsWith(path)) { continue; } if (info.metadata == null || info.metadata.is_deleted || string.IsNullOrEmpty(info.metadata.path)) { item.Path = info.path; item.IsDeleted = true; } else { item.Path = info.metadata.path; item.IsDirectory = info.metadata.is_dir; if (!item.IsDirectory) { item.Modified = info.metadata.modified; item.Nonce = GetNonce(); item.Signature = GetSignature(oauth, info.path, timeStamp, item.Nonce); } } deltas.Add(item); } if (!delta.has_more) { break; } cursor = newCursor; } if (deltas.Count == 0) { throw new InvalidOperationException("the repo is up-to-date."); } var deployInfo = new DropboxDeployInfo { TimeStamp = timeStamp, Token = oauth.Token, ConsumerKey = oauth.ConsumerKey, OAuthVersion = "1.0", SignatureMethod = "HMAC-SHA1", OldCursor = oldCursor, NewCursor = newCursor, Path = path, UserName = account.display_name, Email = account.email, }; deployInfo.Deltas.AddRange(deltas); return(deployInfo); }
private async Task ProcessFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc) { var oauthHeader = GetOAuthHeader(info, delta); int retries = MaxRetries; while (retries >= 0) { retries--; using (var client = new HttpClient { BaseAddress = new Uri(DropboxApiContentUri), Timeout = _timeout }) { client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", oauthHeader); try { string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant()); using (HttpResponseMessage response = await client.GetAsync(requestPath)) { using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync()) { await SafeWriteFile(parent, path, lastModifiedUtc, stream); } } if (retries < MaxRetries - 1) { Interlocked.Increment(ref _retriedCount); } Interlocked.Increment(ref _successCount); break; } catch (Exception ex) { if (retries <= 0) { Interlocked.Increment(ref _failedCount); LogError("Get({0}) '{1}'failed with {2}", MaxRetries - retries - 1, SandboxFilePath + delta.Path, ex.Message); break; } } // First retry is 1s, second retry is 20s assuming rate-limit await Task.Delay(retries == 1 ? TimeSpan.FromSeconds(20) : TimeSpan.FromSeconds(1)); } } }
public virtual async Task ProcessFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc) { var oauthHeader = GetOAuthHeader(info, delta); int retries = 0; while (retries <= MaxRetries) { using (var client = CreateDropboxHttpClient()) { client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", oauthHeader); try { string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant()); using (HttpResponseMessage response = await client.GetAsync(requestPath)) { using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync()) { await SafeWriteFile(parent, path, lastModifiedUtc, stream); } } if (retries > 0) { // Increment the successful retry count Interlocked.Increment(ref _retriedCount); } // Increment the total success counter Interlocked.Increment(ref _successCount); break; } catch (Exception ex) { if (retries == MaxRetries) { Interlocked.Increment(ref _failedCount); LogError("Get({0}) '{1}' failed with {2}", retries, SandboxFilePath + delta.Path, ex.Message); throw; } } // First retry is 1s, second retry is 20s assuming rate-limit await Task.Delay(retries == MaxRetries - 1 ? TimeSpan.FromSeconds(1) : RetryWaitToAvoidRateLimit); retries++; } } }
internal async Task ApplyChangesCore(DropboxDeployInfo info) { string parent = info.Path.TrimEnd('/') + '/'; DateTime updateMessageTime = DateTime.UtcNow; var sem = new SemaphoreSlim(MaxConcurrentRequests, MaxConcurrentRequests); var rateLimiter = new RateLimiter(MaxFilesPerSecs * 10, TimeSpan.FromSeconds(10)); var tasks = new List<Task>(); foreach (DropboxDeltaInfo delta in info.Deltas) { if (!delta.Path.StartsWith(parent, StringComparison.OrdinalIgnoreCase)) { continue; } // Ignore .git and .hg files and folders string pathWithSlash = delta.Path + "/"; if (pathWithSlash.IndexOf("/.git/", StringComparison.OrdinalIgnoreCase) != -1 || pathWithSlash.IndexOf("/.hg/", StringComparison.OrdinalIgnoreCase) != -1) { continue; } var path = delta.Path; if (delta.IsDeleted) { SafeDelete(parent, path); Interlocked.Increment(ref _successCount); } else if (delta.IsDirectory) { SafeCreateDir(parent, path); Interlocked.Increment(ref _successCount); } else { DateTime modified; if (DateTime.TryParse(delta.Modified, out modified)) { modified = modified.ToUniversalTime(); } else { modified = DateTime.UtcNow; } if (!IsFileChanged(parent, path, modified)) { LogInfo("file unchanged {0}", path); Interlocked.Increment(ref _successCount); continue; } try { var task = Task.Run(async () => { await sem.WaitAsync(); try { await rateLimiter.ThrottleAsync(); await ProcessFileAsync(info, delta, parent, path, modified); } finally { Interlocked.Increment(ref _fileCount); sem.Release(); } }); tasks.Add(task); } catch (Exception ex) { LogError("{0}", ex); throw; } } } await Task.WhenAll(tasks); }
private Task<StreamInfo> GetFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, int retries = MaxRetries) { var parameters = new Dictionary<string, string> { { "oauth_consumer_key", info.ConsumerKey }, { "oauth_signature_method", info.SignatureMethod }, { "oauth_timestamp", info.TimeStamp }, { "oauth_nonce", delta.Nonce }, { "oauth_version", info.OAuthVersion }, { "oauth_token", info.Token }, { "oauth_signature", delta.Signature } }; var sb = new StringBuilder(); foreach (var key in parameters.Keys) { if (sb.Length != 0) { sb.Append(','); } sb.AppendFormat("{0}=\"{1}\"", key, parameters[key]); } var client = new HttpClient(); client.BaseAddress = new Uri(DropboxApiContentUri); client.Timeout = _timeout; client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", sb.ToString()); // Using ContinueWith instead of Then to avoid SyncContext deadlock in 4.5 var tcs = new TaskCompletionSource<Task<StreamInfo>>(); client.GetAsync(SandboxFilePath + DropboxPathEncode(delta.Path.ToLower(CultureInfo.CurrentCulture))).ContinueWith(t => { if (t.IsFaulted) { using (client) { if (retries <= 0) { LogError("Get(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + t.Exception); tcs.TrySetException(t.Exception.InnerExceptions); } else { LogError("Retry(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + t.Exception); tcs.TrySetResult(RetryGetFileAsync(info, delta, retries)); } } } else if (t.IsCanceled) { using (client) { tcs.TrySetCanceled(); } } else { try { HttpResponseMessage response = t.Result; if ((int)response.StatusCode < 500 || retries <= 0) { response.EnsureSuccessStatusCode(); tcs.TrySetResult(GetStreamInfo(client, response, delta.Path)); if (retries < MaxRetries) { // Success due to retried Interlocked.Increment(ref _retriedCount); } } else { using (client) { using (response) { LogError("Retry(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + (int)response.StatusCode); tcs.TrySetResult(RetryGetFileAsync(info, delta, retries)); } } } } catch (Exception ex) { LogError("Get(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + ex); tcs.TrySetException(ex); } } }); return tcs.Task.FastUnwrap(); }
private Task<StreamInfo> RetryGetFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, int retries) { // First retry is 1s, second retry is 20s assuming rate-limit Thread.Sleep(retries == 1 ? 20000 : 1000); return GetFileAsync(info, delta, retries - 1); }
public async Task ApplyChangesCoreCreatesDirectoriesForDirectoryDeltas() { // Arrange var helper = CreateDropboxHelper(); var dirDeltaInfo = new DropboxEntryInfo { Path = "foo/qux/", IsDirectory = true }; var deployInfo = new DropboxDeployInfo { Path = "foo" }; deployInfo.Deltas.Add(dirDeltaInfo); string dirPath = Path.Combine(helper.Environment.RepositoryPath, "qux"); // Act await helper.ApplyChangesCore(deployInfo, useOAuth20: false); // Assert Assert.True(Directory.Exists(dirPath)); }
private static string GetOAuthHeader(DropboxDeployInfo info, DropboxDeltaInfo delta) { var parameters = new Dictionary<string, string> { { "oauth_consumer_key", info.ConsumerKey }, { "oauth_signature_method", info.SignatureMethod }, { "oauth_timestamp", info.TimeStamp }, { "oauth_nonce", delta.Nonce }, { "oauth_version", info.OAuthVersion }, { "oauth_token", info.Token }, { "oauth_signature", delta.Signature } }; var sb = new StringBuilder(); foreach (var item in parameters) { if (sb.Length != 0) { sb.Append(','); } sb.AppendFormat("{0}=\"{1}\"", item.Key, item.Value); } return sb.ToString(); }
public async Task ApplyChangesCoreThrowsIfAnyFileTaskFails() { // Arrange var helper = CreateDropboxHelper(); int processedFiles = 0; Mock.Get(helper).Setup(h => h.ProcessFileAsync(It.IsAny<HttpClient>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<DateTime>())) .Callback(() => Interlocked.Increment(ref processedFiles)) .Returns(Task.FromResult(0)); Mock.Get(helper).Setup(h => h.ProcessFileAsync(It.IsAny<HttpClient>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<DateTime>())) .Callback(() => Interlocked.Increment(ref processedFiles)) .Returns(GetFailedTask()); var deployInfo = new DropboxDeployInfo { Path = "foo" }; deployInfo.Deltas.AddRange(new [] { new DropboxEntryInfo { Path = "foo/test.txt" }, new DropboxEntryInfo { Path = "foo/bar.txt" }, new DropboxEntryInfo { Path = "foo/qux.txt" }, new DropboxEntryInfo { Path = "foo/buzz.png" }, new DropboxEntryInfo { Path = "foo/baz.php"}, new DropboxEntryInfo { Path = "foo/file0.php"}, new DropboxEntryInfo { Path = "foo/file1.php"}, new DropboxEntryInfo { Path = "foo/file2.php"}, new DropboxEntryInfo { Path = "foo/file3.php"}, }); // Act await ExceptionAssert.ThrowsAsync<HttpRequestException>(async() => await helper.ApplyChangesCore(deployInfo, useOAuth20: false)); // Assert // Ensure we processed other files Assert.Equal(deployInfo.Deltas.Count, processedFiles); }
public async Task GetDeltasPopulatesEntriesNodeWhenHasMoreIsSet() { // Arrange const string DeltaPayload1 = @"{""reset"": true, ""cursor"": ""cursor1"", ""has_more"": true, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/bar.txt"", ""is_dir"": false}]]}"; const string DeltaPayload2 = @"{""reset"": false, ""cursor"": ""cursor2"", ""has_more"": false, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/qux.txt"", ""is_dir"": false}]]}"; var dropboxInfo = new DropboxDeployInfo { Token = "Some-token", Path = "/foo" }; int i = 0; var handler = new TestMessageHandler(_ => { var content = new StringContent(i++ == 0 ? DeltaPayload1 : DeltaPayload2, Encoding.UTF8, "application/json"); return new HttpResponseMessage { Content = content }; }); var helper = CreateDropboxHelper(handler); // Act await helper.UpdateDropboxDeployInfo(dropboxInfo); // Assert Assert.Equal("cursor2", dropboxInfo.NewCursor); Assert.Equal(2, dropboxInfo.Deltas.Count); Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path); Assert.Equal("/foo/qux.txt", dropboxInfo.Deltas[1].Path); }
public async Task ApplyChangesCoreDeletesFilesForDeltasThatHaveBeenDeleted() { // Arrange var helper = CreateDropboxHelper(); var fileDeltaInfo = new DropboxEntryInfo { Path = "foo/bar.txt", IsDeleted = true }; var dirDeltaInfo = new DropboxEntryInfo { Path = "foo/baz/", IsDeleted = true, IsDirectory = true }; var deployInfo = new DropboxDeployInfo { Path = "foo" }; deployInfo.Deltas.AddRange(new [] { fileDeltaInfo, dirDeltaInfo }); string filePath = Path.Combine(helper.Environment.RepositoryPath, "bar.txt"), dirPath = Path.Combine(helper.Environment.RepositoryPath, "baz"); File.WriteAllBytes(filePath, new byte[0]); Directory.CreateDirectory(dirPath); // Act await helper.ApplyChangesCore(deployInfo, useOAuth20: false); // Assert Assert.False(File.Exists(filePath)); Assert.False(Directory.Exists(dirPath)); }
private void ApplyChanges(DropboxDeployInfo info) { Semaphore sem = new Semaphore(MaxConcurrentRequests, MaxConcurrentRequests); List<Task> tasks = new List<Task>(); string parent = info.Path.TrimEnd('/') + '/'; foreach (DropboxDeltaInfo delta in info.Deltas) { if (!delta.Path.StartsWith(parent, StringComparison.OrdinalIgnoreCase)) { continue; } var path = delta.Path; if (delta.IsDeleted) { SafeDelete(parent, path); } else if (delta.IsDirectory) { SafeCreateDir(parent, path); } else { DateTime modified = DateTime.Parse(delta.Modified).ToUniversalTime(); if (!IsFileChanged(parent, path, modified)) { _tracer.Trace("file unchanged {0}", path); continue; } // throttle concurrent get file dropbox sem.WaitOne(); Task task; try { // Using ContinueWith instead of Then to avoid SyncContext deadlock in 4.5 task = GetFileAsync(info, delta).ContinueWith(t => { sem.Release(); if (!t.IsFaulted && !t.IsCanceled) { using (Stream stream = t.Result) { SafeWriteFile(parent, path, stream, modified); } } return t; }).FastUnwrap(); } catch (Exception ex) { sem.Release(); _tracer.TraceError(ex); Task.WaitAll(tasks.ToArray(), _timeout); throw; } tasks.Add(task); } } if (!Task.WaitAll(tasks.ToArray(), _timeout)) { throw new TimeoutException(RS.Format(Resources.Error_SyncDropboxTimeout, (int)_timeout.TotalSeconds)); } }
private async Task ProcessFileAsyncCore(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc) { var parameters = new Dictionary<string, string> { { "oauth_consumer_key", info.ConsumerKey }, { "oauth_signature_method", info.SignatureMethod }, { "oauth_timestamp", info.TimeStamp }, { "oauth_nonce", delta.Nonce }, { "oauth_version", info.OAuthVersion }, { "oauth_token", info.Token }, { "oauth_signature", delta.Signature } }; var sb = new StringBuilder(); foreach (var key in parameters.Keys) { if (sb.Length != 0) { sb.Append(','); } sb.AppendFormat("{0}=\"{1}\"", key, parameters[key]); } int retries = MaxRetries; while (retries >= 0) { retries--; using (var client = new HttpClient { BaseAddress = new Uri(DropboxApiContentUri), Timeout = _timeout }) { client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", sb.ToString()); try { string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant()); using (HttpResponseMessage response = await client.GetAsync(requestPath)) { using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync()) { await SafeWriteFile(parent, path, lastModifiedUtc, stream); } } if (retries < MaxRetries - 1) { Interlocked.Increment(ref _retriedCount); } Interlocked.Increment(ref _successCount); break; } catch (Exception ex) { if (retries <= 0) { Interlocked.Increment(ref _failedCount); LogError("Get({0}) '{1}'failed with {2}", MaxRetries - retries - 1, SandboxFilePath + delta.Path, ex.Message); break; } else { // First retry is 1s, second retry is 20s assuming rate-limit Thread.Sleep(retries == 1 ? TimeSpan.FromSeconds(20) : TimeSpan.FromSeconds(1)); continue; } } finally { Interlocked.Increment(ref _fileCount); } } } }
private Task<Stream> GetFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta) { var parameters = new Dictionary<string, string> { { "oauth_consumer_key", info.ConsumerKey }, { "oauth_signature_method", info.SignatureMethod }, { "oauth_timestamp", info.TimeStamp }, { "oauth_nonce", delta.Nonce }, { "oauth_version", info.OAuthVersion }, { "oauth_token", info.Token }, { "oauth_signature", delta.Signature } }; var sb = new StringBuilder(); foreach (var key in parameters.Keys) { if (sb.Length != 0) { sb.Append(','); } sb.AppendFormat("{0}=\"{1}\"", key, parameters[key]); } var client = new HttpClient(); client.BaseAddress = new Uri(DropboxApiContentUri); client.Timeout = _timeout; client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", sb.ToString()); // Using ContinueWith instead of Then to avoid SyncContext deadlock in 4.5 TaskCompletionSource<Task<Stream>> tcs = new TaskCompletionSource<Task<Stream>>(); client.GetAsync(SandboxFilePath + delta.Path.ToLower()).ContinueWith(t => { if (t.IsFaulted) { _tracer.TraceError("Get '" + SandboxFilePath + delta.Path + "' failed with " + t.Exception); tcs.TrySetException(t.Exception.InnerExceptions); } else if (t.IsCanceled) { tcs.TrySetCanceled(); } else { try { tcs.TrySetResult(t.Result.EnsureSuccessStatusCode().Content.ReadAsStreamAsync()); } catch (Exception ex) { _tracer.TraceError("Get '" + SandboxFilePath + delta.Path + "' failed with " + ex); tcs.TrySetException(ex); } } }); return tcs.Task.FastUnwrap(); }