Ejemplo n.º 1
0
        public async Task GetDeltasPopulatesEntriesNodeWhenHasMoreIsSet()
        {
            // Arrange
            const string DeltaPayload1 = @"{""reset"": true, ""cursor"": ""cursor1"", ""has_more"": true, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/bar.txt"", ""is_dir"": false}]]}";
            const string DeltaPayload2 = @"{""reset"": false, ""cursor"": ""cursor2"", ""has_more"": false, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/qux.txt"", ""is_dir"": false}]]}";
            var          dropboxInfo   = new DropboxDeployInfo
            {
                Token = "Some-token",
                Path  = "/foo"
            };
            int i       = 0;
            var handler = new TestMessageHandler(_ =>
            {
                var content = new StringContent(i++ == 0 ? DeltaPayload1 : DeltaPayload2, Encoding.UTF8, "application/json");
                return(new HttpResponseMessage {
                    Content = content
                });
            });
            var helper = CreateDropboxHelper(handler);

            // Act
            await helper.UpdateDropboxDeployInfo(dropboxInfo);

            // Assert
            Assert.Equal("cursor2", dropboxInfo.NewCursor);
            Assert.Equal(2, dropboxInfo.Deltas.Count);
            Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path);
            Assert.Equal("/foo/qux.txt", dropboxInfo.Deltas[1].Path);
        }
Ejemplo n.º 2
0
        public async Task ApplyChangesCoreDeletesFilesForDeltasThatHaveBeenDeleted()
        {
            // Arrange
            var helper        = CreateDropboxHelper();
            var fileDeltaInfo = new DropboxEntryInfo {
                Path = "foo/bar.txt", IsDeleted = true
            };
            var dirDeltaInfo = new DropboxEntryInfo {
                Path = "foo/baz/", IsDeleted = true, IsDirectory = true
            };
            var deployInfo = new DropboxDeployInfo {
                Path = "foo"
            };

            deployInfo.Deltas.AddRange(new [] { fileDeltaInfo, dirDeltaInfo });
            string filePath = Path.Combine(helper.Environment.RepositoryPath, "bar.txt"),
                   dirPath  = Path.Combine(helper.Environment.RepositoryPath, "baz");

            File.WriteAllBytes(filePath, new byte[0]);
            Directory.CreateDirectory(dirPath);

            // Act
            await helper.ApplyChangesCore(deployInfo, useOAuth20 : false);

            // Assert
            Assert.False(File.Exists(filePath));
            Assert.False(Directory.Exists(dirPath));
        }
Ejemplo n.º 3
0
        public async Task GetDeltasPopulatesEntriesNodeWhenHasMoreIsSet()
        {
            // Arrange
            const string DeltaPayload1 = @"{""cursor"": ""cursor1"", ""has_more"": true, ""entries"": [{"".tag"": ""file"", ""path_display"": ""/foo/bar.txt"", ""server_modified"": ""2017-08-30T19:20:02Z""}]}";
            const string DeltaPayload2 = @"{""cursor"": ""cursor2"", ""has_more"": false, ""entries"": [{"".tag"": ""file"", ""path_display"": ""/foo/qux.txt"", ""server_modified"": ""2017-08-30T21:40:35Z""}]}";
            var          dropboxInfo   = new DropboxDeployInfo
            {
                Token = "Some-token",
                Path  = "/foo"
            };
            int i       = 0;
            var handler = new TestMessageHandler(_ =>
            {
                var content = new StringContent(i++ == 0 ? DeltaPayload1 : DeltaPayload2, Encoding.UTF8, "application/json");
                return(new HttpResponseMessage {
                    Content = content
                });
            });
            var helper = CreateDropboxHelper(handler);

            // Act
            await helper.UpdateDropboxDeployInfo(dropboxInfo);

            // Assert
            Assert.Equal("cursor2", dropboxInfo.NewCursor);
            Assert.Equal(2, dropboxInfo.Deltas.Count);
            Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path);
            Assert.Equal("/foo/qux.txt", dropboxInfo.Deltas[1].Path);
        }
Ejemplo n.º 4
0
        public async Task TestDropboxBasic()
        {
            OAuthInfo oauth = GetOAuthInfo();

            if (oauth == null)
            {
                // only run in private kudu
                return;
            }

            AccountInfo       account = GetAccountInfo(oauth);
            DropboxDeployInfo deploy  = GetDeployInfo("/BasicTest", oauth, account);

            string appName = "DropboxTest";
            await ApplicationManager.RunAsync(appName, async appManager =>
            {
                HttpClient client = HttpClientHelper.CreateClient(appManager.ServiceUrl, appManager.DeploymentManager.Credentials);
                var result        = await client.PostAsJsonAsync("deploy?scmType=Dropbox", deploy);
                result.EnsureSuccessful();

                await Task.WhenAll(
                    KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/default.html", "Hello Default!"),
                    KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/temp/temp.html", "Hello Temp!"),
                    KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/New Folder/New File.html", "Hello New File!")
                    );
            });
        }
Ejemplo n.º 5
0
        public async Task ApplyChangesCoreThrowsIfAnyFileTaskFails()
        {
            // Arrange
            var helper         = CreateDropboxHelper();
            int processedFiles = 0;

            Mock.Get(helper).Setup(h => h.ProcessFileAsync(It.IsAny <HttpClient>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <DateTime>()))
            .Callback(() => Interlocked.Increment(ref processedFiles))
            .Returns(Task.FromResult(0));

            Mock.Get(helper).Setup(h => h.ProcessFileAsync(It.IsAny <HttpClient>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <DateTime>()))
            .Callback(() => Interlocked.Increment(ref processedFiles))
            .Returns(GetFailedTask());
            var deployInfo = new DropboxDeployInfo {
                Path = "foo"
            };

            deployInfo.Deltas.AddRange(new []
            {
                new DropboxEntryInfo {
                    Path = "foo/test.txt"
                },
                new DropboxEntryInfo {
                    Path = "foo/bar.txt"
                },
                new DropboxEntryInfo {
                    Path = "foo/qux.txt"
                },
                new DropboxEntryInfo {
                    Path = "foo/buzz.png"
                },
                new DropboxEntryInfo {
                    Path = "foo/baz.php"
                },
                new DropboxEntryInfo {
                    Path = "foo/file0.php"
                },
                new DropboxEntryInfo {
                    Path = "foo/file1.php"
                },
                new DropboxEntryInfo {
                    Path = "foo/file2.php"
                },
                new DropboxEntryInfo {
                    Path = "foo/file3.php"
                },
            });

            // Act
            await Assert.ThrowsAsync <HttpRequestException>(async() => await helper.ApplyChangesCore(deployInfo, useOAuth20: false));

            // Assert
            // Ensure we processed other files
            Assert.Equal(deployInfo.Deltas.Count, processedFiles);
        }
Ejemplo n.º 6
0
        public async Task TestDropboxBasicForBasicScenario(Scenario scenario)
        {
            OAuthInfo oauth = GetOAuthInfo();

            if (oauth == null)
            {
                // only run in private kudu
                return;
            }

            AccountInfo       account = GetAccountInfo(oauth);
            DropboxDeployInfo deploy  = GetDeployInfo("/BasicTest", oauth, account);

            string appName = "DropboxTest";
            await ApplicationManager.RunAsync(appName, async appManager =>
            {
                if (scenario == Scenario.NoRepository)
                {
                    await appManager.SettingsManager.SetValue(SettingsKeys.NoRepository, "1");
                }
                else if (scenario == Scenario.InPlace)
                {
                    await appManager.SettingsManager.SetValue(SettingsKeys.RepositoryPath, "wwwroot");
                }

                HttpClient client = HttpClientHelper.CreateClient(appManager.ServiceUrl, appManager.DeploymentManager.Credentials);
                var result        = await client.PostAsJsonAsync("deploy?scmType=Dropbox", deploy);
                result.EnsureSuccessful();

                await Task.WhenAll(
                    KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/default.html", "Hello Default!"),
                    KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/temp/temp.html", "Hello Temp!"),
                    KuduAssert.VerifyUrlAsync(appManager.SiteUrl + "/New Folder/New File.html", "Hello New File!")
                    );

                var repositoryGit = appManager.VfsManager.Exists(@"site\repository\.git");
                var wwwrootGit    = appManager.VfsManager.Exists(@"site\wwwroot\.git");

                if (scenario == Scenario.NoRepository)
                {
                    Assert.False(repositoryGit, @"site\repository\.git should not exist for " + scenario);
                    Assert.False(wwwrootGit, @"site\wwwroot\.git should not exist for " + scenario);
                }
                else if (scenario == Scenario.InPlace)
                {
                    Assert.False(repositoryGit, @"site\repository\.git should not exist for " + scenario);
                    Assert.True(wwwrootGit, @"site\wwwroot\.git should exist for " + scenario);
                }
                else if (scenario == Scenario.Default)
                {
                    Assert.True(repositoryGit, @"site\repository\.git should exist for " + scenario);
                    Assert.False(wwwrootGit, @"site\wwwroot\.git should not exist for " + scenario);
                }
            });
        }
Ejemplo n.º 7
0
        public void TestDropboxConcurrent()
        {
            OAuthInfo oauth = GetOAuthInfo();

            if (oauth == null)
            {
                // only run in private kudu
                return;
            }

            AccountInfo       account = GetAccountInfo(oauth);
            DropboxDeployInfo deploy  = GetDeployInfo("/BasicTest", oauth, account);

            string appName = "DropboxTest";

            ApplicationManager.Run(appName, appManager =>
            {
                HttpClient client = HttpClientHelper.CreateClient(appManager.ServiceUrl, appManager.DeploymentManager.Credentials);
                var tasks         = new Task <HttpResponseMessage>[]
                {
                    client.PostAsJsonAsync("deploy", deploy),
                    client.PostAsJsonAsync("deploy", deploy)
                };

                Task.WaitAll(tasks);

                // both 200 and 202 is success
                tasks[0].Result.EnsureSuccessful();
                tasks[1].Result.EnsureSuccessful();

                var success = (tasks[0].Result.StatusCode == HttpStatusCode.OK) ? tasks[0].Result : tasks[1].Result;
                var failure = !Object.ReferenceEquals(success, tasks[0].Result) ? tasks[0].Result : tasks[1].Result;

                success.EnsureSuccessful();
                Assert.Equal(HttpStatusCode.Accepted, failure.StatusCode);

                var results = appManager.DeploymentManager.GetResultsAsync().Result.ToList();
                Assert.Equal(1, results.Count);
                Assert.Equal(DeployStatus.Success, results[0].Status);
                Assert.Equal("Dropbox", results[0].Deployer);
            });
        }
Ejemplo n.º 8
0
        public async Task ApplyChangesCoreCreatesDirectoriesForDirectoryDeltas()
        {
            // Arrange
            var helper       = CreateDropboxHelper();
            var dirDeltaInfo = new DropboxEntryInfo {
                Path = "foo/qux/", IsDirectory = true
            };
            var deployInfo = new DropboxDeployInfo {
                Path = "foo"
            };

            deployInfo.Deltas.Add(dirDeltaInfo);
            string dirPath = Path.Combine(helper.Environment.RepositoryPath, "qux");

            // Act
            await helper.ApplyChangesCore(deployInfo, useOAuth20 : false);

            // Assert
            Assert.True(Directory.Exists(dirPath));
        }
Ejemplo n.º 9
0
        public async Task GetDeltasPopulatesEntriesNode()
        {
            // Arrange
            const string DeltaPayload = @"{""reset"": true, ""cursor"": ""AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA"", ""has_more"": false, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""thumb_exists"": false, ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/bar.txt"", ""is_dir"": false, ""icon"": ""page_white"", ""root"": ""dropbox"", ""mime_type"": ""application/epub+zip"", ""size"": ""120.7 KB""}]]}";
            var          dropboxInfo  = new DropboxDeployInfo
            {
                Token = "Some-token",
                Path  = "/foo"
            };
            var handler = new TestMessageHandler(DeltaPayload, isJson: true);
            var helper  = CreateDropboxHelper(handler);

            // Act
            await helper.UpdateDropboxDeployInfo(dropboxInfo);

            // Assert
            Assert.Null(dropboxInfo.OldCursor);
            Assert.Equal("AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA", dropboxInfo.NewCursor);
            Assert.Equal(1, dropboxInfo.Deltas.Count);
            Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path);
        }
Ejemplo n.º 10
0
        public async Task GetDeltasPopulatesEntriesNode()
        {
            // Arrange
            const string DeltaPayload = @"{""cursor"": ""AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA"", ""has_more"": false, ""entries"": [{"".tag"": ""file"", ""path_display"": ""/foo/bar.txt"", ""server_modified"": ""2017-08-30T18:50:02Z""}]}";
            var          dropboxInfo  = new DropboxDeployInfo
            {
                Token = "Some-token",
                Path  = "/foo"
            };
            var handler = new TestMessageHandler(DeltaPayload, isJson: true);
            var helper  = CreateDropboxHelper(handler);

            // Act
            await helper.UpdateDropboxDeployInfo(dropboxInfo);

            // Assert
            Assert.Null(dropboxInfo.OldCursor);
            Assert.Equal("AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA", dropboxInfo.NewCursor);
            Assert.Equal(1, dropboxInfo.Deltas.Count);
            Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path);
        }
Ejemplo n.º 11
0
        public void TestDropboxSpecialFolderChars()
        {
            OAuthInfo oauth = GetOAuthInfo();

            if (oauth == null)
            {
                // only run in private kudu
                return;
            }

            AccountInfo       account = GetAccountInfo(oauth);
            DropboxDeployInfo deploy  = GetDeployInfo("/!@#$faiztest$#!@", oauth, account);

            string appName = "SpecialCharsTest";

            ApplicationManager.Run(appName, appManager =>
            {
                HttpClient client = HttpClientHelper.CreateClient(appManager.ServiceUrl, appManager.DeploymentManager.Credentials);
                client.PostAsJsonAsync("deploy?scmType=Dropbox", deploy).Result.EnsureSuccessful();
            });
        }
Ejemplo n.º 12
0
        public ChangeSet Sync(DropboxDeployInfo info, string branch)
        {
            if (_settings.GetValue(CursorKey) != info.OldCursor)
            {
                throw new InvalidOperationException(Resources.Error_MismatchDropboxCursor);
            }

            if (!IsEmptyRepo())
            {
                // git checkout --force <branch>
                _repository.Update(branch);
            }

            ChangeSet changeSet;
            string prefix = "Partially";
            try
            {
                using (_tracer.Step("Synch with Dropbox"))
                {
                    // Sync dropbox => repository directory
                    ApplyChanges(info);
                }

                prefix = "Successfully";
            }
            finally
            {
                // Commit anyway even partial change
                changeSet = _repository.Commit(prefix + " sync with dropbox at " + DateTime.UtcNow.ToString("g"), String.Format("{0} <{1}>", info.UserName, info.Email));
            }

            // Save new dropboc cursor
            _tracer.Trace("Update dropbox cursor");
            _settings.SetValue(CursorKey, info.NewCursor);

            return changeSet;
        }
Ejemplo n.º 13
0
        internal DropboxDeployInfo GetDeployInfo(string path, OAuthInfo oauth, AccountInfo account, string cursor = null)
        {
            List <DropboxEntryInfo> deltas = new List <DropboxEntryInfo>();
            string timeStamp = GetUtcTimeStamp();
            string oldCursor = cursor;
            string newCursor = "";

            while (true)
            {
                DeltaInfo delta = GetDeltaInfo(oauth, cursor);
                newCursor = delta.cursor;
                if (newCursor == oldCursor)
                {
                    break;
                }

                foreach (EntryInfo info in delta.entries)
                {
                    DropboxEntryInfo item = new DropboxEntryInfo();

                    if (info.metadata != null && !info.metadata.path.StartsWith(path))
                    {
                        continue;
                    }

                    if (info.metadata == null || info.metadata.is_deleted || string.IsNullOrEmpty(info.metadata.path))
                    {
                        item.Path      = info.path;
                        item.IsDeleted = true;
                    }
                    else
                    {
                        item.Path        = info.metadata.path;
                        item.IsDirectory = info.metadata.is_dir;
                        if (!item.IsDirectory)
                        {
                            item.Modified  = info.metadata.modified;
                            item.Nonce     = GetNonce();
                            item.Signature = GetSignature(oauth, info.path, timeStamp, item.Nonce);
                        }
                    }

                    deltas.Add(item);
                }

                if (!delta.has_more)
                {
                    break;
                }

                cursor = newCursor;
            }

            if (deltas.Count == 0)
            {
                throw new InvalidOperationException("the repo is up-to-date.");
            }

            var deployInfo = new DropboxDeployInfo
            {
                TimeStamp       = timeStamp,
                Token           = oauth.Token,
                ConsumerKey     = oauth.ConsumerKey,
                OAuthVersion    = "1.0",
                SignatureMethod = "HMAC-SHA1",
                OldCursor       = oldCursor,
                NewCursor       = newCursor,
                Path            = path,
                UserName        = account.display_name,
                Email           = account.email,
            };

            deployInfo.Deltas.AddRange(deltas);

            return(deployInfo);
        }
Ejemplo n.º 14
0
        private async Task ProcessFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc)
        {
            var oauthHeader = GetOAuthHeader(info, delta);

            int retries = MaxRetries;
            while (retries >= 0)
            {
                retries--;

                using (var client = new HttpClient { BaseAddress = new Uri(DropboxApiContentUri), Timeout = _timeout })
                {
                    client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", oauthHeader);
                    try
                    {
                        string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant());
                        using (HttpResponseMessage response = await client.GetAsync(requestPath))
                        {
                            using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync())
                            {
                                await SafeWriteFile(parent, path, lastModifiedUtc, stream);
                            }
                        }
                        if (retries < MaxRetries - 1)
                        {
                            Interlocked.Increment(ref _retriedCount);
                        }
                        Interlocked.Increment(ref _successCount);

                        break;
                    }
                    catch (Exception ex)
                    {
                        if (retries <= 0)
                        {
                            Interlocked.Increment(ref _failedCount);
                            LogError("Get({0}) '{1}'failed with {2}", MaxRetries - retries - 1, SandboxFilePath + delta.Path, ex.Message);
                            break;
                        }
                    }

                    // First retry is 1s, second retry is 20s assuming rate-limit
                    await Task.Delay(retries == 1 ? TimeSpan.FromSeconds(20) : TimeSpan.FromSeconds(1));
                }
            }
        }
Ejemplo n.º 15
0
        public virtual async Task ProcessFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc)
        {
            var oauthHeader = GetOAuthHeader(info, delta);
            int retries = 0;
            while (retries <= MaxRetries)
            {
                using (var client = CreateDropboxHttpClient())
                {
                    client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", oauthHeader);
                    try
                    {
                        string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant());
                        using (HttpResponseMessage response = await client.GetAsync(requestPath))
                        {
                            using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync())
                            {
                                await SafeWriteFile(parent, path, lastModifiedUtc, stream);
                            }
                        }
                        if (retries > 0)
                        {
                            // Increment the successful retry count
                            Interlocked.Increment(ref _retriedCount);
                        }
                        // Increment the total success counter
                        Interlocked.Increment(ref _successCount);

                        break;
                    }
                    catch (Exception ex)
                    {
                        if (retries == MaxRetries)
                        {
                            Interlocked.Increment(ref _failedCount);
                            LogError("Get({0}) '{1}' failed with {2}", retries, SandboxFilePath + delta.Path, ex.Message);
                            throw;
                        }
                    }

                    // First retry is 1s, second retry is 20s assuming rate-limit
                    await Task.Delay(retries == MaxRetries - 1 ? TimeSpan.FromSeconds(1) : RetryWaitToAvoidRateLimit);
                    retries++;
                }
            }
        }
Ejemplo n.º 16
0
        internal async Task ApplyChangesCore(DropboxDeployInfo info)
        {
            string parent = info.Path.TrimEnd('/') + '/';
            DateTime updateMessageTime = DateTime.UtcNow;
            var sem = new SemaphoreSlim(MaxConcurrentRequests, MaxConcurrentRequests);
            var rateLimiter = new RateLimiter(MaxFilesPerSecs * 10, TimeSpan.FromSeconds(10));
            var tasks = new List<Task>();

            foreach (DropboxDeltaInfo delta in info.Deltas)
            {
                if (!delta.Path.StartsWith(parent, StringComparison.OrdinalIgnoreCase))
                {
                    continue;
                }

                // Ignore .git and .hg files and folders
                string pathWithSlash = delta.Path + "/";
                if (pathWithSlash.IndexOf("/.git/", StringComparison.OrdinalIgnoreCase) != -1 ||
                    pathWithSlash.IndexOf("/.hg/", StringComparison.OrdinalIgnoreCase) != -1)
                {
                    continue;
                }

                var path = delta.Path;
                if (delta.IsDeleted)
                {
                    SafeDelete(parent, path);
                    Interlocked.Increment(ref _successCount);
                }
                else if (delta.IsDirectory)
                {
                    SafeCreateDir(parent, path);
                    Interlocked.Increment(ref _successCount);
                }
                else
                {
                    DateTime modified;
                    if (DateTime.TryParse(delta.Modified, out modified))
                    {
                        modified = modified.ToUniversalTime();
                    }
                    else
                    {
                        modified = DateTime.UtcNow;
                    }

                    if (!IsFileChanged(parent, path, modified))
                    {
                        LogInfo("file unchanged {0}", path);
                        Interlocked.Increment(ref _successCount);
                        continue;
                    }

                    try
                    {
                        var task = Task.Run(async () =>
                        {
                            await sem.WaitAsync();
                            try
                            {
                                await rateLimiter.ThrottleAsync();
                                await ProcessFileAsync(info, delta, parent, path, modified);
                            }
                            finally
                            {
                                Interlocked.Increment(ref _fileCount);
                                sem.Release();
                            }
                        });
                        tasks.Add(task);
                    }
                    catch (Exception ex)
                    {
                        LogError("{0}", ex);
                        throw;
                    }
                }
            }
            await Task.WhenAll(tasks);
        }
Ejemplo n.º 17
0
        private Task<StreamInfo> GetFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, int retries = MaxRetries)
        {
            var parameters = new Dictionary<string, string>
            {
                { "oauth_consumer_key", info.ConsumerKey },
                { "oauth_signature_method", info.SignatureMethod },
                { "oauth_timestamp", info.TimeStamp },
                { "oauth_nonce", delta.Nonce },
                { "oauth_version", info.OAuthVersion },
                { "oauth_token", info.Token },
                { "oauth_signature", delta.Signature }
            };

            var sb = new StringBuilder();
            foreach (var key in parameters.Keys)
            {
                if (sb.Length != 0)
                {
                    sb.Append(',');
                }
                sb.AppendFormat("{0}=\"{1}\"", key, parameters[key]);
            }

            var client = new HttpClient();
            client.BaseAddress = new Uri(DropboxApiContentUri);
            client.Timeout = _timeout;
            client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", sb.ToString());

            // Using ContinueWith instead of Then to avoid SyncContext deadlock in 4.5
            var tcs = new TaskCompletionSource<Task<StreamInfo>>();
            client.GetAsync(SandboxFilePath + DropboxPathEncode(delta.Path.ToLower(CultureInfo.CurrentCulture))).ContinueWith(t =>
            {
                if (t.IsFaulted)
                {
                    using (client)
                    {
                        if (retries <= 0)
                        {
                            LogError("Get(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + t.Exception);
                            tcs.TrySetException(t.Exception.InnerExceptions);
                        }
                        else
                        {
                            LogError("Retry(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + t.Exception);
                            tcs.TrySetResult(RetryGetFileAsync(info, delta, retries));
                        }
                    }
                }
                else if (t.IsCanceled)
                {
                    using (client)
                    {
                        tcs.TrySetCanceled();
                    }
                }
                else
                {
                    try
                    {
                        HttpResponseMessage response = t.Result;
                        if ((int)response.StatusCode < 500 || retries <= 0)
                        {
                            response.EnsureSuccessStatusCode();
                            tcs.TrySetResult(GetStreamInfo(client, response, delta.Path));
                            if (retries < MaxRetries)
                            {
                                // Success due to retried
                                Interlocked.Increment(ref _retriedCount);
                            }
                        }
                        else
                        {
                            using (client)
                            {
                                using (response)
                                {
                                    LogError("Retry(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + (int)response.StatusCode);
                                    tcs.TrySetResult(RetryGetFileAsync(info, delta, retries));
                                }
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        LogError("Get(" + retries + ") '" + SandboxFilePath + delta.Path + "' failed with " + ex);
                        tcs.TrySetException(ex);
                    }
                }
            });

            return tcs.Task.FastUnwrap();
        }
Ejemplo n.º 18
0
        private Task<StreamInfo> RetryGetFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, int retries)
        {
            // First retry is 1s, second retry is 20s assuming rate-limit
            Thread.Sleep(retries == 1 ? 20000 : 1000);

            return GetFileAsync(info, delta, retries - 1);
        }
Ejemplo n.º 19
0
        public async Task ApplyChangesCoreCreatesDirectoriesForDirectoryDeltas()
        {
            // Arrange
            var helper = CreateDropboxHelper();
            var dirDeltaInfo = new DropboxEntryInfo { Path = "foo/qux/", IsDirectory = true };
            var deployInfo = new DropboxDeployInfo { Path = "foo" };
            deployInfo.Deltas.Add(dirDeltaInfo);
            string dirPath = Path.Combine(helper.Environment.RepositoryPath, "qux");

            // Act
            await helper.ApplyChangesCore(deployInfo, useOAuth20: false);

            // Assert
            Assert.True(Directory.Exists(dirPath));
        }
Ejemplo n.º 20
0
        private static string GetOAuthHeader(DropboxDeployInfo info, DropboxDeltaInfo delta)
        {
            var parameters = new Dictionary<string, string>
            {
                { "oauth_consumer_key", info.ConsumerKey },
                { "oauth_signature_method", info.SignatureMethod },
                { "oauth_timestamp", info.TimeStamp },
                { "oauth_nonce", delta.Nonce },
                { "oauth_version", info.OAuthVersion },
                { "oauth_token", info.Token },
                { "oauth_signature", delta.Signature }
            };

            var sb = new StringBuilder();
            foreach (var item in parameters)
            {
                if (sb.Length != 0)
                {
                    sb.Append(',');
                }
                sb.AppendFormat("{0}=\"{1}\"", item.Key, item.Value);
            }
            return sb.ToString();
        }
Ejemplo n.º 21
0
        public async Task GetDeltasPopulatesEntriesNode()
        {
            // Arrange
            const string DeltaPayload = @"{""reset"": true, ""cursor"": ""AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA"", ""has_more"": false, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""thumb_exists"": false, ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/bar.txt"", ""is_dir"": false, ""icon"": ""page_white"", ""root"": ""dropbox"", ""mime_type"": ""application/epub+zip"", ""size"": ""120.7 KB""}]]}";
            var dropboxInfo = new DropboxDeployInfo
            {
                Token = "Some-token",
                Path = "/foo"
            };
            var handler = new TestMessageHandler(DeltaPayload, isJson: true);
            var helper = CreateDropboxHelper(handler);

            // Act
            await helper.UpdateDropboxDeployInfo(dropboxInfo);

            // Assert
            Assert.Null(dropboxInfo.OldCursor);
            Assert.Equal("AAGWKUylpghsuMRcKQSdHSpvUW3uPVcIyGINt30oO36wDebzBoqtFFaiqzNCWV568-U_uZwdM1QGyzxYw3GxJRsCWv0G3BlOUiguFrttRsbpmA", dropboxInfo.NewCursor);
            Assert.Equal(1, dropboxInfo.Deltas.Count);
            Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path);
        }
Ejemplo n.º 22
0
        public async Task ApplyChangesCoreThrowsIfAnyFileTaskFails()
        {
            // Arrange
            var helper = CreateDropboxHelper();
            int processedFiles = 0;
            Mock.Get(helper).Setup(h => h.ProcessFileAsync(It.IsAny<HttpClient>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<DateTime>()))
                            .Callback(() => Interlocked.Increment(ref processedFiles))
                            .Returns(Task.FromResult(0));

            Mock.Get(helper).Setup(h => h.ProcessFileAsync(It.IsAny<HttpClient>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<DateTime>()))
                            .Callback(() => Interlocked.Increment(ref processedFiles))
                            .Returns(GetFailedTask());
            var deployInfo = new DropboxDeployInfo { Path = "foo" };
            deployInfo.Deltas.AddRange(new [] 
            { 
                new DropboxEntryInfo { Path = "foo/test.txt" },
                new DropboxEntryInfo { Path = "foo/bar.txt" },
                new DropboxEntryInfo { Path = "foo/qux.txt" },
                new DropboxEntryInfo { Path = "foo/buzz.png" },
                new DropboxEntryInfo { Path = "foo/baz.php"},
                new DropboxEntryInfo { Path = "foo/file0.php"},
                new DropboxEntryInfo { Path = "foo/file1.php"},
                new DropboxEntryInfo { Path = "foo/file2.php"},
                new DropboxEntryInfo { Path = "foo/file3.php"},
            });

            // Act
            await ExceptionAssert.ThrowsAsync<HttpRequestException>(async() => await helper.ApplyChangesCore(deployInfo, useOAuth20: false));

            // Assert
            // Ensure we processed other files
            Assert.Equal(deployInfo.Deltas.Count, processedFiles);
        }
Ejemplo n.º 23
0
        public async Task GetDeltasPopulatesEntriesNodeWhenHasMoreIsSet()
        {
            // Arrange
            const string DeltaPayload1 = @"{""reset"": true, ""cursor"": ""cursor1"", ""has_more"": true, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/bar.txt"", ""is_dir"": false}]]}";
            const string DeltaPayload2 = @"{""reset"": false, ""cursor"": ""cursor2"", ""has_more"": false, ""entries"": [[""/foo/bar.txt"", {""revision"": 1, ""rev"": ""11357a5a5"", ""bytes"": 123641, ""modified"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""client_mtime"": ""Thu, 22 Aug 2013 22:50:24 +0000"", ""path"": ""/foo/qux.txt"", ""is_dir"": false}]]}";
            var dropboxInfo = new DropboxDeployInfo
            {
                Token = "Some-token",
                Path = "/foo"
            };
            int i = 0;
            var handler = new TestMessageHandler(_ => 
            {
                var content = new StringContent(i++ == 0 ? DeltaPayload1 : DeltaPayload2, Encoding.UTF8, "application/json");
                return new HttpResponseMessage { Content = content };
            });
            var helper = CreateDropboxHelper(handler);

            // Act
            await helper.UpdateDropboxDeployInfo(dropboxInfo);

            // Assert
            Assert.Equal("cursor2", dropboxInfo.NewCursor);
            Assert.Equal(2, dropboxInfo.Deltas.Count);
            Assert.Equal("/foo/bar.txt", dropboxInfo.Deltas[0].Path);
            Assert.Equal("/foo/qux.txt", dropboxInfo.Deltas[1].Path);
        }
Ejemplo n.º 24
0
        public async Task ApplyChangesCoreDeletesFilesForDeltasThatHaveBeenDeleted()
        {
            // Arrange
            var helper = CreateDropboxHelper();
            var fileDeltaInfo = new DropboxEntryInfo { Path = "foo/bar.txt", IsDeleted = true };
            var dirDeltaInfo = new DropboxEntryInfo { Path = "foo/baz/", IsDeleted = true, IsDirectory = true };
            var deployInfo = new DropboxDeployInfo { Path = "foo" };
            deployInfo.Deltas.AddRange(new [] { fileDeltaInfo, dirDeltaInfo });
            string filePath = Path.Combine(helper.Environment.RepositoryPath, "bar.txt"),
                   dirPath = Path.Combine(helper.Environment.RepositoryPath, "baz");

            File.WriteAllBytes(filePath, new byte[0]);
            Directory.CreateDirectory(dirPath);

            // Act
            await helper.ApplyChangesCore(deployInfo, useOAuth20: false);

            // Assert
            Assert.False(File.Exists(filePath));
            Assert.False(Directory.Exists(dirPath));
        }
Ejemplo n.º 25
0
        private void ApplyChanges(DropboxDeployInfo info)
        {
            Semaphore sem = new Semaphore(MaxConcurrentRequests, MaxConcurrentRequests);
            List<Task> tasks = new List<Task>();
            string parent = info.Path.TrimEnd('/') + '/';

            foreach (DropboxDeltaInfo delta in info.Deltas)
            {
                if (!delta.Path.StartsWith(parent, StringComparison.OrdinalIgnoreCase))
                {
                    continue;
                }

                var path = delta.Path;
                if (delta.IsDeleted)
                {
                    SafeDelete(parent, path);
                }
                else if (delta.IsDirectory)
                {
                    SafeCreateDir(parent, path);
                }
                else
                {
                    DateTime modified = DateTime.Parse(delta.Modified).ToUniversalTime();
                    if (!IsFileChanged(parent, path, modified))
                    {
                        _tracer.Trace("file unchanged {0}", path);
                        continue;
                    }

                    // throttle concurrent get file dropbox
                    sem.WaitOne();
                    Task task;
                    try
                    {
                        // Using ContinueWith instead of Then to avoid SyncContext deadlock in 4.5
                        task = GetFileAsync(info, delta).ContinueWith(t =>
                        {
                            sem.Release();
                            if (!t.IsFaulted && !t.IsCanceled)
                            {
                                using (Stream stream = t.Result)
                                {
                                    SafeWriteFile(parent, path, stream, modified);
                                }
                            }

                            return t;
                        }).FastUnwrap();
                    }
                    catch (Exception ex)
                    {
                        sem.Release();
                        _tracer.TraceError(ex);
                        Task.WaitAll(tasks.ToArray(), _timeout);
                        throw;
                    }

                    tasks.Add(task);
                }
            }

            if (!Task.WaitAll(tasks.ToArray(), _timeout))
            {
                throw new TimeoutException(RS.Format(Resources.Error_SyncDropboxTimeout, (int)_timeout.TotalSeconds));
            }
        }
Ejemplo n.º 26
0
        private async Task ProcessFileAsyncCore(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc)
        {
            var parameters = new Dictionary<string, string>
            {
                { "oauth_consumer_key", info.ConsumerKey },
                { "oauth_signature_method", info.SignatureMethod },
                { "oauth_timestamp", info.TimeStamp },
                { "oauth_nonce", delta.Nonce },
                { "oauth_version", info.OAuthVersion },
                { "oauth_token", info.Token },
                { "oauth_signature", delta.Signature }
            };

            var sb = new StringBuilder();
            foreach (var key in parameters.Keys)
            {
                if (sb.Length != 0)
                {
                    sb.Append(',');
                }
                sb.AppendFormat("{0}=\"{1}\"", key, parameters[key]);
            }

            int retries = MaxRetries;

            while (retries >= 0)
            {
                retries--;

                using (var client = new HttpClient { BaseAddress = new Uri(DropboxApiContentUri), Timeout = _timeout })
                {
                    client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", sb.ToString());
                    try
                    {
                        string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant());
                        using (HttpResponseMessage response = await client.GetAsync(requestPath))
                        {
                            using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync())
                            {
                                await SafeWriteFile(parent, path, lastModifiedUtc, stream);
                            }
                        }
                        if (retries < MaxRetries - 1)
                        {
                            Interlocked.Increment(ref _retriedCount);
                        }
                        Interlocked.Increment(ref _successCount);

                        break;
                    }
                    catch (Exception ex)
                    {
                        if (retries <= 0)
                        {
                            Interlocked.Increment(ref _failedCount);
                            LogError("Get({0}) '{1}'failed with {2}", MaxRetries - retries - 1, SandboxFilePath + delta.Path, ex.Message);
                            break;
                        }
                        else
                        {
                            // First retry is 1s, second retry is 20s assuming rate-limit
                            Thread.Sleep(retries == 1 ? TimeSpan.FromSeconds(20) : TimeSpan.FromSeconds(1));
                            continue;
                        }
                    }
                    finally
                    {
                        Interlocked.Increment(ref _fileCount);
                    }
                }
            }
        }
Ejemplo n.º 27
0
        private Task<Stream> GetFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta)
        {
            var parameters = new Dictionary<string, string>
            {
                { "oauth_consumer_key", info.ConsumerKey },
                { "oauth_signature_method", info.SignatureMethod },
                { "oauth_timestamp", info.TimeStamp },
                { "oauth_nonce", delta.Nonce },
                { "oauth_version", info.OAuthVersion },
                { "oauth_token", info.Token },
                { "oauth_signature", delta.Signature }
            };

            var sb = new StringBuilder();
            foreach (var key in parameters.Keys)
            {
                if (sb.Length != 0)
                {
                    sb.Append(',');
                }
                sb.AppendFormat("{0}=\"{1}\"", key, parameters[key]);
            }

            var client = new HttpClient();
            client.BaseAddress = new Uri(DropboxApiContentUri);
            client.Timeout = _timeout;
            client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", sb.ToString());

            // Using ContinueWith instead of Then to avoid SyncContext deadlock in 4.5
            TaskCompletionSource<Task<Stream>> tcs = new TaskCompletionSource<Task<Stream>>();
            client.GetAsync(SandboxFilePath + delta.Path.ToLower()).ContinueWith(t =>
            {
                if (t.IsFaulted)
                {
                    _tracer.TraceError("Get '" + SandboxFilePath + delta.Path + "' failed with " + t.Exception);
                    tcs.TrySetException(t.Exception.InnerExceptions);
                }
                else if (t.IsCanceled)
                {
                    tcs.TrySetCanceled();
                }
                else
                {
                    try
                    {
                        tcs.TrySetResult(t.Result.EnsureSuccessStatusCode().Content.ReadAsStreamAsync());
                    }
                    catch (Exception ex)
                    {
                        _tracer.TraceError("Get '" + SandboxFilePath + delta.Path + "' failed with " + ex);
                        tcs.TrySetException(ex);
                    }
                }
            });

            return tcs.Task.FastUnwrap();
        }