private static string GetOAuthHeader(DropboxDeployInfo info, DropboxEntryInfo delta)
{
var parameters = new Dictionary <string, string>
{
{ "oauth_consumer_key", info.ConsumerKey },
{ "oauth_signature_method", info.SignatureMethod },
{ "oauth_timestamp", info.TimeStamp },
{ "oauth_nonce", delta.Nonce },
{ "oauth_version", info.OAuthVersion },
{ "oauth_token", info.Token },
{ "oauth_signature", delta.Signature }
};
var sb = new StringBuilder();
foreach (var item in parameters)
{
if (sb.Length != 0)
{
sb.Append(',');
}
sb.AppendFormat("{0}=\"{1}\"", item.Key, item.Value);
}
return(sb.ToString());
}
public virtual HttpClient CreateDropboxHttpClient(DropboxDeployInfo info, DropboxEntryInfo entry)
{
var oauthToken = GetOAuthHeader(info, entry);
var client = new HttpClient {
BaseAddress = new Uri(DropboxApiContentUri), Timeout = _timeout
};
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", oauthToken);
return(client);
}
private async Task ApplyChanges(DropboxInfo dropboxInfo, bool useOAuth20)
{
DropboxDeployInfo info = dropboxInfo.DeployInfo;
_totals = info.Deltas.Count;
using (new Timer(UpdateStatusFile, state: dropboxInfo.TargetChangeset.Id, dueTime: TimeSpan.FromSeconds(5), period: TimeSpan.FromSeconds(5)))
{
await ApplyChangesCore(info, useOAuth20);
}
}
private async Task ApplyChanges(DropboxHandler.DropboxInfo deploymentInfo)
{
DropboxDeployInfo info = deploymentInfo.DeployInfo;
_totals = info.Deltas.Count;
using (new Timer(UpdateStatusFile, state: deploymentInfo.TargetChangeset.Id, dueTime: TimeSpan.FromSeconds(5), period: TimeSpan.FromSeconds(5)))
{
await ApplyChangesCore(info);
}
}
private async Task ProcessFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc)
{
var oauthHeader = GetOAuthHeader(info, delta);
int retries = MaxRetries;
while (retries >= 0)
{
retries--;
using (var client = new HttpClient {
BaseAddress = new Uri(DropboxApiContentUri), Timeout = _timeout
})
{
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", oauthHeader);
try
{
string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant());
using (HttpResponseMessage response = await client.GetAsync(requestPath))
{
using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync())
{
await SafeWriteFile(parent, path, lastModifiedUtc, stream);
}
}
if (retries < MaxRetries - 1)
{
Interlocked.Increment(ref _retriedCount);
}
Interlocked.Increment(ref _successCount);
break;
}
catch (Exception ex)
{
if (retries <= 0)
{
Interlocked.Increment(ref _failedCount);
LogError("Get({0}) '{1}'failed with {2}", MaxRetries - retries - 1, SandboxFilePath + delta.Path, ex.Message);
break;
}
}
// First retry is 1s, second retry is 20s assuming rate-limit
await Task.Delay(retries == 1?TimeSpan.FromSeconds(20) : TimeSpan.FromSeconds(1));
}
}
}
public async Task UpdateDropboxDeployInfo(DropboxDeployInfo dropboxInfo)
{
string currentCursor = _settings.GetValue(CursorKey),
dropboxToken = dropboxInfo.Token;
dropboxInfo.OldCursor = currentCursor;
string parentPath = dropboxInfo.Path.TrimEnd('/') + '/';
using (_tracer.Step("Updating deploy info"))
using (var client = CreateDropboxV2HttpClient(ListFolderApiUri, dropboxToken))
{
while (true)
{
LogInfo("Fetching delta for cursor '{0}'.", currentCursor);
var content = string.IsNullOrEmpty(currentCursor) ?
new StringContent("{\"path\":\"" + dropboxInfo.Path + "\",\"recursive\":true}", Encoding.UTF8, "application/json") :
new StringContent("{\"cursor\":\"" + currentCursor + "\"}", Encoding.UTF8, "application/json");
var uri = string.IsNullOrEmpty(currentCursor) ? ListFolderApiUri : ListFolderContinueApiUri;
using (var response = await client.PostAsync(uri, content))
{
// The Content-Type of the Json result is text/json which the JsonMediaTypeFormatter does not approve of.
// We'll parse it separately instead.
var deltaResultString = await response.EnsureSuccessStatusCode().Content.ReadAsStringAsync();
var deltaResult = JsonConvert.DeserializeObject <JObject>(deltaResultString);
var entries = deltaResult.Value <JArray>("entries");
if (entries != null && entries.Count > 0)
{
var filteredEntries = entries.Cast <JObject>()
.Select(DropboxEntryInfo.ParseFrom)
.Where(entry => !CanIgnoreEntry(parentPath, entry));
dropboxInfo.Deltas.AddRange(filteredEntries);
}
currentCursor = deltaResult.Value <string>("cursor");
if (!deltaResult.Value <bool>("has_more"))
{
LogInfo("Delta response exhausted. New cursor at '{0}'.", currentCursor);
break;
}
}
}
}
dropboxInfo.NewCursor = currentCursor;
}
public virtual async Task ProcessFileAsync(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc)
{
var oauthHeader = GetOAuthHeader(info, delta);
int retries = 0;
while (retries <= MaxRetries)
{
using (var client = CreateDropboxHttpClient())
{
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", oauthHeader);
try
{
string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant());
using (HttpResponseMessage response = await client.GetAsync(requestPath))
{
using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync())
{
await SafeWriteFile(parent, path, lastModifiedUtc, stream);
}
}
if (retries > 0)
{
// Increment the successful retry count
Interlocked.Increment(ref _retriedCount);
}
// Increment the total success counter
Interlocked.Increment(ref _successCount);
break;
}
catch (Exception ex)
{
if (retries == MaxRetries)
{
Interlocked.Increment(ref _failedCount);
LogError("Get({0}) '{1}' failed with {2}", retries, SandboxFilePath + delta.Path, ex.Message);
throw;
}
}
// First retry is 1s, second retry is 20s assuming rate-limit
await Task.Delay(retries == MaxRetries - 1?TimeSpan.FromSeconds(1) : RetryWaitToAvoidRateLimit);
retries++;
}
}
}
public async Task UpdateDropboxDeployInfo(DropboxDeployInfo dropboxInfo)
{
string currentCursor = _settings.GetValue(CursorKey),
dropboxToken = dropboxInfo.Token;
dropboxInfo.OldCursor = currentCursor;
string parentPath = dropboxInfo.Path.TrimEnd('/') + '/';
using (_tracer.Step("Updating deploy info"))
using (var client = CreateDropboxV2HttpClient(DeltaApiUri, dropboxToken))
{
while (true)
{
LogInfo("Fetching delta for cursor '{0}'.", currentCursor);
string url = String.IsNullOrEmpty(currentCursor) ? String.Empty :
("?cursor=" + HttpUtility.UrlPathEncode(currentCursor));
using (var response = await client.PostAsync(url, content: null))
{
// The Content-Type of the Json result is text/json which the JsonMediaTypeFormatter does not approve of.
// We'll parse it separately instead.
var deltaResultString = await response.EnsureSuccessStatusCode().Content.ReadAsStringAsync();
var deltaResult = JsonConvert.DeserializeObject<JObject>(deltaResultString);
var entries = deltaResult.Value<JArray>("entries");
if (entries != null && entries.Count > 0)
{
var filteredEntries = entries.Cast<JArray>()
.Select(DropboxEntryInfo.ParseFrom)
.Where(entry => !CanIgnoreEntry(parentPath, entry));
dropboxInfo.Deltas.AddRange(filteredEntries);
}
currentCursor = deltaResult.Value<string>("cursor");
if (!deltaResult.Value<bool>("has_more"))
{
LogInfo("Delta response exhausted. New cursor at '{0}'.", currentCursor);
break;
}
}
}
}
dropboxInfo.NewCursor = currentCursor;
}
internal async Task ApplyChangesCore(DropboxDeployInfo info, bool useOAuth20)
{
string parent = info.Path.TrimEnd('/') + '/';
DateTime updateMessageTime = DateTime.UtcNow;
var sem = new SemaphoreSlim(MaxConcurrentRequests, MaxConcurrentRequests);
var rateLimiter = new RateLimiter(MaxFilesPerSecs * 10, TimeSpan.FromSeconds(10));
var tasks = new List<Task>();
foreach (DropboxEntryInfo entry in info.Deltas)
{
if (CanIgnoreEntry(parent, entry))
{
continue;
}
var path = entry.Path;
if (entry.IsDeleted)
{
SafeDelete(parent, path);
Interlocked.Increment(ref _successCount);
}
else if (entry.IsDirectory)
{
SafeCreateDir(parent, path);
Interlocked.Increment(ref _successCount);
}
else
{
DateTime modified;
if (DateTime.TryParse(entry.Modified, out modified))
{
modified = modified.ToUniversalTime();
}
else
{
modified = DateTime.UtcNow;
}
if (!IsFileChanged(parent, path, modified))
{
LogInfo("file unchanged {0}", path);
Interlocked.Increment(ref _successCount);
continue;
}
try
{
var task = Task.Run(async () =>
{
await sem.WaitAsync();
try
{
await rateLimiter.ThrottleAsync();
using (HttpClient client = useOAuth20 ? CreateDropboxV2HttpClient(DropboxApiContentUri, info.Token) :
CreateDropboxHttpClient(info, entry))
{
await ProcessFileAsync(client, entry.Path, parent, modified);
}
}
finally
{
Interlocked.Increment(ref _fileCount);
sem.Release();
}
});
tasks.Add(task);
}
catch (Exception ex)
{
LogError("{0}", ex);
throw;
}
}
}
await Task.WhenAll(tasks);
}
public virtual HttpClient CreateDropboxHttpClient(DropboxDeployInfo info, DropboxEntryInfo entry)
{
var oauthToken = GetOAuthHeader(info, entry);
var client = new HttpClient { BaseAddress = new Uri(DropboxApiContentUri), Timeout = _timeout };
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", oauthToken);
return client;
}
internal async Task ApplyChangesCore(DropboxDeployInfo info, bool useOAuth20)
{
string parent = info.Path.TrimEnd('/') + '/';
DateTime updateMessageTime = DateTime.UtcNow;
var sem = new SemaphoreSlim(MaxConcurrentRequests, MaxConcurrentRequests);
var rateLimiter = new RateLimiter(MaxFilesPerSecs * 10, TimeSpan.FromSeconds(10));
var tasks = new List <Task>();
foreach (DropboxEntryInfo entry in info.Deltas)
{
if (CanIgnoreEntry(parent, entry))
{
continue;
}
var path = entry.Path;
if (entry.IsDeleted)
{
SafeDelete(parent, path);
Interlocked.Increment(ref _successCount);
}
else if (entry.IsDirectory)
{
SafeCreateDir(parent, path);
Interlocked.Increment(ref _successCount);
}
else
{
DateTime modified;
if (useOAuth20 ? DateTime.TryParse(entry.Modified, null, DateTimeStyles.AssumeUniversal, out modified) : DateTime.TryParse(entry.Modified, out modified))
{
modified = modified.ToUniversalTime();
}
else
{
modified = DateTime.UtcNow;
}
if (!IsFileChanged(parent, path, modified))
{
LogInfo("file unchanged {0}", path);
Interlocked.Increment(ref _successCount);
continue;
}
try
{
var task = Task.Run(async() =>
{
await sem.WaitAsync();
try
{
await rateLimiter.ThrottleAsync();
using (HttpClient client = useOAuth20 ? CreateDropboxV2HttpClient(DownloadContentApiUri, info.Token) :
CreateDropboxHttpClient(info, entry))
{
await ProcessFileAsync(client, entry.Path, parent, modified, useOAuth20);
}
}
finally
{
Interlocked.Increment(ref _fileCount);
sem.Release();
}
});
tasks.Add(task);
}
catch (Exception ex)
{
LogError("{0}", ex);
throw;
}
}
}
await Task.WhenAll(tasks);
}
internal async Task <ChangeSet> Sync(DropboxInfo dropboxInfo, string branch, IRepository repository, ITracer tracer)
{
DropboxDeployInfo deployInfo = dropboxInfo.DeployInfo;
// use incoming tracer since it is background work
_tracer = tracer;
ResetStats();
// for Dropbox OAuth V2, the delta is collected and applied by SCM
// simply set OldCursor as current.
if (dropboxInfo.OAuthVersion == 2)
{
deployInfo.OldCursor = _settings.GetValue(CursorKey);
}
else if (_settings.GetValue(CursorKey) != deployInfo.OldCursor)
{
throw new InvalidOperationException(Resources.Error_MismatchDropboxCursor);
}
// initial sync, remove default content
// for simplicity, we do it blindly whether or not in-place
// given the end result is the same
if (String.IsNullOrEmpty(deployInfo.OldCursor) && DeploymentHelper.IsDefaultWebRootContent(_environment.WebRootPath))
{
string hoststarthtml = Path.Combine(_environment.WebRootPath, Constants.HostingStartHtml);
FileSystemHelpers.DeleteFileSafe(hoststarthtml);
}
if (!repository.IsEmpty())
{
// git checkout --force <branch>
repository.Update(branch);
}
ChangeSet changeSet = null;
string message = null;
try
{
using (_tracer.Step("Sync with Dropbox"))
{
if (dropboxInfo.OAuthVersion == 2)
{
// Fetch the deltas
await UpdateDropboxDeployInfo(deployInfo);
}
// Sync dropbox => repository directory
await ApplyChanges(dropboxInfo, useOAuth20 : dropboxInfo.OAuthVersion == 2);
}
message = String.Format(CultureInfo.CurrentCulture,
Resources.Dropbox_Synchronized,
deployInfo.Deltas.Count);
}
catch (Exception)
{
message = String.Format(CultureInfo.CurrentCulture,
Resources.Dropbox_SynchronizedWithFailure,
_successCount,
deployInfo.Deltas.Count,
_failedCount);
throw;
}
finally
{
Logger.Log(message);
Logger.Log(String.Format("{0} downloaded files, {1} successful retries.", _fileCount, _retriedCount));
IDeploymentStatusFile statusFile = _status.Open(dropboxInfo.TargetChangeset.Id);
statusFile.UpdateMessage(message);
statusFile.UpdateProgress(String.Format(CultureInfo.CurrentCulture, Resources.Dropbox_Committing, _successCount));
// Commit anyway even partial change
if (repository.Commit(message, deployInfo.UserName, deployInfo.Email ?? deployInfo.UserName))
{
changeSet = repository.GetChangeSet("HEAD");
}
}
// Save new dropbox cursor
LogInfo("Update dropbox cursor");
_settings.SetValue(CursorKey, deployInfo.NewCursor);
return(changeSet);
}
internal async Task <ChangeSet> Sync(DropboxHandler.DropboxInfo deploymentInfo, string branch, ILogger logger, IRepository repository)
{
DropboxDeployInfo info = deploymentInfo.DeployInfo;
_logger = logger;
_totals = 0;
_successCount = 0;
_fileCount = 0;
_failedCount = 0;
_retriedCount = 0;
if (_settings.GetValue(CursorKey) != info.OldCursor)
{
throw new InvalidOperationException(Resources.Error_MismatchDropboxCursor);
}
if (!repository.IsEmpty())
{
// git checkout --force <branch>
repository.ClearLock();
repository.Update(branch);
}
ChangeSet changeSet;
string message = null;
try
{
using (_tracer.Step("Synch with Dropbox"))
{
// Sync dropbox => repository directory
await ApplyChanges(deploymentInfo);
}
message = String.Format(CultureInfo.CurrentCulture,
Resources.Dropbox_Synchronized,
deploymentInfo.DeployInfo.Deltas.Count());
}
catch (Exception)
{
message = String.Format(CultureInfo.CurrentCulture,
Resources.Dropbox_SynchronizedWithFailure,
_successCount,
deploymentInfo.DeployInfo.Deltas.Count(),
_failedCount);
throw;
}
finally
{
_logger.Log(message);
_logger.Log(String.Format("{0} downloaded files, {1} successful retries.", _fileCount, _retriedCount));
IDeploymentStatusFile statusFile = _status.Open(deploymentInfo.TargetChangeset.Id);
statusFile.UpdateMessage(message);
statusFile.UpdateProgress(String.Format(CultureInfo.CurrentCulture, Resources.Dropbox_Committing, _successCount));
// Commit anyway even partial change
changeSet = repository.Commit(message, String.Format("{0} <{1}>", info.UserName, info.Email));
}
// Save new dropboc cursor
LogInfo("Update dropbox cursor");
_settings.SetValue(CursorKey, info.NewCursor);
return(changeSet);
}
private async Task ProcessFileAsyncCore(DropboxDeployInfo info, DropboxDeltaInfo delta, string parent, string path, DateTime lastModifiedUtc)
{
var parameters = new Dictionary <string, string>
{
{ "oauth_consumer_key", info.ConsumerKey },
{ "oauth_signature_method", info.SignatureMethod },
{ "oauth_timestamp", info.TimeStamp },
{ "oauth_nonce", delta.Nonce },
{ "oauth_version", info.OAuthVersion },
{ "oauth_token", info.Token },
{ "oauth_signature", delta.Signature }
};
var sb = new StringBuilder();
foreach (var key in parameters.Keys)
{
if (sb.Length != 0)
{
sb.Append(',');
}
sb.AppendFormat("{0}=\"{1}\"", key, parameters[key]);
}
int retries = MaxRetries;
while (retries >= 0)
{
retries--;
using (var client = new HttpClient {
BaseAddress = new Uri(DropboxApiContentUri), Timeout = _timeout
})
{
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("OAuth", sb.ToString());
try
{
string requestPath = SandboxFilePath + DropboxPathEncode(delta.Path.ToLowerInvariant());
using (HttpResponseMessage response = await client.GetAsync(requestPath))
{
using (Stream stream = await response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync())
{
await SafeWriteFile(parent, path, lastModifiedUtc, stream);
}
}
if (retries < MaxRetries - 1)
{
Interlocked.Increment(ref _retriedCount);
}
Interlocked.Increment(ref _successCount);
break;
}
catch (Exception ex)
{
if (retries <= 0)
{
Interlocked.Increment(ref _failedCount);
LogError("Get({0}) '{1}'failed with {2}", MaxRetries - retries - 1, SandboxFilePath + delta.Path, ex.Message);
break;
}
else
{
// First retry is 1s, second retry is 20s assuming rate-limit
Thread.Sleep(retries == 1 ? TimeSpan.FromSeconds(20) : TimeSpan.FromSeconds(1));
continue;
}
}
finally
{
Interlocked.Increment(ref _fileCount);
}
}
}
}
private async Task ApplyChangesCore(DropboxDeployInfo info)
{
string parent = info.Path.TrimEnd('/') + '/';
DateTime updateMessageTime = DateTime.UtcNow;
var sem = new SemaphoreSlim(MaxConcurrentRequests, MaxConcurrentRequests);
var rateLimiter = new RateLimiter(MaxFilesPerSecs * 10, TimeSpan.FromSeconds(10));
var tasks = new List <Task>();
foreach (DropboxDeltaInfo delta in info.Deltas)
{
if (!delta.Path.StartsWith(parent, StringComparison.OrdinalIgnoreCase))
{
Interlocked.Increment(ref _successCount);
continue;
}
// Ignore .git and .hg files and folders
string pathWithSlash = delta.Path + "/";
if (pathWithSlash.IndexOf("/.git/", StringComparison.OrdinalIgnoreCase) != -1 ||
pathWithSlash.IndexOf("/.hg/", StringComparison.OrdinalIgnoreCase) != -1)
{
continue;
}
var path = delta.Path;
if (delta.IsDeleted)
{
SafeDelete(parent, path);
Interlocked.Increment(ref _successCount);
}
else if (delta.IsDirectory)
{
SafeCreateDir(parent, path);
Interlocked.Increment(ref _successCount);
}
else
{
DateTime modified;
if (DateTime.TryParse(delta.Modified, out modified))
{
modified = modified.ToUniversalTime();
}
else
{
modified = DateTime.UtcNow;
}
if (!IsFileChanged(parent, path, modified))
{
LogInfo("file unchanged {0}", path);
Interlocked.Increment(ref _successCount);
continue;
}
try
{
var task = Task.Run(async() =>
{
await sem.WaitAsync();
try
{
rateLimiter.Throtte();
await ProcessFileAsyncCore(info, delta, parent, path, modified);
}
finally
{
sem.Release();
}
});
tasks.Add(task);
}
catch (Exception ex)
{
LogError("{0}", ex);
throw;
}
}
}
await Task.WhenAll(tasks);
}
internal DropboxDeployInfo GetDeployInfo(string path, OAuthInfo oauth, AccountInfo account, string cursor = null)
{
List<DropboxEntryInfo> deltas = new List<DropboxEntryInfo>();
string timeStamp = GetUtcTimeStamp();
string oldCursor = cursor;
string newCursor = "";
while (true)
{
DeltaInfo delta = GetDeltaInfo(oauth, cursor);
newCursor = delta.cursor;
if (newCursor == oldCursor)
{
break;
}
foreach (EntryInfo info in delta.entries)
{
DropboxEntryInfo item = new DropboxEntryInfo();
if (info.metadata != null && !info.metadata.path.StartsWith(path))
{
continue;
}
if (info.metadata == null || info.metadata.is_deleted || string.IsNullOrEmpty(info.metadata.path))
{
item.Path = info.path;
item.IsDeleted = true;
}
else
{
item.Path = info.metadata.path;
item.IsDirectory = info.metadata.is_dir;
if (!item.IsDirectory)
{
item.Modified = info.metadata.modified;
item.Nonce = GetNonce();
item.Signature = GetSignature(oauth, info.path, timeStamp, item.Nonce);
}
}
deltas.Add(item);
}
if (!delta.has_more)
{
break;
}
cursor = newCursor;
}
if (deltas.Count == 0)
{
throw new InvalidOperationException("the repo is up-to-date.");
}
var deployInfo = new DropboxDeployInfo
{
TimeStamp = timeStamp,
Token = oauth.Token,
ConsumerKey = oauth.ConsumerKey,
OAuthVersion = "1.0",
SignatureMethod = "HMAC-SHA1",
OldCursor = oldCursor,
NewCursor = newCursor,
Path = path,
UserName = account.display_name,
Email = account.email,
};
deployInfo.Deltas.AddRange(deltas);
return deployInfo;
}
private static string GetOAuthHeader(DropboxDeployInfo info, DropboxEntryInfo delta)
{
var parameters = new Dictionary<string, string>
{
{ "oauth_consumer_key", info.ConsumerKey },
{ "oauth_signature_method", info.SignatureMethod },
{ "oauth_timestamp", info.TimeStamp },
{ "oauth_nonce", delta.Nonce },
{ "oauth_version", info.OAuthVersion },
{ "oauth_token", info.Token },
{ "oauth_signature", delta.Signature }
};
var sb = new StringBuilder();
foreach (var item in parameters)
{
if (sb.Length != 0)
{
sb.Append(',');
}
sb.AppendFormat("{0}=\"{1}\"", item.Key, item.Value);
}
return sb.ToString();
}