/// <summary>
/// Worker Method for the synchronize request.
/// </summary>
/// <param name="databaseName">Database name</param>
/// <param name="pollingUrl">URL for polling</param>
/// <param name="pollingInterval">Polling interval set by the post response</param>
/// <param name="maxNumberOfAttempts">Max number of attempts for each poll before the attempt is declared a failure</param>
/// <returns></returns>
private async Task <ScaleOutServerDatabaseSyncResult> PollSyncStatusWithRetryAsync(string databaseName, Uri pollingUrl, TimeSpan pollingInterval, int maxNumberOfAttempts = 3)
{
return(await Task.Run(async() =>
{
ScaleOutServerDatabaseSyncResult response = null;
var syncCompleted = false;
var retryCount = 0;
while (!syncCompleted && retryCount < maxNumberOfAttempts)
{
// Wait for specified polling interval other than retries.
if (retryCount == 0)
{
await Task.Delay(pollingInterval);
}
else
{
await Task.Delay(DefaultRetryIntervalForPolling);
}
this.AsAzureDataplaneClient.ResetHttpClient();
using (HttpResponseMessage message = await AsAzureDataplaneClient.CallGetAsync(pollingUrl, string.Empty, correlationId))
{
bool shouldRetry = false;
if (message.IsSuccessStatusCode && message.Content != null)
{
var responseString = await message.Content.ReadAsStringAsync();
response = JsonConvert.DeserializeObject <ScaleOutServerDatabaseSyncResult>(responseString);
if (response != null)
{
var state = response.SyncState;
if (state == DatabaseSyncState.Completed || state == DatabaseSyncState.Failed)
{
syncCompleted = true;
}
else
{
pollingUrl = message.Headers.Location ?? pollingUrl;
pollingInterval = message.Headers.RetryAfter.Delta ?? pollingInterval;
}
}
else
{
shouldRetry = true;
}
}
else
{
shouldRetry = true;
}
if (shouldRetry)
{
retryCount++;
response = new ScaleOutServerDatabaseSyncResult()
{
Database = databaseName,
SyncState = DatabaseSyncState.Invalid
};
response.Details = string.Format(
"Http Error code: {0}. Message: {1}",
message.StatusCode.ToString(),
message.Content != null ? await message.Content.ReadAsStringAsync() : string.Empty);
if (message.StatusCode >= (HttpStatusCode)400 && message.StatusCode <= (HttpStatusCode)499)
{
break;
}
}
else
{
retryCount = 0;
}
}
}
return response;
}));
}
/// <summary>
/// Worker Method for the synchronize request.
/// </summary>
/// <param name="context">The AS azure context</param>
/// <param name="syncBaseUri">Base Uri for sync</param>
/// <param name="databaseName">Database name</param>
/// <param name="accessToken">Access token</param>
/// <param name="maxNumberOfAttempts">Max number of retries for get command</param>
/// <returns></returns>
private async Task <ScaleOutServerDatabaseSyncDetails> SynchronizeDatabaseAsync(
AsAzureContext context,
Uri syncBaseUri,
string databaseName,
string accessToken)
{
Tuple <Uri, RetryConditionHeaderValue> pollingUrlAndRetryAfter = new Tuple <Uri, RetryConditionHeaderValue>(null, null);
ScaleOutServerDatabaseSyncDetails syncResult = null;
return(await Task.Run(async() =>
{
try
{
var synchronize = string.Format((string)context.Environment.Endpoints[AsAzureEnvironment.AsRolloutEndpoints.SyncEndpoint], this.serverName, databaseName);
this.AsAzureHttpClient.resetHttpClient();
using (var message = await AsAzureHttpClient.CallPostAsync(
syncBaseUri,
synchronize,
accessToken,
correlationId,
null))
{
this.syncRequestRootActivityId = message.Headers.Contains(RootActivityIdHeaderName) ? message.Headers.GetValues(RootActivityIdHeaderName).FirstOrDefault() : string.Empty;
this.syncRequestTimeStamp = message.Headers.Contains(CurrentUtcDateHeaderName) ? message.Headers.GetValues(CurrentUtcDateHeaderName).FirstOrDefault() : string.Empty;
message.EnsureSuccessStatusCode();
if (message.StatusCode != HttpStatusCode.Accepted)
{
var timestampNow = DateTime.Now;
syncResult = new ScaleOutServerDatabaseSyncDetails
{
CorrelationId = correlationId.ToString(),
Database = databaseName,
SyncState = DatabaseSyncState.Completed,
Details = string.Format("Http status code: {0}. Nothing readonly instances found to replicate databases.", message.StatusCode),
UpdatedAt = timestampNow,
StartedAt = timestampNow
};
return syncResult;
}
pollingUrlAndRetryAfter = new Tuple <Uri, RetryConditionHeaderValue>(message.Headers.Location, message.Headers.RetryAfter);
}
}
catch (Exception e)
{
var timestampNow = DateTime.Now;
// Return sync details with exception message as details
return new ScaleOutServerDatabaseSyncDetails
{
CorrelationId = correlationId.ToString(),
Database = databaseName,
SyncState = DatabaseSyncState.Invalid,
Details = Resources.PostSyncRequestFailureMessage.FormatInvariant(
this.clusterResolveResult.CoreServerName,
this.syncRequestRootActivityId,
this.syncRequestTimeStamp,
string.Format(e.Message)),
UpdatedAt = timestampNow,
StartedAt = timestampNow
};
}
Uri pollingUrl = pollingUrlAndRetryAfter.Item1;
var retryAfter = pollingUrlAndRetryAfter.Item2;
try
{
ScaleOutServerDatabaseSyncResult result = await this.PollSyncStatusWithRetryAsync(
databaseName,
accessToken,
pollingUrl,
retryAfter.Delta ?? DefaultPollingInterval);
syncResult = ScaleOutServerDatabaseSyncDetails.FromResult(result, correlationId.ToString());
}
catch (Exception e)
{
var timestampNow = DateTime.Now;
// Append exception message to sync details and return
syncResult = new ScaleOutServerDatabaseSyncDetails
{
CorrelationId = correlationId.ToString(),
Database = databaseName,
SyncState = DatabaseSyncState.Invalid,
Details = Resources.SyncASPollStatusFailureMessage.FormatInvariant(
serverName,
string.Empty,
timestampNow.ToString(CultureInfo.InvariantCulture),
string.Format(e.StackTrace)),
UpdatedAt = timestampNow,
StartedAt = timestampNow
};
}
return syncResult;
}));
}
/// <summary>
/// Worker Method for the synchronize request.
/// </summary>
/// <param name="syncBaseUri">Base Uri for sync</param>
/// <param name="databaseName">Database name</param>
/// <returns></returns>
private async Task <ScaleOutServerDatabaseSyncDetails> SynchronizeDatabaseAsync(
Uri syncBaseUri,
string databaseName)
{
Tuple <Uri, RetryConditionHeaderValue> pollingUrlAndRetryAfter = new Tuple <Uri, RetryConditionHeaderValue>(null, null);
ScaleOutServerDatabaseSyncDetails syncResult = null;
return(await Task.Run(async() =>
{
var syncEndpoint = string.Format(AsAzureEndpoints.SynchronizeEndpointPathFormat, this.ServerName, databaseName);
this.AsAzureDataplaneClient.ResetHttpClient();
using (var message = await AsAzureDataplaneClient.CallPostAsync(syncBaseUri, syncEndpoint, correlationId))
{
this.syncRequestRootActivityId = message.Headers.Contains(RootActivityIdHeaderName) ? message.Headers.GetValues(RootActivityIdHeaderName).FirstOrDefault() : string.Empty;
this.syncRequestTimeStamp = message.Headers.Contains(CurrentUtcDateHeaderName) ? message.Headers.GetValues(CurrentUtcDateHeaderName).FirstOrDefault() : string.Empty;
if (message.StatusCode != HttpStatusCode.Accepted)
{
var timestampNow = DateTime.Now;
// Return sync details with exception message as details
return new ScaleOutServerDatabaseSyncDetails
{
CorrelationId = correlationId.ToString(),
Database = databaseName,
SyncState = DatabaseSyncState.Invalid,
Details = Resources.PostSyncRequestFailureMessage.FormatInvariant(
ServerName,
this.syncRequestRootActivityId,
this.syncRequestTimeStamp,
await message.Content.ReadAsStringAsync()),
UpdatedAt = timestampNow,
StartedAt = timestampNow
};
}
pollingUrlAndRetryAfter = new Tuple <Uri, RetryConditionHeaderValue>(message.Headers.Location, message.Headers.RetryAfter);
}
Uri pollingUrl = pollingUrlAndRetryAfter.Item1;
var retryAfter = pollingUrlAndRetryAfter.Item2;
try
{
ScaleOutServerDatabaseSyncResult result = await this.PollSyncStatusWithRetryAsync(
databaseName,
pollingUrl,
retryAfter.Delta ?? DefaultPollingInterval);
syncResult = ScaleOutServerDatabaseSyncDetails.FromResult(result, correlationId.ToString());
}
catch (Exception e)
{
var timestampNow = DateTime.Now;
// Append exception message to sync details and return
syncResult = new ScaleOutServerDatabaseSyncDetails
{
CorrelationId = correlationId.ToString(),
Database = databaseName,
SyncState = DatabaseSyncState.Invalid,
Details = Resources.SyncASPollStatusFailureMessage.FormatInvariant(
ServerName,
string.Empty,
timestampNow.ToString(CultureInfo.InvariantCulture),
string.Format(e.StackTrace)),
UpdatedAt = timestampNow,
StartedAt = timestampNow
};
}
return syncResult;
}));
}
/// <summary>
///
/// </summary>
/// <param name="databaseName">Database name</param>
/// <param name="accessToken">Access token</param>
/// <param name="pollingUrl">URL for polling</param>
/// <param name="pollingInterval">Polling interval set by the post response</param>
/// <param name="maxNumberOfAttempts">Max number of attempts for each poll before the attempt is declared a failure</param>
/// <returns></returns>
private async Task <ScaleOutServerDatabaseSyncResult> PollSyncStatusWithRetryAsync(string databaseName, string accessToken, Uri pollingUrl, TimeSpan pollingInterval, int maxNumberOfAttempts = 3)
{
return(await Task.Run(async() =>
{
ScaleOutServerDatabaseSyncResult response = null;
var syncCompleted = false;
do
{
var retryCount = 0;
while (retryCount < maxNumberOfAttempts)
{
// Wait for specified polling interval other than retries.
if (retryCount == 0)
{
// WriteInformation(new InformationRecord(string.Format("Synchronize database {0}. Attempt #{1}. Waiting for {2} seconds to get sync results...", databaseName, retryCount, pollingInterval.TotalSeconds), string.Empty));
await Task.Delay(pollingInterval);
}
else
{
await Task.Delay(DefaultRetryIntervalForPolling);
}
this.AsAzureHttpClient.resetHttpClient();
using (HttpResponseMessage message = await AsAzureHttpClient.CallGetAsync(
pollingUrl,
string.Empty,
accessToken,
correlationId))
{
syncCompleted = !message.StatusCode.Equals(HttpStatusCode.SeeOther);
if (syncCompleted)
{
if (message.IsSuccessStatusCode)
{
var responseString = await message.Content.ReadAsStringAsync();
response = JsonConvert.DeserializeObject <ScaleOutServerDatabaseSyncResult>(responseString);
break;
}
else
{
retryCount++;
if (response == null)
{
response = new ScaleOutServerDatabaseSyncResult()
{
Database = databaseName,
SyncState = DatabaseSyncState.Invalid
};
response.Details = string.Format(
"Http Error code: {0}. {1}",
message.StatusCode.ToString(),
message.Content != null ? await message.Content.ReadAsStringAsync() : string.Empty);
}
if (message.StatusCode >= (HttpStatusCode)400 && message.StatusCode <= (HttpStatusCode)499)
{
break;
}
}
}
else
{
pollingUrl = message.Headers.Location;
pollingInterval = message.Headers.RetryAfter.Delta ?? pollingInterval;
}
}
}
}while (!syncCompleted);
return response;
}));
}