/// <summary> /// Worker Method for the synchronize request. /// </summary> /// <param name="databaseName">Database name</param> /// <param name="pollingUrl">URL for polling</param> /// <param name="pollingInterval">Polling interval set by the post response</param> /// <param name="maxNumberOfAttempts">Max number of attempts for each poll before the attempt is declared a failure</param> /// <returns></returns> private async Task <ScaleOutServerDatabaseSyncResult> PollSyncStatusWithRetryAsync(string databaseName, Uri pollingUrl, TimeSpan pollingInterval, int maxNumberOfAttempts = 3) { return(await Task.Run(async() => { ScaleOutServerDatabaseSyncResult response = null; var syncCompleted = false; var retryCount = 0; while (!syncCompleted && retryCount < maxNumberOfAttempts) { // Wait for specified polling interval other than retries. if (retryCount == 0) { await Task.Delay(pollingInterval); } else { await Task.Delay(DefaultRetryIntervalForPolling); } this.AsAzureDataplaneClient.ResetHttpClient(); using (HttpResponseMessage message = await AsAzureDataplaneClient.CallGetAsync(pollingUrl, string.Empty, correlationId)) { bool shouldRetry = false; if (message.IsSuccessStatusCode && message.Content != null) { var responseString = await message.Content.ReadAsStringAsync(); response = JsonConvert.DeserializeObject <ScaleOutServerDatabaseSyncResult>(responseString); if (response != null) { var state = response.SyncState; if (state == DatabaseSyncState.Completed || state == DatabaseSyncState.Failed) { syncCompleted = true; } else { pollingUrl = message.Headers.Location ?? pollingUrl; pollingInterval = message.Headers.RetryAfter.Delta ?? pollingInterval; } } else { shouldRetry = true; } } else { shouldRetry = true; } if (shouldRetry) { retryCount++; response = new ScaleOutServerDatabaseSyncResult() { Database = databaseName, SyncState = DatabaseSyncState.Invalid }; response.Details = string.Format( "Http Error code: {0}. Message: {1}", message.StatusCode.ToString(), message.Content != null ? await message.Content.ReadAsStringAsync() : string.Empty); if (message.StatusCode >= (HttpStatusCode)400 && message.StatusCode <= (HttpStatusCode)499) { break; } } else { retryCount = 0; } } } return response; })); }
/// <summary> /// Worker Method for the synchronize request. /// </summary> /// <param name="context">The AS azure context</param> /// <param name="syncBaseUri">Base Uri for sync</param> /// <param name="databaseName">Database name</param> /// <param name="accessToken">Access token</param> /// <param name="maxNumberOfAttempts">Max number of retries for get command</param> /// <returns></returns> private async Task <ScaleOutServerDatabaseSyncDetails> SynchronizeDatabaseAsync( AsAzureContext context, Uri syncBaseUri, string databaseName, string accessToken) { Tuple <Uri, RetryConditionHeaderValue> pollingUrlAndRetryAfter = new Tuple <Uri, RetryConditionHeaderValue>(null, null); ScaleOutServerDatabaseSyncDetails syncResult = null; return(await Task.Run(async() => { try { var synchronize = string.Format((string)context.Environment.Endpoints[AsAzureEnvironment.AsRolloutEndpoints.SyncEndpoint], this.serverName, databaseName); this.AsAzureHttpClient.resetHttpClient(); using (var message = await AsAzureHttpClient.CallPostAsync( syncBaseUri, synchronize, accessToken, correlationId, null)) { this.syncRequestRootActivityId = message.Headers.Contains(RootActivityIdHeaderName) ? message.Headers.GetValues(RootActivityIdHeaderName).FirstOrDefault() : string.Empty; this.syncRequestTimeStamp = message.Headers.Contains(CurrentUtcDateHeaderName) ? message.Headers.GetValues(CurrentUtcDateHeaderName).FirstOrDefault() : string.Empty; message.EnsureSuccessStatusCode(); if (message.StatusCode != HttpStatusCode.Accepted) { var timestampNow = DateTime.Now; syncResult = new ScaleOutServerDatabaseSyncDetails { CorrelationId = correlationId.ToString(), Database = databaseName, SyncState = DatabaseSyncState.Completed, Details = string.Format("Http status code: {0}. Nothing readonly instances found to replicate databases.", message.StatusCode), UpdatedAt = timestampNow, StartedAt = timestampNow }; return syncResult; } pollingUrlAndRetryAfter = new Tuple <Uri, RetryConditionHeaderValue>(message.Headers.Location, message.Headers.RetryAfter); } } catch (Exception e) { var timestampNow = DateTime.Now; // Return sync details with exception message as details return new ScaleOutServerDatabaseSyncDetails { CorrelationId = correlationId.ToString(), Database = databaseName, SyncState = DatabaseSyncState.Invalid, Details = Resources.PostSyncRequestFailureMessage.FormatInvariant( this.clusterResolveResult.CoreServerName, this.syncRequestRootActivityId, this.syncRequestTimeStamp, string.Format(e.Message)), UpdatedAt = timestampNow, StartedAt = timestampNow }; } Uri pollingUrl = pollingUrlAndRetryAfter.Item1; var retryAfter = pollingUrlAndRetryAfter.Item2; try { ScaleOutServerDatabaseSyncResult result = await this.PollSyncStatusWithRetryAsync( databaseName, accessToken, pollingUrl, retryAfter.Delta ?? DefaultPollingInterval); syncResult = ScaleOutServerDatabaseSyncDetails.FromResult(result, correlationId.ToString()); } catch (Exception e) { var timestampNow = DateTime.Now; // Append exception message to sync details and return syncResult = new ScaleOutServerDatabaseSyncDetails { CorrelationId = correlationId.ToString(), Database = databaseName, SyncState = DatabaseSyncState.Invalid, Details = Resources.SyncASPollStatusFailureMessage.FormatInvariant( serverName, string.Empty, timestampNow.ToString(CultureInfo.InvariantCulture), string.Format(e.StackTrace)), UpdatedAt = timestampNow, StartedAt = timestampNow }; } return syncResult; })); }
/// <summary> /// Worker Method for the synchronize request. /// </summary> /// <param name="syncBaseUri">Base Uri for sync</param> /// <param name="databaseName">Database name</param> /// <returns></returns> private async Task <ScaleOutServerDatabaseSyncDetails> SynchronizeDatabaseAsync( Uri syncBaseUri, string databaseName) { Tuple <Uri, RetryConditionHeaderValue> pollingUrlAndRetryAfter = new Tuple <Uri, RetryConditionHeaderValue>(null, null); ScaleOutServerDatabaseSyncDetails syncResult = null; return(await Task.Run(async() => { var syncEndpoint = string.Format(AsAzureEndpoints.SynchronizeEndpointPathFormat, this.ServerName, databaseName); this.AsAzureDataplaneClient.ResetHttpClient(); using (var message = await AsAzureDataplaneClient.CallPostAsync(syncBaseUri, syncEndpoint, correlationId)) { this.syncRequestRootActivityId = message.Headers.Contains(RootActivityIdHeaderName) ? message.Headers.GetValues(RootActivityIdHeaderName).FirstOrDefault() : string.Empty; this.syncRequestTimeStamp = message.Headers.Contains(CurrentUtcDateHeaderName) ? message.Headers.GetValues(CurrentUtcDateHeaderName).FirstOrDefault() : string.Empty; if (message.StatusCode != HttpStatusCode.Accepted) { var timestampNow = DateTime.Now; // Return sync details with exception message as details return new ScaleOutServerDatabaseSyncDetails { CorrelationId = correlationId.ToString(), Database = databaseName, SyncState = DatabaseSyncState.Invalid, Details = Resources.PostSyncRequestFailureMessage.FormatInvariant( ServerName, this.syncRequestRootActivityId, this.syncRequestTimeStamp, await message.Content.ReadAsStringAsync()), UpdatedAt = timestampNow, StartedAt = timestampNow }; } pollingUrlAndRetryAfter = new Tuple <Uri, RetryConditionHeaderValue>(message.Headers.Location, message.Headers.RetryAfter); } Uri pollingUrl = pollingUrlAndRetryAfter.Item1; var retryAfter = pollingUrlAndRetryAfter.Item2; try { ScaleOutServerDatabaseSyncResult result = await this.PollSyncStatusWithRetryAsync( databaseName, pollingUrl, retryAfter.Delta ?? DefaultPollingInterval); syncResult = ScaleOutServerDatabaseSyncDetails.FromResult(result, correlationId.ToString()); } catch (Exception e) { var timestampNow = DateTime.Now; // Append exception message to sync details and return syncResult = new ScaleOutServerDatabaseSyncDetails { CorrelationId = correlationId.ToString(), Database = databaseName, SyncState = DatabaseSyncState.Invalid, Details = Resources.SyncASPollStatusFailureMessage.FormatInvariant( ServerName, string.Empty, timestampNow.ToString(CultureInfo.InvariantCulture), string.Format(e.StackTrace)), UpdatedAt = timestampNow, StartedAt = timestampNow }; } return syncResult; })); }
/// <summary> /// /// </summary> /// <param name="databaseName">Database name</param> /// <param name="accessToken">Access token</param> /// <param name="pollingUrl">URL for polling</param> /// <param name="pollingInterval">Polling interval set by the post response</param> /// <param name="maxNumberOfAttempts">Max number of attempts for each poll before the attempt is declared a failure</param> /// <returns></returns> private async Task <ScaleOutServerDatabaseSyncResult> PollSyncStatusWithRetryAsync(string databaseName, string accessToken, Uri pollingUrl, TimeSpan pollingInterval, int maxNumberOfAttempts = 3) { return(await Task.Run(async() => { ScaleOutServerDatabaseSyncResult response = null; var syncCompleted = false; do { var retryCount = 0; while (retryCount < maxNumberOfAttempts) { // Wait for specified polling interval other than retries. if (retryCount == 0) { // WriteInformation(new InformationRecord(string.Format("Synchronize database {0}. Attempt #{1}. Waiting for {2} seconds to get sync results...", databaseName, retryCount, pollingInterval.TotalSeconds), string.Empty)); await Task.Delay(pollingInterval); } else { await Task.Delay(DefaultRetryIntervalForPolling); } this.AsAzureHttpClient.resetHttpClient(); using (HttpResponseMessage message = await AsAzureHttpClient.CallGetAsync( pollingUrl, string.Empty, accessToken, correlationId)) { syncCompleted = !message.StatusCode.Equals(HttpStatusCode.SeeOther); if (syncCompleted) { if (message.IsSuccessStatusCode) { var responseString = await message.Content.ReadAsStringAsync(); response = JsonConvert.DeserializeObject <ScaleOutServerDatabaseSyncResult>(responseString); break; } else { retryCount++; if (response == null) { response = new ScaleOutServerDatabaseSyncResult() { Database = databaseName, SyncState = DatabaseSyncState.Invalid }; response.Details = string.Format( "Http Error code: {0}. {1}", message.StatusCode.ToString(), message.Content != null ? await message.Content.ReadAsStringAsync() : string.Empty); } if (message.StatusCode >= (HttpStatusCode)400 && message.StatusCode <= (HttpStatusCode)499) { break; } } } else { pollingUrl = message.Headers.Location; pollingInterval = message.Headers.RetryAfter.Delta ?? pollingInterval; } } } }while (!syncCompleted); return response; })); }