public async Task ScaleUp(IEnumerable <EventData> events)
{
Guard.ArgumentNotNull(events, nameof(events));
IEnumerable <string> collectionsToProcess = _cosmosDbThrottledEventsFilter.GetUniqueCosmosDBContainerNamesFromEventData(events);
if (collectionsToProcess.IsNullOrEmpty())
{
return;
}
_logger.Information($"Found {collectionsToProcess.Count()} collections to process");
foreach (string containerName in collectionsToProcess)
{
try
{
CosmosCollectionType cosmosRepositoryType = containerName.GetEnumValueFromDescription <CosmosCollectionType>();
CosmosDbScalingCollectionSettings settings = await _scalingConfigRepositoryPolicy.ExecuteAsync(() =>
_cosmosDbScalingConfigRepository.GetCollectionSettingsByRepositoryType(cosmosRepositoryType));
int currentThroughput = settings.CurrentRequestUnits;
if (settings.AvailableRequestUnits == 0)
{
string errorMessage = $"The collection '{containerName}' throughput is already at the maximum of {settings.MaxRequestUnits} RU's";
_logger.Warning(errorMessage);
continue;
}
int incrementalRequestUnitsValue = scaleUpIncrementValue;
int increasedRequestUnits = currentThroughput + incrementalRequestUnitsValue;
if (incrementalRequestUnitsValue > settings.AvailableRequestUnits)
{
increasedRequestUnits = settings.MaxRequestUnits;
incrementalRequestUnitsValue = settings.AvailableRequestUnits;
}
settings.CurrentRequestUnits = await ScaleCollection(cosmosRepositoryType, increasedRequestUnits, settings.MaxRequestUnits);
await UpdateCollectionSettings(settings, CosmosDbScalingDirection.Up, incrementalRequestUnitsValue);
}
catch (NonRetriableException)
{
throw;
}
catch (Exception ex)
{
string errorMessage = $"Failed to increase cosmosdb request units on collection '{containerName}'";
_logger.Error(ex, errorMessage);
throw new RetriableException(errorMessage, ex);
}
}
}
private async Task UpdateCollectionSettings(CosmosDbScalingCollectionSettings settings)
{
HttpStatusCode statusCode = await _scalingConfigRepositoryPolicy.ExecuteAsync(
() => _cosmosDbScalingConfigRepository.UpdateCollectionSettings(settings));
if (!statusCode.IsSuccess())
{
string errorMessage = $"Failed to update cosmos scale config repository type: '{settings.CosmosCollectionType}' with new request units of '{settings.CurrentRequestUnits}' with status code: '{statusCode}'";
_logger.Error(errorMessage);
throw new RetriableException(errorMessage);
}
}
private async Task ScaleUpCollection(CosmosDbScalingConfig cosmosDbScalingConfig, string jobDefinitionId)
{
Guard.ArgumentNotNull(cosmosDbScalingConfig, nameof(cosmosDbScalingConfig));
Guard.IsNullOrWhiteSpace(jobDefinitionId, nameof(jobDefinitionId));
CosmosDbScalingJobConfig cosmosDbScalingJobConfig = cosmosDbScalingConfig.JobRequestUnitConfigs
.FirstOrDefault(m => string.Equals(
m.JobDefinitionId,
jobDefinitionId,
StringComparison.InvariantCultureIgnoreCase));
if (cosmosDbScalingJobConfig == null)
{
string errorMessage = $"A job config does not exist for job definition id {jobDefinitionId}";
_logger.Error(errorMessage);
throw new NonRetriableException(errorMessage);
}
CosmosDbScalingCollectionSettings settings = await _scalingConfigRepositoryPolicy.ExecuteAsync(() =>
_cosmosDbScalingConfigRepository.GetCollectionSettingsByRepositoryType(cosmosDbScalingConfig.RepositoryType));
if (settings == null)
{
string errorMessage = $"A collections settings file does not exist for settings collection type: '{cosmosDbScalingConfig.RepositoryType}'";
_logger.Error(errorMessage);
throw new RetriableException(errorMessage);
}
int currentRequestUnits = settings.CurrentRequestUnits;
if (settings.IsAtBaseLine)
{
settings.CurrentRequestUnits = cosmosDbScalingJobConfig.JobRequestUnits;
}
else
{
settings.CurrentRequestUnits =
settings.AvailableRequestUnits >= cosmosDbScalingJobConfig.JobRequestUnits ?
(settings.CurrentRequestUnits + cosmosDbScalingJobConfig.JobRequestUnits) :
settings.MaxRequestUnits;
}
await ScaleCollection(cosmosDbScalingConfig.RepositoryType, settings.CurrentRequestUnits);
int incrementalRequestUnitsValue = currentRequestUnits - settings.CurrentRequestUnits;
await UpdateCollectionSettings(settings, CosmosDbScalingDirection.Up, incrementalRequestUnitsValue);
}
private async Task UpdateCollectionSettings(CosmosDbScalingCollectionSettings settings, CosmosDbScalingDirection direction, int requestUnits)
{
if (direction == CosmosDbScalingDirection.Up)
{
settings.LastScalingIncrementDateTime = DateTimeOffset.Now;
settings.LastScalingIncrementValue = requestUnits;
}
else
{
settings.LastScalingDecrementDateTime = DateTimeOffset.Now;
settings.LastScalingDecrementValue = requestUnits;
}
await UpdateCollectionSettings(settings);
}
public async Task <IActionResult> SaveConfiguration(ScalingConfigurationUpdateModel scalingConfigurationUpdate)
{
FluentValidation.Results.ValidationResult validationResult = await _scalingConfigurationUpdateModelValidator.ValidateAsync(scalingConfigurationUpdate);
if (!validationResult.IsValid)
{
return(validationResult.AsBadRequest());
}
CosmosDbScalingCollectionSettings cosmosDbScalingCollectionSettings = await _cosmosDbScalingConfigRepository.GetCollectionSettingsByRepositoryType(scalingConfigurationUpdate.RepositoryType);
if (cosmosDbScalingCollectionSettings == null)
{
cosmosDbScalingCollectionSettings = new CosmosDbScalingCollectionSettings()
{
CosmosCollectionType = scalingConfigurationUpdate.RepositoryType,
MaxRequestUnits = scalingConfigurationUpdate.MaxRequestUnits,
MinRequestUnits = scalingConfigurationUpdate.BaseRequestUnits,
};
}
else
{
cosmosDbScalingCollectionSettings.MaxRequestUnits = scalingConfigurationUpdate.MaxRequestUnits;
cosmosDbScalingCollectionSettings.MinRequestUnits = scalingConfigurationUpdate.BaseRequestUnits;
}
HttpStatusCode statusCode = await _scalingConfigRepositoryPolicy.ExecuteAsync(
() => _cosmosDbScalingConfigRepository.UpdateCollectionSettings(cosmosDbScalingCollectionSettings));
if (!statusCode.IsSuccess())
{
string errorMessage = $"Failed to Insert or Update Scaling Collection Setting for repository type: '{scalingConfigurationUpdate.RepositoryType}' with status code: '{statusCode}'";
_logger.Error(errorMessage);
throw new RetriableException(errorMessage);
}
await SaveScalingConfig(scalingConfigurationUpdate);
return(new OkObjectResult(scalingConfigurationUpdate));
}
private async Task UpdateCollectionSettings(CosmosDbScalingCollectionSettings settings, CosmosDbScalingDirection direction, int requestUnits)
{
if (direction == CosmosDbScalingDirection.Up)
{
settings.LastScalingIncrementDateTime = DateTimeOffset.Now;
settings.LastScalingIncrementValue = requestUnits;
}
else
{
settings.LastScalingDecrementDateTime = DateTimeOffset.Now;
settings.LastScalingDecrementValue = requestUnits;
}
HttpStatusCode statusCode = await _scalingConfigRepositoryPolicy.ExecuteAsync(
() => _cosmosDbScalingConfigRepository.UpdateCollectionSettings(settings));
if (!statusCode.IsSuccess())
{
string errorMessage = $"Failed to update cosmos scale config repository type: '{settings.CosmosCollectionType}' with new request units of '{settings.CurrentRequestUnits}' with status code: '{statusCode}'";
_logger.Error(errorMessage);
throw new RetriableException(errorMessage);
}
}
public async Task <HttpStatusCode> UpdateCollectionSettings(CosmosDbScalingCollectionSettings settings)
{
Guard.ArgumentNotNull(settings, nameof(settings));
return(await _cosmosRepository.UpsertAsync <CosmosDbScalingCollectionSettings>(settings));
}
public async Task ScaleDownForJobConfiguration()
{
DateTimeOffset now = DateTimeOffset.UtcNow;
DateTimeOffset hourAgo = now.AddHours(-1);
ApiResponse <IEnumerable <JobSummary> > jobSummariesResponse = await _jobsApiClientPolicy.ExecuteAsync(
() => _jobsApiClient.GetNonCompletedJobsWithinTimeFrame(hourAgo, now));
if (!jobSummariesResponse.StatusCode.IsSuccess())
{
string errorMessage = "Failed to fetch job summaries that are still running within the last hour";
_logger.Error(errorMessage);
throw new RetriableException(errorMessage);
}
IEnumerable <string> jobDefinitionIdsStillActive = jobSummariesResponse.Content?.Select(m => m.JobType).Distinct();
IEnumerable <CosmosDbScalingConfig> cosmosDbScalingConfigs = await GetAllConfigs();
IList <CosmosDbScalingCollectionSettings> settingsToUpdate = new List <CosmosDbScalingCollectionSettings>();
foreach (CosmosDbScalingConfig cosmosDbScalingConfig in cosmosDbScalingConfigs)
{
CosmosDbScalingCollectionSettings settings = await _scalingConfigRepositoryPolicy.ExecuteAsync(() =>
_cosmosDbScalingConfigRepository.GetCollectionSettingsByRepositoryType(cosmosDbScalingConfig.RepositoryType));
bool proceed = !settingsToUpdate.Any(m => m.Id == cosmosDbScalingConfig.Id);
if (proceed)
{
if (!settings.IsAtBaseLine)
{
settingsToUpdate.Add(settings);
}
}
}
if (!settingsToUpdate.IsNullOrEmpty())
{
foreach (CosmosDbScalingCollectionSettings settings in settingsToUpdate)
{
try
{
await ScaleCollection(settings.CosmosCollectionType, settings.MinRequestUnits);
int decrementValue = settings.CurrentRequestUnits - settings.MinRequestUnits;
settings.CurrentRequestUnits = settings.MinRequestUnits;
await UpdateCollectionSettings(settings, CosmosDbScalingDirection.Down, decrementValue);
}
catch (Exception ex)
{
throw new RetriableException($"Failed to scale down collection for repository type '{settings.CosmosCollectionType}'", ex);
}
}
await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.RemoveAsync <List <CosmosDbScalingConfig> >(CacheKeys.AllCosmosScalingConfigs));
}
}
public async Task ScaleDownForJobConfiguration()
{
DateTimeOffset now = DateTimeOffset.UtcNow;
DateTimeOffset windowOfTime = now.AddHours(-2);
IEnumerable <JobSummary> jobSummaries = await _jobManagement.GetNonCompletedJobsWithinTimeFrame(windowOfTime, now);
if (jobSummaries == null)
{
string errorMessage = "Failed to fetch job summaries that are still running within the last hour";
_logger.Error(errorMessage);
throw new RetriableException(errorMessage);
}
List <string> jobDefinitionIdsStillActive = jobSummaries.Select(m => m.JobType).Distinct().ToList();
IEnumerable <CosmosDbScalingConfig> cosmosDbScalingConfigs = await GetAllConfigs();
IList <CosmosDbScalingCollectionSettings> settingsToUpdate = new List <CosmosDbScalingCollectionSettings>();
foreach (CosmosDbScalingConfig cosmosDbScalingConfig in cosmosDbScalingConfigs)
{
CosmosDbScalingCollectionSettings settings = await _scalingConfigRepositoryPolicy.ExecuteAsync(() =>
_cosmosDbScalingConfigRepository.GetCollectionSettingsByRepositoryType(cosmosDbScalingConfig.RepositoryType));
bool jobActive = cosmosDbScalingConfig.JobRequestUnitConfigs.Any(item => jobDefinitionIdsStillActive.Contains(item.JobDefinitionId));
bool proceed = !settingsToUpdate.Any(m => m.Id == cosmosDbScalingConfig.Id);
if (proceed && !jobActive)
{
if (!settings.IsAtBaseLine)
{
settingsToUpdate.Add(settings);
}
}
}
if (!settingsToUpdate.IsNullOrEmpty())
{
foreach (CosmosDbScalingCollectionSettings settings in settingsToUpdate)
{
try
{
int?minimumRequestUnitsAllowed = await GetMinimumThroughput(settings.CosmosCollectionType);
if (minimumRequestUnitsAllowed.HasValue && settings.MinRequestUnits < minimumRequestUnitsAllowed.Value)
{
settings.MinRequestUnits = minimumRequestUnitsAllowed.Value;
}
settings.CurrentRequestUnits = await ScaleCollection(settings.CosmosCollectionType, settings.MinRequestUnits, settings.MaxRequestUnits);
await UpdateCollectionSettings(settings);
}
catch (Exception ex)
{
throw new RetriableException($"Failed to scale down collection for repository type '{settings.CosmosCollectionType}'", ex);
}
}
await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.RemoveAsync <List <CosmosDbScalingConfig> >(CacheKeys.AllCosmosScalingConfigs));
}
}
