public override Task <IProcessorResult> Process(
DataModificationType modificationType,
string dataType,
string inputId,
string inputObjectJson)
{
if (dataType == DataApiClient.GetCollectionName <PostponedProcessingObject>())
{
UpdatePostponedObjects(modificationType, inputId, inputObjectJson);
return(Task.FromResult <IProcessorResult>(new SuccessProcessorResult("Postponed objects updated", true)));
}
var matchingObjects = postponedObjects.Values
.Where(obj => obj.MissingData.Any(dataReference => dataReference.DataType == dataType && dataReference.Id == inputId))
.ToList();
if (!matchingObjects.Any())
{
return(Task.FromResult <IProcessorResult>(new NotInterestedProcessorResult()));
}
foreach (var obj in matchingObjects)
{
obj.MissingData.RemoveAll(dataReference => dataReference.DataType == dataType && dataReference.Id == inputId);
}
UnregisterDataTypeIfNoLongerNeeded(dataType);
var summary = $"{matchingObjects.Count} {nameof(PostponedProcessingObject)} were updated";
var outputObjects = matchingObjects
.Select(obj => new SerializedObject(obj.Id, obj.GetType().Name, JsonConvert.SerializeObject(obj)))
.ToList();
return(Task.FromResult <IProcessorResult>(new SuccessProcessorResult(summary, true, outputObjects)));
}
public PostponedProcessingRunner(
IDataApiClient dataApiClient,
IEnumerable <IProcessor> processors,
IDataProcessingServiceLogger dataProcessingServiceLogger)
: base(nameof(PostponedProcessingRunner), DataApiClient.GetCollectionName <PostponedProcessingObject>())
{
this.dataApiClient = dataApiClient;
this.dataProcessingServiceLogger = dataProcessingServiceLogger;
this.processors = processors.ToDictionary(x => x.DisplayName, x => x);
processorRunner = new ProcessorRunner(dataApiClient, dataProcessingServiceLogger);
}
private void UnregisterUnnecessaryInputTypes()
{
var necessaryInputTypes = postponedObjects.Values.SelectMany(x => x.MissingData).Select(x => x.DataType)
.Concat(new[] { DataApiClient.GetCollectionName <PostponedProcessingObject>() })
.Distinct();
var unnecessaryInputTypes = InputTypes.Except(necessaryInputTypes);
foreach (var unnecessaryInputType in unnecessaryInputTypes)
{
UnregisterInputType(unnecessaryInputType);
}
}
private ValidatorDefinition ExtractValidatorDefinition(Type type)
{
var rules = typeValidationRuleBuilder.Build(type, "");
if (!rules.Any())
{
return(null);
}
var ruleset = rules.Aggregate((a, b) => a + "; " + b);
return(new ValidatorDefinition(DataApiClient.GetCollectionName(type), ValidatorType.TextRules, ruleset));
}
public DataServiceProcessor(IDataApiClient dataApiClient)
: base(nameof(DataServiceProcessor), DataApiClient.GetCollectionName <DataServiceDefinition>())
{
this.dataApiClient = dataApiClient;
dataServiceDefinitions = LoadDataServiceDefinitions();
var dataServiceDefinitionsCopy = dataServiceDefinitions.ToList();
foreach (var dataServiceDefinition in dataServiceDefinitionsCopy)
{
var initTask = TransferExistingDataIfNewTarget(dataServiceDefinition, cancellationTokenSource.Token);
initTasks.Add(dataServiceDefinition.Id, initTask);
}
}
public void GetCollectionNameReturnsRegistererdName()
{
var collectionName = "BinaryData";
DataApiClient.RegisterType <DataBlob>(collectionName);
try
{
Assert.That(DataApiClient.GetCollectionName <DataBlob>(), Is.EqualTo(collectionName));
}
finally
{
DataApiClient.UnregisterType <DataBlob>();
}
}
//TODO: [TestCase(typeof(XXX))]
public async Task GenerateValidatorForSpecificType(Type customType)
{
var validatorDefinition = ExtractValidatorDefinition(customType);
if (validatorDefinition == null)
{
return;
}
var dataType = DataApiClient.GetCollectionName(customType);
await SubmitValidator(dataType, validatorDefinition);
await File.AppendAllLinesAsync(outputFilePath, new[] { JsonConvert.SerializeObject(validatorDefinition, Formatting.Indented) });
}
public async Task <ExecutionResult> Action(CancellationToken cancellationToken)
{
var now = DateTime.UtcNow;
var cutoffDateTime = now - logPreservationTimeSpan;
var query = "SELECT _id "
+ $"FROM {DataApiClient.GetCollectionName<DataProcessingServiceLog>()} "
+ $"WHERE SubmissionTimeUtc < '{cutoffDateTime:yyyy-MM-dd HH:mm:ss}'";
var resultStream = await dataApiClient.SearchAsync(query, ResultFormat.Json);
var ids = (await resultStream.ReadAllSearchResultsAsync()).Select(jObject => jObject.Value <string>("_id"));
var deletionCount = 0;
foreach (var id in ids)
{
cancellationToken.ThrowIfCancellationRequested();
dataApiClient.DeleteAsync <DataProcessingServiceLog>(id).Wait(cancellationToken);
deletionCount++;
}
return(new ExecutionResult(true, deletionCount > 0,
$"Deleted all {nameof(DataProcessingServiceLog)} before '{cutoffDateTime:yyyy-MM-dd HH:mm:ss}' (count: {deletionCount})"));
}
public void GetCollectionNameReturnsClassNameWithoutGeneric()
{
Assert.That(DataApiClient.GetCollectionName <Number <double> >(), Is.EqualTo("Number"));
}
public void DataApiCollectionNameAttributeIsUsedIfNoOtherRules()
{
Assert.That(DataApiClient.GetCollectionName <CollectionNameTaggedClass>(), Is.EqualTo("AttributeCollectionName"));
}
public void GetCollectionNameReturnsClosestCollectionNameAttribute()
{
Assert.That(DataApiClient.GetCollectionName <CompletelyOtherNameImplementation>(), Is.EqualTo("ParameterValidation"));
}
public void GetCollectionNameReturnsDefaultIfNothingElseRegistered()
{
Assert.That(DataApiClient.GetCollectionName <DataBlob>(), Is.EqualTo("DataBlob"));
}
public PostponedProcessingObjectUpdateProcessor(IDataApiClient dataApiClient)
: base(nameof(PostponedProcessingObjectUpdateProcessor), DataApiClient.GetCollectionName <PostponedProcessingObject>())
{
this.dataApiClient = dataApiClient;
}
public async Task MeasureOutputPerformance()
{
UserGenerator.RegisterAndLoginUserWithRole(Role.Analyst, adminDataApiClient, out var dataApiClient);
var resultFormat = ResultFormat.Json;
var resultStream = await dataApiClient.SearchAsync($"SELECT Data.id FROM {DataApiClient.GetCollectionName<UnitTestDataObject1>()}", resultFormat);
var resultTable = await Client.Serialization.SeachResultStreamExtensions.ReadAllSearchResultsAsync(resultStream);
var stopWatch = Stopwatch.StartNew();
var batches = new List <UnitTestDataObject1>();
foreach (var componentId in resultTable.Select(x => x.Value <string>("Data_id")))
{
var batch = dataApiClient.GetAsync <UnitTestDataObject1>(componentId).Result;
batches.Add(batch);
}
stopWatch.Stop();
var rowCount = batches.Count;
var averageRetreivalTime = stopWatch.Elapsed.TotalMilliseconds / rowCount;
Console.WriteLine($"It took {stopWatch.Elapsed.TotalSeconds:F3}s to retreive {rowCount} objects (avg: {averageRetreivalTime:F0}ms)");
UserGenerator.DeleteUser(dataApiClient);
}
public override async Task <IProcessorResult> Process(DataModificationType modificationType, string dataType, string inputId, string inputObjectJson)
{
var initTaskSummaries = initTasks.Select(FormatInitTaskStatus).ToList();
var finishedInitTasks = initTasks.Where(kvp => kvp.Value.Status.InSet(TaskStatus.RanToCompletion, TaskStatus.Canceled, TaskStatus.Faulted)).ToList();
finishedInitTasks.Select(kvp => kvp.Key).ForEach(id => initTasks.Remove(id));
if (dataType == DataApiClient.GetCollectionName <DataServiceDefinition>())
{
var dataServiceDefinitionId = inputId;
UpdateDataServiceDefinitions(modificationType, inputId, inputObjectJson);
if (modificationType != DataModificationType.Deleted)
{
var dataServiceDefinition = JsonConvert.DeserializeObject <DataServiceDefinition>(inputObjectJson);
#pragma warning disable 4014
var initTask = TransferExistingDataIfNewTarget(dataServiceDefinition, cancellationTokenSource.Token);
initTasks.Add(dataServiceDefinition.Id, initTask);
#pragma warning restore 4014
bool initTaskCompleted;
try
{
initTaskCompleted = initTask.Wait(TimeSpan.FromSeconds(5));
}
catch
{
initTaskCompleted = true;
}
if (initTaskCompleted)
{
if (initTask.Status == TaskStatus.RanToCompletion)
{
initTaskSummaries.Add($"Init task for service '{dataServiceDefinitionId}' finished without errors. ");
}
else if (initTask.Status == TaskStatus.Faulted)
{
initTaskSummaries.Add($"Init task for service '{dataServiceDefinitionId}' failed: " + initTask.Exception.InnermostException().Message + ". ");
}
else if (initTask.Status == TaskStatus.Canceled)
{
initTaskSummaries.Add($"Init task for service '{dataServiceDefinitionId}' was cancelled. ");
}
initTasks.Remove(dataServiceDefinition.Id);
}
else
{
initTaskSummaries.Add($"Init task for service '{dataServiceDefinitionId}' is running. ");
}
}
var summary = $"Data service definitions '{dataServiceDefinitionId}' {modificationType.ToString().ToLowerInvariant()}";
if (initTaskSummaries.Any())
{
summary += "\n" + string.Join("\n", initTaskSummaries);
}
return(new SuccessProcessorResult(summary, true));
}
var matchingDataServiceDefinitions = dataServiceDefinitions.Where(x => x.DataType == dataType).ToList();
if (!matchingDataServiceDefinitions.Any())
{
return(new NotInterestedProcessorResult());
}
var failingServices = new List <Tuple <DataServiceDefinition, string> >();
foreach (var dataServiceDefinition in matchingDataServiceDefinitions)
{
try
{
var dataService = await CreateDataServiceAsync(dataServiceDefinition);
if (modificationType == DataModificationType.Deleted)
{
await dataService.DeleteAsync(inputId);
}
else
{
var jObject = JObject.Parse(inputObjectJson);
await dataService.TransferAsync(jObject, dataServiceDefinition.Fields, inputId);
}
}
catch (Exception e)
{
failingServices.Add(new Tuple <DataServiceDefinition, string>(dataServiceDefinition, e.InnermostException().Message));
}
}
if (failingServices.Any())
{
var joinedFailedTargets = string.Join("\n", failingServices.Select(tuple => tuple.Item1.Target + ": " + tuple.Item2));
var summary = modificationType == DataModificationType.Deleted
? $"Could not remove {dataType} with ID '{inputId}' from {failingServices.Count} targets:\n{joinedFailedTargets}\n"
: $"Could not transfer {dataType} with ID '{inputId}' to {failingServices.Count} targets:\n{joinedFailedTargets}\n";
if (matchingDataServiceDefinitions.Count > failingServices.Count)
{
var joinedTargets = string.Join("\n", matchingDataServiceDefinitions.Except(failingServices.Select(x => x.Item1)).Select(x => x.Target.ToString()));
summary += modificationType == DataModificationType.Deleted
? $"Successfully removed from {matchingDataServiceDefinitions.Count} targets:\n{joinedTargets}"
: $"Successfully transferred to {matchingDataServiceDefinitions.Count} targets:\n{joinedTargets}";
}
if (initTaskSummaries.Any())
{
summary += "\n" + string.Join("\n", initTaskSummaries);
}
return(new ErrorProcessorResult(summary));
}
else
{
var joinedTargets = string.Join("\n", matchingDataServiceDefinitions.Select(x => x.Target.ToString()));
var summary = modificationType == DataModificationType.Deleted
? $"Removed {dataType} with ID '{inputId}' from {matchingDataServiceDefinitions.Count} targets:\n{joinedTargets}"
: $"Transferred {dataType} with ID '{inputId}' to {matchingDataServiceDefinitions.Count} targets:\n{joinedTargets}";
if (initTaskSummaries.Any())
{
summary += "\n" + string.Join("\n", initTaskSummaries);
}
return(new SuccessProcessorResult(summary, true));
}
}
public static async Task <T> GetAsync <T>(ApiConfiguration configuration, HttpClient httpClient, string id)
{
var json = await GetAsync(configuration, httpClient, DataApiClient.GetCollectionName <T>(), id);
return(!string.IsNullOrEmpty(json) ? ConfiguredJsonSerializer.Deserialize <T>(json) : default);