public override Task <IProcessorResult> Process(
DataModificationType modificationType,
string dataType,
string inputId,
string inputObjectJson)
{
switch (modificationType)
{
case DataModificationType.Created:
case DataModificationType.Replaced:
var postponedObject = JsonConvert.DeserializeObject <PostponedProcessingObject>(inputObjectJson);
if (!processors.ContainsKey(postponedObject.ProcessorName))
{
return(Task.FromResult <IProcessorResult>(new NotInterestedProcessorResult()));
}
postponedObjects[postponedObject.Id] = postponedObject;
break;
case DataModificationType.Deleted:
postponedObjects.Remove(inputId);
break;
default:
throw new ArgumentOutOfRangeException(nameof(modificationType), modificationType, null);
}
return(Task.FromResult <IProcessorResult>(new SuccessProcessorResult("Updated postponed object", true)));
}
public override async Task <IProcessorResult> Process(
DataModificationType modificationType,
string dataType,
string inputId,
string inputJson)
{
var inputFilePath = $@"C:\temp\{DisplayName}_input.json";
File.WriteAllText(inputFilePath, inputJson);
var processInfo = new ProcessStartInfo(Definition.ScriptPath, inputFilePath)
{
UseShellExecute = false,
CreateNoWindow = true,
ErrorDialog = false,
RedirectStandardError = true,
RedirectStandardOutput = true
};
var process = System.Diagnostics.Process.Start(processInfo);
var errorMessages = new List <string>();
process.ErrorDataReceived += (sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
errorMessages.Add(args.Data);
}
};
process.BeginErrorReadLine();
var output = string.Empty;
process.OutputDataReceived += (sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
output += args.Data;
}
};
process.BeginOutputReadLine();
if (!await Task.Run(() => process.WaitForExit(MaximumExecuteTimeInMilliSeconds)))
{
process.Kill();
return(new ErrorProcessorResult($"Processor {DisplayName} didn't finish within {MaximumExecuteTimeInMilliSeconds} ms"));
}
var isSuccess = process.ExitCode == 0;
if (!isSuccess)
{
if (!errorMessages.Any())
{
return(new ErrorProcessorResult("An unknown error occured"));
}
return(new ErrorProcessorResult(
$"Errors: {Environment.NewLine}" +
$"{errorMessages.Aggregate((a,b) => a + Environment.NewLine + b)}"));
}
return(BuildProcessorResults(output));
}
public override Task <IProcessorResult> Process(
DataModificationType modificationType,
string dataType,
string inputId,
string inputObjectJson)
{
if (dataType == DataApiClient.GetCollectionName <PostponedProcessingObject>())
{
UpdatePostponedObjects(modificationType, inputId, inputObjectJson);
return(Task.FromResult <IProcessorResult>(new SuccessProcessorResult("Postponed objects updated", true)));
}
var matchingObjects = postponedObjects.Values
.Where(obj => obj.MissingData.Any(dataReference => dataReference.DataType == dataType && dataReference.Id == inputId))
.ToList();
if (!matchingObjects.Any())
{
return(Task.FromResult <IProcessorResult>(new NotInterestedProcessorResult()));
}
foreach (var obj in matchingObjects)
{
obj.MissingData.RemoveAll(dataReference => dataReference.DataType == dataType && dataReference.Id == inputId);
}
UnregisterDataTypeIfNoLongerNeeded(dataType);
var summary = $"{matchingObjects.Count} {nameof(PostponedProcessingObject)} were updated";
var outputObjects = matchingObjects
.Select(obj => new SerializedObject(obj.Id, obj.GetType().Name, JsonConvert.SerializeObject(obj)))
.ToList();
return(Task.FromResult <IProcessorResult>(new SuccessProcessorResult(summary, true, outputObjects)));
}
public bool IsOfInterest(DataModificationType modificationType, string inputId, string inputJson)
{
if (!ModificationTypes.Contains(modificationType))
{
return(false);
}
return(true);
}
public DataNotification(
string username,
string dataType,
string dataObjectId,
DataModificationType modificationType)
: this(Guid.NewGuid().ToString(), username, dataType, dataObjectId, modificationType)
{
}
public SubscriptionNotification(
string id,
string dataType,
string dataObjectId,
DataModificationType modificationType)
{
Id = id;
DataType = dataType;
DataObjectId = dataObjectId;
ModificationType = modificationType;
}
private async Task ApplyProcessorsToObject(
DataModificationType modificationType,
string dataType,
string inputId,
string inputObjectJson,
IEnumerable <IProcessor> typeProcessors)
{
foreach (var typeProcessor in typeProcessors)
{
await processorRunner.ProcessObject(modificationType, dataType, inputId, inputObjectJson, typeProcessor);
}
}
private DataNotification(
string id,
string username,
string dataType,
string dataObjectId,
DataModificationType modificationType)
{
Id = id;
Username = username;
DataType = dataType;
DataObjectId = dataObjectId;
ModificationType = modificationType;
}
private void UpdateDataServiceDefinitions(DataModificationType modificationType, string inputId, string inputObjectJson)
{
if (modificationType == DataModificationType.Deleted)
{
dataServiceDefinitions.RemoveAll(x => x.Id == inputId);
dataServices.TryRemove(inputId, out _);
}
else
{
var dataServiceDefinition = JsonConvert.DeserializeObject <DataServiceDefinition>(inputObjectJson);
dataServiceDefinitions.RemoveAll(x => x.Id == inputId);
dataServiceDefinitions.Add(dataServiceDefinition);
}
}
public override Task <IProcessorResult> Process(DataModificationType modificationType, string dataType, string inputId, string inputJson)
{
if (!ModificationTypes.Contains(modificationType))
{
return(Task.FromResult <IProcessorResult>(new NotInterestedProcessorResult()));
}
if (PostponeProcessing)
{
PostponedItemsCount++;
return(Task.FromResult <IProcessorResult>(
new PostponedProcessorResult(
new PostponedProcessingObject(
DisplayName,
modificationType,
dataType,
inputId,
MissingDataReference != null ? new List <DataReference> {
MissingDataReference
} : new List <DataReference>(),
MaxPostponeTime,
PostponeAttemptsCount,
DateTime.UtcNow))));
}
var unitTestInputObject = JsonConvert.DeserializeObject <TIn>(inputJson);
var result = processFunc(unitTestInputObject);
ProcessedItemsCount++;
var processorResult = new SuccessProcessorResult(
"",
true,
new List <SerializedObject>
{
new SerializedObject(
result.Id,
OutputTypeName,
JsonConvert.SerializeObject(result))
});
return(Task.FromResult <IProcessorResult>(processorResult));
}
public PostponedProcessingObject(
string processorName,
DataModificationType modificationType,
string dataType,
string dataId,
List <DataReference> missingData,
TimeSpan maxWaitTime,
int remainingAttempts,
DateTime lastAttempt)
: this(
IdGenerator.FromGuid(),
processorName,
modificationType,
dataType,
dataId,
missingData,
maxWaitTime,
remainingAttempts,
lastAttempt)
{
}
public PostponedProcessingObject(
string id,
string processorName,
DataModificationType modificationType,
string dataType,
string dataId,
List <DataReference> missingData,
TimeSpan maxWaitTime,
int remainingAttempts,
DateTime lastAttempt)
{
Id = id;
ProcessorName = processorName;
DataType = dataType;
DataId = dataId;
MissingData = missingData;
MaxWaitTime = maxWaitTime;
RemainingAttempts = remainingAttempts;
LastAttempt = lastAttempt;
ModificationType = modificationType;
}
private void UpdatePostponedObjects(DataModificationType modificationType, string inputId, string inputObjectJson)
{
switch (modificationType)
{
case DataModificationType.Created:
case DataModificationType.Replaced:
var postponedObject = JsonConvert.DeserializeObject <PostponedProcessingObject>(inputObjectJson);
postponedObjects.AddOrUpdate(inputId, postponedObject, (s, obj) => postponedObject);
foreach (var dataType in postponedObject.MissingData.Select(x => x.DataType))
{
RegisterInputType(dataType);
}
break;
case DataModificationType.Deleted:
postponedObjects.TryRemove(inputId, out _);
UnregisterUnnecessaryInputTypes();
break;
default:
throw new ArgumentOutOfRangeException(nameof(modificationType), modificationType, null);
}
}
public abstract Task <IProcessorResult> Process(
DataModificationType modificationType,
string dataType,
string inputId,
string inputObjectJson);
public async ValueTask <List <DataModificationResultModel> > ModifyDataAsync <T>(T[] records, DataModificationType modificationType, bool allOrNone, CancellationToken token = default) where T : SalesforceObjectModel
{
try
{
token.ThrowIfCancellationRequested();
if (records == null || records.Length == 0)
{
return(Enumerable.Empty <DataModificationResultModel>().ToList());
}
HttpMethod method = modificationType switch
{
DataModificationType.Insert => HttpMethod.Post,
DataModificationType.Update => HttpMethod.Patch,
DataModificationType.Delete => HttpMethod.Delete,
_ => throw new ArgumentException("Unknown data modification type")
};
List <DataModificationResultModel> results = new List <DataModificationResultModel>(records.Length);
for (int i = 0; i < records.Length; i += 200)
{
Range r = i + 200 < records.Length ? new Range(i, i + 200) : new Range(i, records.Length);
T[] current = records[r];
_logger?.LogDebug($"Updating {current.Length} records of type {typeof(T)}");
HttpRequestMessage message = await _messageService.BuildDataChangeMessageAsync <T>(current, method, allOrNone, token).ConfigureAwait(false);
HttpResponseMessage response = await _client.SendAsync(message, token).ConfigureAwait(false);
DataModificationResultModel[] res = await _messageService.ProcessResponseAsync <DataModificationResultModel[]>(response, token).ConfigureAwait(false);
results.AddRange(res);
}
_logger?.LogInformation(DataModificationResultLogString(results));
return(results);
}
catch (OperationCanceledException)
{
return(await Task.FromCanceled <List <DataModificationResultModel> >(token).ConfigureAwait(false));
}
}
public ValueTask <List <DataModificationResultModel> > ModifyDataAsync <T>(IEnumerable <T> records, DataModificationType modificationType, bool allOrNone, CancellationToken token = default) where T : SalesforceObjectModel => ModifyDataAsync <T>(records.ToArray(), modificationType, allOrNone, token);
public async Task NotifyDataChangedAsync(string dataType, string dataObjectId, DataModificationType modificationType)
{
if (dataType == null)
{
throw new ArgumentNullException(nameof(dataType));
}
if (dataObjectId == null)
{
throw new ArgumentNullException(nameof(dataObjectId));
}
using var cursor = await subscriptionCollection
.Find(x => x.DataType == dataType || x.DataType == null)
.ToCursorAsync();
var subscribedUsers = new HashSet <string>();
while (await cursor.MoveNextAsync())
{
foreach (var subscription in cursor.Current)
{
if (subscribedUsers.Contains(subscription.Username))
{
continue;
}
if (!await MatchesFilterAsync(dataType, dataObjectId, subscription.Filter))
{
continue;
}
subscribedUsers.Add(subscription.Username);
}
}
foreach (var subscribedUser in subscribedUsers)
{
var notification = new DataNotification(
subscribedUser,
dataType,
dataObjectId,
modificationType);
await dataNotificationCollection.InsertOneAsync(notification);
}
}
public StoreResult(string id, DataModificationType modificationType, bool isNewCollection)
{
Id = id;
ModificationType = modificationType;
IsNewCollection = isNewCollection;
}
public async Task ProcessObject(DataModificationType modificationType, string dataType, string inputId, string inputObjectJson, IProcessor typeProcessor)
{
//await LogExecutionStarting(typeProcessor);
var stopWatch = Stopwatch.StartNew();
bool isSuccess;
var isWorkDone = true;
string summary;
string outputTypeName = null;
try
{
var processorResult = await typeProcessor.Process(modificationType, dataType, inputId, inputObjectJson);
if (processorResult.Status == ProcessingStatus.NotInterested)
{
return;
}
else if (processorResult.Status == ProcessingStatus.Postponed)
{
var postponedProcessorResult = (PostponedProcessorResult)processorResult;
await dataApiClient.InsertAsync(postponedProcessorResult.PostponedObject);
return;
}
else if (processorResult.Status == ProcessingStatus.Error)
{
isSuccess = false;
var errorProcessorResult = (ErrorProcessorResult)processorResult;
summary = errorProcessorResult.ErrorMessage;
}
else if (processorResult.Status == ProcessingStatus.Success)
{
isSuccess = true;
var successProcessorResult = (SuccessProcessorResult)processorResult;
foreach (var obj in successProcessorResult.Objects)
{
await StoreResult(obj);
}
summary = successProcessorResult.Summary;
}
else
{
throw new InvalidEnumArgumentException($"Invalid enum value '{processorResult.Status}' for {nameof(ProcessingStatus)}");
}
}
catch (Exception e)
{
isSuccess = false;
isWorkDone = false;
summary = e.InnermostException().Message;
if (string.IsNullOrWhiteSpace(summary))
{
summary = e.Message;
}
}
stopWatch.Stop();
await LogExecutionFinished(
typeProcessor,
outputTypeName,
inputId,
summary,
isSuccess,
isWorkDone,
stopWatch);
}
public override async Task <IProcessorResult> Process(DataModificationType modificationType, string dataType, string inputId, string inputObjectJson)
{
var initTaskSummaries = initTasks.Select(FormatInitTaskStatus).ToList();
var finishedInitTasks = initTasks.Where(kvp => kvp.Value.Status.InSet(TaskStatus.RanToCompletion, TaskStatus.Canceled, TaskStatus.Faulted)).ToList();
finishedInitTasks.Select(kvp => kvp.Key).ForEach(id => initTasks.Remove(id));
if (dataType == DataApiClient.GetCollectionName <DataServiceDefinition>())
{
var dataServiceDefinitionId = inputId;
UpdateDataServiceDefinitions(modificationType, inputId, inputObjectJson);
if (modificationType != DataModificationType.Deleted)
{
var dataServiceDefinition = JsonConvert.DeserializeObject <DataServiceDefinition>(inputObjectJson);
#pragma warning disable 4014
var initTask = TransferExistingDataIfNewTarget(dataServiceDefinition, cancellationTokenSource.Token);
initTasks.Add(dataServiceDefinition.Id, initTask);
#pragma warning restore 4014
bool initTaskCompleted;
try
{
initTaskCompleted = initTask.Wait(TimeSpan.FromSeconds(5));
}
catch
{
initTaskCompleted = true;
}
if (initTaskCompleted)
{
if (initTask.Status == TaskStatus.RanToCompletion)
{
initTaskSummaries.Add($"Init task for service '{dataServiceDefinitionId}' finished without errors. ");
}
else if (initTask.Status == TaskStatus.Faulted)
{
initTaskSummaries.Add($"Init task for service '{dataServiceDefinitionId}' failed: " + initTask.Exception.InnermostException().Message + ". ");
}
else if (initTask.Status == TaskStatus.Canceled)
{
initTaskSummaries.Add($"Init task for service '{dataServiceDefinitionId}' was cancelled. ");
}
initTasks.Remove(dataServiceDefinition.Id);
}
else
{
initTaskSummaries.Add($"Init task for service '{dataServiceDefinitionId}' is running. ");
}
}
var summary = $"Data service definitions '{dataServiceDefinitionId}' {modificationType.ToString().ToLowerInvariant()}";
if (initTaskSummaries.Any())
{
summary += "\n" + string.Join("\n", initTaskSummaries);
}
return(new SuccessProcessorResult(summary, true));
}
var matchingDataServiceDefinitions = dataServiceDefinitions.Where(x => x.DataType == dataType).ToList();
if (!matchingDataServiceDefinitions.Any())
{
return(new NotInterestedProcessorResult());
}
var failingServices = new List <Tuple <DataServiceDefinition, string> >();
foreach (var dataServiceDefinition in matchingDataServiceDefinitions)
{
try
{
var dataService = await CreateDataServiceAsync(dataServiceDefinition);
if (modificationType == DataModificationType.Deleted)
{
await dataService.DeleteAsync(inputId);
}
else
{
var jObject = JObject.Parse(inputObjectJson);
await dataService.TransferAsync(jObject, dataServiceDefinition.Fields, inputId);
}
}
catch (Exception e)
{
failingServices.Add(new Tuple <DataServiceDefinition, string>(dataServiceDefinition, e.InnermostException().Message));
}
}
if (failingServices.Any())
{
var joinedFailedTargets = string.Join("\n", failingServices.Select(tuple => tuple.Item1.Target + ": " + tuple.Item2));
var summary = modificationType == DataModificationType.Deleted
? $"Could not remove {dataType} with ID '{inputId}' from {failingServices.Count} targets:\n{joinedFailedTargets}\n"
: $"Could not transfer {dataType} with ID '{inputId}' to {failingServices.Count} targets:\n{joinedFailedTargets}\n";
if (matchingDataServiceDefinitions.Count > failingServices.Count)
{
var joinedTargets = string.Join("\n", matchingDataServiceDefinitions.Except(failingServices.Select(x => x.Item1)).Select(x => x.Target.ToString()));
summary += modificationType == DataModificationType.Deleted
? $"Successfully removed from {matchingDataServiceDefinitions.Count} targets:\n{joinedTargets}"
: $"Successfully transferred to {matchingDataServiceDefinitions.Count} targets:\n{joinedTargets}";
}
if (initTaskSummaries.Any())
{
summary += "\n" + string.Join("\n", initTaskSummaries);
}
return(new ErrorProcessorResult(summary));
}
else
{
var joinedTargets = string.Join("\n", matchingDataServiceDefinitions.Select(x => x.Target.ToString()));
var summary = modificationType == DataModificationType.Deleted
? $"Removed {dataType} with ID '{inputId}' from {matchingDataServiceDefinitions.Count} targets:\n{joinedTargets}"
: $"Transferred {dataType} with ID '{inputId}' to {matchingDataServiceDefinitions.Count} targets:\n{joinedTargets}";
if (initTaskSummaries.Any())
{
summary += "\n" + string.Join("\n", initTaskSummaries);
}
return(new SuccessProcessorResult(summary, true));
}
}
