public override Task <IProcessorResult> Process( DataModificationType modificationType, string dataType, string inputId, string inputObjectJson) { switch (modificationType) { case DataModificationType.Created: case DataModificationType.Replaced: var postponedObject = JsonConvert.DeserializeObject <PostponedProcessingObject>(inputObjectJson); if (!processors.ContainsKey(postponedObject.ProcessorName)) { return(Task.FromResult <IProcessorResult>(new NotInterestedProcessorResult())); } postponedObjects[postponedObject.Id] = postponedObject; break; case DataModificationType.Deleted: postponedObjects.Remove(inputId); break; default: throw new ArgumentOutOfRangeException(nameof(modificationType), modificationType, null); } return(Task.FromResult <IProcessorResult>(new SuccessProcessorResult("Updated postponed object", true))); }
public override async Task <IProcessorResult> Process( DataModificationType modificationType, string dataType, string inputId, string inputJson) { var inputFilePath = $@"C:\temp\{DisplayName}_input.json"; File.WriteAllText(inputFilePath, inputJson); var processInfo = new ProcessStartInfo(Definition.ScriptPath, inputFilePath) { UseShellExecute = false, CreateNoWindow = true, ErrorDialog = false, RedirectStandardError = true, RedirectStandardOutput = true }; var process = System.Diagnostics.Process.Start(processInfo); var errorMessages = new List <string>(); process.ErrorDataReceived += (sender, args) => { if (!string.IsNullOrEmpty(args.Data)) { errorMessages.Add(args.Data); } }; process.BeginErrorReadLine(); var output = string.Empty; process.OutputDataReceived += (sender, args) => { if (!string.IsNullOrEmpty(args.Data)) { output += args.Data; } }; process.BeginOutputReadLine(); if (!await Task.Run(() => process.WaitForExit(MaximumExecuteTimeInMilliSeconds))) { process.Kill(); return(new ErrorProcessorResult($"Processor {DisplayName} didn't finish within {MaximumExecuteTimeInMilliSeconds} ms")); } var isSuccess = process.ExitCode == 0; if (!isSuccess) { if (!errorMessages.Any()) { return(new ErrorProcessorResult("An unknown error occured")); } return(new ErrorProcessorResult( $"Errors: {Environment.NewLine}" + $"{errorMessages.Aggregate((a,b) => a + Environment.NewLine + b)}")); } return(BuildProcessorResults(output)); }
public override Task <IProcessorResult> Process( DataModificationType modificationType, string dataType, string inputId, string inputObjectJson) { if (dataType == DataApiClient.GetCollectionName <PostponedProcessingObject>()) { UpdatePostponedObjects(modificationType, inputId, inputObjectJson); return(Task.FromResult <IProcessorResult>(new SuccessProcessorResult("Postponed objects updated", true))); } var matchingObjects = postponedObjects.Values .Where(obj => obj.MissingData.Any(dataReference => dataReference.DataType == dataType && dataReference.Id == inputId)) .ToList(); if (!matchingObjects.Any()) { return(Task.FromResult <IProcessorResult>(new NotInterestedProcessorResult())); } foreach (var obj in matchingObjects) { obj.MissingData.RemoveAll(dataReference => dataReference.DataType == dataType && dataReference.Id == inputId); } UnregisterDataTypeIfNoLongerNeeded(dataType); var summary = $"{matchingObjects.Count} {nameof(PostponedProcessingObject)} were updated"; var outputObjects = matchingObjects .Select(obj => new SerializedObject(obj.Id, obj.GetType().Name, JsonConvert.SerializeObject(obj))) .ToList(); return(Task.FromResult <IProcessorResult>(new SuccessProcessorResult(summary, true, outputObjects))); }
public bool IsOfInterest(DataModificationType modificationType, string inputId, string inputJson) { if (!ModificationTypes.Contains(modificationType)) { return(false); } return(true); }
public DataNotification( string username, string dataType, string dataObjectId, DataModificationType modificationType) : this(Guid.NewGuid().ToString(), username, dataType, dataObjectId, modificationType) { }
public SubscriptionNotification( string id, string dataType, string dataObjectId, DataModificationType modificationType) { Id = id; DataType = dataType; DataObjectId = dataObjectId; ModificationType = modificationType; }
private async Task ApplyProcessorsToObject( DataModificationType modificationType, string dataType, string inputId, string inputObjectJson, IEnumerable <IProcessor> typeProcessors) { foreach (var typeProcessor in typeProcessors) { await processorRunner.ProcessObject(modificationType, dataType, inputId, inputObjectJson, typeProcessor); } }
private DataNotification( string id, string username, string dataType, string dataObjectId, DataModificationType modificationType) { Id = id; Username = username; DataType = dataType; DataObjectId = dataObjectId; ModificationType = modificationType; }
private void UpdateDataServiceDefinitions(DataModificationType modificationType, string inputId, string inputObjectJson) { if (modificationType == DataModificationType.Deleted) { dataServiceDefinitions.RemoveAll(x => x.Id == inputId); dataServices.TryRemove(inputId, out _); } else { var dataServiceDefinition = JsonConvert.DeserializeObject <DataServiceDefinition>(inputObjectJson); dataServiceDefinitions.RemoveAll(x => x.Id == inputId); dataServiceDefinitions.Add(dataServiceDefinition); } }
public override Task <IProcessorResult> Process(DataModificationType modificationType, string dataType, string inputId, string inputJson) { if (!ModificationTypes.Contains(modificationType)) { return(Task.FromResult <IProcessorResult>(new NotInterestedProcessorResult())); } if (PostponeProcessing) { PostponedItemsCount++; return(Task.FromResult <IProcessorResult>( new PostponedProcessorResult( new PostponedProcessingObject( DisplayName, modificationType, dataType, inputId, MissingDataReference != null ? new List <DataReference> { MissingDataReference } : new List <DataReference>(), MaxPostponeTime, PostponeAttemptsCount, DateTime.UtcNow)))); } var unitTestInputObject = JsonConvert.DeserializeObject <TIn>(inputJson); var result = processFunc(unitTestInputObject); ProcessedItemsCount++; var processorResult = new SuccessProcessorResult( "", true, new List <SerializedObject> { new SerializedObject( result.Id, OutputTypeName, JsonConvert.SerializeObject(result)) }); return(Task.FromResult <IProcessorResult>(processorResult)); }
public PostponedProcessingObject( string processorName, DataModificationType modificationType, string dataType, string dataId, List <DataReference> missingData, TimeSpan maxWaitTime, int remainingAttempts, DateTime lastAttempt) : this( IdGenerator.FromGuid(), processorName, modificationType, dataType, dataId, missingData, maxWaitTime, remainingAttempts, lastAttempt) { }
public PostponedProcessingObject( string id, string processorName, DataModificationType modificationType, string dataType, string dataId, List <DataReference> missingData, TimeSpan maxWaitTime, int remainingAttempts, DateTime lastAttempt) { Id = id; ProcessorName = processorName; DataType = dataType; DataId = dataId; MissingData = missingData; MaxWaitTime = maxWaitTime; RemainingAttempts = remainingAttempts; LastAttempt = lastAttempt; ModificationType = modificationType; }
private void UpdatePostponedObjects(DataModificationType modificationType, string inputId, string inputObjectJson) { switch (modificationType) { case DataModificationType.Created: case DataModificationType.Replaced: var postponedObject = JsonConvert.DeserializeObject <PostponedProcessingObject>(inputObjectJson); postponedObjects.AddOrUpdate(inputId, postponedObject, (s, obj) => postponedObject); foreach (var dataType in postponedObject.MissingData.Select(x => x.DataType)) { RegisterInputType(dataType); } break; case DataModificationType.Deleted: postponedObjects.TryRemove(inputId, out _); UnregisterUnnecessaryInputTypes(); break; default: throw new ArgumentOutOfRangeException(nameof(modificationType), modificationType, null); } }
public abstract Task <IProcessorResult> Process( DataModificationType modificationType, string dataType, string inputId, string inputObjectJson);
public async ValueTask <List <DataModificationResultModel> > ModifyDataAsync <T>(T[] records, DataModificationType modificationType, bool allOrNone, CancellationToken token = default) where T : SalesforceObjectModel { try { token.ThrowIfCancellationRequested(); if (records == null || records.Length == 0) { return(Enumerable.Empty <DataModificationResultModel>().ToList()); } HttpMethod method = modificationType switch { DataModificationType.Insert => HttpMethod.Post, DataModificationType.Update => HttpMethod.Patch, DataModificationType.Delete => HttpMethod.Delete, _ => throw new ArgumentException("Unknown data modification type") }; List <DataModificationResultModel> results = new List <DataModificationResultModel>(records.Length); for (int i = 0; i < records.Length; i += 200) { Range r = i + 200 < records.Length ? new Range(i, i + 200) : new Range(i, records.Length); T[] current = records[r]; _logger?.LogDebug($"Updating {current.Length} records of type {typeof(T)}"); HttpRequestMessage message = await _messageService.BuildDataChangeMessageAsync <T>(current, method, allOrNone, token).ConfigureAwait(false); HttpResponseMessage response = await _client.SendAsync(message, token).ConfigureAwait(false); DataModificationResultModel[] res = await _messageService.ProcessResponseAsync <DataModificationResultModel[]>(response, token).ConfigureAwait(false); results.AddRange(res); } _logger?.LogInformation(DataModificationResultLogString(results)); return(results); } catch (OperationCanceledException) { return(await Task.FromCanceled <List <DataModificationResultModel> >(token).ConfigureAwait(false)); } }
public ValueTask <List <DataModificationResultModel> > ModifyDataAsync <T>(IEnumerable <T> records, DataModificationType modificationType, bool allOrNone, CancellationToken token = default) where T : SalesforceObjectModel => ModifyDataAsync <T>(records.ToArray(), modificationType, allOrNone, token);
public async Task NotifyDataChangedAsync(string dataType, string dataObjectId, DataModificationType modificationType) { if (dataType == null) { throw new ArgumentNullException(nameof(dataType)); } if (dataObjectId == null) { throw new ArgumentNullException(nameof(dataObjectId)); } using var cursor = await subscriptionCollection .Find(x => x.DataType == dataType || x.DataType == null) .ToCursorAsync(); var subscribedUsers = new HashSet <string>(); while (await cursor.MoveNextAsync()) { foreach (var subscription in cursor.Current) { if (subscribedUsers.Contains(subscription.Username)) { continue; } if (!await MatchesFilterAsync(dataType, dataObjectId, subscription.Filter)) { continue; } subscribedUsers.Add(subscription.Username); } } foreach (var subscribedUser in subscribedUsers) { var notification = new DataNotification( subscribedUser, dataType, dataObjectId, modificationType); await dataNotificationCollection.InsertOneAsync(notification); } }
public StoreResult(string id, DataModificationType modificationType, bool isNewCollection) { Id = id; ModificationType = modificationType; IsNewCollection = isNewCollection; }
public async Task ProcessObject(DataModificationType modificationType, string dataType, string inputId, string inputObjectJson, IProcessor typeProcessor) { //await LogExecutionStarting(typeProcessor); var stopWatch = Stopwatch.StartNew(); bool isSuccess; var isWorkDone = true; string summary; string outputTypeName = null; try { var processorResult = await typeProcessor.Process(modificationType, dataType, inputId, inputObjectJson); if (processorResult.Status == ProcessingStatus.NotInterested) { return; } else if (processorResult.Status == ProcessingStatus.Postponed) { var postponedProcessorResult = (PostponedProcessorResult)processorResult; await dataApiClient.InsertAsync(postponedProcessorResult.PostponedObject); return; } else if (processorResult.Status == ProcessingStatus.Error) { isSuccess = false; var errorProcessorResult = (ErrorProcessorResult)processorResult; summary = errorProcessorResult.ErrorMessage; } else if (processorResult.Status == ProcessingStatus.Success) { isSuccess = true; var successProcessorResult = (SuccessProcessorResult)processorResult; foreach (var obj in successProcessorResult.Objects) { await StoreResult(obj); } summary = successProcessorResult.Summary; } else { throw new InvalidEnumArgumentException($"Invalid enum value '{processorResult.Status}' for {nameof(ProcessingStatus)}"); } } catch (Exception e) { isSuccess = false; isWorkDone = false; summary = e.InnermostException().Message; if (string.IsNullOrWhiteSpace(summary)) { summary = e.Message; } } stopWatch.Stop(); await LogExecutionFinished( typeProcessor, outputTypeName, inputId, summary, isSuccess, isWorkDone, stopWatch); }
public override async Task <IProcessorResult> Process(DataModificationType modificationType, string dataType, string inputId, string inputObjectJson) { var initTaskSummaries = initTasks.Select(FormatInitTaskStatus).ToList(); var finishedInitTasks = initTasks.Where(kvp => kvp.Value.Status.InSet(TaskStatus.RanToCompletion, TaskStatus.Canceled, TaskStatus.Faulted)).ToList(); finishedInitTasks.Select(kvp => kvp.Key).ForEach(id => initTasks.Remove(id)); if (dataType == DataApiClient.GetCollectionName <DataServiceDefinition>()) { var dataServiceDefinitionId = inputId; UpdateDataServiceDefinitions(modificationType, inputId, inputObjectJson); if (modificationType != DataModificationType.Deleted) { var dataServiceDefinition = JsonConvert.DeserializeObject <DataServiceDefinition>(inputObjectJson); #pragma warning disable 4014 var initTask = TransferExistingDataIfNewTarget(dataServiceDefinition, cancellationTokenSource.Token); initTasks.Add(dataServiceDefinition.Id, initTask); #pragma warning restore 4014 bool initTaskCompleted; try { initTaskCompleted = initTask.Wait(TimeSpan.FromSeconds(5)); } catch { initTaskCompleted = true; } if (initTaskCompleted) { if (initTask.Status == TaskStatus.RanToCompletion) { initTaskSummaries.Add($"Init task for service '{dataServiceDefinitionId}' finished without errors. "); } else if (initTask.Status == TaskStatus.Faulted) { initTaskSummaries.Add($"Init task for service '{dataServiceDefinitionId}' failed: " + initTask.Exception.InnermostException().Message + ". "); } else if (initTask.Status == TaskStatus.Canceled) { initTaskSummaries.Add($"Init task for service '{dataServiceDefinitionId}' was cancelled. "); } initTasks.Remove(dataServiceDefinition.Id); } else { initTaskSummaries.Add($"Init task for service '{dataServiceDefinitionId}' is running. "); } } var summary = $"Data service definitions '{dataServiceDefinitionId}' {modificationType.ToString().ToLowerInvariant()}"; if (initTaskSummaries.Any()) { summary += "\n" + string.Join("\n", initTaskSummaries); } return(new SuccessProcessorResult(summary, true)); } var matchingDataServiceDefinitions = dataServiceDefinitions.Where(x => x.DataType == dataType).ToList(); if (!matchingDataServiceDefinitions.Any()) { return(new NotInterestedProcessorResult()); } var failingServices = new List <Tuple <DataServiceDefinition, string> >(); foreach (var dataServiceDefinition in matchingDataServiceDefinitions) { try { var dataService = await CreateDataServiceAsync(dataServiceDefinition); if (modificationType == DataModificationType.Deleted) { await dataService.DeleteAsync(inputId); } else { var jObject = JObject.Parse(inputObjectJson); await dataService.TransferAsync(jObject, dataServiceDefinition.Fields, inputId); } } catch (Exception e) { failingServices.Add(new Tuple <DataServiceDefinition, string>(dataServiceDefinition, e.InnermostException().Message)); } } if (failingServices.Any()) { var joinedFailedTargets = string.Join("\n", failingServices.Select(tuple => tuple.Item1.Target + ": " + tuple.Item2)); var summary = modificationType == DataModificationType.Deleted ? $"Could not remove {dataType} with ID '{inputId}' from {failingServices.Count} targets:\n{joinedFailedTargets}\n" : $"Could not transfer {dataType} with ID '{inputId}' to {failingServices.Count} targets:\n{joinedFailedTargets}\n"; if (matchingDataServiceDefinitions.Count > failingServices.Count) { var joinedTargets = string.Join("\n", matchingDataServiceDefinitions.Except(failingServices.Select(x => x.Item1)).Select(x => x.Target.ToString())); summary += modificationType == DataModificationType.Deleted ? $"Successfully removed from {matchingDataServiceDefinitions.Count} targets:\n{joinedTargets}" : $"Successfully transferred to {matchingDataServiceDefinitions.Count} targets:\n{joinedTargets}"; } if (initTaskSummaries.Any()) { summary += "\n" + string.Join("\n", initTaskSummaries); } return(new ErrorProcessorResult(summary)); } else { var joinedTargets = string.Join("\n", matchingDataServiceDefinitions.Select(x => x.Target.ToString())); var summary = modificationType == DataModificationType.Deleted ? $"Removed {dataType} with ID '{inputId}' from {matchingDataServiceDefinitions.Count} targets:\n{joinedTargets}" : $"Transferred {dataType} with ID '{inputId}' to {matchingDataServiceDefinitions.Count} targets:\n{joinedTargets}"; if (initTaskSummaries.Any()) { summary += "\n" + string.Join("\n", initTaskSummaries); } return(new SuccessProcessorResult(summary, true)); } }