public void CreateDataRetrievalForTable_NotAvailableCookerThrows() { var cookedDataRetrieval = new TestCookedDataRetrieval(); var dataExtensionRepository = new TestDataExtensionRepository(); var dataCookerReference = new TestCompositeDataCookerReference(false) { availability = DataExtensionAvailability.Error, Path = new DataCookerPath("CompositeCooker1"), }; var tableReference = new TestTableExtensionReference(false) { availability = DataExtensionAvailability.IndirectError, TableDescriptor = new TableDescriptor( Guid.Parse("{F0F20004-E159-447B-B122-BD820C2A9908}"), "Test Table", "Test Table"), }; dataExtensionRepository.compositeCookersByPath.Add(dataCookerReference.Path, dataCookerReference); dataExtensionRepository.tablesById.Add(tableReference.TableDescriptor.Guid, tableReference); var dataExtensionRetrievalFactory = new DataExtensionRetrievalFactory(cookedDataRetrieval, dataExtensionRepository); Assert.ThrowsException <ArgumentException>(() => dataExtensionRetrievalFactory.CreateDataRetrievalForTable(tableReference.TableDescriptor.Guid)); }
/// <inheritdoc /> public void FinalizeTables() { lock (this.dataProcessor) { if (this.finalizedData) { throw new InvalidOperationException($"{nameof(FinalizeTables)} may only be called once."); } this.finalizedData = true; } // When a plugin has multiple source parsers, then it's possible that the list of table references has some // tables associated with a source parser other than the one associated with the data source that was // opened. so just remove these tables and proceed - this is not an error case // var tablesToRemove = new List <TableDescriptor>(); foreach (var kvp in this.tableReferences) { var tableReference = kvp.Value; tableReference.ProcessDependencies(this.dataExtensionRepository); if (tableReference.Availability != DataExtensionAvailability.Available) { tablesToRemove.Add(kvp.Key); continue; } // check that there are no external sources required foreach (var cookerPath in tableReference.DependencyReferences.RequiredSourceDataCookerPaths) { if (!StringComparer.Ordinal.Equals(cookerPath.SourceParserId, this.dataProcessor.SourceParserId)) { tablesToRemove.Add(kvp.Key); break; } Debug.Assert(!string.IsNullOrWhiteSpace(cookerPath.SourceParserId)); } } foreach (var tableDescriptor in tablesToRemove) { this.tableReferences.Remove(tableDescriptor); } lock (this.dataProcessor) { if (this.dataExtensionRetrievalFactory == null) { this.dataExtensionRetrievalFactory = new DataExtensionRetrievalFactory(this.dataProcessor, dataExtensionRepository); } } }
internal FilteredDataRetrieval( DataExtensionRetrievalFactory dataRetrievalFactory, IDataExtensionDependencies extensionDependencies) { Debug.Assert(dataRetrievalFactory != null, nameof(dataRetrievalFactory)); Debug.Assert(extensionDependencies != null, nameof(extensionDependencies)); this.dataRetrievalFactory = dataRetrievalFactory; this.extensionDependencies = extensionDependencies; }
public void Constructor() { var cookedDataRetrieval = new TestCookedDataRetrieval(); var dataExtensionRepository = new TestDataExtensionRepository(); var dataExtensionRetrievalFactory = new DataExtensionRetrievalFactory(cookedDataRetrieval, dataExtensionRepository); Assert.AreEqual(cookedDataRetrieval, dataExtensionRetrievalFactory.CookedSourceData); Assert.AreEqual(dataExtensionRepository, dataExtensionRetrievalFactory.DataExtensionRepository); }
public void CreateDataRetrievalForCompositeDataCooker_MissingCookerThrows() { var cookedDataRetrieval = new TestCookedDataRetrieval(); var dataExtensionRepository = new TestDataExtensionRepository(); var cookerPath = new DataCookerPath("CompositeCooker1"); var dataExtensionRetrievalFactory = new DataExtensionRetrievalFactory(cookedDataRetrieval, dataExtensionRepository); Assert.ThrowsException <ArgumentException>(() => dataExtensionRetrievalFactory.CreateDataRetrievalForCompositeDataCooker(cookerPath)); }
public void CreateDataRetrievalForTable_MissingCookerThrows() { var cookedDataRetrieval = new TestCookedDataRetrieval(); var dataExtensionRepository = new TestDataExtensionRepository(); var tableId = Guid.Parse("{6BB197C8-B3BE-4926-B23A-6F01451D80A3}"); var dataExtensionRetrievalFactory = new DataExtensionRetrievalFactory(cookedDataRetrieval, dataExtensionRepository); Assert.ThrowsException <ArgumentException>(() => dataExtensionRetrievalFactory.CreateDataRetrievalForTable(tableId)); }
private RuntimeExecutionResults ProcessCore() { this.IsProcessed = true; var allFileAssociations = GroupAllFilesToDataSources( this.CustomDataSources, this.FreeFilesToProcess, this.FilesToProcess); var executors = CreateExecutors(allFileAssociations); var processors = this.repository.EnableDataCookers( executors.Select(x => x.Processor as ICustomDataProcessorWithSourceParser).Where(x => !(x is null)), new HashSet <DataCookerPath>(this.enabledCookers)); var executionResults = new List <ExecutionResult>(executors.Count); foreach (var executor in executors) { try { var executionResult = executor.ExecuteAsync(CancellationToken.None) .ConfigureAwait(false) .GetAwaiter() .GetResult(); executionResults.Add(executionResult); } catch (Exception e) { // // todo: log // Console.Error.WriteLine(e); } } var retrieval = this.factory.CreateCrossParserSourceDataCookerRetrieval(processors); var retrievalFactory = new DataExtensionRetrievalFactory(retrieval, this.repository); var results = new RuntimeExecutionResults( retrieval, retrievalFactory, this.repository); return(results); }
public void CreateDataRetrievalForTableSucceeds() { var cookedDataRetrieval = new TestCookedDataRetrieval(); var dataExtensionRepository = new TestDataExtensionRepository(); var sourceDataCookerReference1 = new TestSourceDataCookerReference(false) { availability = DataExtensionAvailability.Available, Path = new DataCookerPath("Source1", "SourceCooker1"), }; var dataCookerReference = new TestCompositeDataCookerReference(false) { availability = DataExtensionAvailability.Available, Path = new DataCookerPath("CompositeCooker1"), }; dataCookerReference.requiredDataCookers.Add(sourceDataCookerReference1.Path); var tableReference = new TestTableExtensionReference(false) { availability = DataExtensionAvailability.Available, TableDescriptor = new TableDescriptor( Guid.Parse("{F0F20004-E159-447B-B122-BD820C2A9908}"), "Test Table", "Test Table"), }; dataExtensionRepository.sourceCookersByPath.Add(sourceDataCookerReference1.Path, sourceDataCookerReference1); dataExtensionRepository.compositeCookersByPath.Add(dataCookerReference.Path, dataCookerReference); dataExtensionRepository.tablesById.Add(tableReference.TableDescriptor.Guid, tableReference); var dataExtensionRetrievalFactory = new DataExtensionRetrievalFactory(cookedDataRetrieval, dataExtensionRepository); var dataRetrieval = dataExtensionRetrievalFactory.CreateDataRetrievalForTable(tableReference.TableDescriptor.Guid); Assert.IsNotNull(dataRetrieval); // check that the cache is working as expected var dataRetrieval2 = dataExtensionRetrievalFactory.CreateDataRetrievalForTable(tableReference.TableDescriptor.Guid); Assert.IsNotNull(dataRetrieval2); Assert.IsTrue(object.ReferenceEquals(dataRetrieval, dataRetrieval2)); }
public void CreateDataRetrievalForCompositeDataCooker_NotAvailableCookerThrows() { var cookedDataRetrieval = new TestCookedDataRetrieval(); var dataExtensionRepository = new TestDataExtensionRepository(); var dataCookerReference = new TestCompositeDataCookerReference(false) { availability = DataExtensionAvailability.Error, Path = new DataCookerPath("CompositeCooker1"), }; dataExtensionRepository.compositeCookersByPath.Add(dataCookerReference.Path, dataCookerReference); var dataExtensionRetrievalFactory = new DataExtensionRetrievalFactory(cookedDataRetrieval, dataExtensionRepository); Assert.ThrowsException <ArgumentException>(() => dataExtensionRetrievalFactory.CreateDataRetrievalForCompositeDataCooker(dataCookerReference.Path)); }
public void CreateDataRetrievalForCompositeDataCookerSucceeds() { var cookedDataRetrieval = new TestCookedDataRetrieval(); var dataExtensionRepository = new TestDataExtensionRepository(); var sourceDataCookerReference1 = new TestSourceDataCookerReference(false) { availability = DataExtensionAvailability.Available, Path = new DataCookerPath("Source1", "SourceCooker1"), }; var dataCookerReference = new TestCompositeDataCookerReference(false) { availability = DataExtensionAvailability.Available, Path = new DataCookerPath("CompositeCooker1"), }; dataCookerReference.requiredDataCookers.Add(sourceDataCookerReference1.Path); dataExtensionRepository.sourceCookersByPath.Add(sourceDataCookerReference1.Path, sourceDataCookerReference1); dataExtensionRepository.compositeCookersByPath.Add(dataCookerReference.Path, dataCookerReference); var dataExtensionRetrievalFactory = new DataExtensionRetrievalFactory(cookedDataRetrieval, dataExtensionRepository); var dataRetrieval = dataExtensionRetrievalFactory.CreateDataRetrievalForCompositeDataCooker(dataCookerReference.Path); Assert.IsNotNull(dataRetrieval); // check that the cache is working as expected var dataRetrieval2 = dataExtensionRetrievalFactory.CreateDataRetrievalForCompositeDataCooker(dataCookerReference.Path); Assert.IsNotNull(dataRetrieval2); Assert.IsTrue(object.ReferenceEquals(dataRetrieval, dataRetrieval2)); }