public void CreateDataRetrievalForTable_NotAvailableCookerThrows()
{
var cookedDataRetrieval = new TestCookedDataRetrieval();
var dataExtensionRepository = new TestDataExtensionRepository();
var dataCookerReference = new TestCompositeDataCookerReference(false)
{
availability = DataExtensionAvailability.Error,
Path = new DataCookerPath("CompositeCooker1"),
};
var tableReference = new TestTableExtensionReference(false)
{
availability = DataExtensionAvailability.IndirectError,
TableDescriptor = new TableDescriptor(
Guid.Parse("{F0F20004-E159-447B-B122-BD820C2A9908}"), "Test Table", "Test Table"),
};
dataExtensionRepository.compositeCookersByPath.Add(dataCookerReference.Path, dataCookerReference);
dataExtensionRepository.tablesById.Add(tableReference.TableDescriptor.Guid, tableReference);
var dataExtensionRetrievalFactory =
new DataExtensionRetrievalFactory(cookedDataRetrieval, dataExtensionRepository);
Assert.ThrowsException <ArgumentException>(() =>
dataExtensionRetrievalFactory.CreateDataRetrievalForTable(tableReference.TableDescriptor.Guid));
}
/// <inheritdoc />
public void FinalizeTables()
{
lock (this.dataProcessor)
{
if (this.finalizedData)
{
throw new InvalidOperationException($"{nameof(FinalizeTables)} may only be called once.");
}
this.finalizedData = true;
}
// When a plugin has multiple source parsers, then it's possible that the list of table references has some
// tables associated with a source parser other than the one associated with the data source that was
// opened. so just remove these tables and proceed - this is not an error case
//
var tablesToRemove = new List <TableDescriptor>();
foreach (var kvp in this.tableReferences)
{
var tableReference = kvp.Value;
tableReference.ProcessDependencies(this.dataExtensionRepository);
if (tableReference.Availability != DataExtensionAvailability.Available)
{
tablesToRemove.Add(kvp.Key);
continue;
}
// check that there are no external sources required
foreach (var cookerPath in tableReference.DependencyReferences.RequiredSourceDataCookerPaths)
{
if (!StringComparer.Ordinal.Equals(cookerPath.SourceParserId, this.dataProcessor.SourceParserId))
{
tablesToRemove.Add(kvp.Key);
break;
}
Debug.Assert(!string.IsNullOrWhiteSpace(cookerPath.SourceParserId));
}
}
foreach (var tableDescriptor in tablesToRemove)
{
this.tableReferences.Remove(tableDescriptor);
}
lock (this.dataProcessor)
{
if (this.dataExtensionRetrievalFactory == null)
{
this.dataExtensionRetrievalFactory =
new DataExtensionRetrievalFactory(this.dataProcessor, dataExtensionRepository);
}
}
}
internal FilteredDataRetrieval(
DataExtensionRetrievalFactory dataRetrievalFactory,
IDataExtensionDependencies extensionDependencies)
{
Debug.Assert(dataRetrievalFactory != null, nameof(dataRetrievalFactory));
Debug.Assert(extensionDependencies != null, nameof(extensionDependencies));
this.dataRetrievalFactory = dataRetrievalFactory;
this.extensionDependencies = extensionDependencies;
}
public void Constructor()
{
var cookedDataRetrieval = new TestCookedDataRetrieval();
var dataExtensionRepository = new TestDataExtensionRepository();
var dataExtensionRetrievalFactory =
new DataExtensionRetrievalFactory(cookedDataRetrieval, dataExtensionRepository);
Assert.AreEqual(cookedDataRetrieval, dataExtensionRetrievalFactory.CookedSourceData);
Assert.AreEqual(dataExtensionRepository, dataExtensionRetrievalFactory.DataExtensionRepository);
}
public void CreateDataRetrievalForCompositeDataCooker_MissingCookerThrows()
{
var cookedDataRetrieval = new TestCookedDataRetrieval();
var dataExtensionRepository = new TestDataExtensionRepository();
var cookerPath = new DataCookerPath("CompositeCooker1");
var dataExtensionRetrievalFactory =
new DataExtensionRetrievalFactory(cookedDataRetrieval, dataExtensionRepository);
Assert.ThrowsException <ArgumentException>(() =>
dataExtensionRetrievalFactory.CreateDataRetrievalForCompositeDataCooker(cookerPath));
}
public void CreateDataRetrievalForTable_MissingCookerThrows()
{
var cookedDataRetrieval = new TestCookedDataRetrieval();
var dataExtensionRepository = new TestDataExtensionRepository();
var tableId = Guid.Parse("{6BB197C8-B3BE-4926-B23A-6F01451D80A3}");
var dataExtensionRetrievalFactory =
new DataExtensionRetrievalFactory(cookedDataRetrieval, dataExtensionRepository);
Assert.ThrowsException <ArgumentException>(() =>
dataExtensionRetrievalFactory.CreateDataRetrievalForTable(tableId));
}
private RuntimeExecutionResults ProcessCore()
{
this.IsProcessed = true;
var allFileAssociations = GroupAllFilesToDataSources(
this.CustomDataSources,
this.FreeFilesToProcess,
this.FilesToProcess);
var executors = CreateExecutors(allFileAssociations);
var processors = this.repository.EnableDataCookers(
executors.Select(x => x.Processor as ICustomDataProcessorWithSourceParser).Where(x => !(x is null)),
new HashSet <DataCookerPath>(this.enabledCookers));
var executionResults = new List <ExecutionResult>(executors.Count);
foreach (var executor in executors)
{
try
{
var executionResult = executor.ExecuteAsync(CancellationToken.None)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
executionResults.Add(executionResult);
}
catch (Exception e)
{
//
// todo: log
//
Console.Error.WriteLine(e);
}
}
var retrieval = this.factory.CreateCrossParserSourceDataCookerRetrieval(processors);
var retrievalFactory = new DataExtensionRetrievalFactory(retrieval, this.repository);
var results = new RuntimeExecutionResults(
retrieval,
retrievalFactory,
this.repository);
return(results);
}
public void CreateDataRetrievalForTableSucceeds()
{
var cookedDataRetrieval = new TestCookedDataRetrieval();
var dataExtensionRepository = new TestDataExtensionRepository();
var sourceDataCookerReference1 = new TestSourceDataCookerReference(false)
{
availability = DataExtensionAvailability.Available,
Path = new DataCookerPath("Source1", "SourceCooker1"),
};
var dataCookerReference = new TestCompositeDataCookerReference(false)
{
availability = DataExtensionAvailability.Available,
Path = new DataCookerPath("CompositeCooker1"),
};
dataCookerReference.requiredDataCookers.Add(sourceDataCookerReference1.Path);
var tableReference = new TestTableExtensionReference(false)
{
availability = DataExtensionAvailability.Available,
TableDescriptor = new TableDescriptor(
Guid.Parse("{F0F20004-E159-447B-B122-BD820C2A9908}"), "Test Table", "Test Table"),
};
dataExtensionRepository.sourceCookersByPath.Add(sourceDataCookerReference1.Path, sourceDataCookerReference1);
dataExtensionRepository.compositeCookersByPath.Add(dataCookerReference.Path, dataCookerReference);
dataExtensionRepository.tablesById.Add(tableReference.TableDescriptor.Guid, tableReference);
var dataExtensionRetrievalFactory =
new DataExtensionRetrievalFactory(cookedDataRetrieval, dataExtensionRepository);
var dataRetrieval = dataExtensionRetrievalFactory.CreateDataRetrievalForTable(tableReference.TableDescriptor.Guid);
Assert.IsNotNull(dataRetrieval);
// check that the cache is working as expected
var dataRetrieval2 = dataExtensionRetrievalFactory.CreateDataRetrievalForTable(tableReference.TableDescriptor.Guid);
Assert.IsNotNull(dataRetrieval2);
Assert.IsTrue(object.ReferenceEquals(dataRetrieval, dataRetrieval2));
}
public void CreateDataRetrievalForCompositeDataCooker_NotAvailableCookerThrows()
{
var cookedDataRetrieval = new TestCookedDataRetrieval();
var dataExtensionRepository = new TestDataExtensionRepository();
var dataCookerReference = new TestCompositeDataCookerReference(false)
{
availability = DataExtensionAvailability.Error,
Path = new DataCookerPath("CompositeCooker1"),
};
dataExtensionRepository.compositeCookersByPath.Add(dataCookerReference.Path, dataCookerReference);
var dataExtensionRetrievalFactory =
new DataExtensionRetrievalFactory(cookedDataRetrieval, dataExtensionRepository);
Assert.ThrowsException <ArgumentException>(() =>
dataExtensionRetrievalFactory.CreateDataRetrievalForCompositeDataCooker(dataCookerReference.Path));
}
public void CreateDataRetrievalForCompositeDataCookerSucceeds()
{
var cookedDataRetrieval = new TestCookedDataRetrieval();
var dataExtensionRepository = new TestDataExtensionRepository();
var sourceDataCookerReference1 = new TestSourceDataCookerReference(false)
{
availability = DataExtensionAvailability.Available,
Path = new DataCookerPath("Source1", "SourceCooker1"),
};
var dataCookerReference = new TestCompositeDataCookerReference(false)
{
availability = DataExtensionAvailability.Available,
Path = new DataCookerPath("CompositeCooker1"),
};
dataCookerReference.requiredDataCookers.Add(sourceDataCookerReference1.Path);
dataExtensionRepository.sourceCookersByPath.Add(sourceDataCookerReference1.Path, sourceDataCookerReference1);
dataExtensionRepository.compositeCookersByPath.Add(dataCookerReference.Path, dataCookerReference);
var dataExtensionRetrievalFactory =
new DataExtensionRetrievalFactory(cookedDataRetrieval, dataExtensionRepository);
var dataRetrieval = dataExtensionRetrievalFactory.CreateDataRetrievalForCompositeDataCooker(dataCookerReference.Path);
Assert.IsNotNull(dataRetrieval);
// check that the cache is working as expected
var dataRetrieval2 = dataExtensionRetrievalFactory.CreateDataRetrievalForCompositeDataCooker(dataCookerReference.Path);
Assert.IsNotNull(dataRetrieval2);
Assert.IsTrue(object.ReferenceEquals(dataRetrieval, dataRetrieval2));
}