public RuntimeProcessorEnvironment(
IDataExtensionRepository repository)
{
Debug.Assert(repository != null);
this.repository = repository;
}
private void ProcessRequiredExtension(
IDataExtensionReference reference,
string extensionId,
IDataExtensionRepository availableDataExtensions)
{
Guard.NotNull(reference, nameof(reference));
Guard.NotNull(availableDataExtensions, nameof(availableDataExtensions));
Debug.Assert(!string.IsNullOrWhiteSpace(extensionId));
if (reference.Availability == DataExtensionAvailability.Undetermined)
{
reference.ProcessDependencies(availableDataExtensions);
}
if (reference.Availability != DataExtensionAvailability.Error)
{
this.dependencyReferences.AddRequiredExtensionReferences(reference.DependencyReferences);
}
Debug.Assert(
reference.Availability != DataExtensionAvailability.Undetermined,
$"{nameof(this.ProcessDependencies)} returned without establishing Availability.");
this.UpdateAvailabilityFromRequiredExtension(reference, extensionId);
this.target.PerformAdditionalDataExtensionValidation(this, reference);
}
/// <summary>
/// This constructor is associated with a given <see cref="ICustomDataProcessorWithSourceParser"/>, provided
/// as a parameter. <paramref name="dataExtensionRepository"/> is required to retrieve all available data
/// extensions.
/// </summary>
/// <param name="dataProcessor">
/// The data processor with which this object is associated.
/// </param>
/// <param name="dataExtensionRepository">
/// Provides access to a set of data extensions.
/// </param>
public CustomDataProcessorExtensibilitySupport(
ICustomDataProcessorWithSourceParser dataProcessor,
IDataExtensionRepository dataExtensionRepository)
{
this.dataProcessor = dataProcessor;
this.dataExtensionRepository = dataExtensionRepository;
this.tableReferences = new Dictionary <TableDescriptor, ITableExtensionReference>();
}
/// <summary>
/// Creates an instance.
/// </summary>
/// <param name="sourceCookerData">
/// Provides access to source data cooker output.
/// </param>
/// <param name="dataExtensionRepository">
/// Provides a way to generate data extensions other than source data cookers.
/// </param>
public DataExtensionRetrievalFactory(
ICookedDataRetrieval sourceCookerData,
IDataExtensionRepository dataExtensionRepository)
{
Guard.NotNull(sourceCookerData, nameof(sourceCookerData));
Guard.NotNull(dataExtensionRepository, nameof(dataExtensionRepository));
this.CookedSourceData = sourceCookerData;
this.DataExtensionRepository = dataExtensionRepository;
}
/// <inheritdoc />
public void ProcessDependencies(IDataExtensionRepository availableDataExtensions)
{
Guard.NotNull(availableDataExtensions, nameof(availableDataExtensions));
if (this.extensionDependencyState == null)
{
this.extensionDependencyState = new DataExtensionDependencyState(this);
}
this.extensionDependencyState.ProcessDependencies(availableDataExtensions);
}
public void ProcessDependencies(
IDataExtensionRepository availableDataExtensions)
{
if (this.processDependencies != null)
{
this.processDependencies.Invoke(availableDataExtensions);
}
else if (this.DependencyState != null)
{
this.DependencyState.ProcessDependencies(availableDataExtensions);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="RuntimeExecutionResults"/>
/// class.
/// </summary>
/// <param name="cookedDataRetrieval">
/// The retrieval interface for getting to cooked data.
/// </param>
/// <param name="retrievalFactory">
/// The factory for creating retrievals for composite cookers.
/// </param>
/// <param name="repository">
/// The repository that was used to process the data.
/// </param>
/// <exception cref="ArgumentNullException">
/// <paramref name="cookedDataRetrieval"/> is <c>null</c>.
/// - or -
/// <paramref name="retrievalFactory"/> is <c>null</c>.
/// - or -
/// <paramref name="repository"/> is <c>null</c>.
/// </exception>
public RuntimeExecutionResults(
ICookedDataRetrieval cookedDataRetrieval,
IDataExtensionRetrievalFactory retrievalFactory,
IDataExtensionRepository repository)
{
Guard.NotNull(cookedDataRetrieval, nameof(cookedDataRetrieval));
Guard.NotNull(retrievalFactory, nameof(retrievalFactory));
Guard.NotNull(repository, nameof(repository));
this.cookedDataRetrieval = cookedDataRetrieval;
this.retrievalFactory = retrievalFactory;
this.repository = repository;
this.sourceCookers = new HashSet <DataCookerPath>(this.repository.SourceDataCookers);
}
/// <summary>
/// Attempts to get a data cooker reference given a data cooker path.
/// </summary>
/// <param name="self">
/// A data extension repository to search.
/// </param>
/// <param name="dataCookerPath">
/// A path to a data cooker.
/// </param>
/// <param name="cooker">
/// Receives the result: a data cooker reference, or <c>null</c> if not found.
/// </param>
/// <returns>
/// true if the path resulted in a data cooker reference; false otherwise.
/// </returns>
public static bool TryGetDataCookerReference(
this IDataExtensionRepository self,
DataCookerPath dataCookerPath,
out IDataCookerReference cooker)
{
Guard.NotNull(self, nameof(self));
cooker = self.GetSourceDataCookerReference(dataCookerPath);
if (cooker is null)
{
cooker = self.GetCompositeDataCookerReference(dataCookerPath);
}
return(!(cooker is null));
}
/// <summary>
/// Enables a set of data cookers to participate in source processing.
/// </summary>
/// <param name="self">
/// The data extension repository.
/// </param>
/// <param name="processors">
/// The source processors that are available to generate data.
/// </param>
/// <param name="cookersToEnable">
/// The requested data cookers to enable.
/// </param>
/// <returns>
/// The set of custom data processors that have data cookers enabled from the requested set.
/// </returns>
public static ISet <ICustomDataProcessorWithSourceParser> EnableDataCookers(
this IDataExtensionRepository self,
IEnumerable <ICustomDataProcessorWithSourceParser> processors,
HashSet <DataCookerPath> cookersToEnable)
{
Guard.NotNull(self, nameof(self));
Guard.NotNull(processors, nameof(processors));
Guard.NotNull(cookersToEnable, nameof(cookersToEnable));
var customProcessors = new HashSet <ICustomDataProcessorWithSourceParser>();
void EnableCooker(
DataCookerPath cooker,
ICustomDataProcessorWithSourceParser processor)
{
if (self.TryGetDataCookerReference(cooker, out IDataCookerReference cookerReference))
{
if (cookerReference is ISourceDataCookerReference sdcr)
{
var parserId = cookerReference.Path.SourceParserId;
if (!StringComparer.OrdinalIgnoreCase.Equals(processor.SourceParserId, parserId))
{
return;
}
processor.EnableCooker(sdcr);
customProcessors.Add(processor);
}
foreach (var cookerToEnable in cookerReference.RequiredDataCookers)
{
EnableCooker(cookerToEnable, processor);
}
}
}
foreach (var processor in processors)
{
foreach (var cookerToEnable in cookersToEnable)
{
EnableCooker(cookerToEnable, processor);
}
}
return(customProcessors);
}
/// <summary>
/// This will process all of the data cooker requirements for the target data
/// extension (both source and composite). It will recursively descend through requirements,
/// determining if the target data extension may be used.
/// </summary>
/// <param name="availableDataExtensions">
/// Data extensions that have been exposed through the runtime.
/// </param>
private void ValidateDataCookerDependencies(
IDataExtensionRepository availableDataExtensions)
{
Guard.NotNull(availableDataExtensions, nameof(availableDataExtensions));
foreach (var requiredDataCookerPath in this.target.RequiredDataCookers)
{
IDataCookerReference dataCookerReference = null;
var sourceId = requiredDataCookerPath.SourceParserId;
if (string.IsNullOrWhiteSpace(sourceId))
{
// Add composite data cooker
var requiredCookerReference = availableDataExtensions.GetCompositeDataCookerReference(requiredDataCookerPath);
if (requiredCookerReference == null)
{
this.UpdateAvailability(DataExtensionAvailability.MissingRequirement);
this.missingDataCookers.Add(requiredDataCookerPath);
continue;
}
dataCookerReference = requiredCookerReference;
this.dependencyReferences.AddRequiredCompositeDataCookerPath(requiredCookerReference.Path);
}
else
{
// Add source data cooker
var requiredCookerReference = availableDataExtensions.GetSourceDataCookerReference(requiredDataCookerPath);
if (requiredCookerReference == null)
{
this.UpdateAvailability(DataExtensionAvailability.MissingRequirement);
this.missingDataCookers.Add(requiredDataCookerPath);
continue;
}
dataCookerReference = requiredCookerReference;
this.dependencyReferences.AddRequiredSourceDataCookerPath(requiredCookerReference.Path);
}
this.ProcessRequiredExtension(dataCookerReference, requiredDataCookerPath.CookerPath, availableDataExtensions);
}
}
/// <summary>
/// This will process all of the data processor requirements for the target data
/// extension. It will recursively descend through requirements, determining if
/// the target data extension may be used.
/// </summary>
/// <param name="availableDataExtensions">
/// Data extensions that have been exposed through the runtime.
/// </param>
private void ValidateRequiredDataProcessors(
IDataExtensionRepository availableDataExtensions)
{
Guard.NotNull(availableDataExtensions, nameof(availableDataExtensions));
foreach (var processorId in this.target.RequiredDataProcessors)
{
var processorReference = availableDataExtensions.GetDataProcessorReference(processorId);
if (processorReference == null)
{
this.UpdateAvailability(DataExtensionAvailability.MissingRequirement);
this.missingDataProcessors.Add(processorId);
continue;
}
this.ProcessRequiredExtension(processorReference, processorId.Id, availableDataExtensions);
}
}
/// <summary>
/// This is a wrapper around <see cref="ValidateDataCookerDependencies"/>. It performs some prechecks
/// before iterating over all of the dependencies. It also performs meta-checks, looking for errors
/// like circular dependencies.
/// </summary>
/// <param name="availableDataExtensions">
/// Data extensions that have been exposed through the runtime.
/// </param>
public void ProcessDependencies(IDataExtensionRepository availableDataExtensions)
{
Guard.NotNull(availableDataExtensions, nameof(availableDataExtensions));
if (this.establishAvailabilityStatus == EstablishAvailabilityStatus.InProgress)
{
this.AddError($"There is a circular dependency between data extensions: {this.target.Name}.");
this.UpdateAvailability(DataExtensionAvailability.Error);
return;
}
if (this.establishAvailabilityStatus == EstablishAvailabilityStatus.Complete)
{
return;
}
if (this.establishAvailabilityStatus == EstablishAvailabilityStatus.InProgress)
{
this.errors.Add($"There is a circular dependency between data extensions involving {this.target.Name}");
this.UpdateAvailability(DataExtensionAvailability.Error);
return;
}
this.establishAvailabilityStatus = EstablishAvailabilityStatus.InProgress;
this.UpdateAvailability(this.target.InitialAvailability);
if (this.Availability == DataExtensionAvailability.Error)
{
// this means there was an error in the target, so just stop now without any further
// validation.
//
this.establishAvailabilityStatus = EstablishAvailabilityStatus.Complete;
return;
}
this.ValidateDataCookerDependencies(availableDataExtensions);
this.ValidateRequiredDataProcessors(availableDataExtensions);
this.UpdateAvailability(DataExtensionAvailability.Available);
this.establishAvailabilityStatus = EstablishAvailabilityStatus.Complete;
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="dataExtensions">
/// Repository of data extensions.
/// </param>
public TableExtensionSelector(IDataExtensionRepository dataExtensions)
{
Guard.NotNull(dataExtensions, nameof(dataExtensions));
this.dataExtensions = dataExtensions;
var availableTables = new Dictionary <Guid, ITableExtensionReference>();
var availableCategories = new HashSet <string>(StringComparer.Ordinal);
foreach (var kvp in dataExtensions.TablesById)
{
if (kvp.Value.Availability == DataExtensionAvailability.Available)
{
availableTables.Add(kvp.Key, kvp.Value);
availableCategories.Add(kvp.Value.TableDescriptor.Category);
}
}
this.Tables = new ReadOnlyDictionary <Guid, ITableExtensionReference>(availableTables);
this.TableCategories = availableCategories;
}
/// <summary>
/// This enables all required source data cookers for each specified table. Doing so will ensure that the
/// table has the required data to build.
/// </summary>
/// <param name="self">
/// The data extension repository.
/// </param>
/// <param name="processors">
/// The source processors that are available to generate data.
/// </param>
/// <param name="enabledTables">
/// Table references that the caller is interested in building.
/// </param>
/// <returns>
/// The set of custom data processors that have data cookers enabled in order to build the requested set
/// of tables.
/// </returns>
public static ISet <ICustomDataProcessorWithSourceParser> EnableSourceDataCookersForTables(
this IDataExtensionRepository self,
IEnumerable <ICustomDataProcessorWithSourceParser> processors,
HashSet <ITableExtensionReference> enabledTables)
{
Guard.NotNull(self, nameof(self));
Guard.NotNull(processors, nameof(processors));
Guard.NotNull(enabledTables, nameof(enabledTables));
var customProcessors = new HashSet <ICustomDataProcessorWithSourceParser>();
var availableTableExtensions = new TableExtensionSelector(self);
var requiredReferencesBySourceId = availableTableExtensions.GetSourceDataCookersForTables(enabledTables);
foreach (var processor in processors)
{
if (requiredReferencesBySourceId.TryGetValue(
processor.SourceParserId,
out var requiredCookerReferences))
{
if (!requiredCookerReferences.Any())
{
continue;
}
customProcessors.Add(processor);
foreach (var cookerReference in requiredCookerReferences)
{
processor.EnableCooker(cookerReference);
}
}
}
return(customProcessors);
}
/// <summary>
/// Gets a set of table extensions references given a set of table extension identifiers and a set of
/// available source processors.
/// </summary>
/// <param name="self">
/// The data extension repository.
/// </param>
/// <param name="selectedTables">
/// Table identifiers that the caller is interested in,
/// or <c>null</c> if interested in all available tables.
/// </param>
/// <param name="processorsWithParsers">
/// The source processors that are available to generate data.
/// </param>
/// <returns>
/// A set of table extensions, limited to those whose ID was specified and where required source processors
/// are available.
/// </returns>
public static HashSet <ITableExtensionReference> GetEnabledTableExtensionReferences(
this IDataExtensionRepository self,
ISet <Guid> selectedTables,
IEnumerable <ICustomDataProcessorWithSourceParser> processorsWithParsers)
{
//
// selectedTables being null means everything is enabled
//
Guard.NotNull(self, nameof(self));
Guard.NotNull(processorsWithParsers, nameof(processorsWithParsers));
var sourceParserIds =
processorsWithParsers.Select(p => p.SourceParserId);
var uniqueSourceParserIds = new HashSet <string>(sourceParserIds, StringComparer.Ordinal);
bool SourceParsersAvailable(ITableExtensionReference table)
{
foreach (var requiredSource in table.DependencyReferences.RequiredSourceDataCookerPaths)
{
if (!uniqueSourceParserIds.Contains(requiredSource.SourceParserId))
{
return(false);
}
}
return(true);
}
var enabledTables = new HashSet <ITableExtensionReference>();
if (selectedTables != null)
{
foreach (var tableId in selectedTables)
{
if (self.TablesById.TryGetValue(tableId, out var tableReference))
{
Debug.Assert(tableReference.Availability == DataExtensionAvailability.Available);
if (SourceParsersAvailable(tableReference))
{
enabledTables.Add(tableReference);
}
}
}
}
else
{
foreach (var tableReference in self.TablesById.Values)
{
if (tableReference.Availability == DataExtensionAvailability.Available)
{
if (SourceParsersAvailable(tableReference))
{
enabledTables.Add(tableReference);
}
}
}
}
return(enabledTables);
}
