public RuntimeProcessorEnvironment( IDataExtensionRepository repository) { Debug.Assert(repository != null); this.repository = repository; }
private void ProcessRequiredExtension( IDataExtensionReference reference, string extensionId, IDataExtensionRepository availableDataExtensions) { Guard.NotNull(reference, nameof(reference)); Guard.NotNull(availableDataExtensions, nameof(availableDataExtensions)); Debug.Assert(!string.IsNullOrWhiteSpace(extensionId)); if (reference.Availability == DataExtensionAvailability.Undetermined) { reference.ProcessDependencies(availableDataExtensions); } if (reference.Availability != DataExtensionAvailability.Error) { this.dependencyReferences.AddRequiredExtensionReferences(reference.DependencyReferences); } Debug.Assert( reference.Availability != DataExtensionAvailability.Undetermined, $"{nameof(this.ProcessDependencies)} returned without establishing Availability."); this.UpdateAvailabilityFromRequiredExtension(reference, extensionId); this.target.PerformAdditionalDataExtensionValidation(this, reference); }
/// <summary> /// This constructor is associated with a given <see cref="ICustomDataProcessorWithSourceParser"/>, provided /// as a parameter. <paramref name="dataExtensionRepository"/> is required to retrieve all available data /// extensions. /// </summary> /// <param name="dataProcessor"> /// The data processor with which this object is associated. /// </param> /// <param name="dataExtensionRepository"> /// Provides access to a set of data extensions. /// </param> public CustomDataProcessorExtensibilitySupport( ICustomDataProcessorWithSourceParser dataProcessor, IDataExtensionRepository dataExtensionRepository) { this.dataProcessor = dataProcessor; this.dataExtensionRepository = dataExtensionRepository; this.tableReferences = new Dictionary <TableDescriptor, ITableExtensionReference>(); }
/// <summary> /// Creates an instance. /// </summary> /// <param name="sourceCookerData"> /// Provides access to source data cooker output. /// </param> /// <param name="dataExtensionRepository"> /// Provides a way to generate data extensions other than source data cookers. /// </param> public DataExtensionRetrievalFactory( ICookedDataRetrieval sourceCookerData, IDataExtensionRepository dataExtensionRepository) { Guard.NotNull(sourceCookerData, nameof(sourceCookerData)); Guard.NotNull(dataExtensionRepository, nameof(dataExtensionRepository)); this.CookedSourceData = sourceCookerData; this.DataExtensionRepository = dataExtensionRepository; }
/// <inheritdoc /> public void ProcessDependencies(IDataExtensionRepository availableDataExtensions) { Guard.NotNull(availableDataExtensions, nameof(availableDataExtensions)); if (this.extensionDependencyState == null) { this.extensionDependencyState = new DataExtensionDependencyState(this); } this.extensionDependencyState.ProcessDependencies(availableDataExtensions); }
public void ProcessDependencies( IDataExtensionRepository availableDataExtensions) { if (this.processDependencies != null) { this.processDependencies.Invoke(availableDataExtensions); } else if (this.DependencyState != null) { this.DependencyState.ProcessDependencies(availableDataExtensions); } }
/// <summary> /// Initializes a new instance of the <see cref="RuntimeExecutionResults"/> /// class. /// </summary> /// <param name="cookedDataRetrieval"> /// The retrieval interface for getting to cooked data. /// </param> /// <param name="retrievalFactory"> /// The factory for creating retrievals for composite cookers. /// </param> /// <param name="repository"> /// The repository that was used to process the data. /// </param> /// <exception cref="ArgumentNullException"> /// <paramref name="cookedDataRetrieval"/> is <c>null</c>. /// - or - /// <paramref name="retrievalFactory"/> is <c>null</c>. /// - or - /// <paramref name="repository"/> is <c>null</c>. /// </exception> public RuntimeExecutionResults( ICookedDataRetrieval cookedDataRetrieval, IDataExtensionRetrievalFactory retrievalFactory, IDataExtensionRepository repository) { Guard.NotNull(cookedDataRetrieval, nameof(cookedDataRetrieval)); Guard.NotNull(retrievalFactory, nameof(retrievalFactory)); Guard.NotNull(repository, nameof(repository)); this.cookedDataRetrieval = cookedDataRetrieval; this.retrievalFactory = retrievalFactory; this.repository = repository; this.sourceCookers = new HashSet <DataCookerPath>(this.repository.SourceDataCookers); }
/// <summary> /// Attempts to get a data cooker reference given a data cooker path. /// </summary> /// <param name="self"> /// A data extension repository to search. /// </param> /// <param name="dataCookerPath"> /// A path to a data cooker. /// </param> /// <param name="cooker"> /// Receives the result: a data cooker reference, or <c>null</c> if not found. /// </param> /// <returns> /// true if the path resulted in a data cooker reference; false otherwise. /// </returns> public static bool TryGetDataCookerReference( this IDataExtensionRepository self, DataCookerPath dataCookerPath, out IDataCookerReference cooker) { Guard.NotNull(self, nameof(self)); cooker = self.GetSourceDataCookerReference(dataCookerPath); if (cooker is null) { cooker = self.GetCompositeDataCookerReference(dataCookerPath); } return(!(cooker is null)); }
/// <summary> /// Enables a set of data cookers to participate in source processing. /// </summary> /// <param name="self"> /// The data extension repository. /// </param> /// <param name="processors"> /// The source processors that are available to generate data. /// </param> /// <param name="cookersToEnable"> /// The requested data cookers to enable. /// </param> /// <returns> /// The set of custom data processors that have data cookers enabled from the requested set. /// </returns> public static ISet <ICustomDataProcessorWithSourceParser> EnableDataCookers( this IDataExtensionRepository self, IEnumerable <ICustomDataProcessorWithSourceParser> processors, HashSet <DataCookerPath> cookersToEnable) { Guard.NotNull(self, nameof(self)); Guard.NotNull(processors, nameof(processors)); Guard.NotNull(cookersToEnable, nameof(cookersToEnable)); var customProcessors = new HashSet <ICustomDataProcessorWithSourceParser>(); void EnableCooker( DataCookerPath cooker, ICustomDataProcessorWithSourceParser processor) { if (self.TryGetDataCookerReference(cooker, out IDataCookerReference cookerReference)) { if (cookerReference is ISourceDataCookerReference sdcr) { var parserId = cookerReference.Path.SourceParserId; if (!StringComparer.OrdinalIgnoreCase.Equals(processor.SourceParserId, parserId)) { return; } processor.EnableCooker(sdcr); customProcessors.Add(processor); } foreach (var cookerToEnable in cookerReference.RequiredDataCookers) { EnableCooker(cookerToEnable, processor); } } } foreach (var processor in processors) { foreach (var cookerToEnable in cookersToEnable) { EnableCooker(cookerToEnable, processor); } } return(customProcessors); }
/// <summary> /// This will process all of the data cooker requirements for the target data /// extension (both source and composite). It will recursively descend through requirements, /// determining if the target data extension may be used. /// </summary> /// <param name="availableDataExtensions"> /// Data extensions that have been exposed through the runtime. /// </param> private void ValidateDataCookerDependencies( IDataExtensionRepository availableDataExtensions) { Guard.NotNull(availableDataExtensions, nameof(availableDataExtensions)); foreach (var requiredDataCookerPath in this.target.RequiredDataCookers) { IDataCookerReference dataCookerReference = null; var sourceId = requiredDataCookerPath.SourceParserId; if (string.IsNullOrWhiteSpace(sourceId)) { // Add composite data cooker var requiredCookerReference = availableDataExtensions.GetCompositeDataCookerReference(requiredDataCookerPath); if (requiredCookerReference == null) { this.UpdateAvailability(DataExtensionAvailability.MissingRequirement); this.missingDataCookers.Add(requiredDataCookerPath); continue; } dataCookerReference = requiredCookerReference; this.dependencyReferences.AddRequiredCompositeDataCookerPath(requiredCookerReference.Path); } else { // Add source data cooker var requiredCookerReference = availableDataExtensions.GetSourceDataCookerReference(requiredDataCookerPath); if (requiredCookerReference == null) { this.UpdateAvailability(DataExtensionAvailability.MissingRequirement); this.missingDataCookers.Add(requiredDataCookerPath); continue; } dataCookerReference = requiredCookerReference; this.dependencyReferences.AddRequiredSourceDataCookerPath(requiredCookerReference.Path); } this.ProcessRequiredExtension(dataCookerReference, requiredDataCookerPath.CookerPath, availableDataExtensions); } }
/// <summary> /// This will process all of the data processor requirements for the target data /// extension. It will recursively descend through requirements, determining if /// the target data extension may be used. /// </summary> /// <param name="availableDataExtensions"> /// Data extensions that have been exposed through the runtime. /// </param> private void ValidateRequiredDataProcessors( IDataExtensionRepository availableDataExtensions) { Guard.NotNull(availableDataExtensions, nameof(availableDataExtensions)); foreach (var processorId in this.target.RequiredDataProcessors) { var processorReference = availableDataExtensions.GetDataProcessorReference(processorId); if (processorReference == null) { this.UpdateAvailability(DataExtensionAvailability.MissingRequirement); this.missingDataProcessors.Add(processorId); continue; } this.ProcessRequiredExtension(processorReference, processorId.Id, availableDataExtensions); } }
/// <summary> /// This is a wrapper around <see cref="ValidateDataCookerDependencies"/>. It performs some prechecks /// before iterating over all of the dependencies. It also performs meta-checks, looking for errors /// like circular dependencies. /// </summary> /// <param name="availableDataExtensions"> /// Data extensions that have been exposed through the runtime. /// </param> public void ProcessDependencies(IDataExtensionRepository availableDataExtensions) { Guard.NotNull(availableDataExtensions, nameof(availableDataExtensions)); if (this.establishAvailabilityStatus == EstablishAvailabilityStatus.InProgress) { this.AddError($"There is a circular dependency between data extensions: {this.target.Name}."); this.UpdateAvailability(DataExtensionAvailability.Error); return; } if (this.establishAvailabilityStatus == EstablishAvailabilityStatus.Complete) { return; } if (this.establishAvailabilityStatus == EstablishAvailabilityStatus.InProgress) { this.errors.Add($"There is a circular dependency between data extensions involving {this.target.Name}"); this.UpdateAvailability(DataExtensionAvailability.Error); return; } this.establishAvailabilityStatus = EstablishAvailabilityStatus.InProgress; this.UpdateAvailability(this.target.InitialAvailability); if (this.Availability == DataExtensionAvailability.Error) { // this means there was an error in the target, so just stop now without any further // validation. // this.establishAvailabilityStatus = EstablishAvailabilityStatus.Complete; return; } this.ValidateDataCookerDependencies(availableDataExtensions); this.ValidateRequiredDataProcessors(availableDataExtensions); this.UpdateAvailability(DataExtensionAvailability.Available); this.establishAvailabilityStatus = EstablishAvailabilityStatus.Complete; }
/// <summary> /// Constructor /// </summary> /// <param name="dataExtensions"> /// Repository of data extensions. /// </param> public TableExtensionSelector(IDataExtensionRepository dataExtensions) { Guard.NotNull(dataExtensions, nameof(dataExtensions)); this.dataExtensions = dataExtensions; var availableTables = new Dictionary <Guid, ITableExtensionReference>(); var availableCategories = new HashSet <string>(StringComparer.Ordinal); foreach (var kvp in dataExtensions.TablesById) { if (kvp.Value.Availability == DataExtensionAvailability.Available) { availableTables.Add(kvp.Key, kvp.Value); availableCategories.Add(kvp.Value.TableDescriptor.Category); } } this.Tables = new ReadOnlyDictionary <Guid, ITableExtensionReference>(availableTables); this.TableCategories = availableCategories; }
/// <summary> /// This enables all required source data cookers for each specified table. Doing so will ensure that the /// table has the required data to build. /// </summary> /// <param name="self"> /// The data extension repository. /// </param> /// <param name="processors"> /// The source processors that are available to generate data. /// </param> /// <param name="enabledTables"> /// Table references that the caller is interested in building. /// </param> /// <returns> /// The set of custom data processors that have data cookers enabled in order to build the requested set /// of tables. /// </returns> public static ISet <ICustomDataProcessorWithSourceParser> EnableSourceDataCookersForTables( this IDataExtensionRepository self, IEnumerable <ICustomDataProcessorWithSourceParser> processors, HashSet <ITableExtensionReference> enabledTables) { Guard.NotNull(self, nameof(self)); Guard.NotNull(processors, nameof(processors)); Guard.NotNull(enabledTables, nameof(enabledTables)); var customProcessors = new HashSet <ICustomDataProcessorWithSourceParser>(); var availableTableExtensions = new TableExtensionSelector(self); var requiredReferencesBySourceId = availableTableExtensions.GetSourceDataCookersForTables(enabledTables); foreach (var processor in processors) { if (requiredReferencesBySourceId.TryGetValue( processor.SourceParserId, out var requiredCookerReferences)) { if (!requiredCookerReferences.Any()) { continue; } customProcessors.Add(processor); foreach (var cookerReference in requiredCookerReferences) { processor.EnableCooker(cookerReference); } } } return(customProcessors); }
/// <summary> /// Gets a set of table extensions references given a set of table extension identifiers and a set of /// available source processors. /// </summary> /// <param name="self"> /// The data extension repository. /// </param> /// <param name="selectedTables"> /// Table identifiers that the caller is interested in, /// or <c>null</c> if interested in all available tables. /// </param> /// <param name="processorsWithParsers"> /// The source processors that are available to generate data. /// </param> /// <returns> /// A set of table extensions, limited to those whose ID was specified and where required source processors /// are available. /// </returns> public static HashSet <ITableExtensionReference> GetEnabledTableExtensionReferences( this IDataExtensionRepository self, ISet <Guid> selectedTables, IEnumerable <ICustomDataProcessorWithSourceParser> processorsWithParsers) { // // selectedTables being null means everything is enabled // Guard.NotNull(self, nameof(self)); Guard.NotNull(processorsWithParsers, nameof(processorsWithParsers)); var sourceParserIds = processorsWithParsers.Select(p => p.SourceParserId); var uniqueSourceParserIds = new HashSet <string>(sourceParserIds, StringComparer.Ordinal); bool SourceParsersAvailable(ITableExtensionReference table) { foreach (var requiredSource in table.DependencyReferences.RequiredSourceDataCookerPaths) { if (!uniqueSourceParserIds.Contains(requiredSource.SourceParserId)) { return(false); } } return(true); } var enabledTables = new HashSet <ITableExtensionReference>(); if (selectedTables != null) { foreach (var tableId in selectedTables) { if (self.TablesById.TryGetValue(tableId, out var tableReference)) { Debug.Assert(tableReference.Availability == DataExtensionAvailability.Available); if (SourceParsersAvailable(tableReference)) { enabledTables.Add(tableReference); } } } } else { foreach (var tableReference in self.TablesById.Values) { if (tableReference.Availability == DataExtensionAvailability.Available) { if (SourceParsersAvailable(tableReference)) { enabledTables.Add(tableReference); } } } } return(enabledTables); }