private void AssertThatThereIsACacheDataProvider(ILoadMetadata metadata, IEnumerable<IProcessTask> processTasks)
{
const string whatWeExpected = @"(we expected one that was a MEF class implementing ICachedDataProvider since you are trying to execute a cache based data load)";
List<ProcessTask> incompatibleProviders = new List<ProcessTask>();
List<ProcessTask> compatibleProviders = new List<ProcessTask>();
foreach (ProcessTask task in processTasks)
{
//it's not a DataProvider
if (!task.ProcessTaskType.Equals(ProcessTaskType.DataProvider))
continue;
var type = task.CatalogueRepository.MEF.GetType(task.Path);
if (typeof(ICachedDataProvider).IsAssignableFrom(type))
compatibleProviders.Add(task);
else
incompatibleProviders.Add(task);
}
if (!incompatibleProviders.Any() && !compatibleProviders.Any())
throw new CacheDataProviderFindingException("LoadMetadata " + metadata + " does not have ANY process tasks of type ProcessTaskType.DataProvider " + whatWeExpected);
if (!compatibleProviders.Any())
throw new CacheDataProviderFindingException("LoadMetadata " + metadata + " has some DataProviders tasks but none of them wrap classes that implement ICachedDataProvider " + whatWeExpected + " FYI the data providers in your load wrap the following classes:" + string.Join(",", incompatibleProviders.Select(t => t.Path)));
if (compatibleProviders.Count > 1)
throw new CacheDataProviderFindingException("LoadMetadata " + metadata + " has multiple cache DataProviders tasks (" + string.Join(",", compatibleProviders.Select(p => p.ToString())) + "), you are only allowed 1");
}
protected ScheduledJobFactory(int?overrideNumberOfDaysToLoad, ILoadMetadata loadMetadata, ILogManager logManager)
{
OverrideNumberOfDaysToLoad = overrideNumberOfDaysToLoad;
JobDescription = loadMetadata.Name;
LoadMetadata = loadMetadata;
LogManager = logManager;
}
public CheckEntireDataLoadProcess(ILoadMetadata loadMetadata, HICDatabaseConfiguration databaseConfiguration, HICLoadConfigurationFlags loadConfigurationFlags, MEF mef)
{
_databaseConfiguration = databaseConfiguration;
_loadConfigurationFlags = loadConfigurationFlags;
_mef = mef;
LoadMetadata = loadMetadata;
}
public ChooseLoadDirectoryUI(ILoadMetadata loadMetadata)
{
InitializeComponent();
var help = loadMetadata.CatalogueRepository.CommentStore.GetDocumentationIfExists("ILoadMetadata.LocationOfFlatFiles", false, true);
helpIcon1.SetHelpText("Location Of Flat Files", help);
}
/// <summary>
/// Preferred Constructor, creates RAW, STAGING, LIVE connection strings based on the data access points in the LoadMetadata, also respects the ServerDefaults for RAW override (if any)
/// </summary>
/// <param name="lmd"></param>
/// <param name="namer"></param>
public HICDatabaseConfiguration(ILoadMetadata lmd, INameDatabasesAndTablesDuringLoads namer = null) :
this(lmd.GetDistinctLiveDatabaseServer(), namer, lmd.CatalogueRepository.GetServerDefaults(), lmd.OverrideRAWServer)
{
var globalIgnorePattern = GetGlobalIgnorePatternIfAny(lmd.CatalogueRepository);
if (globalIgnorePattern != null && !string.IsNullOrWhiteSpace(globalIgnorePattern.Regex))
{
IgnoreColumns = new Regex(globalIgnorePattern.Regex);
}
}
public ICacheLayout CreateCacheLayout(ILoadProgress loadProgress, ILoadMetadata metadata)
{
AssertThatThereIsACacheDataProvider(metadata, metadata.ProcessTasks.Where(p=>!p.IsDisabled));
var cp = loadProgress.CacheProgress;
var factory = new CachingPipelineUseCase(cp);
var destination = factory.CreateDestinationOnly(new ThrowImmediatelyDataLoadEventListener());
return destination.CreateCacheLayout();
}
public DataLoadProcess(IRDMPPlatformRepositoryServiceLocator repositoryLocator, ILoadMetadata loadMetadata, ICheckable preExecutionChecker, ILogManager logManager, IDataLoadEventListener dataLoadEventListener, IDataLoadExecution loadExecution, HICDatabaseConfiguration configuration)
{
_repositoryLocator = repositoryLocator;
LoadMetadata = loadMetadata;
DataLoadEventListener = dataLoadEventListener;
_configuration = configuration;
LoadExecution = loadExecution;
_preExecutionChecker = preExecutionChecker;
LogManager = logManager;
ExitCode = ExitCodeType.Success;
JobProvider = new JobFactory(loadMetadata, logManager);
}
/// <summary>
/// Creates a new operation in the data load (e.g. copy files from A to B, load all CSV files to RAW table B etc)
/// </summary>
/// <param name="repository"></param>
/// <param name="parent"></param>
/// <param name="stage"></param>
public ProcessTask(ICatalogueRepository repository, ILoadMetadata parent, LoadStage stage)
{
var order = repository.GetAllObjectsWithParent <ProcessTask>(parent).Select(t => t.Order).DefaultIfEmpty().Max() + 1;
repository.InsertAndHydrate(this, new Dictionary <string, object>
{
{ "LoadMetadata_ID", parent.ID },
{ "ProcessTaskType", ProcessTaskType.Executable.ToString() },
{ "LoadStage", stage },
{ "Name", "New Process" + Guid.NewGuid() },
{ "Order", order }
});
}
public LoadProgressSummaryReport(LoadProgress loadProgress)
{
_loadProgress = loadProgress;
_loadMetadata = _loadProgress.LoadMetadata;
_cacheProgress = _loadProgress.CacheProgress;
try
{
dqeRepository = new DQERepository(loadProgress.CatalogueRepository);
}
catch (NotSupportedException)
{
dqeRepository = null;
}
}
public HICDataLoadFactory(ILoadMetadata loadMetadata, HICDatabaseConfiguration databaseConfiguration, HICLoadConfigurationFlags loadConfigurationFlags, ICatalogueRepository repository, ILogManager logManager)
{
_databaseConfiguration = databaseConfiguration;
_loadConfigurationFlags = loadConfigurationFlags;
_repository = repository;
_logManager = logManager;
LoadMetadata = loadMetadata;
// If we are not supplied any catalogues to load, it is expected that we will load all catalogues associated with the provided ILoadMetadata
_cataloguesToLoad = LoadMetadata.GetAllCatalogues().ToList();
if (!_cataloguesToLoad.Any())
{
throw new InvalidOperationException("LoadMetadata " + LoadMetadata.ID + " is not related to any Catalogues, there is nothing to load");
}
}
public LoadArgsDictionary(ILoadMetadata loadMetadata, StandardDatabaseHelper dbDeployInfo)
{
if (string.IsNullOrWhiteSpace(loadMetadata.LocationOfFlatFiles))
{
throw new Exception(@"No Project Directory (LocationOfFlatFiles) has been configured on LoadMetadata " + loadMetadata.Name);
}
_dbDeployInfo = dbDeployInfo;
_loadMetadata = loadMetadata;
LoadArgs = new Dictionary <LoadStage, IStageArgs>();
foreach (LoadStage loadStage in Enum.GetValues(typeof(LoadStage)))
{
LoadArgs.Add(loadStage, CreateLoadArgs(loadStage));
}
}
public ChooseLoadDirectoryUI(IActivateItems activator, ILoadMetadata loadMetadata)
{
InitializeComponent();
SetItemActivator(activator);
var help = loadMetadata.CatalogueRepository.CommentStore.GetDocumentationIfExists("ILoadMetadata.LocationOfFlatFiles", false, true);
helpIcon1.SetHelpText("Location Of Flat Files", help);
if (!string.IsNullOrWhiteSpace(loadMetadata.LocationOfFlatFiles))
{
tbUseExisting.Text = loadMetadata.LocationOfFlatFiles;
CheckExistingProjectDirectory();
}
}
public DataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description, ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener, HICDatabaseConfiguration configuration)
{
_logManager = logManager;
RepositoryLocator = repositoryLocator;
LoadMetadata = loadMetadata;
LoadDirectory = directory;
Configuration = configuration;
_listener = listener;
Description = description;
List <ICatalogue> catalogues = LoadMetadata.GetAllCatalogues().ToList();
if (LoadMetadata != null)
{
_loggingTask = GetLoggingTask(catalogues);
}
RegularTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetTableInfoList(false)).Distinct().ToList();
LookupTablesToLoad = catalogues.SelectMany(catalogue => catalogue.GetLookupTableInfoList()).Distinct().ToList();
}
public ScheduledDataLoadJob(IRDMPPlatformRepositoryServiceLocator repositoryLocator, string description, ILogManager logManager, ILoadMetadata loadMetadata, ILoadDirectory directory, IDataLoadEventListener listener, HICDatabaseConfiguration configuration)
: base(repositoryLocator, description, logManager, loadMetadata, directory, listener, configuration)
{
}
public PreExecutionChecker(ILoadMetadata loadMetadata, HICDatabaseConfiguration overrideDatabaseConfiguration)
{
_loadMetadata = loadMetadata;
_databaseConfiguration = overrideDatabaseConfiguration ?? new HICDatabaseConfiguration(loadMetadata);
_cataloguesToLoad = _loadMetadata.GetAllCatalogues().ToList();
}
public CatalogueLoadChecks(ILoadMetadata loadMetadata, HICLoadConfigurationFlags loadConfigurationFlags, HICDatabaseConfiguration databaseConfiguration)
{
_loadMetadata = loadMetadata;
_loadConfigurationFlags = loadConfigurationFlags;
_databaseConfiguration = databaseConfiguration;
}
public ProcessTaskChecks(ILoadMetadata loadMetadata)
{
_loadMetadata = loadMetadata;
}
public AnyAvailableLoadProgressSelectionStrategy(ILoadMetadata loadMetadata)
{
_loadMetadata = loadMetadata;
}
public SingleScheduledJobFactory(ILoadProgress loadProgress, IJobDateGenerationStrategy jobDateGenerationStrategy, int overrideNumberOfDaysToLoad, ILoadMetadata loadMetadata, ILogManager logManager) : base(overrideNumberOfDaysToLoad, loadMetadata, logManager)
{
_loadProgress = loadProgress;
_jobDateGenerationStrategy = jobDateGenerationStrategy;
}
public JobFactory(ILoadMetadata loadMetadata, ILogManager logManager)
{
_loadMetadata = loadMetadata;
_logManager = logManager;
}
public int Run(IRDMPPlatformRepositoryServiceLocator locator, IDataLoadEventListener listener, ICheckNotifier checkNotifier, GracefulCancellationToken token)
{
ILoadProgress loadProgress = locator.CatalogueRepository.GetObjectByID <LoadProgress>(_options.LoadProgress);
ILoadMetadata loadMetadata = locator.CatalogueRepository.GetObjectByID <LoadMetadata>(_options.LoadMetadata);
if (loadMetadata == null && loadProgress != null)
{
loadMetadata = loadProgress.LoadMetadata;
}
if (loadMetadata == null)
{
throw new ArgumentException("No Load Metadata specified");
}
if (loadProgress != null && loadProgress.LoadMetadata_ID != loadMetadata.ID)
{
throw new ArgumentException("The supplied LoadProgress does not belong to the supplied LoadMetadata load");
}
var databaseConfiguration = new HICDatabaseConfiguration(loadMetadata);
var flags = new HICLoadConfigurationFlags();
flags.ArchiveData = !_options.DoNotArchiveData;
flags.DoLoadToStaging = !_options.StopAfterRAW;
flags.DoMigrateFromStagingToLive = !_options.StopAfterSTAGING;
var checkable = new CheckEntireDataLoadProcess(loadMetadata, databaseConfiguration, flags, locator.CatalogueRepository.MEF);
switch (_options.Command)
{
case CommandLineActivity.run:
var loggingServer = loadMetadata.GetDistinctLoggingDatabase();
var logManager = new LogManager(loggingServer);
// Create the pipeline to pass into the DataLoadProcess object
var dataLoadFactory = new HICDataLoadFactory(loadMetadata, databaseConfiguration, flags, locator.CatalogueRepository, logManager);
IDataLoadExecution execution = dataLoadFactory.Create(listener);
IDataLoadProcess dataLoadProcess;
if (loadMetadata.LoadProgresses.Any())
{
//Then the load is designed to run X days of source data at a time
//Load Progress
ILoadProgressSelectionStrategy whichLoadProgress = loadProgress != null ? (ILoadProgressSelectionStrategy) new SingleLoadProgressSelectionStrategy(loadProgress) : new AnyAvailableLoadProgressSelectionStrategy(loadMetadata);
var jobDateFactory = new JobDateGenerationStrategyFactory(whichLoadProgress);
dataLoadProcess = _options.Iterative
? (IDataLoadProcess) new IterativeScheduledDataLoadProcess(locator, loadMetadata, checkable, execution, jobDateFactory, whichLoadProgress, _options.DaysToLoad, logManager, listener, databaseConfiguration) :
new SingleJobScheduledDataLoadProcess(locator, loadMetadata, checkable, execution, jobDateFactory, whichLoadProgress, _options.DaysToLoad, logManager, listener, databaseConfiguration);
}
else
{
//OnDemand
dataLoadProcess = new DataLoadProcess(locator, loadMetadata, checkable, logManager, listener, execution, databaseConfiguration);
}
var exitCode = dataLoadProcess.Run(token);
//return 0 for success or load not required otherwise return the exit code (which will be non zero so error)
return(exitCode == ExitCodeType.Success || exitCode == ExitCodeType.OperationNotRequired? 0: (int)exitCode);
case CommandLineActivity.check:
checkable.Check(checkNotifier);
return(0);
default:
throw new ArgumentOutOfRangeException();
}
}
public MultipleScheduleJobFactory(Dictionary <ILoadProgress, IJobDateGenerationStrategy> availableSchedules, int?overrideNumberOfDaysToLoad, ILoadMetadata loadMetadata, ILogManager logManager)
: base(overrideNumberOfDaysToLoad, loadMetadata, logManager)
{
_availableSchedules = availableSchedules;
_scheduleList = _availableSchedules.Keys.ToList();
_lastScheduleId = 0;
}
// todo: refactor to cut down on ctor params
public IterativeScheduledDataLoadProcess(IRDMPPlatformRepositoryServiceLocator repositoryLocator, ILoadMetadata loadMetadata, ICheckable preExecutionChecker, IDataLoadExecution loadExecution, JobDateGenerationStrategyFactory jobDateGenerationStrategyFactory, ILoadProgressSelectionStrategy loadProgressSelectionStrategy, int?overrideNumberOfDaysToLoad, ILogManager logManager, IDataLoadEventListener dataLoadEventsreceiver, HICDatabaseConfiguration configuration)
: base(repositoryLocator, loadMetadata, preExecutionChecker, loadExecution, jobDateGenerationStrategyFactory, loadProgressSelectionStrategy, overrideNumberOfDaysToLoad, logManager, dataLoadEventsreceiver, configuration)
{
}
public MetadataLoggingConfigurationChecks(ILoadMetadata loadMetadata)
{
_loadMetadata = loadMetadata;
}
/// <summary>
/// Preferred Constructor, creates RAW, STAGING, LIVE connection strings based on the data access points in the LoadMetadata, also respects the ServerDefaults for RAW override (if any)
/// </summary>
/// <param name="lmd"></param>
/// <param name="namer"></param>
public HICDatabaseConfiguration(ILoadMetadata lmd, INameDatabasesAndTablesDuringLoads namer = null) :
this(lmd.GetDistinctLiveDatabaseServer(), namer, lmd.CatalogueRepository.GetServerDefaults(), lmd.OverrideRAWServer)
{
}
