public override ExitCodeType Run(GracefulCancellationToken loadCancellationToken, object payload = null)
        {
            // single job, so grab the first available LoadProgress and lock it
            var loadProgresses = LoadProgressSelectionStrategy.GetAllLoadProgresses();

            if (!loadProgresses.Any())
            {
                return(ExitCodeType.OperationNotRequired);
            }

            var loadProgress = loadProgresses.First();

            // we don't need any other schedules the strategy may have given us
            loadProgresses.Remove(loadProgress);

            // Create the job factory
            if (_scheduledJobFactory != null)
            {
                throw new Exception("Job factory should only be created once");
            }

            _scheduledJobFactory = new SingleScheduledJobFactory(loadProgress, JobDateGenerationStrategyFactory.Create(loadProgress, DataLoadEventListener), OverrideNumberOfDaysToLoad ?? loadProgress.DefaultNumberOfDaysToLoadEachTime, LoadMetadata, LogManager);

            // If the job factory won't produce any jobs we can bail out here
            if (!_scheduledJobFactory.HasJobs())
            {
                return(ExitCodeType.OperationNotRequired);
            }

            // Run the data load
            JobProvider = _scheduledJobFactory;

            return(base.Run(loadCancellationToken, payload));
        }
예제 #2
0
 protected ScheduledDataLoadProcess(IRDMPPlatformRepositoryServiceLocator repositoryLocator, ILoadMetadata loadMetadata, ICheckable preExecutionChecker, IDataLoadExecution loadExecution, JobDateGenerationStrategyFactory jobDateGenerationStrategyFactory, ILoadProgressSelectionStrategy loadProgressSelectionStrategy, int?overrideNumberOfDaysToLoad, ILogManager logManager, IDataLoadEventListener dataLoadEventListener, HICDatabaseConfiguration configuration)
     : base(repositoryLocator, loadMetadata, preExecutionChecker, logManager, dataLoadEventListener, loadExecution, configuration)
 {
     JobDateGenerationStrategyFactory = jobDateGenerationStrategyFactory;
     LoadProgressSelectionStrategy    = loadProgressSelectionStrategy;
     OverrideNumberOfDaysToLoad       = overrideNumberOfDaysToLoad;
 }
        public void DateKnown_NoCache_SuggestSingleScheduleConsecutiveDateStrategy()
        {
            var lp = Mock.Of <ILoadProgress>(p => p.DataLoadProgress == new DateTime(2001, 01, 01));

            var factory = new JobDateGenerationStrategyFactory(new SingleLoadProgressSelectionStrategy(lp));

            Assert.AreEqual(typeof(SingleScheduleConsecutiveDateStrategy), factory.Create(lp, new ThrowImmediatelyDataLoadEventListener()).GetType());
        }
        public void NoDates()
        {
            var lp = Mock.Of <ILoadProgress>();

            var factory = new JobDateGenerationStrategyFactory(new SingleLoadProgressSelectionStrategy(lp));

            var ex = Assert.Throws <LoadOrCacheProgressUnclearException>(() => factory.Create(lp, new ThrowImmediatelyDataLoadEventListener()));

            Assert.AreEqual("Don't know when to start the data load, both DataLoadProgress and OriginDate are null", ex.Message);
        }
예제 #5
0
        public void LoadProgress_JobFactory_NoDates()
        {
            var lp = WhenIHaveA <LoadProgress>();



            lp.OriginDate = new DateTime(2001, 1, 1);

            // We are fully up-to-date
            lp.DataLoadProgress = DateTime.Now;

            lp.Check(new ThrowImmediatelyCheckNotifier());

            var stratFactory = new JobDateGenerationStrategyFactory(new AnyAvailableLoadProgressSelectionStrategy(lp.LoadMetadata));
            var strat        = stratFactory.Create(lp, new ThrowImmediatelyDataLoadEventListener());

            var dir = LoadDirectory.CreateDirectoryStructure(new DirectoryInfo(TestContext.CurrentContext.WorkDirectory), "LoadProgress_JobFactory_NoDates", true);

            var lmd = lp.LoadMetadata;

            lmd.LocationOfFlatFiles = dir.RootPath.FullName;

            foreach (var cata in lmd.GetAllCatalogues())
            {
                cata.LoggingDataTask = "ff";
                cata.SaveToDatabase();
            }


            lmd.SaveToDatabase();


            var jobFactory = new SingleScheduledJobFactory(lp, strat, 999, lp.LoadMetadata, null);
            var ex         = Assert.Throws <Exception>(() => jobFactory.Create(RepositoryLocator, new ThrowImmediatelyDataLoadEventListener(), null));

            Assert.AreEqual("DatesToRetrieve was empty for load 'MyLoad'.  Possibly the load is already up to date?", ex.Message);

            // We have 1 day to load (date is the last fully loaded date)
            lp.DataLoadProgress = DateTime.Now.AddDays(-2);
            lp.SaveToDatabase();

            strat      = stratFactory.Create(lp, new ThrowImmediatelyDataLoadEventListener());
            jobFactory = new SingleScheduledJobFactory(lp, strat, 999, lp.LoadMetadata, null);

            var job = jobFactory.Create(RepositoryLocator, new ThrowImmediatelyDataLoadEventListener(), null);

            Assert.AreEqual(1, ((ScheduledDataLoadJob)job).DatesToRetrieve.Count);
        }
예제 #6
0
        public void up()
        {
            RepositoryLocator.CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestDataWriter));
            RepositoryLocator.CatalogueRepository.MEF.AddTypeToCatalogForTesting(typeof(TestDataInventor));

            _lmd = new LoadMetadata(CatalogueRepository, "JobDateGenerationStrategyFactoryTestsIntegration");
            _lp  = new LoadProgress(CatalogueRepository, _lmd);

            _lp.DataLoadProgress = new DateTime(2001, 1, 1);
            _lp.SaveToDatabase();

            _cp = new CacheProgress(CatalogueRepository, _lp);


            _server  = new DiscoveredServer(new SqlConnectionStringBuilder("server=localhost;initial catalog=fish"));
            _factory = new JobDateGenerationStrategyFactory(new SingleLoadProgressSelectionStrategy(_lp));
        }
예제 #7
0
        public override ExitCodeType Run(GracefulCancellationToken loadCancellationToken, object payload = null)
        {
            // grab all the load schedules we can and lock them
            var loadProgresses = LoadProgressSelectionStrategy.GetAllLoadProgresses();

            if (!loadProgresses.Any())
            {
                return(ExitCodeType.OperationNotRequired);
            }

            // create job factory
            var progresses  = loadProgresses.ToDictionary(loadProgress => loadProgress, loadProgress => JobDateGenerationStrategyFactory.Create(loadProgress, DataLoadEventListener));
            var jobProvider = new MultipleScheduleJobFactory(progresses, OverrideNumberOfDaysToLoad, LoadMetadata, LogManager);

            // check if the factory will produce any jobs, if not we can stop here
            if (!jobProvider.HasJobs())
            {
                return(ExitCodeType.OperationNotRequired);
            }

            // Run the data load process
            JobProvider = jobProvider;

            //Do a data load
            ExitCodeType result;

            while ((result = base.Run(loadCancellationToken, payload)) == ExitCodeType.Success) //stop if it said not required
            {
                //or if between executions the token is set
                if (loadCancellationToken.IsAbortRequested)
                {
                    return(ExitCodeType.Abort);
                }

                if (loadCancellationToken.IsCancellationRequested)
                {
                    return(ExitCodeType.Success);
                }
            }

            //should be Operation Not Required or Error since the token inside handles stopping
            return(result);
        }
예제 #8
0
파일: DleRunner.cs 프로젝트: 24418863/rdm
        public int Run(IRDMPPlatformRepositoryServiceLocator locator, IDataLoadEventListener listener, ICheckNotifier checkNotifier, GracefulCancellationToken token)
        {
            ILoadProgress loadProgress = locator.CatalogueRepository.GetObjectByID <LoadProgress>(_options.LoadProgress);
            ILoadMetadata loadMetadata = locator.CatalogueRepository.GetObjectByID <LoadMetadata>(_options.LoadMetadata);

            if (loadMetadata == null && loadProgress != null)
            {
                loadMetadata = loadProgress.LoadMetadata;
            }

            if (loadMetadata == null)
            {
                throw new ArgumentException("No Load Metadata specified");
            }

            if (loadProgress != null && loadProgress.LoadMetadata_ID != loadMetadata.ID)
            {
                throw new ArgumentException("The supplied LoadProgress does not belong to the supplied LoadMetadata load");
            }

            var databaseConfiguration = new HICDatabaseConfiguration(loadMetadata);
            var flags = new HICLoadConfigurationFlags();

            flags.ArchiveData                = !_options.DoNotArchiveData;
            flags.DoLoadToStaging            = !_options.StopAfterRAW;
            flags.DoMigrateFromStagingToLive = !_options.StopAfterSTAGING;

            var checkable = new CheckEntireDataLoadProcess(loadMetadata, databaseConfiguration, flags, locator.CatalogueRepository.MEF);

            switch (_options.Command)
            {
            case CommandLineActivity.run:

                var loggingServer = loadMetadata.GetDistinctLoggingDatabase();
                var logManager    = new LogManager(loggingServer);

                // Create the pipeline to pass into the DataLoadProcess object
                var dataLoadFactory = new HICDataLoadFactory(loadMetadata, databaseConfiguration, flags, locator.CatalogueRepository, logManager);

                IDataLoadExecution execution = dataLoadFactory.Create(listener);
                IDataLoadProcess   dataLoadProcess;

                if (loadMetadata.LoadProgresses.Any())
                {
                    //Then the load is designed to run X days of source data at a time
                    //Load Progress
                    ILoadProgressSelectionStrategy whichLoadProgress = loadProgress != null ? (ILoadProgressSelectionStrategy) new SingleLoadProgressSelectionStrategy(loadProgress) : new AnyAvailableLoadProgressSelectionStrategy(loadMetadata);

                    var jobDateFactory = new JobDateGenerationStrategyFactory(whichLoadProgress);

                    dataLoadProcess = _options.Iterative
                            ? (IDataLoadProcess) new IterativeScheduledDataLoadProcess(locator, loadMetadata, checkable, execution, jobDateFactory, whichLoadProgress, _options.DaysToLoad, logManager, listener, databaseConfiguration) :
                                      new SingleJobScheduledDataLoadProcess(locator, loadMetadata, checkable, execution, jobDateFactory, whichLoadProgress, _options.DaysToLoad, logManager, listener, databaseConfiguration);
                }
                else
                {
                    //OnDemand
                    dataLoadProcess = new DataLoadProcess(locator, loadMetadata, checkable, logManager, listener, execution, databaseConfiguration);
                }

                var exitCode = dataLoadProcess.Run(token);

                //return 0 for success or load not required otherwise return the exit code (which will be non zero so error)
                return(exitCode == ExitCodeType.Success || exitCode == ExitCodeType.OperationNotRequired? 0: (int)exitCode);

            case CommandLineActivity.check:

                checkable.Check(checkNotifier);

                return(0);

            default:
                throw new ArgumentOutOfRangeException();
            }
        }