private void RecordSuccessfulFeedRun(IFeedRun feedRun, IExecutionLogLogger executionLogLogger) { feedRun.DateCompleted = _effectiveExecutionEndTime; feedRun.LastExecutionStartDate = executionLogLogger.GetExecutionStartTime(); feedRun.LastExecutionCompletedDate = executionLogLogger.GetExecutionEndTime(); feedRun.ExecutionLog = executionLogLogger.GetExecutionLog(); FeedRunService.EndFeedRun(feedRun, true); }
private void RecordSuccessfulFeedRun(IFeedRun feedRun) { feedRun.DateCompleted = _effectiveExecutionEndTime; feedRun.LastExecutionStartDate = _reportInformation.ExecutionStartTime; feedRun.LastExecutionCompletedDate = _reportInformation.ExecutionEndTime.Value; feedRun.ExecutionLog = _executionInformation.GetExecutionLog(); FeedRunService.EndFeedRun(feedRun, true); }
public void Build(string[] args) { try { //check if another instance is running - if so exit ExecutionLogLogger.SetExecutionStartTime(DateTime.Now); var isProcessRunning = Process.GetProcessesByName(Path.GetFileNameWithoutExtension(Assembly.GetEntryAssembly().Location)).Length > 1; if (isProcessRunning) { Log.Info("Another instance was running, so exiting the application without any processing."); return; } // First check if the "done" file is there. If so, skip job execution and move to file-related steps if (HasDoneFile()) { Log.Info("Done file exists at the drop folder. Skipping job execution and moving to file operations."); ExecuteFileOperations(); return; } Log.Info("Execution started!"); var feed = FeedService.GetFeed(FeedId); var allowPrimary = !feed.IsPaused; if (!allowPrimary) { Log.InfoFormat("{0} feeds are paused. Exiting application without any processing.", feed.Name); return; } // First clean all the files inside the output folder (in case the previous run failed and left out old files behind.) RemoveOldFiles(); _currentRun = FeedRunService.GetOrCreateActiveFeedRun(feed.FeedId, true, false); _effectiveExecutionEndTime = DateTime.Now; Runner.Initialize(ExecutionLogLogger, _currentRun.FeedId); ExecutionLogLogger = Runner.Execute(); if (ExecutionLogLogger.HasError) { Log.Error("Execution failed!!! Exiting application without moving any files."); HandleExit(); return; } HandleSuccess(); var elapsedTime = DateTime.Now - ExecutionLogLogger.GetExecutionStartTime(); Log.InfoFormat("Execution completed in {0}", elapsedTime.ToString(@"dd\.hh\:mm\:ss")); } catch (Exception e) { Log.Error("Execution failed!!!", e); HandleExit(); } }
private DateTime GetFromTime(IFeedRun inventoryFeedRun) { DateTime effectiveStartTime = inventoryFeedRun.DateStarted; // If in an incremental run and a test from date value was specified, use that value if (inventoryFeedRun.FeedRunType == FeedRunType.Incremental && !string.IsNullOrWhiteSpace(TestIncrementalRunFromDate)) { effectiveStartTime = DateTime.Parse(TestIncrementalRunFromDate); Log.InfoFormat("Found TestIncrementalRunFromDate. Using its value {0}", TestIncrementalRunFromDate); } return(effectiveStartTime); }
private static bool ShouldEnforceFullRun(IEnumerable <string> arguments, IFeedRun lastSuccessfulRun) { var enforceFullRun = arguments.ToList().Contains(FullRunParameterName, StringComparer.InvariantCultureIgnoreCase); if (!AllowIncrementalRuns || enforceFullRun) { return(true); } // If there was no successful prior run if (lastSuccessfulRun == null) { return(true); } return(false); }
private static bool ShouldEnforceFullRun(RunType forceRunType, IFeedRun lastSuccessfulRun) { var enforceFullRun = forceRunType == RunType.Full; if (!AllowIncrementalRuns || enforceFullRun) { return(true); } // If there was no successful prior run if (lastSuccessfulRun == null) { return(true); } return(false); }
private bool ShouldEnforceFullRun(IEnumerable <string> arguments, IFeedRun lastSuccessfulPlaFeedRun) { var enforceFullRun = arguments.ToList().Contains(FullRunParameterName, StringComparer.InvariantCultureIgnoreCase); if (!enforceFullRun && !AllowIncrementalRuns) { enforceFullRun = true; } var lastSuccessfulInventoryFeedRun = FeedRunService.GetLastSuccessfulRun(FeedId); // If there was no successful PLA feed or successful Inventory feed, enforce a full run if (lastSuccessfulPlaFeedRun == null && lastSuccessfulInventoryFeedRun == null) { enforceFullRun = true; } return(enforceFullRun); }
private DateTime GetFromTime(bool enforceFullRun, IFeedRun inventoryFeedRun, IFeedRun lastSuccessfulPlaFeedRun) { DateTime effectiveStartTime; if (lastSuccessfulPlaFeedRun == null || inventoryFeedRun.DateStarted > lastSuccessfulPlaFeedRun.DateStarted) { effectiveStartTime = inventoryFeedRun.DateStarted; } else { effectiveStartTime = lastSuccessfulPlaFeedRun.DateStarted; } // If in an incremental run and a test from date value was specified, use that value if (inventoryFeedRun.FeedRunType == FeedRunType.Incremental && !string.IsNullOrWhiteSpace(TestIncrementalRunFromDate)) { effectiveStartTime = DateTime.Parse(TestIncrementalRunFromDate); Log.InfoFormat("Found TestIncrementalRunFromDate. Using its value {0}", TestIncrementalRunFromDate); } return(effectiveStartTime); }
public void Build(string[] args) { try { //check if another instance is running - if so exit ExecutionLogLogger.SetExecutionStartTime(DateTime.Now); var isProcessRunning = Process.GetProcessesByName(Path.GetFileNameWithoutExtension(Assembly.GetEntryAssembly().Location)).Length > 1; if (isProcessRunning) { Log.Info("Another instance was running, so exiting the application without any processing."); return; } // First check if the "done" file is there. If so, skip job execution and move to file-related steps if (HasDoneFile()) { Log.Info("Done file exists at the drop folder. Skipping job execution and moving to file operations."); ExecuteFileOperations(); return; } // Check if the target destination contains any files, if so exit. if (!IsTargetDestinationClean()) { Log.Info("Target destination has files. Exiting application without any processing."); return; } var enforceFullRun = args.ToList().Contains(FullRunParamName, StringComparer.InvariantCultureIgnoreCase); Log.Info("Execution started!"); var feed = FeedService.GetFeed(FeedId); // If the feed is paused, terminate execution if (feed.IsPaused) { Log.Info("This feed is in paused state. Exiting application without any processing"); return; } // First clean all the files inside the output folder (in case the previous run failed) RemoveOldFiles(); if (!enforceFullRun && !AllowIncrementalRuns) { enforceFullRun = true; } _currentRun = FeedRunService.GetOrCreateActiveFeedRun(feed.FeedId, enforceFullRun, AllowIncrementalRuns); _effectiveExecutionEndTime = DateTime.Now; var fromTime = _currentRun.DateStarted; var effectiveStartTime = fromTime.AddHours(-IncrementalRunBufferTimeLength); if (_currentRun.FeedRunType == FeedRunType.Incremental && !string.IsNullOrWhiteSpace(TestIncrementalRunFromDate)) { var time = DateTime.Parse(TestIncrementalRunFromDate); fromTime = time; effectiveStartTime = time.AddHours(-IncrementalRunBufferTimeLength); Log.InfoFormat("Found TestIncrementalRunFromDate. Using its value {0}", TestIncrementalRunFromDate); } if (_currentRun.FeedRunType == FeedRunType.Incremental) { Log.InfoFormat("Executing an incremental run with an effective start date of {0}", effectiveStartTime); } Runner.Initialize(ExecutionLogLogger, _currentRun.FeedId, _currentRun.FeedRunType == FeedRunType.Incremental, fromTime, effectiveStartTime, _effectiveExecutionEndTime); ExecutionLogLogger = Runner.Execute(); if (ExecutionLogLogger.HasError && (_currentRun.FeedRunType == FeedRunType.Incremental || !RRFeedGenerator.Execution.FileFeedWriter.Constants.AllowItemErrorsInFiles)) { Log.Error("Execution failed!!!"); HandleExit(); return; } HandleSuccess(); var elapsedTime = DateTime.Now - ExecutionLogLogger.GetExecutionStartTime(); Log.InfoFormat("Execution completed in {0}", elapsedTime.ToString(@"dd\.hh\:mm\:ss")); } catch (Exception e) { Log.Error("Execution failed!!!", e); HandleExit(); } }
public void Build(string[] arguments) { try { //check if another instance is running - if so exit ExecutionLogLogger.SetExecutionStartTime(DateTime.Now); var isProcessRunning = Process.GetProcessesByName(Path.GetFileNameWithoutExtension(Assembly.GetEntryAssembly().Location)).Length > 1; if (isProcessRunning) { Log.Info("Another instance was running, so exiting the application without any processing."); return; } Log.Info("Execution started!"); if (OutputInstructionProcessor.Initialize()) { Log.Info("Instruction processor processed instructions from the previous exection. Exiting the application without executing a new run."); return; } var feed = FeedService.GetFeed(FeedId); if (feed.IsPaused) { Log.InfoFormat("{0} feed is paused. Exiting application without any processing.", feed.Name); return; } // First decide if a full run should be enforced. var lastSuccessfulRun = FeedRunService.GetLastSuccessfulRun(FeedId); var enforceFullRun = ShouldEnforceFullRun(arguments, lastSuccessfulRun); var isPseudoFullRun = false; // If a full run is being enforced, but there is an existing successful run, execute an incremental run, where we still send products // that haven't been modified/touched by rule changes. This is necessary because a product expires in 30 days in Google if no updates // about the product is sent. The solution we came up with regarding this behavior is having 2 schedules for this feed generator, where one will run // only once a week and enforce a "full run". If we're in such a pseudo full run, then we'll execute an incremental run, while ensuring that unmodified // product data still gets passed. if (enforceFullRun && lastSuccessfulRun != null) { isPseudoFullRun = true; } // Next, need to decide on the effective start time. _currentRun = FeedRunService.GetOrCreateActiveFeedRun(FeedId, enforceFullRun && !isPseudoFullRun, AllowIncrementalRuns); DateTime?fromTime = null; DateTime?effectiveStartTime = null; // Only set these values if we aren't in a "real" full run if (!(enforceFullRun && !isPseudoFullRun)) { fromTime = GetFromTime(_currentRun); // Effective start time is used for callibrating the fromTime to a previous point in time, in case this is needed due to the execution // sequence/timinig of other processes that impact the data that the inventory feed depends on. For example, if results of an inventory-related // process takes 2 hours to get replicated down to the catalogue/website, where as the inventory data has already been updated in Bronte at // the time of execution AND business has updated rules related to the feed in the past 15 minutes, then gathering the incremental data as // if the run started two hours ago but applying rule changes using "now" as the reference point will yield more "accurate" results. effectiveStartTime = fromTime; if (_currentRun.FeedRunType == FeedRunType.Incremental) { effectiveStartTime = fromTime.Value.AddHours(-IncrementalRunBufferTimeLength); } } _effectiveExecutionEndTime = DateTime.Now; Runner.Initialize(ExecutionLogLogger, GooglePlaFeedId, _currentRun.FeedRunType == FeedRunType.Incremental, fromTime, effectiveStartTime, _effectiveExecutionEndTime, isPseudoFullRun); ExecutionLogLogger = Runner.Execute(); // Ensure that we kill the watcher here as well if (ExecutionLogLogger.HasError) { Log.Error("Execution failed!!! Exiting application without further processing."); HandleExit(); return; } HandleSuccess(); var elapsedTime = DateTime.Now - ExecutionLogLogger.GetExecutionStartTime(); Log.InfoFormat("Execution completed in {0}", elapsedTime.ToString(@"dd\.hh\:mm\:ss")); } catch (Exception e) { Log.Error("Execution failed!!!", e); HandleExit(); } }
public void Build(string[] args) { try { //check if another instance is running - if so exit ExecutionLogLogger.SetExecutionStartTime(DateTime.Now); var isProcessRunning = Process.GetProcessesByName(Path.GetFileNameWithoutExtension(Assembly.GetEntryAssembly().Location)).Length > 1; if (isProcessRunning) { Log.Info("Another instance was running, so exiting the application without any processing."); return; } var enforceFullRun = args.ToList().Contains(FullRunParamName, StringComparer.InvariantCultureIgnoreCase); Log.Info("Execution started!"); var feed = FeedService.GetFeed(FeedId); // If the feed is paused, terminate execution if (feed.IsPaused) { Log.Info("This feed is in paused state. Exiting application without any processing"); return; } if (!enforceFullRun && !AllowIncrementalRuns) { enforceFullRun = true; } _currentRun = FeedRunService.GetOrCreateActiveFeedRun(feed.FeedId, enforceFullRun, AllowIncrementalRuns); // First clean all the files inside the download folder (in case the previous run failed) RemoveOldFiles(); // Download the file GetRatingsFileByFtp(); // Unzip the file UnzipRatingsFile(); _effectiveExecutionEndTime = DateTime.Now; var fromTime = _currentRun.DateStarted; var effectiveStartTime = fromTime.AddHours(-IncrementalRunBufferTimeLength); if (_currentRun.FeedRunType == FeedRunType.Incremental && !string.IsNullOrWhiteSpace(TestIncrementalRunFromDate)) { var time = DateTime.Parse(TestIncrementalRunFromDate); fromTime = time; effectiveStartTime = time.AddHours(-IncrementalRunBufferTimeLength); Log.InfoFormat("Found TestIncrementalRunFromDate. Using its value {0}", TestIncrementalRunFromDate); } if (_currentRun.FeedRunType == FeedRunType.Incremental) { Log.InfoFormat("Executing an incremental run with an effective start date of {0}", effectiveStartTime); } Runner.Initialize(ExecutionLogLogger, _currentRun.FeedId, _currentRun.FeedRunType == FeedRunType.Incremental, fromTime, effectiveStartTime, _effectiveExecutionEndTime); ExecutionLogLogger = Runner.Execute(); if (ExecutionLogLogger.HasError) { Log.Error("Execution failed!!!"); HandleExit(); return; } HandleSuccess(); var elapsedTime = DateTime.Now - ExecutionLogLogger.GetExecutionStartTime(); Log.InfoFormat("Execution completed in {0}", elapsedTime.ToString(@"dd\.hh\:mm\:ss")); } catch (Exception e) { Log.Error("Execution failed!!!", e); HandleExit(); } }
public void Build(string[] args) { try { //check if another instance is running - if so exit ExecutionLogLogger.SetExecutionStartTime(DateTime.Now); ExecutionLogLoggerSecondary.SetExecutionStartTime(DateTime.Now); var isProcessRunning = Process.GetProcessesByName(Path.GetFileNameWithoutExtension(Assembly.GetEntryAssembly().Location)).Length > 1; if (isProcessRunning) { Log.Info("Another instance was running, so exiting the application without any processing."); return; } // First check if the "done" file is there. If so, skip job execution and move to file-related steps if (HasDoneFile()) { Log.Info("Done file exists at the drop folder. Skipping job execution and moving to file operations."); ExecuteFileOperations(); return; } Log.Info("Execution started!"); var feed = FeedService.GetFeed(FeedId); IFeed secondaryFeed = null; if (RunMode == RunMode.PrimaryAndSecondary) { secondaryFeed = FeedService.GetFeed(FeedIdSecondary); } var allowPrimary = !feed.IsPaused; var allowSecondary = secondaryFeed != null && !secondaryFeed.IsPaused; if (!allowPrimary && !allowSecondary) { Log.InfoFormat("Both primary {0} and secondary {1} feeds are paused. Exiting application without any processing.", feed.Name, secondaryFeed.Name); return; } // If one of the feeds is paused, notify the user and move on if (!allowPrimary) { Log.InfoFormat("The primary feed {0} is in paused state. Skipping primary feed.", feed.Name); } if (RunMode == RunMode.PrimaryAndSecondary && !allowSecondary) { Log.InfoFormat("The secondary feed {0} is in paused state. Skipping secondary feed.", secondaryFeed.Name); } // First clean all the files inside the output folder (in case the previous run failed and left out old files behind.) RemoveOldFiles(); _currentRun = FeedRunService.GetOrCreateActiveFeedRun(feed.FeedId, true, false); int?secondaryRunId = null; if (RunMode == RunMode.PrimaryAndSecondary) { _currentSecondaryRun = FeedRunService.GetOrCreateActiveFeedRun(secondaryFeed.FeedId, true, false); secondaryRunId = _currentSecondaryRun.FeedId; } _effectiveExecutionEndTime = DateTime.Now; Runner.Initialize(ExecutionLogLogger, ExecutionLogLoggerSecondary, _currentRun.FeedId, secondaryRunId); var results = Runner.Execute().ToList(); ExecutionLogLogger = results[0]; if (RunMode == RunMode.PrimaryAndSecondary) { ExecutionLogLoggerSecondary = results[1]; } if (ExecutionLogLogger.HasError || ExecutionLogLoggerSecondary.HasError) { Log.Error("Execution failed!!! Exiting application without moving any files."); HandleExit(); return; } HandleSuccess(); var elapsedTime = DateTime.Now - ExecutionLogLogger.GetExecutionStartTime(); Log.InfoFormat("Execution completed in {0}", elapsedTime.ToString(@"dd\.hh\:mm\:ss")); } catch (Exception e) { Log.Error("Execution failed!!!", e); HandleExit(); } }
public void Build(RunType requestedRunType, DateTime?effectiveStartTime = null, DateTime?effectiveEndTime = null) { try { NewRelic.Api.Agent.NewRelic.AddCustomParameter("FeedId", FeedId); // start report creation _reportInformation = new ReportInformation(FeedId) { ExecutionStartTime = DateTime.Now, RunType = requestedRunType, EffectiveEndTime = effectiveEndTime }; //check if another instance is running - if so exit var isProcessRunning = Process.GetProcessesByName(Path.GetFileNameWithoutExtension(Assembly.GetEntryAssembly().Location)).Length > 1; if (isProcessRunning) { Log.Info("Another instance was running, so exiting the application without any processing."); return; } Log.Info("Execution started!"); if (!Runner.IsReady()) { Log.Info("Runner is not ready for a new run. Exiting the application without executing a new run."); return; } var feed = FeedService.GetFeed(FeedId); _reportInformation.FeedName = feed.Name; if (feed.IsPaused) { Log.InfoFormat("{0} feed is paused. Exiting application without any processing.", feed.Name); _reportInformation.CustomMessages.Add("Feed is paused. No processing done."); return; } // First decide if a full run should be enforced. var lastSuccessfulRun = FeedRunService.GetLastSuccessfulRun(FeedId); var enforceFullRun = ShouldEnforceFullRun(requestedRunType, lastSuccessfulRun); // If incremental run requested, but full run required first (unless forced) // then abort as the incremental might be configured for different output if (requestedRunType == RunType.Incremental && enforceFullRun) { Log.InfoFormat("{0} feed must execute a full run first. Exiting application without any processing.", feed.Name); _reportInformation.CustomMessages.Add("Must execute a full run first. No processing done."); return; } // Next, need to decide on the effective start time. _currentRun = FeedRunService.GetOrCreateActiveFeedRun(FeedId, enforceFullRun, AllowIncrementalRuns); _reportInformation.FeedRunId = _currentRun.FeedRunId; DateTime?fromTime = null; // Only set these values if we aren't in a "real" full run if (!enforceFullRun && requestedRunType != RunType.OnDemand) { fromTime = GetFromTime(_currentRun); // Effective start time is used for callibrating the fromTime to a previous point in time, in case this is needed due to the execution // sequence/timinig of other processes that impact the data that the inventory feed depends on. For example, if results of an inventory-related // process takes 2 hours to get replicated down to the catalogue/website, where as the inventory data has already been updated in Bronte at // the time of execution AND business has updated rules related to the feed in the past 15 minutes, then gathering the incremental data as // if the run started two hours ago but applying rule changes using "now" as the reference point will yield more "accurate" results. effectiveStartTime = fromTime; if (_currentRun.FeedRunType == FeedRunType.Incremental) { effectiveStartTime = fromTime.Value.AddMinutes(-IncrementalRunBufferTimeLength); } } _reportInformation.EffectiveStartTime = effectiveStartTime; _effectiveExecutionEndTime = DateTime.Now; var runType = _currentRun.FeedRunType == FeedRunType.Incremental ? RunType.Incremental : requestedRunType; Runner.Initialize( runType, effectiveStartTime, effectiveEndTime); _executionInformation = Runner.Execute(); UpdateCountersInReport(_executionInformation, runType); // Ensure that we kill the watcher here as well if (_executionInformation.HasError) { _reportInformation.HasErrors = true; LoggingHelper.Error("Execution failed!!! Exiting application without further processing.", Log); HandleExit(); return; } _reportInformation.ExecutionEndTime = DateTime.Now; HandleSuccess(); var elapsedTime = _reportInformation.ExecutionEndTime.Value - _reportInformation.ExecutionStartTime; Log.InfoFormat("Execution completed in {0}", elapsedTime.ToString(@"dd\.hh\:mm\:ss")); } catch (Exception e) { LoggingHelper.Error(e, "Execution failed!!!", Log); _reportInformation.CustomMessages.Add("Exception during execution. See logs for details."); HandleExit(); } }
public void Build(string[] arguments) { try { //check if another instance is running - if so exit ExecutionLogLogger.SetExecutionStartTime(DateTime.Now); var isProcessRunning = Process.GetProcessesByName(Path.GetFileNameWithoutExtension(Assembly.GetEntryAssembly().Location)).Length > 1; if (isProcessRunning) { Log.Info("Another instance was running, so exiting the application without any processing."); return; } // First check if the "done" file is there. If so, skip job execution and move to file-related steps if (HasDoneFile()) { Log.Info("Done file exists at the drop folder. Skipping job execution and moving to file operations."); ExecuteFileOperations(); return; } Log.Info("Execution started!"); var feed = FeedService.GetFeed(FeedId); if (feed.IsPaused) { Log.InfoFormat("{0} feed is paused. Exiting application without any processing.", feed.Name); return; } // First clean all the files inside the output folder (in case the previous run failed and left out old files behind.) RemoveOldFiles(); // First decide if a full run should be enforced. var lastSuccessfulPlaFeedRun = FeedRunService.GetLastSuccessfulRun(GooglePlaFeedId); var enforceFullRun = ShouldEnforceFullRun(arguments, lastSuccessfulPlaFeedRun); // Next, need to decide on the effective start time. If the previous successful run of the inventory feed // was more recent than the most recent successful run of the Google PLA feed, then use its end time. // Otherwise, use the end time of the most recent successful Google PLA feed run. _currentRun = FeedRunService.GetOrCreateActiveFeedRun(FeedId, enforceFullRun, AllowIncrementalRuns); var fromTime = GetFromTime(enforceFullRun, _currentRun, lastSuccessfulPlaFeedRun); // Effective start time is used for callibrating the fromTime to a previous point in time, in case this is needed due to the execution // sequence/timinig of other processes that impact the data that the inventory feed depends on. For example, if results of an inventory-related // process takes 2 hours to get replicated down to the catalogue/website, where as the inventory data has already been updated in Bronte at // the time of execution AND business has updated rules related to the feed in the past 15 minutes, then gathering the incremental data as // if the run started two hours ago but applying rule changes using "now" as the reference point will yield more "accurate" results. var effectiveStartTime = fromTime; if (_currentRun.FeedRunType == FeedRunType.Incremental) { effectiveStartTime = fromTime.AddHours(-IncrementalRunBufferTimeLength); } _effectiveExecutionEndTime = DateTime.Now; Runner.Initialize(ExecutionLogLogger, GooglePlaFeedId, _currentRun.FeedRunType == FeedRunType.Incremental, fromTime, effectiveStartTime, _effectiveExecutionEndTime); ExecutionLogLogger = Runner.Execute(); if (ExecutionLogLogger.HasError) { Log.Error("Execution failed!!! Exiting application without moving any files."); HandleExit(); return; } HandleSuccess(); var elapsedTime = DateTime.Now - ExecutionLogLogger.GetExecutionStartTime(); Log.InfoFormat("Execution completed in {0}", elapsedTime.ToString(@"dd\.hh\:mm\:ss")); } catch (Exception e) { Log.Error("Execution failed!!!", e); HandleExit(); } }