/// <summary> Continuously execute the processes in a recurring background thread </summary> public void Execute_In_Background() { // Set the variable which will control background execution int time_between_polls = Engine_ApplicationCache_Gateway.Settings.Builder.Override_Seconds_Between_Polls.HasValue ? Engine_ApplicationCache_Gateway.Settings.Builder.Override_Seconds_Between_Polls.Value : 60; if ((time_between_polls < 0) || (MultiInstance_Builder_Settings.Instances.Count == 1)) { time_between_polls = Convert.ToInt32(Engine_ApplicationCache_Gateway.Settings.Builder.Seconds_Between_Polls); } // Determine the new log name string log_name = "incoming_" + controllerStarted.Year + "_" + controllerStarted.Month.ToString().PadLeft(2, '0') + "_" + controllerStarted.Day.ToString().PadLeft(2, '0') + ".html"; string local_log_name = Path.Combine(logFileDirectory, log_name); // Create the new log file LogFileXhtml preloader_logger = new LogFileXhtml(local_log_name, "SobekCM Incoming Packages Log", "UFDC_Builder.exe", true); // start with warnings on imagemagick and ghostscript not being installed if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.ImageMagick_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.ImageMagick_Executable))) { Console.WriteLine("WARNING: Could not find ImageMagick installed. Some image processing will be unavailable."); preloader_logger.AddNonError("WARNING: Could not find ImageMagick installed. Some image processing will be unavailable."); } if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.Ghostscript_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.Ghostscript_Executable))) { Console.WriteLine("WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable."); preloader_logger.AddNonError("WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable."); } // Set the time for the next feed building event to 10 minutes from now feedNextBuildTime = DateTime.Now.Add(new TimeSpan(0, 10, 0)); // First, step through each active configuration and see if building is currently aborted // while doing very minimal processes aborted = false; Console.WriteLine("Checking for initial abort condition"); preloader_logger.AddNonError("Checking for initial abort condition"); string abort_message = String.Empty; int build_instances = 0; foreach (Single_Instance_Configuration dbConfig in instances) { if (!dbConfig.Is_Active) { Console.WriteLine(dbConfig.Name + " is set to INACTIVE"); preloader_logger.AddNonError(dbConfig.Name + " is set to INACTIVE"); } else { SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; // Check that this should not be skipped or aborted Builder_Operation_Flag_Enum operationFlag = Abort_Database_Mechanism.Builder_Operation_Flag; switch (operationFlag) { case Builder_Operation_Flag_Enum.ABORT_REQUESTED: case Builder_Operation_Flag_Enum.ABORTING: // Since this was an abort request at the very beginning, switch back to standard Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.STANDARD_OPERATION; build_instances++; break; case Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED: abort_message = "PREVIOUS NO BUILDING flag found in " + dbConfig.Name; Console.WriteLine(abort_message); preloader_logger.AddNonError(abort_message); break; default: build_instances++; break; } } } // If no instances to run just abort if (build_instances == 0) { // Add messages in each active instance foreach (Single_Instance_Configuration dbConfig in instances) { if (dbConfig.Is_Active) { Console.WriteLine("No active databases set for building in the config file"); preloader_logger.AddError("No active databases set for building in config file... Aborting"); SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", abort_message, String.Empty); // Save information about this last run Engine_Database.Set_Setting("Builder Version", Engine_ApplicationCache_Gateway.Settings.Static.Current_Builder_Version); Engine_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString()); Engine_Database.Set_Setting("Builder Last Message", abort_message); } } // Do nothing else return; } // Build all the bulk loader objects List <Worker_BulkLoader> loaders = new List <Worker_BulkLoader>(); foreach (Single_Instance_Configuration dbConfig in instances) { if (!dbConfig.Is_Active) { loaders.Add(null); continue; } SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; // At this point warn on mossing the Ghostscript and ImageMagick, to get it into each instances database logs if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.ImageMagick_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.ImageMagick_Executable))) { Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find ImageMagick installed. Some image processing will be unavailable.", String.Empty); } if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.Ghostscript_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.Ghostscript_Executable))) { Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable.", String.Empty); } Console.WriteLine(dbConfig.Name + " - Preparing to begin polling"); preloader_logger.AddNonError(dbConfig.Name + " - Preparing to begin polling"); Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Preparing to begin polling", String.Empty); // Create the new bulk loader Worker_BulkLoader newLoader = new Worker_BulkLoader(preloader_logger, dbConfig, verbose, logFileDirectory, pluginRootDirectory); // Try to refresh to test database and engine connectivity if (newLoader.Refresh_Settings_And_Item_List()) { loaders.Add(newLoader); } else { Console.WriteLine(dbConfig.Name + " - Error pulling setting of configuration information"); preloader_logger.AddError(dbConfig.Name + " - Error pulling setting of configuration information"); } } // If no loaders past the tests above, done if (loaders.Count == 0) { Console.WriteLine("Aborting since no valid instances found to process"); preloader_logger.AddError("Aborting since no valid instances found to process"); return; } // Set the maximum number of packages to process before moving to the next instance if (loaders.Count > 1) { MultiInstance_Builder_Settings.Instance_Package_Limit = 100; } bool firstRun = true; // Loop continually until the end hour is achieved Builder_Operation_Flag_Enum abort_flag = Builder_Operation_Flag_Enum.STANDARD_OPERATION; do { // Is it time to build any RSS/XML feeds? if (DateTime.Compare(DateTime.Now, feedNextBuildTime) >= 0) { feedNextBuildTime = DateTime.Now.Add(new TimeSpan(0, 10, 0)); } bool skip_sleep = false; // Step through each instance for (int i = 0; i < loaders.Count; i++) { if (loaders[i] != null) { // Get the instance Single_Instance_Configuration dbInstance = instances[i]; // Set the database connection strings Engine_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; SobekCM_Item_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; // Look for abort if (CheckForAbort()) { aborted = true; if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED) { abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING; } break; } // Refresh all settings, etc.. loaders[i].Refresh_Settings_And_Item_List(); // Pull the abort/pause flag Builder_Operation_Flag_Enum currentPauseFlag = Abort_Database_Mechanism.Builder_Operation_Flag; // If not paused, run the prebuilder if (currentPauseFlag != Builder_Operation_Flag_Enum.PAUSE_REQUESTED) { skip_sleep = skip_sleep || Run_BulkLoader(loaders[i], verbose); // Look for abort if ((!aborted) && (CheckForAbort())) { aborted = true; if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED) { abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING; } break; } } else { preloader_logger.AddNonError(dbInstance.Name + " - Building paused"); Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building temporarily PAUSED", String.Empty); } } } if (aborted) { break; } // Publish the log publish_log_file(local_log_name); // Sleep for correct number of milliseconds if (!skip_sleep) { Thread.Sleep(1000 * time_between_polls); } } while (DateTime.Now.Hour < BULK_LOADER_END_HOUR); // Do the final work for all of the different dbInstances if (!aborted) { for (int i = 0; i < instances.Count; i++) { if (loaders[i] != null) { // Get the instance Single_Instance_Configuration dbInstance = instances[i]; // Set the database flag SobekCM_Item_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; // Pull the abort/pause flag Builder_Operation_Flag_Enum currentPauseFlag2 = Abort_Database_Mechanism.Builder_Operation_Flag; // If not paused, run the prebuilder if (currentPauseFlag2 != Builder_Operation_Flag_Enum.PAUSE_REQUESTED) { // Initiate the recreation of the links between metadata and collections Engine_Database.Admin_Update_Cached_Aggregation_Metadata_Links(); } // Clear the memory loaders[i].ReleaseResources(); } } } else { // Mark the aborted in each instance foreach (Single_Instance_Configuration dbConfig in instances) { if (dbConfig.Is_Active) { Console.WriteLine("Setting abort flag message in " + dbConfig.Name); preloader_logger.AddNonError("Setting abort flag message in " + dbConfig.Name); SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building ABORTED per request from database key", String.Empty); // Save information about this last run Engine_Database.Set_Setting("Builder Version", Engine_ApplicationCache_Gateway.Settings.Static.Current_Builder_Version); Engine_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString()); Engine_Database.Set_Setting("Builder Last Message", "Building ABORTED per request"); // Finally, set the builder flag appropriately if (abort_flag == Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED) { Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; } } } } // Publish the log publish_log_file(local_log_name); //// Initiate a solr/lucene index optimization since we are done loading for a while //if (DateTime.Now.Day % 2 == 0) //{ // if (Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL.Length > 0) // { // Console.WriteLine("Initiating Solr/Lucene document index optimization"); // Solr_Controller.Optimize_Document_Index(Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL); // } //} //else //{ // if (Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL.Length > 0) // { // Console.WriteLine("Initiating Solr/Lucene page index optimization"); // Solr_Controller.Optimize_Page_Index(Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL); // } //} //// Sleep for twenty minutes to end this (the index rebuild might take some time) //Thread.Sleep(1000 * 20 * 60); }
/// <summary> Updates the database-based cached aggregation browse information </summary> /// <param name="Settings"> Instance-wide settings which may be required for this process </param> public override void DoWork(InstanceWide_Settings Settings) { Engine_Database.Admin_Update_Cached_Aggregation_Metadata_Links(); }
/// <summary> Continuously execute the processes in a recurring background thread </summary> public void Execute_In_Background() { // Determine the new log name string log_name = "incoming_" + controllerStarted.Year + "_" + controllerStarted.Month.ToString().PadLeft(2, '0') + "_" + controllerStarted.Day.ToString().PadLeft(2, '0') + ".html"; string local_log_name = Path.Combine(logFileDirectory, log_name); // Create the new log file LogFileXhtml preloader_logger = null; try { preloader_logger = new LogFileXhtml(local_log_name, "SobekCM Incoming Packages Log", "UFDC_Builder.exe", true); } catch (Exception ee) { Console.WriteLine("Error creating logfile: " + ee.Message); } // Configure builders to run and run some basic tests if ((!Configure_Builders_To_Run(preloader_logger)) || (preloader_logger == null)) { return; } // Set the variable which will control background execution int time_between_polls = Engine_ApplicationCache_Gateway.Settings.Builder.Override_Seconds_Between_Polls.HasValue ? Engine_ApplicationCache_Gateway.Settings.Builder.Override_Seconds_Between_Polls.Value : 60; if ((time_between_polls < 0) || (MultiInstance_Builder_Settings.Instances.Count == 1)) { time_between_polls = Convert.ToInt32(Engine_ApplicationCache_Gateway.Settings.Builder.Seconds_Between_Polls); } // Loop continually until the end hour is achieved Builder_Operation_Flag_Enum abort_flag = Builder_Operation_Flag_Enum.STANDARD_OPERATION; do { // Check the current last write time on the config file versus the last read if (File.Exists(configurationFile)) { DateTime lastWriteTime = (new FileInfo(configurationFile)).LastWriteTime; if (lastWriteTime > configReadTime) { write_nonerror("Configuration file change detected ... reloading", preloader_logger); Configure_Builders_To_Run(preloader_logger); } } bool skip_sleep = false; // Step through each instance for (int i = 0; i < loaders.Count; i++) { if (loaders[i] != null) { // Get the instance Single_Instance_Configuration dbInstance = instances[i]; // Set the database connection strings Engine_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; SobekCM_Item_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; // Look for abort if (CheckForAbort()) { aborted = true; if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED) { abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING; } break; } // Refresh all settings, etc.. loaders[i].Refresh_Settings_And_Item_List(); // Pull the abort/pause flag Builder_Operation_Flag_Enum currentPauseFlag = Abort_Database_Mechanism.Builder_Operation_Flag; // If not paused, run the prebuilder if (currentPauseFlag != Builder_Operation_Flag_Enum.PAUSE_REQUESTED) { skip_sleep = skip_sleep || Run_BulkLoader(loaders[i], verbose); // Look for abort if ((!aborted) && (CheckForAbort())) { aborted = true; if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED) { abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING; } break; } } else { preloader_logger.AddNonError(dbInstance.Name + " - Building paused"); Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building temporarily PAUSED", String.Empty); } } } if (aborted) { break; } // Publish the log publish_log_file(local_log_name); // Sleep for correct number of milliseconds if (!skip_sleep) { Thread.Sleep(1000 * time_between_polls); } } while (DateTime.Now.Hour < BULK_LOADER_END_HOUR); // Do the final work for all of the different dbInstances if (!aborted) { for (int i = 0; i < instances.Count; i++) { if (loaders[i] != null) { // Get the instance Single_Instance_Configuration dbInstance = instances[i]; // Set the database flag SobekCM_Item_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; // Pull the abort/pause flag Builder_Operation_Flag_Enum currentPauseFlag2 = Abort_Database_Mechanism.Builder_Operation_Flag; // If not paused, run the prebuilder if (currentPauseFlag2 != Builder_Operation_Flag_Enum.PAUSE_REQUESTED) { // Initiate the recreation of the links between metadata and collections Engine_Database.Admin_Update_Cached_Aggregation_Metadata_Links(); } // Clear the memory loaders[i].ReleaseResources(); } } } else { // Mark the aborted in each instance foreach (Single_Instance_Configuration dbConfig in instances) { if (dbConfig.Is_Active) { Console.WriteLine("Setting abort flag message in " + dbConfig.Name); preloader_logger.AddNonError("Setting abort flag message in " + dbConfig.Name); SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building ABORTED per request from database key", String.Empty); // Save information about this last run Engine_Database.Set_Setting("Builder Version", Engine_ApplicationCache_Gateway.Settings.Static.Current_Builder_Version); Engine_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString()); Engine_Database.Set_Setting("Builder Last Message", "Building ABORTED per request"); // Finally, set the builder flag appropriately if (abort_flag == Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED) { Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; } } } } // Publish the log publish_log_file(local_log_name); //// Initiate a solr/lucene index optimization since we are done loading for a while //if (DateTime.Now.Day % 2 == 0) //{ // if (Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL.Length > 0) // { // Console.WriteLine("Initiating Solr/Lucene document index optimization"); // Solr_Controller.Optimize_Document_Index(Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL); // } //} //else //{ // if (Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL.Length > 0) // { // Console.WriteLine("Initiating Solr/Lucene page index optimization"); // Solr_Controller.Optimize_Page_Index(Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL); // } //} //// Sleep for twenty minutes to end this (the index rebuild might take some time) //Thread.Sleep(1000 * 20 * 60); }