/// <summary> Constructor for a new instance of the Worker_Controller class </summary> /// <param name="Verbose"> Flag indicates if this should be verbose in the log file and console </param> public Worker_Controller(bool Verbose) { verbose = Verbose; controllerStarted = DateTime.Now; aborted = false; // Assign the database connection strings SobekCM_Database.Connection_String = SobekCM_Library_Settings.Database_Connections[0].Connection_String; Library.Database.SobekCM_Database.Connection_String = SobekCM_Library_Settings.Database_Connections[0].Connection_String; // Pull the values from the database and assign other setting values SobekCM_Library_Settings.Local_Log_Directory = Application.StartupPath + "\\Logs\\"; DataSet settings = Library.Database.SobekCM_Database.Get_Builder_Settings_Complete(null); if (settings == null) { Console.WriteLine("FATAL ERROR pulling latest settings from the database: " + Library.Database.SobekCM_Database.Last_Exception.Message); return; } if (!SobekCM_Library_Settings.Refresh(settings)) { Console.WriteLine("Error using database settings to refresh SobekCM_Library_Settings in Worker_Controller constructor"); } // If this starts in an ABORTED mode, set to standard Builder_Operation_Flag_Enum operationFlag = Abort_Database_Mechanism.Builder_Operation_Flag; if ((operationFlag == Builder_Operation_Flag_Enum.ABORTING) || (operationFlag == Builder_Operation_Flag_Enum.ABORT_REQUESTED) || (operationFlag == Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED)) { Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.STANDARD_OPERATION; } }
static void Main(string[] args) { //// Try to read the metadata configuration file string app_start_config = Application.StartupPath + "\\config"; //if ((Directory.Exists(app_start_config)) && (File.Exists(app_start_config + "\\sobekCM_metadata.config"))) //{ // ResourceObjectSettings.MetadataConfig.Read_Metadata_Configuration(app_start_config + "\\sobekCM_metadata.config"); //} bool complete_static_rebuild = false; bool marc_rebuild = false; bool run_preloader = true; bool show_help = false; bool build_production_marcxml_feed = false; bool build_test_marcxml_feed = false; string invalid_arg = String.Empty; bool verbose = false; // Get values from the arguments foreach (string thisArgs in args) { bool arg_handled = false; // Check for versioning option if (thisArgs == "--version") { Console.WriteLine("You are running version " + Engine_ApplicationCache_Gateway.Settings.Static.Current_Builder_Version + " of the SobekCM Builder."); return; } // Always run in BACKGROUND mode if (thisArgs == "--background") { arg_handled = true; } // Check for verbose flag if (thisArgs == "--verbose") { verbose = true; arg_handled = true; } // Check for no loader flag if (thisArgs == "--refresh_oai") { arg_handled = true; } // Check for no oading flag if (thisArgs == "--noload") { run_preloader = false; arg_handled = true; } // Check for static rebuild if (thisArgs == "--staticrebuild") { complete_static_rebuild = true; arg_handled = true; } // Check for static rebuild if (thisArgs == "--createmarc") { marc_rebuild = true; arg_handled = true; } // Check for marc xml feed creation flags if (thisArgs.IndexOf("--marcxml") == 0) { build_production_marcxml_feed = true; arg_handled = true; } if (thisArgs.IndexOf("--testmarcxml") == 0) { build_test_marcxml_feed = true; arg_handled = true; } // Check for help if ((thisArgs == "--help") || (thisArgs == "?") || (thisArgs == "-help")) { show_help = true; arg_handled = true; } // If not handled, set as error if (!arg_handled) { invalid_arg = thisArgs; break; } } // Was there an invalid argument or was help requested if ((invalid_arg.Length > 0) || (show_help)) { StringBuilder builder = new StringBuilder(); builder.Append("\nThis application is used to bulk load SobekCM items, perform post-processing\n"); builder.Append("for items loaded through the web, and perform some regular maintenance activities\n"); builder.Append("in support of a SobekCM web application.\n\n"); builder.Append("Usage: SobekCM_Builder [options]\n\n"); builder.Append("Options:\n\n"); builder.Append(" --version\tDisplays the current version of the SobekCM Builder\n\n"); builder.Append(" --verbose\tFlag indicates to be verbose in the logs and console\n\n"); builder.Append(" --help\t\tShows these instructions\n\n"); builder.Append(" --noload\t\tSupresses the loading portion of the SobekCM Builder\n\n"); builder.Append(" --staticrebuild\tPerform a complete rebuild on static pages\n\n"); builder.Append(" --marcxml\tBuild the marcxml production feed\n\n"); builder.Append(" --testmarcxml\tBuild the marcxml test feed\n\n"); builder.Append(" --createmarc\tRecreate all of the MARC.xml files\n\n"); builder.Append(" --refresh_oai\tResave the OAI-PMH DC data for every item in the library\n\n"); builder.Append("Examples:\n\n"); builder.Append(" 1. To just rebuild all the static pages:\n"); builder.Append(" SobekCM_Builder --nopreload --noload --staticrebuild"); builder.Append(" 2. To have the SobekCM Builder constantly run in the background"); builder.Append(" SobekCM_Builder --background"); // If invalid arg, save to log file if (invalid_arg.Length > 0) { // Show INVALID ARGUMENT error in console Console.WriteLine("\nINVALID ARGUMENT PROVIDED ( " + invalid_arg + " )"); } Console.WriteLine(builder.ToString()); return; } // Now, veryify the configuration file exists string config_file = Application.StartupPath + "\\config\\sobekcm.config"; if (!File.Exists(config_file)) { Console.WriteLine("The configuration file is missing!!\n"); Console.WriteLine("Execution aborted due to missing configuration file."); return; } // Should be a config file now, so read it if (!MultiInstance_Builder_Settings_Reader.Read_Config(config_file)) { Console.WriteLine("Error encountered reading the configuration file!!\n"); Console.WriteLine("Execution aborted due to incorrect configuration file."); return; } // If no instances exist, then the builder has nothing to do if ((MultiInstance_Builder_Settings.Instances.Count == 0) || (String.IsNullOrEmpty(MultiInstance_Builder_Settings.Instances[0].DatabaseConnection.Connection_String))) { Console.WriteLine("Missing database connection string!!\n"); Console.WriteLine("Execution aborted due to configuration file not including any instances to process"); return; } // Assign the connection string and test the connection (if only a single connection listed) if (MultiInstance_Builder_Settings.Instances.Count == 1) { Engine_Database.Connection_String = MultiInstance_Builder_Settings.Instances[0].DatabaseConnection.Connection_String; if (!Engine_Database.Test_Connection()) { Console.WriteLine("Unable to connect to the database using provided connection string:"); Console.WriteLine(); Console.WriteLine(Engine_Database.Connection_String); return; } } // Get the root directory MultiInstance_Builder_Settings.Builder_Executable_Directory = Application.StartupPath; // Verify connectivity and rights on the logs subfolder string logFileDirectory = Path.Combine(MultiInstance_Builder_Settings.Builder_Executable_Directory, "logs"); if (!Directory.Exists(logFileDirectory)) { try { Directory.CreateDirectory(logFileDirectory); } catch { Console.WriteLine("Error creating necessary logs subfolder under the application folder.\n"); Console.WriteLine("Please create manually.\n"); Console.WriteLine(logFileDirectory); return; } } try { StreamWriter testWriter = new StreamWriter(Path.Combine(logFileDirectory, "test.log"), false); testWriter.WriteLine("TEST"); testWriter.Flush(); testWriter.Close(); File.Delete(Path.Combine(logFileDirectory, "test.log")); } catch { Console.WriteLine("The service account needs modify rights on the logs subfolder.\n"); Console.WriteLine("Please correct manually.\n"); Console.WriteLine(logFileDirectory); return; } // Look for Ghostscript from the registry, if not provided in the config file if (Engine_ApplicationCache_Gateway.Settings.Builder.Ghostscript_Executable.Length == 0) { // LOOK FOR THE GHOSTSCRIPT DIRECTORY string possible_ghost = Look_For_Variable_Registry_Key("SOFTWARE\\GPL Ghostscript", "GS_DLL"); if (!String.IsNullOrEmpty(possible_ghost)) { Engine_ApplicationCache_Gateway.Settings.Builder.Ghostscript_Executable = possible_ghost; } } // Look for Imagemagick from the registry, if not provided in the config file string possible_imagemagick = Look_For_Variable_Registry_Key("SOFTWARE\\ImageMagick", "BinPath"); if (!String.IsNullOrEmpty(possible_imagemagick)) { Engine_ApplicationCache_Gateway.Settings.Builder.ImageMagick_Executable = possible_imagemagick; } //// If this is to refresh the OAI, don't use the worker controller //if (( refresh_oai ) && (Engine_ApplicationCache_Gateway.Settings.Database_Connections.Count == 1)) //{ // // Set the item for the current mode // int successes = 0; // DataTable item_list_table = SobekCM_Database.Get_All_Groups_First_VID( ); // foreach (DataRow thisRow in item_list_table.Rows) // { // string bibid = thisRow["BibID"].ToString(); // string vid = thisRow["VID"].ToString(); // int groupid = Convert.ToInt32(thisRow["groupid"]); // string directory = Engine_ApplicationCache_Gateway.Settings.Image_Server_Network + bibid.Substring(0, 2) + "\\" + bibid.Substring(2, 2) + "\\" + bibid.Substring(4, 2) + "\\" + bibid.Substring(6, 2) + "\\" + bibid.Substring(8, 2) + "\\" + vid; // string mets = directory + "\\" + bibid + "_" + vid + ".mets.xml"; // try // { // SobekCM_Item thisItem = SobekCM_Item.Read_METS(mets); // if (thisItem != null) // { // // Get the OAI-PMH dublin core information // StringBuilder oaiDataBuilder = new StringBuilder(1000); // StringWriter writer = new StringWriter(oaiDataBuilder); // DC_METS_dmdSec_ReaderWriter.Write_Simple_Dublin_Core(writer, thisItem.Bib_Info); // // Also add the URL as identifier // oaiDataBuilder.AppendLine("<dc:identifier>" + Engine_ApplicationCache_Gateway.Settings.System_Base_URL + bibid + "</dc:identifier>"); // Resource_Object.Database.SobekCM_Database.Save_Item_Group_OAI(groupid, oaiDataBuilder.ToString(), "oai_dc", true); // writer.Flush(); // writer.Close(); // successes++; // if (successes%1000 == 0) // Console.WriteLine(@"{0} complete", successes); // } // } // catch // { // } // } // return; //} // Controller always runs in background mode Worker_Controller controller = new Worker_Controller(verbose); controller.Execute_In_Background(); // If this was set to aborting, set to last execution aborted Builder_Operation_Flag_Enum operationFlag = Abort_Database_Mechanism.Builder_Operation_Flag; if ((operationFlag == Builder_Operation_Flag_Enum.ABORTING) || (operationFlag == Builder_Operation_Flag_Enum.ABORT_REQUESTED)) { Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; } }
static void Main(string[] args) { // Try to read the metadata configuration file string app_start_config = Application.StartupPath + "\\config"; if ((Directory.Exists(app_start_config)) && (File.Exists(app_start_config + "\\sobekCM_metadata.config"))) { Metadata_Configuration.Read_Metadata_Configuration(app_start_config + "\\sobekCM_metadata.config"); } bool complete_static_rebuild = false; bool marc_rebuild = false; bool run_preloader = true; bool run_background = false; bool show_help = false; bool build_production_marcxml_feed = false; bool build_test_marcxml_feed = false; string invalid_arg = String.Empty; bool refresh_oai = false; bool verbose = false; // Get values from the arguments foreach (string thisArgs in args) { bool arg_handled = false; // Check for the config flag if (thisArgs == "--config") { if (File.Exists(app_start_config + "\\SobekCM_Builder_Configuration.exe")) { Process.Start(app_start_config + "\\SobekCM_Builder_Configuration.exe"); return; } Console.WriteLine("ERROR: Unable to find configuration executable file!!"); return; } // Check for versioning option if (thisArgs == "--version") { Console.WriteLine("You are running version " + SobekCM_Library_Settings.CURRENT_BUILDER_VERSION + " of the SobekCM Builder."); return; } // Check for no loader flag if (thisArgs == "--background") { run_background = true; arg_handled = true; } // Check for verbose flag if (thisArgs == "--verbose") { verbose = true; arg_handled = true; } // Check for no loader flag if (thisArgs == "--refresh_oai") { refresh_oai = true; arg_handled = true; } // Check for no oading flag if (thisArgs == "--noload") { run_preloader = false; arg_handled = true; } // Check for static rebuild if (thisArgs == "--staticrebuild") { complete_static_rebuild = true; arg_handled = true; } // Check for static rebuild if (thisArgs == "--createmarc") { marc_rebuild = true; arg_handled = true; } // Check for marc xml feed creation flags if (thisArgs.IndexOf("--marcxml") == 0) { build_production_marcxml_feed = true; arg_handled = true; } if (thisArgs.IndexOf("--testmarcxml") == 0) { build_test_marcxml_feed = true; arg_handled = true; } // Check for help if ((thisArgs == "--help") || (thisArgs == "?") || (thisArgs == "-help")) { show_help = true; arg_handled = true; } // If not handled, set as error if (!arg_handled) { invalid_arg = thisArgs; break; } } // Was there an invalid argument or was help requested if ((invalid_arg.Length > 0) || (show_help)) { StringBuilder builder = new StringBuilder(); builder.Append("\nThis application is used to bulk load SobekCM items, perform post-processing\n"); builder.Append("for items loaded through the web, and perform some regular maintenance activities\n"); builder.Append("in support of a SobekCM web application.\n\n"); builder.Append("Usage: SobekCM_Builder [options]\n\n"); builder.Append("Options:\n\n"); builder.Append(" --config\tRuns the configuration tool\n\n"); builder.Append(" --version\tDisplays the current version of the SobekCM Builder\n\n"); builder.Append(" --verbose\tFlag indicates to be verbose in the logs and console\n\n"); builder.Append(" --background\tContinues to run in the background\n\n"); builder.Append(" --help\t\tShows these instructions\n\n"); builder.Append(" --noload\t\tSupresses the loading portion of the SobekCM Builder\n\n"); builder.Append(" --staticrebuild\tPerform a complete rebuild on static pages\n\n"); builder.Append(" --marcxml\tBuild the marcxml production feed\n\n"); builder.Append(" --testmarcxml\tBuild the marcxml test feed\n\n"); builder.Append(" --createmarc\tRecreate all of the MARC.xml files\n\n"); builder.Append(" --refresh_oai\tResave the OAI-PMH DC data for every item in the library\n\n"); builder.Append("Examples:\n\n"); builder.Append(" 1. To just rebuild all the static pages:\n"); builder.Append(" SobekCM_Builder --nopreload --noload --staticrebuild"); builder.Append(" 2. To have the SobekCM Builder constantly run in the background"); builder.Append(" SobekCM_Builder --background"); // If invalid arg, save to log file if (invalid_arg.Length > 0) { // Show INVALID ARGUMENT error in console Console.WriteLine("\nINVALID ARGUMENT PROVIDED ( " + invalid_arg + " )"); } Console.WriteLine(builder.ToString()); return; } // Now, veryify the configuration file exists string config_file = Application.StartupPath + "\\config\\sobekcm.config"; if (!File.Exists(config_file)) { Console.WriteLine("The configuration file is missing!!\n"); Console.Write("Would you like to run the configuration tool? [Y/N]: "); string result = Console.ReadLine().ToUpper(); if ((result == "Y") || (result == "YES")) { // Does the config app exist? if (File.Exists(Application.StartupPath + "\\config\\SobekCM_Builder_Configuration.exe")) { // Run the config app Process configProcess = new Process { StartInfo = { FileName = Application.StartupPath + "\\config\\SobekCM_Builder_Configuration.exe" } }; configProcess.Start(); configProcess.WaitForExit(); // If still no config file, just abort if (!File.Exists(config_file)) { Console.WriteLine("Execution aborted due to missing configuration file."); return; } } else { Console.WriteLine("ERROR: Unable to find configuration executable file!!"); return; } } else { Console.WriteLine("Execution aborted due to missing configuration file."); return; } } // Should be a config file now, so read it SobekCM_Library_Settings.Read_Configuration_File(config_file); if ((SobekCM_Library_Settings.Database_Connections.Count == 0) || (SobekCM_Library_Settings.Database_Connections[0].Connection_String.Length == 0)) { Console.WriteLine("Missing database connection string!!\n"); Console.Write("Would you like to run the configuration tool? [Y/N]: "); string result = Console.ReadLine().ToUpper(); if ((result == "Y") || (result == "YES")) { // Does the config app exist? if (File.Exists(Application.StartupPath + "\\config\\SobekCM_Builder_Configuration.exe")) { // Run the config app Process configProcess = new Process { StartInfo = { FileName = Application.StartupPath + "\\config\\SobekCM_Builder_Configuration.exe" } }; configProcess.Start(); configProcess.WaitForExit(); // If still no config file, just abort if (!File.Exists(config_file)) { Console.WriteLine("Execution aborted due to missing configuration file."); return; } } else { Console.WriteLine("ERROR: Unable to find configuration executable file!!"); return; } } else { Console.WriteLine("Execution aborted due to missing configuration file."); return; } } // Assign the connection string and test the connection (if only a single connection listed) if (SobekCM_Library_Settings.Database_Connections.Count == 1) { SobekCM_Database.Connection_String = SobekCM_Library_Settings.Database_Connections[0].Connection_String; if (!SobekCM_Database.Test_Connection()) { Console.WriteLine("Unable to connect to the database using provided connection string:"); Console.WriteLine(); Console.WriteLine(SobekCM_Database.Connection_String); Console.WriteLine(); Console.WriteLine("Run this application with an argument of '--config' to launch the configuration tool."); return; } } // Verify connectivity and rights on the logs subfolder SobekCM_Library_Settings.Local_Log_Directory = Application.StartupPath + "\\logs"; if (!Directory.Exists(SobekCM_Library_Settings.Local_Log_Directory)) { try { Directory.CreateDirectory(SobekCM_Library_Settings.Local_Log_Directory); } catch { Console.WriteLine("Error creating necessary logs subfolder under the application folder.\n"); Console.WriteLine("Please create manually.\n"); Console.WriteLine(SobekCM_Library_Settings.Local_Log_Directory); return; } } try { StreamWriter testWriter = new StreamWriter(SobekCM_Library_Settings.Local_Log_Directory + "\\test.log", false); testWriter.WriteLine("TEST"); testWriter.Flush(); testWriter.Close(); File.Delete(SobekCM_Library_Settings.Local_Log_Directory + "\\test.log"); } catch { Console.WriteLine("The service account needs modify rights on the logs subfolder.\n"); Console.WriteLine("Please correct manually.\n"); Console.WriteLine(SobekCM_Library_Settings.Local_Log_Directory); return; } // Look for Ghostscript from the registry, if not provided in the config file if (SobekCM_Library_Settings.Ghostscript_Executable.Length == 0) { // LOOK FOR THE GHOSTSCRIPT DIRECTORY string possible_ghost = Look_For_Variable_Registry_Key("SOFTWARE\\GPL Ghostscript", "GS_DLL"); if (!String.IsNullOrEmpty(possible_ghost)) { SobekCM_Library_Settings.Ghostscript_Executable = possible_ghost; } } // Look for Imagemagick from the registry, if not provided in the config file string possible_imagemagick = Look_For_Variable_Registry_Key("SOFTWARE\\ImageMagick", "BinPath"); if (!String.IsNullOrEmpty(possible_imagemagick)) { SobekCM_Library_Settings.ImageMagick_Executable = possible_imagemagick; } // If this is to refresh the OAI, don't use the worker controller if ((refresh_oai) && (SobekCM_Library_Settings.Database_Connections.Count == 1)) { // Set the item for the current mode int successes = 0; DataTable item_list_table = SobekCM_Database.Get_All_Groups_First_VID( ); foreach (DataRow thisRow in item_list_table.Rows) { string bibid = thisRow["BibID"].ToString(); string vid = thisRow["VID"].ToString(); int groupid = Convert.ToInt32(thisRow["groupid"]); string directory = SobekCM_Library_Settings.Image_Server_Network + bibid.Substring(0, 2) + "\\" + bibid.Substring(2, 2) + "\\" + bibid.Substring(4, 2) + "\\" + bibid.Substring(6, 2) + "\\" + bibid.Substring(8, 2) + "\\" + vid; string mets = directory + "\\" + bibid + "_" + vid + ".mets.xml"; try { SobekCM_Item thisItem = SobekCM_Item.Read_METS(mets); if (thisItem != null) { // Get the OAI-PMH dublin core information StringBuilder oaiDataBuilder = new StringBuilder(1000); StringWriter writer = new StringWriter(oaiDataBuilder); DC_METS_dmdSec_ReaderWriter.Write_Simple_Dublin_Core(writer, thisItem.Bib_Info); // Also add the URL as identifier oaiDataBuilder.AppendLine("<dc:identifier>" + SobekCM_Library_Settings.System_Base_URL + bibid + "</dc:identifier>"); Resource_Object.Database.SobekCM_Database.Save_Item_Group_OAI(groupid, oaiDataBuilder.ToString(), "oai_dc", true); writer.Flush(); writer.Close(); successes++; if (successes % 1000 == 0) { Console.WriteLine(@"{0} complete", successes); } } } catch { } } return; } // Two ways to run this... constantly in background or once Worker_Controller controller = new Worker_Controller(verbose); if (!run_background) { controller.Execute_Immediately(build_production_marcxml_feed, build_test_marcxml_feed, run_preloader, complete_static_rebuild, marc_rebuild); } else { controller.Execute_In_Background(); } // If this was set to aborting, set to last execution aborted Builder_Operation_Flag_Enum operationFlag = Abort_Database_Mechanism.Builder_Operation_Flag; if ((operationFlag == Builder_Operation_Flag_Enum.ABORTING) || (operationFlag == Builder_Operation_Flag_Enum.ABORT_REQUESTED)) { Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; } }
/// <summary> Continuously execute the processes in a recurring background thread </summary> public void Execute_In_Background() { // Load all the settings SobekCM_Library_Settings.Refresh(Library.Database.SobekCM_Database.Get_Settings_Complete(null)); // Set the variable which will control background execution int time_between_polls = SobekCM_Library_Settings.Builder_Override_Seconds_Between_Polls; if ((time_between_polls < 0) || (SobekCM_Library_Settings.Database_Connections.Count == 1)) { time_between_polls = Convert.ToInt32(SobekCM_Library_Settings.Builder_Seconds_Between_Polls); } // Determine the new log name string log_name = "incoming_" + controllerStarted.Year + "_" + controllerStarted.Month.ToString().PadLeft(2, '0') + "_" + controllerStarted.Day.ToString().PadLeft(2, '0') + ".html"; string local_log_name = SobekCM_Library_Settings.Local_Log_Directory + "\\" + log_name; // Create the new log file LogFileXHTML preloader_logger = new LogFileXHTML(local_log_name, "SobekCM Incoming Packages Log", "UFDC_Builder.exe", true); // start with warnings on imagemagick and ghostscript not being installed if (SobekCM_Library_Settings.ImageMagick_Executable.Length == 0) { Console.WriteLine("WARNING: Could not find ImageMagick installed. Some image processing will be unavailable."); preloader_logger.AddNonError("WARNING: Could not find ImageMagick installed. Some image processing will be unavailable."); } if (SobekCM_Library_Settings.Ghostscript_Executable.Length == 0) { Console.WriteLine("WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable."); preloader_logger.AddNonError("WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable."); } // Set the time for the next feed building event to 10 minutes from now feedNextBuildTime = DateTime.Now.Add(new TimeSpan(0, 10, 0)); // First, step through each active configuration and see if building is currently aborted // while doing very minimal processes aborted = false; Console.WriteLine("Checking for initial abort condition"); preloader_logger.AddNonError("Checking for initial abort condition"); string abort_message = String.Empty; Builder_Operation_Flag_Enum abort_flag = Builder_Operation_Flag_Enum.STANDARD_OPERATION; foreach (Database_Instance_Configuration dbConfig in SobekCM_Library_Settings.Database_Connections) { if ((!aborted) && (dbConfig.Is_Active) && (dbConfig.Can_Abort)) { SobekCM_Database.Connection_String = dbConfig.Connection_String; Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String; // Check that this should not be skipped or aborted Builder_Operation_Flag_Enum operationFlag = Abort_Database_Mechanism.Builder_Operation_Flag; switch (operationFlag) { case Builder_Operation_Flag_Enum.ABORT_REQUESTED: case Builder_Operation_Flag_Enum.ABORTING: abort_message = "PREVIOUS ABORT flag found in " + dbConfig.Name; abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; Console.WriteLine(abort_message); preloader_logger.AddNonError(abort_message); aborted = true; Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; break; case Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED: abort_message = "PREVIOUS NO BUILDING flag found in " + dbConfig.Name; Console.WriteLine(abort_message); preloader_logger.AddNonError(abort_message); aborted = true; break; } } } // If initially aborted, step through each instance and set a message if (aborted) { // Add messages in each active instance foreach (Database_Instance_Configuration dbConfig in SobekCM_Library_Settings.Database_Connections) { if (dbConfig.Is_Active) { Console.WriteLine("Setting previous abort flag message in " + dbConfig.Name); preloader_logger.AddNonError("Setting previous abort flag message in " + dbConfig.Name); SobekCM_Database.Connection_String = dbConfig.Connection_String; Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String; Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", abort_message, String.Empty); // Save information about this last run Library.Database.SobekCM_Database.Set_Setting("Builder Version", SobekCM_Library_Settings.CURRENT_BUILDER_VERSION); Library.Database.SobekCM_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString()); Library.Database.SobekCM_Database.Set_Setting("Builder Last Message", abort_message); // Finally, set the builder flag appropriately if (abort_flag != Builder_Operation_Flag_Enum.STANDARD_OPERATION) { Abort_Database_Mechanism.Builder_Operation_Flag = abort_flag; } } } // Do nothing else return; } // Build all the bulk loader objects List <Worker_BulkLoader> loaders = new List <Worker_BulkLoader>(); bool activeInstanceFound = false; foreach (Database_Instance_Configuration dbConfig in SobekCM_Library_Settings.Database_Connections) { if (!dbConfig.Is_Active) { loaders.Add(null); Console.WriteLine(dbConfig.Name + " is set to INACTIVE"); preloader_logger.AddNonError(dbConfig.Name + " is set to INACTIVE"); } else { activeInstanceFound = true; SobekCM_Database.Connection_String = dbConfig.Connection_String; Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String; // At this point warn on mossing the Ghostscript and ImageMagick if (SobekCM_Library_Settings.ImageMagick_Executable.Length == 0) { Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find ImageMagick installed. Some image processing will be unavailable.", String.Empty); } if (SobekCM_Library_Settings.Ghostscript_Executable.Length == 0) { Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable.", String.Empty); } Console.WriteLine(dbConfig.Name + " - Preparing to begin polling"); preloader_logger.AddNonError(dbConfig.Name + " - Preparing to begin polling"); Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Preparing to begin polling", String.Empty); Worker_BulkLoader newLoader = new Worker_BulkLoader(preloader_logger, verbose, dbConfig.Name, dbConfig.Can_Abort); loaders.Add(newLoader); } } // If no active instances, just exit if (!activeInstanceFound) { Console.WriteLine("No active databases in the config file"); preloader_logger.AddError("No active databases in config file... Aborting"); return; } bool firstRun = true; // Loop continually until the end hour is achieved do { // Is it time to build any RSS/XML feeds? bool rebuildRssFeeds = false; if (DateTime.Compare(DateTime.Now, feedNextBuildTime) >= 0) { rebuildRssFeeds = true; feedNextBuildTime = DateTime.Now.Add(new TimeSpan(0, 10, 0)); } // Step through each instance for (int i = 0; i < SobekCM_Library_Settings.Database_Connections.Count; i++) { if (loaders[i] != null) { // Get the instance Database_Instance_Configuration dbInstance = SobekCM_Library_Settings.Database_Connections[i]; // Set the database connection strings SobekCM_Database.Connection_String = dbInstance.Connection_String; Library.Database.SobekCM_Database.Connection_String = dbInstance.Connection_String; // Look for abort if ((dbInstance.Can_Abort) && (CheckForAbort())) { aborted = true; if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED) { abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING; } break; } // Refresh all settings, etc.. loaders[i].Refresh_Settings_And_Item_List(); // Pull the abort/pause flag Builder_Operation_Flag_Enum currentPauseFlag = Abort_Database_Mechanism.Builder_Operation_Flag; // If not paused, run the prebuilder if (currentPauseFlag != Builder_Operation_Flag_Enum.PAUSE_REQUESTED) { if (firstRun) { // // Always build an endeca feed first (so it occurs once a day) // if (SobekCM_Library_Settings.Build_MARC_Feed_By_Default) // { // Create_Complete_MarcXML_Feed(false); // } //} // CLear the old logs Console.WriteLine(dbInstance.Name + " - Expiring old log entries"); preloader_logger.AddNonError(dbInstance.Name + " - Expiring old log entries"); Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Expiring old log entries", String.Empty); Library.Database.SobekCM_Database.Builder_Expire_Log_Entries(SobekCM_Library_Settings.Builder_Log_Expiration_Days); // Rebuild all the static pages Console.WriteLine(dbInstance.Name + " - Rebuilding all static pages"); preloader_logger.AddNonError(dbInstance.Name + " - Rebuilding all static pages"); long staticRebuildLogId = Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Rebuilding all static pages", String.Empty); Static_Pages_Builder builder = new Static_Pages_Builder(SobekCM_Library_Settings.Application_Server_URL, SobekCM_Library_Settings.Static_Pages_Location, SobekCM_Library_Settings.Application_Server_Network); builder.Rebuild_All_Static_Pages(preloader_logger, false, SobekCM_Library_Settings.Local_Log_Directory, dbInstance.Name, staticRebuildLogId); // Process any pending FDA reports from the FDA Report DropBox Process_Any_Pending_FDA_Reports(loaders[i]); } Run_BulkLoader(loaders[i], verbose); // Look for abort if ((!aborted) && (dbInstance.Can_Abort) && (CheckForAbort())) { aborted = true; if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED) { abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING; } break; } if (rebuildRssFeeds) { loaders[i].Build_Feeds(); } // Clear memory from this loader loaders[i].Clear_Item_List(); } else { preloader_logger.AddNonError(dbInstance.Name + " - Building paused"); Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building temporarily PAUSED", String.Empty); } } } if (aborted) { break; } // No longer the first run firstRun = false; // Publish the log publish_log_file(local_log_name); // Sleep for correct number of milliseconds Thread.Sleep(1000 * time_between_polls); } while (DateTime.Now.Hour < BULK_LOADER_END_HOUR); // Do the final work for all of the different dbInstances if (!aborted) { for (int i = 0; i < SobekCM_Library_Settings.Database_Connections.Count; i++) { if (loaders[i] != null) { // Get the instance Database_Instance_Configuration dbInstance = SobekCM_Library_Settings.Database_Connections[i]; // Set the database flag SobekCM_Database.Connection_String = dbInstance.Connection_String; // Pull the abort/pause flag Builder_Operation_Flag_Enum currentPauseFlag2 = Abort_Database_Mechanism.Builder_Operation_Flag; // If not paused, run the prebuilder if (currentPauseFlag2 != Builder_Operation_Flag_Enum.PAUSE_REQUESTED) { // Refresh all settings, etc.. loaders[i].Refresh_Settings_And_Item_List(); // Initiate the recreation of the links between metadata and collections Library.Database.SobekCM_Database.Admin_Update_Cached_Aggregation_Metadata_Links(); } // Clear the memory loaders[i].Clear_Item_List(); } } } else { // Mark the aborted in each instance foreach (Database_Instance_Configuration dbConfig in SobekCM_Library_Settings.Database_Connections) { if (dbConfig.Is_Active) { Console.WriteLine("Setting abort flag message in " + dbConfig.Name); preloader_logger.AddNonError("Setting abort flag message in " + dbConfig.Name); SobekCM_Database.Connection_String = dbConfig.Connection_String; Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String; Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building ABORTED per request from database key", String.Empty); // Save information about this last run Library.Database.SobekCM_Database.Set_Setting("Builder Version", SobekCM_Library_Settings.CURRENT_BUILDER_VERSION); Library.Database.SobekCM_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString()); Library.Database.SobekCM_Database.Set_Setting("Builder Last Message", "Building ABORTED per request"); // Finally, set the builder flag appropriately if (abort_flag == Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED) { Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; } } } } // Publish the log publish_log_file(local_log_name); //// Initiate a solr/lucene index optimization since we are done loading for a while //if (DateTime.Now.Day % 2 == 0) //{ // if (SobekCM_Library_Settings.Document_Solr_Index_URL.Length > 0) // { // Console.WriteLine("Initiating Solr/Lucene document index optimization"); // Solr_Controller.Optimize_Document_Index(SobekCM_Library_Settings.Document_Solr_Index_URL); // } //} //else //{ // if (SobekCM_Library_Settings.Page_Solr_Index_URL.Length > 0) // { // Console.WriteLine("Initiating Solr/Lucene page index optimization"); // Solr_Controller.Optimize_Page_Index(SobekCM_Library_Settings.Page_Solr_Index_URL); // } //} //// Sleep for twenty minutes to end this (the index rebuild might take some time) //Thread.Sleep(1000 * 20 * 60); }
protected override void OnStart(string[] args) { // Get the application data path string appDataPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "SobekCM", "Builder"); try { if (!Directory.Exists(appDataPath)) { Directory.CreateDirectory(appDataPath); } } catch (Exception ee) { EventLog.WriteEntry("SobekCM Builder Service", "Error creating the application data folder, which is missing.\n\n" + appDataPath + "\n\n" + ee.Message, EventLogEntryType.Error); return; } //// Try to read the metadata configuration file string app_start_config = appDataPath + "\\config"; //if ((Directory.Exists(app_start_config)) && (File.Exists(app_start_config + "\\sobekCM_metadata.config"))) //{ // ResourceObjectSettings.MetadataConfig.Read_Metadata_Configuration(app_start_config + "\\sobekCM_metadata.config"); //} // Now, verify the configuration file exists string config_file = appDataPath + "\\config\\sobekcm.config"; if (!File.Exists(config_file)) { EventLog.WriteEntry("SobekCM Builder Service", "The configuration file is missing!!\n\nExecution aborted due to missing configuration file.\n\n" + config_file, EventLogEntryType.Error); return; } // Assign the connection string and test the connection (if only a single connection listed) if (MultiInstance_Builder_Settings.Instances.Count == 1) { Engine_Database.Connection_String = MultiInstance_Builder_Settings.Instances[0].DatabaseConnection.Connection_String; if (!Engine_Database.Test_Connection()) { if (Engine_Database.Last_Exception != null) { EventLog.WriteEntry("SobekCM Builder Service", "Unable to connect to the database using provided connection string:\n\n" + Engine_Database.Connection_String + "\n\n" + Engine_Database.Last_Exception.Message, EventLogEntryType.Error); } else { EventLog.WriteEntry("SobekCM Builder Service", "Unable to connect to the database using provided connection string:\n\n" + Engine_Database.Connection_String, EventLogEntryType.Error); } return; } } // Verify connectivity and rights on the logs subfolder string startUpFile = System.Reflection.Assembly.GetExecutingAssembly().CodeBase.Replace("file://", ""); MultiInstance_Builder_Settings.Builder_Executable_Directory = (new FileInfo(startUpFile)).Directory.FullName; string logFileDirectory = Path.Combine(MultiInstance_Builder_Settings.Builder_Executable_Directory, "logs"); if (!Directory.Exists(logFileDirectory)) { try { Directory.CreateDirectory(logFileDirectory); } catch { EventLog.WriteEntry("SobekCM Builder Service", "Error creating necessary logs subfolder under the application folder.\n\nPlease create manually.\n\n" + logFileDirectory, EventLogEntryType.Error); return; } } try { StreamWriter testWriter = new StreamWriter(Path.Combine(logFileDirectory, "test.log"), false); testWriter.WriteLine("TEST"); testWriter.Flush(); testWriter.Close(); File.Delete(Path.Combine(logFileDirectory, "test.log")); } catch { EventLog.WriteEntry("SobekCM Builder Service", "The service account needs modify rights on the logs subfolder.\n\nPlease create manually.\n\n" + logFileDirectory, EventLogEntryType.Error); return; } // Look for Ghostscript from the registry, if not provided in the config file if (Engine_ApplicationCache_Gateway.Settings.Builder.Ghostscript_Executable.Length == 0) { // LOOK FOR THE GHOSTSCRIPT DIRECTORY string possible_ghost = Look_For_Variable_Registry_Key("SOFTWARE\\GPL Ghostscript", "GS_DLL"); if (!String.IsNullOrEmpty(possible_ghost)) { Engine_ApplicationCache_Gateway.Settings.Builder.Ghostscript_Executable = possible_ghost; } } // Look for Imagemagick from the registry, if not provided in the config file string possible_imagemagick = Look_For_Variable_Registry_Key("SOFTWARE\\ImageMagick", "BinPath"); if (!String.IsNullOrEmpty(possible_imagemagick)) { Engine_ApplicationCache_Gateway.Settings.Builder.ImageMagick_Executable = possible_imagemagick; } // Two ways to run this... constantly in background or once Worker_Controller controller = new Worker_Controller(true); controller.Execute_In_Background(); // If this was set to aborting, set to last execution aborted Builder_Operation_Flag_Enum operationFlag = Abort_Database_Mechanism.Builder_Operation_Flag; if ((operationFlag == Builder_Operation_Flag_Enum.ABORTING) || (operationFlag == Builder_Operation_Flag_Enum.ABORT_REQUESTED)) { Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; } }
/// <summary> Continuously execute the processes in a recurring background thread </summary> public void Execute_In_Background() { // Set the variable which will control background execution int time_between_polls = Engine_ApplicationCache_Gateway.Settings.Builder.Override_Seconds_Between_Polls.HasValue ? Engine_ApplicationCache_Gateway.Settings.Builder.Override_Seconds_Between_Polls.Value : 60; if ((time_between_polls < 0) || (MultiInstance_Builder_Settings.Instances.Count == 1)) { time_between_polls = Convert.ToInt32(Engine_ApplicationCache_Gateway.Settings.Builder.Seconds_Between_Polls); } // Determine the new log name string log_name = "incoming_" + controllerStarted.Year + "_" + controllerStarted.Month.ToString().PadLeft(2, '0') + "_" + controllerStarted.Day.ToString().PadLeft(2, '0') + ".html"; string local_log_name = Path.Combine(logFileDirectory, log_name); // Create the new log file LogFileXhtml preloader_logger = new LogFileXhtml(local_log_name, "SobekCM Incoming Packages Log", "UFDC_Builder.exe", true); // start with warnings on imagemagick and ghostscript not being installed if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.ImageMagick_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.ImageMagick_Executable))) { Console.WriteLine("WARNING: Could not find ImageMagick installed. Some image processing will be unavailable."); preloader_logger.AddNonError("WARNING: Could not find ImageMagick installed. Some image processing will be unavailable."); } if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.Ghostscript_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.Ghostscript_Executable))) { Console.WriteLine("WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable."); preloader_logger.AddNonError("WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable."); } // Set the time for the next feed building event to 10 minutes from now feedNextBuildTime = DateTime.Now.Add(new TimeSpan(0, 10, 0)); // First, step through each active configuration and see if building is currently aborted // while doing very minimal processes aborted = false; Console.WriteLine("Checking for initial abort condition"); preloader_logger.AddNonError("Checking for initial abort condition"); string abort_message = String.Empty; int build_instances = 0; foreach (Single_Instance_Configuration dbConfig in instances) { if (!dbConfig.Is_Active) { Console.WriteLine(dbConfig.Name + " is set to INACTIVE"); preloader_logger.AddNonError(dbConfig.Name + " is set to INACTIVE"); } else { SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; // Check that this should not be skipped or aborted Builder_Operation_Flag_Enum operationFlag = Abort_Database_Mechanism.Builder_Operation_Flag; switch (operationFlag) { case Builder_Operation_Flag_Enum.ABORT_REQUESTED: case Builder_Operation_Flag_Enum.ABORTING: // Since this was an abort request at the very beginning, switch back to standard Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.STANDARD_OPERATION; build_instances++; break; case Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED: abort_message = "PREVIOUS NO BUILDING flag found in " + dbConfig.Name; Console.WriteLine(abort_message); preloader_logger.AddNonError(abort_message); break; default: build_instances++; break; } } } // If no instances to run just abort if (build_instances == 0) { // Add messages in each active instance foreach (Single_Instance_Configuration dbConfig in instances) { if (dbConfig.Is_Active) { Console.WriteLine("No active databases set for building in the config file"); preloader_logger.AddError("No active databases set for building in config file... Aborting"); SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", abort_message, String.Empty); // Save information about this last run Engine_Database.Set_Setting("Builder Version", Engine_ApplicationCache_Gateway.Settings.Static.Current_Builder_Version); Engine_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString()); Engine_Database.Set_Setting("Builder Last Message", abort_message); } } // Do nothing else return; } // Build all the bulk loader objects List <Worker_BulkLoader> loaders = new List <Worker_BulkLoader>(); foreach (Single_Instance_Configuration dbConfig in instances) { if (!dbConfig.Is_Active) { loaders.Add(null); continue; } SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; // At this point warn on mossing the Ghostscript and ImageMagick, to get it into each instances database logs if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.ImageMagick_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.ImageMagick_Executable))) { Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find ImageMagick installed. Some image processing will be unavailable.", String.Empty); } if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.Ghostscript_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.Ghostscript_Executable))) { Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable.", String.Empty); } Console.WriteLine(dbConfig.Name + " - Preparing to begin polling"); preloader_logger.AddNonError(dbConfig.Name + " - Preparing to begin polling"); Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Preparing to begin polling", String.Empty); // Create the new bulk loader Worker_BulkLoader newLoader = new Worker_BulkLoader(preloader_logger, dbConfig, verbose, logFileDirectory, pluginRootDirectory); // Try to refresh to test database and engine connectivity if (newLoader.Refresh_Settings_And_Item_List()) { loaders.Add(newLoader); } else { Console.WriteLine(dbConfig.Name + " - Error pulling setting of configuration information"); preloader_logger.AddError(dbConfig.Name + " - Error pulling setting of configuration information"); } } // If no loaders past the tests above, done if (loaders.Count == 0) { Console.WriteLine("Aborting since no valid instances found to process"); preloader_logger.AddError("Aborting since no valid instances found to process"); return; } // Set the maximum number of packages to process before moving to the next instance if (loaders.Count > 1) { MultiInstance_Builder_Settings.Instance_Package_Limit = 100; } bool firstRun = true; // Loop continually until the end hour is achieved Builder_Operation_Flag_Enum abort_flag = Builder_Operation_Flag_Enum.STANDARD_OPERATION; do { // Is it time to build any RSS/XML feeds? if (DateTime.Compare(DateTime.Now, feedNextBuildTime) >= 0) { feedNextBuildTime = DateTime.Now.Add(new TimeSpan(0, 10, 0)); } bool skip_sleep = false; // Step through each instance for (int i = 0; i < loaders.Count; i++) { if (loaders[i] != null) { // Get the instance Single_Instance_Configuration dbInstance = instances[i]; // Set the database connection strings Engine_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; SobekCM_Item_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; // Look for abort if (CheckForAbort()) { aborted = true; if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED) { abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING; } break; } // Refresh all settings, etc.. loaders[i].Refresh_Settings_And_Item_List(); // Pull the abort/pause flag Builder_Operation_Flag_Enum currentPauseFlag = Abort_Database_Mechanism.Builder_Operation_Flag; // If not paused, run the prebuilder if (currentPauseFlag != Builder_Operation_Flag_Enum.PAUSE_REQUESTED) { skip_sleep = skip_sleep || Run_BulkLoader(loaders[i], verbose); // Look for abort if ((!aborted) && (CheckForAbort())) { aborted = true; if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED) { abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING; } break; } } else { preloader_logger.AddNonError(dbInstance.Name + " - Building paused"); Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building temporarily PAUSED", String.Empty); } } } if (aborted) { break; } // Publish the log publish_log_file(local_log_name); // Sleep for correct number of milliseconds if (!skip_sleep) { Thread.Sleep(1000 * time_between_polls); } } while (DateTime.Now.Hour < BULK_LOADER_END_HOUR); // Do the final work for all of the different dbInstances if (!aborted) { for (int i = 0; i < instances.Count; i++) { if (loaders[i] != null) { // Get the instance Single_Instance_Configuration dbInstance = instances[i]; // Set the database flag SobekCM_Item_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; // Pull the abort/pause flag Builder_Operation_Flag_Enum currentPauseFlag2 = Abort_Database_Mechanism.Builder_Operation_Flag; // If not paused, run the prebuilder if (currentPauseFlag2 != Builder_Operation_Flag_Enum.PAUSE_REQUESTED) { // Initiate the recreation of the links between metadata and collections Engine_Database.Admin_Update_Cached_Aggregation_Metadata_Links(); } // Clear the memory loaders[i].ReleaseResources(); } } } else { // Mark the aborted in each instance foreach (Single_Instance_Configuration dbConfig in instances) { if (dbConfig.Is_Active) { Console.WriteLine("Setting abort flag message in " + dbConfig.Name); preloader_logger.AddNonError("Setting abort flag message in " + dbConfig.Name); SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building ABORTED per request from database key", String.Empty); // Save information about this last run Engine_Database.Set_Setting("Builder Version", Engine_ApplicationCache_Gateway.Settings.Static.Current_Builder_Version); Engine_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString()); Engine_Database.Set_Setting("Builder Last Message", "Building ABORTED per request"); // Finally, set the builder flag appropriately if (abort_flag == Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED) { Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; } } } } // Publish the log publish_log_file(local_log_name); //// Initiate a solr/lucene index optimization since we are done loading for a while //if (DateTime.Now.Day % 2 == 0) //{ // if (Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL.Length > 0) // { // Console.WriteLine("Initiating Solr/Lucene document index optimization"); // Solr_Controller.Optimize_Document_Index(Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL); // } //} //else //{ // if (Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL.Length > 0) // { // Console.WriteLine("Initiating Solr/Lucene page index optimization"); // Solr_Controller.Optimize_Page_Index(Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL); // } //} //// Sleep for twenty minutes to end this (the index rebuild might take some time) //Thread.Sleep(1000 * 20 * 60); }
private bool Configure_Builders_To_Run(LogFileXhtml PreloaderLogger) { // Verify connectivity and rights on the logs subfolder if (!Directory.Exists(logFileDirectory)) { try { Directory.CreateDirectory(logFileDirectory); } catch { Console.WriteLine("Error creating necessary logs subfolder under the application folder.\n"); Console.WriteLine("Please create manually.\n"); Console.WriteLine(logFileDirectory); return(false); } } try { StreamWriter testWriter = new StreamWriter(Path.Combine(logFileDirectory, "test.log"), false); testWriter.WriteLine("TEST"); testWriter.Flush(); testWriter.Close(); File.Delete(Path.Combine(logFileDirectory, "test.log")); } catch { Console.WriteLine("The service account needs modify rights on the logs subfolder.\n"); Console.WriteLine("Please correct manually.\n"); Console.WriteLine(logFileDirectory); return(false); } // Verify connectivity and rights on the plugins subfolder if (!Directory.Exists(pluginRootDirectory)) { try { Directory.CreateDirectory(pluginRootDirectory); } catch { Console.WriteLine("Error creating necessary plugins subfolder under the application folder.\n"); Console.WriteLine("Please create manually.\n"); Console.WriteLine(pluginRootDirectory); return(false); } } try { StreamWriter testWriter = new StreamWriter(Path.Combine(pluginRootDirectory, "test.log"), false); testWriter.WriteLine("TEST"); testWriter.Flush(); testWriter.Close(); File.Delete(Path.Combine(pluginRootDirectory, "test.log")); } catch { Console.WriteLine("The service account needs modify rights on the plugins subfolder.\n"); Console.WriteLine("Please correct manually.\n"); Console.WriteLine(pluginRootDirectory); return(false); } // Now, veryify the configuration file exists configurationFile = Path.Combine(MultiInstance_Builder_Settings.Builder_Executable_Directory, "config", "sobekcm.config"); if (!File.Exists(configurationFile)) { write_error("The configuration file is missing!!", PreloaderLogger); write_error("Execution aborted due to missing configuration file.", PreloaderLogger); return(false); } // Should be a config file now, so read it if (!MultiInstance_Builder_Settings_Reader.Read_Config(configurationFile)) { write_error("Error encountered reading the configuration file!!", PreloaderLogger); write_error("Execution aborted due to incorrect configuration file.", PreloaderLogger); return(false); } // Since the configuration was read, save the time configReadTime = DateTime.Now; // If no instances exist, then the builder has nothing to do if ((MultiInstance_Builder_Settings.Instances.Count == 0) || (String.IsNullOrEmpty(MultiInstance_Builder_Settings.Instances[0].DatabaseConnection.Connection_String))) { write_error("No instances listed in the congfiguration file", PreloaderLogger); write_error("Execution aborted due to configuration file not including any instances to process", PreloaderLogger); return(false); } // This is a bit of shortcut in the case there is only one instance if (MultiInstance_Builder_Settings.Instances.Count == 1) { // If no database connection on the single instance, return if (String.IsNullOrEmpty(MultiInstance_Builder_Settings.Instances[0].DatabaseConnection.Connection_String)) { write_error("Single instance configuration is missing the database connection string", PreloaderLogger); write_error("Execution aborted", PreloaderLogger); return(false); } Engine_Database.Connection_String = MultiInstance_Builder_Settings.Instances[0].DatabaseConnection.Connection_String; if (!Engine_Database.Test_Connection()) { write_error("Unable to connect to the database using provided connection string:", PreloaderLogger); write_error(Engine_Database.Connection_String, PreloaderLogger); write_error("Execution aborted", PreloaderLogger); return(false); } } // Look for the valid Image magick file information in the registry and configuration if ((String.IsNullOrEmpty(Engine_ApplicationCache_Gateway.Settings.Builder.ImageMagick_Executable)) || (!File.Exists(Engine_ApplicationCache_Gateway.Settings.Builder.ImageMagick_Executable))) { string possible_imagemagick = Look_For_Variable_Registry_Key("SOFTWARE\\ImageMagick", "BinPath"); if ((!String.IsNullOrEmpty(possible_imagemagick)) && (Directory.Exists(possible_imagemagick)) && (File.Exists(Path.Combine(possible_imagemagick, "convert.exe")))) { MultiInstance_Builder_Settings.ImageMagick_Executable = Path.Combine(possible_imagemagick, "convert.exe"); } } else { MultiInstance_Builder_Settings.ImageMagick_Executable = Engine_ApplicationCache_Gateway.Settings.Builder.ImageMagick_Executable; } // If no ImageMagick file found, add a warning if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.ImageMagick_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.ImageMagick_Executable))) { write_nonerror("WARNING: Could not find ImageMagick installed. Some image processing will be unavailable.", PreloaderLogger); } // Look for a valid ghostscript file information in the registry and configuration if ((String.IsNullOrEmpty(Engine_ApplicationCache_Gateway.Settings.Builder.Ghostscript_Executable)) || (!File.Exists(Engine_ApplicationCache_Gateway.Settings.Builder.Ghostscript_Executable))) { string possible_ghost = Look_For_Variable_Registry_Key("SOFTWARE\\GPL Ghostscript", "GS_DLL"); if (!String.IsNullOrEmpty(possible_ghost)) { string gsPath = Path.GetDirectoryName(possible_ghost); if ((!String.IsNullOrEmpty(gsPath)) && (Directory.Exists(gsPath)) && ((File.Exists(Path.Combine(gsPath, "gswin32c.exe"))) || (File.Exists(Path.Combine(gsPath, "gswin64c.exe"))))) { if (File.Exists(Path.Combine(gsPath, "gswin64c.exe"))) { MultiInstance_Builder_Settings.Ghostscript_Executable = Path.Combine(gsPath, "gswin64c.exe"); } else { MultiInstance_Builder_Settings.Ghostscript_Executable = Path.Combine(gsPath, "gswin32c.exe"); } } } } else { MultiInstance_Builder_Settings.Ghostscript_Executable = Engine_ApplicationCache_Gateway.Settings.Builder.Ghostscript_Executable; } // If no Ghostscript file found, add a warning if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.Ghostscript_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.Ghostscript_Executable))) { write_nonerror("WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable.", PreloaderLogger); } // Save the list of instances instances.Clear(); foreach (Single_Instance_Configuration dbInfo in MultiInstance_Builder_Settings.Instances) { instances.Add(dbInfo); } // First, step through each active configuration and see if building is currently aborted // while doing very minimal processes aborted = false; write_nonerror("Checking for initial abort condition", PreloaderLogger); string abort_message = String.Empty; int build_instances = 0; foreach (Single_Instance_Configuration dbConfig in instances) { if (!dbConfig.Is_Active) { write_nonerror(dbConfig.Name + " is set to INACTIVE", PreloaderLogger); } else { SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; // Check that this should not be skipped or aborted Builder_Operation_Flag_Enum operationFlag = Abort_Database_Mechanism.Builder_Operation_Flag; switch (operationFlag) { case Builder_Operation_Flag_Enum.ABORT_REQUESTED: case Builder_Operation_Flag_Enum.ABORTING: // Since this was an abort request at the very beginning, switch back to standard Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.STANDARD_OPERATION; build_instances++; break; case Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED: abort_message = "PREVIOUS NO BUILDING flag found in " + dbConfig.Name; write_nonerror(abort_message, PreloaderLogger); break; default: build_instances++; break; } } } // If no instances to run just abort if (build_instances == 0) { // Add messages in each active instance foreach (Single_Instance_Configuration dbConfig in instances) { if (dbConfig.Is_Active) { write_error("No active databases set for building in config file... Aborting", PreloaderLogger); SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", abort_message, String.Empty); // Save information about this last run Engine_Database.Set_Setting("Builder Version", Engine_ApplicationCache_Gateway.Settings.Static.Current_Builder_Version); Engine_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString()); Engine_Database.Set_Setting("Builder Last Message", abort_message); } } // Do nothing else return(false); } // Build all the bulk loader objects loaders.Clear(); foreach (Single_Instance_Configuration dbConfig in instances) { if (!dbConfig.Is_Active) { loaders.Add(null); continue; } SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; // At this point warn on mossing the Ghostscript and ImageMagick, to get it into each instances database logs if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.ImageMagick_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.ImageMagick_Executable))) { Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find ImageMagick installed. Some image processing will be unavailable.", String.Empty); } if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.Ghostscript_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.Ghostscript_Executable))) { Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable.", String.Empty); } write_nonerror(dbConfig.Name + " - Preparing to begin polling", PreloaderLogger); Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Preparing to begin polling", String.Empty); // Create the new bulk loader Worker_BulkLoader newLoader = new Worker_BulkLoader(PreloaderLogger, dbConfig, verbose, logFileDirectory, pluginRootDirectory); // Try to refresh to test database and engine connectivity if (newLoader.Refresh_Settings_And_Item_List()) { loaders.Add(newLoader); } else { write_error(dbConfig.Name + " - Error pulling setting of configuration information", PreloaderLogger); } } // If no loaders past the tests above, done if (loaders.Count == 0) { write_error("Aborting since no valid instances found to process", PreloaderLogger); return(false); } // Set the maximum number of packages to process before moving to the next instance if (loaders.Count > 1) { MultiInstance_Builder_Settings.Instance_Package_Limit = 100; } return(true); }
/// <summary> Continuously execute the processes in a recurring background thread </summary> public void Execute_In_Background() { // Determine the new log name string log_name = "incoming_" + controllerStarted.Year + "_" + controllerStarted.Month.ToString().PadLeft(2, '0') + "_" + controllerStarted.Day.ToString().PadLeft(2, '0') + ".html"; string local_log_name = Path.Combine(logFileDirectory, log_name); // Create the new log file LogFileXhtml preloader_logger = null; try { preloader_logger = new LogFileXhtml(local_log_name, "SobekCM Incoming Packages Log", "UFDC_Builder.exe", true); } catch (Exception ee) { Console.WriteLine("Error creating logfile: " + ee.Message); } // Configure builders to run and run some basic tests if ((!Configure_Builders_To_Run(preloader_logger)) || (preloader_logger == null)) { return; } // Set the variable which will control background execution int time_between_polls = Engine_ApplicationCache_Gateway.Settings.Builder.Override_Seconds_Between_Polls.HasValue ? Engine_ApplicationCache_Gateway.Settings.Builder.Override_Seconds_Between_Polls.Value : 60; if ((time_between_polls < 0) || (MultiInstance_Builder_Settings.Instances.Count == 1)) { time_between_polls = Convert.ToInt32(Engine_ApplicationCache_Gateway.Settings.Builder.Seconds_Between_Polls); } // Loop continually until the end hour is achieved Builder_Operation_Flag_Enum abort_flag = Builder_Operation_Flag_Enum.STANDARD_OPERATION; do { // Check the current last write time on the config file versus the last read if (File.Exists(configurationFile)) { DateTime lastWriteTime = (new FileInfo(configurationFile)).LastWriteTime; if (lastWriteTime > configReadTime) { write_nonerror("Configuration file change detected ... reloading", preloader_logger); Configure_Builders_To_Run(preloader_logger); } } bool skip_sleep = false; // Step through each instance for (int i = 0; i < loaders.Count; i++) { if (loaders[i] != null) { // Get the instance Single_Instance_Configuration dbInstance = instances[i]; // Set the database connection strings Engine_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; SobekCM_Item_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; // Look for abort if (CheckForAbort()) { aborted = true; if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED) { abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING; } break; } // Refresh all settings, etc.. loaders[i].Refresh_Settings_And_Item_List(); // Pull the abort/pause flag Builder_Operation_Flag_Enum currentPauseFlag = Abort_Database_Mechanism.Builder_Operation_Flag; // If not paused, run the prebuilder if (currentPauseFlag != Builder_Operation_Flag_Enum.PAUSE_REQUESTED) { skip_sleep = skip_sleep || Run_BulkLoader(loaders[i], verbose); // Look for abort if ((!aborted) && (CheckForAbort())) { aborted = true; if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED) { abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING; } break; } } else { preloader_logger.AddNonError(dbInstance.Name + " - Building paused"); Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building temporarily PAUSED", String.Empty); } } } if (aborted) { break; } // Publish the log publish_log_file(local_log_name); // Sleep for correct number of milliseconds if (!skip_sleep) { Thread.Sleep(1000 * time_between_polls); } } while (DateTime.Now.Hour < BULK_LOADER_END_HOUR); // Do the final work for all of the different dbInstances if (!aborted) { for (int i = 0; i < instances.Count; i++) { if (loaders[i] != null) { // Get the instance Single_Instance_Configuration dbInstance = instances[i]; // Set the database flag SobekCM_Item_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String; // Pull the abort/pause flag Builder_Operation_Flag_Enum currentPauseFlag2 = Abort_Database_Mechanism.Builder_Operation_Flag; // If not paused, run the prebuilder if (currentPauseFlag2 != Builder_Operation_Flag_Enum.PAUSE_REQUESTED) { // Initiate the recreation of the links between metadata and collections Engine_Database.Admin_Update_Cached_Aggregation_Metadata_Links(); } // Clear the memory loaders[i].ReleaseResources(); } } } else { // Mark the aborted in each instance foreach (Single_Instance_Configuration dbConfig in instances) { if (dbConfig.Is_Active) { Console.WriteLine("Setting abort flag message in " + dbConfig.Name); preloader_logger.AddNonError("Setting abort flag message in " + dbConfig.Name); SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String; Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building ABORTED per request from database key", String.Empty); // Save information about this last run Engine_Database.Set_Setting("Builder Version", Engine_ApplicationCache_Gateway.Settings.Static.Current_Builder_Version); Engine_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString()); Engine_Database.Set_Setting("Builder Last Message", "Building ABORTED per request"); // Finally, set the builder flag appropriately if (abort_flag == Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED) { Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; } } } } // Publish the log publish_log_file(local_log_name); //// Initiate a solr/lucene index optimization since we are done loading for a while //if (DateTime.Now.Day % 2 == 0) //{ // if (Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL.Length > 0) // { // Console.WriteLine("Initiating Solr/Lucene document index optimization"); // Solr_Controller.Optimize_Document_Index(Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL); // } //} //else //{ // if (Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL.Length > 0) // { // Console.WriteLine("Initiating Solr/Lucene page index optimization"); // Solr_Controller.Optimize_Page_Index(Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL); // } //} //// Sleep for twenty minutes to end this (the index rebuild might take some time) //Thread.Sleep(1000 * 20 * 60); }
static void Main(string[] Args) { bool show_help = false; string invalid_arg = String.Empty; bool verbose = false; // Get values from the arguments foreach (string thisArgs in Args) { bool arg_handled = false; // Check for versioning option if (thisArgs == "--version") { Console.WriteLine("You are running version " + Engine_ApplicationCache_Gateway.Settings.Static.Current_Builder_Version + " of the SobekCM Builder."); return; } // Always run in BACKGROUND mode if (thisArgs == "--background") { arg_handled = true; } // Check for verbose flag if (thisArgs == "--verbose") { verbose = true; arg_handled = true; } // Check for no loader flag if (thisArgs == "--refresh_oai") { arg_handled = true; } // Check for help if ((thisArgs == "--help") || (thisArgs == "?") || (thisArgs == "-help")) { show_help = true; arg_handled = true; } // If not handled, set as error if (!arg_handled) { invalid_arg = thisArgs; break; } } // Was there an invalid argument or was help requested if ((invalid_arg.Length > 0) || (show_help)) { StringBuilder builder = new StringBuilder(); builder.Append("\nThis application is used to bulk load SobekCM items, perform post-processing\n"); builder.Append("for items loaded through the web, and perform some regular maintenance activities\n"); builder.Append("in support of a SobekCM web application.\n\n"); builder.Append("Usage: SobekCM_Builder [options]\n\n"); builder.Append("Options:\n\n"); builder.Append(" --version\tDisplays the current version of the SobekCM Builder\n\n"); builder.Append(" --verbose\tFlag indicates to be verbose in the logs and console\n\n"); builder.Append(" --help\t\tShows these instructions\n\n"); // If invalid arg, save to log file if (invalid_arg.Length > 0) { // Show INVALID ARGUMENT error in console Console.WriteLine("\nINVALID ARGUMENT PROVIDED ( " + invalid_arg + " )"); } Console.WriteLine(builder.ToString()); return; } // Controller always runs in background mode Worker_Controller controller = new Worker_Controller(verbose); controller.Execute_In_Background(); // If this was set to aborting, set to last execution aborted Builder_Operation_Flag_Enum operationFlag = Abort_Database_Mechanism.Builder_Operation_Flag; if ((operationFlag == Builder_Operation_Flag_Enum.ABORTING) || (operationFlag == Builder_Operation_Flag_Enum.ABORT_REQUESTED)) { Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; } }