/// <summary> Constructor for a new instance of the Worker_BulkLoader class </summary> /// <param name="Logger"> Log file object for logging progress </param> /// <param name="Verbose"> Flag indicates if the builder is in verbose mode, where it should log alot more information </param> /// <param name="InstanceInfo"> Information for the instance of SobekCM to be processed by this bulk loader </param> /// <param name="LogFileDirectory"> Directory where any log files would be written </param> /// <param name="PluginRootDirectory"> Root directory where all the plug-ins are stored locally for the builder </param> public Worker_BulkLoader(LogFileXhtml Logger, Single_Instance_Configuration InstanceInfo, bool Verbose, string LogFileDirectory, string PluginRootDirectory ) { // Save the log file and verbose flag logger = Logger; verbose = Verbose; multiInstanceBuilder = ( MultiInstance_Builder_Settings.Instances.Count > 1); logFileDirectory = LogFileDirectory; pluginRootDirectory = PluginRootDirectory; instanceInfo = InstanceInfo; // If this is processing multiple instances, limit the numbe of packages that should be processed // before allowing the builder to move to the next instance and poll if (multiInstanceBuilder) newItemLimit = 100; else newItemLimit = -1; Add_NonError_To_Log("Worker_BulkLoader.Constructor: Start", verbose, String.Empty, String.Empty, -1); // Create new list of collections to build aggregationsToRefresh = new List<string>(); processedItems = new List<BibVidStruct>(); deletedItems = new List<BibVidStruct>(); // Set some defaults aborted = false; refreshed = false; noFoldersReported = false; firstrun = true; Add_NonError_To_Log("Worker_BulkLoader.Constructor: Done", verbose, String.Empty, String.Empty, -1); }
/// <summary> Constructor for a new instance of the Worker_BulkLoader class </summary> /// <param name="Logger"> Log file object for logging progress </param> /// <param name="Verbose"> Flag indicates if the builder is in verbose mode, where it should log alot more information </param> /// <param name="DbInstance"> This database instance </param> /// <param name="MultiInstanceBuilder"> Flag indicates if this is set to be a multi-instance builder configuration </param> /// <param name="LogFileDirectory"> Directory where any log files would be written </param> public Worker_BulkLoader(LogFileXhtml Logger, bool Verbose, Database_Instance_Configuration DbInstance, bool MultiInstanceBuilder, string LogFileDirectory ) { // Save the log file and verbose flag logger = Logger; verbose = Verbose; instanceName = DbInstance.Name; canAbort = DbInstance.Can_Abort; multiInstanceBuilder = MultiInstanceBuilder; dbInstance = DbInstance; logFileDirectory = LogFileDirectory; if (multiInstanceBuilder) newItemLimit = 100; else newItemLimit = -1; Add_NonError_To_Log("Worker_BulkLoader.Constructor: Start", verbose, String.Empty, String.Empty, -1); // Create new list of collections to build aggregationsToRefresh = new List<string>(); processedItems = new List<BibVidStruct>(); deletedItems = new List<BibVidStruct>(); // get all the info settings = InstanceWide_Settings_Builder.Build_Settings(dbInstance); Refresh_Settings_And_Item_List(); // Ensure there is SOME instance name if (instanceName.Length == 0) instanceName = settings.System.System_Name; if (verbose) settings.Builder.Verbose_Flag = true; Add_NonError_To_Log("Worker_BulkLoader.Constructor: Created Static Pages Builder", verbose, String.Empty, String.Empty, -1); // Set some defaults aborted = false; Add_NonError_To_Log("Worker_BulkLoader.Constructor: Building modules for pre, post, and item processing", verbose, String.Empty, String.Empty, -1); Add_NonError_To_Log("Worker_BulkLoader.Constructor: Done", verbose, String.Empty, String.Empty, -1); }
/// <summary>Private method used to generate the error log for the packages</summary> /// <param name="Folder_Name"></param> /// <param name="ErrorMessage"></param> /// <param name="Resource_Folder"></param> /// <param name="MetsType"></param> /// <param name="BaseErrorMessage"></param> private void Create_Error_Log(string Resource_Folder, string Folder_Name, string ErrorMessage, string MetsType, string BaseErrorMessage) { // Split the message into seperate lines string[] errors = ErrorMessage.Split(new char[] { '\n' }); long errorLogId = OnError(BaseErrorMessage, Folder_Name.Replace("_", ":"), MetsType, -1); try { LogFileXhtml errorLog = new LogFileXhtml(Resource_Folder + "\\" + Folder_Name + ".log.html", "Package Processing Log", "SobekCM Builder Errors"); errorLog.New(); errorLog.AddComplete("Error Log for " + Folder_Name + " processed at: " + DateTime.Now.ToString()); errorLog.AddComplete(""); foreach (string thisError in errors) { errorLog.AddError(thisError); OnError(thisError, Folder_Name.Replace("_", ":"), MetsType, errorLogId); } errorLog.Close(); } catch { } }
private void Run_BulkLoader( bool Verbose ) { // Create the local log directories if (!Directory.Exists(Engine_ApplicationCache_Gateway.Settings.Local_Log_Directory)) { Console.WriteLine("Creating local log directory: " + Engine_ApplicationCache_Gateway.Settings.Local_Log_Directory); Directory.CreateDirectory(Engine_ApplicationCache_Gateway.Settings.Local_Log_Directory); } // Determine the new log name string log_name = "incoming_" + controllerStarted.Year + "_" + controllerStarted.Month.ToString().PadLeft(2, '0') + "_" + controllerStarted.Day.ToString().PadLeft(2, '0') + ".html"; string local_log_name = Engine_ApplicationCache_Gateway.Settings.Local_Log_Directory + "\\" + log_name; // Create the new log file LogFileXhtml preloader_logger = new LogFileXhtml(local_log_name, "SobekCM Incoming Packages Log", "UFDC_Builder.exe", true); // Step through each database instance foreach (Database_Instance_Configuration dbConfig in Engine_ApplicationCache_Gateway.Settings.Database_Connections) { try { if (!dbConfig.Is_Active) { Console.WriteLine( dbConfig.Name + " is set to INACTIVE"); preloader_logger.AddNonError(dbConfig.Name + " is set to INACTIVE"); } else { SobekCM_Database.Connection_String = dbConfig.Connection_String; Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String; Worker_BulkLoader newLoader = new Worker_BulkLoader(preloader_logger, verbose, dbConfig, (instances.Count > 1), logFileDirectory); newLoader.Perform_BulkLoader(Verbose); // Save information about this last run Library.Database.SobekCM_Database.Set_Setting("Builder Version", Engine_ApplicationCache_Gateway.Settings.Current_Builder_Version); Library.Database.SobekCM_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString()); Library.Database.SobekCM_Database.Set_Setting("Builder Last Message", newLoader.Final_Message); } } catch { } } }
/// <summary> Continuously execute the processes in a recurring background thread </summary> public void Execute_In_Background() { // Set the variable which will control background execution int time_between_polls = Engine_ApplicationCache_Gateway.Settings.Builder_Override_Seconds_Between_Polls; if (( time_between_polls < 0 ) || ( Engine_ApplicationCache_Gateway.Settings.Database_Connections.Count == 1 )) time_between_polls = Convert.ToInt32(Engine_ApplicationCache_Gateway.Settings.Builder.Seconds_Between_Polls); // Determine the new log name string log_name = "incoming_" + controllerStarted.Year + "_" + controllerStarted.Month.ToString().PadLeft(2, '0') + "_" + controllerStarted.Day.ToString().PadLeft(2, '0') + ".html"; string local_log_name = Engine_ApplicationCache_Gateway.Settings.Local_Log_Directory + "\\" + log_name; if (String.IsNullOrEmpty(Engine_ApplicationCache_Gateway.Settings.Local_Log_Directory)) { local_log_name = logFileDirectory + "\\" + log_name; } // Create the new log file LogFileXhtml preloader_logger = new LogFileXhtml(local_log_name, "SobekCM Incoming Packages Log", "UFDC_Builder.exe", true); // start with warnings on imagemagick and ghostscript not being installed if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.ImageMagick_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.ImageMagick_Executable))) { Console.WriteLine("WARNING: Could not find ImageMagick installed. Some image processing will be unavailable."); preloader_logger.AddNonError("WARNING: Could not find ImageMagick installed. Some image processing will be unavailable."); } if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.Ghostscript_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.Ghostscript_Executable))) { Console.WriteLine("WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable."); preloader_logger.AddNonError("WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable."); } // Set the time for the next feed building event to 10 minutes from now feedNextBuildTime = DateTime.Now.Add(new TimeSpan(0, 10, 0)); // First, step through each active configuration and see if building is currently aborted // while doing very minimal processes aborted = false; Console.WriteLine("Checking for initial abort condition"); preloader_logger.AddNonError("Checking for initial abort condition"); string abort_message = String.Empty; Builder_Operation_Flag_Enum abort_flag = Builder_Operation_Flag_Enum.STANDARD_OPERATION; foreach (Database_Instance_Configuration dbConfig in instances) { if ((!aborted) && (dbConfig.Is_Active) && (dbConfig.Can_Abort)) { SobekCM_Database.Connection_String = dbConfig.Connection_String; Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String; // Check that this should not be skipped or aborted Builder_Operation_Flag_Enum operationFlag = Abort_Database_Mechanism.Builder_Operation_Flag; switch (operationFlag) { case Builder_Operation_Flag_Enum.ABORT_REQUESTED: case Builder_Operation_Flag_Enum.ABORTING: abort_message = "PREVIOUS ABORT flag found in " + dbConfig.Name; abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; Console.WriteLine(abort_message); preloader_logger.AddNonError(abort_message); aborted = true; Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; break; case Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED: abort_message = "PREVIOUS NO BUILDING flag found in " + dbConfig.Name; Console.WriteLine(abort_message); preloader_logger.AddNonError(abort_message); aborted = true; break; } } } // If initially aborted, step through each instance and set a message if (aborted) { // Add messages in each active instance foreach (Database_Instance_Configuration dbConfig in instances) { if (dbConfig.Is_Active) { Console.WriteLine("Setting previous abort flag message in " + dbConfig.Name); preloader_logger.AddNonError("Setting previous abort flag message in " + dbConfig.Name); SobekCM_Database.Connection_String = dbConfig.Connection_String; Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String; Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", abort_message, String.Empty); // Save information about this last run Library.Database.SobekCM_Database.Set_Setting("Builder Version", Engine_ApplicationCache_Gateway.Settings.Current_Builder_Version); Library.Database.SobekCM_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString()); Library.Database.SobekCM_Database.Set_Setting("Builder Last Message", abort_message); // Finally, set the builder flag appropriately if (abort_flag != Builder_Operation_Flag_Enum.STANDARD_OPERATION) Abort_Database_Mechanism.Builder_Operation_Flag = abort_flag; } } // Do nothing else return; } // Build all the bulk loader objects List<Worker_BulkLoader> loaders = new List<Worker_BulkLoader>(); bool activeInstanceFound = false; foreach (Database_Instance_Configuration dbConfig in instances) { if (!dbConfig.Is_Active) { loaders.Add(null); Console.WriteLine(dbConfig.Name + " is set to INACTIVE"); preloader_logger.AddNonError(dbConfig.Name + " is set to INACTIVE"); } else { activeInstanceFound = true; SobekCM_Database.Connection_String = dbConfig.Connection_String; Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String; // At this point warn on mossing the Ghostscript and ImageMagick, to get it into each instances database logs if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.ImageMagick_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.ImageMagick_Executable))) { Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find ImageMagick installed. Some image processing will be unavailable.", String.Empty); } if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.Ghostscript_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.Ghostscript_Executable))) { Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable.", String.Empty); } Console.WriteLine(dbConfig.Name + " - Preparing to begin polling"); preloader_logger.AddNonError(dbConfig.Name + " - Preparing to begin polling"); Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Preparing to begin polling", String.Empty); Worker_BulkLoader newLoader = new Worker_BulkLoader(preloader_logger, verbose, dbConfig, (instances.Count > 1), logFileDirectory); loaders.Add(newLoader); } } // If no active instances, just exit if (!activeInstanceFound) { Console.WriteLine("No active databases in the config file"); preloader_logger.AddError("No active databases in config file... Aborting"); return; } // Set the maximum number of packages to process before moving to the next instance if (loaders.Count > 1) MultiInstance_Builder_Settings.Instance_Package_Limit = 100; bool firstRun = true; // Loop continually until the end hour is achieved do { // Is it time to build any RSS/XML feeds? if (DateTime.Compare(DateTime.Now, feedNextBuildTime) >= 0) { feedNextBuildTime = DateTime.Now.Add(new TimeSpan(0, 10, 0)); } bool skip_sleep = false; // Step through each instance for (int i = 0; i < instances.Count; i++) { if (loaders[i] != null) { // Get the instance Database_Instance_Configuration dbInstance = instances[i]; // Set the database connection strings Engine_Database.Connection_String = dbInstance.Connection_String; SobekCM_Database.Connection_String = dbInstance.Connection_String; Library.Database.SobekCM_Database.Connection_String = dbInstance.Connection_String; // Look for abort if ((dbInstance.Can_Abort) && (CheckForAbort())) { aborted = true; if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED) { abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING; } break; } // Refresh all settings, etc.. loaders[i].Refresh_Settings_And_Item_List(); // Pull the abort/pause flag Builder_Operation_Flag_Enum currentPauseFlag = Abort_Database_Mechanism.Builder_Operation_Flag; // If not paused, run the prebuilder if (currentPauseFlag != Builder_Operation_Flag_Enum.PAUSE_REQUESTED) { if (firstRun) { // // Always build an endeca feed first (so it occurs once a day) // if (Engine_ApplicationCache_Gateway.Settings.Build_MARC_Feed_By_Default) // { // Create_Complete_MarcXML_Feed(false); // } //} // CLear the old logs Console.WriteLine(dbInstance.Name + " - Expiring old log entries"); preloader_logger.AddNonError(dbInstance.Name + " - Expiring old log entries"); Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Expiring old log entries", String.Empty); Library.Database.SobekCM_Database.Builder_Expire_Log_Entries(Engine_ApplicationCache_Gateway.Settings.Builder.Log_Expiration_Days); // Rebuild all the static pages Console.WriteLine(dbInstance.Name + " - Rebuilding all static pages"); preloader_logger.AddNonError(dbInstance.Name + " - Rebuilding all static pages"); long staticRebuildLogId = Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Rebuilding all static pages", String.Empty); Static_Pages_Builder builder = new Static_Pages_Builder(Engine_ApplicationCache_Gateway.Settings.Servers.Application_Server_URL, Engine_ApplicationCache_Gateway.Settings.Servers.Static_Pages_Location, Engine_ApplicationCache_Gateway.URL_Portals.Default_Portal.Default_Web_Skin); builder.Rebuild_All_Static_Pages(preloader_logger, false, Engine_ApplicationCache_Gateway.Settings.Local_Log_Directory, dbInstance.Name, staticRebuildLogId); } skip_sleep = skip_sleep || Run_BulkLoader(loaders[i], verbose); // Look for abort if ((!aborted) && (dbInstance.Can_Abort) && (CheckForAbort())) { aborted = true; if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED) { abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING; } break; } } else { preloader_logger.AddNonError( dbInstance.Name + " - Building paused"); Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building temporarily PAUSED", String.Empty); } } } if (aborted) break; // No longer the first run firstRun = false; // Publish the log publish_log_file(local_log_name); // Sleep for correct number of milliseconds if ( !skip_sleep ) Thread.Sleep(1000 * time_between_polls); } while (DateTime.Now.Hour < BULK_LOADER_END_HOUR); // Do the final work for all of the different dbInstances if (!aborted) { for (int i = 0; i < instances.Count; i++) { if (loaders[i] != null) { // Get the instance Database_Instance_Configuration dbInstance = instances[i]; // Set the database flag SobekCM_Database.Connection_String = dbInstance.Connection_String; Library.Database.SobekCM_Database.Connection_String = dbInstance.Connection_String; // Pull the abort/pause flag Builder_Operation_Flag_Enum currentPauseFlag2 = Abort_Database_Mechanism.Builder_Operation_Flag; // If not paused, run the prebuilder if (currentPauseFlag2 != Builder_Operation_Flag_Enum.PAUSE_REQUESTED) { // Initiate the recreation of the links between metadata and collections Library.Database.SobekCM_Database.Admin_Update_Cached_Aggregation_Metadata_Links(); } // Clear the memory loaders[i].ReleaseResources(); } } } else { // Mark the aborted in each instance foreach (Database_Instance_Configuration dbConfig in instances ) { if (dbConfig.Is_Active) { Console.WriteLine("Setting abort flag message in " + dbConfig.Name); preloader_logger.AddNonError("Setting abort flag message in " + dbConfig.Name); SobekCM_Database.Connection_String = dbConfig.Connection_String; Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String; Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building ABORTED per request from database key", String.Empty); // Save information about this last run Library.Database.SobekCM_Database.Set_Setting("Builder Version", Engine_ApplicationCache_Gateway.Settings.Current_Builder_Version); Library.Database.SobekCM_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString()); Library.Database.SobekCM_Database.Set_Setting("Builder Last Message", "Building ABORTED per request"); // Finally, set the builder flag appropriately if ( abort_flag == Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED ) Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED; } } } // Publish the log publish_log_file(local_log_name); //// Initiate a solr/lucene index optimization since we are done loading for a while //if (DateTime.Now.Day % 2 == 0) //{ // if (Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL.Length > 0) // { // Console.WriteLine("Initiating Solr/Lucene document index optimization"); // Solr_Controller.Optimize_Document_Index(Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL); // } //} //else //{ // if (Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL.Length > 0) // { // Console.WriteLine("Initiating Solr/Lucene page index optimization"); // Solr_Controller.Optimize_Page_Index(Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL); // } //} //// Sleep for twenty minutes to end this (the index rebuild might take some time) //Thread.Sleep(1000 * 20 * 60); }
/// <summary> Immediately perform all requested tasks </summary> /// <param name="BuildProductionMarcxmlFeed"> Flag indicates if the MarcXML feed for OPACs should be produced </param> /// <param name="BuildTestMarcxmlFeed"> Flag indicates if the items set to be put in a TEST feed should have their MarcXML feed produced</param> /// <param name="RunBulkloader"> Flag indicates if the preload </param> /// <param name="CompleteStaticRebuild"> Flag indicates whether to rebuild all the item static pages </param> /// <param name="MarcRebuild"> Flag indicates if all the MarcXML files for each resource should be rewritten from the METS/MODS metadata files </param> public void Execute_Immediately(bool BuildProductionMarcxmlFeed, bool BuildTestMarcxmlFeed, bool RunBulkloader, bool CompleteStaticRebuild, bool MarcRebuild ) { // start with warnings on imagemagick and ghostscript not being installed if (Engine_ApplicationCache_Gateway.Settings.ImageMagick_Executable.Length == 0) { Console.WriteLine("WARNING: Could not find ImageMagick installed. Some image processing will be unavailable."); } if (Engine_ApplicationCache_Gateway.Settings.Ghostscript_Executable.Length == 0) { Console.WriteLine("WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable."); } if (CompleteStaticRebuild) { Console.WriteLine("Beginning static rebuild"); LogFileXhtml staticRebuildLog = new LogFileXhtml( logFileDirectory + "\\static_rebuild.html"); Static_Pages_Builder builder = new Static_Pages_Builder(Engine_ApplicationCache_Gateway.Settings.Servers.Application_Server_URL, Engine_ApplicationCache_Gateway.Settings.Servers.Static_Pages_Location, Engine_ApplicationCache_Gateway.Settings.Servers.Application_Server_Network); builder.Rebuild_All_Static_Pages(staticRebuildLog, true, Engine_ApplicationCache_Gateway.Settings.Local_Log_Directory, String.Empty, -1); } if ( MarcRebuild ) { Static_Pages_Builder builder = new Static_Pages_Builder(Engine_ApplicationCache_Gateway.Settings.Servers.Application_Server_URL, Engine_ApplicationCache_Gateway.Settings.Servers.Static_Pages_Location, Engine_ApplicationCache_Gateway.Settings.Servers.Application_Server_Network); builder.Rebuild_All_MARC_Files(Engine_ApplicationCache_Gateway.Settings.Servers.Image_Server_Network); } if (BuildProductionMarcxmlFeed) { Create_Complete_MarcXML_Feed(false); } if (BuildTestMarcxmlFeed) { Create_Complete_MarcXML_Feed(true); } // Create the log string directory = Engine_ApplicationCache_Gateway.Settings.Local_Log_Directory; if (!Directory.Exists(directory)) Directory.CreateDirectory(directory); // Run the PRELOADER if (RunBulkloader) { Run_BulkLoader( verbose ); } else { Console.WriteLine("PreLoader skipped per command line arguments"); } }
/// <summary> Rebuilds all static pages, including the RSS feeds and site maps </summary> /// <param name="Logger"> Log file to record progress </param> /// <param name="BuildAllCitationPages"> Flag indicates to build the individual static HTML pages for each digital resource </param> /// <param name="RssFeedLocation"> Location where the RSS feeds should be updated to </param> /// <param name="InstanceName"> Name of this instance </param> /// <param name="PrimaryLogId"> Log ID in the case this is the builder and it has been pre-logged </param> /// <returns> The number of encountered errors </returns> public int Rebuild_All_Static_Pages(LogFileXhtml Logger, bool BuildAllCitationPages, string RssFeedLocation, string InstanceName, long PrimaryLogId ) { if (InstanceName.Length > 0) InstanceName = InstanceName + " - "; if (PrimaryLogId < 0) { Console.WriteLine("Rebuilding all static pages"); Logger.AddNonError(InstanceName + "Rebuilding all static pages"); PrimaryLogId = SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Rebuilding all static pages", String.Empty); } // Set the correct base directory if (staticSobekcmLocation.Length > 0) UI_ApplicationCache_Gateway.Settings.Servers.Base_Directory = staticSobekcmLocation; // Set the item for the current mode errors = 0; if (BuildAllCitationPages) { Console.WriteLine(InstanceName + "Rebuildig all citation pages"); SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Rebuilding all item-level citation static pages", String.Empty); DataSet item_list_table = SobekCM_Database.Get_Item_List(false, null); // Get the item list and build the hashtable Item_Lookup_Object itemList = new Item_Lookup_Object(); Engine_Database.Populate_Item_Lookup_Object(false, itemList, tracer); foreach (DataRow thisRow in item_list_table.Tables[0].Rows) { //if (errors > 10000) //{ // logger.AddError("10000 errors encountered!"); // Console.WriteLine("10000 errors encountered"); // return errors; //} string bibid = thisRow["BibID"].ToString(); string vid = thisRow["VID"].ToString(); string itemDirectory = UI_ApplicationCache_Gateway.Settings.Servers.Image_Server_Network + bibid.Substring(0, 2) + "\\" + bibid.Substring(2, 2) + "\\" + bibid.Substring(4, 2) + "\\" + bibid.Substring(6, 2) + "\\" + bibid.Substring(8, 2) + "\\" + vid; string staticDirectory = itemDirectory + "\\" + UI_ApplicationCache_Gateway.Settings.Resources.Backup_Files_Folder_Name; // ********** TEMPORARY CLEAN UP ***********************// // TODO: REMOVE THIS! // *******************************************// try { if (Directory.Exists(itemDirectory)) { if (!Directory.Exists(staticDirectory)) Directory.CreateDirectory(staticDirectory); if ( File.Exists(itemDirectory + "\\thumbs.db")) File.Delete(itemDirectory + "\\thumbs.db"); if ( File.Exists(itemDirectory + "\\doc.xml")) File.Delete(itemDirectory + "\\doc.xml"); if ( File.Exists(itemDirectory + "\\citation_mets.xml")) File.Delete(itemDirectory + "\\citation_mets.xml"); if ( File.Exists(itemDirectory + "\\" + bibid + "_" + vid + ".html")) File.Delete(itemDirectory + "\\" + bibid + "_" + vid + ".html"); if ( File.Exists(itemDirectory + "\\errors.xml")) File.Delete(itemDirectory + "\\errors.xml"); if ( File.Exists(itemDirectory + "\\qc_error.html")) File.Delete(itemDirectory + "\\qc_error.html"); string[] files = Directory.GetFileSystemEntries(itemDirectory, "*.job"); foreach( string thisFile in files ) File.Delete(thisFile); files = Directory.GetFileSystemEntries(itemDirectory, "*.bridge*"); foreach (string thisFile in files) File.Delete(thisFile); files = Directory.GetFileSystemEntries(itemDirectory, "*.qc.jpg"); foreach (string thisFile in files) File.Delete(thisFile); files = Directory.GetFileSystemEntries(itemDirectory, "*_qc.jpg"); foreach (string thisFile in files) File.Delete(thisFile); files = Directory.GetFileSystemEntries(itemDirectory, "*_INGEST_xml.txt"); foreach (string thisFile in files) File.Delete(thisFile); files = Directory.GetFileSystemEntries(itemDirectory, "*_INGEST.xml"); if (files.Length > 0) { if (!Directory.Exists(itemDirectory + "\\fda_files")) Directory.CreateDirectory(itemDirectory + "\\fda_files"); foreach (string thisFile in files) { string filename = Path.GetFileName(thisFile); File.Move(thisFile, itemDirectory + "\\fda_files\\" + filename); } } if (File.Exists(itemDirectory + "\\original.mets.xml")) File.Move(itemDirectory + "\\original.mets.xml", staticDirectory + "\\original.mets.xml"); files = Directory.GetFileSystemEntries(itemDirectory, "recd_*.mets.xml"); foreach (string thisFile in files) { string filename = Path.GetFileName(thisFile); File.Move(thisFile, staticDirectory+ "\\" + filename); } files = Directory.GetFileSystemEntries(itemDirectory, "*.mets.bak"); foreach (string thisFile in files) { string filename = Path.GetFileName(thisFile); File.Move(thisFile, staticDirectory + "\\" + filename); } } } catch { Console.WriteLine(InstanceName + "....Error cleaning up folder " + bibid + ":" + vid); Logger.AddError("....Error cleaning up folder " + bibid + ":" + vid); } // ************* END TEMPORARY CLEAN UP ******************// try { if (!Directory.Exists(staticDirectory)) Directory.CreateDirectory(staticDirectory); string static_file = staticDirectory + "\\" + bibid + "_" + vid + ".html"; if (Create_Item_Citation_HTML(bibid, vid, static_file, itemDirectory, itemList)) { // Also copy to the static page location server string web_server_directory = UI_ApplicationCache_Gateway.Settings.Servers.Static_Pages_Location + bibid.Substring(0, 2) + "\\" + bibid.Substring(2, 2) + "\\" + bibid.Substring(4, 2) + "\\" + bibid.Substring(6, 2) + "\\" + bibid.Substring(8, 2) + "\\" + vid; if (!Directory.Exists(web_server_directory)) Directory.CreateDirectory(web_server_directory); string web_server_file_version = web_server_directory + "\\" + bibid + "_" + vid + ".html"; File.Copy(static_file, web_server_file_version, true); } else { errors++; Console.WriteLine(InstanceName + "....Error creating citation file for: " + bibid + ":" + vid); Logger.AddError("....Error creating citation file for: " + bibid + ":" + vid); } } catch (Exception ee) { Console.WriteLine(InstanceName + "....Exception caught while creating citation file for: " + bibid + ":" + vid); Logger.AddError("....Exception caught while creating citation file for: " + bibid + ":" + vid); Console.WriteLine("........." + ee.Message); Logger.AddError("........." + ee.Message); Logger.AddError("........." + ee.StackTrace); } } Console.WriteLine(InstanceName + "Done rebuilding all item-level citation static pages"); Logger.AddNonError("Done rebuilding all item-level citation static pages"); SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Done rebuilding all item-level citation static pages", String.Empty); } // Set the mode away from the display item mode currentMode.Mode = Display_Mode_Enum.Aggregation; currentMode.Aggregation_Type = Aggregation_Type_Enum.Home; currentMode.Skin = defaultSkin; // Get the default web skin Web_Skin_Object defaultSkinObject = assistant.Get_HTML_Skin(currentMode, Engine_ApplicationCache_Gateway.Web_Skin_Collection, false, null); // Get the list of all collections DataTable allCollections = SobekCM_Database.Get_Codes_Item_Aggregations( null); DataView collectionView = new DataView(allCollections) {Sort = "Name ASC"}; // Build the basic site map first StreamWriter sitemap_writer = new StreamWriter(staticSobekcmDataLocation + "sitemap_collections.xml", false); sitemap_writer.WriteLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"); sitemap_writer.WriteLine("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">"); sitemap_writer.WriteLine("\t<url>"); sitemap_writer.WriteLine("\t\t<loc>" + primaryWebServerUrl + "</loc>"); sitemap_writer.WriteLine("\t</url>"); // Prepare to build all the links to static pages StringBuilder static_browse_links = new StringBuilder(); StringBuilder recent_rss_link_builder = new StringBuilder(); StringBuilder all_rss_link_builder = new StringBuilder(); int col = 2; DataSet items; List<string> processed_codes = new List<string>(); foreach (DataRowView thisCollectionView in collectionView) { // Clear the tracer tracer.Clear(); if (!Convert.ToBoolean(thisCollectionView.Row["Hidden"])) { // Build the static links pages string code = thisCollectionView.Row["Code"].ToString().ToLower(); if ((!processed_codes.Contains(code)) && (code != "all")) { processed_codes.Add(code); // Add this to the sitemap sitemap_writer.WriteLine("\t<url>"); sitemap_writer.WriteLine("\t\t<loc>" + primaryWebServerUrl + code + "</loc>"); sitemap_writer.WriteLine("\t</url>"); Logger.AddNonError(InstanceName + ".....Building static links page for " + code); Console.WriteLine(InstanceName + @"Building static links page for {0}", code); SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building static links page for " + code, String.Empty); //Get the list of items for this collection items = Engine_Database.Simple_Item_List(code, tracer); // Get the item aggregation object Item_Aggregation aggregation = SobekEngineClient.Aggregations.Get_Aggregation(code.ToLower(), UI_ApplicationCache_Gateway.Settings.System.Default_UI_Language, UI_ApplicationCache_Gateway.Settings.System.Default_UI_Language, tracer); // Build the static browse pages if (Build_All_Browse(aggregation, items)) { static_browse_links.Append("<td><a href=\"" + code + "_all.html\">" + code + "</a></td>" + Environment.NewLine); col++; } if (col > 5) { static_browse_links.Append("</tr>" + Environment.NewLine + "<tr>"); col = 1; } // Build the RSS feeds Logger.AddNonError(InstanceName + ".....Building RSS feeds for " + code); Console.WriteLine(InstanceName + @"Building RSS feeds for {0}", code); SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building RSS feeds for " + code, String.Empty); if (Create_RSS_Feed(code, staticSobekcmDataLocation + "rss\\", thisCollectionView.Row["Name"].ToString(), items)) { recent_rss_link_builder.Append("<img src=\"http://cdn.sobekrepository.org/images/misc/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\"> <a href=\"" + primaryWebServerUrl + "rss/" + code + "_short_rss.xml\">" + thisCollectionView.Row["Name"] + "</a><br />" + Environment.NewLine); all_rss_link_builder.Append("<img src=\"http://cdn.sobekrepository.org/images/misc/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\"> <a href=\"" + primaryWebServerUrl + "rss/" + code + "_rss.xml\">" + thisCollectionView.Row["Name"] + "</a><br />" + Environment.NewLine); } } } } // Finish out the collection sitemap sitemap_writer.WriteLine("</urlset>"); sitemap_writer.Flush(); sitemap_writer.Close(); items = Engine_Database.Simple_Item_List(String.Empty, tracer); Logger.AddNonError(InstanceName + ".....Building static links page for ALL ITEMS"); Console.WriteLine(InstanceName + @"Building static links page for ALL ITEMS"); SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building static links page for ALL ITEMS", String.Empty); Item_Aggregation allAggregation = SobekEngineClient.Aggregations.Get_Aggregation("all", UI_ApplicationCache_Gateway.Settings.System.Default_UI_Language, UI_ApplicationCache_Gateway.Settings.System.Default_UI_Language, tracer); Build_All_Browse(allAggregation, items); Console.WriteLine(InstanceName + @"Building RSS feeds ALL ITEMS"); Logger.AddNonError(InstanceName + ".....Building RSS feeds for ALL ITEMS"); SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building RSS feeds for ALL ITEMS", String.Empty); Create_RSS_Feed("all", staticSobekcmDataLocation + "rss\\", "All " + UI_ApplicationCache_Gateway.Settings.System.System_Abbreviation + " Items", items); // Build the site maps Logger.AddNonError(InstanceName + ".....Building site maps"); Console.WriteLine(InstanceName + @"Building site maps"); SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building site maps", String.Empty); int sitemaps = Build_Site_Maps(); // Output the main browse and rss links pages Logger.AddNonError(InstanceName + "....Building main sitemaps and links page"); Console.WriteLine(InstanceName + @"Building main sitemaps and links page"); SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building main sitemaps and links page", String.Empty); StreamWriter allListWriter = new StreamWriter(staticSobekcmDataLocation + "index.html", false); allListWriter.WriteLine("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">"); allListWriter.WriteLine("<html xmlns=\"http://www.w3.org/1999/xhtml\" >"); allListWriter.WriteLine("<head>"); allListWriter.WriteLine(" <title>" + UI_ApplicationCache_Gateway.Settings.System.System_Name + " Site Map Links</title>"); allListWriter.WriteLine(); allListWriter.WriteLine(" <meta name=\"robots\" content=\"index, follow\">"); allListWriter.WriteLine(" <link href=\"" + primaryWebServerUrl + "default/SobekCM.css\" rel=\"stylesheet\" type=\"text/css\" title=\"standard\" />"); if (defaultSkinObject.CSS_Style.Length > 0) { allListWriter.WriteLine(" <link href=\"" + UI_ApplicationCache_Gateway.Settings.Servers.System_Base_URL + defaultSkinObject.CSS_Style + "\" rel=\"stylesheet\" type=\"text/css\" />"); } allListWriter.WriteLine("</head>"); allListWriter.WriteLine("<body>"); allListWriter.WriteLine("<div id=\"container-inner\">"); string banner = "<div id=\"sbkHmw_BannerDiv\"><a alt=\"All Collections\" href=\"" + primaryWebServerUrl + "\" style=\"padding-bottom:0px;margin-bottom:0px\"><img id=\"mainBanner\" src=\"" + primaryWebServerUrl + "design/aggregations/all/images/banners/coll.jpg\" alt=\"\" /></a></div>"; Display_Header(allListWriter, defaultSkinObject, banner); allListWriter.WriteLine("<br /><br />This page is to provide static links to all resources in " + UI_ApplicationCache_Gateway.Settings.System.System_Abbreviation + ". <br />"); allListWriter.WriteLine("Click <a href=\"" + primaryWebServerUrl + "\">HERE</a> to return to main library. <br />"); allListWriter.WriteLine("<br />"); allListWriter.WriteLine("<br />"); allListWriter.WriteLine("SITE MAPS<br />"); allListWriter.WriteLine("<br />"); allListWriter.WriteLine("<a href=\"sitemap_collections.xml\">Map to all the collection home pages</a><br />"); if (sitemaps > 0) { for (int i = 1; i <= sitemaps; i++) { allListWriter.WriteLine("<a href=\"sitemap" + i + ".xml\">Site Map File " + i + "</a><br />"); } allListWriter.WriteLine("<br />"); allListWriter.WriteLine("<br />"); } else { allListWriter.WriteLine("NO SITE MAPS GENERATED!"); } Display_Footer(allListWriter, defaultSkinObject); allListWriter.WriteLine("</div>"); allListWriter.WriteLine("</body>"); allListWriter.WriteLine("</html>"); allListWriter.Flush(); allListWriter.Close(); // Create the list of all the RSS feeds try { Logger.AddNonError(InstanceName + ".....Building main rss feed page"); Console.WriteLine(InstanceName + @"Building main rss feed page"); SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building main rss feed page", String.Empty); StreamWriter writer = new StreamWriter(staticSobekcmDataLocation + "rss\\index.htm", false); writer.WriteLine("<!DOCTYPE html>"); writer.WriteLine("<html>"); writer.WriteLine("<head>"); writer.WriteLine(" <title>RSS Feeds for " + UI_ApplicationCache_Gateway.Settings.System.System_Abbreviation + "</title>"); writer.WriteLine(); writer.WriteLine(" <meta name=\"robots\" content=\"index, follow\">"); writer.WriteLine(" <link href=\"" + primaryWebServerUrl + "default/SobekCM.css\" rel=\"stylesheet\" type=\"text/css\" title=\"standard\" />"); if (defaultSkinObject.CSS_Style.Length > 0) { writer.WriteLine(" <link href=\"" + UI_ApplicationCache_Gateway.Settings.Servers.System_Base_URL + defaultSkinObject.CSS_Style + "\" rel=\"stylesheet\" type=\"text/css\" />"); } writer.WriteLine("</head>"); writer.WriteLine("<body>"); writer.WriteLine("<div id=\"container-inner\">"); string banner2 = "<div id=\"sbkHmw_BannerDiv\"><a alt=\"All Collections\" href=\"" + primaryWebServerUrl + "\" style=\"padding-bottom:0px;margin-bottom:0px\"><img id=\"mainBanner\" src=\"" + primaryWebServerUrl + "design/aggregations/all/images/banners/coll.jpg\" alt=\"\" /></a></div>"; Display_Header(writer, defaultSkinObject, banner2); writer.WriteLine("<div id=\"pagecontainer\">"); writer.WriteLine("<div class=\"ViewsBrowsesRow\">"); writer.WriteLine(" <ul class=\"sbk_FauxUpwardTabsList\">"); writer.WriteLine(" <li><a href=\"" + primaryWebServerUrl + "\">" + UI_ApplicationCache_Gateway.Settings.System.System_Abbreviation + " HOME</a></li>"); writer.WriteLine(" <li class=\"current\">RSS FEEDS</li>"); writer.WriteLine(" </ul>"); writer.WriteLine("</div>"); writer.WriteLine(); writer.WriteLine("<div class=\"SobekSearchPanel\">"); writer.WriteLine(" <h1>RSS Feeds for the " + UI_ApplicationCache_Gateway.Settings.System.System_Name + "</h1>"); writer.WriteLine("</div>"); writer.WriteLine(); writer.WriteLine("<div class=\"SobekHomeText\">"); writer.WriteLine("<table width=\"700\" border=\"0\" align=\"center\">"); writer.WriteLine(" <tr>"); writer.WriteLine(" <td>"); writer.WriteLine(" <br />"); writer.WriteLine(" This page provides links to RSS feeds for items within " + UI_ApplicationCache_Gateway.Settings.System.System_Name + ". The first group of RSS feeds below contains the last 20 items added to the collection. The second group of items contains links to every item in a collection. These rss feeds can grow quite lengthy and the load time is often non-trivial.<br />"); writer.WriteLine(" <br />"); writer.WriteLine(" In addition, the following three RSS feeds are provided:"); writer.WriteLine(" <blockquote>"); writer.WriteLine(" <img src=\"http://cdn.sobekrepository.org/images/misc/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\"> <a href=\"" + primaryWebServerUrl + "rss/all_rss.xml\">All items in " + UI_ApplicationCache_Gateway.Settings.System.System_Abbreviation + "</a><br />"); writer.WriteLine(" <img src=\"http://cdn.sobekrepository.org/images/misc/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\"> <a href=\"" + primaryWebServerUrl + "rss/all_short_rss.xml\">Most recently added items in " + UI_ApplicationCache_Gateway.Settings.System.System_Abbreviation + " (last 100)</a><br />"); writer.WriteLine(" </blockquote>"); writer.WriteLine(" RSS feeds are a way to keep up-to-date on new materials that are added to the Digital Collections. RSS feeds are written in XML and require a news reader to access.<br />"); writer.WriteLine(" <br />"); writer.WriteLine(" You can download and install a <a href=\"http://dmoz.org/Reference/Libraries/Library_and_Information_Science/Technical_Services/Cataloguing/Metadata/RDF/Applications/RSS/News_Readers/\">news reader</a>. Or, you can use a Web-based reader such as <a href=\"http://www.google.com/reader\">Google Reader </a>or <a href=\"http://my.yahoo.com/\">My Yahoo!</a>."); writer.WriteLine(" Follow the instructions in your reader to subscribe to the feed of your choice. You will usually need to copy and paste the feed URL into the reader. <br />"); writer.WriteLine(" <br />"); writer.WriteLine(" </td>"); writer.WriteLine(" </tr>"); writer.WriteLine("</table>"); writer.WriteLine("<table width=\"750\" border=\"0\" align=\"center\" cellpadding=\"1\" cellspacing=\"0\">"); writer.WriteLine(" <tr>"); writer.WriteLine(" <td bgcolor=\"#cccccc\">"); writer.WriteLine(" <table width=\"100%\" border=\"0\" align=\"center\" cellpadding=\"2\" cellspacing=\"0\">"); writer.WriteLine(" <tr>"); writer.WriteLine(" <td bgcolor=\"#f4f4f4\"><span class=\"groupname\"><span class=\"groupnamecaps\"> M</span>OST <span class=\"groupnamecaps\">R</span>ECENT <span class=\"groupnamecaps\">I</span>TEMS (BY COLLECTION)</span></td>"); writer.WriteLine(" </tr>"); writer.WriteLine(" </table>"); writer.WriteLine(" </td>"); writer.WriteLine(" </tr>"); writer.WriteLine("<table>"); writer.WriteLine("<div class=\"SobekHomeText\">"); writer.WriteLine("<table width=\"700\" border=\"0\" align=\"center\">"); writer.WriteLine(" <tr>"); writer.WriteLine(" <td>"); writer.WriteLine(recent_rss_link_builder.ToString()); writer.WriteLine(" <br />"); writer.WriteLine(" </td>"); writer.WriteLine(" </tr>"); writer.WriteLine("</table>"); writer.WriteLine("<table width=\"750\" border=\"0\" align=\"center\" cellpadding=\"1\" cellspacing=\"0\">"); writer.WriteLine(" <tr>"); writer.WriteLine(" <td bgcolor=\"#cccccc\">"); writer.WriteLine(" <table width=\"100%\" border=\"0\" align=\"center\" cellpadding=\"2\" cellspacing=\"0\">"); writer.WriteLine(" <tr>"); writer.WriteLine(" <td bgcolor=\"#f4f4f4\"><span class=\"groupname\"><span class=\"groupnamecaps\"> A</span>LL <span class=\"groupnamecaps\">I</span>TEMS (BY COLLECTION) </span></td>"); writer.WriteLine(" </tr>"); writer.WriteLine(" </table>"); writer.WriteLine(" </td>"); writer.WriteLine(" </tr>"); writer.WriteLine("<table>"); writer.WriteLine("<div class=\"SobekHomeText\">"); writer.WriteLine("<table width=\"700\" border=\"0\" align=\"center\">"); writer.WriteLine(" <tr>"); writer.WriteLine(" <td>"); writer.WriteLine(all_rss_link_builder.ToString()); writer.WriteLine(" <br />"); writer.WriteLine(" </td>"); writer.WriteLine(" </tr>"); writer.WriteLine("</table>"); writer.WriteLine("<br />"); writer.WriteLine("</div>"); writer.WriteLine("</div>"); Display_Footer(writer, defaultSkinObject); writer.WriteLine("</div>"); writer.WriteLine("</body>"); writer.WriteLine("</html>"); writer.Flush(); writer.Close(); } catch { Logger.AddError("ERROR BUILDING RSS INDEX.HTM FILE"); Console.WriteLine(@"Error building RSS index.htm file"); SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Error", "Error building the main RSS feed page", String.Empty); } return errors; }