/// <summary> Continuously execute the processes in a recurring background thread </summary>
        public void Execute_In_Background()
        {
            // Set the variable which will control background execution
            int time_between_polls = Engine_ApplicationCache_Gateway.Settings.Builder_Override_Seconds_Between_Polls;
            if (( time_between_polls < 0 ) || ( Engine_ApplicationCache_Gateway.Settings.Database_Connections.Count == 1 ))
                time_between_polls = Convert.ToInt32(Engine_ApplicationCache_Gateway.Settings.Builder.Seconds_Between_Polls);

            // Determine the new log name
            string log_name = "incoming_" + controllerStarted.Year + "_" + controllerStarted.Month.ToString().PadLeft(2, '0') + "_" + controllerStarted.Day.ToString().PadLeft(2, '0') + ".html";
            string local_log_name = Engine_ApplicationCache_Gateway.Settings.Local_Log_Directory + "\\" + log_name;
            if (String.IsNullOrEmpty(Engine_ApplicationCache_Gateway.Settings.Local_Log_Directory))
            {
                local_log_name = logFileDirectory + "\\" + log_name;
            }

            // Create the new log file
            LogFileXhtml preloader_logger = new LogFileXhtml(local_log_name, "SobekCM Incoming Packages Log", "UFDC_Builder.exe", true);

            // start with warnings on imagemagick and ghostscript not being installed
            if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.ImageMagick_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.ImageMagick_Executable)))
            {
                Console.WriteLine("WARNING: Could not find ImageMagick installed.  Some image processing will be unavailable.");
                preloader_logger.AddNonError("WARNING: Could not find ImageMagick installed.  Some image processing will be unavailable.");
            }

            if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.Ghostscript_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.Ghostscript_Executable)))
            {
                Console.WriteLine("WARNING: Could not find GhostScript installed.  Some PDF processing will be unavailable.");
                preloader_logger.AddNonError("WARNING: Could not find GhostScript installed.  Some PDF processing will be unavailable.");
            }

            // Set the time for the next feed building event to 10 minutes from now
            feedNextBuildTime = DateTime.Now.Add(new TimeSpan(0, 10, 0));

            // First, step through each active configuration and see if building is currently aborted
            // while doing very minimal processes
            aborted = false;
            Console.WriteLine("Checking for initial abort condition");
            preloader_logger.AddNonError("Checking for initial abort condition");
            string abort_message = String.Empty;
            Builder_Operation_Flag_Enum abort_flag = Builder_Operation_Flag_Enum.STANDARD_OPERATION;
            foreach (Database_Instance_Configuration dbConfig in instances)
            {
                if ((!aborted) && (dbConfig.Is_Active) && (dbConfig.Can_Abort))
                {
                    SobekCM_Database.Connection_String = dbConfig.Connection_String;
                    Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String;

                    // Check that this should not be skipped or aborted
                    Builder_Operation_Flag_Enum operationFlag = Abort_Database_Mechanism.Builder_Operation_Flag;
                    switch (operationFlag)
                    {
                        case Builder_Operation_Flag_Enum.ABORT_REQUESTED:
                        case Builder_Operation_Flag_Enum.ABORTING:
                            abort_message = "PREVIOUS ABORT flag found in " + dbConfig.Name;
                            abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
                            Console.WriteLine(abort_message);
                            preloader_logger.AddNonError(abort_message);
                            aborted = true;
                            Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
                            break;

                        case Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED:
                            abort_message = "PREVIOUS NO BUILDING flag found in " + dbConfig.Name;
                            Console.WriteLine(abort_message);
                            preloader_logger.AddNonError(abort_message);
                            aborted = true;
                            break;

                    }
                }
            }

            // If initially aborted, step through each instance and set a message
            if (aborted)
            {
                // Add messages in each active instance
                foreach (Database_Instance_Configuration dbConfig in instances)
                {
                    if (dbConfig.Is_Active)
                    {
                        Console.WriteLine("Setting previous abort flag message in " + dbConfig.Name);
                        preloader_logger.AddNonError("Setting previous abort flag message in " + dbConfig.Name);
                        SobekCM_Database.Connection_String = dbConfig.Connection_String;
                        Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String;
                        Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", abort_message, String.Empty);

                        // Save information about this last run
                        Library.Database.SobekCM_Database.Set_Setting("Builder Version", Engine_ApplicationCache_Gateway.Settings.Current_Builder_Version);
                        Library.Database.SobekCM_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString());
                        Library.Database.SobekCM_Database.Set_Setting("Builder Last Message", abort_message);

                        // Finally, set the builder flag appropriately
                        if (abort_flag != Builder_Operation_Flag_Enum.STANDARD_OPERATION)
                            Abort_Database_Mechanism.Builder_Operation_Flag = abort_flag;
                    }
                }

                // Do nothing else
                return;
            }

            // Build all the bulk loader objects
            List<Worker_BulkLoader> loaders = new List<Worker_BulkLoader>();
            bool activeInstanceFound = false;
            foreach (Database_Instance_Configuration dbConfig in instances)
            {
                if (!dbConfig.Is_Active)
                {
                    loaders.Add(null);
                    Console.WriteLine(dbConfig.Name + " is set to INACTIVE");
                    preloader_logger.AddNonError(dbConfig.Name + " is set to INACTIVE");
                }
                else
                {
                    activeInstanceFound = true;
                    SobekCM_Database.Connection_String = dbConfig.Connection_String;
                    Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String;

                    // At this point warn on mossing the Ghostscript and ImageMagick, to get it into each instances database logs
                    if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.ImageMagick_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.ImageMagick_Executable)))
                    {
                        Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find ImageMagick installed.  Some image processing will be unavailable.", String.Empty);
                    }

                    if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.Ghostscript_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.Ghostscript_Executable)))
                    {
                        Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find GhostScript installed.  Some PDF processing will be unavailable.", String.Empty);
                    }

                    Console.WriteLine(dbConfig.Name + " - Preparing to begin polling");
                    preloader_logger.AddNonError(dbConfig.Name + " - Preparing to begin polling");
                    Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Preparing to begin polling", String.Empty);

                    Worker_BulkLoader newLoader = new Worker_BulkLoader(preloader_logger, verbose, dbConfig, (instances.Count > 1), logFileDirectory);
                    loaders.Add(newLoader);
                }
            }

            // If no active instances, just exit
            if (!activeInstanceFound)
            {
                Console.WriteLine("No active databases in the config file");
                preloader_logger.AddError("No active databases in config file... Aborting");
                return;
            }

            // Set the maximum number of packages to process before moving to the next instance
            if (loaders.Count > 1)
                MultiInstance_Builder_Settings.Instance_Package_Limit = 100;

            bool firstRun = true;

            // Loop continually until the end hour is achieved
            do
            {
                // Is it time to build any RSS/XML feeds?
                if (DateTime.Compare(DateTime.Now, feedNextBuildTime) >= 0)
                {
                    feedNextBuildTime = DateTime.Now.Add(new TimeSpan(0, 10, 0));
                }

                bool skip_sleep = false;

                // Step through each instance
                for (int i = 0; i < instances.Count; i++)
                {
                    if (loaders[i] != null)
                    {
                        // Get the instance
                        Database_Instance_Configuration dbInstance = instances[i];

                        // Set the database connection strings
                        Engine_Database.Connection_String = dbInstance.Connection_String;
                        SobekCM_Database.Connection_String = dbInstance.Connection_String;
                        Library.Database.SobekCM_Database.Connection_String = dbInstance.Connection_String;

                        // Look for abort
                        if ((dbInstance.Can_Abort) && (CheckForAbort()))
                        {
                            aborted = true;
                            if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED)
                            {
                                abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
                                Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING;
                            }
                            break;
                        }

                        // Refresh all settings, etc..
                        loaders[i].Refresh_Settings_And_Item_List();

                        // Pull the abort/pause flag
                        Builder_Operation_Flag_Enum currentPauseFlag = Abort_Database_Mechanism.Builder_Operation_Flag;

                        // If not paused, run the prebuilder
                        if (currentPauseFlag != Builder_Operation_Flag_Enum.PAUSE_REQUESTED)
                        {
                            if (firstRun)
                            {

                                //    // Always build an endeca feed first (so it occurs once a day)
                                //    if (Engine_ApplicationCache_Gateway.Settings.Build_MARC_Feed_By_Default)
                                //    {
                                //        Create_Complete_MarcXML_Feed(false);
                                //    }
                                //}

                                // CLear the old logs
                                Console.WriteLine(dbInstance.Name + " - Expiring old log entries");
                                preloader_logger.AddNonError(dbInstance.Name + " - Expiring old log entries");
                                Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Expiring old log entries", String.Empty);
                                Library.Database.SobekCM_Database.Builder_Expire_Log_Entries(Engine_ApplicationCache_Gateway.Settings.Builder.Log_Expiration_Days);

                                // Rebuild all the static pages
                                Console.WriteLine(dbInstance.Name + " - Rebuilding all static pages");
                                preloader_logger.AddNonError(dbInstance.Name + " - Rebuilding all static pages");
                                long staticRebuildLogId = Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Rebuilding all static pages", String.Empty);

                                Static_Pages_Builder builder = new Static_Pages_Builder(Engine_ApplicationCache_Gateway.Settings.Servers.Application_Server_URL, Engine_ApplicationCache_Gateway.Settings.Servers.Static_Pages_Location, Engine_ApplicationCache_Gateway.URL_Portals.Default_Portal.Default_Web_Skin);
                                builder.Rebuild_All_Static_Pages(preloader_logger, false, Engine_ApplicationCache_Gateway.Settings.Local_Log_Directory, dbInstance.Name, staticRebuildLogId);

                            }

                            skip_sleep = skip_sleep || Run_BulkLoader(loaders[i], verbose);

                            // Look for abort
                            if ((!aborted) && (dbInstance.Can_Abort) && (CheckForAbort()))
                            {
                                aborted = true;
                                if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED)
                                {
                                    abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
                                    Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING;
                                }
                                break;
                            }
                        }
                        else
                        {
                            preloader_logger.AddNonError( dbInstance.Name +  " - Building paused");
                            Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building temporarily PAUSED", String.Empty);
                        }
                    }
                }

                if (aborted)
                    break;

                // No longer the first run
                firstRun = false;

                // Publish the log
                publish_log_file(local_log_name);

                // Sleep for correct number of milliseconds
                if ( !skip_sleep )
                    Thread.Sleep(1000 * time_between_polls);

            } while (DateTime.Now.Hour < BULK_LOADER_END_HOUR);

            // Do the final work for all of the different dbInstances
            if (!aborted)
            {
                for (int i = 0; i < instances.Count; i++)
                {
                    if (loaders[i] != null)
                    {
                        // Get the instance
                        Database_Instance_Configuration dbInstance = instances[i];

                        // Set the database flag
                        SobekCM_Database.Connection_String = dbInstance.Connection_String;
                        Library.Database.SobekCM_Database.Connection_String = dbInstance.Connection_String;

                        // Pull the abort/pause flag
                        Builder_Operation_Flag_Enum currentPauseFlag2 = Abort_Database_Mechanism.Builder_Operation_Flag;

                        // If not paused, run the prebuilder
                        if (currentPauseFlag2 != Builder_Operation_Flag_Enum.PAUSE_REQUESTED)
                        {
                            // Initiate the recreation of the links between metadata and collections
                            Library.Database.SobekCM_Database.Admin_Update_Cached_Aggregation_Metadata_Links();
                        }

                        // Clear the memory
                        loaders[i].ReleaseResources();
                    }
                }
            }
            else
            {
                // Mark the aborted in each instance
                foreach (Database_Instance_Configuration dbConfig in instances )
                {
                    if (dbConfig.Is_Active)
                    {
                        Console.WriteLine("Setting abort flag message in " + dbConfig.Name);
                        preloader_logger.AddNonError("Setting abort flag message in " + dbConfig.Name);
                        SobekCM_Database.Connection_String = dbConfig.Connection_String;
                        Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String;
                        Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building ABORTED per request from database key", String.Empty);

                        // Save information about this last run
                        Library.Database.SobekCM_Database.Set_Setting("Builder Version", Engine_ApplicationCache_Gateway.Settings.Current_Builder_Version);
                        Library.Database.SobekCM_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString());
                        Library.Database.SobekCM_Database.Set_Setting("Builder Last Message", "Building ABORTED per request");

                        // Finally, set the builder flag appropriately
                        if ( abort_flag == Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED )
                            Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
                    }
                }
            }

            // Publish the log
            publish_log_file(local_log_name);

            //// Initiate a solr/lucene index optimization since we are done loading for a while
            //if (DateTime.Now.Day % 2 == 0)
            //{
            //	if (Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL.Length > 0)
            //	{
            //		Console.WriteLine("Initiating Solr/Lucene document index optimization");
            //		Solr_Controller.Optimize_Document_Index(Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL);
            //	}
            //}
            //else
            //{
            //	if (Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL.Length > 0)
            //	{
            //		Console.WriteLine("Initiating Solr/Lucene page index optimization");
            //		Solr_Controller.Optimize_Page_Index(Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL);
            //	}
            //}
            //// Sleep for twenty minutes to end this (the index rebuild might take some time)
            //Thread.Sleep(1000 * 20 * 60);
        }
        /// <summary>Private method used to generate the error log for the packages</summary>
        /// <param name="Folder_Name"></param>
        /// <param name="ErrorMessage"></param>
        /// <param name="Resource_Folder"></param>
        /// <param name="MetsType"></param>
        /// <param name="BaseErrorMessage"></param>
        private void Create_Error_Log(string Resource_Folder, string Folder_Name, string ErrorMessage, string MetsType, string BaseErrorMessage)
        {
            // Split the message into seperate lines
            string[] errors = ErrorMessage.Split(new char[] { '\n' });

            long errorLogId = OnError(BaseErrorMessage, Folder_Name.Replace("_", ":"), MetsType, -1);

            try
            {
                LogFileXhtml errorLog = new LogFileXhtml(Resource_Folder + "\\" + Folder_Name + ".log.html", "Package Processing Log", "SobekCM Builder Errors");
                errorLog.New();
                errorLog.AddComplete("Error Log for " + Folder_Name + " processed at: " + DateTime.Now.ToString());
                errorLog.AddComplete("");

                foreach (string thisError in errors)
                {
                    errorLog.AddError(thisError);
                    OnError(thisError, Folder_Name.Replace("_", ":"), MetsType, errorLogId);
                }
                errorLog.Close();
            }
            catch
            {

            }
        }
        /// <summary> Rebuilds all static pages, including the RSS feeds and site maps </summary>
        /// <param name="Logger"> Log file to record progress </param>
        /// <param name="BuildAllCitationPages"> Flag indicates to build the individual static HTML pages for each digital resource </param>
        /// <param name="RssFeedLocation"> Location where the RSS feeds should be updated to </param>
        /// <param name="InstanceName"> Name of this instance </param>
        /// <param name="PrimaryLogId"> Log ID in the case this is the builder and it has been pre-logged </param>
        /// <returns> The number of encountered errors </returns>
        public int Rebuild_All_Static_Pages(LogFileXhtml Logger, bool BuildAllCitationPages, string RssFeedLocation, string InstanceName, long PrimaryLogId )
        {
            if (InstanceName.Length > 0)
                InstanceName = InstanceName + " - ";

            if (PrimaryLogId < 0)
            {
                Console.WriteLine("Rebuilding all static pages");
                Logger.AddNonError(InstanceName + "Rebuilding all static pages");
                PrimaryLogId = SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Rebuilding all static pages", String.Empty);
            }

            // Set the correct base directory
            if (staticSobekcmLocation.Length > 0)
                UI_ApplicationCache_Gateway.Settings.Servers.Base_Directory = staticSobekcmLocation;

            // Set the item for the current mode
            errors = 0;
            if (BuildAllCitationPages)
            {
                Console.WriteLine(InstanceName + "Rebuildig all citation pages");
                SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Rebuilding all item-level citation static pages", String.Empty);

                DataSet item_list_table = SobekCM_Database.Get_Item_List(false, null);

                // Get the item list and build the hashtable
                Item_Lookup_Object itemList = new Item_Lookup_Object();
                Engine_Database.Populate_Item_Lookup_Object(false, itemList, tracer);

                foreach (DataRow thisRow in item_list_table.Tables[0].Rows)
                {
                    //if (errors > 10000)
                    //{
                    //    logger.AddError("10000 errors encountered!");
                    //    Console.WriteLine("10000 errors encountered");

                    //    return errors;
                    //}
                    string bibid = thisRow["BibID"].ToString();
                    string vid = thisRow["VID"].ToString();
                    string itemDirectory = UI_ApplicationCache_Gateway.Settings.Servers.Image_Server_Network + bibid.Substring(0, 2) + "\\" + bibid.Substring(2, 2) + "\\" + bibid.Substring(4, 2) + "\\" + bibid.Substring(6, 2) + "\\" + bibid.Substring(8, 2) + "\\" + vid;
                    string staticDirectory = itemDirectory + "\\" + UI_ApplicationCache_Gateway.Settings.Resources.Backup_Files_Folder_Name;

                    // ********** TEMPORARY CLEAN UP ***********************//
                    // TODO: REMOVE THIS!
                    // *******************************************//
                    try
                    {
                        if (Directory.Exists(itemDirectory))
                        {
                            if (!Directory.Exists(staticDirectory))
                                Directory.CreateDirectory(staticDirectory);

                            if ( File.Exists(itemDirectory + "\\thumbs.db"))
                                File.Delete(itemDirectory + "\\thumbs.db");

                            if ( File.Exists(itemDirectory + "\\doc.xml"))
                                File.Delete(itemDirectory + "\\doc.xml");

                            if ( File.Exists(itemDirectory + "\\citation_mets.xml"))
                                File.Delete(itemDirectory + "\\citation_mets.xml");

                            if ( File.Exists(itemDirectory + "\\" + bibid + "_" + vid + ".html"))
                                File.Delete(itemDirectory + "\\" + bibid + "_" + vid + ".html");

                            if ( File.Exists(itemDirectory + "\\errors.xml"))
                                File.Delete(itemDirectory + "\\errors.xml");

                            if ( File.Exists(itemDirectory + "\\qc_error.html"))
                                File.Delete(itemDirectory + "\\qc_error.html");

                            string[] files = Directory.GetFileSystemEntries(itemDirectory, "*.job");
                            foreach( string thisFile in files )
                                File.Delete(thisFile);

                            files = Directory.GetFileSystemEntries(itemDirectory, "*.bridge*");
                            foreach (string thisFile in files)
                                File.Delete(thisFile);

                            files = Directory.GetFileSystemEntries(itemDirectory, "*.qc.jpg");
                            foreach (string thisFile in files)
                                File.Delete(thisFile);

                            files = Directory.GetFileSystemEntries(itemDirectory, "*_qc.jpg");
                            foreach (string thisFile in files)
                                File.Delete(thisFile);

                            files = Directory.GetFileSystemEntries(itemDirectory, "*_INGEST_xml.txt");
                            foreach (string thisFile in files)
                                File.Delete(thisFile);

                            files = Directory.GetFileSystemEntries(itemDirectory, "*_INGEST.xml");
                            if (files.Length > 0)
                            {
                                if (!Directory.Exists(itemDirectory + "\\fda_files"))
                                    Directory.CreateDirectory(itemDirectory + "\\fda_files");

                                foreach (string thisFile in files)
                                {
                                    string filename = Path.GetFileName(thisFile);
                                    File.Move(thisFile, itemDirectory + "\\fda_files\\" + filename);
                                }
                            }

                            if (File.Exists(itemDirectory + "\\original.mets.xml"))
                                File.Move(itemDirectory + "\\original.mets.xml", staticDirectory + "\\original.mets.xml");

                            files = Directory.GetFileSystemEntries(itemDirectory, "recd_*.mets.xml");
                            foreach (string thisFile in files)
                            {
                                string filename = Path.GetFileName(thisFile);
                                File.Move(thisFile, staticDirectory+ "\\" + filename);
                            }

                            files = Directory.GetFileSystemEntries(itemDirectory, "*.mets.bak");
                            foreach (string thisFile in files)
                            {
                                string filename = Path.GetFileName(thisFile);
                                File.Move(thisFile, staticDirectory + "\\" + filename);
                            }
                        }
                    }
                    catch
                    {
                        Console.WriteLine(InstanceName + "....Error cleaning up folder " + bibid + ":" + vid);
                        Logger.AddError("....Error cleaning up folder " + bibid + ":" + vid);
                    }
                    // ************* END TEMPORARY CLEAN UP ******************//

                    try
                    {
                        if (!Directory.Exists(staticDirectory))
                            Directory.CreateDirectory(staticDirectory);

                        string static_file = staticDirectory + "\\" + bibid + "_" + vid + ".html";

                        if (Create_Item_Citation_HTML(bibid, vid, static_file, itemDirectory, itemList))
                        {
                            // Also copy to the static page location server
                            string web_server_directory = UI_ApplicationCache_Gateway.Settings.Servers.Static_Pages_Location + bibid.Substring(0, 2) + "\\" + bibid.Substring(2, 2) + "\\" + bibid.Substring(4, 2) + "\\" + bibid.Substring(6, 2) + "\\" + bibid.Substring(8, 2) + "\\" + vid;
                            if (!Directory.Exists(web_server_directory))
                                Directory.CreateDirectory(web_server_directory);

                            string web_server_file_version = web_server_directory + "\\" + bibid + "_" + vid + ".html";
                            File.Copy(static_file, web_server_file_version, true);

                        }
                        else
                        {
                            errors++;
                            Console.WriteLine(InstanceName + "....Error creating citation file for: " + bibid + ":" + vid);
                            Logger.AddError("....Error creating citation file for: " + bibid + ":" + vid);
                        }
                    }
                    catch (Exception ee)
                    {
                        Console.WriteLine(InstanceName + "....Exception caught while creating citation file for: " + bibid + ":" + vid);
                        Logger.AddError("....Exception caught while creating citation file for: " + bibid + ":" + vid);

                        Console.WriteLine("........." + ee.Message);
                        Logger.AddError("........." + ee.Message);
                        Logger.AddError("........." + ee.StackTrace);
                    }
                }

                Console.WriteLine(InstanceName + "Done rebuilding all item-level citation static pages");
                Logger.AddNonError("Done rebuilding all item-level citation static pages");
                SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Done rebuilding all item-level citation static pages", String.Empty);

            }

            // Set the mode away from the display item mode
            currentMode.Mode = Display_Mode_Enum.Aggregation;
            currentMode.Aggregation_Type = Aggregation_Type_Enum.Home;
            currentMode.Skin = defaultSkin;

            // Get the default web skin
            Web_Skin_Object defaultSkinObject = assistant.Get_HTML_Skin(currentMode, Engine_ApplicationCache_Gateway.Web_Skin_Collection, false, null);

            // Get the list of all collections
            DataTable allCollections = SobekCM_Database.Get_Codes_Item_Aggregations( null);
            DataView collectionView = new DataView(allCollections) {Sort = "Name ASC"};

            // Build the basic site map first
            StreamWriter sitemap_writer = new StreamWriter(staticSobekcmDataLocation + "sitemap_collections.xml", false);
            sitemap_writer.WriteLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
            sitemap_writer.WriteLine("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
            sitemap_writer.WriteLine("\t<url>");
            sitemap_writer.WriteLine("\t\t<loc>" + primaryWebServerUrl + "</loc>");
            sitemap_writer.WriteLine("\t</url>");

            // Prepare to build all the links to static pages
            StringBuilder static_browse_links = new StringBuilder();
            StringBuilder recent_rss_link_builder = new StringBuilder();
            StringBuilder all_rss_link_builder = new StringBuilder();
            int col = 2;
            DataSet items;
            List<string> processed_codes = new List<string>();
            foreach (DataRowView thisCollectionView in collectionView)
            {
                // Clear the tracer
                tracer.Clear();

                if (!Convert.ToBoolean(thisCollectionView.Row["Hidden"]))
                {
                    // Build the static links pages
                    string code = thisCollectionView.Row["Code"].ToString().ToLower();
                    if ((!processed_codes.Contains(code)) && (code != "all"))
                    {
                        processed_codes.Add(code);

                        // Add this to the sitemap
                        sitemap_writer.WriteLine("\t<url>");
                        sitemap_writer.WriteLine("\t\t<loc>" + primaryWebServerUrl + code + "</loc>");
                        sitemap_writer.WriteLine("\t</url>");

                        Logger.AddNonError(InstanceName + ".....Building static links page for " + code);
                        Console.WriteLine(InstanceName + @"Building static links page for {0}", code);
                        SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building static links page for " + code, String.Empty);

                        //Get the list of items for this collection
                        items = Engine_Database.Simple_Item_List(code, tracer);

                        // Get the item aggregation object
                        Item_Aggregation aggregation = SobekEngineClient.Aggregations.Get_Aggregation(code.ToLower(), UI_ApplicationCache_Gateway.Settings.System.Default_UI_Language, UI_ApplicationCache_Gateway.Settings.System.Default_UI_Language, tracer);

                        // Build the static browse pages
                        if (Build_All_Browse(aggregation, items))
                        {
                            static_browse_links.Append("<td><a href=\"" + code + "_all.html\">" + code + "</a></td>" + Environment.NewLine);
                            col++;
                        }

                        if (col > 5)
                        {
                            static_browse_links.Append("</tr>" + Environment.NewLine + "<tr>");
                            col = 1;
                        }

                        // Build the RSS feeds
                        Logger.AddNonError(InstanceName + ".....Building RSS feeds for " + code);
                        Console.WriteLine(InstanceName + @"Building RSS feeds for {0}", code);
                        SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building RSS feeds for " + code, String.Empty);

                        if (Create_RSS_Feed(code, staticSobekcmDataLocation + "rss\\", thisCollectionView.Row["Name"].ToString(), items))
                        {
                            recent_rss_link_builder.Append("<img src=\"http://cdn.sobekrepository.org/images/misc/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\">&nbsp;<a href=\"" + primaryWebServerUrl + "rss/" + code + "_short_rss.xml\">" + thisCollectionView.Row["Name"] + "</a><br />" + Environment.NewLine);
                            all_rss_link_builder.Append("<img src=\"http://cdn.sobekrepository.org/images/misc/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\">&nbsp;<a href=\"" + primaryWebServerUrl + "rss/" + code + "_rss.xml\">" + thisCollectionView.Row["Name"] + "</a><br />" + Environment.NewLine);
                        }
                    }
                }
            }

            // Finish out the collection sitemap
            sitemap_writer.WriteLine("</urlset>");
            sitemap_writer.Flush();
            sitemap_writer.Close();

            items = Engine_Database.Simple_Item_List(String.Empty, tracer);
            Logger.AddNonError(InstanceName + ".....Building static links page for ALL ITEMS");
            Console.WriteLine(InstanceName + @"Building static links page for ALL ITEMS");
            SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building static links page for ALL ITEMS", String.Empty);

            Item_Aggregation allAggregation = SobekEngineClient.Aggregations.Get_Aggregation("all", UI_ApplicationCache_Gateway.Settings.System.Default_UI_Language, UI_ApplicationCache_Gateway.Settings.System.Default_UI_Language, tracer);

            Build_All_Browse(allAggregation, items);

            Console.WriteLine(InstanceName + @"Building RSS feeds ALL ITEMS");
            Logger.AddNonError(InstanceName + ".....Building RSS feeds for ALL ITEMS");
            SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building RSS feeds for ALL ITEMS", String.Empty);

            Create_RSS_Feed("all", staticSobekcmDataLocation + "rss\\", "All " + UI_ApplicationCache_Gateway.Settings.System.System_Abbreviation + " Items", items);

            // Build the site maps
            Logger.AddNonError(InstanceName + ".....Building site maps");
            Console.WriteLine(InstanceName + @"Building site maps");
            SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building site maps", String.Empty);

            int sitemaps = Build_Site_Maps();

            // Output the main browse and rss links pages
            Logger.AddNonError(InstanceName + "....Building main sitemaps and links page");
            Console.WriteLine(InstanceName + @"Building main sitemaps and links page");
            SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building main sitemaps and links page", String.Empty);

            StreamWriter allListWriter = new StreamWriter(staticSobekcmDataLocation + "index.html", false);
            allListWriter.WriteLine("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">");
            allListWriter.WriteLine("<html xmlns=\"http://www.w3.org/1999/xhtml\" >");
            allListWriter.WriteLine("<head>");
            allListWriter.WriteLine("  <title>" + UI_ApplicationCache_Gateway.Settings.System.System_Name + " Site Map Links</title>");
            allListWriter.WriteLine();
            allListWriter.WriteLine("  <meta name=\"robots\" content=\"index, follow\">");
            allListWriter.WriteLine("  <link href=\"" + primaryWebServerUrl + "default/SobekCM.css\" rel=\"stylesheet\" type=\"text/css\" title=\"standard\" />");
            if (defaultSkinObject.CSS_Style.Length > 0)
            {
                allListWriter.WriteLine("  <link href=\"" + UI_ApplicationCache_Gateway.Settings.Servers.System_Base_URL + defaultSkinObject.CSS_Style + "\" rel=\"stylesheet\" type=\"text/css\" />");
            }

            allListWriter.WriteLine("</head>");
            allListWriter.WriteLine("<body>");

            allListWriter.WriteLine("<div id=\"container-inner\">");

            string banner = "<div id=\"sbkHmw_BannerDiv\"><a alt=\"All Collections\" href=\"" + primaryWebServerUrl + "\" style=\"padding-bottom:0px;margin-bottom:0px\"><img id=\"mainBanner\" src=\"" + primaryWebServerUrl + "design/aggregations/all/images/banners/coll.jpg\" alt=\"\" /></a></div>";
            Display_Header(allListWriter, defaultSkinObject, banner);

            allListWriter.WriteLine("<br /><br />This page is to provide static links to all resources in " + UI_ApplicationCache_Gateway.Settings.System.System_Abbreviation + ". <br />");
            allListWriter.WriteLine("Click <a href=\"" + primaryWebServerUrl + "\">HERE</a> to return to main library. <br />");
            allListWriter.WriteLine("<br />");
            allListWriter.WriteLine("<br />");
            allListWriter.WriteLine("SITE MAPS<br />");
            allListWriter.WriteLine("<br />");
            allListWriter.WriteLine("<a href=\"sitemap_collections.xml\">Map to all the collection home pages</a><br />");
            if (sitemaps > 0)
            {
                for (int i = 1; i <= sitemaps; i++)
                {
                    allListWriter.WriteLine("<a href=\"sitemap" + i + ".xml\">Site Map File " + i + "</a><br />");
                }
                allListWriter.WriteLine("<br />");
                allListWriter.WriteLine("<br />");
            }
            else
            {
                allListWriter.WriteLine("NO SITE MAPS GENERATED!");
            }

            Display_Footer(allListWriter, defaultSkinObject);
            allListWriter.WriteLine("</div>");
            allListWriter.WriteLine("</body>");
            allListWriter.WriteLine("</html>");
            allListWriter.Flush();
            allListWriter.Close();

            // Create the list of all the RSS feeds
            try
            {
                Logger.AddNonError(InstanceName + ".....Building main rss feed page");
                Console.WriteLine(InstanceName + @"Building main rss feed page");
                SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building main rss feed page", String.Empty);

                StreamWriter writer = new StreamWriter(staticSobekcmDataLocation + "rss\\index.htm", false);
                writer.WriteLine("<!DOCTYPE html>");
                writer.WriteLine("<html>");
                writer.WriteLine("<head>");
                writer.WriteLine("  <title>RSS Feeds for " + UI_ApplicationCache_Gateway.Settings.System.System_Abbreviation + "</title>");
                writer.WriteLine();
                writer.WriteLine("  <meta name=\"robots\" content=\"index, follow\">");
                writer.WriteLine("  <link href=\"" + primaryWebServerUrl + "default/SobekCM.css\" rel=\"stylesheet\" type=\"text/css\" title=\"standard\" />");
                if (defaultSkinObject.CSS_Style.Length > 0)
                {
                    writer.WriteLine("  <link href=\"" + UI_ApplicationCache_Gateway.Settings.Servers.System_Base_URL + defaultSkinObject.CSS_Style + "\" rel=\"stylesheet\" type=\"text/css\" />");
                }
                writer.WriteLine("</head>");
                writer.WriteLine("<body>");

                writer.WriteLine("<div id=\"container-inner\">");

                string banner2 = "<div id=\"sbkHmw_BannerDiv\"><a alt=\"All Collections\" href=\"" + primaryWebServerUrl + "\" style=\"padding-bottom:0px;margin-bottom:0px\"><img id=\"mainBanner\" src=\"" + primaryWebServerUrl + "design/aggregations/all/images/banners/coll.jpg\" alt=\"\" /></a></div>";
                Display_Header(writer, defaultSkinObject, banner2);

                writer.WriteLine("<div id=\"pagecontainer\">");

                writer.WriteLine("<div class=\"ViewsBrowsesRow\">");
                writer.WriteLine("  <ul class=\"sbk_FauxUpwardTabsList\">");
                writer.WriteLine("    <li><a href=\"" + primaryWebServerUrl + "\">" + UI_ApplicationCache_Gateway.Settings.System.System_Abbreviation + " HOME</a></li>");
                writer.WriteLine("    <li class=\"current\">RSS FEEDS</li>");
                writer.WriteLine("  </ul>");
                writer.WriteLine("</div>");
                writer.WriteLine();
                writer.WriteLine("<div class=\"SobekSearchPanel\">");
                writer.WriteLine("  <h1>RSS Feeds for the " + UI_ApplicationCache_Gateway.Settings.System.System_Name + "</h1>");
                writer.WriteLine("</div>");
                writer.WriteLine();

                writer.WriteLine("<div class=\"SobekHomeText\">");
                writer.WriteLine("<table width=\"700\" border=\"0\" align=\"center\">");
                writer.WriteLine("  <tr>");
                writer.WriteLine("    <td>");
                writer.WriteLine("      <br />");
                writer.WriteLine("      This page provides links to RSS feeds for items within " + UI_ApplicationCache_Gateway.Settings.System.System_Name + ".  The first group of RSS feeds below contains the last 20 items added to the collection.  The second group of items contains links to every item in a collection.  These rss feeds can grow quite lengthy and the load time is often non-trivial.<br />");
                writer.WriteLine("      <br />");
                writer.WriteLine("      In addition, the following three RSS feeds are provided:");
                writer.WriteLine("      <blockquote>");
                writer.WriteLine("        <img src=\"http://cdn.sobekrepository.org/images/misc/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\">&nbsp;<a href=\"" + primaryWebServerUrl + "rss/all_rss.xml\">All items in " + UI_ApplicationCache_Gateway.Settings.System.System_Abbreviation + "</a><br />");
                writer.WriteLine("        <img src=\"http://cdn.sobekrepository.org/images/misc/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\">&nbsp;<a href=\"" + primaryWebServerUrl + "rss/all_short_rss.xml\">Most recently added items in " + UI_ApplicationCache_Gateway.Settings.System.System_Abbreviation + " (last 100)</a><br />");
                writer.WriteLine("      </blockquote>");
                writer.WriteLine("      RSS feeds	are a way to keep up-to-date on new materials that are added to the Digital Collections. RSS feeds are written in XML    and require a news reader to access.<br />");
                writer.WriteLine("      <br />");
                writer.WriteLine("      You can download and install a <a href=\"http://dmoz.org/Reference/Libraries/Library_and_Information_Science/Technical_Services/Cataloguing/Metadata/RDF/Applications/RSS/News_Readers/\">news reader</a>.  Or, you can use a Web-based reader such as <a href=\"http://www.google.com/reader\">Google Reader </a>or <a href=\"http://my.yahoo.com/\">My Yahoo!</a>.");
                writer.WriteLine("      Follow the instructions in your reader to subscribe to the feed of   your choice. You will usually need to copy and paste the feed URL into the reader. <br />");
                writer.WriteLine("      <br />");
                writer.WriteLine("    </td>");
                writer.WriteLine("  </tr>");
                writer.WriteLine("</table>");

                writer.WriteLine("<table width=\"750\" border=\"0\" align=\"center\" cellpadding=\"1\" cellspacing=\"0\">");
                writer.WriteLine("	<tr>");
                writer.WriteLine("    <td bgcolor=\"#cccccc\">");
                writer.WriteLine("      <table width=\"100%\" border=\"0\" align=\"center\" cellpadding=\"2\" cellspacing=\"0\">");
                writer.WriteLine("		  <tr>");
                writer.WriteLine("          <td bgcolor=\"#f4f4f4\"><span class=\"groupname\"><span class=\"groupnamecaps\"> &nbsp; M</span>OST <span class=\"groupnamecaps\">R</span>ECENT <span class=\"groupnamecaps\">I</span>TEMS (BY COLLECTION)</span></td>");
                writer.WriteLine("        </tr>");
                writer.WriteLine("      </table>");
                writer.WriteLine("    </td>");
                writer.WriteLine("  </tr>");
                writer.WriteLine("<table>");

                writer.WriteLine("<div class=\"SobekHomeText\">");
                writer.WriteLine("<table width=\"700\" border=\"0\" align=\"center\">");
                writer.WriteLine("  <tr>");
                writer.WriteLine("    <td>");
                writer.WriteLine(recent_rss_link_builder.ToString());
                writer.WriteLine("      <br />");
                writer.WriteLine("    </td>");
                writer.WriteLine("  </tr>");
                writer.WriteLine("</table>");

                writer.WriteLine("<table width=\"750\" border=\"0\" align=\"center\" cellpadding=\"1\" cellspacing=\"0\">");
                writer.WriteLine("	<tr>");
                writer.WriteLine("    <td bgcolor=\"#cccccc\">");
                writer.WriteLine("      <table width=\"100%\" border=\"0\" align=\"center\" cellpadding=\"2\" cellspacing=\"0\">");
                writer.WriteLine("		  <tr>");
                writer.WriteLine("          <td bgcolor=\"#f4f4f4\"><span class=\"groupname\"><span class=\"groupnamecaps\"> &nbsp; A</span>LL <span class=\"groupnamecaps\">I</span>TEMS (BY COLLECTION) </span></td>");
                writer.WriteLine("        </tr>");
                writer.WriteLine("      </table>");
                writer.WriteLine("    </td>");
                writer.WriteLine("  </tr>");
                writer.WriteLine("<table>");

                writer.WriteLine("<div class=\"SobekHomeText\">");
                writer.WriteLine("<table width=\"700\" border=\"0\" align=\"center\">");
                writer.WriteLine("  <tr>");
                writer.WriteLine("    <td>");
                writer.WriteLine(all_rss_link_builder.ToString());
                writer.WriteLine("      <br />");
                writer.WriteLine("    </td>");
                writer.WriteLine("  </tr>");
                writer.WriteLine("</table>");

                writer.WriteLine("<br />");

                writer.WriteLine("</div>");
                writer.WriteLine("</div>");
                Display_Footer(writer, defaultSkinObject);
                writer.WriteLine("</div>");
                writer.WriteLine("</body>");
                writer.WriteLine("</html>");

                writer.Flush();
                writer.Close();
            }
            catch
            {
                Logger.AddError("ERROR BUILDING RSS INDEX.HTM FILE");
                Console.WriteLine(@"Error building RSS index.htm file");
                SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Error", "Error building the main RSS feed page", String.Empty);

            }

            return errors;
        }