/// <summary> Constructor for a new instance of the Worker_BulkLoader class </summary>
/// <param name="Logger"> Log file object for logging progress </param>
/// <param name="Verbose"> Flag indicates if the builder is in verbose mode, where it should log alot more information </param>
/// <param name="InstanceName"> Name of this instance, likely from the Builder config </param>
/// <param name="Can_Abort"></param>
public Worker_BulkLoader(LogFileXHTML Logger, bool Verbose, string InstanceName, bool Can_Abort )
{
// Save the log file and verbose flag
logger = Logger;
verbose = Verbose;
instanceName = InstanceName;
canAbort = Can_Abort;
multiInstanceBuilder = SobekCM_Library_Settings.Database_Connections.Count > 1;
Add_NonError_To_Log("Worker_BulkLoader.Constructor: Start", verbose, String.Empty, String.Empty, -1);
// Create the METS validation objects
thisMetsValidator = new SobekCM_METS_Validator( String.Empty );
metsSchemeValidator = new METS_Validator_Object(false);
Add_NonError_To_Log("Worker_BulkLoader.Constructor: Created Validators", verbose, String.Empty, String.Empty, -1);
// Create new list of collections to build
aggregations_to_refresh = new List<string>();
items_by_aggregation = new Dictionary<string, List<string>>();
// get all the info
Refresh_Settings_And_Item_List();
// Ensure there is SOME instance name
if (instanceName.Length == 0)
instanceName = SobekCM_Library_Settings.System_Name;
// Create the new statics page builder
staticBuilder = new Static_Pages_Builder(SobekCM_Library_Settings.Application_Server_URL, SobekCM_Library_Settings.Static_Pages_Location, SobekCM_Library_Settings.Application_Server_Network);
Add_NonError_To_Log("Worker_BulkLoader.Constructor: Created Static Pages Builder", verbose, String.Empty, String.Empty, -1);
// Get the list of collection codes
codeManager = new Aggregation_Code_Manager();
SobekCM_Database.Populate_Code_Manager(codeManager, null);
Add_NonError_To_Log("Worker_BulkLoader.Constructor: Populated code manager with " + codeManager.All_Aggregations.Count + " aggregations.", verbose, String.Empty, String.Empty, -1);
// Set some defaults
aborted = false;
// Get the executable path/file for ghostscript and imagemagick
ghostscript_executable = SobekCM_Library_Settings.Ghostscript_Executable;
imagemagick_executable = SobekCM_Library_Settings.ImageMagick_Executable;
Add_NonError_To_Log("Worker_BulkLoader.Constructor: Done", verbose, String.Empty, String.Empty, -1);
}
/// <summary> Rebuilds all static pages, including the RSS feeds and site maps </summary>
/// <param name="Logger"> Log file to record progress </param>
/// <param name="BuildAllCitationPages"> Flag indicates to build the individual static HTML pages for each digital resource </param>
/// <param name="RssFeedLocation"> Location where the RSS feeds should be updated to </param>
/// <param name="InstanceName"> Name of this instance </param>
/// <param name="PrimaryLogId"> Log ID in the case this is the builder and it has been pre-logged </param>
/// <returns> The number of encountered errors </returns>
public int Rebuild_All_Static_Pages(LogFileXHTML Logger, bool BuildAllCitationPages, string RssFeedLocation, string InstanceName, long PrimaryLogId )
{
if (InstanceName.Length > 0)
InstanceName = InstanceName + " - ";
if (PrimaryLogId < 0)
{
Console.WriteLine("Rebuilding all static pages");
Logger.AddNonError(InstanceName + "Rebuilding all static pages");
PrimaryLogId = SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Rebuilding all static pages", String.Empty);
}
// Set the correct base directory
if (staticSobekcmLocation.Length > 0)
SobekCM_Library_Settings.Base_Directory = staticSobekcmLocation;
// Set the item for the current mode
errors = 0;
if (BuildAllCitationPages)
{
Console.WriteLine(InstanceName + "Rebuildig all citation pages");
SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Rebuilding all item-level citation static pages", String.Empty);
DataSet item_list_table = SobekCM_Database.Get_Item_List(false, null);
// Get the item list and build the hashtable
Item_Lookup_Object itemList = new Item_Lookup_Object();
SobekCM_Database.Populate_Item_Lookup_Object(false, itemList, tracer);
foreach (DataRow thisRow in item_list_table.Tables[0].Rows)
{
//if (errors > 10000)
//{
// logger.AddError("10000 errors encountered!");
// Console.WriteLine("10000 errors encountered");
// return errors;
//}
string bibid = thisRow["BibID"].ToString();
string vid = thisRow["VID"].ToString();
string itemDirectory = SobekCM_Library_Settings.Image_Server_Network + bibid.Substring(0, 2) + "\\" + bibid.Substring(2, 2) + "\\" + bibid.Substring(4, 2) + "\\" + bibid.Substring(6, 2) + "\\" + bibid.Substring(8, 2) + "\\" + vid;
string staticDirectory = itemDirectory + "\\" + SobekCM_Library_Settings.BACKUP_FILES_FOLDER_NAME;
// ********** TEMPORARY CLEAN UP ***********************//
/// TODO: REMOVE THIS!
// *******************************************//
try
{
if (Directory.Exists(itemDirectory))
{
if (!Directory.Exists(staticDirectory))
Directory.CreateDirectory(staticDirectory);
if ( File.Exists(itemDirectory + "\\thumbs.db"))
File.Delete(itemDirectory + "\\thumbs.db");
if ( File.Exists(itemDirectory + "\\doc.xml"))
File.Delete(itemDirectory + "\\doc.xml");
if ( File.Exists(itemDirectory + "\\citation_mets.xml"))
File.Delete(itemDirectory + "\\citation_mets.xml");
if ( File.Exists(itemDirectory + "\\" + bibid + "_" + vid + ".html"))
File.Delete(itemDirectory + "\\" + bibid + "_" + vid + ".html");
if ( File.Exists(itemDirectory + "\\errors.xml"))
File.Delete(itemDirectory + "\\errors.xml");
if ( File.Exists(itemDirectory + "\\qc_error.html"))
File.Delete(itemDirectory + "\\qc_error.html");
string[] files = Directory.GetFileSystemEntries(itemDirectory, "*.job");
foreach( string thisFile in files )
File.Delete(thisFile);
files = Directory.GetFileSystemEntries(itemDirectory, "*.bridge*");
foreach (string thisFile in files)
File.Delete(thisFile);
files = Directory.GetFileSystemEntries(itemDirectory, "*.qc.jpg");
foreach (string thisFile in files)
File.Delete(thisFile);
files = Directory.GetFileSystemEntries(itemDirectory, "*_qc.jpg");
foreach (string thisFile in files)
File.Delete(thisFile);
files = Directory.GetFileSystemEntries(itemDirectory, "*_INGEST_xml.txt");
foreach (string thisFile in files)
File.Delete(thisFile);
files = Directory.GetFileSystemEntries(itemDirectory, "*_INGEST.xml");
if (files.Length > 0)
{
if (!Directory.Exists(itemDirectory + "\\fda_files"))
Directory.CreateDirectory(itemDirectory + "\\fda_files");
foreach (string thisFile in files)
{
string filename = Path.GetFileName(thisFile);
File.Move(thisFile, itemDirectory + "\\fda_files\\" + filename);
}
}
if (File.Exists(itemDirectory + "\\original.mets.xml"))
File.Move(itemDirectory + "\\original.mets.xml", staticDirectory + "\\original.mets.xml");
files = Directory.GetFileSystemEntries(itemDirectory, "recd_*.mets.xml");
foreach (string thisFile in files)
{
string filename = Path.GetFileName(thisFile);
File.Move(thisFile, staticDirectory+ "\\" + filename);
}
files = Directory.GetFileSystemEntries(itemDirectory, "*.mets.bak");
foreach (string thisFile in files)
{
string filename = Path.GetFileName(thisFile);
File.Move(thisFile, staticDirectory + "\\" + filename);
}
}
}
catch
{
Console.WriteLine(InstanceName + "....Error cleaning up folder " + bibid + ":" + vid);
Logger.AddError("....Error cleaning up folder " + bibid + ":" + vid);
}
// ************* END TEMPORARY CLEAN UP ******************//
try
{
if (!Directory.Exists(staticDirectory))
Directory.CreateDirectory(staticDirectory);
string static_file = staticDirectory + "\\" + bibid + "_" + vid + ".html";
if (Create_Item_Citation_HTML(bibid, vid, static_file, itemDirectory, itemList))
{
// Also copy to the static page location server
string web_server_directory = SobekCM_Library_Settings.Static_Pages_Location + bibid.Substring(0, 2) + "\\" + bibid.Substring(2, 2) + "\\" + bibid.Substring(4, 2) + "\\" + bibid.Substring(6, 2) + "\\" + bibid.Substring(8, 2) + "\\" + vid;
if (!Directory.Exists(web_server_directory))
Directory.CreateDirectory(web_server_directory);
string web_server_file_version = web_server_directory + "\\" + bibid + "_" + vid + ".html";
File.Copy(static_file, web_server_file_version, true);
}
else
{
errors++;
Console.WriteLine(InstanceName + "....Error creating citation file for: " + bibid + ":" + vid);
Logger.AddError("....Error creating citation file for: " + bibid + ":" + vid);
}
}
catch (Exception ee)
{
Console.WriteLine(InstanceName + "....Exception caught while creating citation file for: " + bibid + ":" + vid);
Logger.AddError("....Exception caught while creating citation file for: " + bibid + ":" + vid);
Console.WriteLine("........." + ee.Message);
Logger.AddError("........." + ee.Message);
Logger.AddError("........." + ee.StackTrace);
}
}
Console.WriteLine(InstanceName + "Done rebuilding all item-level citation static pages");
Logger.AddNonError("Done rebuilding all item-level citation static pages");
SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Done rebuilding all item-level citation static pages", String.Empty);
}
// Set the mode away from the display item mode
currentMode.Mode = Display_Mode_Enum.Aggregation;
currentMode.Aggregation_Type = Aggregation_Type_Enum.Home;
currentMode.Skin = defaultSkin;
// Get the default web skin
SobekCM_Skin_Object defaultSkinObject = skinsCollection[defaultSkin] ?? assistant.Get_HTML_Skin(currentMode, skinsCollection, false, null);
// Get the list of all collections
DataTable allCollections = SobekCM_Database.Get_Codes_Item_Aggregations( null);
DataView collectionView = new DataView(allCollections) {Sort = "Name ASC"};
// Build the basic site map first
StreamWriter sitemap_writer = new StreamWriter(staticSobekcmDataLocation + "sitemap_collections.xml", false);
sitemap_writer.WriteLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
sitemap_writer.WriteLine("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
sitemap_writer.WriteLine("\t<url>");
sitemap_writer.WriteLine("\t\t<loc>" + primaryWebServerUrl + "</loc>");
sitemap_writer.WriteLine("\t</url>");
// Prepare to build all the links to static pages
StringBuilder static_browse_links = new StringBuilder();
StringBuilder recent_rss_link_builder = new StringBuilder();
StringBuilder all_rss_link_builder = new StringBuilder();
int col = 2;
DataSet items;
List<string> processed_codes = new List<string>();
foreach (DataRowView thisCollectionView in collectionView)
{
// Clear the tracer
tracer.Clear();
if (!Convert.ToBoolean(thisCollectionView.Row["Hidden"]))
{
// Build the static links pages
string code = thisCollectionView.Row["Code"].ToString().ToLower();
if ((!processed_codes.Contains(code)) && (code != "all"))
{
processed_codes.Add(code);
// Add this to the sitemap
sitemap_writer.WriteLine("\t<url>");
sitemap_writer.WriteLine("\t\t<loc>" + primaryWebServerUrl + code + "</loc>");
sitemap_writer.WriteLine("\t</url>");
Logger.AddNonError(InstanceName + ".....Building static links page for " + code);
Console.WriteLine(InstanceName + @"Building static links page for {0}", code);
SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building static links page for " + code, String.Empty);
//Get the list of items for this collection
items = SobekCM_Database.Simple_Item_List(code, tracer);
// Get the item aggregation object
Item_Aggregation aggregation = Item_Aggregation_Builder.Get_Item_Aggregation(code.ToLower(), "en", null, true, false, null);
// Build the static browse pages
if (Build_All_Browse(aggregation, items))
{
static_browse_links.Append("<td><a href=\"" + code + "_all.html\">" + code + "</a></td>" + Environment.NewLine);
col++;
}
if (col > 5)
{
static_browse_links.Append("</tr>" + Environment.NewLine + "<tr>");
col = 1;
}
// Build the RSS feeds
Logger.AddNonError(InstanceName + ".....Building RSS feeds for " + code);
Console.WriteLine(InstanceName + @"Building RSS feeds for {0}", code);
SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building RSS feeds for " + code, String.Empty);
if (Create_RSS_Feed(code, staticSobekcmDataLocation + "rss\\", thisCollectionView.Row["Name"].ToString(), items))
{
recent_rss_link_builder.Append("<img src=\"" + primaryWebServerUrl + "default/images/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\"> <a href=\"" + primaryWebServerUrl + "rss/" + code + "_short_rss.xml\">" + thisCollectionView.Row["Name"] + "</a><br />" + Environment.NewLine);
all_rss_link_builder.Append("<img src=\"" + primaryWebServerUrl + "default/images/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\"> <a href=\"" + primaryWebServerUrl + "rss/" + code + "_rss.xml\">" + thisCollectionView.Row["Name"] + "</a><br />" + Environment.NewLine);
}
}
}
}
// Finish out the collection sitemap
sitemap_writer.WriteLine("</urlset>");
sitemap_writer.Flush();
sitemap_writer.Close();
items = SobekCM_Database.Simple_Item_List(String.Empty, tracer);
Logger.AddNonError(InstanceName + ".....Building static links page for ALL ITEMS");
Console.WriteLine(InstanceName + @"Building static links page for ALL ITEMS");
SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building static links page for ALL ITEMS", String.Empty);
Item_Aggregation allAggregation = Item_Aggregation_Builder.Get_Item_Aggregation("all", "en", null, true, false, null);
Build_All_Browse(allAggregation, items);
Console.WriteLine(InstanceName + @"Building RSS feeds ALL ITEMS");
Logger.AddNonError(InstanceName + ".....Building RSS feeds for ALL ITEMS");
SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building RSS feeds for ALL ITEMS", String.Empty);
Create_RSS_Feed("all", staticSobekcmDataLocation + "rss\\", "All " + SobekCM_Library_Settings.System_Abbreviation + " Items", items);
// Build the site maps
Logger.AddNonError(InstanceName + ".....Building site maps");
Console.WriteLine(InstanceName + @"Building site maps");
SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building site maps", String.Empty);
int sitemaps = Build_Site_Maps();
// Output the main browse and rss links pages
Logger.AddNonError(InstanceName + "....Building main sitemaps and links page");
Console.WriteLine(InstanceName + @"Building main sitemaps and links page");
SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building main sitemaps and links page", String.Empty);
StreamWriter allListWriter = new StreamWriter(staticSobekcmDataLocation + "index.html", false);
allListWriter.WriteLine("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">");
allListWriter.WriteLine("<html xmlns=\"http://www.w3.org/1999/xhtml\" >");
allListWriter.WriteLine("<head>");
allListWriter.WriteLine(" <title>" + SobekCM_Library_Settings.System_Name + " Site Map Links</title>");
allListWriter.WriteLine();
allListWriter.WriteLine(" <meta name=\"robots\" content=\"index, follow\">");
allListWriter.WriteLine(" <link href=\"" + primaryWebServerUrl + "default/SobekCM.css\" rel=\"stylesheet\" type=\"text/css\" title=\"standard\" />");
if (defaultSkinObject.CSS_Style.Length > 0)
{
allListWriter.WriteLine(" <link href=\"" + SobekCM_Library_Settings.System_Base_URL + defaultSkinObject.CSS_Style + "\" rel=\"stylesheet\" type=\"text/css\" />");
}
allListWriter.WriteLine("</head>");
allListWriter.WriteLine("<body>");
allListWriter.WriteLine("<div id=\"container-inner\">");
string banner = "<div id=\"sbkHmw_BannerDiv\"><a alt=\"All Collections\" href=\"" + primaryWebServerUrl + "\" style=\"padding-bottom:0px;margin-bottom:0px\"><img id=\"mainBanner\" src=\"" + primaryWebServerUrl + "design/aggregations/all/images/banners/coll.jpg\" alt=\"\" /></a></div>";
Display_Header(allListWriter, defaultSkinObject, banner);
allListWriter.WriteLine("<br /><br />This page is to provide static links to all resources in " + SobekCM_Library_Settings.System_Abbreviation + ". <br />");
allListWriter.WriteLine("Click <a href=\"" + primaryWebServerUrl + "\">HERE</a> to return to main library. <br />");
allListWriter.WriteLine("<br />");
allListWriter.WriteLine("<br />");
allListWriter.WriteLine("SITE MAPS<br />");
allListWriter.WriteLine("<br />");
allListWriter.WriteLine("<a href=\"sitemap_collections.xml\">Map to all the collection home pages</a><br />");
if (sitemaps > 0)
{
for (int i = 1; i <= sitemaps; i++)
{
allListWriter.WriteLine("<a href=\"sitemap" + i + ".xml\">Site Map File " + i + "</a><br />");
}
allListWriter.WriteLine("<br />");
allListWriter.WriteLine("<br />");
}
else
{
allListWriter.WriteLine("NO SITE MAPS GENERATED!");
}
Display_Footer(allListWriter, defaultSkinObject);
allListWriter.WriteLine("</div>");
allListWriter.WriteLine("</body>");
allListWriter.WriteLine("</html>");
allListWriter.Flush();
allListWriter.Close();
// Create the list of all the RSS feeds
try
{
Logger.AddNonError(InstanceName + ".....Building main rss feed page");
Console.WriteLine(InstanceName + @"Building main rss feed page");
SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Standard", "Building main rss feed page", String.Empty);
StreamWriter writer = new StreamWriter(staticSobekcmDataLocation + "rss\\index.htm", false);
writer.WriteLine("<!DOCTYPE html>");
writer.WriteLine("<html>");
writer.WriteLine("<head>");
writer.WriteLine(" <title>RSS Feeds for " + SobekCM_Library_Settings.System_Abbreviation + "</title>");
writer.WriteLine();
writer.WriteLine(" <meta name=\"robots\" content=\"index, follow\">");
writer.WriteLine(" <link href=\"" + primaryWebServerUrl + "default/SobekCM.css\" rel=\"stylesheet\" type=\"text/css\" title=\"standard\" />");
if (defaultSkinObject.CSS_Style.Length > 0)
{
writer.WriteLine(" <link href=\"" + SobekCM_Library_Settings.System_Base_URL + defaultSkinObject.CSS_Style + "\" rel=\"stylesheet\" type=\"text/css\" />");
}
writer.WriteLine("</head>");
writer.WriteLine("<body>");
writer.WriteLine("<div id=\"container-inner\">");
string banner2 = "<div id=\"sbkHmw_BannerDiv\"><a alt=\"All Collections\" href=\"" + primaryWebServerUrl + "\" style=\"padding-bottom:0px;margin-bottom:0px\"><img id=\"mainBanner\" src=\"" + primaryWebServerUrl + "design/aggregations/all/images/banners/coll.jpg\" alt=\"\" /></a></div>";
Display_Header(writer, defaultSkinObject, banner2);
writer.WriteLine("<div id=\"pagecontainer\">");
writer.WriteLine("<div class=\"ViewsBrowsesRow\">");
writer.WriteLine(" <ul class=\"sbk_FauxUpwardTabsList\">");
writer.WriteLine(" <li><a href=\"" + primaryWebServerUrl + "\">" + SobekCM_Library_Settings.System_Abbreviation + " HOME</a></li>");
writer.WriteLine(" <li class=\"current\">RSS FEEDS</li>");
writer.WriteLine(" </ul>");
writer.WriteLine("</div>");
writer.WriteLine();
writer.WriteLine("<div class=\"SobekSearchPanel\">");
writer.WriteLine(" <h1>RSS Feeds for the " + SobekCM_Library_Settings.System_Name + "</h1>");
writer.WriteLine("</div>");
writer.WriteLine();
writer.WriteLine("<div class=\"SobekHomeText\">");
writer.WriteLine("<table width=\"700\" border=\"0\" align=\"center\">");
writer.WriteLine(" <tr>");
writer.WriteLine(" <td>");
writer.WriteLine(" <br />");
writer.WriteLine(" This page provides links to RSS feeds for items within " + SobekCM_Library_Settings.System_Name + ". The first group of RSS feeds below contains the last 20 items added to the collection. The second group of items contains links to every item in a collection. These rss feeds can grow quite lengthy and the load time is often non-trivial.<br />");
writer.WriteLine(" <br />");
writer.WriteLine(" In addition, the following three RSS feeds are provided:");
writer.WriteLine(" <blockquote>");
writer.WriteLine(" <img src=\"" + primaryWebServerUrl + "default/images/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\"> <a href=\"" + primaryWebServerUrl + "rss/all_rss.xml\">All items in " + SobekCM_Library_Settings.System_Abbreviation + "</a><br />");
writer.WriteLine(" <img src=\"" + primaryWebServerUrl + "default/images/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\"> <a href=\"" + primaryWebServerUrl + "rss/all_short_rss.xml\">Most recently added items in " + SobekCM_Library_Settings.System_Abbreviation + " (last 100)</a><br />");
writer.WriteLine(" </blockquote>");
writer.WriteLine(" RSS feeds are a way to keep up-to-date on new materials that are added to the Digital Collections. RSS feeds are written in XML and require a news reader to access.<br />");
writer.WriteLine(" <br />");
writer.WriteLine(" You can download and install a <a href=\"http://dmoz.org/Reference/Libraries/Library_and_Information_Science/Technical_Services/Cataloguing/Metadata/RDF/Applications/RSS/News_Readers/\">news reader</a>. Or, you can use a Web-based reader such as <a href=\"http://www.google.com/reader\">Google Reader </a>or <a href=\"http://my.yahoo.com/\">My Yahoo!</a>.");
writer.WriteLine(" Follow the instructions in your reader to subscribe to the feed of your choice. You will usually need to copy and paste the feed URL into the reader. <br />");
writer.WriteLine(" <br />");
writer.WriteLine(" </td>");
writer.WriteLine(" </tr>");
writer.WriteLine("</table>");
writer.WriteLine("<table width=\"750\" border=\"0\" align=\"center\" cellpadding=\"1\" cellspacing=\"0\">");
writer.WriteLine(" <tr>");
writer.WriteLine(" <td bgcolor=\"#cccccc\">");
writer.WriteLine(" <table width=\"100%\" border=\"0\" align=\"center\" cellpadding=\"2\" cellspacing=\"0\">");
writer.WriteLine(" <tr>");
writer.WriteLine(" <td bgcolor=\"#f4f4f4\"><span class=\"groupname\"><span class=\"groupnamecaps\"> M</span>OST <span class=\"groupnamecaps\">R</span>ECENT <span class=\"groupnamecaps\">I</span>TEMS (BY COLLECTION)</span></td>");
writer.WriteLine(" </tr>");
writer.WriteLine(" </table>");
writer.WriteLine(" </td>");
writer.WriteLine(" </tr>");
writer.WriteLine("<table>");
writer.WriteLine("<div class=\"SobekHomeText\">");
writer.WriteLine("<table width=\"700\" border=\"0\" align=\"center\">");
writer.WriteLine(" <tr>");
writer.WriteLine(" <td>");
writer.WriteLine(recent_rss_link_builder.ToString());
writer.WriteLine(" <br />");
writer.WriteLine(" </td>");
writer.WriteLine(" </tr>");
writer.WriteLine("</table>");
writer.WriteLine("<table width=\"750\" border=\"0\" align=\"center\" cellpadding=\"1\" cellspacing=\"0\">");
writer.WriteLine(" <tr>");
writer.WriteLine(" <td bgcolor=\"#cccccc\">");
writer.WriteLine(" <table width=\"100%\" border=\"0\" align=\"center\" cellpadding=\"2\" cellspacing=\"0\">");
writer.WriteLine(" <tr>");
writer.WriteLine(" <td bgcolor=\"#f4f4f4\"><span class=\"groupname\"><span class=\"groupnamecaps\"> A</span>LL <span class=\"groupnamecaps\">I</span>TEMS (BY COLLECTION) </span></td>");
writer.WriteLine(" </tr>");
writer.WriteLine(" </table>");
writer.WriteLine(" </td>");
writer.WriteLine(" </tr>");
writer.WriteLine("<table>");
writer.WriteLine("<div class=\"SobekHomeText\">");
writer.WriteLine("<table width=\"700\" border=\"0\" align=\"center\">");
writer.WriteLine(" <tr>");
writer.WriteLine(" <td>");
writer.WriteLine(all_rss_link_builder.ToString());
writer.WriteLine(" <br />");
writer.WriteLine(" </td>");
writer.WriteLine(" </tr>");
writer.WriteLine("</table>");
writer.WriteLine("<br />");
writer.WriteLine("</div>");
writer.WriteLine("</div>");
Display_Footer(writer, defaultSkinObject);
writer.WriteLine("</div>");
writer.WriteLine("</body>");
writer.WriteLine("</html>");
writer.Flush();
writer.Close();
}
catch
{
Logger.AddError("ERROR BUILDING RSS INDEX.HTM FILE");
Console.WriteLine(@"Error building RSS index.htm file");
SobekCM_Database.Builder_Add_Log_Entry(PrimaryLogId, String.Empty, "Error", "Error building the main RSS feed page", String.Empty);
}
return errors;
}
/// <summary>Private method used to generate the error log for the packages</summary>
/// <param name="Folder_Name"></param>
/// <param name="ErrorMessage"></param>
/// <param name="Resource_Folder"></param>
/// <param name="MetsType"></param>
/// <param name="BaseErrorMessage"></param>
private void Create_Error_Log( string Resource_Folder, string Folder_Name, string ErrorMessage, string MetsType, string BaseErrorMessage )
{
// Split the message into seperate lines
string[] errors = ErrorMessage.Split(new char[] { '\n' });
long errorLogId = Add_Error_To_Log(BaseErrorMessage, Folder_Name.Replace("_", ":"), MetsType, -1);
try
{
LogFileXHTML errorLog = new LogFileXHTML(Resource_Folder + "\\" + Folder_Name + ".log.html", "Package Processing Log", "SobekCM Builder Errors");
errorLog.New();
errorLog.AddComplete("Error Log for " + Folder_Name + " processed at: " + DateTime.Now.ToString());
errorLog.AddComplete("");
foreach (string thisError in errors)
{
errorLog.AddError(thisError);
Add_Error_To_Log(thisError, Folder_Name.Replace("_",":"), MetsType, errorLogId );
}
errorLog.Close();
}
catch
{
}
}
/// <summary> Rebuilds all static pages, including the RSS feeds and site maps </summary>
/// <param name="logger"> Log file to record progress </param>
/// <param name="build_all_citation_pages"> Flag indicates to build the individual static HTML pages for each digital resource </param>
/// <param name="rss_feed_location"> Location where the RSS feeds should be updated to </param>
/// <returns> The number of encountered errors </returns>
public int Rebuild_All_Static_Pages(LogFileXHTML logger, bool build_all_citation_pages, string rss_feed_location)
{
logger.AddNonError("Rebuilding all static pages");
// Set the item for the current mode
errors = 0;
if (build_all_citation_pages)
{
DataSet item_list_table = SobekCM_Database.Get_Item_List(false, null);
foreach (DataRow thisRow in item_list_table.Tables[0].Rows)
{
//if (errors > 10000)
//{
// logger.AddError("10000 errors encountered!");
// Console.WriteLine("10000 errors encountered");
// return errors;
//}
string bibid = thisRow["BibID"].ToString();
string vid = thisRow["VID"].ToString();
if (Create_Item_Citation_HTML(bibid, vid, thisRow["File_Location"] + "\\" + bibid + "_" + vid + ".html", String.Empty))
{
//successes.WriteLine(bibid + "\t" + vid);
}
else
{
errors++;
//failures.WriteLine(bibid + "\t" + vid);
}
}
}
// Set the mode away from the display item mode
currentMode.Mode = Display_Mode_Enum.Aggregation_Home;
// Get the list of all collections
DataTable allCollections = SobekCM_Database.Get_Codes_Item_Aggregations( null);
DataView collectionView = new DataView(allCollections) {Sort = "Name ASC"};
// Build the basic site map first
StreamWriter sitemap_writer = new StreamWriter(staticSobekcmDataLocation + "sitemap_collections.xml", false);
sitemap_writer.WriteLine("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
sitemap_writer.WriteLine("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">");
sitemap_writer.WriteLine("\t<url>");
sitemap_writer.WriteLine("\t\t<loc>" + primaryWebServerUrl + "</loc>");
sitemap_writer.WriteLine("\t</url>");
// Prepare to build all the links to static pages
StringBuilder static_browse_links = new StringBuilder();
StringBuilder recent_rss_link_builder = new StringBuilder();
StringBuilder all_rss_link_builder = new StringBuilder();
int col = 2;
DataSet items;
List<string> processed_codes = new List<string>();
foreach (DataRowView thisCollectionView in collectionView)
{
// Clear the tracer
tracer.Clear();
if (!Convert.ToBoolean(thisCollectionView.Row["Hidden"]))
{
// Build the static links pages
string code = thisCollectionView.Row["Code"].ToString().ToLower();
if ((!processed_codes.Contains(code)) && ( code != "all" ))
{
processed_codes.Add(code);
// Add this to the sitemap
sitemap_writer.WriteLine("\t<url>");
sitemap_writer.WriteLine("\t\t<loc>" + primaryWebServerUrl + code + "</loc>");
sitemap_writer.WriteLine("\t</url>");
logger.AddNonError(".....Building static links page... " + code);
Console.WriteLine(@"Building static links page... {0}", code);
//Get the list of items for this collection
items = SobekCM_Database.Simple_Item_List(code, tracer);
// Continue if there were items
if ((items != null) && (items.Tables[0].Rows.Count > 0))
{
// Build the static browse pages
if (Build_All_Browse(code, items))
{
static_browse_links.Append("<td width=\"150px\"><a href=\"" + code + "_list.html\">" + code + "</a></td>" + Environment.NewLine );
col++;
}
if (col > 5)
{
static_browse_links.Append("</tr>" + Environment.NewLine + "<tr>");
col = 1;
}
// Build the RSS feeds
logger.AddNonError(".....Building RSS feed... " + code);
Console.WriteLine(@"Building RSS feed... {0}", code);
if (Create_RSS_Feed(code, staticSobekcmDataLocation + "rss\\", thisCollectionView.Row["Name"].ToString(), items))
{
recent_rss_link_builder.Append("<img src=\"" + primaryWebServerUrl + "default/images/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\"> <a href=\"" + primaryWebServerUrl + "rss/" + code + "_short_rss.xml\">" + thisCollectionView.Row["Name"] + "</a><br />" + Environment.NewLine );
all_rss_link_builder.Append("<img src=\"" + primaryWebServerUrl + "default/images/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\"> <a href=\"" + primaryWebServerUrl + "rss/" + code + "_rss.xml\">" + thisCollectionView.Row["Name"] + "</a><br />" + Environment.NewLine );
}
}
}
}
}
// Finish out the collection sitemap
sitemap_writer.WriteLine("</urlset>");
sitemap_writer.Flush();
sitemap_writer.Close();
items = SobekCM_Database.Simple_Item_List(String.Empty, tracer);
logger.AddNonError(".....Building static links page... ALL ITEMS");
Console.WriteLine(@"Building static links page... ALL ITEMS");
Build_All_Browse(String.Empty, items);
Console.WriteLine(@"Building RSS feed... ALL ITEMS");
logger.AddNonError(".....Building RSS feed... ALL ITEMS");
Create_RSS_Feed("all", staticSobekcmDataLocation + "rss\\", "All UFDC Items", items);
// Build the site maps
logger.AddNonError(".....Building site maps");
Console.WriteLine(@"Building site maps");
int sitemaps = Build_Site_Maps();
// Output the main browse and rss links pages
logger.AddNonError("....Building main browse html page");
Console.WriteLine(@"Building main browse html page");
StreamWriter allListWriter = new StreamWriter(staticSobekcmDataLocation + "index.html", false);
allListWriter.WriteLine("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">");
allListWriter.WriteLine("<html xmlns=\"http://www.w3.org/1999/xhtml\" >");
allListWriter.WriteLine("<head>");
allListWriter.WriteLine(" <title>UFDC Site Map Links</title>");
allListWriter.WriteLine();
allListWriter.WriteLine(" <!-- Static HTML generated by application written by Mark Sullivan -->");
allListWriter.WriteLine(" <meta name=\"robots\" content=\"index, follow\">");
allListWriter.WriteLine(" <link href=\"" + primaryWebServerUrl + "default/SobekCM.css\" rel=\"stylesheet\" type=\"text/css\" title=\"standard\" />");
allListWriter.WriteLine(" <style type=\"text/css\" media=\"screen\">");
allListWriter.WriteLine(" @import url( http://ufdc.ufl.edu/design/skins/ufdc/ufdc.css );");
allListWriter.WriteLine(" </style>");
allListWriter.WriteLine("</head>");
allListWriter.WriteLine("<body>");
allListWriter.WriteLine("<div id=\"container-inner\">");
Display_Header(allListWriter, ufdcInterface);
allListWriter.WriteLine("<div id=\"pagecontainer\">");
allListWriter.WriteLine("<center><a href=\"" + primaryWebServerUrl + "\"><img id=\"mainBanner\" src=\"" + primaryWebServerUrl + "design/aggregations/all/images/banners/coll.jpg\" alt=\"MISSING BANNER\" /></a></center>");
allListWriter.WriteLine("<br /><br />This page is to provide static links to all resources in UFDC. <br />");
allListWriter.WriteLine("Click <a href=\"" + primaryWebServerUrl + "\">HERE</a> to return to main library. <br />");
allListWriter.WriteLine("<br />");
allListWriter.WriteLine("<br />");
allListWriter.WriteLine("SITE MAPS<br />");
allListWriter.WriteLine("<br />");
allListWriter.WriteLine("<a href=\"sitemap_collections.xml\">Map to all the collection home pages</a><br />");
if (sitemaps > 0)
{
for (int i = 1; i <= sitemaps; i++)
{
allListWriter.WriteLine("<a href=\"sitemap" + i + ".xml\">Site Map File " + i + "</a><br />");
}
allListWriter.WriteLine("<br />");
allListWriter.WriteLine("<br />");
}
else
{
allListWriter.WriteLine("NO SITE MAPS GENERATED!");
}
Display_Footer(allListWriter, ufdcInterface);
allListWriter.WriteLine("</div>");
allListWriter.WriteLine("</body>");
allListWriter.WriteLine("</html>");
allListWriter.Flush();
allListWriter.Close();
// Create the list of all the RSS feeds
try
{
logger.AddNonError(".....Building main rss feed page");
Console.WriteLine(@"Building main rss feed page");
StreamWriter writer = new StreamWriter(staticSobekcmDataLocation + "rss\\index.htm", false);
writer.WriteLine("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">");
writer.WriteLine("<html xmlns=\"http://www.w3.org/1999/xhtml\" >");
writer.WriteLine("<head>");
writer.WriteLine(" <title>RSS Feeds for UFDC</title>");
writer.WriteLine();
writer.WriteLine(" <!-- Static HTML generated by application written by Mark Sullivan -->");
writer.WriteLine(" <meta name=\"robots\" content=\"index, follow\">");
writer.WriteLine(" <link href=\"" + primaryWebServerUrl + "default/SobekCM.css\" rel=\"stylesheet\" type=\"text/css\" title=\"standard\" />");
writer.WriteLine(" <style type=\"text/css\" media=\"screen\">");
writer.WriteLine(" @import url( http://ufdc.ufl.edu/design/skins/ufdc/ufdc.css );");
writer.WriteLine(" </style>");
writer.WriteLine("</head>");
writer.WriteLine("<body>");
writer.WriteLine("<div id=\"container-inner\">");
Display_Header(writer, ufdcInterface);
writer.WriteLine("<div id=\"pagecontainer\">");
writer.WriteLine("<center><a href=\"" + primaryWebServerUrl + "\"><img id=\"mainBanner\" src=\"" + primaryWebServerUrl + "design/aggregations/all/images/banners/coll.jpg\" alt=\"MISSING BANNER\" /></a></center>");
writer.WriteLine("<div class=\"ViewsBrowsesRow\">");
writer.WriteLine(" <a href=\"" + primaryWebServerUrl + "\"><img src=\"" + primaryWebServerUrl + "design/skins/ufdc/tabs/cL.gif\" border=\"0\" alt=\"\" /><span class=\"tab\"> UFDC HOME </span><img src=\"" + primaryWebServerUrl + "design/skins/ufdc/tabs/cR.gif\" border=\"0\" alt=\"\" /></a>");
writer.WriteLine(" <img src=\"" + primaryWebServerUrl + "design/skins/ufdc/tabs/cL_s.gif\" border=\"0\" alt=\"\" /><span class=\"tab_s\"> RSS FEEDS </span><img src=\"" + primaryWebServerUrl + "design/skins/ufdc/tabs/cR_s.gif\" border=\"0\" alt=\"\" />");
writer.WriteLine("</div>");
writer.WriteLine();
writer.WriteLine("<div class=\"SobekSearchPanel\">");
writer.WriteLine(" <h1>RSS Feeds for the University of Florida Digital Collections</h1>");
writer.WriteLine("</div>");
writer.WriteLine();
writer.WriteLine("<div class=\"SobekHomeText\">");
writer.WriteLine("<table width=\"700\" border=\"0\" align=\"center\">");
writer.WriteLine(" <tr>");
writer.WriteLine(" <td>");
writer.WriteLine(" <br />");
writer.WriteLine(" This page provides links to RSS feeds for items within UFDC. The first group of RSS feeds below contains the last 20 items added to the collection. The second group of items contains links to every item in a collection. These rss feeds can grow quite lengthy and the load time is often non-trivial.<br />");
writer.WriteLine(" <br />");
writer.WriteLine(" In addition, the following three RSS feeds are provided:");
writer.WriteLine(" <blockquote>");
writer.WriteLine(" <img src=\"" + primaryWebServerUrl + "default/images/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\"> <a href=\"" + primaryWebServerUrl + "rss/all_rss.xml\">All items in UFDC</a><br />");
writer.WriteLine(" <img src=\"" + primaryWebServerUrl + "default/images/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\"> <a href=\"" + primaryWebServerUrl + "rss/all_short_rss.xml\">Most recently added items in UFDC (last 100)</a><br />");
writer.WriteLine(" <img src=\"" + primaryWebServerUrl + "default/images/16px-Feed-icon.svg.png\" alt=\"RSS\" width=\"16\" height=\"16\"> <a href=\"" + primaryWebServerUrl + "rss/ufdc_news_rss.xml\">Recent UFDC news and documentation changes</a><br />");
writer.WriteLine(" </blockquote>");
writer.WriteLine(" RSS feeds are a way to keep up-to-date on new materials that are added to the Digital Collections. RSS feeds are written in XML and require a news reader to access.<br />");
writer.WriteLine(" <br />");
writer.WriteLine(" You can download and install a <a href=\"http://dmoz.org/Reference/Libraries/Library_and_Information_Science/Technical_Services/Cataloguing/Metadata/RDF/Applications/RSS/News_Readers/\">news reader</a>. Or, you can use a Web-based reader such as <a href=\"http://www.google.com/reader\">Google Reader </a>or <a href=\"http://my.yahoo.com/\">My Yahoo!</a>.");
writer.WriteLine(" Follow the instructions in your reader to subscribe to the feed of your choice. You will usually need to copy and paste the feed URL into the reader. <br />");
writer.WriteLine(" <br />");
writer.WriteLine(" </td>");
writer.WriteLine(" </tr>");
writer.WriteLine("</table>");
writer.WriteLine("<table width=\"750\" border=\"0\" align=\"center\" cellpadding=\"1\" cellspacing=\"0\">");
writer.WriteLine(" <tr>");
writer.WriteLine(" <td bgcolor=\"#cccccc\">");
writer.WriteLine(" <table width=\"100%\" border=\"0\" align=\"center\" cellpadding=\"2\" cellspacing=\"0\">");
writer.WriteLine(" <tr>");
writer.WriteLine(" <td bgcolor=\"#f4f4f4\"><span class=\"groupname\"><span class=\"groupnamecaps\"> M</span>OST <span class=\"groupnamecaps\">R</span>ECENT <span class=\"groupnamecaps\">I</span>TEMS (BY COLLECTION)</span></td>");
writer.WriteLine(" </tr>");
writer.WriteLine(" </table>");
writer.WriteLine(" </td>");
writer.WriteLine(" </tr>");
writer.WriteLine("<table>");
writer.WriteLine("<div class=\"SobekHomeText\">");
writer.WriteLine("<table width=\"700\" border=\"0\" align=\"center\">");
writer.WriteLine(" <tr>");
writer.WriteLine(" <td>");
writer.WriteLine(recent_rss_link_builder.ToString());
writer.WriteLine(" <br />");
writer.WriteLine(" </td>");
writer.WriteLine(" </tr>");
writer.WriteLine("</table>");
writer.WriteLine("<table width=\"750\" border=\"0\" align=\"center\" cellpadding=\"1\" cellspacing=\"0\">");
writer.WriteLine(" <tr>");
writer.WriteLine(" <td bgcolor=\"#cccccc\">");
writer.WriteLine(" <table width=\"100%\" border=\"0\" align=\"center\" cellpadding=\"2\" cellspacing=\"0\">");
writer.WriteLine(" <tr>");
writer.WriteLine(" <td bgcolor=\"#f4f4f4\"><span class=\"groupname\"><span class=\"groupnamecaps\"> A</span>LL <span class=\"groupnamecaps\">I</span>TEMS (BY COLLECTION) </span></td>");
writer.WriteLine(" </tr>");
writer.WriteLine(" </table>");
writer.WriteLine(" </td>");
writer.WriteLine(" </tr>");
writer.WriteLine("<table>");
writer.WriteLine("<div class=\"SobekHomeText\">");
writer.WriteLine("<table width=\"700\" border=\"0\" align=\"center\">");
writer.WriteLine(" <tr>");
writer.WriteLine(" <td>");
writer.WriteLine(all_rss_link_builder.ToString());
writer.WriteLine(" <br />");
writer.WriteLine(" </td>");
writer.WriteLine(" </tr>");
writer.WriteLine("</table>");
writer.WriteLine("<br />");
Display_Footer(writer, ufdcInterface);
writer.WriteLine("</div>");
writer.WriteLine("</body>");
writer.WriteLine("</html>");
writer.Flush();
writer.Close();
}
catch
{
logger.AddError("ERROR BUILDING RSS INDEX.HTM FILE");
Console.WriteLine(@"Error building RSS index.htm file");
}
return errors;
}
/// <summary> Immediately perform all requested tasks </summary>
/// <param name="BuildProductionMarcxmlFeed"> Flag indicates if the MarcXML feed for OPACs should be produced </param>
/// <param name="BuildTestMarcxmlFeed"> Flag indicates if the items set to be put in a TEST feed should have their MarcXML feed produced</param>
/// <param name="RunBulkloader"> Flag indicates if the preload </param>
/// <param name="CompleteStaticRebuild"> Flag indicates whether to rebuild all the item static pages </param>
/// <param name="MarcRebuild"> Flag indicates if all the MarcXML files for each resource should be rewritten from the METS/MODS metadata files </param>
public void Execute_Immediately(bool BuildProductionMarcxmlFeed, bool BuildTestMarcxmlFeed, bool RunBulkloader, bool CompleteStaticRebuild, bool MarcRebuild )
{
// start with warnings on imagemagick and ghostscript not being installed
if (SobekCM_Library_Settings.ImageMagick_Executable.Length == 0)
{
Console.WriteLine("WARNING: Could not find ImageMagick installed. Some image processing will be unavailable.");
}
if (SobekCM_Library_Settings.Ghostscript_Executable.Length == 0)
{
Console.WriteLine("WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable.");
}
if (CompleteStaticRebuild)
{
Console.WriteLine("Beginning static rebuild");
LogFileXHTML staticRebuildLog = new LogFileXHTML(Application.StartupPath + "/Logs/static_rebuild.html");
Static_Pages_Builder builder = new Static_Pages_Builder(SobekCM_Library_Settings.Application_Server_URL, SobekCM_Library_Settings.Static_Pages_Location, SobekCM_Library_Settings.Application_Server_Network);
builder.Rebuild_All_Static_Pages(staticRebuildLog, true, SobekCM_Library_Settings.Local_Log_Directory, String.Empty, -1);
}
if ( MarcRebuild )
{
Static_Pages_Builder builder = new Static_Pages_Builder(SobekCM_Library_Settings.Application_Server_URL, SobekCM_Library_Settings.Static_Pages_Location, SobekCM_Library_Settings.Application_Server_Network);
builder.Rebuild_All_MARC_Files( SobekCM_Library_Settings.Image_Server_Network );
}
if (BuildProductionMarcxmlFeed)
{
Create_Complete_MarcXML_Feed(false);
}
if (BuildTestMarcxmlFeed)
{
Create_Complete_MarcXML_Feed(true);
}
// Create the log
string directory = SobekCM_Library_Settings.Local_Log_Directory;
if (!Directory.Exists(directory))
Directory.CreateDirectory(directory);
// Run the PRELOADER
if (RunBulkloader)
{
Run_BulkLoader( verbose );
}
else
{
Console.WriteLine("PreLoader skipped per command line arguments");
}
}
private void Run_BulkLoader( bool Verbose )
{
// Create the local log directories
if (!Directory.Exists(SobekCM_Library_Settings.Local_Log_Directory))
{
Console.WriteLine("Creating local log directory: " + SobekCM_Library_Settings.Local_Log_Directory);
Directory.CreateDirectory(SobekCM_Library_Settings.Local_Log_Directory);
}
// Determine the new log name
string log_name = "incoming_" + controllerStarted.Year + "_" + controllerStarted.Month.ToString().PadLeft(2, '0') + "_" + controllerStarted.Day.ToString().PadLeft(2, '0') + ".html";
string local_log_name = SobekCM_Library_Settings.Local_Log_Directory + "\\" + log_name;
// Create the new log file
LogFileXHTML preloader_logger = new LogFileXHTML(local_log_name, "SobekCM Incoming Packages Log", "UFDC_Builder.exe", true);
// Step through each database instance
foreach (Database_Instance_Configuration dbConfig in SobekCM_Library_Settings.Database_Connections)
{
try
{
if (!dbConfig.Is_Active)
{
Console.WriteLine( dbConfig.Name + " is set to INACTIVE");
preloader_logger.AddNonError(dbConfig.Name + " is set to INACTIVE");
}
else
{
SobekCM_Database.Connection_String = dbConfig.Connection_String;
Worker_BulkLoader newLoader = new Worker_BulkLoader(preloader_logger, verbose, dbConfig.Name, dbConfig.Can_Abort);
newLoader.Perform_BulkLoader(Verbose);
// Save information about this last run
Library.Database.SobekCM_Database.Set_Setting("Builder Version", SobekCM_Library_Settings.CURRENT_BUILDER_VERSION);
Library.Database.SobekCM_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString());
Library.Database.SobekCM_Database.Set_Setting("Builder Last Message", newLoader.Final_Message);
}
}
catch
{
}
}
}
/// <summary> Continuously execute the processes in a recurring background thread </summary>
public void Execute_In_Background()
{
// Load all the settings
SobekCM_Library_Settings.Refresh(Library.Database.SobekCM_Database.Get_Settings_Complete(null));
// Set the variable which will control background execution
int time_between_polls = SobekCM_Library_Settings.Builder_Override_Seconds_Between_Polls;
if (( time_between_polls < 0 ) || ( SobekCM_Library_Settings.Database_Connections.Count == 1 ))
time_between_polls = Convert.ToInt32(SobekCM_Library_Settings.Builder_Seconds_Between_Polls);
// Determine the new log name
string log_name = "incoming_" + controllerStarted.Year + "_" + controllerStarted.Month.ToString().PadLeft(2, '0') + "_" + controllerStarted.Day.ToString().PadLeft(2, '0') + ".html";
string local_log_name = SobekCM_Library_Settings.Local_Log_Directory + "\\" + log_name;
// Create the new log file
LogFileXHTML preloader_logger = new LogFileXHTML(local_log_name, "SobekCM Incoming Packages Log", "UFDC_Builder.exe", true);
// start with warnings on imagemagick and ghostscript not being installed
if (SobekCM_Library_Settings.ImageMagick_Executable.Length == 0)
{
Console.WriteLine("WARNING: Could not find ImageMagick installed. Some image processing will be unavailable.");
preloader_logger.AddNonError("WARNING: Could not find ImageMagick installed. Some image processing will be unavailable.");
}
if (SobekCM_Library_Settings.Ghostscript_Executable.Length == 0)
{
Console.WriteLine("WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable.");
preloader_logger.AddNonError("WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable.");
}
// Set the time for the next feed building event to 10 minutes from now
feedNextBuildTime = DateTime.Now.Add(new TimeSpan(0, 10, 0));
// First, step through each active configuration and see if building is currently aborted
// while doing very minimal processes
aborted = false;
Console.WriteLine("Checking for initial abort condition");
preloader_logger.AddNonError("Checking for initial abort condition");
string abort_message = String.Empty;
Builder_Operation_Flag_Enum abort_flag = Builder_Operation_Flag_Enum.STANDARD_OPERATION;
foreach (Database_Instance_Configuration dbConfig in SobekCM_Library_Settings.Database_Connections)
{
if ((!aborted) && (dbConfig.Is_Active) && (dbConfig.Can_Abort))
{
SobekCM_Database.Connection_String = dbConfig.Connection_String;
Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String;
// Check that this should not be skipped or aborted
Builder_Operation_Flag_Enum operationFlag = Abort_Database_Mechanism.Builder_Operation_Flag;
switch (operationFlag)
{
case Builder_Operation_Flag_Enum.ABORT_REQUESTED:
case Builder_Operation_Flag_Enum.ABORTING:
abort_message = "PREVIOUS ABORT flag found in " + dbConfig.Name;
abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
Console.WriteLine(abort_message);
preloader_logger.AddNonError(abort_message);
aborted = true;
Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
break;
case Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED:
abort_message = "PREVIOUS NO BUILDING flag found in " + dbConfig.Name;
Console.WriteLine(abort_message);
preloader_logger.AddNonError(abort_message);
aborted = true;
break;
}
}
}
// If initially aborted, step through each instance and set a message
if (aborted)
{
// Add messages in each active instance
foreach (Database_Instance_Configuration dbConfig in SobekCM_Library_Settings.Database_Connections)
{
if (dbConfig.Is_Active)
{
Console.WriteLine("Setting previous abort flag message in " + dbConfig.Name);
preloader_logger.AddNonError("Setting previous abort flag message in " + dbConfig.Name);
SobekCM_Database.Connection_String = dbConfig.Connection_String;
Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String;
Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", abort_message, String.Empty);
// Save information about this last run
Library.Database.SobekCM_Database.Set_Setting("Builder Version", SobekCM_Library_Settings.CURRENT_BUILDER_VERSION);
Library.Database.SobekCM_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString());
Library.Database.SobekCM_Database.Set_Setting("Builder Last Message", abort_message);
// Finally, set the builder flag appropriately
if (abort_flag != Builder_Operation_Flag_Enum.STANDARD_OPERATION)
Abort_Database_Mechanism.Builder_Operation_Flag = abort_flag;
}
}
// Do nothing else
return;
}
// Build all the bulk loader objects
List<Worker_BulkLoader> loaders = new List<Worker_BulkLoader>();
bool activeInstanceFound = false;
foreach (Database_Instance_Configuration dbConfig in SobekCM_Library_Settings.Database_Connections)
{
if (!dbConfig.Is_Active)
{
loaders.Add(null);
Console.WriteLine(dbConfig.Name + " is set to INACTIVE");
preloader_logger.AddNonError(dbConfig.Name + " is set to INACTIVE");
}
else
{
activeInstanceFound = true;
SobekCM_Database.Connection_String = dbConfig.Connection_String;
Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String;
// At this point warn on mossing the Ghostscript and ImageMagick
if (SobekCM_Library_Settings.ImageMagick_Executable.Length == 0)
{
Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find ImageMagick installed. Some image processing will be unavailable.", String.Empty);
}
if (SobekCM_Library_Settings.Ghostscript_Executable.Length == 0)
{
Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable.", String.Empty);
}
Console.WriteLine(dbConfig.Name + " - Preparing to begin polling");
preloader_logger.AddNonError(dbConfig.Name + " - Preparing to begin polling");
Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Preparing to begin polling", String.Empty);
Worker_BulkLoader newLoader = new Worker_BulkLoader(preloader_logger, verbose, dbConfig.Name, dbConfig.Can_Abort);
loaders.Add(newLoader);
}
}
// If no active instances, just exit
if (!activeInstanceFound)
{
Console.WriteLine("No active databases in the config file");
preloader_logger.AddError("No active databases in config file... Aborting");
return;
}
bool firstRun = true;
// Loop continually until the end hour is achieved
do
{
// Is it time to build any RSS/XML feeds?
bool rebuildRssFeeds = false;
if (DateTime.Compare(DateTime.Now, feedNextBuildTime) >= 0)
{
rebuildRssFeeds = true;
feedNextBuildTime = DateTime.Now.Add(new TimeSpan(0, 10, 0));
}
// Step through each instance
for (int i = 0; i < SobekCM_Library_Settings.Database_Connections.Count; i++)
{
if (loaders[i] != null)
{
// Get the instance
Database_Instance_Configuration dbInstance = SobekCM_Library_Settings.Database_Connections[i];
// Set the database connection strings
SobekCM_Database.Connection_String = dbInstance.Connection_String;
Library.Database.SobekCM_Database.Connection_String = dbInstance.Connection_String;
// Look for abort
if ((dbInstance.Can_Abort) && (CheckForAbort()))
{
aborted = true;
if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED)
{
abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING;
}
break;
}
// Refresh all settings, etc..
loaders[i].Refresh_Settings_And_Item_List();
// Pull the abort/pause flag
Builder_Operation_Flag_Enum currentPauseFlag = Abort_Database_Mechanism.Builder_Operation_Flag;
// If not paused, run the prebuilder
if (currentPauseFlag != Builder_Operation_Flag_Enum.PAUSE_REQUESTED)
{
if (firstRun)
{
// // Always build an endeca feed first (so it occurs once a day)
// if (SobekCM_Library_Settings.Build_MARC_Feed_By_Default)
// {
// Create_Complete_MarcXML_Feed(false);
// }
//}
// CLear the old logs
Console.WriteLine(dbInstance.Name + " - Expiring old log entries");
preloader_logger.AddNonError(dbInstance.Name + " - Expiring old log entries");
Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Expiring old log entries", String.Empty);
Library.Database.SobekCM_Database.Builder_Expire_Log_Entries(SobekCM_Library_Settings.Builder_Log_Expiration_Days);
// Rebuild all the static pages
Console.WriteLine(dbInstance.Name + " - Rebuilding all static pages");
preloader_logger.AddNonError(dbInstance.Name + " - Rebuilding all static pages");
long staticRebuildLogId = Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Rebuilding all static pages", String.Empty);
Static_Pages_Builder builder = new Static_Pages_Builder(SobekCM_Library_Settings.Application_Server_URL, SobekCM_Library_Settings.Static_Pages_Location, SobekCM_Library_Settings.Application_Server_Network);
builder.Rebuild_All_Static_Pages(preloader_logger, false, SobekCM_Library_Settings.Local_Log_Directory, dbInstance.Name, staticRebuildLogId);
// Process any pending FDA reports from the FDA Report DropBox
Process_Any_Pending_FDA_Reports(loaders[i]);
}
Run_BulkLoader(loaders[i], verbose);
// Look for abort
if ((!aborted) && (dbInstance.Can_Abort) && (CheckForAbort()))
{
aborted = true;
if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED)
{
abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING;
}
break;
}
if (rebuildRssFeeds)
loaders[i].Build_Feeds();
// Clear memory from this loader
loaders[i].Clear_Item_List();
}
else
{
preloader_logger.AddNonError( dbInstance.Name + " - Building paused");
Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building temporarily PAUSED", String.Empty);
}
}
}
if (aborted)
break;
// No longer the first run
firstRun = false;
// Publish the log
publish_log_file(local_log_name);
// Sleep for correct number of milliseconds
Thread.Sleep(1000 * time_between_polls);
} while (DateTime.Now.Hour < BULK_LOADER_END_HOUR);
// Do the final work for all of the different dbInstances
if (!aborted)
{
for (int i = 0; i < SobekCM_Library_Settings.Database_Connections.Count; i++)
{
if (loaders[i] != null)
{
// Get the instance
Database_Instance_Configuration dbInstance = SobekCM_Library_Settings.Database_Connections[i];
// Set the database flag
SobekCM_Database.Connection_String = dbInstance.Connection_String;
// Pull the abort/pause flag
Builder_Operation_Flag_Enum currentPauseFlag2 = Abort_Database_Mechanism.Builder_Operation_Flag;
// If not paused, run the prebuilder
if (currentPauseFlag2 != Builder_Operation_Flag_Enum.PAUSE_REQUESTED)
{
// Refresh all settings, etc..
loaders[i].Refresh_Settings_And_Item_List();
// Initiate the recreation of the links between metadata and collections
Library.Database.SobekCM_Database.Admin_Update_Cached_Aggregation_Metadata_Links();
}
// Clear the memory
loaders[i].Clear_Item_List();
}
}
}
else
{
// Mark the aborted in each instance
foreach (Database_Instance_Configuration dbConfig in SobekCM_Library_Settings.Database_Connections)
{
if (dbConfig.Is_Active)
{
Console.WriteLine("Setting abort flag message in " + dbConfig.Name);
preloader_logger.AddNonError("Setting abort flag message in " + dbConfig.Name);
SobekCM_Database.Connection_String = dbConfig.Connection_String;
Library.Database.SobekCM_Database.Connection_String = dbConfig.Connection_String;
Library.Database.SobekCM_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building ABORTED per request from database key", String.Empty);
// Save information about this last run
Library.Database.SobekCM_Database.Set_Setting("Builder Version", SobekCM_Library_Settings.CURRENT_BUILDER_VERSION);
Library.Database.SobekCM_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString());
Library.Database.SobekCM_Database.Set_Setting("Builder Last Message", "Building ABORTED per request");
// Finally, set the builder flag appropriately
if ( abort_flag == Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED )
Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
}
}
}
// Publish the log
publish_log_file(local_log_name);
//// Initiate a solr/lucene index optimization since we are done loading for a while
//if (DateTime.Now.Day % 2 == 0)
//{
// if (SobekCM_Library_Settings.Document_Solr_Index_URL.Length > 0)
// {
// Console.WriteLine("Initiating Solr/Lucene document index optimization");
// Solr_Controller.Optimize_Document_Index(SobekCM_Library_Settings.Document_Solr_Index_URL);
// }
//}
//else
//{
// if (SobekCM_Library_Settings.Page_Solr_Index_URL.Length > 0)
// {
// Console.WriteLine("Initiating Solr/Lucene page index optimization");
// Solr_Controller.Optimize_Page_Index(SobekCM_Library_Settings.Page_Solr_Index_URL);
// }
//}
//// Sleep for twenty minutes to end this (the index rebuild might take some time)
//Thread.Sleep(1000 * 20 * 60);
}