/// <summary> Constructor for a new instance of the Worker_BulkLoader class </summary>
/// <param name="Logger"> Log file object for logging progress </param>
/// <param name="Verbose"> Flag indicates if the builder is in verbose mode, where it should log alot more information </param>
/// <param name="InstanceInfo"> Information for the instance of SobekCM to be processed by this bulk loader </param>
/// <param name="LogFileDirectory"> Directory where any log files would be written </param>
/// <param name="PluginRootDirectory"> Root directory where all the plug-ins are stored locally for the builder </param>
public Worker_BulkLoader(LogFileXhtml Logger, Single_Instance_Configuration InstanceInfo, bool Verbose, string LogFileDirectory, string PluginRootDirectory)
{
// Save the log file and verbose flag
logger = Logger;
verbose = Verbose;
multiInstanceBuilder = (MultiInstance_Builder_Settings.Instances.Count > 1);
logFileDirectory = LogFileDirectory;
pluginRootDirectory = PluginRootDirectory;
instanceInfo = InstanceInfo;
// If this is processing multiple instances, limit the numbe of packages that should be processed
// before allowing the builder to move to the next instance and poll
if (multiInstanceBuilder)
{
newItemLimit = 100;
}
else
{
newItemLimit = -1;
}
Add_NonError_To_Log("Worker_BulkLoader.Constructor: Start", verbose, String.Empty, String.Empty, -1);
// Create new list of collections to build
aggregationsToRefresh = new List <string>();
processedItems = new List <BibVidStruct>();
deletedItems = new List <BibVidStruct>();
// Set some defaults
aborted = false;
refreshed = false;
noFoldersReported = false;
firstrun = true;
Add_NonError_To_Log("Worker_BulkLoader.Constructor: Done", verbose, String.Empty, String.Empty, -1);
}
/// <summary> Continuously execute the processes in a recurring background thread </summary>
public void Execute_In_Background()
{
// Set the variable which will control background execution
int time_between_polls = Engine_ApplicationCache_Gateway.Settings.Builder.Override_Seconds_Between_Polls.HasValue ? Engine_ApplicationCache_Gateway.Settings.Builder.Override_Seconds_Between_Polls.Value : 60;
if ((time_between_polls < 0) || (MultiInstance_Builder_Settings.Instances.Count == 1))
{
time_between_polls = Convert.ToInt32(Engine_ApplicationCache_Gateway.Settings.Builder.Seconds_Between_Polls);
}
// Determine the new log name
string log_name = "incoming_" + controllerStarted.Year + "_" + controllerStarted.Month.ToString().PadLeft(2, '0') + "_" + controllerStarted.Day.ToString().PadLeft(2, '0') + ".html";
string local_log_name = Path.Combine(logFileDirectory, log_name);
// Create the new log file
LogFileXhtml preloader_logger = new LogFileXhtml(local_log_name, "SobekCM Incoming Packages Log", "UFDC_Builder.exe", true);
// start with warnings on imagemagick and ghostscript not being installed
if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.ImageMagick_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.ImageMagick_Executable)))
{
Console.WriteLine("WARNING: Could not find ImageMagick installed. Some image processing will be unavailable.");
preloader_logger.AddNonError("WARNING: Could not find ImageMagick installed. Some image processing will be unavailable.");
}
if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.Ghostscript_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.Ghostscript_Executable)))
{
Console.WriteLine("WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable.");
preloader_logger.AddNonError("WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable.");
}
// Set the time for the next feed building event to 10 minutes from now
feedNextBuildTime = DateTime.Now.Add(new TimeSpan(0, 10, 0));
// First, step through each active configuration and see if building is currently aborted
// while doing very minimal processes
aborted = false;
Console.WriteLine("Checking for initial abort condition");
preloader_logger.AddNonError("Checking for initial abort condition");
string abort_message = String.Empty;
int build_instances = 0;
foreach (Single_Instance_Configuration dbConfig in instances)
{
if (!dbConfig.Is_Active)
{
Console.WriteLine(dbConfig.Name + " is set to INACTIVE");
preloader_logger.AddNonError(dbConfig.Name + " is set to INACTIVE");
}
else
{
SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String;
Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String;
// Check that this should not be skipped or aborted
Builder_Operation_Flag_Enum operationFlag = Abort_Database_Mechanism.Builder_Operation_Flag;
switch (operationFlag)
{
case Builder_Operation_Flag_Enum.ABORT_REQUESTED:
case Builder_Operation_Flag_Enum.ABORTING:
// Since this was an abort request at the very beginning, switch back to standard
Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.STANDARD_OPERATION;
build_instances++;
break;
case Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED:
abort_message = "PREVIOUS NO BUILDING flag found in " + dbConfig.Name;
Console.WriteLine(abort_message);
preloader_logger.AddNonError(abort_message);
break;
default:
build_instances++;
break;
}
}
}
// If no instances to run just abort
if (build_instances == 0)
{
// Add messages in each active instance
foreach (Single_Instance_Configuration dbConfig in instances)
{
if (dbConfig.Is_Active)
{
Console.WriteLine("No active databases set for building in the config file");
preloader_logger.AddError("No active databases set for building in config file... Aborting");
SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String;
Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String;
Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", abort_message, String.Empty);
// Save information about this last run
Engine_Database.Set_Setting("Builder Version", Engine_ApplicationCache_Gateway.Settings.Static.Current_Builder_Version);
Engine_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString());
Engine_Database.Set_Setting("Builder Last Message", abort_message);
}
}
// Do nothing else
return;
}
// Build all the bulk loader objects
List <Worker_BulkLoader> loaders = new List <Worker_BulkLoader>();
foreach (Single_Instance_Configuration dbConfig in instances)
{
if (!dbConfig.Is_Active)
{
loaders.Add(null);
continue;
}
SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String;
Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String;
// At this point warn on mossing the Ghostscript and ImageMagick, to get it into each instances database logs
if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.ImageMagick_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.ImageMagick_Executable)))
{
Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find ImageMagick installed. Some image processing will be unavailable.", String.Empty);
}
if ((String.IsNullOrEmpty(MultiInstance_Builder_Settings.Ghostscript_Executable)) || (!File.Exists(MultiInstance_Builder_Settings.Ghostscript_Executable)))
{
Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "WARNING: Could not find GhostScript installed. Some PDF processing will be unavailable.", String.Empty);
}
Console.WriteLine(dbConfig.Name + " - Preparing to begin polling");
preloader_logger.AddNonError(dbConfig.Name + " - Preparing to begin polling");
Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Preparing to begin polling", String.Empty);
// Create the new bulk loader
Worker_BulkLoader newLoader = new Worker_BulkLoader(preloader_logger, dbConfig, verbose, logFileDirectory, pluginRootDirectory);
// Try to refresh to test database and engine connectivity
if (newLoader.Refresh_Settings_And_Item_List())
{
loaders.Add(newLoader);
}
else
{
Console.WriteLine(dbConfig.Name + " - Error pulling setting of configuration information");
preloader_logger.AddError(dbConfig.Name + " - Error pulling setting of configuration information");
}
}
// If no loaders past the tests above, done
if (loaders.Count == 0)
{
Console.WriteLine("Aborting since no valid instances found to process");
preloader_logger.AddError("Aborting since no valid instances found to process");
return;
}
// Set the maximum number of packages to process before moving to the next instance
if (loaders.Count > 1)
{
MultiInstance_Builder_Settings.Instance_Package_Limit = 100;
}
bool firstRun = true;
// Loop continually until the end hour is achieved
Builder_Operation_Flag_Enum abort_flag = Builder_Operation_Flag_Enum.STANDARD_OPERATION;
do
{
// Is it time to build any RSS/XML feeds?
if (DateTime.Compare(DateTime.Now, feedNextBuildTime) >= 0)
{
feedNextBuildTime = DateTime.Now.Add(new TimeSpan(0, 10, 0));
}
bool skip_sleep = false;
// Step through each instance
for (int i = 0; i < loaders.Count; i++)
{
if (loaders[i] != null)
{
// Get the instance
Single_Instance_Configuration dbInstance = instances[i];
// Set the database connection strings
Engine_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String;
SobekCM_Item_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String;
// Look for abort
if (CheckForAbort())
{
aborted = true;
if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED)
{
abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING;
}
break;
}
// Refresh all settings, etc..
loaders[i].Refresh_Settings_And_Item_List();
// Pull the abort/pause flag
Builder_Operation_Flag_Enum currentPauseFlag = Abort_Database_Mechanism.Builder_Operation_Flag;
// If not paused, run the prebuilder
if (currentPauseFlag != Builder_Operation_Flag_Enum.PAUSE_REQUESTED)
{
skip_sleep = skip_sleep || Run_BulkLoader(loaders[i], verbose);
// Look for abort
if ((!aborted) && (CheckForAbort()))
{
aborted = true;
if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED)
{
abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING;
}
break;
}
}
else
{
preloader_logger.AddNonError(dbInstance.Name + " - Building paused");
Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building temporarily PAUSED", String.Empty);
}
}
}
if (aborted)
{
break;
}
// Publish the log
publish_log_file(local_log_name);
// Sleep for correct number of milliseconds
if (!skip_sleep)
{
Thread.Sleep(1000 * time_between_polls);
}
} while (DateTime.Now.Hour < BULK_LOADER_END_HOUR);
// Do the final work for all of the different dbInstances
if (!aborted)
{
for (int i = 0; i < instances.Count; i++)
{
if (loaders[i] != null)
{
// Get the instance
Single_Instance_Configuration dbInstance = instances[i];
// Set the database flag
SobekCM_Item_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String;
Engine_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String;
// Pull the abort/pause flag
Builder_Operation_Flag_Enum currentPauseFlag2 = Abort_Database_Mechanism.Builder_Operation_Flag;
// If not paused, run the prebuilder
if (currentPauseFlag2 != Builder_Operation_Flag_Enum.PAUSE_REQUESTED)
{
// Initiate the recreation of the links between metadata and collections
Engine_Database.Admin_Update_Cached_Aggregation_Metadata_Links();
}
// Clear the memory
loaders[i].ReleaseResources();
}
}
}
else
{
// Mark the aborted in each instance
foreach (Single_Instance_Configuration dbConfig in instances)
{
if (dbConfig.Is_Active)
{
Console.WriteLine("Setting abort flag message in " + dbConfig.Name);
preloader_logger.AddNonError("Setting abort flag message in " + dbConfig.Name);
SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String;
Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String;
Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building ABORTED per request from database key", String.Empty);
// Save information about this last run
Engine_Database.Set_Setting("Builder Version", Engine_ApplicationCache_Gateway.Settings.Static.Current_Builder_Version);
Engine_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString());
Engine_Database.Set_Setting("Builder Last Message", "Building ABORTED per request");
// Finally, set the builder flag appropriately
if (abort_flag == Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED)
{
Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
}
}
}
}
// Publish the log
publish_log_file(local_log_name);
//// Initiate a solr/lucene index optimization since we are done loading for a while
//if (DateTime.Now.Day % 2 == 0)
//{
// if (Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL.Length > 0)
// {
// Console.WriteLine("Initiating Solr/Lucene document index optimization");
// Solr_Controller.Optimize_Document_Index(Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL);
// }
//}
//else
//{
// if (Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL.Length > 0)
// {
// Console.WriteLine("Initiating Solr/Lucene page index optimization");
// Solr_Controller.Optimize_Page_Index(Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL);
// }
//}
//// Sleep for twenty minutes to end this (the index rebuild might take some time)
//Thread.Sleep(1000 * 20 * 60);
}
/// <summary> Continuously execute the processes in a recurring background thread </summary>
public void Execute_In_Background()
{
// Determine the new log name
string log_name = "incoming_" + controllerStarted.Year + "_" + controllerStarted.Month.ToString().PadLeft(2, '0') + "_" + controllerStarted.Day.ToString().PadLeft(2, '0') + ".html";
string local_log_name = Path.Combine(logFileDirectory, log_name);
// Create the new log file
LogFileXhtml preloader_logger = null;
try
{
preloader_logger = new LogFileXhtml(local_log_name, "SobekCM Incoming Packages Log", "UFDC_Builder.exe", true);
}
catch (Exception ee)
{
Console.WriteLine("Error creating logfile: " + ee.Message);
}
// Configure builders to run and run some basic tests
if ((!Configure_Builders_To_Run(preloader_logger)) || (preloader_logger == null))
{
return;
}
// Set the variable which will control background execution
int time_between_polls = Engine_ApplicationCache_Gateway.Settings.Builder.Override_Seconds_Between_Polls.HasValue ? Engine_ApplicationCache_Gateway.Settings.Builder.Override_Seconds_Between_Polls.Value : 60;
if ((time_between_polls < 0) || (MultiInstance_Builder_Settings.Instances.Count == 1))
{
time_between_polls = Convert.ToInt32(Engine_ApplicationCache_Gateway.Settings.Builder.Seconds_Between_Polls);
}
// Loop continually until the end hour is achieved
Builder_Operation_Flag_Enum abort_flag = Builder_Operation_Flag_Enum.STANDARD_OPERATION;
do
{
// Check the current last write time on the config file versus the last read
if (File.Exists(configurationFile))
{
DateTime lastWriteTime = (new FileInfo(configurationFile)).LastWriteTime;
if (lastWriteTime > configReadTime)
{
write_nonerror("Configuration file change detected ... reloading", preloader_logger);
Configure_Builders_To_Run(preloader_logger);
}
}
bool skip_sleep = false;
// Step through each instance
for (int i = 0; i < loaders.Count; i++)
{
if (loaders[i] != null)
{
// Get the instance
Single_Instance_Configuration dbInstance = instances[i];
// Set the database connection strings
Engine_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String;
SobekCM_Item_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String;
// Look for abort
if (CheckForAbort())
{
aborted = true;
if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED)
{
abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING;
}
break;
}
// Refresh all settings, etc..
loaders[i].Refresh_Settings_And_Item_List();
// Pull the abort/pause flag
Builder_Operation_Flag_Enum currentPauseFlag = Abort_Database_Mechanism.Builder_Operation_Flag;
// If not paused, run the prebuilder
if (currentPauseFlag != Builder_Operation_Flag_Enum.PAUSE_REQUESTED)
{
skip_sleep = skip_sleep || Run_BulkLoader(loaders[i], verbose);
// Look for abort
if ((!aborted) && (CheckForAbort()))
{
aborted = true;
if (Abort_Database_Mechanism.Builder_Operation_Flag != Builder_Operation_Flag_Enum.NO_BUILDING_REQUESTED)
{
abort_flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING;
}
break;
}
}
else
{
preloader_logger.AddNonError(dbInstance.Name + " - Building paused");
Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building temporarily PAUSED", String.Empty);
}
}
}
if (aborted)
{
break;
}
// Publish the log
publish_log_file(local_log_name);
// Sleep for correct number of milliseconds
if (!skip_sleep)
{
Thread.Sleep(1000 * time_between_polls);
}
} while (DateTime.Now.Hour < BULK_LOADER_END_HOUR);
// Do the final work for all of the different dbInstances
if (!aborted)
{
for (int i = 0; i < instances.Count; i++)
{
if (loaders[i] != null)
{
// Get the instance
Single_Instance_Configuration dbInstance = instances[i];
// Set the database flag
SobekCM_Item_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String;
Engine_Database.Connection_String = dbInstance.DatabaseConnection.Connection_String;
// Pull the abort/pause flag
Builder_Operation_Flag_Enum currentPauseFlag2 = Abort_Database_Mechanism.Builder_Operation_Flag;
// If not paused, run the prebuilder
if (currentPauseFlag2 != Builder_Operation_Flag_Enum.PAUSE_REQUESTED)
{
// Initiate the recreation of the links between metadata and collections
Engine_Database.Admin_Update_Cached_Aggregation_Metadata_Links();
}
// Clear the memory
loaders[i].ReleaseResources();
}
}
}
else
{
// Mark the aborted in each instance
foreach (Single_Instance_Configuration dbConfig in instances)
{
if (dbConfig.Is_Active)
{
Console.WriteLine("Setting abort flag message in " + dbConfig.Name);
preloader_logger.AddNonError("Setting abort flag message in " + dbConfig.Name);
SobekCM_Item_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String;
Engine_Database.Connection_String = dbConfig.DatabaseConnection.Connection_String;
Engine_Database.Builder_Add_Log_Entry(-1, String.Empty, "Standard", "Building ABORTED per request from database key", String.Empty);
// Save information about this last run
Engine_Database.Set_Setting("Builder Version", Engine_ApplicationCache_Gateway.Settings.Static.Current_Builder_Version);
Engine_Database.Set_Setting("Builder Last Run Finished", DateTime.Now.ToString());
Engine_Database.Set_Setting("Builder Last Message", "Building ABORTED per request");
// Finally, set the builder flag appropriately
if (abort_flag == Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED)
{
Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.LAST_EXECUTION_ABORTED;
}
}
}
}
// Publish the log
publish_log_file(local_log_name);
//// Initiate a solr/lucene index optimization since we are done loading for a while
//if (DateTime.Now.Day % 2 == 0)
//{
// if (Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL.Length > 0)
// {
// Console.WriteLine("Initiating Solr/Lucene document index optimization");
// Solr_Controller.Optimize_Document_Index(Engine_ApplicationCache_Gateway.Settings.Document_Solr_Index_URL);
// }
//}
//else
//{
// if (Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL.Length > 0)
// {
// Console.WriteLine("Initiating Solr/Lucene page index optimization");
// Solr_Controller.Optimize_Page_Index(Engine_ApplicationCache_Gateway.Settings.Page_Solr_Index_URL);
// }
//}
//// Sleep for twenty minutes to end this (the index rebuild might take some time)
//Thread.Sleep(1000 * 20 * 60);
}
