private void Process_Single_Incoming_Package(Incoming_Digital_Resource ResourcePackage)
{
ResourcePackage.BuilderLogId = Add_NonError_To_Log("........Processing '" + ResourcePackage.Folder_Name + "'", "Standard", ResourcePackage.BibID + ":" + ResourcePackage.VID, ResourcePackage.METS_Type_String, -1);
// Clear any existing error linked to this item
Engine_Database.Builder_Clear_Item_Error_Log(ResourcePackage.BibID, ResourcePackage.VID, "SobekCM Builder");
// Before we save this or anything, let's see if this is truly a new resource
ResourcePackage.NewPackage = (Engine_Database.Get_Item_Information(ResourcePackage.BibID, ResourcePackage.VID, null) == null);
ResourcePackage.Package_Time = DateTime.Now;
try
{
// Do all the item processing per instance config
foreach (iSubmissionPackageModule thisModule in builderModules.ItemProcessModules)
{
//if ( superverbose)
//{
// Add_NonError_To_Log("Running module " + thisModule.GetType().ToString(), true, ResourcePackage.BibID + ":" + ResourcePackage.VID, String.Empty, ResourcePackage.BuilderLogId);
//}
if (!thisModule.DoWork(ResourcePackage))
{
Add_Error_To_Log("Unable to complete new/replacement for " + ResourcePackage.BibID + ":" + ResourcePackage.VID, ResourcePackage.BibID + ":" + ResourcePackage.VID, String.Empty, ResourcePackage.BuilderLogId);
// Try to move the whole package to the failures folder
string final_failures_folder = Path.Combine(ResourcePackage.Source_Folder.Failures_Folder, ResourcePackage.BibID + "_" + ResourcePackage.VID);
if (Directory.Exists(final_failures_folder))
{
final_failures_folder = final_failures_folder + "_" + DateTime.Now.Year + "_" + DateTime.Now.Month.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Day.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Hour.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Minute.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Second.ToString().PadLeft(2, '0');
}
try
{
Directory.Move(ResourcePackage.Resource_Folder, final_failures_folder);
}
catch
{
}
return;
}
}
// Save these collections to mark them for refreshing the RSS feeds, etc..
Add_Process_Info_To_PostProcess_Lists(ResourcePackage.BibID, ResourcePackage.VID, ResourcePackage.Metadata.Behaviors.Aggregation_Code_List);
// Finally, clear the memory a little bit
ResourcePackage.Clear_METS();
}
catch (Exception ee)
{
StreamWriter errorWriter = new StreamWriter(logFileDirectory + "\\error.log", true);
errorWriter.WriteLine("Message: " + ee.Message);
errorWriter.WriteLine("Stack Trace: " + ee.StackTrace);
errorWriter.Flush();
errorWriter.Close();
Add_Error_To_Log("Unable to complete new/replacement for " + ResourcePackage.BibID + ":" + ResourcePackage.VID, ResourcePackage.BibID + ":" + ResourcePackage.VID, String.Empty, ResourcePackage.BuilderLogId, ee);
}
}
/// <summary> Indexes all the items within a SobekCM library or a single item aggregation within a SobekCM library </summary>
/// <param name="SolrDocumentUrl"> URL for the solr/lucene core used for searching for a single document within the library </param>
/// <param name="SolrPageUrl"> URL for the solr/lucene core used for searching within a single document for matching pages </param>
/// <param name="File_Location"> Location where all resource files are located </param>
/// <param name="Collection"> Code the item aggreagtion to index, or empty string to index the entire library </param>
public static void Index_Collection(string SolrDocumentUrl, string SolrPageUrl, string File_Location, string Collection)
{
// Initialize the document-level Solr/Lucene worker and add the solr url
Startup.Init <SolrDocument>(SolrDocumentUrl);
var solrDocumentWorker = ServiceLocator.Current.GetInstance <ISolrOperations <SolrDocument> >();
// Initialize the page-level Solr/Lucene worker and add the solr url
Startup.Init <SolrPage>(SolrPageUrl);
var solrPageWorker = ServiceLocator.Current.GetInstance <ISolrOperations <SolrPage> >();
// Get the start time
DateTime startTime = DateTime.Now;
// Get the list of all items in this collection
int itemcount = 1;
int sincelastcommit = 0;
DataSet items = Engine_Database.Simple_Item_List(Collection, null);
List <SolrDocument> index_files = new List <SolrDocument>();
List <SolrPage> index_pages = new List <SolrPage>();
// Temporarily write each bib:vid
StreamWriter bibVidWriter = new StreamWriter("bib_vid_list.txt");
foreach (DataRow thisRow in items.Tables[0].Rows)
{
string bibid = thisRow[0].ToString();
string vid = thisRow[1].ToString();
bibVidWriter.WriteLine(bibid + ":" + vid);
}
bibVidWriter.Flush();
bibVidWriter.Close();
// Temporarily log this
StreamWriter logWriter = new StreamWriter("log" + DateTime.Now.Year + DateTime.Now.Month.ToString().PadLeft(2, '0') + DateTime.Now.Day.ToString().PadLeft(2, '0') + ".txt", false);
// Step through each row
foreach (DataRow thisRow in items.Tables[0].Rows)
{
string bibid = thisRow[0].ToString();
string vid = thisRow[1].ToString();
string directory = File_Location + bibid.Substring(0, 2) + "\\" + bibid.Substring(2, 2) + "\\" + bibid.Substring(4, 2) + "\\" + bibid.Substring(6, 2) + "\\" + bibid.Substring(8) + "\\" + vid.PadLeft(5, '0');
string metsFile = directory + "\\" + bibid + "_" + vid + ".mets.xml";
if ((Directory.Exists(directory)) && (File.Exists(metsFile)))
{
Console.WriteLine(itemcount.ToString() + @":" + bibid + @":" + vid);
// Read a METS file
SobekCM_Item item = SobekCM_Item.Read_METS(metsFile);
// Only continue if this is not NULL
if (item != null)
{
logWriter.WriteLine(itemcount.ToString() + ":" + bibid + ":" + vid);
// Pull some data from the database
DataSet itemInfoSet = Engine_Database.Get_Item_Information(bibid, vid, true, null);
if ((itemInfoSet != null) && (itemInfoSet.Tables[0].Rows.Count > 0))
{
DataRow itemRow = itemInfoSet.Tables[0].Rows[0];
// Copy over the serial hierarchy
item.Behaviors.Serial_Info.Clear();
string level1_text = itemRow["Level1_Text"].ToString();
if (level1_text.Length > 0)
{
item.Behaviors.Serial_Info.Add_Hierarchy(0, Convert.ToInt32(itemRow["Level1_Index"]), level1_text);
string level2_text = itemRow["Level2_Text"].ToString();
if (level2_text.Length > 0)
{
item.Behaviors.Serial_Info.Add_Hierarchy(0, Convert.ToInt32(itemRow["Level2_Index"]), level2_text);
string level3_text = itemRow["Level3_Text"].ToString();
if (level1_text.Length > 0)
{
item.Behaviors.Serial_Info.Add_Hierarchy(0, Convert.ToInt32(itemRow["Level3_Index"]), level3_text);
}
}
}
// Copy the main thumbnail
item.Behaviors.Main_Thumbnail = itemRow["MainThumbnailFile"].ToString();
long aleph = Convert.ToInt64(itemRow["ALEPH_Number"]);
long oclc = Convert.ToInt64(itemRow["OCLC_Number"]);
if (aleph > 1)
{
item.Bib_Info.ALEPH_Record = aleph.ToString();
}
if (oclc > 1)
{
item.Bib_Info.OCLC_Record = oclc.ToString();
}
// Set the aggregationPermissions
item.Behaviors.Clear_Aggregations();
foreach (DataRow thisAggrRow in itemInfoSet.Tables[1].Rows)
{
string code = thisAggrRow["Code"].ToString();
string name = thisAggrRow["Name"].ToString();
item.Behaviors.Add_Aggregation(code, name);
}
}
// Add this document to the list of documents to index
index_files.Add(new SolrDocument(item, directory));
// Index five documents at a time, since this could be alot of pages at a time
if (index_files.Count > 4)
{
logWriter.Flush();
// Add to document index
logWriter.WriteLine("ADDING TO DOCUMENT INDEX");
Console.WriteLine(@"Adding to Lucene/Solr Document Index");
bool document_success = false;
int document_attempts = 0;
while (!document_success)
{
try
{
solrDocumentWorker.Add(index_files);
document_success = true;
}
catch (Exception)
{
if (document_attempts > 5)
{
throw;
}
document_attempts++;
logWriter.WriteLine("ERROR " + document_attempts);
Console.WriteLine(@"ERROR " + document_attempts);
Thread.Sleep(document_attempts * 1000);
}
}
// Add each page to be indexed
foreach (SolrDocument document in index_files)
{
index_pages.AddRange(document.Solr_Pages);
}
// Add to page index
logWriter.WriteLine("ADDING TO PAGE INDEX");
Console.WriteLine(@"Adding to Lucene/Solr Page Index");
bool page_success = false;
int page_attempts = 0;
while (!page_success)
{
try
{
solrPageWorker.Add(index_pages);
page_success = true;
}
catch (Exception)
{
if (page_attempts > 5)
{
throw;
}
page_attempts++;
logWriter.WriteLine("ERROR " + page_attempts);
Console.WriteLine(@"ERROR " + page_attempts);
Thread.Sleep(page_attempts * 1000);
}
}
// Clear the documents and pages
index_files.Clear();
index_pages.Clear();
if (sincelastcommit > 500)
{
logWriter.WriteLine("DOCUMENT COMMIT ( " + DateTime.Now.ToString() + " )");
Console.WriteLine(@"Comitting Changes to Lucene/Solr Document Index ( {0} )", DateTime.Now.ToString());
try
{
solrDocumentWorker.Commit();
}
catch
{
logWriter.WriteLine("ERROR CAUGHT DURING COMMIT ( " + DateTime.Now.ToString() + " )");
Console.WriteLine(@"Error caught during document commit ( {0} )", DateTime.Now.ToString());
Thread.Sleep(10 * 60 * 1000);
}
logWriter.WriteLine("PAGE COMMIT ( " + DateTime.Now.ToString() + " )");
Console.WriteLine(@"Comitting Changes to Lucene/Solr Page Index ( {0} )", DateTime.Now.ToString());
try
{
solrPageWorker.Commit();
}
catch
{
logWriter.WriteLine("ERROR CAUGHT DURING COMMIT ( " + DateTime.Now.ToString() + " )");
Console.WriteLine(@"Error caught during document commit ( {0} )", DateTime.Now.ToString());
Thread.Sleep(10 * 60 * 1000);
}
sincelastcommit = 0;
//if (commitssinceoptimize >= 5)
//{
// logWriter.WriteLine("DOCUMENT OPTIMIZE ( " + DateTime.Now.ToString() + " )");
// Console.WriteLine("Optimizing Lucene/Solr Document Index ( " + DateTime.Now.ToString() + " )");
// try
// {
// solrDocumentWorker.Optimize();
// }
// catch (Exception ee)
// {
// logWriter.WriteLine("ERROR CAUGHT DURING OPTIMIZE ( " + DateTime.Now.ToString() + " )");
// Console.WriteLine("Error caught during document optimize ( " + DateTime.Now.ToString() + " )");
// Thread.Sleep(10 * 60 * 1000);
// }
// logWriter.WriteLine("PAGE OPTIMIZE ( " + DateTime.Now.ToString() + " )");
// Console.WriteLine("Optimizing Lucene/Solr Page Index ( " + DateTime.Now.ToString() + " )");
// try
// {
// solrPageWorker.Optimize();
// }
// catch (Exception ee)
// {
// logWriter.WriteLine("ERROR CAUGHT DURING OPTIMIZE ( " + DateTime.Now.ToString() + " )");
// Console.WriteLine("Error caught during document optimize ( " + DateTime.Now.ToString() + " )");
// Thread.Sleep(10 * 60 * 1000);
// }
// commitssinceoptimize = 0;
//}
}
}
}
sincelastcommit++;
}
itemcount++;
}
if (index_files.Count > 0)
{
logWriter.Flush();
// Add to document index
Console.WriteLine(@"Adding to Lucene/Solr Document Index");
solrDocumentWorker.Add(index_files);
// Add each page to be indexed
foreach (SolrDocument document in index_files)
{
index_pages.AddRange(document.Solr_Pages);
}
// Add to page index
Console.WriteLine(@"Adding to Lucene/Solr Page Index");
solrPageWorker.Add(index_pages);
// Clear the documents and pages
index_files.Clear();
index_pages.Clear();
}
// Comit the changes to the solr/lucene index
logWriter.WriteLine("DOCUMENT COMMIT ( " + DateTime.Now.ToString() + " )");
Console.WriteLine(@"Comitting Changes to Lucene/Solr Document Index ( {0} )", DateTime.Now.ToString());
try
{
solrDocumentWorker.Commit();
}
catch
{
logWriter.WriteLine("ERROR CAUGHT DURING COMMIT ( " + DateTime.Now.ToString() + " )");
Console.WriteLine(@"Error caught during document commit ( {0} )", DateTime.Now.ToString());
Thread.Sleep(10 * 60 * 1000);
}
logWriter.WriteLine("PAGE COMMIT ( " + DateTime.Now.ToString() + " )");
Console.WriteLine(@"Comitting Changes to Lucene/Solr Page Index ( {0} )", DateTime.Now.ToString());
try
{
solrPageWorker.Commit();
}
catch
{
logWriter.WriteLine("ERROR CAUGHT DURING COMMIT ( " + DateTime.Now.ToString() + " )");
Console.WriteLine(@"Error caught during document commit ( {0} )", DateTime.Now.ToString());
Thread.Sleep(10 * 60 * 1000);
}
logWriter.WriteLine("Final document optimize");
Console.WriteLine(@"Final document optimize");
try
{
solrDocumentWorker.Optimize();
}
catch (Exception)
{
// Do not do anything here. It may throw an exception when it runs very longs
}
Thread.Sleep(30 * 60 * 1000);
logWriter.WriteLine("Final page optimize");
Console.WriteLine(@"Final page optimize");
try
{
solrPageWorker.Optimize();
}
catch (Exception)
{
// Do not do anything here. It may throw an exception when it runs very longs
}
Thread.Sleep(30 * 60 * 1000);
// Add final meessage
Console.WriteLine(@"Process Complete at {0}", DateTime.Now.ToString());
Console.WriteLine(@"Process Started at {0}", startTime.ToString());
Console.WriteLine();
Console.WriteLine(@"Enter any key to exit:");
Console.ReadKey();
logWriter.Flush();
logWriter.Close();
}
private void Process_All_Deletes(List <Incoming_Digital_Resource> Deletes)
{
if (Deletes.Count == 0)
{
return;
}
Add_NonError_To_Log("....Processing delete packages", "Standard", String.Empty, String.Empty, -1);
Deletes.Sort();
foreach (Incoming_Digital_Resource deleteResource in Deletes)
{
// Check for abort
if (CheckForAbort())
{
Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING;
return;
}
// Save these collections to mark them for search index building
Add_Delete_Info_To_PostProcess_Lists(deleteResource.BibID, deleteResource.VID, deleteResource.Metadata.Behaviors.Aggregation_Code_List);
// Only continue if this bibid/vid exists
if (Engine_Database.Get_Item_Information(deleteResource.BibID, deleteResource.VID, null) != null)
{
// Do all the item processing per instance config
foreach (iSubmissionPackageModule thisModule in builderModules.DeleteItemModules)
{
if (!thisModule.DoWork(deleteResource))
{
Add_Error_To_Log("Unable to complete delete for " + deleteResource.BibID + ":" + deleteResource.VID, deleteResource.BibID + ":" + deleteResource.VID, String.Empty, deleteResource.BuilderLogId);
// Try to move the whole package to the failures folder
string final_failures_folder = Path.Combine(deleteResource.Source_Folder.Failures_Folder, deleteResource.BibID + "_" + deleteResource.VID);
if (Directory.Exists(final_failures_folder))
{
final_failures_folder = final_failures_folder + "_" + DateTime.Now.Year + "_" + DateTime.Now.Month.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Day.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Hour.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Minute.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Second.ToString().PadLeft(2, '0');
}
try
{
Directory.Move(deleteResource.Resource_Folder, final_failures_folder);
}
catch
{
// Do nothing if not able to move?
}
return;
}
}
}
else
{
Add_Error_To_Log("Delete ( " + deleteResource.BibID + ":" + deleteResource.VID + " ) invalid... no pre-existing resource", deleteResource.BibID + ":" + deleteResource.VID, deleteResource.METS_Type_String, deleteResource.BuilderLogId);
// Finally, clear the memory a little bit
deleteResource.Clear_METS();
// Delete the handled METS file and package
deleteResource.Delete();
}
}
}
