private void Process_Single_Incoming_Package(Incoming_Digital_Resource ResourcePackage) { ResourcePackage.BuilderLogId = Add_NonError_To_Log("........Processing '" + ResourcePackage.Folder_Name + "'", "Standard", ResourcePackage.BibID + ":" + ResourcePackage.VID, ResourcePackage.METS_Type_String, -1); // Clear any existing error linked to this item Engine_Database.Builder_Clear_Item_Error_Log(ResourcePackage.BibID, ResourcePackage.VID, "SobekCM Builder"); // Before we save this or anything, let's see if this is truly a new resource ResourcePackage.NewPackage = (Engine_Database.Get_Item_Information(ResourcePackage.BibID, ResourcePackage.VID, null) == null); ResourcePackage.Package_Time = DateTime.Now; try { // Do all the item processing per instance config foreach (iSubmissionPackageModule thisModule in builderModules.ItemProcessModules) { //if ( superverbose) //{ // Add_NonError_To_Log("Running module " + thisModule.GetType().ToString(), true, ResourcePackage.BibID + ":" + ResourcePackage.VID, String.Empty, ResourcePackage.BuilderLogId); //} if (!thisModule.DoWork(ResourcePackage)) { Add_Error_To_Log("Unable to complete new/replacement for " + ResourcePackage.BibID + ":" + ResourcePackage.VID, ResourcePackage.BibID + ":" + ResourcePackage.VID, String.Empty, ResourcePackage.BuilderLogId); // Try to move the whole package to the failures folder string final_failures_folder = Path.Combine(ResourcePackage.Source_Folder.Failures_Folder, ResourcePackage.BibID + "_" + ResourcePackage.VID); if (Directory.Exists(final_failures_folder)) { final_failures_folder = final_failures_folder + "_" + DateTime.Now.Year + "_" + DateTime.Now.Month.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Day.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Hour.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Minute.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Second.ToString().PadLeft(2, '0'); } try { Directory.Move(ResourcePackage.Resource_Folder, final_failures_folder); } catch { } return; } } // Save these collections to mark them for refreshing the RSS feeds, etc.. Add_Process_Info_To_PostProcess_Lists(ResourcePackage.BibID, ResourcePackage.VID, ResourcePackage.Metadata.Behaviors.Aggregation_Code_List); // Finally, clear the memory a little bit ResourcePackage.Clear_METS(); } catch (Exception ee) { StreamWriter errorWriter = new StreamWriter(logFileDirectory + "\\error.log", true); errorWriter.WriteLine("Message: " + ee.Message); errorWriter.WriteLine("Stack Trace: " + ee.StackTrace); errorWriter.Flush(); errorWriter.Close(); Add_Error_To_Log("Unable to complete new/replacement for " + ResourcePackage.BibID + ":" + ResourcePackage.VID, ResourcePackage.BibID + ":" + ResourcePackage.VID, String.Empty, ResourcePackage.BuilderLogId, ee); } }
/// <summary> Indexes all the items within a SobekCM library or a single item aggregation within a SobekCM library </summary> /// <param name="SolrDocumentUrl"> URL for the solr/lucene core used for searching for a single document within the library </param> /// <param name="SolrPageUrl"> URL for the solr/lucene core used for searching within a single document for matching pages </param> /// <param name="File_Location"> Location where all resource files are located </param> /// <param name="Collection"> Code the item aggreagtion to index, or empty string to index the entire library </param> public static void Index_Collection(string SolrDocumentUrl, string SolrPageUrl, string File_Location, string Collection) { // Initialize the document-level Solr/Lucene worker and add the solr url Startup.Init <SolrDocument>(SolrDocumentUrl); var solrDocumentWorker = ServiceLocator.Current.GetInstance <ISolrOperations <SolrDocument> >(); // Initialize the page-level Solr/Lucene worker and add the solr url Startup.Init <SolrPage>(SolrPageUrl); var solrPageWorker = ServiceLocator.Current.GetInstance <ISolrOperations <SolrPage> >(); // Get the start time DateTime startTime = DateTime.Now; // Get the list of all items in this collection int itemcount = 1; int sincelastcommit = 0; DataSet items = Engine_Database.Simple_Item_List(Collection, null); List <SolrDocument> index_files = new List <SolrDocument>(); List <SolrPage> index_pages = new List <SolrPage>(); // Temporarily write each bib:vid StreamWriter bibVidWriter = new StreamWriter("bib_vid_list.txt"); foreach (DataRow thisRow in items.Tables[0].Rows) { string bibid = thisRow[0].ToString(); string vid = thisRow[1].ToString(); bibVidWriter.WriteLine(bibid + ":" + vid); } bibVidWriter.Flush(); bibVidWriter.Close(); // Temporarily log this StreamWriter logWriter = new StreamWriter("log" + DateTime.Now.Year + DateTime.Now.Month.ToString().PadLeft(2, '0') + DateTime.Now.Day.ToString().PadLeft(2, '0') + ".txt", false); // Step through each row foreach (DataRow thisRow in items.Tables[0].Rows) { string bibid = thisRow[0].ToString(); string vid = thisRow[1].ToString(); string directory = File_Location + bibid.Substring(0, 2) + "\\" + bibid.Substring(2, 2) + "\\" + bibid.Substring(4, 2) + "\\" + bibid.Substring(6, 2) + "\\" + bibid.Substring(8) + "\\" + vid.PadLeft(5, '0'); string metsFile = directory + "\\" + bibid + "_" + vid + ".mets.xml"; if ((Directory.Exists(directory)) && (File.Exists(metsFile))) { Console.WriteLine(itemcount.ToString() + @":" + bibid + @":" + vid); // Read a METS file SobekCM_Item item = SobekCM_Item.Read_METS(metsFile); // Only continue if this is not NULL if (item != null) { logWriter.WriteLine(itemcount.ToString() + ":" + bibid + ":" + vid); // Pull some data from the database DataSet itemInfoSet = Engine_Database.Get_Item_Information(bibid, vid, true, null); if ((itemInfoSet != null) && (itemInfoSet.Tables[0].Rows.Count > 0)) { DataRow itemRow = itemInfoSet.Tables[0].Rows[0]; // Copy over the serial hierarchy item.Behaviors.Serial_Info.Clear(); string level1_text = itemRow["Level1_Text"].ToString(); if (level1_text.Length > 0) { item.Behaviors.Serial_Info.Add_Hierarchy(0, Convert.ToInt32(itemRow["Level1_Index"]), level1_text); string level2_text = itemRow["Level2_Text"].ToString(); if (level2_text.Length > 0) { item.Behaviors.Serial_Info.Add_Hierarchy(0, Convert.ToInt32(itemRow["Level2_Index"]), level2_text); string level3_text = itemRow["Level3_Text"].ToString(); if (level1_text.Length > 0) { item.Behaviors.Serial_Info.Add_Hierarchy(0, Convert.ToInt32(itemRow["Level3_Index"]), level3_text); } } } // Copy the main thumbnail item.Behaviors.Main_Thumbnail = itemRow["MainThumbnailFile"].ToString(); long aleph = Convert.ToInt64(itemRow["ALEPH_Number"]); long oclc = Convert.ToInt64(itemRow["OCLC_Number"]); if (aleph > 1) { item.Bib_Info.ALEPH_Record = aleph.ToString(); } if (oclc > 1) { item.Bib_Info.OCLC_Record = oclc.ToString(); } // Set the aggregationPermissions item.Behaviors.Clear_Aggregations(); foreach (DataRow thisAggrRow in itemInfoSet.Tables[1].Rows) { string code = thisAggrRow["Code"].ToString(); string name = thisAggrRow["Name"].ToString(); item.Behaviors.Add_Aggregation(code, name); } } // Add this document to the list of documents to index index_files.Add(new SolrDocument(item, directory)); // Index five documents at a time, since this could be alot of pages at a time if (index_files.Count > 4) { logWriter.Flush(); // Add to document index logWriter.WriteLine("ADDING TO DOCUMENT INDEX"); Console.WriteLine(@"Adding to Lucene/Solr Document Index"); bool document_success = false; int document_attempts = 0; while (!document_success) { try { solrDocumentWorker.Add(index_files); document_success = true; } catch (Exception) { if (document_attempts > 5) { throw; } document_attempts++; logWriter.WriteLine("ERROR " + document_attempts); Console.WriteLine(@"ERROR " + document_attempts); Thread.Sleep(document_attempts * 1000); } } // Add each page to be indexed foreach (SolrDocument document in index_files) { index_pages.AddRange(document.Solr_Pages); } // Add to page index logWriter.WriteLine("ADDING TO PAGE INDEX"); Console.WriteLine(@"Adding to Lucene/Solr Page Index"); bool page_success = false; int page_attempts = 0; while (!page_success) { try { solrPageWorker.Add(index_pages); page_success = true; } catch (Exception) { if (page_attempts > 5) { throw; } page_attempts++; logWriter.WriteLine("ERROR " + page_attempts); Console.WriteLine(@"ERROR " + page_attempts); Thread.Sleep(page_attempts * 1000); } } // Clear the documents and pages index_files.Clear(); index_pages.Clear(); if (sincelastcommit > 500) { logWriter.WriteLine("DOCUMENT COMMIT ( " + DateTime.Now.ToString() + " )"); Console.WriteLine(@"Comitting Changes to Lucene/Solr Document Index ( {0} )", DateTime.Now.ToString()); try { solrDocumentWorker.Commit(); } catch { logWriter.WriteLine("ERROR CAUGHT DURING COMMIT ( " + DateTime.Now.ToString() + " )"); Console.WriteLine(@"Error caught during document commit ( {0} )", DateTime.Now.ToString()); Thread.Sleep(10 * 60 * 1000); } logWriter.WriteLine("PAGE COMMIT ( " + DateTime.Now.ToString() + " )"); Console.WriteLine(@"Comitting Changes to Lucene/Solr Page Index ( {0} )", DateTime.Now.ToString()); try { solrPageWorker.Commit(); } catch { logWriter.WriteLine("ERROR CAUGHT DURING COMMIT ( " + DateTime.Now.ToString() + " )"); Console.WriteLine(@"Error caught during document commit ( {0} )", DateTime.Now.ToString()); Thread.Sleep(10 * 60 * 1000); } sincelastcommit = 0; //if (commitssinceoptimize >= 5) //{ // logWriter.WriteLine("DOCUMENT OPTIMIZE ( " + DateTime.Now.ToString() + " )"); // Console.WriteLine("Optimizing Lucene/Solr Document Index ( " + DateTime.Now.ToString() + " )"); // try // { // solrDocumentWorker.Optimize(); // } // catch (Exception ee) // { // logWriter.WriteLine("ERROR CAUGHT DURING OPTIMIZE ( " + DateTime.Now.ToString() + " )"); // Console.WriteLine("Error caught during document optimize ( " + DateTime.Now.ToString() + " )"); // Thread.Sleep(10 * 60 * 1000); // } // logWriter.WriteLine("PAGE OPTIMIZE ( " + DateTime.Now.ToString() + " )"); // Console.WriteLine("Optimizing Lucene/Solr Page Index ( " + DateTime.Now.ToString() + " )"); // try // { // solrPageWorker.Optimize(); // } // catch (Exception ee) // { // logWriter.WriteLine("ERROR CAUGHT DURING OPTIMIZE ( " + DateTime.Now.ToString() + " )"); // Console.WriteLine("Error caught during document optimize ( " + DateTime.Now.ToString() + " )"); // Thread.Sleep(10 * 60 * 1000); // } // commitssinceoptimize = 0; //} } } } sincelastcommit++; } itemcount++; } if (index_files.Count > 0) { logWriter.Flush(); // Add to document index Console.WriteLine(@"Adding to Lucene/Solr Document Index"); solrDocumentWorker.Add(index_files); // Add each page to be indexed foreach (SolrDocument document in index_files) { index_pages.AddRange(document.Solr_Pages); } // Add to page index Console.WriteLine(@"Adding to Lucene/Solr Page Index"); solrPageWorker.Add(index_pages); // Clear the documents and pages index_files.Clear(); index_pages.Clear(); } // Comit the changes to the solr/lucene index logWriter.WriteLine("DOCUMENT COMMIT ( " + DateTime.Now.ToString() + " )"); Console.WriteLine(@"Comitting Changes to Lucene/Solr Document Index ( {0} )", DateTime.Now.ToString()); try { solrDocumentWorker.Commit(); } catch { logWriter.WriteLine("ERROR CAUGHT DURING COMMIT ( " + DateTime.Now.ToString() + " )"); Console.WriteLine(@"Error caught during document commit ( {0} )", DateTime.Now.ToString()); Thread.Sleep(10 * 60 * 1000); } logWriter.WriteLine("PAGE COMMIT ( " + DateTime.Now.ToString() + " )"); Console.WriteLine(@"Comitting Changes to Lucene/Solr Page Index ( {0} )", DateTime.Now.ToString()); try { solrPageWorker.Commit(); } catch { logWriter.WriteLine("ERROR CAUGHT DURING COMMIT ( " + DateTime.Now.ToString() + " )"); Console.WriteLine(@"Error caught during document commit ( {0} )", DateTime.Now.ToString()); Thread.Sleep(10 * 60 * 1000); } logWriter.WriteLine("Final document optimize"); Console.WriteLine(@"Final document optimize"); try { solrDocumentWorker.Optimize(); } catch (Exception) { // Do not do anything here. It may throw an exception when it runs very longs } Thread.Sleep(30 * 60 * 1000); logWriter.WriteLine("Final page optimize"); Console.WriteLine(@"Final page optimize"); try { solrPageWorker.Optimize(); } catch (Exception) { // Do not do anything here. It may throw an exception when it runs very longs } Thread.Sleep(30 * 60 * 1000); // Add final meessage Console.WriteLine(@"Process Complete at {0}", DateTime.Now.ToString()); Console.WriteLine(@"Process Started at {0}", startTime.ToString()); Console.WriteLine(); Console.WriteLine(@"Enter any key to exit:"); Console.ReadKey(); logWriter.Flush(); logWriter.Close(); }
private void Process_All_Deletes(List <Incoming_Digital_Resource> Deletes) { if (Deletes.Count == 0) { return; } Add_NonError_To_Log("....Processing delete packages", "Standard", String.Empty, String.Empty, -1); Deletes.Sort(); foreach (Incoming_Digital_Resource deleteResource in Deletes) { // Check for abort if (CheckForAbort()) { Abort_Database_Mechanism.Builder_Operation_Flag = Builder_Operation_Flag_Enum.ABORTING; return; } // Save these collections to mark them for search index building Add_Delete_Info_To_PostProcess_Lists(deleteResource.BibID, deleteResource.VID, deleteResource.Metadata.Behaviors.Aggregation_Code_List); // Only continue if this bibid/vid exists if (Engine_Database.Get_Item_Information(deleteResource.BibID, deleteResource.VID, null) != null) { // Do all the item processing per instance config foreach (iSubmissionPackageModule thisModule in builderModules.DeleteItemModules) { if (!thisModule.DoWork(deleteResource)) { Add_Error_To_Log("Unable to complete delete for " + deleteResource.BibID + ":" + deleteResource.VID, deleteResource.BibID + ":" + deleteResource.VID, String.Empty, deleteResource.BuilderLogId); // Try to move the whole package to the failures folder string final_failures_folder = Path.Combine(deleteResource.Source_Folder.Failures_Folder, deleteResource.BibID + "_" + deleteResource.VID); if (Directory.Exists(final_failures_folder)) { final_failures_folder = final_failures_folder + "_" + DateTime.Now.Year + "_" + DateTime.Now.Month.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Day.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Hour.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Minute.ToString().PadLeft(2, '0') + "_" + DateTime.Now.Second.ToString().PadLeft(2, '0'); } try { Directory.Move(deleteResource.Resource_Folder, final_failures_folder); } catch { // Do nothing if not able to move? } return; } } } else { Add_Error_To_Log("Delete ( " + deleteResource.BibID + ":" + deleteResource.VID + " ) invalid... no pre-existing resource", deleteResource.BibID + ":" + deleteResource.VID, deleteResource.METS_Type_String, deleteResource.BuilderLogId); // Finally, clear the memory a little bit deleteResource.Clear_METS(); // Delete the handled METS file and package deleteResource.Delete(); } } }