private static DictionaryBasedObject ReadFromStream_BINARY(byte[] data) { DictionaryBasedObject dbo = (DictionaryBasedObject)SerializeFile.LoadFromByteArray(data); FeatureTrackingManager.Instance.UseFeature(Features.Legacy_Metadata_Binary); return(dbo); }
private static DictionaryBasedObject ReadFromDisk_BINARY(string filename) { DictionaryBasedObject dbo = (DictionaryBasedObject)SerializeFile.LoadRedundant(filename); FeatureTrackingManager.Instance.UseFeature(Features.Legacy_Metadata_Binary); return(dbo); }
private void saveBranchToDisk(object key1) { try { SerializeFile.Save(base_filename + "." + key1.ToString(), trunk[key1]); } catch (Exception e) { Console.WriteLine("Error saving branch {0}: {1}", key1, e.Message); } }
// --- private void fetchBranchFromDisk(object key1) { try { Hashtable branch = (Hashtable)SerializeFile.Load(base_filename + "." + key1.ToString()); trunk[key1] = branch; } catch (Exception e) { Console.WriteLine("Error loading branch {0}: {1}", key1, e.Message); } }
public void RebuildExpedition(int num_topics, bool add_autotags, bool add_tags) { try { ExpeditionDataSource eds = ExpeditionBuilder.BuildExpeditionDataSource(Library, num_topics, add_autotags, add_tags); SerializeFile.SaveSafely(Filename_Store, eds); expedition_data_source = eds; } finally { } Logging.Info("-Rebuilding Expedition"); }
private void LoadKnownWebLibraries(string filename, bool only_load_those_libraries_which_are_actually_present) { Logging.Info("+Loading known Web Libraries"); try { if (File.Exists(filename)) { KnownWebLibrariesFile known_web_libraries_file = SerializeFile.ProtoLoad <KnownWebLibrariesFile>(filename); if (null != known_web_libraries_file.web_library_details) { //... } } } catch (Exception ex) { Logging.Error(ex, "There was a problem loading the known Web Libraries from config file {0}", filename); } Logging.Info("-Loading known Web Libraries"); }
public AITagManager(Library library) { this.library = new TypedWeakReference <Library>(library); current_ai_tags_record = new AITags(); // Attempt to load the existing tags try { if (File.Exists(Filename_Store)) { Stopwatch clk = Stopwatch.StartNew(); Logging.Info("+Loading AutoTags"); current_ai_tags_record = SerializeFile.ProtoLoad <AITags>(Filename_Store); Logging.Info("-Loading AutoTags (time spent: {0} ms)", clk.ElapsedMilliseconds); } } catch (Exception ex) { Logging.Warn(ex, "There was a problem loading existing AutoTags"); } }
internal static void ReadFromDisk(PDFDocument_ThreadUnsafe pdf_document, PDFInkList inks, Dictionary <string, byte[]> library_items_inks_cache) { try { byte[] inks_data = null; if (null != library_items_inks_cache) { library_items_inks_cache.TryGetValue(pdf_document.Fingerprint, out inks_data); } else { List <LibraryDB.LibraryItem> library_items = pdf_document.Library.LibraryDB.GetLibraryItems(pdf_document.Fingerprint, PDFDocumentFileLocations.INKS); if (0 < library_items.Count) { inks_data = library_items[0].data; } } if (null != inks_data) { Dictionary <int, byte[]> page_ink_blobs = SerializeFile.ProtoLoadFromByteArray <Dictionary <int, byte[]> >(inks_data); if (null != page_ink_blobs) { foreach (var pair in page_ink_blobs) { inks.AddPageInkBlob(pair.Key, pair.Value); } } } } catch (Exception ex) { Logging.Error(ex, "There was a problem loading the Inks for document {0}", pdf_document.Fingerprint); } }
private void SaveKnownWebLibraries(string filename = null) { if (null == filename) { filename = KNOWN_WEB_LIBRARIES_FILENAME; } Logging.Info("+Saving known Web Libraries to {0}", filename); try { KnownWebLibrariesFile known_web_libraries_file = new KnownWebLibrariesFile(); known_web_libraries_file.web_library_details = new List <WebLibraryDetail>(); foreach (WebLibraryDetail web_library_detail in web_library_details.Values) { // ************************************************************************************************************* // *** MIGRATION TO OPEN SOURCE CODE *************************************************************************** // ************************************************************************************************************* // Don't remember the web libraries - let them be discovered by this if ("Legacy" == web_library_detail.LibraryType()) { continue; } // ************************************************************************************************************* known_web_libraries_file.web_library_details.Add(web_library_detail); } SerializeFile.ProtoSave <KnownWebLibrariesFile>(filename, known_web_libraries_file); } catch (Exception ex) { Logging.Error(ex, "There was a problem saving the known web libraries to file {0}", filename); } Logging.Info("-Saving known Web Libraries to {0}", filename); }
public static void TestFastSaveLoad() { // Random data double alpha = 0.1; double beta = 0.2; int num_topics = 200; int num_words = 10000; int num_docs = 20000; int[][] WORDS_IN_DOCS = new int[num_docs][]; for (int i = 0; i < num_docs; ++i) { WORDS_IN_DOCS[i] = new int[i % 100 + 1]; } LDASampler lda_sampler = new LDASampler(alpha, beta, num_topics, num_words, num_docs, WORDS_IN_DOCS); { string lda_sampler_filename = @"C:\temp\ldatest_old.dat"; Logging.Info("+OldSave"); SerializeFile.Save(lda_sampler_filename, lda_sampler); Logging.Info("-OldSave"); Logging.Info("+OldLoad"); lda_sampler = (LDASampler)SerializeFile.Load(lda_sampler_filename); Logging.Info("-OldLoad"); } { string lda_sampler_filename = @"C:\temp\ldatest_new.dat"; Logging.Info("+NewSave"); lda_sampler.FastSave(lda_sampler_filename); Logging.Info("-NewSave"); Logging.Info("+NewLoad"); lda_sampler = FastLoad(lda_sampler_filename, WORDS_IN_DOCS); Logging.Info("-NewLoad"); } }
internal static Dictionary <string, string> GetHistoricalSyncFile(Library library) { Dictionary <string, string> historical_sync_file = (Dictionary <string, string>)SerializeFile.LoadSafely(GetFILENAME(library)); if (null == historical_sync_file) { Logging.Info("There is no sync history, so creating a new one"); historical_sync_file = new Dictionary <string, string>(); } return(historical_sync_file); }
private static List <DictionaryBasedObject> ReadFromStream_BINARY(byte[] data) { List <DictionaryBasedObject> annotation_dictionaries = (List <DictionaryBasedObject>)SerializeFile.LoadFromByteArray(data); return(annotation_dictionaries); }
private static List <PDFHighlight> ReadFromStream_PROTOBUF(byte[] data) { List <PDFHighlight> highlights_list = SerializeFile.ProtoLoadFromByteArray <List <PDFHighlight> >(data); return(highlights_list); }
private static DictionaryBasedObject ReadFromDisk_BINARY(string filename) { DictionaryBasedObject dbo = (DictionaryBasedObject)SerializeFile.LoadRedundant(filename); return(dbo); }
// ------------------------------------------------------------------------------------------------------------ private void WritePasswordFile(string filename, Dictionary <string, string> passwords) { Logging.Info("Writing password file {0}.", filename); SerializeFile.SaveRedundant(filename, passwords); }
internal static void PutHistoricalSyncFile(Library library, Dictionary <string, string> historical_sync_file) { SerializeFile.SaveSafely(GetFILENAME(library), historical_sync_file); }
private void Init() { lock (libraryIndexInit_is_pending_lock) { Logging.Info("Try to load a historical progress file: {0}", Filename_DocumentProgressList); try { if (File.Exists(Filename_DocumentProgressList)) { Stopwatch clk = Stopwatch.StartNew(); Logging.Info("+Loading historical progress file: {0}", Filename_DocumentProgressList); //Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start(); lock (pdf_documents_in_library_lock) { //l1_clk.LockPerfTimerStop(); pdf_documents_in_library = (Dictionary <string, PDFDocumentInLibrary>)SerializeFile.LoadSafely(Filename_DocumentProgressList); } Logging.Info("-Loaded historical progress file: {0} (time spent: {1} ms)", Filename_DocumentProgressList, clk.ElapsedMilliseconds); } } catch (Exception ex) { Logging.Error(ex, "FAILED to load historical progress file \"{0}\". Will start indexing afresh.", Filename_DocumentProgressList); //Utilities.LockPerfTimer l2_clk = Utilities.LockPerfChecker.Start(); lock (pdf_documents_in_library_lock) { //l2_clk.LockPerfTimerStop(); pdf_documents_in_library = null; } } // If there was no historical progress file, start afresh //Utilities.LockPerfTimer l3_clk = Utilities.LockPerfChecker.Start(); lock (pdf_documents_in_library_lock) { //l3_clk.LockPerfTimerStop(); if (null == pdf_documents_in_library) { Logging.Warn("Cound not find any indexing progress, so starting from scratch."); pdf_documents_in_library = new Dictionary <string, PDFDocumentInLibrary>(); } } } }
private void Init(WebLibraryDetail web_library_detail) { // have we been here before? if (LibraryIndexIsLoaded) { return; } // Utilities.LockPerfTimer l5_clk = Utilities.LockPerfChecker.Start(); lock (libraryIndexInit_is_pending_lock) { // l5_clk.LockPerfTimerStop(); //Utilities.LockPerfTimer l4_clk = Utilities.LockPerfChecker.Start(); lock (pdf_documents_in_library_lock) { lock (word_index_manager_lock) { //l4_clk.LockPerfTimerStop(); if (null != pdf_documents_in_library && null != word_index_manager) { Logging.Warn("LibraryIndex has already been initialized."); return; } } } Logging.Info("Try to load a historical progress file: {0}", web_library_detail.FILENAME_DOCUMENT_PROGRESS_LIST); try { if (File.Exists(web_library_detail.FILENAME_DOCUMENT_PROGRESS_LIST)) { Stopwatch clk = Stopwatch.StartNew(); Logging.Info("+Loading historical progress file: {0}", web_library_detail.FILENAME_DOCUMENT_PROGRESS_LIST); //Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start(); lock (pdf_documents_in_library_lock) { //l1_clk.LockPerfTimerStop(); pdf_documents_in_library = (Dictionary <string, PDFDocumentInLibrary>)SerializeFile.LoadSafely(web_library_detail.FILENAME_DOCUMENT_PROGRESS_LIST); } Logging.Info("-Loaded historical progress file: {0} (time spent: {1} ms)", web_library_detail.FILENAME_DOCUMENT_PROGRESS_LIST, clk.ElapsedMilliseconds); } } catch (Exception ex) { Logging.Error(ex, "FAILED to load historical progress file \"{0}\". Will start indexing afresh.", web_library_detail.FILENAME_DOCUMENT_PROGRESS_LIST); //Utilities.LockPerfTimer l2_clk = Utilities.LockPerfChecker.Start(); lock (pdf_documents_in_library_lock) { //l2_clk.LockPerfTimerStop(); pdf_documents_in_library = null; } } // If there was no historical progress file, start afresh //Utilities.LockPerfTimer l3_clk = Utilities.LockPerfChecker.Start(); lock (pdf_documents_in_library_lock) { //l3_clk.LockPerfTimerStop(); if (null == pdf_documents_in_library) { Logging.Warn("Could not find any indexing progress, so starting from scratch."); pdf_documents_in_library = new Dictionary <string, PDFDocumentInLibrary>(); } } // Utilities.LockPerfTimer l6_clk = Utilities.LockPerfChecker.Start(); lock (word_index_manager_lock) { // l6_clk.LockPerfTimerStop(); word_index_manager = new LuceneIndex(web_library_detail.LIBRARY_INDEX_BASE_PATH); word_index_manager.WriteMasterList(); } LibraryIndexIsLoaded = true; } }
private static DictionaryBasedObject ReadFromStream_BINARY(byte[] data) { DictionaryBasedObject dbo = (DictionaryBasedObject)SerializeFile.LoadFromByteArray(data); return(dbo); }