private static DictionaryBasedObject ReadFromStream_BINARY(byte[] data)
{
DictionaryBasedObject dbo = (DictionaryBasedObject)SerializeFile.LoadFromByteArray(data);
FeatureTrackingManager.Instance.UseFeature(Features.Legacy_Metadata_Binary);
return(dbo);
}
private static DictionaryBasedObject ReadFromDisk_BINARY(string filename)
{
DictionaryBasedObject dbo = (DictionaryBasedObject)SerializeFile.LoadRedundant(filename);
FeatureTrackingManager.Instance.UseFeature(Features.Legacy_Metadata_Binary);
return(dbo);
}
private void saveBranchToDisk(object key1)
{
try
{
SerializeFile.Save(base_filename + "." + key1.ToString(), trunk[key1]);
}
catch (Exception e)
{
Console.WriteLine("Error saving branch {0}: {1}", key1, e.Message);
}
}
// ---
private void fetchBranchFromDisk(object key1)
{
try
{
Hashtable branch = (Hashtable)SerializeFile.Load(base_filename + "." + key1.ToString());
trunk[key1] = branch;
}
catch (Exception e)
{
Console.WriteLine("Error loading branch {0}: {1}", key1, e.Message);
}
}
public void RebuildExpedition(int num_topics, bool add_autotags, bool add_tags)
{
try
{
ExpeditionDataSource eds = ExpeditionBuilder.BuildExpeditionDataSource(Library, num_topics, add_autotags, add_tags);
SerializeFile.SaveSafely(Filename_Store, eds);
expedition_data_source = eds;
}
finally
{
}
Logging.Info("-Rebuilding Expedition");
}
private void LoadKnownWebLibraries(string filename, bool only_load_those_libraries_which_are_actually_present)
{
Logging.Info("+Loading known Web Libraries");
try
{
if (File.Exists(filename))
{
KnownWebLibrariesFile known_web_libraries_file = SerializeFile.ProtoLoad <KnownWebLibrariesFile>(filename);
if (null != known_web_libraries_file.web_library_details)
{
//...
}
}
}
catch (Exception ex)
{
Logging.Error(ex, "There was a problem loading the known Web Libraries from config file {0}", filename);
}
Logging.Info("-Loading known Web Libraries");
}
public AITagManager(Library library)
{
this.library = new TypedWeakReference <Library>(library);
current_ai_tags_record = new AITags();
// Attempt to load the existing tags
try
{
if (File.Exists(Filename_Store))
{
Stopwatch clk = Stopwatch.StartNew();
Logging.Info("+Loading AutoTags");
current_ai_tags_record = SerializeFile.ProtoLoad <AITags>(Filename_Store);
Logging.Info("-Loading AutoTags (time spent: {0} ms)", clk.ElapsedMilliseconds);
}
}
catch (Exception ex)
{
Logging.Warn(ex, "There was a problem loading existing AutoTags");
}
}
internal static void ReadFromDisk(PDFDocument_ThreadUnsafe pdf_document, PDFInkList inks, Dictionary <string, byte[]> library_items_inks_cache)
{
try
{
byte[] inks_data = null;
if (null != library_items_inks_cache)
{
library_items_inks_cache.TryGetValue(pdf_document.Fingerprint, out inks_data);
}
else
{
List <LibraryDB.LibraryItem> library_items = pdf_document.Library.LibraryDB.GetLibraryItems(pdf_document.Fingerprint, PDFDocumentFileLocations.INKS);
if (0 < library_items.Count)
{
inks_data = library_items[0].data;
}
}
if (null != inks_data)
{
Dictionary <int, byte[]> page_ink_blobs = SerializeFile.ProtoLoadFromByteArray <Dictionary <int, byte[]> >(inks_data);
if (null != page_ink_blobs)
{
foreach (var pair in page_ink_blobs)
{
inks.AddPageInkBlob(pair.Key, pair.Value);
}
}
}
}
catch (Exception ex)
{
Logging.Error(ex, "There was a problem loading the Inks for document {0}", pdf_document.Fingerprint);
}
}
private void SaveKnownWebLibraries(string filename = null)
{
if (null == filename)
{
filename = KNOWN_WEB_LIBRARIES_FILENAME;
}
Logging.Info("+Saving known Web Libraries to {0}", filename);
try
{
KnownWebLibrariesFile known_web_libraries_file = new KnownWebLibrariesFile();
known_web_libraries_file.web_library_details = new List <WebLibraryDetail>();
foreach (WebLibraryDetail web_library_detail in web_library_details.Values)
{
// *************************************************************************************************************
// *** MIGRATION TO OPEN SOURCE CODE ***************************************************************************
// *************************************************************************************************************
// Don't remember the web libraries - let them be discovered by this
if ("Legacy" == web_library_detail.LibraryType())
{
continue;
}
// *************************************************************************************************************
known_web_libraries_file.web_library_details.Add(web_library_detail);
}
SerializeFile.ProtoSave <KnownWebLibrariesFile>(filename, known_web_libraries_file);
}
catch (Exception ex)
{
Logging.Error(ex, "There was a problem saving the known web libraries to file {0}", filename);
}
Logging.Info("-Saving known Web Libraries to {0}", filename);
}
public static void TestFastSaveLoad()
{
// Random data
double alpha = 0.1;
double beta = 0.2;
int num_topics = 200;
int num_words = 10000;
int num_docs = 20000;
int[][] WORDS_IN_DOCS = new int[num_docs][];
for (int i = 0; i < num_docs; ++i)
{
WORDS_IN_DOCS[i] = new int[i % 100 + 1];
}
LDASampler lda_sampler = new LDASampler(alpha, beta, num_topics, num_words, num_docs, WORDS_IN_DOCS);
{
string lda_sampler_filename = @"C:\temp\ldatest_old.dat";
Logging.Info("+OldSave");
SerializeFile.Save(lda_sampler_filename, lda_sampler);
Logging.Info("-OldSave");
Logging.Info("+OldLoad");
lda_sampler = (LDASampler)SerializeFile.Load(lda_sampler_filename);
Logging.Info("-OldLoad");
}
{
string lda_sampler_filename = @"C:\temp\ldatest_new.dat";
Logging.Info("+NewSave");
lda_sampler.FastSave(lda_sampler_filename);
Logging.Info("-NewSave");
Logging.Info("+NewLoad");
lda_sampler = FastLoad(lda_sampler_filename, WORDS_IN_DOCS);
Logging.Info("-NewLoad");
}
}
internal static Dictionary <string, string> GetHistoricalSyncFile(Library library)
{
Dictionary <string, string> historical_sync_file = (Dictionary <string, string>)SerializeFile.LoadSafely(GetFILENAME(library));
if (null == historical_sync_file)
{
Logging.Info("There is no sync history, so creating a new one");
historical_sync_file = new Dictionary <string, string>();
}
return(historical_sync_file);
}
private static List <DictionaryBasedObject> ReadFromStream_BINARY(byte[] data)
{
List <DictionaryBasedObject> annotation_dictionaries = (List <DictionaryBasedObject>)SerializeFile.LoadFromByteArray(data);
return(annotation_dictionaries);
}
private static List <PDFHighlight> ReadFromStream_PROTOBUF(byte[] data)
{
List <PDFHighlight> highlights_list = SerializeFile.ProtoLoadFromByteArray <List <PDFHighlight> >(data);
return(highlights_list);
}
private static DictionaryBasedObject ReadFromDisk_BINARY(string filename)
{
DictionaryBasedObject dbo = (DictionaryBasedObject)SerializeFile.LoadRedundant(filename);
return(dbo);
}
// ------------------------------------------------------------------------------------------------------------
private void WritePasswordFile(string filename, Dictionary <string, string> passwords)
{
Logging.Info("Writing password file {0}.", filename);
SerializeFile.SaveRedundant(filename, passwords);
}
internal static void PutHistoricalSyncFile(Library library, Dictionary <string, string> historical_sync_file)
{
SerializeFile.SaveSafely(GetFILENAME(library), historical_sync_file);
}
private void Init()
{
lock (libraryIndexInit_is_pending_lock)
{
Logging.Info("Try to load a historical progress file: {0}", Filename_DocumentProgressList);
try
{
if (File.Exists(Filename_DocumentProgressList))
{
Stopwatch clk = Stopwatch.StartNew();
Logging.Info("+Loading historical progress file: {0}", Filename_DocumentProgressList);
//Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (pdf_documents_in_library_lock)
{
//l1_clk.LockPerfTimerStop();
pdf_documents_in_library = (Dictionary <string, PDFDocumentInLibrary>)SerializeFile.LoadSafely(Filename_DocumentProgressList);
}
Logging.Info("-Loaded historical progress file: {0} (time spent: {1} ms)", Filename_DocumentProgressList, clk.ElapsedMilliseconds);
}
}
catch (Exception ex)
{
Logging.Error(ex, "FAILED to load historical progress file \"{0}\". Will start indexing afresh.", Filename_DocumentProgressList);
//Utilities.LockPerfTimer l2_clk = Utilities.LockPerfChecker.Start();
lock (pdf_documents_in_library_lock)
{
//l2_clk.LockPerfTimerStop();
pdf_documents_in_library = null;
}
}
// If there was no historical progress file, start afresh
//Utilities.LockPerfTimer l3_clk = Utilities.LockPerfChecker.Start();
lock (pdf_documents_in_library_lock)
{
//l3_clk.LockPerfTimerStop();
if (null == pdf_documents_in_library)
{
Logging.Warn("Cound not find any indexing progress, so starting from scratch.");
pdf_documents_in_library = new Dictionary <string, PDFDocumentInLibrary>();
}
}
}
}
private void Init(WebLibraryDetail web_library_detail)
{
// have we been here before?
if (LibraryIndexIsLoaded)
{
return;
}
// Utilities.LockPerfTimer l5_clk = Utilities.LockPerfChecker.Start();
lock (libraryIndexInit_is_pending_lock)
{
// l5_clk.LockPerfTimerStop();
//Utilities.LockPerfTimer l4_clk = Utilities.LockPerfChecker.Start();
lock (pdf_documents_in_library_lock)
{
lock (word_index_manager_lock)
{
//l4_clk.LockPerfTimerStop();
if (null != pdf_documents_in_library && null != word_index_manager)
{
Logging.Warn("LibraryIndex has already been initialized.");
return;
}
}
}
Logging.Info("Try to load a historical progress file: {0}", web_library_detail.FILENAME_DOCUMENT_PROGRESS_LIST);
try
{
if (File.Exists(web_library_detail.FILENAME_DOCUMENT_PROGRESS_LIST))
{
Stopwatch clk = Stopwatch.StartNew();
Logging.Info("+Loading historical progress file: {0}", web_library_detail.FILENAME_DOCUMENT_PROGRESS_LIST);
//Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (pdf_documents_in_library_lock)
{
//l1_clk.LockPerfTimerStop();
pdf_documents_in_library = (Dictionary <string, PDFDocumentInLibrary>)SerializeFile.LoadSafely(web_library_detail.FILENAME_DOCUMENT_PROGRESS_LIST);
}
Logging.Info("-Loaded historical progress file: {0} (time spent: {1} ms)", web_library_detail.FILENAME_DOCUMENT_PROGRESS_LIST, clk.ElapsedMilliseconds);
}
}
catch (Exception ex)
{
Logging.Error(ex, "FAILED to load historical progress file \"{0}\". Will start indexing afresh.", web_library_detail.FILENAME_DOCUMENT_PROGRESS_LIST);
//Utilities.LockPerfTimer l2_clk = Utilities.LockPerfChecker.Start();
lock (pdf_documents_in_library_lock)
{
//l2_clk.LockPerfTimerStop();
pdf_documents_in_library = null;
}
}
// If there was no historical progress file, start afresh
//Utilities.LockPerfTimer l3_clk = Utilities.LockPerfChecker.Start();
lock (pdf_documents_in_library_lock)
{
//l3_clk.LockPerfTimerStop();
if (null == pdf_documents_in_library)
{
Logging.Warn("Could not find any indexing progress, so starting from scratch.");
pdf_documents_in_library = new Dictionary <string, PDFDocumentInLibrary>();
}
}
// Utilities.LockPerfTimer l6_clk = Utilities.LockPerfChecker.Start();
lock (word_index_manager_lock)
{
// l6_clk.LockPerfTimerStop();
word_index_manager = new LuceneIndex(web_library_detail.LIBRARY_INDEX_BASE_PATH);
word_index_manager.WriteMasterList();
}
LibraryIndexIsLoaded = true;
}
}
private static DictionaryBasedObject ReadFromStream_BINARY(byte[] data)
{
DictionaryBasedObject dbo = (DictionaryBasedObject)SerializeFile.LoadFromByteArray(data);
return(dbo);
}
