public void RebuildExpedition(int num_topics, bool add_autotags, bool add_tags, RebuiltExpeditionCompleteDelegate rebuiltexpeditioncompletedelegate)
{
WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
StatusManager.Instance.UpdateStatus("Expedition", "Rebuilding Expedition");
try
{
Library.IsBusyRegeneratingTags = true;
ExpeditionDataSource eds = ExpeditionBuilder.BuildExpeditionDataSource(LibraryRef, num_topics, add_autotags, add_tags, ExpeditionBuilderProgressUpdate);
if (eds != null)
{
SerializeFile.SaveSafely(Filename_Store, eds);
expedition_data_source = eds;
}
}
finally
{
Library.IsBusyRegeneratingTags = false;
StatusManager.Instance.ClearCancelled("Expedition");
}
Logging.Info("-Rebuilding Expedition");
if (null != rebuiltexpeditioncompletedelegate)
{
Logging.Info("+Notifying of rebuilt Expedition");
rebuiltexpeditioncompletedelegate();
Logging.Info("-Notifying of rebuilt Expedition");
}
}
private static DictionaryBasedObject ReadFromStream_BINARY(byte[] data)
{
DictionaryBasedObject dbo = (DictionaryBasedObject)SerializeFile.LoadFromByteArray(data);
FeatureTrackingManager.Instance.UseFeature(Features.Legacy_Metadata_Binary);
return(dbo);
}
public void IncrementalBuildIndex()
{
if (DateTime.UtcNow.Subtract(time_of_last_library_scan).TotalSeconds > LIBRARY_SCAN_PERIOD_SECONDS)
{
RescanLibrary();
time_of_last_library_scan = DateTime.UtcNow;
}
bool did_some_work = IncrementalBuildNextDocuments();
// Flush to disk
if (did_some_work)
{
Logging.Info("+Writing the index master list");
word_index_manager.WriteMasterList();
lock (pdf_documents_in_library)
{
SerializeFile.SaveSafely(Filename_DocumentProgressList, pdf_documents_in_library);
}
Logging.Info("-Wrote the index master list");
// Report to user
UpdateStatus();
}
}
void SaveKnownWebLibraries()
{
Logging.Info("+Saving known Web Libraries");
try
{
KnownWebLibrariesFile known_web_libraries_file = new KnownWebLibrariesFile();
known_web_libraries_file.web_library_details = new List <WebLibraryDetail>();
foreach (WebLibraryDetail web_library_detail in web_library_details.Values)
{
// *************************************************************************************************************
// *** MIGRATION TO OPEN SOURCE CODE ***************************************************************************
// *************************************************************************************************************
// Don't remember the web libraries - let them be discovered by this
if ("UNKNOWN" == web_library_detail.LibraryType())
{
continue;
}
// *************************************************************************************************************
known_web_libraries_file.web_library_details.Add(web_library_detail);
}
SerializeFile.ProtoSave <KnownWebLibrariesFile>(KNOWN_WEB_LIBRARIES_FILENAME, known_web_libraries_file);
}
catch (Exception ex)
{
Logging.Error(ex, "There was a problem saving the known web libraries");
}
Logging.Info("-Saving known Web Libraries");
}
internal static void ReadFromDisk(PDFDocument pdf_document, PDFInkList inks)
{
WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
try
{
byte[] inks_data = null;
List <LibraryDB.LibraryItem> library_items = pdf_document.LibraryRef.Xlibrary.LibraryDB.GetLibraryItems(PDFDocumentFileLocations.INKS, new List <string>()
{
pdf_document.Fingerprint
});
ASSERT.Test(library_items.Count < 2);
if (0 < library_items.Count)
{
inks_data = library_items[0].data;
}
if (null != inks_data)
{
Dictionary <int, byte[]> page_ink_blobs = SerializeFile.ProtoLoadFromByteArray <Dictionary <int, byte[]> >(inks_data);
if (null != page_ink_blobs)
{
foreach (var pair in page_ink_blobs)
{
pdf_document.AddPageInkBlob(pair.Key, pair.Value);
}
}
}
}
catch (Exception ex)
{
Logging.Error(ex, "There was a problem loading the Inks for document {0}", pdf_document.Fingerprint);
}
}
private void SaveKnownWebLibraries(string filename = null)
{
WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
if (null == filename)
{
filename = KNOWN_WEB_LIBRARIES_FILENAME;
}
Logging.Info("+Saving known Web Libraries to {0}", filename);
try
{
// do NOT save to disk when ANY of the DEV/TEST settings tweak the default Qiqqa behaviour:
ConfigurationManager.ThrowWhenActionIsNotEnabled(nameof(SaveKnownWebLibraries));
ConfigurationManager.ThrowWhenActionIsNotEnabled(nameof(LoadKnownWebLibraries));
ConfigurationManager.ThrowWhenActionIsNotEnabled(nameof(AddLegacyWebLibrariesThatCanBeFoundOnDisk));
KnownWebLibrariesFile known_web_libraries_file = new KnownWebLibrariesFile();
known_web_libraries_file.web_library_details = new List <WebLibraryDetail>();
foreach (WebLibraryDetail web_library_detail in web_library_details.Values)
{
known_web_libraries_file.web_library_details.Add(web_library_detail);
}
SerializeFile.ProtoSave <KnownWebLibrariesFile>(filename, known_web_libraries_file);
}
catch (Exception ex)
{
Logging.Error(ex, "There was a problem saving the known web libraries to file {0}", filename);
}
Logging.Info("-Saving known Web Libraries to {0}", filename);
}
private static DictionaryBasedObject ReadFromDisk_BINARY(string filename)
{
DictionaryBasedObject dbo = (DictionaryBasedObject)SerializeFile.LoadRedundant(filename);
FeatureTrackingManager.Instance.UseFeature(Features.Legacy_Metadata_Binary);
return(dbo);
}
private static void ExtractConnections(string fileToOpen)
{
System.IO.FileInfo FilePath = new System.IO.FileInfo(fileToOpen);
ConnectionsStore myStore = new ConnectionsStore();
myStore = SerializeFile.Deserialize(FilePath.FullName);
Connections.SaveDictionary(myStore.LocalConnections);
}
private void LoadKnownWebLibraries(string filename, bool only_load_those_libraries_which_are_actually_present)
{
WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
Logging.Info("+Loading known Web Libraries");
try
{
if (File.Exists(filename))
{
ConfigurationManager.ThrowWhenActionIsNotEnabled(nameof(LoadKnownWebLibraries));
KnownWebLibrariesFile known_web_libraries_file = SerializeFile.ProtoLoad <KnownWebLibrariesFile>(filename);
if (null != known_web_libraries_file.web_library_details)
{
foreach (WebLibraryDetail new_web_library_detail in known_web_libraries_file.web_library_details)
{
Logging.Info("We have known details for library '{0}' ({1})", new_web_library_detail.Title, new_web_library_detail.Id);
if (!new_web_library_detail.IsPurged)
{
// Intranet libraries had their readonly flag set on the user's current premium status...
if (new_web_library_detail.IsIntranetLibrary ||
new_web_library_detail.IsLocalGuestLibrary ||
new_web_library_detail.IsWebLibrary ||
new_web_library_detail.IsBundleLibrary)
{
new_web_library_detail.IsReadOnly = false;
}
string libdir_path = Library.GetLibraryBasePathForId(new_web_library_detail.Id);
string libfile_path = LibraryDB.GetLibraryDBPath(libdir_path);
if (File.Exists(libfile_path) || !only_load_those_libraries_which_are_actually_present)
{
UpdateKnownWebLibrary(new_web_library_detail);
}
else
{
Logging.Info("Not loading library {0} with Id {1} as it does not exist on disk.", new_web_library_detail.Title, new_web_library_detail.Id);
}
}
else
{
Logging.Info("Not loading purged library {0} with id {1}", new_web_library_detail.Title, new_web_library_detail.Id);
}
}
}
}
}
catch (Exception ex)
{
Logging.Error(ex, "There was a problem loading the known Web Libraries from config file {0}", filename);
}
Logging.Info("-Loading known Web Libraries");
}
public void IncrementalBuildIndex(WebLibraryDetail web_library_detail)
{
if (ShutdownableManager.Instance.IsShuttingDown)
{
Logging.Debug特("LibraryIndex::IncrementalBuildIndex: Breaking out due to application termination");
return;
}
if (Common.Configuration.ConfigurationManager.Instance.ConfigurationRecord.DisableAllBackgroundTasks)
{
Logging.Debug特("LibraryIndex::IncrementalBuildIndex: Breaking due to DisableAllBackgroundTasks");
return;
}
Init(web_library_detail);
if (time_of_last_library_scan.ElapsedMilliseconds >= LIBRARY_SCAN_PERIOD_SECONDS * 1000)
{
if (RescanLibrary(web_library_detail))
{
time_of_last_library_scan.Restart();
}
}
bool did_some_work = IncrementalBuildNextDocuments(web_library_detail);
// Flush to disk
if (did_some_work)
{
Stopwatch clk = Stopwatch.StartNew();
Logging.Info("+Writing the index master list");
// Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (word_index_manager_lock)
{
// l1_clk.LockPerfTimerStop();
word_index_manager?.WriteMasterList();
}
//Utilities.LockPerfTimer l2_clk = Utilities.LockPerfChecker.Start();
lock (pdf_documents_in_library_lock)
{
//l2_clk.LockPerfTimerStop();
SerializeFile.SaveSafely(web_library_detail.FILENAME_DOCUMENT_PROGRESS_LIST, pdf_documents_in_library);
}
Logging.Info("-Wrote the index master list (time spent: {0} ms", clk.ElapsedMilliseconds);
// Report to user
UpdateStatus();
}
}
public void IncrementalBuildIndex()
{
if (Utilities.Shutdownable.ShutdownableManager.Instance.IsShuttingDown)
{
Logging.Debug特("LibraryIndex::IncrementalBuildIndex: Breaking out due to application termination");
return;
}
if (Common.Configuration.ConfigurationManager.Instance.ConfigurationRecord.DisableAllBackgroundTasks)
{
Logging.Debug特("LibraryIndex::IncrementalBuildIndex: Breaking due to DisableAllBackgroundTasks");
return;
}
Init();
if (DateTime.UtcNow.Subtract(time_of_last_library_scan).TotalSeconds > LIBRARY_SCAN_PERIOD_SECONDS)
{
if (RescanLibrary())
{
time_of_last_library_scan = DateTime.UtcNow;
}
}
bool did_some_work = IncrementalBuildNextDocuments();
// Flush to disk
if (did_some_work)
{
Stopwatch clk = Stopwatch.StartNew();
Logging.Info("+Writing the index master list");
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (word_index_manager_lock)
{
l1_clk.LockPerfTimerStop();
word_index_manager.WriteMasterList();
}
Utilities.LockPerfTimer l2_clk = Utilities.LockPerfChecker.Start();
lock (pdf_documents_in_library_lock)
{
l2_clk.LockPerfTimerStop();
SerializeFile.SaveSafely(Filename_DocumentProgressList, pdf_documents_in_library);
}
Logging.Info("-Wrote the index master list (time spent: {0} ms", clk.ElapsedMilliseconds);
// Report to user
UpdateStatus();
}
}
private void saveBranchToDisk(object key1)
{
try
{
SerializeFile.Save(base_filename + "." + key1.ToString(), trunk[key1]);
}
catch (Exception e)
{
Console.WriteLine("Error saving branch {0}: {1}", key1, e.Message);
}
}
// ---
private void fetchBranchFromDisk(object key1)
{
try
{
Hashtable branch = (Hashtable)SerializeFile.Load(base_filename + "." + key1.ToString());
trunk[key1] = branch;
}
catch (Exception e)
{
Console.WriteLine("Error loading branch {0}: {1}", key1, e.Message);
}
}
private void LoadKnownWebLibraries(string filename, bool only_load_those_libraries_which_are_actually_present)
{
Logging.Info("+Loading known Web Libraries");
try
{
if (File.Exists(filename))
{
KnownWebLibrariesFile known_web_libraries_file = SerializeFile.ProtoLoad <KnownWebLibrariesFile>(filename);
if (null != known_web_libraries_file.web_library_details)
{
foreach (WebLibraryDetail new_web_library_detail in known_web_libraries_file.web_library_details)
{
Logging.Info("We have known details for library '{0}' ({1})", new_web_library_detail.Title, new_web_library_detail.Id);
if (!new_web_library_detail.IsPurged)
{
// Intranet libraries had their readonly flag set on the user's current premium status...
if (new_web_library_detail.IsIntranetLibrary ||
new_web_library_detail.IsLocalGuestLibrary ||
new_web_library_detail.IsWebLibrary ||
new_web_library_detail.IsBundleLibrary)
{
new_web_library_detail.IsReadOnly = false;
}
string libdir_path = Library.GetLibraryBasePathForId(new_web_library_detail.Id);
string libfile_path = Path.GetFullPath(Path.Combine(libdir_path, @"Qiqqa.library"));
if (File.Exists(libfile_path) || !only_load_those_libraries_which_are_actually_present)
{
UpdateKnownWebLibrary(new_web_library_detail);
}
else
{
Logging.Info("Not loading library {0} with Id {1} as it does not exist on disk.", new_web_library_detail.Title, new_web_library_detail.Id);
}
}
else
{
Logging.Info("Not loading purged library {0} with id {1}", new_web_library_detail.Title, new_web_library_detail.Id);
}
}
}
}
}
catch (Exception ex)
{
Logging.Error(ex, "There was a problem loading the known Web Libraries from config file {0}", filename);
}
Logging.Info("-Loading known Web Libraries");
}
internal static void WriteToDisk(PDFDocument pdf_document)
{
Dictionary <int, byte[]> page_ink_blobs = new Dictionary <int, byte[]>();
foreach (var pair in pdf_document.Inks.PageInkBlobs)
{
page_ink_blobs.Add(pair.Key, pair.Value);
}
// We only write to disk if we have at least one page of blobbies to write...
if (page_ink_blobs.Count > 0)
{
byte[] data = SerializeFile.ProtoSaveToByteArray <Dictionary <int, byte[]> >(page_ink_blobs);
pdf_document.Library.LibraryDB.PutBlob(pdf_document.Fingerprint, PDFDocumentFileLocations.INKS, data);
}
}
private void saveConnectionsToFileToolStripMenuItem_Click(object sender, EventArgs e)
{
ConnectionsStore myStore = new ConnectionsStore();
myStore.LocalConnections = Connections.RetrieveDictionary();
SaveFileDialog saveFileDialog1 = new SaveFileDialog();
saveFileDialog1.Filter = "cfp files | *.cfp|All files (*.*)|*.*";
saveFileDialog1.FilterIndex = 2;
saveFileDialog1.RestoreDirectory = true;
saveFileDialog1.ShowDialog();
if (saveFileDialog1.FileName != "")
{
SerializeFile.Serialize(saveFileDialog1.FileName, myStore);
}
}
public void RebuildExpedition(int num_topics, bool add_autotags, bool add_tags, RebuiltExpeditionCompleteDelegate rebuiltexpeditioncompletedelegate)
{
Logging.Info("+Rebuilding Expedition");
StatusManager.Instance.ClearCancelled("Expedition");
ExpeditionDataSource eds = ExpeditionBuilder.BuildExpeditionDataSource(library, num_topics, add_autotags, add_tags, ExpeditionBuilderProgressUpdate);
SerializeFile.SaveSafely(Filename_Store, eds);
expedition_data_source = eds;
Logging.Info("-Rebuilding Expedition");
if (null != rebuiltexpeditioncompletedelegate)
{
Logging.Info("+Notifying of rebuilt Expedition");
rebuiltexpeditioncompletedelegate();
Logging.Info("-Notifying of rebuilt Expedition");
}
}
protected SerializeFile[] lstImgFile(string file_kind, string id)
{
string up_path_tpl_o = "~/_Code/SysUpFiles/{0}/{1}/{2}";
string up_path_tpl_s = "~/_Code/SysUpFiles/{0}/{1}";
string web_path_org = string.Format(up_path_tpl_o, file_kind, id, "origin");
string server_path_org = System.Web.Hosting.HostingEnvironment.MapPath(web_path_org);
string web_path_icon = string.Format(up_path_tpl_o, file_kind, id, "icon");
List <SerializeFile> l_files = new List <SerializeFile>();
string file_json_web_path = string.Format(up_path_tpl_s, file_kind, id);
string file_json_server_path = System.Web.Hosting.HostingEnvironment.MapPath(file_json_web_path) + "\\file.json";
string web_path_s = string.Format(up_path_tpl_s, file_kind, id, "origin");
string server_path_s = System.Web.Hosting.HostingEnvironment.MapPath(web_path_s);
if (System.IO.File.Exists(file_json_server_path))
{
var read_json = System.IO.File.ReadAllText(file_json_server_path);
var get_file_json_object = JsonConvert.DeserializeObject <IList <JsonFileInfo> >(read_json).OrderBy(x => x.sort);
foreach (var m in get_file_json_object)
{
string get_file = server_path_org + "//" + m.fileName;
if (System.IO.File.Exists(get_file))
{
FileInfo file_info = new FileInfo(get_file);
SerializeFile file_object = new SerializeFile()
{
guid = m.guid,
fileName = file_info.Name,
fileKind = id,
iconPath = Url.Content(web_path_icon + "/" + file_info.Name),
originPath = Url.Content(web_path_org + "/" + file_info.Name),
size = file_info.Length,
isImage = true
};
l_files.Add(file_object);
}
}
}
return(l_files.ToArray());
}
protected SerializeFile[] listImgFiles(int id, string file_kind, string category1, string category2)
{
string web_path_org = string.Format(upload_path_tpl_o, category1, category2, id, file_kind, "origin");
string server_path_org = Server.MapPath(web_path_org);
string web_path_icon = string.Format(upload_path_tpl_o, category1, category2, id, file_kind, "icon");
List <SerializeFile> l_files = new List <SerializeFile>();
string file_json_web_path = string.Format(upload_path_tpl_s, category1, category2, id, file_kind);
string file_json_server_path = Server.MapPath(file_json_web_path) + "\\file.json";
string web_path_s = string.Format(upload_path_tpl_s, category1, category2, id, file_kind, "origin");
string server_path_s = Server.MapPath(web_path_s);
if (System.IO.File.Exists(file_json_server_path))
{
var read_json = System.IO.File.ReadAllText(file_json_server_path);
var get_file_json_object = JsonConvert.DeserializeObject <IList <JsonFileInfo> >(read_json).OrderBy(x => x.sort);
foreach (var m in get_file_json_object)
{
string get_file = server_path_org + "//" + m.fileName;
if (System.IO.File.Exists(get_file))
{
FileInfo file_info = new FileInfo(get_file);
SerializeFile file_object = new SerializeFile()
{
fileName = file_info.Name,
fileKind = file_kind,
iconPath = Url.Content(web_path_icon + "/" + file_info.Name),
originPath = Url.Content(web_path_org + "/" + file_info.Name),
size = file_info.Length,
isImage = true
};
l_files.Add(file_object);
}
}
}
return(l_files.ToArray());
}
public AITagManager(Library library)
{
this.library = library;
current_ai_tags_record = new AITags();
// Attempt to load the existing tags
try
{
if (File.Exists(Filename_Store))
{
Logging.Info("+Loading AutoTags");
current_ai_tags_record = SerializeFile.ProtoLoad <AITags>(Filename_Store);
Logging.Info("-Loading AutoTags");
}
}
catch (Exception ex)
{
Logging.Warn(ex, "There was a problem loading existing AutoTags");
}
}
void LoadKnownWebLibraries()
{
Logging.Info("+Loading known Web Libraries");
try
{
if (File.Exists(KNOWN_WEB_LIBRARIES_FILENAME))
{
KnownWebLibrariesFile known_web_libraries_file = SerializeFile.ProtoLoad <KnownWebLibrariesFile>(KNOWN_WEB_LIBRARIES_FILENAME);
if (null != known_web_libraries_file.web_library_details)
{
foreach (WebLibraryDetail new_web_library_detail in known_web_libraries_file.web_library_details)
{
Logging.Info("We have known details for library '{0}' ({1})", new_web_library_detail.Title, new_web_library_detail.Id);
if (!new_web_library_detail.IsPurged)
{
// Intranet libraries have their readonly flag set on the user's current premium status...
if (new_web_library_detail.IsIntranetLibrary)
{
new_web_library_detail.IsReadOnly = false;
}
new_web_library_detail.library = new Library(new_web_library_detail);
web_library_details[new_web_library_detail.Id] = new_web_library_detail;
}
else
{
Logging.Info("Not loading purged library {0} with id {1}", new_web_library_detail.Title, new_web_library_detail.Id);
}
}
}
}
}
catch (Exception ex)
{
Logging.Error(ex, "There was a problem loading the known Web Libraries");
}
Logging.Info("-Loading known Web Libraries");
}
internal static void ReadFromDisk(PDFDocument_ThreadUnsafe pdf_document, PDFInkList inks, Dictionary <string, byte[]> library_items_inks_cache)
{
WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
try
{
byte[] inks_data = null;
if (null != library_items_inks_cache)
{
library_items_inks_cache.TryGetValue(pdf_document.Fingerprint, out inks_data);
}
else
{
List <LibraryDB.LibraryItem> library_items = pdf_document.Library.LibraryDB.GetLibraryItems(pdf_document.Fingerprint, PDFDocumentFileLocations.INKS);
if (0 < library_items.Count)
{
inks_data = library_items[0].data;
}
}
if (null != inks_data)
{
Dictionary <int, byte[]> page_ink_blobs = SerializeFile.ProtoLoadFromByteArray <Dictionary <int, byte[]> >(inks_data);
if (null != page_ink_blobs)
{
foreach (var pair in page_ink_blobs)
{
inks.AddPageInkBlob(pair.Key, pair.Value);
}
}
}
}
catch (Exception ex)
{
Logging.Error(ex, "There was a problem loading the Inks for document {0}", pdf_document.Fingerprint);
}
}
public AITagManager(Library library)
{
this.library = library;
current_ai_tags_record = new AITags();
// Attempt to load the existing tags
try
{
if (File.Exists(Filename_Store))
{
Stopwatch clk = new Stopwatch();
clk.Start();
Logging.Info("+Loading AutoTags");
current_ai_tags_record = SerializeFile.ProtoLoad <AITags>(Filename_Store);
Logging.Info("-Loading AutoTags (time spent: {0} ms)", clk.ElapsedMilliseconds);
}
}
catch (Exception ex)
{
Logging.Warn(ex, "There was a problem loading existing AutoTags");
}
}
public static void TestFastSaveLoad()
{
// Random data
double alpha = 0.1;
double beta = 0.2;
int num_topics = 200;
int num_words = 10000;
int num_docs = 20000;
int[][] WORDS_IN_DOCS = new int[num_docs][];
for (int i = 0; i < num_docs; ++i)
{
WORDS_IN_DOCS[i] = new int[i % 100 + 1];
}
LDASampler lda_sampler = new LDASampler(alpha, beta, num_topics, num_words, num_docs, WORDS_IN_DOCS);
{
string lda_sampler_filename = @"C:\temp\ldatest_old.dat";
Logging.Info("+OldSave");
SerializeFile.Save(lda_sampler_filename, lda_sampler);
Logging.Info("-OldSave");
Logging.Info("+OldLoad");
lda_sampler = (LDASampler)SerializeFile.Load(lda_sampler_filename);
Logging.Info("-OldLoad");
}
{
string lda_sampler_filename = @"C:\temp\ldatest_new.dat";
Logging.Info("+NewSave");
lda_sampler.FastSave(lda_sampler_filename);
Logging.Info("-NewSave");
Logging.Info("+NewLoad");
lda_sampler = FastLoad(lda_sampler_filename, WORDS_IN_DOCS);
Logging.Info("-NewLoad");
}
}
public AITagManager(Library library)
{
WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();
this.library = new TypedWeakReference <Library>(library);
current_ai_tags_record = new AITags();
// Attempt to load the existing tags
try
{
if (File.Exists(Filename_Store))
{
Stopwatch clk = Stopwatch.StartNew();
Logging.Info("+Loading AutoTags");
current_ai_tags_record = SerializeFile.ProtoLoad <AITags>(Filename_Store);
Logging.Info("-Loading AutoTags (time spent: {0} ms)", clk.ElapsedMilliseconds);
}
}
catch (Exception ex)
{
Logging.Warn(ex, "There was a problem loading existing AutoTags");
}
}
private void Init()
{
// have we been here before?
if (LibraryIndexIsLoaded)
{
return;
}
Utilities.LockPerfTimer l5_clk = Utilities.LockPerfChecker.Start();
lock (libraryIndexInit_is_pending_lock)
{
l5_clk.LockPerfTimerStop();
Utilities.LockPerfTimer l4_clk = Utilities.LockPerfChecker.Start();
lock (pdf_documents_in_library_lock)
{
lock (word_index_manager_lock)
{
l4_clk.LockPerfTimerStop();
if (null != pdf_documents_in_library && null != word_index_manager)
{
Logging.Warn("LibraryIndex has already been initialized.");
return;
}
}
}
Logging.Info("Try to load a historical progress file: {0}", Filename_DocumentProgressList);
try
{
if (File.Exists(Filename_DocumentProgressList))
{
Stopwatch clk = new Stopwatch();
clk.Start();
Logging.Info("+Loading historical progress file: {0}", Filename_DocumentProgressList);
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (pdf_documents_in_library_lock)
{
l1_clk.LockPerfTimerStop();
pdf_documents_in_library = (Dictionary <string, PDFDocumentInLibrary>)SerializeFile.LoadSafely(Filename_DocumentProgressList);
}
Logging.Info("-Loaded historical progress file: {0} (time spent: {1} ms)", Filename_DocumentProgressList, clk.ElapsedMilliseconds);
}
}
catch (Exception ex)
{
Logging.Error(ex, "FAILED to load historical progress file \"{0}\". Will start indexing afresh.", Filename_DocumentProgressList);
Utilities.LockPerfTimer l2_clk = Utilities.LockPerfChecker.Start();
lock (pdf_documents_in_library_lock)
{
l2_clk.LockPerfTimerStop();
pdf_documents_in_library = null;
}
}
// If there was no historical progress file, start afresh
Utilities.LockPerfTimer l3_clk = Utilities.LockPerfChecker.Start();
lock (pdf_documents_in_library_lock)
{
l3_clk.LockPerfTimerStop();
if (null == pdf_documents_in_library)
{
Logging.Warn("Cound not find any indexing progress, so starting from scratch.");
pdf_documents_in_library = new Dictionary <string, PDFDocumentInLibrary>();
}
}
Utilities.LockPerfTimer l6_clk = Utilities.LockPerfChecker.Start();
lock (word_index_manager_lock)
{
l6_clk.LockPerfTimerStop();
word_index_manager = new LuceneIndex(library.LIBRARY_INDEX_BASE_PATH);
word_index_manager.WriteMasterList();
}
LibraryIndexIsLoaded = true;
}
}
private static List <PDFHighlight> ReadFromStream_PROTOBUF(byte[] data)
{
List <PDFHighlight> highlights_list = SerializeFile.ProtoLoadFromByteArray <List <PDFHighlight> >(data);
return(highlights_list);
}
public void Regenerate(AsyncCallback callback)
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (in_progress_lock)
{
l1_clk.LockPerfTimerStop();
if (regenerating_in_progress)
{
Logging.Info("Not regenerating AutoTags because a regeneration is already in progress.");
return;
}
regenerating_in_progress = true;
}
Stopwatch clk = Stopwatch.StartNew();
try
{
Logging.Info("+AITagManager is starting regenerating");
StatusManager.Instance.UpdateStatusBusy("AITags", "Loading documents");
List <PDFDocument> pdf_documents = library.PDFDocuments;
int count_title_by_user = 0;
int could_title_by_suggest = 0;
StatusManager.Instance.UpdateStatusBusy("AITags", "Deciding whether to use suggested titles");
foreach (PDFDocument pdf_document in pdf_documents)
{
if (pdf_document.IsTitleGeneratedByUser)
{
++count_title_by_user;
}
else
{
++could_title_by_suggest;
}
}
bool use_suggested_titles = could_title_by_suggest > count_title_by_user;
StatusManager.Instance.UpdateStatusBusy("AITags", "Scanning titles");
List <string> titles = new List <string>();
foreach (PDFDocument pdf_document in pdf_documents)
{
if (use_suggested_titles || pdf_document.IsTitleGeneratedByUser)
{
titles.Add(pdf_document.TitleCombined);
}
}
StatusManager.Instance.UpdateStatusBusy("AITags", "Generating AutoTags");
// Get the black/whitelists
List <string> words_blacklist = new List <string>();
List <string> words_whitelist = new List <string>();
{
List <BlackWhiteListEntry> entries = library.BlackWhiteListManager.ReadList();
foreach (var entry in entries)
{
if (entry.is_deleted)
{
continue;
}
switch (entry.list_type)
{
case BlackWhiteListEntry.ListType.White:
words_whitelist.Add(entry.word);
break;
case BlackWhiteListEntry.ListType.Black:
words_blacklist.Add(entry.word);
break;
default:
Logging.Warn("Unknown black/whitelist type " + entry.list_type);
break;
}
}
}
// Generate them
CountingDictionary <NGram> ai_tags = BuzzwordGenerator.GenerateBuzzwords(titles, words_blacklist, words_whitelist, true);
Logging.Info("Generated {0} autotags", ai_tags.Count);
if (ai_tags.Count < 20)
{
Logging.Warn("There are too few autotags (only {0}), so not supressing Scrabble words...", ai_tags.Count);
ai_tags = BuzzwordGenerator.GenerateBuzzwords(titles, words_blacklist, words_whitelist, false);
Logging.Info("Generated {0} autotags without Scrabble suppression", ai_tags.Count);
}
StatusManager.Instance.UpdateStatusBusy("AITags", "AutoTagging documents");
AITags ai_tags_record = new AITags();
// Go through each ngram and see what documents contain it
StatusManager.Instance.ClearCancelled("AITags");
List <NGram> ai_tags_list = new List <NGram>(ai_tags.Keys);
for (int i = 0; i < ai_tags_list.Count; ++i)
{
try
{
NGram ai_tag = ai_tags_list[i];
string tag = ai_tag.text;
if (StatusManager.Instance.IsCancelled("AITags"))
{
break;
}
StatusManager.Instance.UpdateStatusBusy("AITags", String.Format("AutoTagging papers with '{0}'", tag), i, ai_tags_list.Count, true);
// Surround the tag with quotes and search the index
string search_tag = "\"" + tag + "\"";
List <IndexPageResult> fingerprints_potential = LibrarySearcher.FindAllPagesMatchingQuery(library, search_tag);
if (null != fingerprints_potential)
{
// Skip this tag if too many documents have it...
if (ai_tag.is_acronym && fingerprints_potential.Count > 0.05 * pdf_documents.Count)
{
Logging.Info("Skipping AutoTag {0} because too many documents have it...", tag);
continue;
}
foreach (var fingerprint_potential in fingerprints_potential)
{
// Non-acronyms are definitely tagged
if (!ai_tag.is_acronym)
{
ai_tags_record.Associate(tag, fingerprint_potential.fingerprint);
}
else
{
// Acronyms need to be done manually because we only want the upper case ones...
PDFDocument pdf_document = library.GetDocumentByFingerprint(fingerprint_potential.fingerprint);
if (null != pdf_document && !pdf_document.Deleted)
{
bool have_tag = false;
if (!have_tag)
{
string doc_title = pdf_document.TitleCombined;
if (!String.IsNullOrEmpty(doc_title))
{
if (!ai_tag.is_acronym)
{
doc_title = doc_title.ToLower();
}
if (doc_title.Contains(tag))
{
have_tag = true;
}
}
}
if (!have_tag)
{
string doc_comment = pdf_document.Comments;
if (!String.IsNullOrEmpty(doc_comment))
{
if (!ai_tag.is_acronym)
{
doc_comment = doc_comment.ToLower();
}
if (doc_comment.Contains(tag))
{
have_tag = true;
}
}
}
if (!have_tag && pdf_document.DocumentExists)
{
foreach (var page_result in fingerprint_potential.page_results)
{
if (have_tag)
{
break;
}
int page = page_result.page;
WordList page_word_list = pdf_document.PDFRenderer.GetOCRText(page);
if (null != page_word_list)
{
foreach (Word word in page_word_list)
{
if (tag == word.Text)
{
have_tag = true;
break;
}
}
}
}
}
// If we have this tag, record it
if (have_tag)
{
ai_tags_record.Associate(tag, fingerprint_potential.fingerprint);
}
}
else
{
Logging.Warn("Could not find a document matching fingerprint {0}", fingerprint_potential);
}
}
}
}
}
catch (Exception ex)
{
Logging.Error(ex, "There was an exception while processing one of the autotags");
}
}
bool use_new_autotags = true;
if (StatusManager.Instance.IsCancelled("AITags"))
{
if (!MessageBoxes.AskQuestion("You cancelled the generation of your AutoTags. Do you want to use the partially generated AutoTags (YES) or keep your old AutoTags (NO)?"))
{
use_new_autotags = false;
}
}
if (use_new_autotags)
{
StatusManager.Instance.UpdateStatusBusy("AITags", "Saving AutoTags");
SerializeFile.ProtoSave <AITags>(Filename_Store, ai_tags_record);
current_ai_tags_record = ai_tags_record;
}
StatusManager.Instance.UpdateStatus("AITags", "AutoTags generated!");
}
finally
{
Utilities.LockPerfTimer l2_clk = Utilities.LockPerfChecker.Start();
lock (in_progress_lock)
{
l2_clk.LockPerfTimerStop();
regenerating_in_progress = false;
}
Logging.Info("-AITagManager is finished regenerating (time spent: {0} ms)", clk.ElapsedMilliseconds);
}
// Call any callback that might be interested
callback?.Invoke(null);
}
internal static Dictionary <string, string> GetHistoricalSyncFile(Library library)
{
StatusManager.Instance.UpdateStatus(StatusCodes.SYNC_META(library), "Loading sync history");
Dictionary <string, string> historical_sync_file = (Dictionary <string, string>)SerializeFile.LoadSafely(GetFILENAME(library));
if (null == historical_sync_file)
{
Logging.Info("There is no sync history, so creating a new one");
historical_sync_file = new Dictionary <string, string>();
}
return(historical_sync_file);
}
//Проект
private void зберегтиToolStripMenuItem_Click(object sender, EventArgs e)
{
string fileName;
saveFileDialog1.ShowDialog();
fileName = saveFileDialog1.FileName;
SerializeFile serializeFile = new SerializeFile();
bool boolBackgroundImage = false;
if (BackgroundImage != null)
{
serializeFile.Serialize(fileName + " BackgroundImage", BackgroundImage);
boolBackgroundImage = true;
}
serializeFile.Serialize(fileName, boolBackgroundImage);
serializeFile.Serialize(fileName + " x_location", x_location);
serializeFile.Serialize(fileName + " y_location", y_location);
serializeFile.Serialize(fileName + " n_ovalshape", n_ovalshape);
serializeFile.Serialize(fileName + " n_conekt", n_conekt);
//serializeFile.Serialize(fileName + " n_shljahiv", n_shljahiv);
serializeFile.Serialize(fileName + " segment_name", segment_name);
serializeFile.Serialize(fileName + " matrix_sumizn", matrix_sumizn);
//serializeFile.Serialize(fileName + " begin", begin);
//serializeFile.Serialize(fileName + " end", end);
//serializeFile.Serialize(fileName + " sort_shljah", sort_shljah);
//serializeFile.Serialize(fileName + " len_shljah", len_shljah);
string[] ovalshapeName=new string[n_ovalshape+1];
Point[] ovalshapeLocation = new Point[n_ovalshape + 1];
string[] tbName = new string[n_ovalshape + 1];
string[] tbText = new string[n_ovalshape + 1];
Point[] tbLocation = new Point[n_ovalshape + 1];
string[] tb_lengthName = new string[n_conekt + 1];
string[] tb_lengthText = new string[n_conekt + 1];
Point[] tb_lengthLocation = new Point[n_conekt + 1];
string[] lineshapeName = new string[n_conekt+1];
Point[] lineshapeStartPoint = new Point[n_conekt+1];
Point[] lineshapeEndPoint = new Point[n_conekt+1];
for (int i = 0; i <= n_ovalshape; i++)
{
ovalshapeName[i] = ovalshape[i].Name;
ovalshapeLocation[i] = ovalshape[i].Location;
tbName[i] = tb[i].Name;
tbText[i] = tb[i].Text;
tbLocation[i] = tb[i].Location;
}
for (int i = 0; i <= n_conekt; i++)
{
tb_lengthName[i] = tb_length[i].Name;
tb_lengthText[i] = tb_length[i].Text;
tb_lengthLocation[i] = tb_length[i].Location;
lineshapeName[i] = lineshape[i].Name;
lineshapeStartPoint[i] = lineshape[i].StartPoint;
lineshapeEndPoint[i] = lineshape[i].EndPoint;
}
serializeFile.Serialize(fileName + " lineshapeName", lineshapeName);
serializeFile.Serialize(fileName + " lineshapeStartPoint", lineshapeStartPoint);
serializeFile.Serialize(fileName + " lineshapeEndPoint", lineshapeEndPoint);
serializeFile.Serialize(fileName + " ovalshapeName", ovalshapeName);
serializeFile.Serialize(fileName + " ovalshapeLocation", ovalshapeLocation);
serializeFile.Serialize(fileName + " tbName", tbName);
serializeFile.Serialize(fileName + " tbText", tbText);
serializeFile.Serialize(fileName + " tbLocation", tbLocation);
serializeFile.Serialize(fileName + " tb_lengthName", tb_lengthName);
serializeFile.Serialize(fileName + " tb_lengthText", tb_lengthText);
serializeFile.Serialize(fileName + " tb_lengthLocation", tb_lengthLocation);
DialogResult dialogrezult = MessageBox.Show("Збереження завершено!", "Збереження",
MessageBoxButtons.OK);
}
internal static void PutHistoricalSyncFile(Library library, Dictionary <string, string> historical_sync_file)
{
StatusManager.Instance.UpdateStatus(StatusCodes.SYNC_META(library), "Saving sync history");
SerializeFile.SaveSafely(GetFILENAME(library), historical_sync_file);
}
private void завантажитиToolStripMenuItem_Click(object sender, EventArgs e)
{
DialogResult dialogrezult1 = MessageBox.Show("Проект повинен бути пустим!", "Інформація",
MessageBoxButtons.OK);
DialogResult dialogrezult = MessageBox.Show("Вказати потрібно перше слово файлів", "Інформація",
MessageBoxButtons.OK);
string fileName;
openFileDialog1.ShowDialog();
fileName = openFileDialog1.FileName;
SerializeFile serializeFile = new SerializeFile();
x_location = (int[])serializeFile.Deserialize(fileName + " x_location");
y_location = (int[])serializeFile.Deserialize(fileName + " y_location");
n_ovalshape = (int)serializeFile.Deserialize(fileName + " n_ovalshape");
n_conekt = (int)serializeFile.Deserialize(fileName + " n_conekt");
segment_name = (int[,])serializeFile.Deserialize(fileName + " segment_name");
matrix_sumizn = (double[,])serializeFile.Deserialize(fileName + " matrix_sumizn");
string[] ovalshapeName = (string[])serializeFile.Deserialize(fileName + " ovalshapeName");
Point[] ovalshapeLocation = (Point[])serializeFile.Deserialize(fileName + " ovalshapeLocation");
string[] tbName = (string[])serializeFile.Deserialize(fileName + " tbName");
string[] tbText = (string[])serializeFile.Deserialize(fileName + " tbText");
Point[] tbLocation = (Point[])serializeFile.Deserialize(fileName + " tbLocation");
string[] tb_lengthName = (string[])serializeFile.Deserialize(fileName + " tb_lengthName");
string[] tb_lengthText = (string[])serializeFile.Deserialize(fileName + " tb_lengthText");
Point[] tb_lengthLocation = (Point[])serializeFile.Deserialize(fileName + " tb_lengthLocation");
string[] lineshapeName = (string[])serializeFile.Deserialize(fileName + " lineshapeName");
Point[] lineshapeStartPoint = (Point[])serializeFile.Deserialize(fileName + " lineshapeStartPoint");
Point[] lineshapeEndPoint = (Point[])serializeFile.Deserialize(fileName + " lineshapeEndPoint");
//створення точок
for (int i = 0; i <=n_ovalshape; i++)
{
ovalshape[i].Name = ovalshapeName[i];
ovalshape[i].Location = ovalshapeLocation[i];
rectangleShape1.Parent.Shapes.Add(ovalshape[i]);
}
//-----------------------------------------------------------------------------
//Створення полів дя вuводу назв населених пунктів
for (int i = 0; i <= n_ovalshape; i++)
{
tb[i].Name = tbName[i];
tb[i].Text = tbText[i];
tb[i].Location = tbLocation[i];
Controls.Add(tb[i]);
}
//------------------------------------------------------------------------------
//Створення ліній
for (int i = 0; i <=n_conekt; i++)
{
lineshape[i].Name = lineshapeName[i];
lineshape[i].StartPoint = lineshapeStartPoint[i];
lineshape[i].EndPoint = lineshapeEndPoint[i];
rectangleShape1.Parent.Shapes.Add(lineshape[i]);
}
//------------------------------------------------------------------------------
//Створення текстових полів для введення величин
for (int i = 0; i <=n_conekt; i++)
{
tb_length[i].Name = tb_lengthName[i];
tb_length[i].Text = tb_lengthText[i];
tb_length[i].Location = tb_lengthLocation[i];
Controls.Add(tb_length[i]);
}
if ((bool)serializeFile.Deserialize(fileName))
BackgroundImage = (Image)serializeFile.Deserialize(fileName + " BackgroundImage");
}