public void RebuildExpedition(int num_topics, bool add_autotags, bool add_tags, RebuiltExpeditionCompleteDelegate rebuiltexpeditioncompletedelegate)
        {
            WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();

            StatusManager.Instance.UpdateStatus("Expedition", "Rebuilding Expedition");

            try
            {
                Library.IsBusyRegeneratingTags = true;

                ExpeditionDataSource eds = ExpeditionBuilder.BuildExpeditionDataSource(LibraryRef, num_topics, add_autotags, add_tags, ExpeditionBuilderProgressUpdate);
                if (eds != null)
                {
                    SerializeFile.SaveSafely(Filename_Store, eds);
                    expedition_data_source = eds;
                }
            }
            finally
            {
                Library.IsBusyRegeneratingTags = false;

                StatusManager.Instance.ClearCancelled("Expedition");
            }
            Logging.Info("-Rebuilding Expedition");

            if (null != rebuiltexpeditioncompletedelegate)
            {
                Logging.Info("+Notifying of rebuilt Expedition");
                rebuiltexpeditioncompletedelegate();
                Logging.Info("-Notifying of rebuilt Expedition");
            }
        }
        private static DictionaryBasedObject ReadFromStream_BINARY(byte[] data)
        {
            DictionaryBasedObject dbo = (DictionaryBasedObject)SerializeFile.LoadFromByteArray(data);

            FeatureTrackingManager.Instance.UseFeature(Features.Legacy_Metadata_Binary);
            return(dbo);
        }
        public void IncrementalBuildIndex()
        {
            if (DateTime.UtcNow.Subtract(time_of_last_library_scan).TotalSeconds > LIBRARY_SCAN_PERIOD_SECONDS)
            {
                RescanLibrary();
                time_of_last_library_scan = DateTime.UtcNow;
            }

            bool did_some_work = IncrementalBuildNextDocuments();

            // Flush to disk
            if (did_some_work)
            {
                Logging.Info("+Writing the index master list");
                word_index_manager.WriteMasterList();
                lock (pdf_documents_in_library)
                {
                    SerializeFile.SaveSafely(Filename_DocumentProgressList, pdf_documents_in_library);
                }
                Logging.Info("-Wrote the index master list");

                // Report to user
                UpdateStatus();
            }
        }
        void SaveKnownWebLibraries()
        {
            Logging.Info("+Saving known Web Libraries");

            try
            {
                KnownWebLibrariesFile known_web_libraries_file = new KnownWebLibrariesFile();
                known_web_libraries_file.web_library_details = new List <WebLibraryDetail>();
                foreach (WebLibraryDetail web_library_detail in web_library_details.Values)
                {
                    // *************************************************************************************************************
                    // *** MIGRATION TO OPEN SOURCE CODE ***************************************************************************
                    // *************************************************************************************************************
                    // Don't remember the web libraries - let them be discovered by this
                    if ("UNKNOWN" == web_library_detail.LibraryType())
                    {
                        continue;
                    }
                    // *************************************************************************************************************

                    known_web_libraries_file.web_library_details.Add(web_library_detail);
                }
                SerializeFile.ProtoSave <KnownWebLibrariesFile>(KNOWN_WEB_LIBRARIES_FILENAME, known_web_libraries_file);
            }
            catch (Exception ex)
            {
                Logging.Error(ex, "There was a problem saving the known web libraries");
            }

            Logging.Info("-Saving known Web Libraries");
        }
        internal static void ReadFromDisk(PDFDocument pdf_document, PDFInkList inks)
        {
            WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();

            try
            {
                byte[] inks_data = null;
                List <LibraryDB.LibraryItem> library_items = pdf_document.LibraryRef.Xlibrary.LibraryDB.GetLibraryItems(PDFDocumentFileLocations.INKS, new List <string>()
                {
                    pdf_document.Fingerprint
                });
                ASSERT.Test(library_items.Count < 2);
                if (0 < library_items.Count)
                {
                    inks_data = library_items[0].data;
                }

                if (null != inks_data)
                {
                    Dictionary <int, byte[]> page_ink_blobs = SerializeFile.ProtoLoadFromByteArray <Dictionary <int, byte[]> >(inks_data);

                    if (null != page_ink_blobs)
                    {
                        foreach (var pair in page_ink_blobs)
                        {
                            pdf_document.AddPageInkBlob(pair.Key, pair.Value);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Logging.Error(ex, "There was a problem loading the Inks for document {0}", pdf_document.Fingerprint);
            }
        }
        private void SaveKnownWebLibraries(string filename = null)
        {
            WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();

            if (null == filename)
            {
                filename = KNOWN_WEB_LIBRARIES_FILENAME;
            }

            Logging.Info("+Saving known Web Libraries to {0}", filename);

            try
            {
                // do NOT save to disk when ANY of the DEV/TEST settings tweak the default Qiqqa behaviour:
                ConfigurationManager.ThrowWhenActionIsNotEnabled(nameof(SaveKnownWebLibraries));
                ConfigurationManager.ThrowWhenActionIsNotEnabled(nameof(LoadKnownWebLibraries));
                ConfigurationManager.ThrowWhenActionIsNotEnabled(nameof(AddLegacyWebLibrariesThatCanBeFoundOnDisk));

                KnownWebLibrariesFile known_web_libraries_file = new KnownWebLibrariesFile();
                known_web_libraries_file.web_library_details = new List <WebLibraryDetail>();
                foreach (WebLibraryDetail web_library_detail in web_library_details.Values)
                {
                    known_web_libraries_file.web_library_details.Add(web_library_detail);
                }
                SerializeFile.ProtoSave <KnownWebLibrariesFile>(filename, known_web_libraries_file);
            }
            catch (Exception ex)
            {
                Logging.Error(ex, "There was a problem saving the known web libraries to file {0}", filename);
            }

            Logging.Info("-Saving known Web Libraries to {0}", filename);
        }
        private static DictionaryBasedObject ReadFromDisk_BINARY(string filename)
        {
            DictionaryBasedObject dbo = (DictionaryBasedObject)SerializeFile.LoadRedundant(filename);

            FeatureTrackingManager.Instance.UseFeature(Features.Legacy_Metadata_Binary);
            return(dbo);
        }
Пример #8
0
        private static void ExtractConnections(string fileToOpen)
        {
            System.IO.FileInfo FilePath = new System.IO.FileInfo(fileToOpen);
            ConnectionsStore   myStore  = new ConnectionsStore();

            myStore = SerializeFile.Deserialize(FilePath.FullName);
            Connections.SaveDictionary(myStore.LocalConnections);
        }
Пример #9
0
        private void LoadKnownWebLibraries(string filename, bool only_load_those_libraries_which_are_actually_present)
        {
            WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();

            Logging.Info("+Loading known Web Libraries");
            try
            {
                if (File.Exists(filename))
                {
                    ConfigurationManager.ThrowWhenActionIsNotEnabled(nameof(LoadKnownWebLibraries));

                    KnownWebLibrariesFile known_web_libraries_file = SerializeFile.ProtoLoad <KnownWebLibrariesFile>(filename);
                    if (null != known_web_libraries_file.web_library_details)
                    {
                        foreach (WebLibraryDetail new_web_library_detail in known_web_libraries_file.web_library_details)
                        {
                            Logging.Info("We have known details for library '{0}' ({1})", new_web_library_detail.Title, new_web_library_detail.Id);

                            if (!new_web_library_detail.IsPurged)
                            {
                                // Intranet libraries had their readonly flag set on the user's current premium status...
                                if (new_web_library_detail.IsIntranetLibrary ||
                                    new_web_library_detail.IsLocalGuestLibrary ||
                                    new_web_library_detail.IsWebLibrary ||
                                    new_web_library_detail.IsBundleLibrary)
                                {
                                    new_web_library_detail.IsReadOnly = false;
                                }

                                string libdir_path  = Library.GetLibraryBasePathForId(new_web_library_detail.Id);
                                string libfile_path = LibraryDB.GetLibraryDBPath(libdir_path);

                                if (File.Exists(libfile_path) || !only_load_those_libraries_which_are_actually_present)
                                {
                                    UpdateKnownWebLibrary(new_web_library_detail);
                                }
                                else
                                {
                                    Logging.Info("Not loading library {0} with Id {1} as it does not exist on disk.", new_web_library_detail.Title, new_web_library_detail.Id);
                                }
                            }
                            else
                            {
                                Logging.Info("Not loading purged library {0} with id {1}", new_web_library_detail.Title, new_web_library_detail.Id);
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Logging.Error(ex, "There was a problem loading the known Web Libraries from config file {0}", filename);
            }
            Logging.Info("-Loading known Web Libraries");
        }
Пример #10
0
        public void IncrementalBuildIndex(WebLibraryDetail web_library_detail)
        {
            if (ShutdownableManager.Instance.IsShuttingDown)
            {
                Logging.Debug特("LibraryIndex::IncrementalBuildIndex: Breaking out due to application termination");
                return;
            }

            if (Common.Configuration.ConfigurationManager.Instance.ConfigurationRecord.DisableAllBackgroundTasks)
            {
                Logging.Debug特("LibraryIndex::IncrementalBuildIndex: Breaking due to DisableAllBackgroundTasks");
                return;
            }

            Init(web_library_detail);

            if (time_of_last_library_scan.ElapsedMilliseconds >= LIBRARY_SCAN_PERIOD_SECONDS * 1000)
            {
                if (RescanLibrary(web_library_detail))
                {
                    time_of_last_library_scan.Restart();
                }
            }

            bool did_some_work = IncrementalBuildNextDocuments(web_library_detail);

            // Flush to disk
            if (did_some_work)
            {
                Stopwatch clk = Stopwatch.StartNew();

                Logging.Info("+Writing the index master list");

                // Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
                lock (word_index_manager_lock)
                {
                    // l1_clk.LockPerfTimerStop();

                    word_index_manager?.WriteMasterList();
                }

                //Utilities.LockPerfTimer l2_clk = Utilities.LockPerfChecker.Start();
                lock (pdf_documents_in_library_lock)
                {
                    //l2_clk.LockPerfTimerStop();
                    SerializeFile.SaveSafely(web_library_detail.FILENAME_DOCUMENT_PROGRESS_LIST, pdf_documents_in_library);
                }

                Logging.Info("-Wrote the index master list (time spent: {0} ms", clk.ElapsedMilliseconds);

                // Report to user
                UpdateStatus();
            }
        }
Пример #11
0
        public void IncrementalBuildIndex()
        {
            if (Utilities.Shutdownable.ShutdownableManager.Instance.IsShuttingDown)
            {
                Logging.Debug特("LibraryIndex::IncrementalBuildIndex: Breaking out due to application termination");
                return;
            }

            if (Common.Configuration.ConfigurationManager.Instance.ConfigurationRecord.DisableAllBackgroundTasks)
            {
                Logging.Debug特("LibraryIndex::IncrementalBuildIndex: Breaking due to DisableAllBackgroundTasks");
                return;
            }

            Init();

            if (DateTime.UtcNow.Subtract(time_of_last_library_scan).TotalSeconds > LIBRARY_SCAN_PERIOD_SECONDS)
            {
                if (RescanLibrary())
                {
                    time_of_last_library_scan = DateTime.UtcNow;
                }
            }

            bool did_some_work = IncrementalBuildNextDocuments();

            // Flush to disk
            if (did_some_work)
            {
                Stopwatch clk = Stopwatch.StartNew();

                Logging.Info("+Writing the index master list");

                Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
                lock (word_index_manager_lock)
                {
                    l1_clk.LockPerfTimerStop();

                    word_index_manager.WriteMasterList();
                }

                Utilities.LockPerfTimer l2_clk = Utilities.LockPerfChecker.Start();
                lock (pdf_documents_in_library_lock)
                {
                    l2_clk.LockPerfTimerStop();
                    SerializeFile.SaveSafely(Filename_DocumentProgressList, pdf_documents_in_library);
                }

                Logging.Info("-Wrote the index master list (time spent: {0} ms", clk.ElapsedMilliseconds);

                // Report to user
                UpdateStatus();
            }
        }
Пример #12
0
        private void saveBranchToDisk(object key1)
        {
            try
            {
                SerializeFile.Save(base_filename + "." + key1.ToString(), trunk[key1]);
            }

            catch (Exception e)
            {
                Console.WriteLine("Error saving branch {0}: {1}", key1, e.Message);
            }
        }
Пример #13
0
        // ---

        private void fetchBranchFromDisk(object key1)
        {
            try
            {
                Hashtable branch = (Hashtable)SerializeFile.Load(base_filename + "." + key1.ToString());
                trunk[key1] = branch;
            }

            catch (Exception e)
            {
                Console.WriteLine("Error loading branch {0}: {1}", key1, e.Message);
            }
        }
        private void LoadKnownWebLibraries(string filename, bool only_load_those_libraries_which_are_actually_present)
        {
            Logging.Info("+Loading known Web Libraries");
            try
            {
                if (File.Exists(filename))
                {
                    KnownWebLibrariesFile known_web_libraries_file = SerializeFile.ProtoLoad <KnownWebLibrariesFile>(filename);
                    if (null != known_web_libraries_file.web_library_details)
                    {
                        foreach (WebLibraryDetail new_web_library_detail in known_web_libraries_file.web_library_details)
                        {
                            Logging.Info("We have known details for library '{0}' ({1})", new_web_library_detail.Title, new_web_library_detail.Id);

                            if (!new_web_library_detail.IsPurged)
                            {
                                // Intranet libraries had their readonly flag set on the user's current premium status...
                                if (new_web_library_detail.IsIntranetLibrary ||
                                    new_web_library_detail.IsLocalGuestLibrary ||
                                    new_web_library_detail.IsWebLibrary ||
                                    new_web_library_detail.IsBundleLibrary)
                                {
                                    new_web_library_detail.IsReadOnly = false;
                                }

                                string libdir_path  = Library.GetLibraryBasePathForId(new_web_library_detail.Id);
                                string libfile_path = Path.GetFullPath(Path.Combine(libdir_path, @"Qiqqa.library"));

                                if (File.Exists(libfile_path) || !only_load_those_libraries_which_are_actually_present)
                                {
                                    UpdateKnownWebLibrary(new_web_library_detail);
                                }
                                else
                                {
                                    Logging.Info("Not loading library {0} with Id {1} as it does not exist on disk.", new_web_library_detail.Title, new_web_library_detail.Id);
                                }
                            }
                            else
                            {
                                Logging.Info("Not loading purged library {0} with id {1}", new_web_library_detail.Title, new_web_library_detail.Id);
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Logging.Error(ex, "There was a problem loading the known Web Libraries from config file {0}", filename);
            }
            Logging.Info("-Loading known Web Libraries");
        }
        internal static void WriteToDisk(PDFDocument pdf_document)
        {
            Dictionary <int, byte[]> page_ink_blobs = new Dictionary <int, byte[]>();

            foreach (var pair in pdf_document.Inks.PageInkBlobs)
            {
                page_ink_blobs.Add(pair.Key, pair.Value);
            }

            // We only write to disk if we have at least one page of blobbies to write...
            if (page_ink_blobs.Count > 0)
            {
                byte[] data = SerializeFile.ProtoSaveToByteArray <Dictionary <int, byte[]> >(page_ink_blobs);
                pdf_document.Library.LibraryDB.PutBlob(pdf_document.Fingerprint, PDFDocumentFileLocations.INKS, data);
            }
        }
Пример #16
0
        private void saveConnectionsToFileToolStripMenuItem_Click(object sender, EventArgs e)
        {
            ConnectionsStore myStore = new ConnectionsStore();

            myStore.LocalConnections = Connections.RetrieveDictionary();
            SaveFileDialog saveFileDialog1 = new SaveFileDialog();

            saveFileDialog1.Filter           = "cfp files | *.cfp|All files (*.*)|*.*";
            saveFileDialog1.FilterIndex      = 2;
            saveFileDialog1.RestoreDirectory = true;
            saveFileDialog1.ShowDialog();

            if (saveFileDialog1.FileName != "")
            {
                SerializeFile.Serialize(saveFileDialog1.FileName, myStore);
            }
        }
        public void RebuildExpedition(int num_topics, bool add_autotags, bool add_tags, RebuiltExpeditionCompleteDelegate rebuiltexpeditioncompletedelegate)
        {
            Logging.Info("+Rebuilding Expedition");
            StatusManager.Instance.ClearCancelled("Expedition");
            ExpeditionDataSource eds = ExpeditionBuilder.BuildExpeditionDataSource(library, num_topics, add_autotags, add_tags, ExpeditionBuilderProgressUpdate);

            SerializeFile.SaveSafely(Filename_Store, eds);
            expedition_data_source = eds;
            Logging.Info("-Rebuilding Expedition");

            if (null != rebuiltexpeditioncompletedelegate)
            {
                Logging.Info("+Notifying of rebuilt Expedition");
                rebuiltexpeditioncompletedelegate();
                Logging.Info("-Notifying of rebuilt Expedition");
            }
        }
Пример #18
0
        protected SerializeFile[] lstImgFile(string file_kind, string id)
        {
            string up_path_tpl_o = "~/_Code/SysUpFiles/{0}/{1}/{2}";
            string up_path_tpl_s = "~/_Code/SysUpFiles/{0}/{1}";

            string web_path_org    = string.Format(up_path_tpl_o, file_kind, id, "origin");
            string server_path_org = System.Web.Hosting.HostingEnvironment.MapPath(web_path_org);
            string web_path_icon   = string.Format(up_path_tpl_o, file_kind, id, "icon");

            List <SerializeFile> l_files = new List <SerializeFile>();

            string file_json_web_path    = string.Format(up_path_tpl_s, file_kind, id);
            string file_json_server_path = System.Web.Hosting.HostingEnvironment.MapPath(file_json_web_path) + "\\file.json";

            string web_path_s    = string.Format(up_path_tpl_s, file_kind, id, "origin");
            string server_path_s = System.Web.Hosting.HostingEnvironment.MapPath(web_path_s);

            if (System.IO.File.Exists(file_json_server_path))
            {
                var read_json            = System.IO.File.ReadAllText(file_json_server_path);
                var get_file_json_object = JsonConvert.DeserializeObject <IList <JsonFileInfo> >(read_json).OrderBy(x => x.sort);

                foreach (var m in get_file_json_object)
                {
                    string get_file = server_path_org + "//" + m.fileName;
                    if (System.IO.File.Exists(get_file))
                    {
                        FileInfo      file_info   = new FileInfo(get_file);
                        SerializeFile file_object = new SerializeFile()
                        {
                            guid       = m.guid,
                            fileName   = file_info.Name,
                            fileKind   = id,
                            iconPath   = Url.Content(web_path_icon + "/" + file_info.Name),
                            originPath = Url.Content(web_path_org + "/" + file_info.Name),
                            size       = file_info.Length,
                            isImage    = true
                        };
                        l_files.Add(file_object);
                    }
                }
            }

            return(l_files.ToArray());
        }
Пример #19
0
        protected SerializeFile[] listImgFiles(int id, string file_kind, string category1, string category2)
        {
            string web_path_org    = string.Format(upload_path_tpl_o, category1, category2, id, file_kind, "origin");
            string server_path_org = Server.MapPath(web_path_org);
            string web_path_icon   = string.Format(upload_path_tpl_o, category1, category2, id, file_kind, "icon");

            List <SerializeFile> l_files = new List <SerializeFile>();

            string file_json_web_path    = string.Format(upload_path_tpl_s, category1, category2, id, file_kind);
            string file_json_server_path = Server.MapPath(file_json_web_path) + "\\file.json";

            string web_path_s    = string.Format(upload_path_tpl_s, category1, category2, id, file_kind, "origin");
            string server_path_s = Server.MapPath(web_path_s);

            if (System.IO.File.Exists(file_json_server_path))
            {
                var read_json            = System.IO.File.ReadAllText(file_json_server_path);
                var get_file_json_object = JsonConvert.DeserializeObject <IList <JsonFileInfo> >(read_json).OrderBy(x => x.sort);
                foreach (var m in get_file_json_object)
                {
                    string get_file = server_path_org + "//" + m.fileName;
                    if (System.IO.File.Exists(get_file))
                    {
                        FileInfo      file_info   = new FileInfo(get_file);
                        SerializeFile file_object = new SerializeFile()
                        {
                            fileName   = file_info.Name,
                            fileKind   = file_kind,
                            iconPath   = Url.Content(web_path_icon + "/" + file_info.Name),
                            originPath = Url.Content(web_path_org + "/" + file_info.Name),
                            size       = file_info.Length,
                            isImage    = true
                        };
                        l_files.Add(file_object);
                    }
                }
            }

            return(l_files.ToArray());
        }
Пример #20
0
        public AITagManager(Library library)
        {
            this.library = library;

            current_ai_tags_record = new AITags();


            // Attempt to load the existing tags
            try
            {
                if (File.Exists(Filename_Store))
                {
                    Logging.Info("+Loading AutoTags");
                    current_ai_tags_record = SerializeFile.ProtoLoad <AITags>(Filename_Store);
                    Logging.Info("-Loading AutoTags");
                }
            }
            catch (Exception ex)
            {
                Logging.Warn(ex, "There was a problem loading existing AutoTags");
            }
        }
        void LoadKnownWebLibraries()
        {
            Logging.Info("+Loading known Web Libraries");
            try
            {
                if (File.Exists(KNOWN_WEB_LIBRARIES_FILENAME))
                {
                    KnownWebLibrariesFile known_web_libraries_file = SerializeFile.ProtoLoad <KnownWebLibrariesFile>(KNOWN_WEB_LIBRARIES_FILENAME);
                    if (null != known_web_libraries_file.web_library_details)
                    {
                        foreach (WebLibraryDetail new_web_library_detail in known_web_libraries_file.web_library_details)
                        {
                            Logging.Info("We have known details for library '{0}' ({1})", new_web_library_detail.Title, new_web_library_detail.Id);

                            if (!new_web_library_detail.IsPurged)
                            {
                                // Intranet libraries have their readonly flag set on the user's current premium status...
                                if (new_web_library_detail.IsIntranetLibrary)
                                {
                                    new_web_library_detail.IsReadOnly = false;
                                }

                                new_web_library_detail.library = new Library(new_web_library_detail);
                                web_library_details[new_web_library_detail.Id] = new_web_library_detail;
                            }
                            else
                            {
                                Logging.Info("Not loading purged library {0} with id {1}", new_web_library_detail.Title, new_web_library_detail.Id);
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Logging.Error(ex, "There was a problem loading the known Web Libraries");
            }
            Logging.Info("-Loading known Web Libraries");
        }
        internal static void ReadFromDisk(PDFDocument_ThreadUnsafe pdf_document, PDFInkList inks, Dictionary <string, byte[]> library_items_inks_cache)
        {
            WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();

            try
            {
                byte[] inks_data = null;
                if (null != library_items_inks_cache)
                {
                    library_items_inks_cache.TryGetValue(pdf_document.Fingerprint, out inks_data);
                }
                else
                {
                    List <LibraryDB.LibraryItem> library_items = pdf_document.Library.LibraryDB.GetLibraryItems(pdf_document.Fingerprint, PDFDocumentFileLocations.INKS);
                    if (0 < library_items.Count)
                    {
                        inks_data = library_items[0].data;
                    }
                }

                if (null != inks_data)
                {
                    Dictionary <int, byte[]> page_ink_blobs = SerializeFile.ProtoLoadFromByteArray <Dictionary <int, byte[]> >(inks_data);

                    if (null != page_ink_blobs)
                    {
                        foreach (var pair in page_ink_blobs)
                        {
                            inks.AddPageInkBlob(pair.Key, pair.Value);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Logging.Error(ex, "There was a problem loading the Inks for document {0}", pdf_document.Fingerprint);
            }
        }
Пример #23
0
        public AITagManager(Library library)
        {
            this.library = library;

            current_ai_tags_record = new AITags();

            // Attempt to load the existing tags
            try
            {
                if (File.Exists(Filename_Store))
                {
                    Stopwatch clk = new Stopwatch();
                    clk.Start();
                    Logging.Info("+Loading AutoTags");
                    current_ai_tags_record = SerializeFile.ProtoLoad <AITags>(Filename_Store);
                    Logging.Info("-Loading AutoTags (time spent: {0} ms)", clk.ElapsedMilliseconds);
                }
            }
            catch (Exception ex)
            {
                Logging.Warn(ex, "There was a problem loading existing AutoTags");
            }
        }
Пример #24
0
        public static void TestFastSaveLoad()
        {
            // Random data
            double alpha      = 0.1;
            double beta       = 0.2;
            int    num_topics = 200;
            int    num_words  = 10000;
            int    num_docs   = 20000;

            int[][] WORDS_IN_DOCS = new int[num_docs][];
            for (int i = 0; i < num_docs; ++i)
            {
                WORDS_IN_DOCS[i] = new int[i % 100 + 1];
            }

            LDASampler lda_sampler = new LDASampler(alpha, beta, num_topics, num_words, num_docs, WORDS_IN_DOCS);

            {
                string lda_sampler_filename = @"C:\temp\ldatest_old.dat";
                Logging.Info("+OldSave");
                SerializeFile.Save(lda_sampler_filename, lda_sampler);
                Logging.Info("-OldSave");
                Logging.Info("+OldLoad");
                lda_sampler = (LDASampler)SerializeFile.Load(lda_sampler_filename);
                Logging.Info("-OldLoad");
            }
            {
                string lda_sampler_filename = @"C:\temp\ldatest_new.dat";
                Logging.Info("+NewSave");
                lda_sampler.FastSave(lda_sampler_filename);
                Logging.Info("-NewSave");
                Logging.Info("+NewLoad");
                lda_sampler = FastLoad(lda_sampler_filename, WORDS_IN_DOCS);
                Logging.Info("-NewLoad");
            }
        }
Пример #25
0
        public AITagManager(Library library)
        {
            WPFDoEvents.AssertThisCodeIs_NOT_RunningInTheUIThread();

            this.library = new TypedWeakReference <Library>(library);

            current_ai_tags_record = new AITags();

            // Attempt to load the existing tags
            try
            {
                if (File.Exists(Filename_Store))
                {
                    Stopwatch clk = Stopwatch.StartNew();
                    Logging.Info("+Loading AutoTags");
                    current_ai_tags_record = SerializeFile.ProtoLoad <AITags>(Filename_Store);
                    Logging.Info("-Loading AutoTags (time spent: {0} ms)", clk.ElapsedMilliseconds);
                }
            }
            catch (Exception ex)
            {
                Logging.Warn(ex, "There was a problem loading existing AutoTags");
            }
        }
Пример #26
0
        private void Init()
        {
            // have we been here before?
            if (LibraryIndexIsLoaded)
            {
                return;
            }

            Utilities.LockPerfTimer l5_clk = Utilities.LockPerfChecker.Start();
            lock (libraryIndexInit_is_pending_lock)
            {
                l5_clk.LockPerfTimerStop();

                Utilities.LockPerfTimer l4_clk = Utilities.LockPerfChecker.Start();
                lock (pdf_documents_in_library_lock)
                {
                    lock (word_index_manager_lock)
                    {
                        l4_clk.LockPerfTimerStop();
                        if (null != pdf_documents_in_library && null != word_index_manager)
                        {
                            Logging.Warn("LibraryIndex has already been initialized.");
                            return;
                        }
                    }
                }

                Logging.Info("Try to load a historical progress file: {0}", Filename_DocumentProgressList);
                try
                {
                    if (File.Exists(Filename_DocumentProgressList))
                    {
                        Stopwatch clk = new Stopwatch();
                        clk.Start();
                        Logging.Info("+Loading historical progress file: {0}", Filename_DocumentProgressList);
                        Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
                        lock (pdf_documents_in_library_lock)
                        {
                            l1_clk.LockPerfTimerStop();
                            pdf_documents_in_library = (Dictionary <string, PDFDocumentInLibrary>)SerializeFile.LoadSafely(Filename_DocumentProgressList);
                        }
                        Logging.Info("-Loaded historical progress file: {0} (time spent: {1} ms)", Filename_DocumentProgressList, clk.ElapsedMilliseconds);
                    }
                }
                catch (Exception ex)
                {
                    Logging.Error(ex, "FAILED to load historical progress file \"{0}\". Will start indexing afresh.", Filename_DocumentProgressList);
                    Utilities.LockPerfTimer l2_clk = Utilities.LockPerfChecker.Start();
                    lock (pdf_documents_in_library_lock)
                    {
                        l2_clk.LockPerfTimerStop();
                        pdf_documents_in_library = null;
                    }
                }

                // If there was no historical progress file, start afresh
                Utilities.LockPerfTimer l3_clk = Utilities.LockPerfChecker.Start();
                lock (pdf_documents_in_library_lock)
                {
                    l3_clk.LockPerfTimerStop();
                    if (null == pdf_documents_in_library)
                    {
                        Logging.Warn("Cound not find any indexing progress, so starting from scratch.");
                        pdf_documents_in_library = new Dictionary <string, PDFDocumentInLibrary>();
                    }
                }

                Utilities.LockPerfTimer l6_clk = Utilities.LockPerfChecker.Start();
                lock (word_index_manager_lock)
                {
                    l6_clk.LockPerfTimerStop();
                    word_index_manager = new LuceneIndex(library.LIBRARY_INDEX_BASE_PATH);
                    word_index_manager.WriteMasterList();
                }

                LibraryIndexIsLoaded = true;
            }
        }
Пример #27
0
        private static List <PDFHighlight> ReadFromStream_PROTOBUF(byte[] data)
        {
            List <PDFHighlight> highlights_list = SerializeFile.ProtoLoadFromByteArray <List <PDFHighlight> >(data);

            return(highlights_list);
        }
Пример #28
0
        public void Regenerate(AsyncCallback callback)
        {
            Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
            lock (in_progress_lock)
            {
                l1_clk.LockPerfTimerStop();
                if (regenerating_in_progress)
                {
                    Logging.Info("Not regenerating AutoTags because a regeneration is already in progress.");
                    return;
                }

                regenerating_in_progress = true;
            }

            Stopwatch clk = Stopwatch.StartNew();

            try
            {
                Logging.Info("+AITagManager is starting regenerating");

                StatusManager.Instance.UpdateStatusBusy("AITags", "Loading documents");
                List <PDFDocument> pdf_documents = library.PDFDocuments;

                int count_title_by_user    = 0;
                int could_title_by_suggest = 0;
                StatusManager.Instance.UpdateStatusBusy("AITags", "Deciding whether to use suggested titles");
                foreach (PDFDocument pdf_document in pdf_documents)
                {
                    if (pdf_document.IsTitleGeneratedByUser)
                    {
                        ++count_title_by_user;
                    }
                    else
                    {
                        ++could_title_by_suggest;
                    }
                }

                bool use_suggested_titles = could_title_by_suggest > count_title_by_user;

                StatusManager.Instance.UpdateStatusBusy("AITags", "Scanning titles");
                List <string> titles = new List <string>();
                foreach (PDFDocument pdf_document in pdf_documents)
                {
                    if (use_suggested_titles || pdf_document.IsTitleGeneratedByUser)
                    {
                        titles.Add(pdf_document.TitleCombined);
                    }
                }

                StatusManager.Instance.UpdateStatusBusy("AITags", "Generating AutoTags");

                // Get the black/whitelists
                List <string> words_blacklist = new List <string>();
                List <string> words_whitelist = new List <string>();
                {
                    List <BlackWhiteListEntry> entries = library.BlackWhiteListManager.ReadList();
                    foreach (var entry in entries)
                    {
                        if (entry.is_deleted)
                        {
                            continue;
                        }

                        switch (entry.list_type)
                        {
                        case BlackWhiteListEntry.ListType.White:
                            words_whitelist.Add(entry.word);
                            break;

                        case BlackWhiteListEntry.ListType.Black:
                            words_blacklist.Add(entry.word);
                            break;

                        default:
                            Logging.Warn("Unknown black/whitelist type " + entry.list_type);
                            break;
                        }
                    }
                }

                // Generate them
                CountingDictionary <NGram> ai_tags = BuzzwordGenerator.GenerateBuzzwords(titles, words_blacklist, words_whitelist, true);
                Logging.Info("Generated {0} autotags", ai_tags.Count);
                if (ai_tags.Count < 20)
                {
                    Logging.Warn("There are too few autotags (only {0}), so not supressing Scrabble words...", ai_tags.Count);
                    ai_tags = BuzzwordGenerator.GenerateBuzzwords(titles, words_blacklist, words_whitelist, false);
                    Logging.Info("Generated {0} autotags without Scrabble suppression", ai_tags.Count);
                }

                StatusManager.Instance.UpdateStatusBusy("AITags", "AutoTagging documents");
                AITags ai_tags_record = new AITags();

                // Go through each ngram and see what documents contain it
                StatusManager.Instance.ClearCancelled("AITags");
                List <NGram> ai_tags_list = new List <NGram>(ai_tags.Keys);
                for (int i = 0; i < ai_tags_list.Count; ++i)
                {
                    try
                    {
                        NGram  ai_tag = ai_tags_list[i];
                        string tag    = ai_tag.text;

                        if (StatusManager.Instance.IsCancelled("AITags"))
                        {
                            break;
                        }

                        StatusManager.Instance.UpdateStatusBusy("AITags", String.Format("AutoTagging papers with '{0}'", tag), i, ai_tags_list.Count, true);

                        // Surround the tag with quotes and search the index
                        string search_tag = "\"" + tag + "\"";
                        List <IndexPageResult> fingerprints_potential = LibrarySearcher.FindAllPagesMatchingQuery(library, search_tag);

                        if (null != fingerprints_potential)
                        {
                            // Skip this tag if too many documents have it...
                            if (ai_tag.is_acronym && fingerprints_potential.Count > 0.05 * pdf_documents.Count)
                            {
                                Logging.Info("Skipping AutoTag {0} because too many documents have it...", tag);
                                continue;
                            }

                            foreach (var fingerprint_potential in fingerprints_potential)
                            {
                                // Non-acronyms are definitely tagged
                                if (!ai_tag.is_acronym)
                                {
                                    ai_tags_record.Associate(tag, fingerprint_potential.fingerprint);
                                }
                                else
                                {
                                    // Acronyms need to be done manually because we only want the upper case ones...
                                    PDFDocument pdf_document = library.GetDocumentByFingerprint(fingerprint_potential.fingerprint);
                                    if (null != pdf_document && !pdf_document.Deleted)
                                    {
                                        bool have_tag = false;

                                        if (!have_tag)
                                        {
                                            string doc_title = pdf_document.TitleCombined;
                                            if (!String.IsNullOrEmpty(doc_title))
                                            {
                                                if (!ai_tag.is_acronym)
                                                {
                                                    doc_title = doc_title.ToLower();
                                                }
                                                if (doc_title.Contains(tag))
                                                {
                                                    have_tag = true;
                                                }
                                            }
                                        }

                                        if (!have_tag)
                                        {
                                            string doc_comment = pdf_document.Comments;
                                            if (!String.IsNullOrEmpty(doc_comment))
                                            {
                                                if (!ai_tag.is_acronym)
                                                {
                                                    doc_comment = doc_comment.ToLower();
                                                }
                                                if (doc_comment.Contains(tag))
                                                {
                                                    have_tag = true;
                                                }
                                            }
                                        }

                                        if (!have_tag && pdf_document.DocumentExists)
                                        {
                                            foreach (var page_result in fingerprint_potential.page_results)
                                            {
                                                if (have_tag)
                                                {
                                                    break;
                                                }

                                                int      page           = page_result.page;
                                                WordList page_word_list = pdf_document.PDFRenderer.GetOCRText(page);
                                                if (null != page_word_list)
                                                {
                                                    foreach (Word word in page_word_list)
                                                    {
                                                        if (tag == word.Text)
                                                        {
                                                            have_tag = true;
                                                            break;
                                                        }
                                                    }
                                                }
                                            }
                                        }

                                        // If we have this tag, record it
                                        if (have_tag)
                                        {
                                            ai_tags_record.Associate(tag, fingerprint_potential.fingerprint);
                                        }
                                    }
                                    else
                                    {
                                        Logging.Warn("Could not find a document matching fingerprint {0}", fingerprint_potential);
                                    }
                                }
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        Logging.Error(ex, "There was an exception while processing one of the autotags");
                    }
                }

                bool use_new_autotags = true;

                if (StatusManager.Instance.IsCancelled("AITags"))
                {
                    if (!MessageBoxes.AskQuestion("You cancelled the generation of your AutoTags.  Do you want to use the partially generated AutoTags (YES) or keep your old AutoTags (NO)?"))
                    {
                        use_new_autotags = false;
                    }
                }

                if (use_new_autotags)
                {
                    StatusManager.Instance.UpdateStatusBusy("AITags", "Saving AutoTags");
                    SerializeFile.ProtoSave <AITags>(Filename_Store, ai_tags_record);
                    current_ai_tags_record = ai_tags_record;
                }

                StatusManager.Instance.UpdateStatus("AITags", "AutoTags generated!");
            }
            finally
            {
                Utilities.LockPerfTimer l2_clk = Utilities.LockPerfChecker.Start();
                lock (in_progress_lock)
                {
                    l2_clk.LockPerfTimerStop();
                    regenerating_in_progress = false;
                }

                Logging.Info("-AITagManager is finished regenerating (time spent: {0} ms)", clk.ElapsedMilliseconds);
            }

            // Call any callback that might be interested
            callback?.Invoke(null);
        }
        internal static Dictionary <string, string> GetHistoricalSyncFile(Library library)
        {
            StatusManager.Instance.UpdateStatus(StatusCodes.SYNC_META(library), "Loading sync history");

            Dictionary <string, string> historical_sync_file = (Dictionary <string, string>)SerializeFile.LoadSafely(GetFILENAME(library));

            if (null == historical_sync_file)
            {
                Logging.Info("There is no sync history, so creating a new one");
                historical_sync_file = new Dictionary <string, string>();
            }

            return(historical_sync_file);
        }
Пример #30
0
        //Проект
        private void зберегтиToolStripMenuItem_Click(object sender, EventArgs e)
        {
            string fileName;
            saveFileDialog1.ShowDialog();
            fileName = saveFileDialog1.FileName;
            SerializeFile serializeFile = new SerializeFile();
            bool boolBackgroundImage = false;
            if (BackgroundImage != null)
            {
                serializeFile.Serialize(fileName + " BackgroundImage", BackgroundImage);
                boolBackgroundImage = true;
            }
            serializeFile.Serialize(fileName, boolBackgroundImage);
            serializeFile.Serialize(fileName + " x_location", x_location);
            serializeFile.Serialize(fileName + " y_location", y_location);
            serializeFile.Serialize(fileName + " n_ovalshape", n_ovalshape);
            serializeFile.Serialize(fileName + " n_conekt", n_conekt);
            //serializeFile.Serialize(fileName + " n_shljahiv", n_shljahiv);
            serializeFile.Serialize(fileName + " segment_name", segment_name);
            serializeFile.Serialize(fileName + " matrix_sumizn", matrix_sumizn);
            //serializeFile.Serialize(fileName + " begin", begin);
            //serializeFile.Serialize(fileName + " end", end);
            //serializeFile.Serialize(fileName + " sort_shljah", sort_shljah);
            //serializeFile.Serialize(fileName + " len_shljah", len_shljah);
            string[] ovalshapeName=new string[n_ovalshape+1];
            Point[] ovalshapeLocation = new Point[n_ovalshape + 1];
            string[] tbName = new string[n_ovalshape + 1];
            string[] tbText = new string[n_ovalshape + 1];
            Point[] tbLocation = new Point[n_ovalshape + 1];
            string[] tb_lengthName = new string[n_conekt + 1];
            string[] tb_lengthText = new string[n_conekt + 1];
            Point[] tb_lengthLocation = new Point[n_conekt + 1];
            string[] lineshapeName = new string[n_conekt+1];
            Point[] lineshapeStartPoint = new Point[n_conekt+1];
            Point[] lineshapeEndPoint = new Point[n_conekt+1];
            for (int i = 0; i <= n_ovalshape; i++)
            {
                ovalshapeName[i] = ovalshape[i].Name;
                ovalshapeLocation[i] = ovalshape[i].Location;

                tbName[i] = tb[i].Name;
                tbText[i] = tb[i].Text;
                tbLocation[i] = tb[i].Location;
            }
            for (int i = 0; i <= n_conekt; i++)
            {
                tb_lengthName[i] = tb_length[i].Name;
                tb_lengthText[i] = tb_length[i].Text;
                tb_lengthLocation[i] = tb_length[i].Location;

                lineshapeName[i] = lineshape[i].Name;
                lineshapeStartPoint[i] = lineshape[i].StartPoint;
                lineshapeEndPoint[i] = lineshape[i].EndPoint;
            }
            serializeFile.Serialize(fileName + " lineshapeName", lineshapeName);
            serializeFile.Serialize(fileName + " lineshapeStartPoint", lineshapeStartPoint);
            serializeFile.Serialize(fileName + " lineshapeEndPoint", lineshapeEndPoint);
            serializeFile.Serialize(fileName + " ovalshapeName", ovalshapeName);
            serializeFile.Serialize(fileName + " ovalshapeLocation", ovalshapeLocation);
            serializeFile.Serialize(fileName + " tbName", tbName);
            serializeFile.Serialize(fileName + " tbText", tbText);
            serializeFile.Serialize(fileName + " tbLocation", tbLocation);
            serializeFile.Serialize(fileName + " tb_lengthName", tb_lengthName);
            serializeFile.Serialize(fileName + " tb_lengthText", tb_lengthText);
            serializeFile.Serialize(fileName + " tb_lengthLocation", tb_lengthLocation);
            DialogResult dialogrezult = MessageBox.Show("Збереження завершено!", "Збереження",
                MessageBoxButtons.OK);
        }
 internal static void PutHistoricalSyncFile(Library library, Dictionary <string, string> historical_sync_file)
 {
     StatusManager.Instance.UpdateStatus(StatusCodes.SYNC_META(library), "Saving sync history");
     SerializeFile.SaveSafely(GetFILENAME(library), historical_sync_file);
 }
Пример #32
0
 private void завантажитиToolStripMenuItem_Click(object sender, EventArgs e)
 {
     DialogResult dialogrezult1 = MessageBox.Show("Проект повинен бути пустим!", "Інформація",
        MessageBoxButtons.OK);
     DialogResult dialogrezult = MessageBox.Show("Вказати потрібно перше слово файлів", "Інформація",
        MessageBoxButtons.OK);
     string fileName;
     openFileDialog1.ShowDialog();
     fileName = openFileDialog1.FileName;
     SerializeFile serializeFile = new SerializeFile();
     x_location = (int[])serializeFile.Deserialize(fileName + " x_location");
     y_location = (int[])serializeFile.Deserialize(fileName + " y_location");
     n_ovalshape = (int)serializeFile.Deserialize(fileName + " n_ovalshape");
     n_conekt = (int)serializeFile.Deserialize(fileName + " n_conekt");
     segment_name = (int[,])serializeFile.Deserialize(fileName + " segment_name");
     matrix_sumizn = (double[,])serializeFile.Deserialize(fileName + " matrix_sumizn");
     string[] ovalshapeName = (string[])serializeFile.Deserialize(fileName + " ovalshapeName");
     Point[] ovalshapeLocation = (Point[])serializeFile.Deserialize(fileName + " ovalshapeLocation");
     string[] tbName = (string[])serializeFile.Deserialize(fileName + " tbName");
     string[] tbText = (string[])serializeFile.Deserialize(fileName + " tbText");
     Point[] tbLocation = (Point[])serializeFile.Deserialize(fileName + " tbLocation");
     string[] tb_lengthName = (string[])serializeFile.Deserialize(fileName + " tb_lengthName");
     string[] tb_lengthText = (string[])serializeFile.Deserialize(fileName + " tb_lengthText");
     Point[] tb_lengthLocation = (Point[])serializeFile.Deserialize(fileName + " tb_lengthLocation");
      string[] lineshapeName = (string[])serializeFile.Deserialize(fileName + " lineshapeName");
     Point[] lineshapeStartPoint = (Point[])serializeFile.Deserialize(fileName + " lineshapeStartPoint");
     Point[] lineshapeEndPoint = (Point[])serializeFile.Deserialize(fileName + " lineshapeEndPoint");
     //створення точок
     for (int i = 0; i <=n_ovalshape; i++)
     {
         ovalshape[i].Name = ovalshapeName[i];
         ovalshape[i].Location = ovalshapeLocation[i];
         rectangleShape1.Parent.Shapes.Add(ovalshape[i]);
     }
     //-----------------------------------------------------------------------------
     //Створення полів дя вuводу назв населених пунктів
     for (int i = 0; i <= n_ovalshape; i++)
     {
         tb[i].Name = tbName[i];
         tb[i].Text = tbText[i];
         tb[i].Location = tbLocation[i];
         Controls.Add(tb[i]);
     }
     //------------------------------------------------------------------------------
     //Створення ліній
     for (int i = 0; i <=n_conekt; i++)
     {
         lineshape[i].Name = lineshapeName[i];
         lineshape[i].StartPoint = lineshapeStartPoint[i];
         lineshape[i].EndPoint = lineshapeEndPoint[i];
         rectangleShape1.Parent.Shapes.Add(lineshape[i]);
     }
     //------------------------------------------------------------------------------
     //Створення текстових полів для введення величин
     for (int i = 0; i <=n_conekt; i++)
     {
         tb_length[i].Name = tb_lengthName[i];
         tb_length[i].Text = tb_lengthText[i];
         tb_length[i].Location = tb_lengthLocation[i];
         Controls.Add(tb_length[i]);
     }
     if ((bool)serializeFile.Deserialize(fileName))
         BackgroundImage = (Image)serializeFile.Deserialize(fileName + " BackgroundImage");
 }