protected virtual void Dispose(bool disposing)
{
Logging.Debug("LuceneIndex::Dispose({0}) @{1}", disposing, dispose_count);
try
{
if (dispose_count == 0)
{
// Get rid of managed resources
Logging.Info("Disposing the lucene index writer");
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (index_writer_lock)
{
l1_clk.LockPerfTimerStop();
FlushIndexWriter_LOCK();
}
}
Utilities.LockPerfTimer l2_clk = Utilities.LockPerfChecker.Start();
lock (index_writer_lock)
{
l2_clk.LockPerfTimerStop();
index_writer = null;
}
}
catch (Exception ex)
{
Logging.Error(ex);
}
++dispose_count;
}
private static void LogError(string msg)
{
msg = LogAssist.CheckToBreakIntoDebugger(LogAssist.PrefixMemUsage(LogAssist.AppendStackTrace(msg)));
ILog l = log;
if (l != null)
{
l.Error(msg);
}
else
{
BufferMessage(msg);
}
#if TEST
object l1 = new object();
object l2 = new object();
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (l1)
{
l1_clk.LockPerfTimerStop();
msg += "x";
Utilities.LockPerfTimer l2_clk = Utilities.LockPerfChecker.Start();
System.Threading.Thread.Sleep(10 * 1000);
l2_clk.LockPerfTimerStop();
msg += "x";
}
#endif
}
private void RollAfterDrag()
{
const double ROLL_THRESHOLD = 1;
const double ROLL_FALLOFF = 1.05;
current_scroll_gamma.X /= ROLL_FALLOFF;
current_scroll_gamma.Y /= ROLL_FALLOFF;
Scroll(current_scroll_gamma);
// Check if we should keep going
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (thread_lock)
{
l1_clk.LockPerfTimerStop();
if (Math.Abs(current_scroll_gamma.X) < ROLL_THRESHOLD && Math.Abs(current_scroll_gamma.Y) < ROLL_THRESHOLD)
{
is_someone_scrolling = false;
}
else
{
Dispatcher.BeginInvoke(new Action(() => RollAfterDrag()), DispatcherPriority.Background);
}
}
}
internal FadeManager(FrameworkElement fe, double from_opacity, double to_opacity)
{
this.fe = fe;
this.from_opacity = from_opacity;
this.to_opacity = to_opacity;
// Check that we actually have something to do
if (fe.Opacity == to_opacity)
{
return;
}
// Do the before animation stuff
fe.Opacity = from_opacity;
fe.Visibility = Visibility.Visible;
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (responsible_managers_lock)
{
l1_clk.LockPerfTimerStop();
//Logging.Info("FadeManager: Current {0} animations running", responsible_managers.Count);
responsible_managers[fe] = this;
DoubleAnimation da = new DoubleAnimation(from_opacity, to_opacity, new Duration(TimeSpan.FromMilliseconds(250)));
da.Completed += da_Completed;
fe.BeginAnimation(FrameworkElement.OpacityProperty, da, HandoffBehavior.SnapshotAndReplace);
}
}
public int GetDocumentCountForKeyword(string word)
{
word = word.ToLower();
if (use_make_reasonable_word)
{
word = ReasonableWord.MakeReasonableWord(word);
}
if (null == word)
{
return(0);
}
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (locker)
{
l1_clk.LockPerfTimerStop();
WordInWordIndex wiw;
if (word_in_word_index_lookups.TryGetValue(word, out wiw))
{
return(wiw.DocumentCount);
}
else
{
return(0);
}
}
}
private void AddDocumentPage_INTERNAL(string fingerprint, int page, Document document)
{
// Write to the index
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (index_writer_lock)
{
l1_clk.LockPerfTimerStop();
if (null == index_writer)
{
Logging.Info("+Creating a new lucene IndexWriter");
index_writer = new Lucene.Net.Index.IndexWriter(LIBRARY_INDEX_BASE_PATH, analyzer, IndexWriter.MaxFieldLength.UNLIMITED);
Logging.Info("-Creating a new lucene IndexWriter");
}
// Delete the document if it already exists
Lucene.Net.Search.BooleanQuery bq = new Lucene.Net.Search.BooleanQuery();
bq.Add(new Lucene.Net.Search.TermQuery(new Lucene.Net.Index.Term("fingerprint", fingerprint)), Lucene.Net.Search.BooleanClause.Occur.MUST);
bq.Add(new Lucene.Net.Search.TermQuery(new Lucene.Net.Index.Term("page", System.Convert.ToString(page))), Lucene.Net.Search.BooleanClause.Occur.MUST);
index_writer.DeleteDocuments(bq);
// Add the new document
if (null != document)
{
index_writer.AddDocument(document);
}
}
}
private void FirePropertyChangedEventHandler(object sender, PropertyChangedEventArgs e)
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (callback_wrappers_lock)
{
l1_clk.LockPerfTimerStop();
if (0 < callback_wrappers.Count)
{
object[] parameters = new object[] { sender, e };
for (int i = callback_wrappers.Count - 1; i >= 0; --i)
{
object target = callback_wrappers[i].object_to_callback.Target;
if (null != target)
{
callback_wrappers[i].method_to_call.Invoke(target, parameters);
}
else
{
Logging.Debug特("Removing garbage collected callback");
callback_wrappers.RemoveAt(i);
}
}
}
}
}
public void Shutdown()
{
Logging.Info("ShutdownableManager is shutting down all shutdownables:");
IsShuttingDown = true;
while (true)
{
ShutdownDelegate shutdown_delegate = null;
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (shutdown_delegates_lock)
{
l1_clk.LockPerfTimerStop();
if (!shutdown_delegates.Any())
{
break;
}
shutdown_delegate = shutdown_delegates[0];
shutdown_delegates.RemoveAt(0);
}
try
{
Logging.Info("ShutdownableManager is shutting down {0}", shutdown_delegate.Target);
shutdown_delegate();
}
catch (Exception ex)
{
Logging.Error(ex, "There was a problem shutting down Shutdownable {0}", shutdown_delegate.Target);
}
}
}
private void MC_THREAD(object thread_object)
{
int thread = (int)thread_object;
int total_topic_shifts = 0;
for (int i = 0; i < total_words_in_corpus; i += NUM_THREADS)
{
{
// Get the next doc to process
int doc = random_mc_orderings[random_mt[thread].NextIntExclusive(random_mc_orderings.Count)];
int doc_word_index = random_mt[thread].NextIntExclusive(lda_sampler.WORDS_IN_DOCS[doc].Length);
// Get the associated word/topic with this position in the doc
int word = lda_sampler.WORDS_IN_DOCS[doc][doc_word_index];
// What is the most likely topic for this document/word?
int new_topic = SampleTopicForWordInDoc(doc, word, thread);
// Has the word moved topic?
int old_topic = lda_sampler.topic_of_word_in_doc[doc][doc_word_index];
if (new_topic != old_topic)
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (lda_sampler)
{
l1_clk.LockPerfTimerStop();
// Resample the old topic, because it might have been moved already...
old_topic = lda_sampler.topic_of_word_in_doc[doc][doc_word_index];
if (new_topic != old_topic)
{
++total_topic_shifts;
if (old_topic == lda_sampler.topic_of_word_in_doc[doc][doc_word_index])
{
// Remove the word from its existing topic
--lda_sampler.number_of_times_doc_has_a_specific_topic[doc, old_topic];
--lda_sampler.number_of_times_a_topic_has_any_word[old_topic];
--lda_sampler.number_of_times_a_topic_has_a_specific_word[old_topic, word];
// Give the word a new topic
++lda_sampler.number_of_times_doc_has_a_specific_topic[doc, new_topic];
++lda_sampler.number_of_times_a_topic_has_any_word[new_topic];
++lda_sampler.number_of_times_a_topic_has_a_specific_word[new_topic, word];
}
else
{
Logging.Info("It must have been moved on already!");
}
lda_sampler.topic_of_word_in_doc[doc][doc_word_index] = new_topic;
}
}
}
}
}
//Logging.Info("{0} words changed topic", total_topic_shifts);
}
public void WriteMasterList()
{
Logging.Info("+WriteMasterList");
string filename_temp = Path.GetTempFileName();
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (locker)
{
l1_clk.LockPerfTimerStop();
FlushAllWords_LOCK();
PurgeAllWords_LOCK();
using (FileStream fs = File.Open(filename_temp, FileMode.Create))
{
Headers headers = new Headers();
// First the documents
{
headers.documents = new List <DocumentMapHeader>();
foreach (var pair in fingerprint_to_document_ids)
{
DocumentMapHeader header = new DocumentMapHeader
{
Fingerprint = pair.Key,
DocumentId = pair.Value
};
headers.documents.Add(header);
}
}
// Then the words
{
headers.words = new List <WordMapHeader>();
foreach (WordInWordIndex word_in_word_index in word_in_word_indexes)
{
WordMapHeader header = new WordMapHeader
{
Word = word_in_word_index.Word,
WordId = word_in_word_index.WordId,
DocCount = word_in_word_index.DocumentCount
};
headers.words.Add(header);
}
}
Serializer.Serialize <Headers>(fs, headers);
}
}
Logging.Info("-WriteMasterList");
// Move the temp file over the library filename
Directory.CreateDirectory(Path.GetDirectoryName(GetFilename_MasterList()));
FileTools.MoveSafelyWithOverwriting(filename_temp, GetFilename_MasterList());
// Write the version of the index
File.WriteAllText(VersionFilename, INDEX_VERSION);
}
private void wcb_DownloadDataCompleted(object sender, DownloadDataCompletedEventArgs e)
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (progress_lock)
{
l1_clk.LockPerfTimerStop();
Logging.Info("Download complete");
}
}
private void FlushAllWords()
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (locker)
{
l1_clk.LockPerfTimerStop();
FlushAllWords_LOCK();
}
}
public bool IsCancelled(string key)
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (cancelled_items_lock)
{
l1_clk.LockPerfTimerStop();
return(cancelled_items.Contains(key));
}
}
public void BumpHoldOffPendingLevel()
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (hold_off_lock)
{
l1_clk.LockPerfTimerStop();
hold_off--;
}
}
public void ClearCancelled(string key)
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (cancelled_items_lock)
{
l1_clk.LockPerfTimerStop();
cancelled_items.Remove(key);
}
}
public void WriteMasterList()
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (index_writer_lock)
{
l1_clk.LockPerfTimerStop();
FlushIndexWriter_LOCK();
}
}
public void Flush()
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (cache)
{
l1_clk.LockPerfTimerStop();
cache.Flush();
}
}
public static string StaticDecryptString(string TextValue)
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (Instance)
{
l1_clk.LockPerfTimerStop();
return(Instance.DecryptString(TextValue));
}
}
public void Register(ShutdownDelegate shutdown_delegate)
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (shutdown_delegates_lock)
{
l1_clk.LockPerfTimerStop();
Logging.Info("ShutdownableManager is registering {0}", shutdown_delegate.Target);
shutdown_delegates.Add(shutdown_delegate);
}
}
internal void ProcessDocument(PDFDocument pdf_document)
{
if (null != pdf_document.ReadingStage && !Choices.ReadingStages.Contains(pdf_document.ReadingStage))
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (tags_lock)
{
l1_clk.LockPerfTimerStop();
tags.Add(pdf_document.ReadingStage);
}
}
}
internal void wc_DownloadDataCompleted(object sender, DownloadDataCompletedEventArgs e)
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (progress_lock)
{
l1_clk.LockPerfTimerStop();
Logging.Info("Download complete");
DownloadDataCompletedEventArgs = e;
}
CleanupAfterDownload();
}
private void wcb_DownloadProgressChanged(object sender, DownloadProgressChangedEventArgs e)
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (progress_lock)
{
l1_clk.LockPerfTimerStop();
if (progress_percentage < e.ProgressPercentage)
{
Logging.Info("Downloaded {0} / {1} ({2}%)", e.BytesReceived, e.TotalBytesToReceive, e.ProgressPercentage);
progress_percentage = e.ProgressPercentage;
}
}
}
private void ReadMasterList()
{
Logging.Info("+ReadMasterList");
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (locker)
{
l1_clk.LockPerfTimerStop();
try
{
using (FileStream fs = File.OpenRead(GetFilename_MasterList()))
{
Headers headers = Serializer.Deserialize <Headers>(fs);
// First the documents
{
foreach (var header in headers.documents)
{
fingerprint_to_document_ids[header.Fingerprint] = header.DocumentId;
document_id_to_fingerprints[header.DocumentId] = header.Fingerprint;
}
}
// Then the words
{
foreach (var header in headers.words)
{
WordInWordIndex wiwi = new WordInWordIndex(header.Word, header.WordId, header.DocCount);
// Sanity check that they are in the right order
if (wiwi.WordId != word_in_word_indexes.Count)
{
throw new Exception("The ordering of the word index is corrupt");
}
// Add to our maps
word_in_word_indexes.Add(wiwi);
word_in_word_index_lookups[wiwi.Word] = wiwi;
}
}
}
}
catch (Exception ex)
{
Logging.Warn(ex, "Unable to load index master list, so starting from scratch");
}
}
Logging.Info("-ReadMasterList");
}
internal void wc_DownloadProgressChanged(object sender, DownloadProgressChangedEventArgs e)
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (progress_lock)
{
l1_clk.LockPerfTimerStop();
// Limit the logging frequency
if (ProgressPercentage < e.ProgressPercentage)
{
Logging.Info("Downloaded {0} / {1} ({2}%)", e.BytesReceived, e.TotalBytesToReceive, e.ProgressPercentage);
ProgressPercentage = e.ProgressPercentage;
}
}
}
// ------------------------------------------------------------------------------------------------------------------------
public byte[] GetPageByHeightAsImage(int page, double height)
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (cache)
{
l1_clk.LockPerfTimerStop();
byte[] bitmap = cache.Get(page, height);
if (null == bitmap)
{
bitmap = SoraxPDFRendererDLLWrapper.GetPageByHeightAsImage(pdf_filename, pdf_owner_password, pdf_user_password, page, height);
cache.Put(page, height, bitmap);
}
return(bitmap);
}
}
public SearchResult Search(string word)
{
SearchResult search_result = new SearchResult();
search_result.word = word;
if (null == word)
{
return(search_result);
}
word = word.ToLower();
if (use_make_reasonable_word)
{
word = ReasonableWord.MakeReasonableWord(word);
}
if (null == word)
{
return(search_result);
}
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (locker)
{
l1_clk.LockPerfTimerStop();
WordInWordIndex word_in_word_index = GetWordInWordIndex_LOCKER(word, false);
if (null == word_in_word_index)
{
return(search_result);
}
search_result.doc_count = word_in_word_index.DocumentCount;
for (int i = 0; i < word_in_word_index.DocumentIds.Count; ++i)
{
int document_id = word_in_word_index.DocumentIds[i];
int document_id_count = word_in_word_index.DocumentIdsCount[i];
string document_fingerprint;
if (document_id_to_fingerprints.TryGetValue(document_id, out document_fingerprint))
{
search_result.doc_counts[document_fingerprint] = document_id_count;
}
}
return(search_result);
}
}
private void BackgroundQueueRollAfterDrag(Point gamma)
{
current_scroll_gamma.X = gamma.X;
current_scroll_gamma.Y = gamma.Y;
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (thread_lock)
{
l1_clk.LockPerfTimerStop();
if (!is_someone_scrolling)
{
is_someone_scrolling = true;
Dispatcher.BeginInvoke(new Action(() => RollAfterDrag()), DispatcherPriority.Background);
}
}
}
public void AddDocumentWord(string document_fingerprint, string word)
{
if (String.IsNullOrEmpty(word))
{
throw new Exception("Can not index null word");
}
word = word.ToLower();
if (use_make_reasonable_word)
{
word = ReasonableWord.MakeReasonableWord(word);
}
if (null == word)
{
return;
}
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (locker)
{
l1_clk.LockPerfTimerStop();
// Get the doc_id
int document_id;
if (!fingerprint_to_document_ids.TryGetValue(document_fingerprint, out document_id))
{
if (0 == document_id_to_fingerprints.Count)
{
document_id = 1;
}
else
{
document_id = document_id_to_fingerprints.Keys.Max() + 1;
}
fingerprint_to_document_ids[document_fingerprint] = document_id;
document_id_to_fingerprints[document_id] = document_fingerprint;
}
// Get the word from the index or add it if it does not exist
WordInWordIndex word_in_word_index = GetWordInWordIndex_LOCKER(word, true);
if (null == word_in_word_index)
{
throw new Exception("Not expecting to get back a null WordInWordIndex");
}
word_in_word_index.TallyDocId(document_id);
}
}
private static CountingDictionary <NGram> FilterSubNGrams(CountingDictionary <NGram> source_ngrams)
{
CountingDictionary <NGram> ngrams = new CountingDictionary <NGram>();
object ngrams_lock = new object();
Parallel.ForEach(source_ngrams, ngram_sub =>
//foreach (var ngram_sub in source_ngrams)
{
bool is_bad = false;
string text_sub = " " + ngram_sub.Key.text + " ";
foreach (var ngram_sup in source_ngrams)
{
if (ngram_sub.Key == ngram_sup.Key)
{
continue;
}
string text_sup = " " + ngram_sup.Key.text + " ";
if (text_sup.Contains(text_sub))
{
if (ngram_sup.Value / (double)ngram_sub.Value > 0.65)
{
// Logging.Info("Dropping sub-ngram '{0}' as it is subsumed by '{1}'", ngram_sub.Key.text, ngram_sup.Key.text);
is_bad = true;
break;
}
}
}
if (!is_bad)
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (ngrams_lock)
{
l1_clk.LockPerfTimerStop();
ngrams[ngram_sub.Key] = ngram_sub.Value;
}
}
}
);
return(ngrams);
}
public void SetAllCancelled()
{
Utilities.LockPerfTimer l1_clk = Utilities.LockPerfChecker.Start();
lock (status_entries_lock)
{
l1_clk.LockPerfTimerStop();
Utilities.LockPerfTimer l2_clk = Utilities.LockPerfChecker.Start();
lock (cancelled_items_lock)
{
l2_clk.LockPerfTimerStop();
foreach (string key in status_entries.Keys)
{
cancelled_items.Add(key);
}
}
}
}
