public LuceneQueryable(string index_name, int minor_version, bool read_only_mode) { this.index_name = index_name; this.minor_version = minor_version; this.read_only_mode = read_only_mode; driver = BuildLuceneQueryingDriver(this.index_name, this.minor_version, this.read_only_mode); our_hit_filter = new LuceneCommon.HitFilter(this.HitFilter); backend_query_part_hook = new LuceneCommon.QueryPartHook(this.QueryPartHook); // If the queryable is in read-only more, don't // instantiate an indexer for it. // FIXME: --indexing-delay -1 is a hack for --read-only; need to fix this if (read_only_mode || QueryDriver.IndexingDelay < 0) { return; } indexer = LocalIndexerHook(); if (indexer == null && indexer_hook != null) { indexer = indexer_hook(this.index_name, this.minor_version); } OptimizeAllEvent += OnOptimizeAllEvent; // Schedule an optimize, just in case ScheduleOptimize(); Shutdown.ShutdownEvent += new Shutdown.ShutdownHandler(OnShutdownEvent); }
///////////////////////////////////////////////////////// // Merge an external Beagrep index to the current index. Merging will // join the primary- and secondary lucene indexes and if available, the // file attributes store. static void ExecuteMerge(string index_dir, string index_to_merge) { LuceneIndexingDriver driver = new LuceneIndexingDriver(index_dir, false); if (!Path.IsPathRooted(index_to_merge)) { index_to_merge = Path.GetFullPath(index_to_merge); } if (!Directory.Exists(index_to_merge)) { Console.WriteLine("Could not find index to merge: {0}", index_to_merge); Environment.Exit(1); } // Set system priorities so we don't slow down the system SystemPriorities.ReduceIoPriority(); SystemPriorities.SetSchedulerPolicyBatch(); LuceneQueryingDriver driver_to_merge = new LuceneQueryingDriver(index_to_merge, -1, false); Stopwatch watch = new Stopwatch(); watch.Start(); // Merge the lucene index try { driver.Merge(driver_to_merge); } catch (Exception e) { Console.WriteLine("Index merging (lucene) failed: {0}", e); Environment.Exit(1); } // Merge file attributes stores FileAttributesStore_Sqlite store; try { store = new FileAttributesStore_Sqlite(driver.TopDirectory, driver.Fingerprint); store.Merge(new FileAttributesStore_Sqlite(driver_to_merge.TopDirectory, driver_to_merge.Fingerprint)); } catch (Exception e) { Console.WriteLine("Index merging (attributes store) failed: {0}", e); Environment.Exit(1); } watch.Stop(); Console.WriteLine("Successfully merged index {0} into {1} in {2}", index_to_merge, driver.TopDirectory, watch); }
// Remapping hack from DumpIndex static Uri RemapUri (LuceneQueryingDriver driver, Uri uri) { // We only need to remap URIs in the file system backend if (driver.IndexName != "FileSystemIndex") return uri; FileAttributesStore fa_store = new FileAttributesStore (new FileAttributesStore_Mixed (Path.Combine (PathFinder.IndexDir, "FileSystemIndex"), driver.Fingerprint)); string path = uri.LocalPath; Beagrep.Daemon.FileAttributes attr = fa_store.Read (path); if (attr == null) { Console.WriteLine ("No file attribute info for {0}", uri); return uri; } return new Uri ("uid:" + GuidFu.ToShortString (attr.UniqueId) + uri.Fragment); }
static void Main (string[] args) { if (args.Length != 2) { Console.WriteLine ("Usage: beagrep-master-delete-button index-name uri-to-delete"); return; } string index_name = args [0]; LuceneQueryingDriver driver = new LuceneQueryingDriver (index_name, -1, true); Uri uri = new Uri (args [1], false); Uri uri_to_delete = RemapUri (driver, uri); LuceneIndexingDriver indexer = new LuceneIndexingDriver (index_name, false); Indexable indexable = new Indexable (uri_to_delete); indexable.Type = IndexableType.Remove; IndexerRequest request = new IndexerRequest (); request.Add (indexable); IndexerReceipt [] receipts = indexer.Flush (request); if (receipts == null || receipts.Length == 0) { Console.WriteLine ("Uri {0} not found in {1}", uri, index_name); return; } IndexerRemovedReceipt r = receipts [0] as IndexerRemovedReceipt; if (r == null || r.NumRemoved == 0) { Console.WriteLine ("Uri {0} not found in {1}", uri, index_name); return; } Console.WriteLine ("Uri {0} deleted", uri); }
static ArrayList RemapUris (LuceneQueryingDriver driver, ArrayList uris) { // We only need to remap URIs in the file system backend if (driver.IndexName != "FileSystemIndex") return uris; FileAttributesStore fa_store = new FileAttributesStore (new FileAttributesStore_Mixed (Path.Combine (PathFinder.IndexDir, "FileSystemIndex"), driver.Fingerprint)); for (int i = 0; i < uris.Count; i++) { Uri uri = (Uri) uris [i]; string path = uri.LocalPath; Beagrep.Daemon.FileAttributes attr = fa_store.Read (path); if (attr == null) { Console.WriteLine ("No file attribute info for {0}", uri); continue; } Uri internal_uri = new Uri ("uid:" + GuidFu.ToShortString (attr.UniqueId) + uri.Fragment); uris [i] = internal_uri; } return uris; }
///////////////////////////////////////////////////////// // Get the total number of entries from the index. static void ExecuteInfo(string index_dir) { LuceneQueryingDriver driver = new LuceneQueryingDriver(index_dir, true); Console.WriteLine("Total number of entries in index: {0}", driver.GetItemCount()); }
public PropertyRemovalGenerator(LuceneQueryingDriver driver, Property prop) { this.driver = driver; this.prop_to_match = prop; }
///////////////////////////////////////////////////////// // Merge an external Beagrep index to the current index. Merging will // join the primary- and secondary lucene indexes and if available, the // file attributes store. static void ExecuteMerge(string index_dir, string index_to_merge) { LuceneIndexingDriver driver = new LuceneIndexingDriver (index_dir, false); if (!Path.IsPathRooted (index_to_merge)) index_to_merge = Path.GetFullPath (index_to_merge); if (!Directory.Exists (index_to_merge)) { Console.WriteLine ("Could not find index to merge: {0}", index_to_merge); Environment.Exit (1); } // Set system priorities so we don't slow down the system SystemPriorities.ReduceIoPriority (); SystemPriorities.SetSchedulerPolicyBatch (); LuceneQueryingDriver driver_to_merge = new LuceneQueryingDriver (index_to_merge, -1, false); Stopwatch watch = new Stopwatch (); watch.Start (); // Merge the lucene index try { driver.Merge (driver_to_merge); } catch (Exception e) { Console.WriteLine ("Index merging (lucene) failed: {0}", e); Environment.Exit (1); } // Merge file attributes stores FileAttributesStore_Sqlite store; try { store = new FileAttributesStore_Sqlite (driver.TopDirectory, driver.Fingerprint); store.Merge (new FileAttributesStore_Sqlite (driver_to_merge.TopDirectory, driver_to_merge.Fingerprint)); } catch (Exception e) { Console.WriteLine ("Index merging (attributes store) failed: {0}", e); Environment.Exit (1); } watch.Stop (); Console.WriteLine ("Successfully merged index {0} into {1} in {2}", index_to_merge, driver.TopDirectory, watch); }
///////////////////////////////////////////////////////// // Get the total number of entries from the index. static void ExecuteInfo(string index_dir) { LuceneQueryingDriver driver = new LuceneQueryingDriver (index_dir, true); Console.WriteLine ("Total number of entries in index: {0}", driver.GetItemCount()); }
public UidManager (FileAttributesStore fa_store, LuceneQueryingDriver driver) { this.fa_store = fa_store; this.name_resolver = (LuceneNameResolver) driver; }
static int DumpOneIndex_Metadata (string index_name, ArrayList uris, bool show_properties) { LuceneQueryingDriver driver; driver = new LuceneQueryingDriver (index_name, -1, true); Hashtable all_hits_by_uri = null; ArrayList all_hits = null; if (uris.Count == 0 || index_name == "FileSystemIndex") { all_hits_by_uri = driver.GetAllHitsByUri (); all_hits = new ArrayList (all_hits_by_uri.Values); } // A hard-wired hack if (index_name == "FileSystemIndex") { foreach (Hit hit in all_hits) { string internal_uri; if (hit [Property.IsChildPropKey] == "true") { string path = RemapUriToPath (all_hits_by_uri, hit); internal_uri = UriFu.UriToEscapedString (hit.ParentUri); hit.ParentUri = UriFu.PathToFileUri (path); hit.Uri = UriFu.AddFragment (UriFu.PathToFileUri (path), hit.Uri.Fragment, true); } else { internal_uri = UriFu.UriToEscapedString (hit.Uri); hit.Uri = UriFu.PathToFileUri (RemapUriToPath (all_hits_by_uri, hit)); hit.AddProperty (Property.NewUnsearched ("beagrep:InternalUri", internal_uri)); } } } ArrayList matching_hits; if (uris.Count == 0) matching_hits = all_hits; else { matching_hits = new ArrayList (driver.GetHitsForUris (RemapUris (driver, uris))); if (index_name == "FileSystemIndex") { for (int i = 0; i < matching_hits.Count; i++) { Hit hit = (Hit) matching_hits [i]; Hit mapped_hit = (Hit) all_hits_by_uri [hit.Uri]; matching_hits [i] = mapped_hit; } } } matching_hits.Sort (new HitByUriComparer ()); foreach (Hit hit in matching_hits) { if (! show_properties) { Console.WriteLine ("{0}: {1}", index_name, hit.Uri); continue; } Console.WriteLine (" Index: {0}", index_name); Console.WriteLine (" Uri: {0}", hit.Uri); if (hit.ParentUri != null) Console.WriteLine ("Parent: {0}", hit.ParentUri); Console.WriteLine (" MimeT: {0}", hit.MimeType); Console.WriteLine (" Type: {0}", hit.Type); Console.WriteLine ("Source: {0}", hit.Source); ArrayList props; props = new ArrayList (hit.Properties); props.Sort (); foreach (Property prop in props) { char [] legend = new char [4]; legend [0] = prop.IsMutable ? 'm' : ' '; legend [1] = prop.IsSearched ? 's' : ' '; legend [2] = prop.IsPersistent ? 'p' : ' '; legend [3] = prop.Type == PropertyType.Text ? 't' : ' '; Console.WriteLine (" Prop: [{0}] {1} = '{2}'", new String (legend), prop.Key, prop.Value); } Console.WriteLine (); } return matching_hits.Count; }
// Dump the fields: we do this via direct Lucene access. static void DumpOneIndex_Fields (string index_name) { LuceneQueryingDriver driver; driver = new LuceneQueryingDriver (index_name, -1, true); IndexReader reader; reader = IndexReader.Open (driver.PrimaryStore); Console.WriteLine (" -- Primary Index --"); foreach (DictionaryEntry fi in reader.GetFieldNames (IndexReader.FieldOption.ALL)) Console.WriteLine ("- [{0}]", fi.Key); reader.Close (); reader = IndexReader.Open (driver.SecondaryStore); if (reader.MaxDoc () != 0) { Console.WriteLine ("\n -- Secondary Index --"); foreach (DictionaryEntry fi in reader.GetFieldNames (IndexReader.FieldOption.ALL)) Console.WriteLine ("- [{0}]", fi.Key); reader.Close (); } Console.WriteLine (); }
// Dump the term frequencies: we do this via direct Lucene access. static void DumpOneIndex_TermFrequencies (string index_name) { LuceneQueryingDriver driver; driver = new LuceneQueryingDriver (index_name, -1, true); IndexReader reader; reader = IndexReader.Open (driver.PrimaryStore); TermEnum term_enum; term_enum = reader.Terms (new Term("", "")); int distinct_term_count = 0; int term_count = 0; // from LuceneFAQ // Terms are sorted first by field, then by text // so all terms with a given field are adjacent in enumerations. if (term_enum.Term () != null) { while (true) { Console.WriteLine("fieldname: {0}, text: {1}", term_enum.Term().Field(), term_enum.Term().Text()); if (term_enum.Term().Field() != "Text") { if (!term_enum.Next ()) { break; } continue; } int freq; freq = term_enum.DocFreq (); Console.WriteLine ("{0} '{1}' {2}", index_name, term_enum.Term ().Text (), freq); // FIXME: spew these as a count ++distinct_term_count; term_count += freq; if (!term_enum.Next ()) break; } } term_enum.Close (); reader.Close (); Console.WriteLine (); }
public LuceneQueryable (string index_name, int minor_version, bool read_only_mode) { this.index_name = index_name; this.minor_version = minor_version; this.read_only_mode = read_only_mode; driver = BuildLuceneQueryingDriver (this.index_name, this.minor_version, this.read_only_mode); our_hit_filter = new LuceneCommon.HitFilter (this.HitFilter); backend_query_part_hook = new LuceneCommon.QueryPartHook (this.QueryPartHook); // If the queryable is in read-only more, don't // instantiate an indexer for it. // FIXME: --indexing-delay -1 is a hack for --read-only; need to fix this if (read_only_mode || QueryDriver.IndexingDelay < 0) return; indexer = LocalIndexerHook (); if (indexer == null && indexer_hook != null) indexer = indexer_hook (this.index_name, this.minor_version); OptimizeAllEvent += OnOptimizeAllEvent; // Schedule an optimize, just in case ScheduleOptimize (); Shutdown.ShutdownEvent += new Shutdown.ShutdownHandler (OnShutdownEvent); }
public PropertyRemovalGenerator (LuceneQueryingDriver driver, Property prop) { this.driver = driver; this.prop_to_match = prop; }