private IndexerReceipt [] Flush_Unlocked(IndexerRequest request) { ArrayList receipt_queue; receipt_queue = new ArrayList(); IndexReader primary_reader, secondary_reader; primary_reader = IndexReader.Open(PrimaryStore); secondary_reader = IndexReader.Open(SecondaryStore); // Step #1: Make our first pass over the list of // indexables that make up our request. For each add // or property change in the request, get the Lucene // documents so we can move forward any persistent // properties (for adds) or all old properties (for // property changes). // // Then, for each add or remove in the request, // delete the associated documents from the index. // Note that we previously cached added documents so // that we can move persistent properties forward. // parent_child_old_props is double-nested hashtable (depth-2 tree) // indexed by the parent uri, it stores another hashtable indexed by the (parent+child documents) // FIXME: 2-level hashtable is a waste for any non-child document. // Replace this by a better data structure. Hashtable parent_child_old_props = UriFu.NewHashtable(); TermDocs term_docs = secondary_reader.TermDocs(); int delete_count = 0; IEnumerable request_indexables = request.Indexables; foreach (Indexable indexable in request_indexables) { string uri_str = UriFu.UriToEscapedString(indexable.Uri); Term term; // Store the necessary properties from old documents for re-addition if (indexable.Type == IndexableType.Add || indexable.Type == IndexableType.PropertyChange) { term = new Term("Uri", uri_str); term_docs.Seek(term); Hashtable this_parent_child_props = null; if (term_docs.Next()) { this_parent_child_props = UriFu.NewHashtable(); this_parent_child_props [indexable.Uri] = secondary_reader.Document(term_docs.Doc()); parent_child_old_props [indexable.Uri] = this_parent_child_props; } term = new Term("ParentUri", uri_str); term_docs.Seek(term); while (term_docs.Next()) { Document doc = secondary_reader.Document(term_docs.Doc()); string child_uri_str = doc.Get("Uri"); Uri child_uri = UriFu.EscapedStringToUri(child_uri_str); // Any valid lucene document *should* have a Uri, so no need to check for null // Store the child documents too, to save persistent-properties // of child documents this_parent_child_props [child_uri] = doc; } } // Now remove (non-remove indexables will be re-added in next block) Logger.Log.Debug("-{0}", indexable.DisplayUri); int num_delete = 0; term = new Term("Uri", uri_str); // For property changes, only secondary index is modified secondary_reader.DeleteDocuments(term); // Now remove from everywhere else (if asked to remove or if asked to add, in which case // we first remove and then add) // So we also need to remove child documents if (indexable.Type != IndexableType.PropertyChange) { num_delete = primary_reader.DeleteDocuments(term); // When we delete an indexable, also delete any children. // FIXME: Shouldn't we also delete any children of children, etc.? term = new Term("ParentUri", uri_str); num_delete += primary_reader.DeleteDocuments(term); secondary_reader.DeleteDocuments(term); } // If this is a strict removal (and not a deletion that // we are doing in anticipation of adding something back), // queue up a removed receipt. if (indexable.Type == IndexableType.Remove) { IndexerRemovedReceipt r; r = new IndexerRemovedReceipt(indexable.Id); r.NumRemoved = num_delete; receipt_queue.Add(r); } delete_count += num_delete; } term_docs.Close(); if (HaveItemCount) { AdjustItemCount(-delete_count); } else { SetItemCount(primary_reader); } // We are now done with the readers, so we close them. // And also free them. Somehow not freeing them is preventing them from // GCed at all. primary_reader.Close(); primary_reader = null; secondary_reader.Close(); secondary_reader = null; // FIXME: If we crash at exactly this point, we are in // trouble. Items will have been dropped from the index // without the proper replacements being added. We can // hopefully fix this when we move to Lucene 2.1. // Step #2: Make another pass across our list of indexables // and write out any new documents. if (text_cache != null) { text_cache.BeginTransaction(); } IndexWriter primary_writer, secondary_writer; // FIXME: Lock obtain time-out can happen here; if that happens, // an exception will be thrown and this method will break in the middle // leaving IndexWriters unclosed! Same for any Lucene.Net-index modification // methods. primary_writer = new IndexWriter(PrimaryStore, IndexingAnalyzer, false); secondary_writer = null; foreach (Indexable indexable in request_indexables) { // If shutdown has been started, break here // FIXME: Some more processing will continue, a lot of them // concerning receipts, but the daemon will anyway ignore receipts // now, what is the fastest way to stop from here ? if (Shutdown.ShutdownRequested) { Log.Debug("Shutdown initiated. Breaking while flushing indexables."); break; } // Receipts for removes were generated in the // previous block. Now we just have to remove // items from the text cache. if (indexable.Type == IndexableType.Remove) { if (text_cache != null) { text_cache.Delete(indexable.Uri); } continue; } IndexerAddedReceipt r; Hashtable prop_change_docs = (Hashtable)parent_child_old_props [indexable.Uri]; if (indexable.Type == IndexableType.PropertyChange) { Logger.Log.Debug("+{0} (props only)", indexable.DisplayUri); r = new IndexerAddedReceipt(indexable.Id); r.PropertyChangesOnly = true; receipt_queue.Add(r); Document doc; if (prop_change_docs == null) { doc = null; } else { doc = (Document)prop_change_docs [indexable.Uri]; } Document new_doc; new_doc = RewriteDocument(doc, indexable); // Write out the new document... if (secondary_writer == null) { secondary_writer = new IndexWriter(SecondaryStore, IndexingAnalyzer, false); } secondary_writer.AddDocument(new_doc); // Get child property change indexables... ArrayList prop_change_indexables; prop_change_indexables = GetChildPropertyChange(prop_change_docs, indexable); // and store them; no need to delete them first, since they were already removed from the index if (prop_change_indexables == null) { continue; } foreach (Indexable prop_change_indexable in prop_change_indexables) { Log.Debug("+{0} (props only, generated indexable)", prop_change_indexable.Uri); doc = (Document)prop_change_docs [prop_change_indexable.Uri]; new_doc = RewriteDocument(doc, prop_change_indexable); secondary_writer.AddDocument(new_doc); } continue; // ...and proceed to the next Indexable } // If we reach this point we know we are dealing with an IndexableType.Add if (indexable.Type != IndexableType.Add) { throw new Exception("When I said it was an IndexableType.Add, I meant it!"); } r = AddIndexableToIndex(indexable, primary_writer, ref secondary_writer, prop_change_docs); if (r != null) { receipt_queue.Add(r); } } if (text_cache != null) { text_cache.CommitTransaction(); } if (Shutdown.ShutdownRequested) { foreach (DeferredInfo di in deferred_indexables) { di.Cleanup(); } deferred_indexables.Clear(); foreach (Indexable indexable in request_indexables) { indexable.Cleanup(); } primary_writer.Close(); if (secondary_writer != null) { secondary_writer.Close(); } return(null); } if (request.OptimizeIndex) { Stopwatch watch = new Stopwatch(); Logger.Log.Debug("Optimizing {0}", IndexName); watch.Start(); primary_writer.Optimize(); if (secondary_writer == null) { secondary_writer = new IndexWriter(SecondaryStore, IndexingAnalyzer, false); } secondary_writer.Optimize(); watch.Stop(); Logger.Log.Debug("{0} optimized in {1}", IndexName, watch); } // Step #4. Close our writers and return the events to // indicate what has happened. primary_writer.Close(); if (secondary_writer != null) { secondary_writer.Close(); } // Send a single IndexerIndexablesReceipt if there were deferred indexables if (deferred_indexables.Count > 0) { Log.Debug("{0} indexables generated more indexables; asking daemon to schedule their indexing.", deferred_indexables.Count); IndexerIndexablesReceipt r = new IndexerIndexablesReceipt(); receipt_queue.Add(r); } IndexerReceipt [] receipt_array; receipt_array = new IndexerReceipt [receipt_queue.Count]; for (int i = 0; i < receipt_queue.Count; ++i) { receipt_array [i] = (IndexerReceipt)receipt_queue [i]; } return(receipt_array); }
protected bool Exists () { if (! (Directory.Exists (top_dir) && File.Exists (VersionFile) && File.Exists (FingerprintFile) && Directory.Exists (PrimaryIndexDirectory) && IndexReader.IndexExists (PrimaryIndexDirectory) && Directory.Exists (SecondaryIndexDirectory) && IndexReader.IndexExists (SecondaryIndexDirectory) && Directory.Exists (LockDirectory))) return false; // Check the index's version number. If it is wrong, // declare the index non-existent. StreamReader version_reader; string version_str; version_reader = new StreamReader (VersionFile); version_str = version_reader.ReadLine (); version_reader.Close (); int current_major_version, current_minor_version; int i = version_str.IndexOf ('.'); try { if (i != -1) { current_major_version = Convert.ToInt32 (version_str.Substring (0, i)); current_minor_version = Convert.ToInt32 (version_str.Substring (i+1)); } else { current_minor_version = Convert.ToInt32 (version_str); current_major_version = 0; } } catch (FormatException) { // Something wrong with the version file. return false; } if (current_major_version != MAJOR_VERSION || (minor_version >= 0 && current_minor_version != minor_version)) { Logger.Log.Debug ("Version mismatch in {0}", index_name); Logger.Log.Debug ("Index has version {0}.{1}, expected {2}.{3}", current_major_version, current_minor_version, MAJOR_VERSION, minor_version); return false; } // Check the lock directory: If there is a dangling write lock, // assume that the index is corrupted and declare it non-existent. DirectoryInfo lock_dir_info; lock_dir_info = new DirectoryInfo (LockDirectory); bool dangling_lock = false; foreach (FileInfo info in lock_dir_info.GetFiles ()) { if (IsDanglingLock (info)) { Logger.Log.Warn ("Found a dangling index lock on {0}.", info.FullName); dangling_lock = true; } } if (dangling_lock) { Beagle.Util.Stopwatch w = new Beagle.Util.Stopwatch (); w.Start (); if (VerifyLuceneIndex (PrimaryIndexDirectory) && VerifyLuceneIndex (SecondaryIndexDirectory)) { w.Stop (); Log.Warn ("Indexes verified in {0}. Deleting stale lock files.", w); try { foreach (FileInfo info in lock_dir_info.GetFiles ()) info.Delete (); } catch { Log.Warn ("Could not delete lock files."); return false; } return true; } else return false; } return true; }
static void DoMain (string [] args) { SystemInformation.SetProcessName ("beagle-build-index"); if (args.Length < 2) PrintUsage (); ArrayList allowed_patterns = new ArrayList (); ArrayList denied_patterns = new ArrayList (); int i = 0; while (i < args.Length) { string arg = args [i]; ++i; string next_arg = i < args.Length ? args [i] : null; switch (arg) { case "-h": case "--help": PrintUsage (); break; case "--tag": if (next_arg != null) arg_tag = next_arg; ++i; break; case "-r": case "--recursive": arg_recursive = true; break; case "--enable-deletion": arg_delete = true; break; case "--disable-directories": arg_disable_directories = true; break; case "--enable-text-cache": arg_cache_text = true; break; case "--target": if (next_arg != null) arg_output = Path.IsPathRooted (next_arg) ? next_arg : Path.GetFullPath (next_arg); ++i; break; case "--disable-filtering": arg_disable_filtering = true; break; case "--disable-on-battery": arg_disable_on_battery = true; break; case "--allow-pattern": if (next_arg == null) break; if (next_arg.IndexOf (',') != -1) { foreach (string pattern in next_arg.Split (',')) allowed_patterns.Add (pattern); } else { allowed_patterns.Add (next_arg); } ++i; break; case "--deny-pattern": if (next_arg == null) break; if (next_arg.IndexOf (',') != -1) { foreach (string pattern in next_arg.Split (',')) denied_patterns.Add (pattern); } else { denied_patterns.Add (next_arg); } ++i; break; case "--disable-restart": arg_disable_restart = true; break; case "--source": if (next_arg == null) break; arg_source = next_arg; ++i; break; case "--removable": arg_removable = true; break; default: if (arg.StartsWith ("-") || arg.StartsWith ("--")) PrintUsage (); string path = Path.IsPathRooted (arg) ? arg : Path.GetFullPath (arg); if (path != "/" && path.EndsWith ("/")) path = path.TrimEnd ('/'); if (Directory.Exists (path)) pending_directories.Enqueue (new DirectoryInfo (path)); else if (File.Exists (path)) pending_files.Enqueue (new FileInfo (path)); break; } } ///////////////////////////////////////////////////////// if (arg_output == null) { Logger.Log.Error ("--target must be specified"); Environment.Exit (1); } // Set the storage dir, this should be used to store log messages // and filterver.dat PathFinder.StorageDir = arg_output; foreach (FileSystemInfo info in pending_directories) { if (Path.GetFullPath (arg_output) == info.FullName) { Logger.Log.Error ("Target directory cannot be one of the source paths."); Environment.Exit (1); } } foreach (FileSystemInfo info in pending_files) { if (Path.GetFullPath (arg_output) == info.FullName) { Logger.Log.Error ("Target directory cannot be one of the source paths."); Environment.Exit (1); } } if (!Directory.Exists (Path.GetDirectoryName (arg_output))) { Logger.Log.Error ("Index directory not available for construction: {0}", arg_output); Environment.Exit (1); } // Be *EXTRA PARANOID* about the contents of the target // directory, because creating an indexing driver will // nuke it. if (Directory.Exists (arg_output)) { foreach (FileInfo info in DirectoryWalker.GetFileInfos (arg_output)) { if (Array.IndexOf (allowed_files, info.Name) == -1) { Logger.Log.Error ("{0} doesn't look safe to delete: non-Beagle file {1} was found", arg_output, info.FullName); Environment.Exit (1); } } foreach (DirectoryInfo info in DirectoryWalker.GetDirectoryInfos (arg_output)) { if (Array.IndexOf (allowed_dirs, info.Name) == -1) { Logger.Log.Error ("{0} doesn't look safe to delete: non-Beagle directory {1} was found", arg_output, info.FullName); Environment.Exit (1); } } } string config_file_path = Path.Combine (arg_output, "StaticIndex.xml"); string prev_source = null; if (File.Exists (config_file_path)) { Config static_index_config = Conf.LoadFrom (config_file_path); if (static_index_config == null) { Log.Error ("Invalid configuation file {0}", config_file_path); Environment.Exit (1); } prev_source = static_index_config.GetOption ("Source", null); if (arg_source != null && prev_source != arg_source) { Log.Error ("Source already set to {0} for existing static index. Cannot set source to {1}.", prev_source, arg_source); Environment.Exit (1); } bool prev_removable = static_index_config.GetOption ("Removable", false); if (arg_removable != prev_removable) { Log.Error ("Index previously created for {0}-removable path", (prev_removable ? "" : "non")); Environment.Exit (1); } else { volume_label = static_index_config.GetOption ("VolumeLabel", null); } // If arg_source is not given, and prev_source is present, use prev_source // as the arg_source. This is useful for re-running build-index without // giving --arg_source for already existing static index arg_source = prev_source; } // Removable media related options if (arg_removable) { if (pending_files.Count > 0) { Log.Error ("Indexing individual files is not allowed for removable media."); Environment.Exit (1); } else if (pending_directories.Count > 1) { Log.Error ("Indexing multiple root directories is not allowed for removable media."); Environment.Exit (1); } mnt_dir = ((DirectoryInfo) pending_directories.Peek ()).FullName; if (mnt_dir.Length != 1) mnt_dir = mnt_dir.TrimEnd ('/'); // compute volume label // (1) directory name if block.is_volume is false // (2) hal volume.label if set // (3) hal volume.uuid if set Hal.Manager manager = new Hal.Manager (new Hal.Context ()); Hal.Device mnt_device = null; foreach (Hal.Device device in manager.FindDeviceStringMatch ("volume.mount_point", mnt_dir)) mnt_device = device; string new_volume_label = null; if (mnt_device != null) { new_volume_label = mnt_device.GetPropertyString ("volume.label"); if (String.IsNullOrEmpty (new_volume_label)) new_volume_label = mnt_device.GetPropertyString ("volume.uuid"); } if (new_volume_label == null) new_volume_label = ((DirectoryInfo) pending_directories.Peek ()).Name; // Sanity check // Volume label is part of the URI, so cannot be changed once set if (volume_label == null) { volume_label = new_volume_label; } else if (volume_label != new_volume_label) { Log.Error ("Volume label (earlier '{0}') changed (to '{1}')! You need to create a new index.", volume_label, new_volume_label); Environment.Exit (1); } } if (arg_source == null) { DirectoryInfo dir = new DirectoryInfo (StringFu.SanitizePath (arg_output)); arg_source = dir.Name; } if (! BatteryMonitor.UsingAC && arg_disable_on_battery) { Log.Always ("Indexer is disabled when on battery power (--disable-on-battery)"); Environment.Exit (0); } string global_files_config = Path.Combine (PathFinder.ConfigDataDir, "config-files"); global_files_config = Path.Combine (global_files_config, Conf.Names.FilesQueryableConfig + ".xml"); if (! File.Exists (global_files_config)) { Log.Error ("Global configuration file not found {0}", global_files_config); Environment.Exit (0); } // Setup regexes for allowed/denied patterns if (allowed_patterns.Count > 0) { allowed_regex = StringFu.GetPatternRegex (allowed_patterns); } else { // Read the exclude values from config // For system-wide indexes, only the global config value will be used Config config = Conf.Get (Conf.Names.FilesQueryableConfig); List<string[]> values = config.GetListOptionValues (Conf.Names.ExcludePattern); if (values != null) foreach (string[] exclude in values) denied_patterns.Add (exclude [0]); if (denied_patterns.Count > 0) denied_regex = StringFu.GetPatternRegex (denied_patterns); } Log.Always ("Starting beagle-build-index (pid {0}) at {1}", Process.GetCurrentProcess ().Id, DateTime.Now); // Set system priorities so we don't slow down the system SystemPriorities.ReduceIoPriority (); SystemPriorities.SetSchedulerPolicyBatch (); SystemPriorities.Renice (19); driver = new LuceneIndexingDriver (arg_output, MINOR_VERSION, false); driver.TextCache = (arg_cache_text) ? new TextCache (arg_output) : null; if (driver.TextCache != null) driver.TextCache.WorldReadable = true; backing_fa_store = new FileAttributesStore_Sqlite (driver.TopDirectory, driver.Fingerprint); fa_store = new FileAttributesStore (backing_fa_store); // Set up signal handlers #if MONO_1_9 Shutdown.SetupSignalHandlers (delegate (int signal) { if (signal == (int) Mono.Unix.Native.Signum.SIGINT || signal == (int) Mono.Unix.Native.Signum.SIGTERM) Shutdown.BeginShutdown (); }); #else SetupSignalHandlers (); #endif Thread monitor_thread = null; Stopwatch watch = new Stopwatch (); watch.Start (); if (!arg_disable_restart) { // Start the thread that monitors memory usage. monitor_thread = ExceptionHandlingThread.Start (new ThreadStart (MemoryMonitorWorker)); } // Start indexworker to do the crawling and indexing IndexWorker (); // Join any threads so that we know that we're the only thread still running if (monitor_thread != null) monitor_thread.Join (); watch.Stop (); Logger.Log.Debug ("Elapsed time {0}.", watch); // Write this after indexing is done. This is because, if creating a new index, // LuceneIndexingDriver.Create() is called which purges the entire directory. if (prev_source == null) { Config static_index_config = Conf.LoadNew ("StaticIndex.xml"); // Write StaticIndex.xml containing: // The name of the source static_index_config.SetOption ("Source", arg_source); static_index_config ["Source"].Description = "Source of the static index"; if (arg_removable) { static_index_config.SetOption ("VolumeLabel", volume_label); static_index_config ["VolumeLabel"].Description = "Volume label of the removable source"; static_index_config.SetOption ("Removable", true); static_index_config ["Removable"].Description = "Removable source"; } Conf.SaveTo (static_index_config, config_file_path); } if (restart) { Logger.Log.Debug ("Restarting beagle-build-index"); Process p = new Process (); p.StartInfo.UseShellExecute = false; // FIXME: Maybe this isn't the right way to do things? It should be ok, // the PATH is inherited from the shell script which runs mono itself. p.StartInfo.FileName = "mono"; p.StartInfo.Arguments = String.Join (" ", Environment.GetCommandLineArgs ()); p.Start (); } Log.Always ("Exiting beagle-build-index (pid {0}) at {1}", Process.GetCurrentProcess ().Id, DateTime.Now); }
static void LaunchHelper() { // If we are in the process of shutting down, return immediately. if (Shutdown.ShutdownRequested) { return; } lock (helper_lock) { // If a helper appears to be running, return immediately. if (CheckHelper()) { return; } Logger.Log.Debug("Launching helper process"); SafeProcess p = new SafeProcess(); string[] args = new string [3]; args [0] = helper_path; if (BeagleDaemon.DisableTextCache) { args [1] = "--disable-text-cache"; } else { args [1] = String.Empty; } if (Log.Level == LogLevel.Debug) { args [2] = "--debug"; } else { args [2] = String.Empty; } p.Arguments = args; p.RedirectStandardOutput = false; p.RedirectStandardError = false; p.Start(); Logger.Log.Debug("IndexHelper PID is {0}", p.Id); // Poll the helper's socket. Wait up to a minute // (500 ms * 120 times) for the helper to be ready // to handle requests. Stopwatch watch = new Stopwatch(); watch.Start(); int poll_count = 0; bool found_helper; do { Thread.Sleep(500); ++poll_count; found_helper = CheckHelper(); } while (poll_count < 120 && !found_helper && !Shutdown.ShutdownRequested); watch.Stop(); if (!found_helper) { throw new Exception(String.Format("Couldn't launch helper process {0}", p.Id)); } Logger.Log.Debug("Found IndexHelper ({0}) in {1}", p.Id, watch); helper_pid = p.Id; } }
static void DoMain(string [] args) { SystemInformation.SetProcessName("beagle-build-index"); if (args.Length < 2) { PrintUsage(); } ArrayList allowed_patterns = new ArrayList(); ArrayList denied_patterns = new ArrayList(); int i = 0; while (i < args.Length) { string arg = args [i]; ++i; string next_arg = i < args.Length ? args [i] : null; switch (arg) { case "-h": case "--help": PrintUsage(); break; case "--tag": if (next_arg != null) { arg_tag = next_arg; } ++i; break; case "-r": case "--recursive": arg_recursive = true; break; case "--enable-deletion": arg_delete = true; break; case "--disable-directories": arg_disable_directories = true; break; case "--enable-text-cache": arg_cache_text = true; break; case "--target": if (next_arg != null) { arg_output = Path.IsPathRooted(next_arg) ? next_arg : Path.GetFullPath(next_arg); } ++i; break; case "--disable-filtering": arg_disable_filtering = true; break; case "--disable-on-battery": arg_disable_on_battery = true; break; case "--allow-pattern": if (next_arg == null) { break; } if (next_arg.IndexOf(',') != -1) { foreach (string pattern in next_arg.Split(',')) { allowed_patterns.Add(pattern); } } else { allowed_patterns.Add(next_arg); } ++i; break; case "--deny-pattern": if (next_arg == null) { break; } if (next_arg.IndexOf(',') != -1) { foreach (string pattern in next_arg.Split(',')) { denied_patterns.Add(pattern); } } else { denied_patterns.Add(next_arg); } ++i; break; case "--disable-restart": arg_disable_restart = true; break; case "--source": if (next_arg == null) { break; } arg_source = next_arg; ++i; break; case "--removable": arg_removable = true; break; default: if (arg.StartsWith("-") || arg.StartsWith("--")) { PrintUsage(); } string path = Path.IsPathRooted(arg) ? arg : Path.GetFullPath(arg); if (path != "/" && path.EndsWith("/")) { path = path.TrimEnd('/'); } if (Directory.Exists(path)) { pending_directories.Enqueue(new DirectoryInfo(path)); } else if (File.Exists(path)) { pending_files.Enqueue(new FileInfo(path)); } break; } } ///////////////////////////////////////////////////////// if (arg_output == null) { Logger.Log.Error("--target must be specified"); Environment.Exit(1); } // Set the storage dir, this should be used to store log messages // and filterver.dat PathFinder.StorageDir = arg_output; foreach (FileSystemInfo info in pending_directories) { if (Path.GetFullPath(arg_output) == info.FullName) { Logger.Log.Error("Target directory cannot be one of the source paths."); Environment.Exit(1); } } foreach (FileSystemInfo info in pending_files) { if (Path.GetFullPath(arg_output) == info.FullName) { Logger.Log.Error("Target directory cannot be one of the source paths."); Environment.Exit(1); } } if (!Directory.Exists(Path.GetDirectoryName(arg_output))) { Logger.Log.Error("Index directory not available for construction: {0}", arg_output); Environment.Exit(1); } // Be *EXTRA PARANOID* about the contents of the target // directory, because creating an indexing driver will // nuke it. if (Directory.Exists(arg_output)) { foreach (FileInfo info in DirectoryWalker.GetFileInfos(arg_output)) { if (Array.IndexOf(allowed_files, info.Name) == -1) { Logger.Log.Error("{0} doesn't look safe to delete: non-Beagle file {1} was found", arg_output, info.FullName); Environment.Exit(1); } } foreach (DirectoryInfo info in DirectoryWalker.GetDirectoryInfos(arg_output)) { if (Array.IndexOf(allowed_dirs, info.Name) == -1) { Logger.Log.Error("{0} doesn't look safe to delete: non-Beagle directory {1} was found", arg_output, info.FullName); Environment.Exit(1); } } } string config_file_path = Path.Combine(arg_output, "StaticIndex.xml"); string prev_source = null; if (File.Exists(config_file_path)) { Config static_index_config = Conf.LoadFrom(config_file_path); if (static_index_config == null) { Log.Error("Invalid configuation file {0}", config_file_path); Environment.Exit(1); } prev_source = static_index_config.GetOption("Source", null); if (arg_source != null && prev_source != arg_source) { Log.Error("Source already set to {0} for existing static index. Cannot set source to {1}.", prev_source, arg_source); Environment.Exit(1); } bool prev_removable = static_index_config.GetOption("Removable", false); if (arg_removable != prev_removable) { Log.Error("Index previously created for {0}-removable path", (prev_removable ? "" : "non")); Environment.Exit(1); } else { volume_label = static_index_config.GetOption("VolumeLabel", null); } // If arg_source is not given, and prev_source is present, use prev_source // as the arg_source. This is useful for re-running build-index without // giving --arg_source for already existing static index arg_source = prev_source; } // Removable media related options if (arg_removable) { if (pending_files.Count > 0) { Log.Error("Indexing individual files is not allowed for removable media."); Environment.Exit(1); } else if (pending_directories.Count > 1) { Log.Error("Indexing multiple root directories is not allowed for removable media."); Environment.Exit(1); } mnt_dir = ((DirectoryInfo)pending_directories.Peek()).FullName; if (mnt_dir.Length != 1) { mnt_dir = mnt_dir.TrimEnd('/'); } // compute volume label // (1) directory name if block.is_volume is false // (2) hal volume.label if set // (3) hal volume.uuid if set Hal.Manager manager = new Hal.Manager(new Hal.Context()); Hal.Device mnt_device = null; foreach (Hal.Device device in manager.FindDeviceStringMatch("volume.mount_point", mnt_dir)) { mnt_device = device; } string new_volume_label = null; if (mnt_device != null) { new_volume_label = mnt_device.GetPropertyString("volume.label"); if (String.IsNullOrEmpty(new_volume_label)) { new_volume_label = mnt_device.GetPropertyString("volume.uuid"); } } if (new_volume_label == null) { new_volume_label = ((DirectoryInfo)pending_directories.Peek()).Name; } // Sanity check // Volume label is part of the URI, so cannot be changed once set if (volume_label == null) { volume_label = new_volume_label; } else if (volume_label != new_volume_label) { Log.Error("Volume label (earlier '{0}') changed (to '{1}')! You need to create a new index.", volume_label, new_volume_label); Environment.Exit(1); } } if (arg_source == null) { DirectoryInfo dir = new DirectoryInfo(StringFu.SanitizePath(arg_output)); arg_source = dir.Name; } if (!BatteryMonitor.UsingAC && arg_disable_on_battery) { Log.Always("Indexer is disabled when on battery power (--disable-on-battery)"); Environment.Exit(0); } string global_files_config = Path.Combine(PathFinder.ConfigDataDir, "config-files"); global_files_config = Path.Combine(global_files_config, Conf.Names.FilesQueryableConfig + ".xml"); if (!File.Exists(global_files_config)) { Log.Error("Global configuration file not found {0}", global_files_config); Environment.Exit(0); } // Setup regexes for allowed/denied patterns if (allowed_patterns.Count > 0) { allowed_regex = StringFu.GetPatternRegex(allowed_patterns); } else { // Read the exclude values from config // For system-wide indexes, only the global config value will be used Config config = Conf.Get(Conf.Names.FilesQueryableConfig); List <string[]> values = config.GetListOptionValues(Conf.Names.ExcludePattern); if (values != null) { foreach (string[] exclude in values) { denied_patterns.Add(exclude [0]); } } if (denied_patterns.Count > 0) { denied_regex = StringFu.GetPatternRegex(denied_patterns); } } Log.Always("Starting beagle-build-index (pid {0}) at {1}", Process.GetCurrentProcess().Id, DateTime.Now); // Set system priorities so we don't slow down the system SystemPriorities.ReduceIoPriority(); SystemPriorities.SetSchedulerPolicyBatch(); SystemPriorities.Renice(19); driver = new LuceneIndexingDriver(arg_output, MINOR_VERSION, false); driver.TextCache = (arg_cache_text) ? new TextCache(arg_output) : null; if (driver.TextCache != null) { driver.TextCache.WorldReadable = true; } backing_fa_store = new FileAttributesStore_Sqlite(driver.TopDirectory, driver.Fingerprint); fa_store = new FileAttributesStore(backing_fa_store); // Set up signal handlers #if MONO_1_9 Shutdown.SetupSignalHandlers(delegate(int signal) { if (signal == (int)Mono.Unix.Native.Signum.SIGINT || signal == (int)Mono.Unix.Native.Signum.SIGTERM) { Shutdown.BeginShutdown(); } }); #else SetupSignalHandlers(); #endif Thread monitor_thread = null; Stopwatch watch = new Stopwatch(); watch.Start(); if (!arg_disable_restart) { // Start the thread that monitors memory usage. monitor_thread = ExceptionHandlingThread.Start(new ThreadStart(MemoryMonitorWorker)); } // Start indexworker to do the crawling and indexing IndexWorker(); // Join any threads so that we know that we're the only thread still running if (monitor_thread != null) { monitor_thread.Join(); } watch.Stop(); Logger.Log.Debug("Elapsed time {0}.", watch); // Write this after indexing is done. This is because, if creating a new index, // LuceneIndexingDriver.Create() is called which purges the entire directory. if (prev_source == null) { Config static_index_config = Conf.LoadNew("StaticIndex.xml"); // Write StaticIndex.xml containing: // The name of the source static_index_config.SetOption("Source", arg_source); static_index_config ["Source"].Description = "Source of the static index"; if (arg_removable) { static_index_config.SetOption("VolumeLabel", volume_label); static_index_config ["VolumeLabel"].Description = "Volume label of the removable source"; static_index_config.SetOption("Removable", true); static_index_config ["Removable"].Description = "Removable source"; } Conf.SaveTo(static_index_config, config_file_path); } if (restart) { Logger.Log.Debug("Restarting beagle-build-index"); Process p = new Process(); p.StartInfo.UseShellExecute = false; // FIXME: Maybe this isn't the right way to do things? It should be ok, // the PATH is inherited from the shell script which runs mono itself. p.StartInfo.FileName = "mono"; p.StartInfo.Arguments = String.Join(" ", Environment.GetCommandLineArgs()); p.Start(); } Log.Always("Exiting beagle-build-index (pid {0}) at {1}", Process.GetCurrentProcess().Id, DateTime.Now); }
public static bool StartupProcess () { // Profile our initialization Stopwatch stopwatch = new Stopwatch (); stopwatch.Start (); // Fire up our server if (! StartServer ()) { if (! arg_replace) { Logger.Log.Error ("Could not set up the listener for beagle requests. " + "There is probably another beagled instance running. " + "Use --replace to replace the running service"); Environment.Exit (1); } ReplaceExisting (); } // Set up out-of-process indexing LuceneQueryable.IndexerHook = new LuceneQueryable.IndexerCreator (RemoteIndexer.NewRemoteIndexer); Config config = Conf.Get (Conf.Names.DaemonConfig); // Initialize synchronization to keep the indexes local if PathFinder.StorageDir // is on a non-block device, or if BEAGLE_SYNCHRONIZE_LOCALLY is set if ((! SystemInformation.IsPathOnBlockDevice (PathFinder.StorageDir) && config.GetOption (Conf.Names.IndexSynchronization, true)) || Environment.GetEnvironmentVariable ("BEAGLE_SYNCHRONIZE_LOCALLY") != null) IndexSynchronization.Initialize (); // Start the query driver. Logger.Log.Debug ("Starting QueryDriver"); QueryDriver.Start (); // Start our battery monitor so we can shut down the // scheduler if needed. BatteryMonitor.Init (); bool initially_on_battery = ! BatteryMonitor.UsingAC && ! config.GetOption (Conf.Names.IndexOnBattery, false); // Start the Global Scheduler thread if (! arg_disable_scheduler) { if (! initially_on_battery) { Logger.Log.Debug ("Starting Scheduler thread"); Scheduler.Global.Start (); } else { Log.Debug ("Beagle started on battery, not starting scheduler thread"); } } // Start our Inotify threads Inotify.Start (); // Test if the FileAdvise stuff is working: This will print a // warning if not. The actual advice calls will fail silently. FileAdvise.TestAdvise (); #if ENABLE_AVAHI zeroconf = new Beagle.Daemon.Network.Zeroconf (); #endif Conf.WatchForUpdates (); stopwatch.Stop (); Logger.Log.Debug ("Daemon initialization finished after {0}", stopwatch); SystemInformation.LogMemoryUsage (); if (arg_indexing_test_mode) { Thread.Sleep (1000); // Ugly paranoia: wait a second for the backends to settle. Logger.Log.Debug ("Running in indexing test mode"); Scheduler.Global.EmptyQueueEvent += OnEmptySchedulerQueue; Scheduler.Global.Add (null); // pulse the scheduler } return false; }
//////////////////////////////////////////////////////////////// public int DoCountMatchQuery (Query query, QueryPartHook query_part_hook) { if (Debug) Logger.Log.Debug ("###### {0}: Starting low-level queries", IndexName); Stopwatch total; total = new Stopwatch (); total.Start (); ArrayList primary_required_part_queries; ArrayList secondary_required_part_queries; LNS.BooleanQuery primary_prohibited_part_query; LNS.BooleanQuery secondary_prohibited_part_query; AndHitFilter all_hit_filters; ArrayList term_list; term_list = AssembleQuery ( query, query_part_hook, null, out primary_required_part_queries, out secondary_required_part_queries, out primary_prohibited_part_query, out secondary_prohibited_part_query, out all_hit_filters); // If we have no required parts, give up. if (primary_required_part_queries == null) return 0; IndexReader primary_reader; LNS.IndexSearcher primary_searcher; IndexReader secondary_reader; LNS.IndexSearcher secondary_searcher; if (! BuildSearchers (out primary_reader, out primary_searcher, out secondary_reader, out secondary_searcher)) return 0; // Build whitelists and blacklists for search subsets. LuceneBitArray primary_whitelist, secondary_whitelist; CreateQueryWhitelists (null, primary_searcher, secondary_searcher, primary_prohibited_part_query, secondary_prohibited_part_query, out primary_whitelist, out secondary_whitelist); // Now run the low level queries against our indexes. BetterBitArray primary_matches = null; if (primary_required_part_queries != null) { if (secondary_searcher != null) primary_matches = DoRequiredQueries_TwoIndex (primary_searcher, secondary_searcher, primary_required_part_queries, secondary_required_part_queries, primary_whitelist, secondary_whitelist); else primary_matches = DoRequiredQueries (primary_searcher, primary_required_part_queries, primary_whitelist); } int result = 0; // FIXME: Pass the count through uri-filter and other validation checks if (primary_matches != null) result = primary_matches.TrueCount; CloseSearchers (primary_reader, primary_searcher, secondary_reader, secondary_searcher); total.Stop (); if (Debug) Logger.Log.Debug ("###### {0}: Total query run in {1}", IndexName, total); return result; }
//////////////////////////////////////////////////////////////// public void DoQuery (Query query, IQueryResult result, ICollection search_subset_uris, // should be internal uris QueryPartHook query_part_hook, HitFilter hit_filter) { if (Debug) Logger.Log.Debug ("###### {0}: Starting low-level queries", IndexName); Stopwatch total, a, b, c, d, e, f; total = new Stopwatch (); a = new Stopwatch (); b = new Stopwatch (); c = new Stopwatch (); d = new Stopwatch (); e = new Stopwatch (); f = new Stopwatch (); total.Start (); a.Start (); ArrayList primary_required_part_queries; ArrayList secondary_required_part_queries; LNS.BooleanQuery primary_prohibited_part_query; LNS.BooleanQuery secondary_prohibited_part_query; AndHitFilter all_hit_filters; ArrayList term_list; // Assemble all of the parts into a bunch of Lucene queries term_list = AssembleQuery (query, query_part_hook, hit_filter, out primary_required_part_queries, out secondary_required_part_queries, out primary_prohibited_part_query, out secondary_prohibited_part_query, out all_hit_filters); a.Stop (); if (Debug) Log.Debug ("###### {0}: Building queries took {1}", IndexName, a); // If we have no required parts, give up. if (primary_required_part_queries == null) return; b.Start (); // // Now that we have all of these nice queries, let's execute them! // IndexReader primary_reader; LNS.IndexSearcher primary_searcher; IndexReader secondary_reader; LNS.IndexSearcher secondary_searcher; // Create the searchers that we will need. if (! BuildSearchers (out primary_reader, out primary_searcher, out secondary_reader, out secondary_searcher)) return; b.Stop (); if (Debug) Log.Debug ("###### {0}: Readers/searchers built in {1}", IndexName, b); // Build whitelists and blacklists for search subsets. c.Start (); // Possibly create our whitelists from the search subset. LuceneBitArray primary_whitelist, secondary_whitelist; CreateQueryWhitelists (search_subset_uris, primary_searcher, secondary_searcher, primary_prohibited_part_query, secondary_prohibited_part_query, out primary_whitelist, out secondary_whitelist); c.Stop (); if (Debug) Log.Debug ("###### {0}: Whitelists and blacklists built in {1}", IndexName, c); // Now run the low level queries against our indexes. d.Start (); BetterBitArray primary_matches = null; if (primary_required_part_queries != null) { if (secondary_searcher != null) primary_matches = DoRequiredQueries_TwoIndex (primary_searcher, secondary_searcher, primary_required_part_queries, secondary_required_part_queries, primary_whitelist, secondary_whitelist); else primary_matches = DoRequiredQueries (primary_searcher, primary_required_part_queries, primary_whitelist); } d.Stop (); if (Debug) Logger.Log.Debug ("###### {0}: Low-level queries finished in {1}", IndexName, d); e.Start (); // Only generate results if we got some matches if (primary_matches != null && primary_matches.ContainsTrue ()) { GenerateQueryResults (primary_reader, secondary_reader, primary_matches, result, term_list, query.MaxHits, new HitFilter (all_hit_filters.HitFilter), IndexName); } e.Stop (); if (Debug) Log.Debug ("###### {0}: Query results generated in {1}", IndexName, e); // // Finally, we clean up after ourselves. // f.Start (); CloseSearchers (primary_reader, primary_searcher, secondary_reader, secondary_searcher); f.Stop (); if (Debug) Log.Debug ("###### {0}: Readers/searchers released in {1}", IndexName, f); total.Stop (); if (Debug) { Log.Debug ("###### {0}: Query time breakdown:", IndexName); Log.Debug ("###### {0}: Build queries {1,6} ({2:0.0}%)", IndexName, a, 100 * a.ElapsedTime / total.ElapsedTime); Log.Debug ("###### {0}: Got readers {1,6} ({2:0.0}%)", IndexName, b, 100 * b.ElapsedTime / total.ElapsedTime); Log.Debug ("###### {0}: Whitelists {1,6} ({2:0.0}%)", IndexName, c, 100 * c.ElapsedTime / total.ElapsedTime); Log.Debug ("###### {0}: Queries {1,6} ({2:0.0}%)", IndexName, d, 100 * d.ElapsedTime / total.ElapsedTime); Log.Debug ("###### {0}: Gen'd Results {1,6} ({2:0.0}%)", IndexName, e, 100 * e.ElapsedTime / total.ElapsedTime); Log.Debug ("###### {0}: Reader cleanup {1,6} ({2:0.0}%)", IndexName, f, 100 * f.ElapsedTime / total.ElapsedTime); Log.Debug ("###### {0}: TOTAL {1,6}", IndexName, total); Logger.Log.Debug ("###### {0}: Total query run in {1}", IndexName, total); } }
private static ArrayList FindRecentResults (IndexReader primary_reader, IndexReader secondary_reader, BetterBitArray primary_matches, Dictionary<int, Hit> hits_by_id, int max_results, ref int total_number_of_matches, HitFilter hit_filter, string index_name) { Stopwatch b = new Stopwatch (); b.Start (); int count = 0; Document doc; ArrayList all_docs = null; TopScores top_docs = null; TermDocs term_docs = null; if (primary_matches.TrueCount > max_results) top_docs = new TopScores (max_results); else all_docs = new ArrayList (primary_matches.TrueCount); if (secondary_reader != null) term_docs = secondary_reader.TermDocs (); for (int match_index = primary_matches.Count; ; match_index --) { // Walk across the matches backwards, since newer // documents are more likely to be at the end of // the index. match_index = primary_matches.GetPreviousTrueIndex (match_index); if (match_index < 0) break; count++; doc = primary_reader.Document (match_index, fields_timestamp_uri); // Check the timestamp --- if we have already reached our // limit, we might be able to reject it immediately. string timestamp_str; long timestamp_num = 0; timestamp_str = doc.Get ("Timestamp"); if (timestamp_str == null) { Logger.Log.Warn ("No timestamp on {0}!", GetUriFromDocument (doc)); } else { timestamp_num = Int64.Parse (doc.Get ("Timestamp")); if (top_docs != null && ! top_docs.WillAccept (timestamp_num)) continue; } // Get the actual hit now // doc was created with only 2 fields, so first get the complete lucene document for primary document. // Also run our hit_filter now, if we have one. Since we insist of returning max_results // most recent hits, any hits that would be filtered out should happen now and not later. Hit hit = CreateHit (primary_reader.Document (match_index), secondary_reader, term_docs); if (hit_filter != null && ! hit_filter (hit)) { if (Debug) Log.Debug ("Filtered out {0}", hit.Uri); total_number_of_matches --; continue; } hits_by_id [match_index] = hit; // Add the document to the appropriate data structure. // We use the timestamp_num as the score, so high // scores correspond to more-recent timestamps. if (all_docs != null) all_docs.Add (hit); else top_docs.Add (timestamp_num, hit); } if (term_docs != null) term_docs.Close (); b.Stop (); if (Debug) Log.Debug (">>> {0}: Instantiated and scanned {1} documents in {2}", index_name, count, b); if (all_docs != null) { // Sort results before sending all_docs.Sort (); return all_docs; } else { return top_docs.TopScoringObjects; } }
private ICollection DoLowLevelRDFQuery (Query query, PropertyType pred_type, string predicate, string field_value, TextCache text_cache) { Stopwatch total, a, b, c, d, e, f; total = new Stopwatch (); a = new Stopwatch (); b = new Stopwatch (); c = new Stopwatch (); d = new Stopwatch (); e = new Stopwatch (); f = new Stopwatch (); total.Start (); a.Start (); // Assemble all of the parts into a bunch of Lucene queries ArrayList primary_required_part_queries; ArrayList secondary_required_part_queries; LNS.BooleanQuery primary_prohibited_part_query; LNS.BooleanQuery secondary_prohibited_part_query; AndHitFilter all_hit_filters; ArrayList term_list; // Assemble all of the parts into a bunch of Lucene queries term_list = AssembleQuery (query, null, null, out primary_required_part_queries, out secondary_required_part_queries, out primary_prohibited_part_query, out secondary_prohibited_part_query, out all_hit_filters); a.Stop (); if (Debug) Log.Debug ("###### {0}: Building queries took {1}", IndexName, a); // If we have no required parts, give up. if (primary_required_part_queries == null) return null; b.Start (); // // Now that we have all of these nice queries, let's execute them! // // Create the searchers that we will need. IndexReader primary_reader; LNS.IndexSearcher primary_searcher; IndexReader secondary_reader; LNS.IndexSearcher secondary_searcher; // Create the searchers that we will need. if (! BuildSearchers (out primary_reader, out primary_searcher, out secondary_reader, out secondary_searcher)) return null; b.Stop (); if (Debug) Log.Debug ("###### {0}: Readers/searchers built in {1}", IndexName, b); // Build whitelists and blacklists for search subsets. c.Start (); // Possibly create our whitelists from the search subset. LuceneBitArray primary_whitelist, secondary_whitelist; CreateQueryWhitelists (null, primary_searcher, secondary_searcher, primary_prohibited_part_query, secondary_prohibited_part_query, out primary_whitelist, out secondary_whitelist); c.Stop (); if (Debug) Log.Debug ("###### {0}: Whitelists and blacklists built in {1}", IndexName, c); // Now run the low level queries against our indexes. d.Start (); BetterBitArray primary_matches = null; if (primary_required_part_queries != null) { if (secondary_searcher != null) primary_matches = DoRequiredQueries_TwoIndex (primary_searcher, secondary_searcher, primary_required_part_queries, secondary_required_part_queries, primary_whitelist, secondary_whitelist); else primary_matches = DoRequiredQueries (primary_searcher, primary_required_part_queries, primary_whitelist); } d.Stop (); if (Debug) Logger.Log.Debug ("###### {0}: Low-level queries finished in {1} and returned {2} matches", IndexName, d, primary_matches.TrueCount); e.Start (); int count = 0; Document doc; ArrayList hits = new ArrayList (primary_matches.TrueCount); TermDocs secondary_term_docs = null; if (secondary_searcher != null) secondary_term_docs = secondary_searcher.Reader.TermDocs (); FieldSelector fields = null; if (predicate != null) fields = new MapFieldSelector (new string[] { "Uri", "Timestamp", PropertyToFieldName (pred_type, predicate)}); for (int match_index = primary_matches.GetNextTrueIndex (0); match_index < primary_matches.Count; match_index = primary_matches.GetNextTrueIndex (++ match_index)) { count++; // If we have a HitFilter, apply it. // RDF FIXME: Ignore Hit Filter for now // If predicate was not specified but object was specified, // then figure out the right predicate if (predicate == null && field_value != null) { Hit hit = new Hit (); doc = primary_searcher.Doc (match_index); hit.Uri = GetUriFromDocument (doc); hit.Timestamp = StringFu.StringToDateTime (doc.Get ("Timestamp")); bool found_matching_predicate = false; foreach (Field field in doc.Fields ()) { if (! FieldIsPredicate (field, field_value)) continue; Property prop = new Property (); prop.Type = pred_type; prop.Key = predicate; prop.Value = field_value; hit.AddProperty (prop); found_matching_predicate = true; } // Now get the matching predicate from the secondary index if (secondary_searcher == null) { doc = null; } else { Term term = new Term ("Uri", doc.Get ("Uri")); secondary_term_docs.Seek (term); if (secondary_term_docs.Next ()) doc = secondary_searcher.Doc (secondary_term_docs.Doc ()); } if (doc != null) { foreach (Field field in doc.Fields ()) { if (! FieldIsPredicate (field, field_value)) continue; Property prop = new Property (); prop.Type = pred_type; prop.Key = predicate; prop.Value = field_value; hit.AddProperty (prop); found_matching_predicate = true; } } if (! found_matching_predicate) { // No matching predicate found // This means some unstored field matched the query // FIXME: Add a synthetic property #text hit.AddProperty (Property.New ("#text", field_value)); } hits.Add (hit); } else if (predicate == "TextLinks") { // Special treatment: TextLinks is not stored but can be queried doc = primary_searcher.Doc (match_index, fields_timestamp_uri); Hit hit = CreateHit (doc, secondary_reader, secondary_term_docs, fields); if (field_value != null) hit.AddProperty (Property.New ("TextLinks", field_value)); else { foreach (Property text_link_property in GetTextLinks (hit.Uri, text_cache)) hit.AddProperty (text_link_property); } hits.Add (hit); } else { doc = primary_searcher.Doc (match_index, fields); Hit hit = CreateHit (doc, secondary_reader, secondary_term_docs, fields); foreach (Property prop in hit.Properties) { if (prop.Key == predicate) prop.Value = field_value; } hits.Add (hit); } } e.Stop (); if (Debug) Log.Debug ("###### {0}: Query results generated in {1}", IndexName, e); // // Finally, we clean up after ourselves. // f.Start (); CloseSearchers (primary_reader, primary_searcher, secondary_reader, secondary_searcher); f.Stop (); if (Debug) Log.Debug ("###### {0}: Readers/searchers released in {1}", IndexName, f); total.Stop (); if (Debug) { Log.Debug ("###### {0}: Query time breakdown:", IndexName); Log.Debug ("###### {0}: Build queries {1,6} ({2:0.0}%)", IndexName, a, 100 * a.ElapsedTime / total.ElapsedTime); Log.Debug ("###### {0}: Got readers {1,6} ({2:0.0}%)", IndexName, b, 100 * b.ElapsedTime / total.ElapsedTime); Log.Debug ("###### {0}: Whitelists {1,6} ({2:0.0}%)", IndexName, c, 100 * c.ElapsedTime / total.ElapsedTime); Log.Debug ("###### {0}: Queries {1,6} ({2:0.0}%)", IndexName, d, 100 * d.ElapsedTime / total.ElapsedTime); Log.Debug ("###### {0}: Gen'd Results {1,6} ({2:0.0}%)", IndexName, e, 100 * e.ElapsedTime / total.ElapsedTime); Log.Debug ("###### {0}: Reader cleanup {1,6} ({2:0.0}%)", IndexName, f, 100 * f.ElapsedTime / total.ElapsedTime); Log.Debug ("###### {0}: TOTAL {1,6}", IndexName, total); Logger.Log.Debug ("###### {0}: Total query run in {1}", IndexName, total); } return hits; }
// There are two ways we can determine the max_results // most recent items: // // One is to instantiate Lucene documents for each of // the document IDs in primary_matches. This is a // fairly expensive operation. // // The other is to walk through the list of all // document IDs in descending time order. This is // a less expensive operation, but adds up over time // on large data sets. // // We can walk about 2.5 docs for every Document we // instantiate. So what we'll do, if we have more // matches than available hits, is walk (m * 1.25) // docs to see if we can fill out the top 100 hits. // If not, we'll fall back to creating documents // for all of them. private static ArrayList ScanRecentDocs (IndexReader primary_reader, IndexReader secondary_reader, BetterBitArray primary_matches, Dictionary<int, Hit> hits_by_id, int max_results, ref int total_number_of_matches, HitFilter hit_filter, string index_name) { Stopwatch a = new Stopwatch (); a.Start (); TermDocs docs = primary_reader.TermDocs (); TermEnum enumerator = primary_reader.Terms (new Term ("InvertedTimestamp", String.Empty)); ArrayList results = new ArrayList (max_results); int docs_found = 0; int docs_walked = 0; int hit_filter_removed = 0; int max_docs = (int) (primary_matches.TrueCount * 1.25); Term term; TermDocs secondary_term_docs = null; if (secondary_reader != null) secondary_term_docs = secondary_reader.TermDocs (); do { term = enumerator.Term (); if (term.Field () != "InvertedTimestamp") break; docs.Seek (enumerator); while (docs.Next () && docs_found < max_results && docs_walked < max_docs) { int doc_id = docs.Doc (); if (primary_matches.Get (doc_id)) { Document doc = primary_reader.Document (doc_id); Hit hit = CreateHit (doc, secondary_reader, secondary_term_docs); // If we have a HitFilter, apply it. if (hit_filter != null && ! hit_filter (hit)) { if (Debug) Log.Debug ("Filtered out {0}", hit.Uri); hit_filter_removed ++; continue; } hits_by_id [doc_id] = hit; // Add the result, last modified first results.Add (hit); docs_found++; } docs_walked++; } } while (enumerator.Next () && docs_found < max_results && docs_walked < max_docs); docs.Close (); if (secondary_term_docs != null) secondary_term_docs.Close (); // If we've found all the docs we can return in a subset! // Fantastic, we've probably short circuited a slow search. if (docs_found != max_results) { // Otherwise bad luck! Not all docs found // Start afresh - this time traversing all results results = null; } else { // Adjust total_number_of_matches. We need to do this to avoid scenarios like the following: // max_hits = 100. Matched 100 results. But hit filter removed 30. So 70 results will be returned. // We want to avoid saying "Showing top 70 of 100". Note that since we are not passing // every document in the index through the hit_filter, when we say "Showing top 100 of 1234", the // 1234 could actually be much less. But since max_hits was 100, that will not mislead the user. total_number_of_matches -= hit_filter_removed; } a.Stop (); if (Debug) { Log.Debug (">>> {0}: Walked {1} items, populated an enum with {2} items in {3}", index_name, docs_walked, docs_found, a); if (docs_found == max_results) Log.Debug (">>> {0}: Successfully short circuited timestamp ordering!", index_name); } return results; }
private static void GenerateQueryResults (IndexReader primary_reader, IndexReader secondary_reader, BetterBitArray primary_matches, IQueryResult result, ICollection query_term_list, int max_results, HitFilter hit_filter, string index_name) { int num_hits; if (Debug) Logger.Log.Debug (">>> {0}: Initially handed {1} matches", index_name, primary_matches.TrueCount); if (primary_matches.TrueCount <= max_results) { if (Debug) Logger.Log.Debug (">>> {0}: Initial count is within our limit of {1}", index_name, max_results); num_hits = primary_matches.TrueCount; } else { if (Debug) Logger.Log.Debug (">>> {0}: Number of hits is capped at {1}", index_name, max_results); num_hits = max_results; } Stopwatch total, d, e; total = new Stopwatch (); d = new Stopwatch (); e = new Stopwatch (); total.Start (); ArrayList final_list_of_hits = null; // This is used only for scoring Dictionary<int, Hit> hits_by_id = new Dictionary<int, Hit> (num_hits); int total_number_of_matches = primary_matches.TrueCount; if (primary_matches.TrueCount > max_results) final_list_of_hits = ScanRecentDocs (primary_reader, secondary_reader, primary_matches, hits_by_id, max_results, ref total_number_of_matches, hit_filter, index_name); if (final_list_of_hits == null) final_list_of_hits = FindRecentResults (primary_reader, secondary_reader, primary_matches, hits_by_id, max_results, ref total_number_of_matches, hit_filter, index_name); d.Start (); ScoreHits (hits_by_id, primary_reader, query_term_list); hits_by_id = null; d.Stop (); if (Debug) Log.Debug (">>> {0}: Scored hits in {1}", index_name, d); e.Start (); // 25 hits seems to be the sweet spot: anything lower // and serialization overhead gets us, higher takes // longer to send out. const int MAX_QUEUED_HITS = 25; int sent_index = 0; // Break up the hits into reasonably sized chunks for // sending over the wire. for (int i = 0; i < final_list_of_hits.Count; ++i) { // Flush our hits if (i > 0 && i % MAX_QUEUED_HITS == 0) { result.Add (final_list_of_hits.GetRange (0, MAX_QUEUED_HITS)); final_list_of_hits.RemoveRange (0, MAX_QUEUED_HITS); i -= MAX_QUEUED_HITS; } } // Flush the remaining hits result.Add (final_list_of_hits, total_number_of_matches); final_list_of_hits = null; e.Stop (); if (Debug) Log.Debug (">>> {0}: Hit filters executed and results sent in {1}", index_name, e); total.Stop (); if (Debug) { Logger.Log.Debug (">>> {0}: GenerateQueryResults time statistics:", index_name); //Logger.Log.Debug (">>> {0}: Short circuit {1,6} ({2:0.0}%)", index_name, a == null ? "N/A" : a.ToString (), a == null ? 0.0 : 100 * a.ElapsedTime / total.ElapsedTime); //Logger.Log.Debug (">>> {0}: Create docs {1,6} ({2:0.0}%)", index_name, b, 100 * b.ElapsedTime / total.ElapsedTime); //Logger.Log.Debug (">>> {0}: Hit assembly {1,6} ({2:0.0}%)", index_name, c, 100 * c.ElapsedTime / total.ElapsedTime); Logger.Log.Debug (">>> {0}: Scored hits {1,6} ({2:0.0}%)", index_name, d, 100 * d.ElapsedTime / total.ElapsedTime); Logger.Log.Debug (">>> {0}: Results sent {1,6} ({2:0.0}%)", index_name, e, 100 * e.ElapsedTime / total.ElapsedTime); Logger.Log.Debug (">>> {0}: TOTAL {1,6}", index_name, total); } }
public static bool StartupProcess() { // Profile our initialization Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); // Fire up our server if (!StartServer()) { if (!arg_replace) { Logger.Log.Error("Could not set up the listener for beagle requests. " + "There is probably another beagled instance running. " + "Use --replace to replace the running service"); Environment.Exit(1); } ReplaceExisting(); } // Set up out-of-process indexing LuceneQueryable.IndexerHook = new LuceneQueryable.IndexerCreator(RemoteIndexer.NewRemoteIndexer); Config config = Conf.Get(Conf.Names.DaemonConfig); // Initialize synchronization to keep the indexes local if PathFinder.StorageDir // is on a non-block device, or if BEAGLE_SYNCHRONIZE_LOCALLY is set if ((!SystemInformation.IsPathOnBlockDevice(PathFinder.StorageDir) && config.GetOption(Conf.Names.IndexSynchronization, true)) || Environment.GetEnvironmentVariable("BEAGLE_SYNCHRONIZE_LOCALLY") != null) { IndexSynchronization.Initialize(); } // Start the query driver. Logger.Log.Debug("Starting QueryDriver"); QueryDriver.Start(); // Start our battery monitor so we can shut down the // scheduler if needed. BatteryMonitor.Init(); bool initially_on_battery = !BatteryMonitor.UsingAC && !config.GetOption(Conf.Names.IndexOnBattery, false); // Start the Global Scheduler thread if (!arg_disable_scheduler) { if (!initially_on_battery) { Logger.Log.Debug("Starting Scheduler thread"); Scheduler.Global.Start(); } else { Log.Debug("Beagle started on battery, not starting scheduler thread"); } } // Start our Inotify threads Inotify.Start(); // Test if the FileAdvise stuff is working: This will print a // warning if not. The actual advice calls will fail silently. FileAdvise.TestAdvise(); #if ENABLE_AVAHI zeroconf = new Beagle.Daemon.Network.Zeroconf(); #endif Conf.WatchForUpdates(); stopwatch.Stop(); Logger.Log.Debug("Daemon initialization finished after {0}", stopwatch); SystemInformation.LogMemoryUsage(); if (arg_indexing_test_mode) { Thread.Sleep(1000); // Ugly paranoia: wait a second for the backends to settle. Logger.Log.Debug("Running in indexing test mode"); Scheduler.Global.EmptyQueueEvent += OnEmptySchedulerQueue; Scheduler.Global.Add(null); // pulse the scheduler } return(false); }
private IndexerReceipt [] Flush_Unlocked (IndexerRequest request) { ArrayList receipt_queue; receipt_queue = new ArrayList (); IndexReader primary_reader, secondary_reader; primary_reader = IndexReader.Open (PrimaryStore); secondary_reader = IndexReader.Open (SecondaryStore); // Step #1: Make our first pass over the list of // indexables that make up our request. For each add // or property change in the request, get the Lucene // documents so we can move forward any persistent // properties (for adds) or all old properties (for // property changes). // // Then, for each add or remove in the request, // delete the associated documents from the index. // Note that we previously cached added documents so // that we can move persistent properties forward. // parent_child_old_props is double-nested hashtable (depth-2 tree) // indexed by the parent uri, it stores another hashtable indexed by the (parent+child documents) // FIXME: 2-level hashtable is a waste for any non-child document. // Replace this by a better data structure. Hashtable parent_child_old_props = UriFu.NewHashtable (); TermDocs term_docs = secondary_reader.TermDocs (); int delete_count = 0; IEnumerable request_indexables = request.Indexables; foreach (Indexable indexable in request_indexables) { string uri_str = UriFu.UriToEscapedString (indexable.Uri); Term term; // Store the necessary properties from old documents for re-addition if (indexable.Type == IndexableType.Add || indexable.Type == IndexableType.PropertyChange) { term = new Term ("Uri", uri_str); term_docs.Seek (term); Hashtable this_parent_child_props = null; if (term_docs.Next ()) { this_parent_child_props = UriFu.NewHashtable (); this_parent_child_props [indexable.Uri] = secondary_reader.Document (term_docs.Doc ()); parent_child_old_props [indexable.Uri] = this_parent_child_props; } term = new Term ("ParentUri", uri_str); term_docs.Seek (term); while (term_docs.Next ()) { Document doc = secondary_reader.Document (term_docs.Doc ()); string child_uri_str = doc.Get ("Uri"); Uri child_uri = UriFu.EscapedStringToUri (child_uri_str); // Any valid lucene document *should* have a Uri, so no need to check for null // Store the child documents too, to save persistent-properties // of child documents this_parent_child_props [child_uri] = doc; } } // Now remove (non-remove indexables will be re-added in next block) Logger.Log.Debug ("-{0}", indexable.DisplayUri); int num_delete = 0; term = new Term ("Uri", uri_str); // For property changes, only secondary index is modified secondary_reader.DeleteDocuments (term); // Now remove from everywhere else (if asked to remove or if asked to add, in which case // we first remove and then add) // So we also need to remove child documents if (indexable.Type != IndexableType.PropertyChange) { num_delete = primary_reader.DeleteDocuments (term); // When we delete an indexable, also delete any children. // FIXME: Shouldn't we also delete any children of children, etc.? term = new Term ("ParentUri", uri_str); num_delete += primary_reader.DeleteDocuments (term); secondary_reader.DeleteDocuments (term); } // If this is a strict removal (and not a deletion that // we are doing in anticipation of adding something back), // queue up a removed receipt. if (indexable.Type == IndexableType.Remove) { IndexerRemovedReceipt r; r = new IndexerRemovedReceipt (indexable.Id); r.NumRemoved = num_delete; receipt_queue.Add (r); } delete_count += num_delete; } term_docs.Close (); if (HaveItemCount) AdjustItemCount (-delete_count); else SetItemCount (primary_reader); // We are now done with the readers, so we close them. // And also free them. Somehow not freeing them is preventing them from // GCed at all. primary_reader.Close (); primary_reader = null; secondary_reader.Close (); secondary_reader = null; // FIXME: If we crash at exactly this point, we are in // trouble. Items will have been dropped from the index // without the proper replacements being added. We can // hopefully fix this when we move to Lucene 2.1. // Step #2: Make another pass across our list of indexables // and write out any new documents. if (text_cache != null) text_cache.BeginTransaction (); IndexWriter primary_writer, secondary_writer; // FIXME: Lock obtain time-out can happen here; if that happens, // an exception will be thrown and this method will break in the middle // leaving IndexWriters unclosed! Same for any Lucene.Net-index modification // methods. primary_writer = new IndexWriter (PrimaryStore, IndexingAnalyzer, false); secondary_writer = null; foreach (Indexable indexable in request_indexables) { // If shutdown has been started, break here // FIXME: Some more processing will continue, a lot of them // concerning receipts, but the daemon will anyway ignore receipts // now, what is the fastest way to stop from here ? if (Shutdown.ShutdownRequested) { Log.Debug ("Shutdown initiated. Breaking while flushing indexables."); break; } // Receipts for removes were generated in the // previous block. Now we just have to remove // items from the text cache. if (indexable.Type == IndexableType.Remove) { if (text_cache != null) text_cache.Delete (indexable.Uri); continue; } IndexerAddedReceipt r; Hashtable prop_change_docs = (Hashtable) parent_child_old_props [indexable.Uri]; if (indexable.Type == IndexableType.PropertyChange) { Logger.Log.Debug ("+{0} (props only)", indexable.DisplayUri); r = new IndexerAddedReceipt (indexable.Id); r.PropertyChangesOnly = true; receipt_queue.Add (r); Document doc; if (prop_change_docs == null) doc = null; else doc = (Document) prop_change_docs [indexable.Uri]; Document new_doc; new_doc = RewriteDocument (doc, indexable); // Write out the new document... if (secondary_writer == null) secondary_writer = new IndexWriter (SecondaryStore, IndexingAnalyzer, false); secondary_writer.AddDocument (new_doc); // Get child property change indexables... ArrayList prop_change_indexables; prop_change_indexables = GetChildPropertyChange (prop_change_docs, indexable); // and store them; no need to delete them first, since they were already removed from the index if (prop_change_indexables == null) continue; foreach (Indexable prop_change_indexable in prop_change_indexables) { Log.Debug ("+{0} (props only, generated indexable)", prop_change_indexable.Uri); doc = (Document) prop_change_docs [prop_change_indexable.Uri]; new_doc = RewriteDocument (doc, prop_change_indexable); secondary_writer.AddDocument (new_doc); } continue; // ...and proceed to the next Indexable } // If we reach this point we know we are dealing with an IndexableType.Add if (indexable.Type != IndexableType.Add) throw new Exception ("When I said it was an IndexableType.Add, I meant it!"); r = AddIndexableToIndex (indexable, primary_writer, ref secondary_writer, prop_change_docs); if (r != null) receipt_queue.Add (r); } if (text_cache != null) text_cache.CommitTransaction (); if (Shutdown.ShutdownRequested) { foreach (DeferredInfo di in deferred_indexables) di.Cleanup (); deferred_indexables.Clear (); foreach (Indexable indexable in request_indexables) indexable.Cleanup (); primary_writer.Close (); if (secondary_writer != null) secondary_writer.Close (); return null; } if (request.OptimizeIndex) { Stopwatch watch = new Stopwatch (); Logger.Log.Debug ("Optimizing {0}", IndexName); watch.Start (); primary_writer.Optimize (); if (secondary_writer == null) secondary_writer = new IndexWriter (SecondaryStore, IndexingAnalyzer, false); secondary_writer.Optimize (); watch.Stop (); Logger.Log.Debug ("{0} optimized in {1}", IndexName, watch); } // Step #4. Close our writers and return the events to // indicate what has happened. primary_writer.Close (); if (secondary_writer != null) secondary_writer.Close (); // Send a single IndexerIndexablesReceipt if there were deferred indexables if (deferred_indexables.Count > 0) { Log.Debug ("{0} indexables generated more indexables; asking daemon to schedule their indexing.", deferred_indexables.Count); IndexerIndexablesReceipt r = new IndexerIndexablesReceipt (); receipt_queue.Add (r); } IndexerReceipt [] receipt_array; receipt_array = new IndexerReceipt [receipt_queue.Count]; for (int i = 0; i < receipt_queue.Count; ++i) receipt_array [i] = (IndexerReceipt) receipt_queue [i]; return receipt_array; }
static void LaunchHelper () { // If we are in the process of shutting down, return immediately. if (Shutdown.ShutdownRequested) return; lock (helper_lock) { // If a helper appears to be running, return immediately. if (CheckHelper ()) return; Logger.Log.Debug ("Launching helper process"); SafeProcess p = new SafeProcess (); string[] args = new string [3]; args [0] = helper_path; if (BeagleDaemon.DisableTextCache) args [1] = "--disable-text-cache"; else args [1] = String.Empty; if (Log.Level == LogLevel.Debug) args [2] = "--debug"; else args [2] = String.Empty; p.Arguments = args; p.RedirectStandardOutput = false; p.RedirectStandardError = false; p.Start (); Logger.Log.Debug ("IndexHelper PID is {0}", p.Id); // Poll the helper's socket. Wait up to a minute // (500 ms * 120 times) for the helper to be ready // to handle requests. Stopwatch watch = new Stopwatch (); watch.Start (); int poll_count = 0; bool found_helper; do { Thread.Sleep (500); ++poll_count; found_helper = CheckHelper (); } while (poll_count < 120 && ! found_helper && ! Shutdown.ShutdownRequested); watch.Stop (); if (! found_helper) throw new Exception (String.Format ("Couldn't launch helper process {0}", p.Id)); Logger.Log.Debug ("Found IndexHelper ({0}) in {1}", p.Id, watch); helper_pid = p.Id; } }