/////////////////////////////////////////////////////////
// Merge an external Beagrep index to the current index. Merging will
// join the primary- and secondary lucene indexes and if available, the
// file attributes store.
static void ExecuteMerge(string index_dir, string index_to_merge)
{
LuceneIndexingDriver driver = new LuceneIndexingDriver(index_dir, false);
if (!Path.IsPathRooted(index_to_merge))
{
index_to_merge = Path.GetFullPath(index_to_merge);
}
if (!Directory.Exists(index_to_merge))
{
Console.WriteLine("Could not find index to merge: {0}", index_to_merge);
Environment.Exit(1);
}
// Set system priorities so we don't slow down the system
SystemPriorities.ReduceIoPriority();
SystemPriorities.SetSchedulerPolicyBatch();
LuceneQueryingDriver driver_to_merge = new LuceneQueryingDriver(index_to_merge, -1, false);
Stopwatch watch = new Stopwatch();
watch.Start();
// Merge the lucene index
try {
driver.Merge(driver_to_merge);
} catch (Exception e) {
Console.WriteLine("Index merging (lucene) failed: {0}", e);
Environment.Exit(1);
}
// Merge file attributes stores
FileAttributesStore_Sqlite store;
try {
store = new FileAttributesStore_Sqlite(driver.TopDirectory, driver.Fingerprint);
store.Merge(new FileAttributesStore_Sqlite(driver_to_merge.TopDirectory, driver_to_merge.Fingerprint));
} catch (Exception e) {
Console.WriteLine("Index merging (attributes store) failed: {0}", e);
Environment.Exit(1);
}
watch.Stop();
Console.WriteLine("Successfully merged index {0} into {1} in {2}", index_to_merge, driver.TopDirectory, watch);
}
/////////////////////////////////////////////////////////
// Execute a lucene optimize-task on the index.
static void ExecuteOptimize(string index_dir)
{
LuceneIndexingDriver driver = new LuceneIndexingDriver(index_dir, false);
// Set system priorities so we don't slow down the system
SystemPriorities.ReduceIoPriority();
SystemPriorities.SetSchedulerPolicyBatch();
Stopwatch watch = new Stopwatch();
watch.Start();
driver.OptimizeNow();
watch.Stop();
Console.WriteLine("Optimized index {0} in {1}", driver.TopDirectory, watch);
}
private static void DoMain(string [] args)
{
SystemInformation.SetProcessName("beagled-helper");
bool run_by_hand = (Environment.GetEnvironmentVariable("BEAGLE_RUN_HELPER_BY_HAND") != null);
bool log_in_fg = (Environment.GetEnvironmentVariable("BEAGLE_LOG_IN_THE_FOREGROUND_PLEASE") != null);
bool debug = false, disable_textcache = false;
foreach (string arg in args)
{
if (arg == "--disable-text-cache")
{
disable_textcache = true;
}
else if (arg == "--debug")
{
debug = true;
}
}
last_activity = DateTime.Now;
Log.Initialize(PathFinder.LogDir,
"IndexHelper",
debug ? LogLevel.Debug : LogLevel.Warn,
run_by_hand || log_in_fg);
Log.Always("Starting Index Helper process (version {0})", ExternalStringsHack.Version);
Log.Always("Running on {0}", SystemInformation.MonoRuntimeVersion);
Log.Always("Extended attributes are {0}", ExtendedAttribute.Supported ? "supported" : "not supported");
Log.Always("Command Line: {0}",
Environment.CommandLine != null ? Environment.CommandLine : "(null)");
if (disable_textcache)
{
Log.Always("Text cache is disabled.");
}
// Initialize GObject type system
g_type_init();
// Set the IO priority to idle, nice ourselves, and set
// a batch scheduling policy so we that we play nice
// on the system
if (Environment.GetEnvironmentVariable("BEAGLE_EXERCISE_THE_DOG") != null)
{
Log.Always("BEAGLE_EXERCISE_THE_DOG is set");
}
SystemPriorities.ReduceIoPriority();
int nice_to_set;
// We set different nice values because the
// internal implementation of SCHED_BATCH
// unconditionally imposes a +5 penalty on
// processes, and we want to be at nice +17,
// because it has a nice timeslice.
if (SystemPriorities.SetSchedulerPolicyBatch())
{
nice_to_set = 12;
}
else
{
nice_to_set = 17;
}
SystemPriorities.Renice(nice_to_set);
Server.Init();
#if MONO_1_9
Shutdown.SetupSignalHandlers(new Shutdown.SignalHandler(HandleSignal));
#else
SetupSignalHandlers();
#endif
Shutdown.ShutdownEvent += OnShutdown;
main_loop = new MainLoop();
Shutdown.RegisterMainLoop(main_loop);
// Start the server
Log.Debug("Starting messaging server");
bool server_has_been_started = false;
try {
server = new Server("socket-helper", true, false);
server.Start();
server_has_been_started = true;
} catch (InvalidOperationException ex) {
Logger.Log.Error(ex, "Couldn't start server. Exiting immediately.");
}
if (server_has_been_started)
{
// Whether we should generate heap-shot snapshots
heap_shot = (Environment.GetEnvironmentVariable("_HEY_LETS_DO_A_HEAP_SHOT") != null);
if (!run_by_hand)
{
// Start the monitor thread, which keeps an eye on memory usage and idle time.
ExceptionHandlingThread.Start(new ThreadStart(MemoryAndIdleMonitorWorker));
// Start a thread that watches the daemon and begins a shutdown
// if it terminates.
ExceptionHandlingThread.Start(new ThreadStart(DaemonMonitorWorker));
}
// Start the main loop
main_loop.Run();
ExceptionHandlingThread.JoinAllThreads();
// If we placed our sockets in a temp directory, try to clean it up
// Note: this may fail because the daemon is still running
if (PathFinder.GetRemoteStorageDir(false) != PathFinder.StorageDir)
{
try {
Directory.Delete(PathFinder.GetRemoteStorageDir(false));
} catch (IOException) { }
}
Log.Always("Index helper process shut down cleanly.");
}
}
public static void DoMain(string[] args)
{
SystemInformation.InternalCallInitializer.Init();
SystemInformation.SetProcessName("beagrepd");
// Process the command-line arguments
bool arg_debug = false;
bool arg_debug_memory = false;
bool arg_fg = false;
int i = 0;
while (i < args.Length)
{
string arg = args [i];
++i;
string next_arg = i < args.Length ? args [i] : null;
switch (arg)
{
case "-h":
case "--help":
PrintUsage();
Environment.Exit(0);
break;
case "--mdb":
case "--mono-debug":
// Silently ignore these arguments: they get handled
// in the wrapper script.
break;
case "--list-backends":
Console.WriteLine("Current available backends:");
Console.Write(QueryDriver.ListBackends());
Environment.Exit(0);
break;
case "--fg":
case "--foreground":
arg_fg = true;
break;
case "--bg":
case "--background":
arg_fg = false;
break;
case "--replace":
arg_replace = true;
break;
case "--debug":
arg_debug = true;
break;
case "--heap-shot":
arg_heap_shot = true;
arg_debug = true;
arg_debug_memory = true;
break;
case "--no-snapshots":
case "--no-snapshot":
arg_heap_shot_snapshots = false;
break;
case "--heap-buddy":
case "--debug-memory":
arg_debug = true;
arg_debug_memory = true;
break;
case "--indexing-test-mode":
arg_indexing_test_mode = true;
arg_fg = true;
break;
case "--backend":
if (next_arg == null)
{
Console.WriteLine("--backend requires a backend name");
Environment.Exit(1);
break;
}
if (next_arg.StartsWith("--"))
{
Console.WriteLine("--backend requires a backend name. Invalid name '{0}'", next_arg);
Environment.Exit(1);
break;
}
if (next_arg [0] != '+' && next_arg [0] != '-')
{
QueryDriver.OnlyAllow(next_arg);
}
else
{
if (next_arg [0] == '+')
{
QueryDriver.Allow(next_arg.Substring(1));
}
else
{
QueryDriver.Deny(next_arg.Substring(1));
}
}
++i; // we used next_arg
break;
case "--add-static-backend":
if (next_arg != null)
{
QueryDriver.AddStaticQueryable(next_arg);
}
++i;
break;
case "--disable-scheduler":
arg_disable_scheduler = true;
break;
case "--indexing-delay":
if (next_arg != null)
{
try {
QueryDriver.IndexingDelay = Int32.Parse(next_arg);
} catch {
Console.WriteLine("'{0}' is not a valid number of seconds", next_arg);
Environment.Exit(1);
}
}
++i;
break;
case "--autostarted":
// FIXME: This option is deprecated and will be removed in a future release.
break;
case "--disable-text-cache":
disable_textcache = true;
break;
case "--version":
VersionFu.PrintVersion();
Environment.Exit(0);
break;
default:
Console.WriteLine("Unknown argument '{0}'", arg);
Environment.Exit(1);
break;
}
}
if (Environment.GetEnvironmentVariable("SABAYON_SESSION_RUNNING") == "yes")
{
Console.WriteLine("Beagrep is running underneath Sabayon, exiting.");
Environment.Exit(0);
}
if (arg_indexing_test_mode)
{
LuceneQueryable.OptimizeRightAway = true;
}
// Bail out if we are trying to run as root
if (Environment.UserName == "root" && Environment.GetEnvironmentVariable("SUDO_USER") != null)
{
Console.WriteLine("You appear to be running beagrep using sudo. This can cause problems with");
Console.WriteLine("permissions in your .beagrep and .wapi directories if you later try to run");
Console.WriteLine("as an unprivileged user. If you need to run beagrep as root, please use");
Console.WriteLine("'su -c' instead.");
Environment.Exit(-1);
}
if (Environment.UserName == "root" && !Conf.Daemon.GetOption(Conf.Names.AllowRoot, false))
{
Console.WriteLine("You can not run beagrep as root. Beagrep is designed to run from your own");
Console.WriteLine("user account. If you want to create multiuser or system-wide indexes, use");
Console.WriteLine("the beagrep-build-index tool.");
Console.WriteLine();
Console.WriteLine("You can override this setting using the beagrep-config or beagrep-settings tools.");
Environment.Exit(-1);
}
try {
string tmp = PathFinder.HomeDir;
} catch (Exception e) {
Console.WriteLine("Unable to start the daemon: {0}", e.Message);
Environment.Exit(-1);
}
MainLoopThread = Thread.CurrentThread;
// FIXME: We always turn on full debugging output! We are still
// debugging this code, after all...
// arg_debug ? LogLevel.Debug : LogLevel.Warn
Log.Initialize(PathFinder.LogDir, "Beagrep", LogLevel.Debug, arg_fg);
Log.Always("Starting Beagrep Daemon (version {0})", ExternalStringsHack.Version);
Log.Always("Running on {0}", SystemInformation.MonoRuntimeVersion);
Log.Always("Command Line: {0}",
Environment.CommandLine != null ? Environment.CommandLine : "(null)");
if (!ExtendedAttribute.Supported)
{
Logger.Log.Warn("Extended attributes are not supported on this filesystem. " +
"Performance will suffer as a result.");
}
if (disable_textcache)
{
Log.Warn("Running with text-cache disabled!");
Log.Warn("*** Snippets will not be returned for documents indexed in this session.");
}
// Check if global configuration files are installed
if (!Conf.CheckGlobalConfig())
{
Console.WriteLine("Global configuration files not found in '{0}'", PathFinder.ConfigDataDir);
Environment.Exit(-1);
}
// Start our memory-logging thread
if (arg_debug_memory)
{
ExceptionHandlingThread.Start(new ThreadStart(LogMemoryUsage));
}
// Do BEAGREP_EXERCISE_THE_DOG_HARDER-related processing.
ExerciseTheDogHarder();
// Initialize GObject type system
g_type_init();
// Lower our CPU priority
SystemPriorities.Renice(7);
QueryDriver.Init();
Server.Init();
#if MONO_1_9
Shutdown.SetupSignalHandlers(new Shutdown.SignalHandler(HandleSignal));
#else
SetupSignalHandlers();
#endif
Shutdown.ShutdownEvent += OnShutdown;
main_loop = new MainLoop();
Shutdown.RegisterMainLoop(main_loop);
// Defer all actual startup until the main loop is
// running. That way shutdowns during the startup
// process work correctly.
GLib.Idle.Add(new GLib.IdleHandler(StartupProcess));
// Start our event loop.
main_loop.Run();
// We're out of the main loop now, join all the
// running threads so we can exit cleanly.
ExceptionHandlingThread.JoinAllThreads();
// If we placed our sockets in a temp directory, try to clean it up
// Note: this may fail because the helper is still running
if (PathFinder.GetRemoteStorageDir(false) != PathFinder.StorageDir)
{
try {
Directory.Delete(PathFinder.GetRemoteStorageDir(false));
} catch (IOException) { }
}
Log.Always("Beagrep daemon process shut down cleanly.");
}
static void DoMain(string [] args)
{
SystemInformation.SetProcessName("beagle-build-index");
if (args.Length < 2)
{
PrintUsage();
}
ArrayList allowed_patterns = new ArrayList();
ArrayList denied_patterns = new ArrayList();
int i = 0;
while (i < args.Length)
{
string arg = args [i];
++i;
string next_arg = i < args.Length ? args [i] : null;
switch (arg)
{
case "-h":
case "--help":
PrintUsage();
break;
case "--tag":
if (next_arg != null)
{
arg_tag = next_arg;
}
++i;
break;
case "-r":
case "--recursive":
arg_recursive = true;
break;
case "--enable-deletion":
arg_delete = true;
break;
case "--disable-directories":
arg_disable_directories = true;
break;
case "--enable-text-cache":
arg_cache_text = true;
break;
case "--target":
if (next_arg != null)
{
arg_output = Path.IsPathRooted(next_arg) ? next_arg : Path.GetFullPath(next_arg);
}
++i;
break;
case "--disable-filtering":
arg_disable_filtering = true;
break;
case "--disable-on-battery":
arg_disable_on_battery = true;
break;
case "--allow-pattern":
if (next_arg == null)
{
break;
}
if (next_arg.IndexOf(',') != -1)
{
foreach (string pattern in next_arg.Split(','))
{
allowed_patterns.Add(pattern);
}
}
else
{
allowed_patterns.Add(next_arg);
}
++i;
break;
case "--deny-pattern":
if (next_arg == null)
{
break;
}
if (next_arg.IndexOf(',') != -1)
{
foreach (string pattern in next_arg.Split(','))
{
denied_patterns.Add(pattern);
}
}
else
{
denied_patterns.Add(next_arg);
}
++i;
break;
case "--disable-restart":
arg_disable_restart = true;
break;
case "--source":
if (next_arg == null)
{
break;
}
arg_source = next_arg;
++i;
break;
case "--removable":
arg_removable = true;
break;
default:
if (arg.StartsWith("-") || arg.StartsWith("--"))
{
PrintUsage();
}
string path = Path.IsPathRooted(arg) ? arg : Path.GetFullPath(arg);
if (path != "/" && path.EndsWith("/"))
{
path = path.TrimEnd('/');
}
if (Directory.Exists(path))
{
pending_directories.Enqueue(new DirectoryInfo(path));
}
else if (File.Exists(path))
{
pending_files.Enqueue(new FileInfo(path));
}
break;
}
}
/////////////////////////////////////////////////////////
if (arg_output == null)
{
Logger.Log.Error("--target must be specified");
Environment.Exit(1);
}
// Set the storage dir, this should be used to store log messages
// and filterver.dat
PathFinder.StorageDir = arg_output;
foreach (FileSystemInfo info in pending_directories)
{
if (Path.GetFullPath(arg_output) == info.FullName)
{
Logger.Log.Error("Target directory cannot be one of the source paths.");
Environment.Exit(1);
}
}
foreach (FileSystemInfo info in pending_files)
{
if (Path.GetFullPath(arg_output) == info.FullName)
{
Logger.Log.Error("Target directory cannot be one of the source paths.");
Environment.Exit(1);
}
}
if (!Directory.Exists(Path.GetDirectoryName(arg_output)))
{
Logger.Log.Error("Index directory not available for construction: {0}", arg_output);
Environment.Exit(1);
}
// Be *EXTRA PARANOID* about the contents of the target
// directory, because creating an indexing driver will
// nuke it.
if (Directory.Exists(arg_output))
{
foreach (FileInfo info in DirectoryWalker.GetFileInfos(arg_output))
{
if (Array.IndexOf(allowed_files, info.Name) == -1)
{
Logger.Log.Error("{0} doesn't look safe to delete: non-Beagle file {1} was found", arg_output, info.FullName);
Environment.Exit(1);
}
}
foreach (DirectoryInfo info in DirectoryWalker.GetDirectoryInfos(arg_output))
{
if (Array.IndexOf(allowed_dirs, info.Name) == -1)
{
Logger.Log.Error("{0} doesn't look safe to delete: non-Beagle directory {1} was found", arg_output, info.FullName);
Environment.Exit(1);
}
}
}
string config_file_path = Path.Combine(arg_output, "StaticIndex.xml");
string prev_source = null;
if (File.Exists(config_file_path))
{
Config static_index_config = Conf.LoadFrom(config_file_path);
if (static_index_config == null)
{
Log.Error("Invalid configuation file {0}", config_file_path);
Environment.Exit(1);
}
prev_source = static_index_config.GetOption("Source", null);
if (arg_source != null && prev_source != arg_source)
{
Log.Error("Source already set to {0} for existing static index. Cannot set source to {1}.", prev_source, arg_source);
Environment.Exit(1);
}
bool prev_removable = static_index_config.GetOption("Removable", false);
if (arg_removable != prev_removable)
{
Log.Error("Index previously created for {0}-removable path",
(prev_removable ? "" : "non"));
Environment.Exit(1);
}
else
{
volume_label = static_index_config.GetOption("VolumeLabel", null);
}
// If arg_source is not given, and prev_source is present, use prev_source
// as the arg_source. This is useful for re-running build-index without
// giving --arg_source for already existing static index
arg_source = prev_source;
}
// Removable media related options
if (arg_removable)
{
if (pending_files.Count > 0)
{
Log.Error("Indexing individual files is not allowed for removable media.");
Environment.Exit(1);
}
else if (pending_directories.Count > 1)
{
Log.Error("Indexing multiple root directories is not allowed for removable media.");
Environment.Exit(1);
}
mnt_dir = ((DirectoryInfo)pending_directories.Peek()).FullName;
if (mnt_dir.Length != 1)
{
mnt_dir = mnt_dir.TrimEnd('/');
}
// compute volume label
// (1) directory name if block.is_volume is false
// (2) hal volume.label if set
// (3) hal volume.uuid if set
Hal.Manager manager = new Hal.Manager(new Hal.Context());
Hal.Device mnt_device = null;
foreach (Hal.Device device in manager.FindDeviceStringMatch("volume.mount_point", mnt_dir))
{
mnt_device = device;
}
string new_volume_label = null;
if (mnt_device != null)
{
new_volume_label = mnt_device.GetPropertyString("volume.label");
if (String.IsNullOrEmpty(new_volume_label))
{
new_volume_label = mnt_device.GetPropertyString("volume.uuid");
}
}
if (new_volume_label == null)
{
new_volume_label = ((DirectoryInfo)pending_directories.Peek()).Name;
}
// Sanity check
// Volume label is part of the URI, so cannot be changed once set
if (volume_label == null)
{
volume_label = new_volume_label;
}
else if (volume_label != new_volume_label)
{
Log.Error("Volume label (earlier '{0}') changed (to '{1}')! You need to create a new index.", volume_label, new_volume_label);
Environment.Exit(1);
}
}
if (arg_source == null)
{
DirectoryInfo dir = new DirectoryInfo(StringFu.SanitizePath(arg_output));
arg_source = dir.Name;
}
if (!BatteryMonitor.UsingAC && arg_disable_on_battery)
{
Log.Always("Indexer is disabled when on battery power (--disable-on-battery)");
Environment.Exit(0);
}
string global_files_config = Path.Combine(PathFinder.ConfigDataDir, "config-files");
global_files_config = Path.Combine(global_files_config, Conf.Names.FilesQueryableConfig + ".xml");
if (!File.Exists(global_files_config))
{
Log.Error("Global configuration file not found {0}", global_files_config);
Environment.Exit(0);
}
// Setup regexes for allowed/denied patterns
if (allowed_patterns.Count > 0)
{
allowed_regex = StringFu.GetPatternRegex(allowed_patterns);
}
else
{
// Read the exclude values from config
// For system-wide indexes, only the global config value will be used
Config config = Conf.Get(Conf.Names.FilesQueryableConfig);
List <string[]> values = config.GetListOptionValues(Conf.Names.ExcludePattern);
if (values != null)
{
foreach (string[] exclude in values)
{
denied_patterns.Add(exclude [0]);
}
}
if (denied_patterns.Count > 0)
{
denied_regex = StringFu.GetPatternRegex(denied_patterns);
}
}
Log.Always("Starting beagle-build-index (pid {0}) at {1}", Process.GetCurrentProcess().Id, DateTime.Now);
// Set system priorities so we don't slow down the system
SystemPriorities.ReduceIoPriority();
SystemPriorities.SetSchedulerPolicyBatch();
SystemPriorities.Renice(19);
driver = new LuceneIndexingDriver(arg_output, MINOR_VERSION, false);
driver.TextCache = (arg_cache_text) ? new TextCache(arg_output) : null;
if (driver.TextCache != null)
{
driver.TextCache.WorldReadable = true;
}
backing_fa_store = new FileAttributesStore_Sqlite(driver.TopDirectory, driver.Fingerprint);
fa_store = new FileAttributesStore(backing_fa_store);
// Set up signal handlers
#if MONO_1_9
Shutdown.SetupSignalHandlers(delegate(int signal)
{
if (signal == (int)Mono.Unix.Native.Signum.SIGINT ||
signal == (int)Mono.Unix.Native.Signum.SIGTERM)
{
Shutdown.BeginShutdown();
}
});
#else
SetupSignalHandlers();
#endif
Thread monitor_thread = null;
Stopwatch watch = new Stopwatch();
watch.Start();
if (!arg_disable_restart)
{
// Start the thread that monitors memory usage.
monitor_thread = ExceptionHandlingThread.Start(new ThreadStart(MemoryMonitorWorker));
}
// Start indexworker to do the crawling and indexing
IndexWorker();
// Join any threads so that we know that we're the only thread still running
if (monitor_thread != null)
{
monitor_thread.Join();
}
watch.Stop();
Logger.Log.Debug("Elapsed time {0}.", watch);
// Write this after indexing is done. This is because, if creating a new index,
// LuceneIndexingDriver.Create() is called which purges the entire directory.
if (prev_source == null)
{
Config static_index_config = Conf.LoadNew("StaticIndex.xml");
// Write StaticIndex.xml containing:
// The name of the source
static_index_config.SetOption("Source", arg_source);
static_index_config ["Source"].Description = "Source of the static index";
if (arg_removable)
{
static_index_config.SetOption("VolumeLabel", volume_label);
static_index_config ["VolumeLabel"].Description = "Volume label of the removable source";
static_index_config.SetOption("Removable", true);
static_index_config ["Removable"].Description = "Removable source";
}
Conf.SaveTo(static_index_config, config_file_path);
}
if (restart)
{
Logger.Log.Debug("Restarting beagle-build-index");
Process p = new Process();
p.StartInfo.UseShellExecute = false;
// FIXME: Maybe this isn't the right way to do things? It should be ok,
// the PATH is inherited from the shell script which runs mono itself.
p.StartInfo.FileName = "mono";
p.StartInfo.Arguments = String.Join(" ", Environment.GetCommandLineArgs());
p.Start();
}
Log.Always("Exiting beagle-build-index (pid {0}) at {1}", Process.GetCurrentProcess().Id, DateTime.Now);
}
