private void StoreOnDisk()
{
if (stream != null)
{
throw new Exception("Already writing to a file on disk.");
}
//Log.Debug ("Large cached text, storing in file {0}", path);
FileStream file_stream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.ReadWrite);
// We don't expect to need this again in the near future.
FileAdvise.FlushCache(file_stream);
if (!world_readable)
{
// Make files only readable by the owner.
Mono.Unix.Native.Syscall.chmod(
path,
Mono.Unix.Native.FilePermissions.S_IRUSR |
Mono.Unix.Native.FilePermissions.S_IWUSR);
}
stream = file_stream;
stream.Write(buffer, 0, buffer_pos);
buffer_pos = 0;
buffer = null;
}
public bool Open(FileSystemInfo info)
{
isFinished = false;
textPool = new ArrayList();
hotPool = new ArrayList();
text_builder.Length = 0;
currentInfo = info;
if (info is FileInfo)
{
// Open a stream for this file.
currentStream = new FileStream(info.FullName,
FileMode.Open,
FileAccess.Read,
FileShare.Read);
if (preload)
{
// Our default assumption is sequential reads.
// FIXME: Is this the right thing to do here?
FileAdvise.IncreaseReadAhead((FileStream)currentStream);
// Give the OS a hint that we will be reading this
// file soon.
FileAdvise.PreLoad((FileStream)currentStream);
}
}
try {
DoOpen(info);
if (IsFinished)
{
return(true);
}
else if (HasError)
{
return(false);
}
return(Open(currentStream, false));
} catch (Exception e) {
Log.Warn(e, "Unable to filter {0}:", info.FullName);
Cleanup(); // clean up temporary files on an exception
return(false);
}
}
public bool Open(TextReader reader)
{
tempFile = Path.GetTempFileName();
FileStream file_stream = File.OpenWrite(tempFile);
if (Debug)
{
Logger.Log.Debug("Storing text in tempFile {0}", tempFile);
}
// When we dump the contents of a reader into a file, we
// expect to use it again soon.
FileAdvise.PreLoad(file_stream);
// Make sure the temporary file is only readable by the owner.
// FIXME: There is probably a race here. Could some malicious program
// do something to the file between creation and the chmod?
Mono.Unix.Native.Syscall.chmod(tempFile, (Mono.Unix.Native.FilePermissions) 256);
BufferedStream buffered_stream = new BufferedStream(file_stream);
StreamWriter writer = new StreamWriter(buffered_stream);
const int BUFFER_SIZE = 8192;
char [] buffer = new char [BUFFER_SIZE];
int read;
do
{
read = reader.Read(buffer, 0, BUFFER_SIZE);
if (read > 0)
{
writer.Write(buffer, 0, read);
}
} while (read > 0);
writer.Close();
return(Open(new FileInfo(tempFile)));
}
//////////////////////////
private static Uri TextReaderToTempFileUri(TextReader reader)
{
if (reader == null)
{
return(null);
}
string filename = Path.GetTempFileName();
FileStream fileStream = File.OpenWrite(filename);
// When we dump the contents of an indexable into a file, we
// expect to use it again soon.
FileAdvise.PreLoad(fileStream);
// Make sure the temporary file is only readable by the owner.
// FIXME: There is probably a race here. Could some malicious program
// do something to the file between creation and the chmod?
Mono.Unix.Native.Syscall.chmod(filename, Mono.Unix.Native.FilePermissions.S_IRUSR);
BufferedStream bufferedStream = new BufferedStream(fileStream);
StreamWriter writer = new StreamWriter(bufferedStream);
char [] buffer;
buffer = GetCharBuffer();
int read;
do
{
read = reader.Read(buffer, 0, buffer.Length);
if (read > 0)
{
writer.Write(buffer, 0, read);
}
} while (read > 0);
writer.Close();
return(UriFu.PathToFileUri(filename));
}
private void Close()
{
if (closed)
{
return;
}
Cleanup();
DoClose();
if (currentReader != null)
{
currentReader.Close();
}
if (currentStream != null)
{
// When crawling, give the OS a hint that we don't
// need to keep this file around in the page cache.
if (indexable.FlushBufferCache && currentStream is FileStream)
{
FileAdvise.FlushCache((FileStream)currentStream);
}
currentStream.Close();
currentStream = null;
}
if (snippetWriter != null)
{
snippetWriter.Close();
}
closed = true;
}
public static bool StartupProcess()
{
// Profile our initialization
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
// Fire up our server
if (!StartServer())
{
if (!arg_replace)
{
Logger.Log.Error("Could not set up the listener for beagrep requests. "
+ "There is probably another beagrepd instance running. "
+ "Use --replace to replace the running service");
Environment.Exit(1);
}
ReplaceExisting();
}
// Set up out-of-process indexing
LuceneQueryable.IndexerHook = new LuceneQueryable.IndexerCreator(RemoteIndexer.NewRemoteIndexer);
Config config = Conf.Get(Conf.Names.DaemonConfig);
// Initialize synchronization to keep the indexes local if PathFinder.StorageDir
// is on a non-block device, or if BEAGREP_SYNCHRONIZE_LOCALLY is set
if ((!SystemInformation.IsPathOnBlockDevice(PathFinder.StorageDir) &&
config.GetOption(Conf.Names.IndexSynchronization, true)) ||
Environment.GetEnvironmentVariable("BEAGREP_SYNCHRONIZE_LOCALLY") != null)
{
IndexSynchronization.Initialize();
}
// Start the query driver.
Logger.Log.Debug("Starting QueryDriver");
QueryDriver.Start();
// Start our battery monitor so we can shut down the
// scheduler if needed.
BatteryMonitor.Init();
bool initially_on_battery = !BatteryMonitor.UsingAC && !config.GetOption(Conf.Names.IndexOnBattery, false);
// Start the Global Scheduler thread
if (!arg_disable_scheduler)
{
if (!initially_on_battery)
{
Logger.Log.Debug("Starting Scheduler thread");
Scheduler.Global.Start();
}
else
{
Log.Debug("Beagrep started on battery, not starting scheduler thread");
}
}
// Start our Inotify threads
Inotify.Start();
// Test if the FileAdvise stuff is working: This will print a
// warning if not. The actual advice calls will fail silently.
FileAdvise.TestAdvise();
#if ENABLE_AVAHI
zeroconf = new Beagrep.Daemon.Network.Zeroconf();
#endif
Conf.WatchForUpdates();
stopwatch.Stop();
Logger.Log.Debug("Daemon initialization finished after {0}", stopwatch);
SystemInformation.LogMemoryUsage();
if (arg_indexing_test_mode)
{
Thread.Sleep(1000); // Ugly paranoia: wait a second for the backends to settle.
Logger.Log.Debug("Running in indexing test mode");
Scheduler.Global.EmptyQueueEvent += OnEmptySchedulerQueue;
Scheduler.Global.Add(null); // pulse the scheduler
}
return(false);
}
private string StoreStreamInTempFile(Stream stream, string extension, DateTime mtime)
{
if (stream == null)
{
return(null);
}
string filename = FileSystem.GetTempFileName(extension);
FileStream file_stream = new FileStream(filename, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite); // FileShare.ReadWrite needed for setting the mtime
// When we dump the contents of an indexable into a file, we
// expect to use it again soon.
FileAdvise.PreLoad(file_stream);
//Log.Debug ("Storing archive contents in {0}", filename);
File.SetLastWriteTimeUtc(filename, mtime); // change this before making read-only
Mono.Unix.Native.Syscall.chmod(filename, Mono.Unix.Native.FilePermissions.S_IRUSR);
BufferedStream buffered_stream = new BufferedStream(file_stream);
byte [] buffer = new byte [8192];
long prev_pos = -1;
int read, total_bytes_read = 0;
bool skip_file = false;
int stuck_count = 0;
do
{
try {
read = stream.Read(buffer, 0, buffer.Length);
} catch (Exception e) {
Log.Error(e, "Caught exception extracting data from archive {0}", Indexable.DisplayUri.ToString());
skip_file = true;
break;
}
total_bytes_read += read;
// Don't extract big files, to avoid filling up /tmp
if (total_bytes_read > MAX_SINGLE_FILE)
{
Log.Debug("10 meg threshold hit, skipping over {0}", Indexable.DisplayUri.ToString());
skip_file = true;
break;
}
if (read > 0)
{
buffered_stream.Write(buffer, 0, read);
}
// Lame workaround for some gzip files which loop
// forever with SharpZipLib. We have to check for
// the parser getting stuck on a certain stream
// position.
if (stream is GZipInputStream && read == buffer.Length)
{
if (stream.Position == prev_pos)
{
stuck_count++;
// 20 is a fairly arbitrary value
if (stuck_count == 20)
{
Log.Debug("{0} appears to be broken, skipping", Indexable.DisplayUri.ToString());
skip_file = true;
break;
}
}
else
{
stuck_count = 0;
prev_pos = stream.Position;
}
}
} while (read > 0);
buffered_stream.Close();
if (skip_file)
{
FileSystem.PosixDelete(filename);
return(null);
}
return(filename);
}
