private void StoreOnDisk() { if (stream != null) { throw new Exception("Already writing to a file on disk."); } //Log.Debug ("Large cached text, storing in file {0}", path); FileStream file_stream = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.ReadWrite); // We don't expect to need this again in the near future. FileAdvise.FlushCache(file_stream); if (!world_readable) { // Make files only readable by the owner. Mono.Unix.Native.Syscall.chmod( path, Mono.Unix.Native.FilePermissions.S_IRUSR | Mono.Unix.Native.FilePermissions.S_IWUSR); } stream = file_stream; stream.Write(buffer, 0, buffer_pos); buffer_pos = 0; buffer = null; }
public bool Open(FileSystemInfo info) { isFinished = false; textPool = new ArrayList(); hotPool = new ArrayList(); text_builder.Length = 0; currentInfo = info; if (info is FileInfo) { // Open a stream for this file. currentStream = new FileStream(info.FullName, FileMode.Open, FileAccess.Read, FileShare.Read); if (preload) { // Our default assumption is sequential reads. // FIXME: Is this the right thing to do here? FileAdvise.IncreaseReadAhead((FileStream)currentStream); // Give the OS a hint that we will be reading this // file soon. FileAdvise.PreLoad((FileStream)currentStream); } } try { DoOpen(info); if (IsFinished) { return(true); } else if (HasError) { return(false); } return(Open(currentStream, false)); } catch (Exception e) { Log.Warn(e, "Unable to filter {0}:", info.FullName); Cleanup(); // clean up temporary files on an exception return(false); } }
public bool Open(TextReader reader) { tempFile = Path.GetTempFileName(); FileStream file_stream = File.OpenWrite(tempFile); if (Debug) { Logger.Log.Debug("Storing text in tempFile {0}", tempFile); } // When we dump the contents of a reader into a file, we // expect to use it again soon. FileAdvise.PreLoad(file_stream); // Make sure the temporary file is only readable by the owner. // FIXME: There is probably a race here. Could some malicious program // do something to the file between creation and the chmod? Mono.Unix.Native.Syscall.chmod(tempFile, (Mono.Unix.Native.FilePermissions) 256); BufferedStream buffered_stream = new BufferedStream(file_stream); StreamWriter writer = new StreamWriter(buffered_stream); const int BUFFER_SIZE = 8192; char [] buffer = new char [BUFFER_SIZE]; int read; do { read = reader.Read(buffer, 0, BUFFER_SIZE); if (read > 0) { writer.Write(buffer, 0, read); } } while (read > 0); writer.Close(); return(Open(new FileInfo(tempFile))); }
////////////////////////// private static Uri TextReaderToTempFileUri(TextReader reader) { if (reader == null) { return(null); } string filename = Path.GetTempFileName(); FileStream fileStream = File.OpenWrite(filename); // When we dump the contents of an indexable into a file, we // expect to use it again soon. FileAdvise.PreLoad(fileStream); // Make sure the temporary file is only readable by the owner. // FIXME: There is probably a race here. Could some malicious program // do something to the file between creation and the chmod? Mono.Unix.Native.Syscall.chmod(filename, Mono.Unix.Native.FilePermissions.S_IRUSR); BufferedStream bufferedStream = new BufferedStream(fileStream); StreamWriter writer = new StreamWriter(bufferedStream); char [] buffer; buffer = GetCharBuffer(); int read; do { read = reader.Read(buffer, 0, buffer.Length); if (read > 0) { writer.Write(buffer, 0, read); } } while (read > 0); writer.Close(); return(UriFu.PathToFileUri(filename)); }
private void Close() { if (closed) { return; } Cleanup(); DoClose(); if (currentReader != null) { currentReader.Close(); } if (currentStream != null) { // When crawling, give the OS a hint that we don't // need to keep this file around in the page cache. if (indexable.FlushBufferCache && currentStream is FileStream) { FileAdvise.FlushCache((FileStream)currentStream); } currentStream.Close(); currentStream = null; } if (snippetWriter != null) { snippetWriter.Close(); } closed = true; }
public static bool StartupProcess() { // Profile our initialization Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); // Fire up our server if (!StartServer()) { if (!arg_replace) { Logger.Log.Error("Could not set up the listener for beagrep requests. " + "There is probably another beagrepd instance running. " + "Use --replace to replace the running service"); Environment.Exit(1); } ReplaceExisting(); } // Set up out-of-process indexing LuceneQueryable.IndexerHook = new LuceneQueryable.IndexerCreator(RemoteIndexer.NewRemoteIndexer); Config config = Conf.Get(Conf.Names.DaemonConfig); // Initialize synchronization to keep the indexes local if PathFinder.StorageDir // is on a non-block device, or if BEAGREP_SYNCHRONIZE_LOCALLY is set if ((!SystemInformation.IsPathOnBlockDevice(PathFinder.StorageDir) && config.GetOption(Conf.Names.IndexSynchronization, true)) || Environment.GetEnvironmentVariable("BEAGREP_SYNCHRONIZE_LOCALLY") != null) { IndexSynchronization.Initialize(); } // Start the query driver. Logger.Log.Debug("Starting QueryDriver"); QueryDriver.Start(); // Start our battery monitor so we can shut down the // scheduler if needed. BatteryMonitor.Init(); bool initially_on_battery = !BatteryMonitor.UsingAC && !config.GetOption(Conf.Names.IndexOnBattery, false); // Start the Global Scheduler thread if (!arg_disable_scheduler) { if (!initially_on_battery) { Logger.Log.Debug("Starting Scheduler thread"); Scheduler.Global.Start(); } else { Log.Debug("Beagrep started on battery, not starting scheduler thread"); } } // Start our Inotify threads Inotify.Start(); // Test if the FileAdvise stuff is working: This will print a // warning if not. The actual advice calls will fail silently. FileAdvise.TestAdvise(); #if ENABLE_AVAHI zeroconf = new Beagrep.Daemon.Network.Zeroconf(); #endif Conf.WatchForUpdates(); stopwatch.Stop(); Logger.Log.Debug("Daemon initialization finished after {0}", stopwatch); SystemInformation.LogMemoryUsage(); if (arg_indexing_test_mode) { Thread.Sleep(1000); // Ugly paranoia: wait a second for the backends to settle. Logger.Log.Debug("Running in indexing test mode"); Scheduler.Global.EmptyQueueEvent += OnEmptySchedulerQueue; Scheduler.Global.Add(null); // pulse the scheduler } return(false); }
private string StoreStreamInTempFile(Stream stream, string extension, DateTime mtime) { if (stream == null) { return(null); } string filename = FileSystem.GetTempFileName(extension); FileStream file_stream = new FileStream(filename, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite); // FileShare.ReadWrite needed for setting the mtime // When we dump the contents of an indexable into a file, we // expect to use it again soon. FileAdvise.PreLoad(file_stream); //Log.Debug ("Storing archive contents in {0}", filename); File.SetLastWriteTimeUtc(filename, mtime); // change this before making read-only Mono.Unix.Native.Syscall.chmod(filename, Mono.Unix.Native.FilePermissions.S_IRUSR); BufferedStream buffered_stream = new BufferedStream(file_stream); byte [] buffer = new byte [8192]; long prev_pos = -1; int read, total_bytes_read = 0; bool skip_file = false; int stuck_count = 0; do { try { read = stream.Read(buffer, 0, buffer.Length); } catch (Exception e) { Log.Error(e, "Caught exception extracting data from archive {0}", Indexable.DisplayUri.ToString()); skip_file = true; break; } total_bytes_read += read; // Don't extract big files, to avoid filling up /tmp if (total_bytes_read > MAX_SINGLE_FILE) { Log.Debug("10 meg threshold hit, skipping over {0}", Indexable.DisplayUri.ToString()); skip_file = true; break; } if (read > 0) { buffered_stream.Write(buffer, 0, read); } // Lame workaround for some gzip files which loop // forever with SharpZipLib. We have to check for // the parser getting stuck on a certain stream // position. if (stream is GZipInputStream && read == buffer.Length) { if (stream.Position == prev_pos) { stuck_count++; // 20 is a fairly arbitrary value if (stuck_count == 20) { Log.Debug("{0} appears to be broken, skipping", Indexable.DisplayUri.ToString()); skip_file = true; break; } } else { stuck_count = 0; prev_pos = stream.Position; } } } while (read > 0); buffered_stream.Close(); if (skip_file) { FileSystem.PosixDelete(filename); return(null); } return(filename); }