/// <summary> /// Attempts to create an index writer, it will retry on failure 5 times and on the last time will try to forcefully unlock the index files /// </summary> /// <param name="baseLuceneDirectory"></param> /// <param name="analyzer"></param> /// <returns></returns> private Attempt <IndexWriter> TryCreateWriterWithRetry(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer) { var maxTries = 5; var result = DelegateExtensions.RetryUntilSuccessOrMaxAttempts((currentTry) => { //last try... if (currentTry == maxTries) { LogHelper.Info <LocalTempStorageIndexer>("Could not acquire index lock, attempting to force unlock it..."); //unlock it! IndexWriter.Unlock(baseLuceneDirectory); } var writerAttempt = TryCreateWriter(baseLuceneDirectory, analyzer); if (writerAttempt) { return(writerAttempt); } LogHelper.Info <LocalTempStorageIndexer>("Could not create writer on {0}, retrying ....", baseLuceneDirectory.ToString); return(Attempt <IndexWriter> .Fail()); }, 5, TimeSpan.FromSeconds(1)); return(result); }
private static void CheckIndexAndRecover(Lucene.Net.Store.Directory directory, string indexDirectory) { startupLog.Warn("Unclean shutdown detected on {0}, checking the index for errors. This may take a while.", indexDirectory); var memoryStream = new MemoryStream(); var stringWriter = new StreamWriter(memoryStream); var checkIndex = new CheckIndex(directory); if (startupLog.IsWarnEnabled) { checkIndex.SetInfoStream(stringWriter); } var sp = Stopwatch.StartNew(); var status = checkIndex.CheckIndex_Renamed_Method(); sp.Stop(); if (startupLog.IsWarnEnabled) { startupLog.Warn("Checking index {0} took: {1}, clean: {2}", indexDirectory, sp.Elapsed, status.clean); memoryStream.Position = 0; log.Warn(new StreamReader(memoryStream).ReadToEnd()); } if (status.clean) { return; } startupLog.Warn("Attempting to fix index: {0}", indexDirectory); sp.Restart(); checkIndex.FixIndex(status); startupLog.Warn("Fixed index {0} in {1}", indexDirectory, sp.Elapsed); }
private bool IsIndexCorrupted(Lucene.Net.Store.Directory directory) { if (directory is FSDirectory && IndexReader.IndexExists(directory)) { bool unexpectedProcessTermination = IndexReader.IsLocked(directory); if (unexpectedProcessTermination) { try { _log.Error("Try to cleanup index after unexpected Process Termination..."); Directory.Delete(_settings.IndexPath, true); Directory.CreateDirectory(_settings.IndexPath); return(true); } catch (Exception accessEx) { _log.Error("Could not cleanup index after unexpected Process Termination. Try simply unlock now.", accessEx); try { IndexReader.Unlock(directory); } catch (Exception ex) { _log.Error("Could not unlock index after unexpected Process Termination. Giving up now.", ex); } } } } return(false); }
private bool ClearLuceneDirFiles(Lucene.Net.Store.Directory baseLuceneDirectory) { try { //unlock it! IndexWriter.Unlock(baseLuceneDirectory); var fileLuceneDirectory = baseLuceneDirectory as FSDirectory; if (fileLuceneDirectory != null) { foreach (var file in fileLuceneDirectory.ListAll()) { try { fileLuceneDirectory.DeleteFile(file); } catch (IOException) { if (file.InvariantEquals("write.lock")) { LogHelper.Warn <LocalTempStorageIndexer>("The lock file could not be deleted but should be removed when the writer is disposed"); } } } return(true); } return(false); } catch (Exception ex) { LogHelper.Error <LocalTempStorageIndexer>("Could not clear corrupt index from main index folder, the index cannot be used", ex); return(false); } }
private static void WriteIndexVersion(Lucene.Net.Store.Directory directory) { using (var indexOutput = directory.CreateOutput("index.version")) { indexOutput.WriteString(IndexVersion); indexOutput.Flush(); } }
public LuceneIndexModifier(LuceneSettings settings) { this.settings = settings; this.indexBaseDirectory = settings.GetIndexDirectory(); this.Init(); StartIndexerThread(); }
private static void EnsureIndexVersionMatches(string indexName, Lucene.Net.Store.Directory directory) { if (directory.FileExists("index.version") == false) { throw new InvalidOperationException("Could not find index.version " + indexName + ", resetting index"); } using (var indexInput = directory.OpenInput("index.version")) { var versionFromDisk = indexInput.ReadString(); if (versionFromDisk != IndexVersion) { throw new InvalidOperationException("Index " + indexName + " is of version " + versionFromDisk + " which is not compatible with " + IndexVersion + ", resetting index"); } } }
public void Initialize(NameValueCollection config, string configuredPath, Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer) { var codegenPath = HttpRuntime.CodegenDir; _tempPath = Path.Combine(codegenPath, configuredPath.TrimStart('~', '/').Replace("/", "\\")); if (config != null) { if (config["syncStorage"] != null) { var attempt = config["syncStorage"].TryConvertTo <bool>(); if (attempt) { _syncStorage = attempt.Result; } } } InitializeLocalIndexAndDirectory(baseLuceneDirectory, analyzer, configuredPath); }
private Attempt <IndexWriter> TryCreateWriter(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer) { try { var w = new IndexWriter( //read from the underlying/default directory, not the temp codegen dir baseLuceneDirectory, analyzer, Snapshotter, IndexWriter.MaxFieldLength.UNLIMITED); //Done! return(Attempt.Succeed(w)); } catch (Exception ex) { LogHelper.WarnWithException <LocalTempStorageIndexer>("Could not create index writer with snapshot policy for copying... retrying...", ex); return(Attempt <IndexWriter> .Fail(ex)); } }
public void CreateIndexes() { Lucene.Net.Store.Directory dir = FSDirectory.GetDirectory(IndexPath, true); Analyzer analyzer = new StandardAnalyzer(); var indexWriter = new IndexWriter(dir, analyzer, true, new IndexWriter.MaxFieldLength(25000)); foreach (var File in Directory.GetFiles(HelpFilePath, "*.xaml")) { var text = GetTextFromXaml(File).Replace("\r\n", " ").Replace("\n", " "); var doc = new Document(); var fldContent = new Field(ContentField, text, Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.WITH_OFFSETS); var fldName = new Field(PluginField, Path.GetFileNameWithoutExtension(Path.GetFileName(File)), Field.Store.YES, Field.Index.NO, Field.TermVector.NO); doc.Add(fldContent); doc.Add(fldName); indexWriter.AddDocument(doc); } indexWriter.Optimize(); indexWriter.Close(); }
private bool TryWaitForDirectoryUnlock(Lucene.Net.Store.Directory dir) { var maxTries = 5; var result = DelegateExtensions.RetryUntilSuccessOrMaxAttempts((currentTry) => { //last try... if (currentTry == maxTries) { LogHelper.Info <LocalTempStorageIndexer>("Could not acquire directory lock, attempting to force unlock it..."); //unlock it! IndexWriter.Unlock(dir); } if (IndexWriter.IsLocked(dir) == false) { return(Attempt.Succeed(true)); } LogHelper.Info <LocalTempStorageIndexer>("Could not acquire directory lock for {0} writer, retrying ....", dir.ToString); return(Attempt <bool> .Fail()); }, 5, TimeSpan.FromSeconds(1)); return(result); }
private Index CreateIndexImplementation(string directoryPath, IndexDefinition indexDefinition, Lucene.Net.Store.Directory directory) { var viewGenerator = indexDefinitionStorage.GetViewGenerator(indexDefinition.Name); var indexImplementation = indexDefinition.IsMapReduce ? (Index) new MapReduceIndex(directory, directoryPath, indexDefinition, viewGenerator, documentDatabase.WorkContext) : new SimpleIndex(directory, directoryPath, indexDefinition, viewGenerator, documentDatabase.WorkContext); configuration.Container.SatisfyImportsOnce(indexImplementation); return(indexImplementation); }
private void InitializeLocalIndexAndDirectory(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer, string configuredPath) { lock (Locker) { if (!Directory.Exists(_tempPath)) { Directory.CreateDirectory(_tempPath); } else { //if we are syncing storage to the main file system to temp files, then clear out whatever is //currently there since we'll re-copy it over if (_syncStorage) { //clear it! Directory.Delete(_tempPath, true); //recreate it Directory.CreateDirectory(_tempPath); } } //if we are syncing storage to the main file system to temp files, then sync from the main FS to our temp FS if (_syncStorage) { //copy index using (new IndexWriter( //read from the underlying/default directory, not the temp codegen dir baseLuceneDirectory, analyzer, Snapshotter, IndexWriter.MaxFieldLength.UNLIMITED)) { try { var basePath = IOHelper.MapPath(configuredPath); var commit = Snapshotter.Snapshot(); var fileNames = commit.GetFileNames(); foreach (var fileName in fileNames) { File.Copy( Path.Combine(basePath, "Index", fileName), Path.Combine(_tempPath, Path.GetFileName(fileName)), true); } var segments = commit.GetSegmentsFileName(); if (segments.IsNullOrWhiteSpace() == false) { File.Copy( Path.Combine(basePath, "Index", segments), Path.Combine(_tempPath, Path.GetFileName(segments)), true); } } finally { Snapshotter.Release(); } } //create the custom lucene directory which will keep the main and temp FS's in sync LuceneDirectory = new TempStorageDirectory( new DirectoryInfo(_tempPath), baseLuceneDirectory); } else { //just return a normal lucene directory that uses the codegen folder LuceneDirectory = FSDirectory.Open(new DirectoryInfo(_tempPath)); } } }
private static Index CreateIndexImplementation(string name, IndexDefinition indexDefinition, Lucene.Net.Store.Directory directory) { return(indexDefinition.IsMapReduce ? (Index) new MapReduceIndex(directory, name, indexDefinition) : new SimpleIndex(directory, name, indexDefinition)); }
private InitializeDirectoryFlags InitializeLocalIndexAndDirectory(Lucene.Net.Store.Directory baseLuceneDirectory, Analyzer analyzer, string configuredPath) { lock (_locker) { if (Directory.Exists(TempPath) == false) { Directory.CreateDirectory(TempPath); } //copy index if it exists, don't do anything if it's not there if (IndexReader.IndexExists(baseLuceneDirectory) == false) { return(InitializeDirectoryFlags.SuccessNoIndexExists); } var writerAttempt = TryCreateWriterWithRetry(baseLuceneDirectory, analyzer); if (writerAttempt.Success == false) { LogHelper.Error <LocalTempStorageIndexer>("Could not create index writer with snapshot policy for copying, the index cannot be used", writerAttempt.Exception); return(InitializeDirectoryFlags.FailedLocked); } //Try to open the reader from the source, this will fail if the index is corrupt and we'll need to handle that try { //NOTE: To date I've not seen this error occur using (writerAttempt.Result.GetReader()) { } } catch (Exception ex) { writerAttempt.Result.Dispose(); LogHelper.Error <LocalTempStorageIndexer>( string.Format("Could not open an index reader, {0} is empty or corrupt... attempting to clear index files in master folder", configuredPath), ex); if (ClearLuceneDirFiles(baseLuceneDirectory) == false) { //hrm, not much we can do in this situation, but this shouldn't happen LogHelper.Error <LocalTempStorageIndexer>("Could not open an index reader, index is corrupt.", ex); return(InitializeDirectoryFlags.FailedCorrupt); } //the main index is now blank, we'll proceed as normal with a new empty index... writerAttempt = TryCreateWriter(baseLuceneDirectory, analyzer); if (writerAttempt.Success == false) { //ultra fail... LogHelper.Error <LocalTempStorageIndexer>("Could not create index writer with snapshot policy for copying, the index cannot be used", writerAttempt.Exception); return(InitializeDirectoryFlags.FailedLocked); } } using (writerAttempt.Result) { try { var basePath = IOHelper.MapPath(configuredPath); var commit = Snapshotter.Snapshot(); var allSnapshotFiles = commit.GetFileNames() .Concat(new[] { commit.GetSegmentsFileName(), //we need to manually include the segments.gen file "segments.gen" }) .Distinct() .ToArray(); var tempDir = new DirectoryInfo(TempPath); //Get all files in the temp storage that don't exist in the snapshot collection, we want to remove these var toRemove = tempDir.GetFiles() .Select(x => x.Name) .Except(allSnapshotFiles); using (var tempDirectory = new SimpleFSDirectory(tempDir)) { if (TryWaitForDirectoryUnlock(tempDirectory)) { foreach (var file in toRemove) { try { File.Delete(Path.Combine(TempPath, file)); } catch (IOException ex) { if (file.InvariantEquals("write.lock")) { //This might happen if the writer is open LogHelper.Warn <LocalTempStorageIndexer>("The lock file could not be deleted but should be removed when the writer is disposed"); } LogHelper.Debug <LocalTempStorageIndexer>("Could not delete non synced index file file, index sync will continue but old index files will remain - this shouldn't affect indexing/searching operations. {0}", () => ex.ToString()); } } } else { //quit here, this shouldn't happen with all the checks above. LogHelper.Warn <LocalTempStorageIndexer>("Cannot sync index files from main storage, the temp file index is currently locked"); return(InitializeDirectoryFlags.FailedLocked); } foreach (var fileName in allSnapshotFiles.Where(f => f.IsNullOrWhiteSpace() == false)) { var destination = Path.Combine(TempPath, Path.GetFileName(fileName)); //don't copy if it's already there, lucene is 'write once' so this file is meant to be there already if (File.Exists(destination)) { continue; } try { File.Copy( Path.Combine(basePath, "Index", fileName), destination); } catch (IOException ex) { LogHelper.Error <LocalTempStorageIndexer>("Could not copy index file, could not sync from main storage", ex); //quit here return(InitializeDirectoryFlags.FailedFileSync); } } } } finally { Snapshotter.Release(); } } LogHelper.Info <LocalTempStorageIndexer>("Successfully sync'd main index to local temp storage for index: {0}", () => configuredPath); return(InitializeDirectoryFlags.Success); } }
public Indexer(string indexDir) { IndexDirectory = FSDirectory.Open(indexDir); Analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30); }