public MockSingleIntIndexInput(Directory dir, string fileName, IOContext context) { @in = dir.OpenInput(fileName, context); CodecUtil.CheckHeader(@in, MockSingleIntIndexOutput.CODEC, MockSingleIntIndexOutput.VERSION_START, MockSingleIntIndexOutput.VERSION_START); }
public void TestSetup() { UmbracoExamineSearcher.DisableInitializationCheck = true; BaseUmbracoIndexer.DisableInitializationCheck = true; //we'll copy over the pdf files first var svc = new TestDataService(); var path = svc.MapPath("/App_Data/Converting_file_to_PDF.pdf"); var f = new FileInfo(path); var dir = f.Directory; //ensure the folder is there System.IO.Directory.CreateDirectory(dir.FullName); var pdfs = new[] { TestFiles.Converting_file_to_PDF, TestFiles.PDFStandards, TestFiles.SurviorFlipCup, TestFiles.windows_vista }; var names = new[] { "Converting_file_to_PDF.pdf", "PDFStandards.pdf", "SurviorFlipCup.pdf", "windows_vista.pdf" }; for (int index = 0; index < pdfs.Length; index++) { var p = pdfs[index]; using (var writer = File.Create(Path.Combine(dir.FullName, names[index]))) { writer.Write(p, 0, p.Length); } } _luceneDir = new RAMDirectory(); _indexer = IndexInitializer.GetPdfIndexer(_luceneDir); _indexer.RebuildIndex(); _searcher = IndexInitializer.GetUmbracoSearcher(_luceneDir); }
private void CheckSplitting(Directory dir, Term splitTerm, int leftCount, int rightCount) { using (Directory dir1 = NewDirectory()) { using (Directory dir2 = NewDirectory()) { PKIndexSplitter splitter = new PKIndexSplitter(dir, dir1, dir2, splitTerm, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())), NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); splitter.Split(); using (IndexReader ir1 = DirectoryReader.Open(dir1)) { using (IndexReader ir2 = DirectoryReader.Open(dir2)) { assertEquals(leftCount, ir1.NumDocs); assertEquals(rightCount, ir2.NumDocs); CheckContents(ir1, "1"); CheckContents(ir2, "2"); } } } } }
public int CloseStaleReaders(Directory dir, TimeSpan ts) { lock (_locker) { var now = DateTime.Now; var readersForDir = _oldReaders.Where(x => x.Item1.Directory().GetLockID() == dir.GetLockID()).ToList(); var newest = readersForDir.OrderByDescending(x => x.Item2).FirstOrDefault(); readersForDir.Remove(newest); var stale = readersForDir.Where(x => now - x.Item2 >= ts).ToArray(); foreach (var reader in stale) { //close reader and remove from list try { reader.Item1.Close(); } catch (AlreadyClosedException) { //if this happens, more than one instance has decreased referenced, this could occur if this //somehow gets called in conjuction with the shutdown code or manually, etc... } _oldReaders.Remove(reader); } return stale.Length; } }
public override void SetUp() { base.SetUp(); dir = NewDirectory(); RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, true), Similarity, TimeZone); for (int i = 900; i < 1112; i++) { Document doc = new Document(); string num = Regex.Replace(Regex.Replace(English.IntToEnglish(i), "[-]", " "), "[,]", ""); doc.Add(NewTextField("numbers", num, Field.Store.NO)); writer.AddDocument(doc); } { Document doc = new Document(); doc.Add(NewTextField("numbers", "thou hast sand betwixt thy toes", Field.Store.NO)); writer.AddDocument(doc); } { Document doc = new Document(); doc.Add(NewTextField("numbers", "hundredeight eightyeight yeight", Field.Store.NO)); writer.AddDocument(doc); } { Document doc = new Document(); doc.Add(NewTextField("numbers", "tres y cinco", Field.Store.NO)); writer.AddDocument(doc); } writer.Commit(); writer.Dispose(); }
public override void SetUp() { base.SetUp(); dir = NewDirectory(); var iw = new RandomIndexWriter(Random(), dir, Similarity, TimeZone); int numDocs = TestUtil.NextInt(Random(), 2049, 4000); for (int i = 0; i < numDocs; i++) { var document = new Document { NewTextField("english", English.IntToEnglish(i), Field.Store.NO), NewTextField("oddeven", (i%2 == 0) ? "even" : "odd", Field.Store.NO ), NewStringField("byte", string.Empty + (unchecked((byte) Random().Next ())), Field.Store.NO), NewStringField("short", string.Empty + ((short) Random().Next()), Field.Store .NO), new IntField("int", Random().Next(), Field.Store.NO), new LongField("long", Random().NextLong(), Field.Store.NO), new FloatField("float", Random().NextFloat(), Field.Store.NO), new DoubleField("double", Random().NextDouble(), Field.Store.NO), new NumericDocValuesField("intdocvalues", Random().Next()), new FloatDocValuesField("floatdocvalues", Random().NextFloat()) }; iw.AddDocument(document); } reader = iw.Reader; iw.Dispose(); searcher = NewSearcher(reader); }
public override void SetUp() { base.SetUp(); _dir = NewDirectory(); _indexWriter = new RandomIndexWriter(Random(), _dir, new MockAnalyzer(Random()), Similarity, TimeZone); FieldType ft = new FieldType(TextField.TYPE_STORED); ft.StoreTermVectors = true; ft.StoreTermVectorOffsets = true; ft.StoreTermVectorPositions = true; Analyzer analyzer = new MockAnalyzer(Random()); Document doc; for (int i = 0; i < 100; i++) { doc = new Document(); doc.Add(new Field(_idFieldName, Random().toString(), ft)); doc.Add(new Field(_textFieldName, new StringBuilder(Random().toString()).append(Random().toString()).append( Random().toString()).toString(), ft)); doc.Add(new Field(_classFieldName, Random().toString(), ft)); _indexWriter.AddDocument(doc, analyzer); } _indexWriter.Commit(); _originalIndex = SlowCompositeReaderWrapper.Wrap(_indexWriter.Reader); }
public MapReduceIndex(Directory directory, int id, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator, WorkContext context) : base(directory, id, indexDefinition, viewGenerator, context) { jsonSerializer = JsonExtensions.CreateDefaultJsonSerializer(); jsonSerializer.Converters = MapReduceConverters; }
public NAppIndexUpdater(Configuration.ConfigManager config, Database currentDb) { indexFullPath = config.GetSetting(SettingKeys.Index_Directory, System.IO.Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Index")); indexFullPath = System.IO.Path.GetFullPath(indexFullPath); directory = FSDirectory.Open(new System.IO.DirectoryInfo(indexFullPath)); this.currentDb = currentDb; }
public override void SetUp() { base.SetUp(); dir = NewDirectory(); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer (Random())); iwc.SetMergePolicy(NewLogMergePolicy()); var iw = new RandomIndexWriter(Random(), dir, iwc); var doc = new Document { NewStringField("id", "1", Field.Store.YES), NewTextField("body", "some contents and more contents", Field.Store.NO), new NumericDocValuesField("popularity", 5) }; iw.AddDocument(doc); doc = new Document { NewStringField("id", "2", Field.Store.YES), NewTextField("body", "another document with different contents", Field.Store .NO), new NumericDocValuesField("popularity", 20) }; iw.AddDocument(doc); doc = new Document { NewStringField("id", "3", Field.Store.YES), NewTextField("body", "crappy contents", Field.Store.NO), new NumericDocValuesField("popularity", 2) }; iw.AddDocument(doc); iw.ForceMerge(1); reader = iw.Reader; iw.Dispose(); }
public static void Finish() { s.IndexReader.Dispose(); s = null; Dir.Dispose(); Dir = null; }
public MultiIndexLockFactory(Lucene.Net.Store.Directory master, Lucene.Net.Store.Directory child) { if (master == null) throw new ArgumentNullException("master"); if (child == null) throw new ArgumentNullException("child"); _master = master; _child = child; }
public override void SetUp() { base.SetUp(); store = NewDirectory(); IndexWriter writer = new IndexWriter(store, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false))); Document doc; doc = new Document(); doc.Add(NewTextField("aaa", "foo", Field.Store.YES)); writer.AddDocument(doc); doc = new Document(); doc.Add(NewTextField("aaa", "foo", Field.Store.YES)); writer.AddDocument(doc); doc = new Document(); doc.Add(NewTextField("contents", "Tom", Field.Store.YES)); writer.AddDocument(doc); doc = new Document(); doc.Add(NewTextField("contents", "Jerry", Field.Store.YES)); writer.AddDocument(doc); doc = new Document(); doc.Add(NewTextField("zzz", "bar", Field.Store.YES)); writer.AddDocument(doc); writer.ForceMerge(1); writer.Dispose(); }
public AuditLog(int DaysToKeepLog, int DaysToKeepRecords) { string dir = GetDataDirectory(); _logDays = DaysToKeepLog; _recordDays = DaysToKeepRecords; Task.Run(() => { while (IsFileLocked(new FileInfo($"{dir}/write.lock"))) { Thread.Sleep(500); } if (File.Exists($"{dir}/write.lock")) File.Delete($"{dir}/write.lock"); _dir = FSDirectory.Open(new DirectoryInfo(GetDataDirectory())); var reader = GetSearcher(); var terms = reader.IndexReader.Terms(); while (terms.Next()) { if (terms.Term.Field == "content") trie[terms.Term.Text] = terms.Term.Text; if (terms.Term.Field == "user") users.Add(terms.Term.Text); } ValidateBackup(); }); }
public void TestSetup() { _luceneDir = new RAMDirectory(); _indexer = IndexInitializer.GetUmbracoIndexer(_luceneDir); _indexer.RebuildIndex(); _searcher = IndexInitializer.GetUmbracoSearcher(_luceneDir); }
public void Dispose() { if (!_disposed) { if (_index != null) { _index.Dispose(); _index = null; } if (_analyzer != null) { _analyzer.Dispose(); _analyzer = null; } if (_directory != null) { _directory.Dispose(); _directory = null; } _disposed = true; } }
public MockRAMDirectory(Directory dir) : base(dir) { if (openFiles == null) { openFiles = new System.Collections.Hashtable(); } }
public void SearchFiltered(IndexWriter writer, Directory directory, Filter filter, bool optimize) { try { for (int i = 0; i < 60; i++) {//Simple docs Document doc = new Document(); doc.Add(new Field(FIELD, i.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED)); writer.AddDocument(doc); } if (optimize) writer.Optimize(); writer.Close(); BooleanQuery booleanQuery = new BooleanQuery(); booleanQuery.Add(new TermQuery(new Term(FIELD, "36")), Occur.SHOULD); IndexSearcher indexSearcher = new IndexSearcher(directory); ScoreDoc[] hits = indexSearcher.Search(booleanQuery, filter, 1000).ScoreDocs; Assert.AreEqual(1, hits.Length, "Number of matched documents"); } catch (System.IO.IOException e) { Assert.Fail(e.Message); } }
private void CreateRandomIndexes(int maxSegments) { dir = NewDirectory(); numDocs = AtLeast(150); int numTerms = TestUtil.NextInt(Random(), 1, numDocs / 5); ISet<string> randomTerms = new HashSet<string>(); while (randomTerms.size() < numTerms) { randomTerms.add(TestUtil.RandomSimpleString(Random())); } terms = new List<string>(randomTerms); int seed = Random().Next(); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(new Random(seed))); iwc.SetMergePolicy(TestSortingMergePolicy.NewSortingMergePolicy(sort)); iw = new RandomIndexWriter(new Random(seed), dir, iwc); for (int i = 0; i < numDocs; ++i) { Document doc = RandomDocument(); iw.AddDocument(doc); if (i == numDocs / 2 || (i != numDocs - 1 && Random().nextInt(8) == 0)) { iw.Commit(); } if (Random().nextInt(15) == 0) { string term = RandomInts.RandomFrom(Random(), terms); iw.DeleteDocuments(new Term("s", term)); } } reader = iw.Reader; }
private RAMDirectory(Directory dir, bool closeDir) { System.String[] files = dir.List(); byte[] buf = new byte[BufferedIndexOutput.BUFFER_SIZE]; for (int i = 0; i < files.Length; i++) { // make place on ram disk IndexOutput os = CreateOutput(System.IO.Path.GetFileName(files[i])); // read current file IndexInput is_Renamed = dir.OpenInput(files[i]); // and copy to ram disk long len = (int) is_Renamed.Length(); long readCount = 0; while (readCount < len) { int toRead = readCount + BufferedIndexOutput.BUFFER_SIZE > len ? (int) (len - readCount) : BufferedIndexOutput.BUFFER_SIZE; is_Renamed.ReadBytes(buf, 0, toRead); os.WriteBytes(buf, toRead); readCount += toRead; } // graceful cleanup is_Renamed.Close(); os.Close(); } if (closeDir) dir.Close(); }
public SuggestionQueryIndexExtension( WorkContext workContext, string key, StringDistance distanceType, bool isRunInMemory, string field, float accuracy) { this.workContext = workContext; this.key = key; this.field = field; if (isRunInMemory) { directory = new RAMDirectory(); } else { directory = FSDirectory.Open(new DirectoryInfo(key)); } this.spellChecker = new SpellChecker.Net.Search.Spell.SpellChecker(directory, null); this.spellChecker.SetAccuracy(accuracy); this.spellChecker.setStringDistance(distanceType); }
public override void SetUp() { base.SetUp(); DirectoryInfo file = CreateTempDir("testIndex"); // use a simple FSDir here, to be sure to have SimpleFSInputs Dir = new SimpleFSDirectory(file, null); }
public override void SetUp() { base.SetUp(); dir = NewDirectory(); var iw = new RandomIndexWriter(Random(), dir, Similarity, TimeZone); var doc = new Document { NewStringField("id", "1", Field.Store.YES), NewTextField("body", "some contents and more contents", Field.Store.NO), new NumericDocValuesField("popularity", 5) }; iw.AddDocument(doc); doc = new Document { NewStringField("id", "2", Field.Store.YES), NewTextField("body", "another document with different contents", Field.Store .NO), new NumericDocValuesField("popularity", 20) }; iw.AddDocument(doc); doc = new Document { NewStringField("id", "3", Field.Store.YES), NewTextField("body", "crappy contents", Field.Store.NO), new NumericDocValuesField("popularity", 2) }; iw.AddDocument(doc); reader = iw.Reader; searcher = new IndexSearcher(reader); iw.Dispose(); }
public void Initialize() { _cwsDir = new RAMDirectory(); _pdfDir = new RAMDirectory(); _simpleDir = new RAMDirectory(); _conventionDir = new RAMDirectory(); //get all of the indexers and rebuild them all first var indexers = new IIndexer[] { IndexInitializer.GetUmbracoIndexer(_cwsDir), IndexInitializer.GetSimpleIndexer(_simpleDir), IndexInitializer.GetUmbracoIndexer(_conventionDir) }; foreach (var i in indexers) { try { i.RebuildIndex(); } finally { var d = i as IDisposable; if (d != null) d.Dispose(); } } //now get the multi index searcher for all indexes _searcher = IndexInitializer.GetMultiSearcher(_pdfDir, _simpleDir, _conventionDir, _cwsDir); }
public static void Main(string[] args) { if (args.Length < 3) { Console.Error.WriteLine("Usage: IndexMergeTool <mergedIndex> <index1> <index2> [index3] ..."); Environment.Exit(1); } FSDirectory mergedIndex = FSDirectory.Open(new System.IO.DirectoryInfo(args[0])); #pragma warning disable 612, 618 using (IndexWriter writer = new IndexWriter(mergedIndex, new IndexWriterConfig(LuceneVersion.LUCENE_CURRENT, null) .SetOpenMode(IndexWriterConfig.OpenMode_e.CREATE))) #pragma warning restore 612, 618 { Directory[] indexes = new Directory[args.Length - 1]; for (int i = 1; i < args.Length; i++) { indexes[i - 1] = FSDirectory.Open(new System.IO.DirectoryInfo(args[i])); } Console.WriteLine("Merging..."); writer.AddIndexes(indexes); Console.WriteLine("Full merge..."); writer.ForceMerge(1); } Console.WriteLine("Done."); }
public SearchAutoComplete(Directory autoCompleteDir, int maxResults = 8) { this.m_directory = autoCompleteDir; MaxResults = maxResults; ReplaceSearcher(); }
public override void TearDown() { base.TearDown(); dir.Dispose(); dir = null; anlzr = null; }
/// <summary> /// ctor for unit tests /// </summary> internal LocalPackageIndex(LuceneDirectory directory, IPackageSearchEngine engine, IReflectorFactory reflectorFactory, ILog logger) { _directory = directory; _engine = engine; _reflectorFactory = reflectorFactory; Logger = logger; }
public FileSwitchDirectory(System.Collections.Hashtable primaryExtensions, Directory primaryDir, Directory secondaryDir, bool doClose) { this.primaryExtensions = primaryExtensions; this.primaryDir = primaryDir; this.secondaryDir = secondaryDir; this.doClose = doClose; this.lockFactory = primaryDir.GetLockFactory(); }
public LuceneIndexingService(Func <EntitiesContext> contextThunk, bool indexContainsAllVersions) { if (contextThunk == null) { throw new ArgumentNullException("contextThunk"); } _contextThunk = contextThunk; _indexContainsAllVersions = indexContainsAllVersions; _directory = new LuceneFileSystem(LuceneCommon.IndexDirectory); _getShouldAutoUpdate = () => true; }
public override void TestSetup() { _luceneDir = new RAMDirectory(); _indexer = IndexInitializer.GetSimpleIndexer(_luceneDir); _reIndexDateTime = DateTime.Now; Thread.Sleep(1000); _indexer.RebuildIndex(); _searcher = IndexInitializer.GetLuceneSearcher(_luceneDir); }
public void TestConsistencyOnExceptions() { // so the handler's index isn't empty replicator.Publish(CreateRevision(1)); client.UpdateNow(); client.Dispose(); callback.Dispose(); // Replicator violates write-once policy. It may be that the // handler copies files to the index dir, then fails to copy a // file and reverts the copy operation. On the next attempt, it // will copy the same file again. There is nothing wrong with this // in a real system, but it does violate write-once, and MDW // doesn't like it. Disabling it means that we won't catch cases // where the handler overwrites an existing index file, but // there's nothing currently we can do about it, unless we don't // use MDW. handlerIndexDir.PreventDoubleWrite = (false); handlerTaxoDir.PreventDoubleWrite = (false); // wrap sourceDirFactory to return a MockDirWrapper so we can simulate errors ISourceDirectoryFactory @in = sourceDirFactory; AtomicInt32 failures = new AtomicInt32(AtLeast(10)); sourceDirFactory = new SourceDirectoryFactoryAnonymousClass(this, @in, failures); handler = new IndexAndTaxonomyReplicationHandler(handlerIndexDir, handlerTaxoDir, () => { if (Random.NextDouble() < 0.2 && failures > 0) { throw RuntimeException.Create("random exception from callback"); } }); client = new ReplicationClientAnonymousClass(this, replicator, handler, @in, failures); client.StartUpdateThread(10, "indexAndTaxo"); Directory baseHandlerIndexDir = handlerIndexDir.Delegate; int numRevisions = AtLeast(20) + 2; for (int i = 2; i < numRevisions; i++) { replicator.Publish(CreateRevision(i)); AssertHandlerRevision(i, baseHandlerIndexDir); } // disable errors -- maybe randomness didn't exhaust all allowed failures, // and we don't want e.g. CheckIndex to hit false errors. handlerIndexDir.MaxSizeInBytes = (0); handlerIndexDir.RandomIOExceptionRate = (0.0); handlerIndexDir.RandomIOExceptionRateOnOpen = (0.0); handlerTaxoDir.MaxSizeInBytes = (0); handlerTaxoDir.RandomIOExceptionRate = (0.0); handlerTaxoDir.RandomIOExceptionRateOnOpen = (0.0); }
public int getLemmaIDbyLemmaName(string lemmaName) { string directory = System.IO.Directory.GetCurrentDirectory(); string[] splitDir = directory.Split('\\'); if (splitDir[splitDir.Length - 1] == "Debug") { System.IO.Directory.SetCurrentDirectory("..\\..\\"); } string[] splitLemmaName = lemmaName.Split('('); string results = null; int id = -1; string indexDir = "Index"; using (Lucene.Net.Store.Directory dir = FSDirectory.Open(indexDir)) using (IndexSearcher searcher = new IndexSearcher(dir)) { string[] splited; Term term; WildcardQuery q = null; BooleanQuery bq = new BooleanQuery(); if (splitLemmaName[0].Split(' ').Length > 1) { splited = lemmaName.Split(' '); for (int i = 0; i < splited.Length; i++) { term = new Term("title", "*" + splited[i].ToLower() + "*"); q = new WildcardQuery(term); bq.Add(q, Occur.SHOULD); } } else { term = new Term("title", "*" + splitLemmaName[0].ToLower() + "*"); q = new WildcardQuery(term); bq.Add(q, Occur.SHOULD); } TopDocs hits = searcher.Search(bq, 1); foreach (ScoreDoc d in hits.ScoreDocs) { Lucene.Net.Documents.Document doc = searcher.Doc(d.Doc); results = doc.Get("LID").ToString(); } } Int32.TryParse(results, out id); return(id); }
/// <summary> /// Copies the provided list of files from the <paramref name="source"/> <see cref="Directory"/> to the /// <paramref name="target"/> <see cref="Directory"/>, if they are not the same. /// </summary> /// <exception cref="IOException"></exception> public static void CopyFiles(Directory source, Directory target, IList <string> files) { if (source.Equals(target)) { return; } foreach (string file in files) { source.Copy(target, file, file, IOContext.READ_ONCE); } }
public static UmbracoContentIndexer GetUmbracoIndexer(Lucene.Net.Store.Directory luceneDir) { var i = new UmbracoContentIndexer(CreateCriteria(), luceneDir, //custom lucene directory new TestDataService(), new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), false); i.IndexingError += IndexingError; return(i); }
public void TestOpen() { Directory indexDir = NewDirectory(); IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null); conf.IndexDeletionPolicy = new SnapshotDeletionPolicy(conf.IndexDeletionPolicy); IndexWriter indexWriter = new IndexWriter(indexDir, conf); Directory taxoDir = NewDirectory(); IndexAndTaxonomyRevision.SnapshotDirectoryTaxonomyWriter taxoWriter = new IndexAndTaxonomyRevision.SnapshotDirectoryTaxonomyWriter(taxoDir); try { indexWriter.AddDocument(NewDocument(taxoWriter)); indexWriter.Commit(); taxoWriter.Commit(); IRevision rev = new IndexAndTaxonomyRevision(indexWriter, taxoWriter); foreach (var e in rev.SourceFiles) { string source = e.Key; Directory dir = source.Equals(IndexAndTaxonomyRevision.INDEX_SOURCE, StringComparison.Ordinal) ? indexDir : taxoDir; foreach (RevisionFile file in e.Value) { using (IndexInput src = dir.OpenInput(file.FileName, IOContext.READ_ONCE)) using (System.IO.Stream @in = rev.Open(source, file.FileName)) { assertEquals(src.Length, @in.Length); byte[] srcBytes = new byte[(int)src.Length]; byte[] inBytes = new byte[(int)src.Length]; int offset = 0; if (Random.nextBoolean()) { int skip = Random.Next(10); if (skip >= src.Length) { skip = 0; } @in.Seek(skip, SeekOrigin.Current); src.Seek(skip); offset = skip; } src.ReadBytes(srcBytes, offset, srcBytes.Length - offset); @in.Read(inBytes, offset, inBytes.Length - offset); assertArrayEquals(srcBytes, inBytes); } } } } finally { IOUtils.Dispose(indexWriter, taxoWriter, taxoDir, indexDir); } }
public TermVectorsWriter(Directory directory, System.String segment, FieldInfos fieldInfos, IState state) { // Open files for TermVector storage tvx = directory.CreateOutput(segment + "." + IndexFileNames.VECTORS_INDEX_EXTENSION, state); tvx.WriteInt(TermVectorsReader.FORMAT_CURRENT); tvd = directory.CreateOutput(segment + "." + IndexFileNames.VECTORS_DOCUMENTS_EXTENSION, state); tvd.WriteInt(TermVectorsReader.FORMAT_CURRENT); tvf = directory.CreateOutput(segment + "." + IndexFileNames.VECTORS_FIELDS_EXTENSION, state); tvf.WriteInt(TermVectorsReader.FORMAT_CURRENT); this.fieldInfos = fieldInfos; }
/// <summary> /// Creates the index at a given path /// </summary> /// <param name="indexPath">The pathname to create the index</param> public void CreateIndex(string indexPath) { luceneIndexDirectory = Lucene.Net.Store.FSDirectory.Open(indexPath); IndexWriter.MaxFieldLength mfl = new IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH); writer = new Lucene.Net.Index.IndexWriter(luceneIndexDirectory, analyzer, true, mfl); writer.SetSimilarity(new NewSimilarity()); //writer.SetSimilarity(newSimilarity); // for similarity measure }
/// <summary>Initializes a <see cref="NuGetSearcherManager"/> instance.</summary> /// <param name="directory"> /// Optionally, the Lucene directory to read the index from. If <c>null</c> is provided, the directory /// implementation is determined based off of the configuration (<see cref="config"/>). /// </param> /// <param name="loader"> /// Optionally, the loader used to read the JSON data files. If <c>null</c> is provided, the loader /// implementation is determined based off of the configuration (<see cref="config"/>). /// </param> /// <param name="config"> /// The configuration to read which primarily determines whether the resulting instance will read from the local /// disk or from blob storage. /// </param> /// <param name="loggerFactory"> /// Optionally, the logger factory defined by the consuming application. /// </param> /// <returns>The resulting <see cref="NuGetSearcherManager"/> instance.</returns> public static NuGetSearcherManager Create(IndexingConfiguration config, ILoggerFactory loggerFactory, Directory directory = null, ILoader loader = null) { if (loggerFactory == null) { throw new ArgumentNullException(nameof(loggerFactory)); } LoggerFactory = loggerFactory; var logger = loggerFactory.CreateLogger <NuGetSearcherManager>(); // If a local Lucene directory has been specified, create a directory and loader for the specified directory. var luceneDirectory = config.LocalLuceneDirectory; if (!string.IsNullOrEmpty(luceneDirectory)) { directory = directory ?? new SimpleFSDirectory(new DirectoryInfo(luceneDirectory)); loader = loader ?? new FileLoader(config.LocalDataDirectory); } IIndexDirectoryProvider indexProvider; if (directory == null) { // If no directory has been provided, create a CloudIndexDirectoryProvider from the configuration. indexProvider = new CloudIndexDirectoryProvider(config, logger); } else { // Use the specified directory to create a FixedIndexDirectoryProvider. var indexContainerName = luceneDirectory ?? config.IndexContainer; indexProvider = new LocalIndexDirectoryProvider(directory, indexContainerName); } // If a loader has been specified, use it. // Otherwise, create a StorageLoader from the configuration. loader = loader ?? new StorageLoader(config, logger); var searcherManager = new NuGetSearcherManager(logger, indexProvider, loader, config.AuxiliaryDataRefreshRateSec, config.IndexReloadRateSec); var registrationBaseAddress = config.RegistrationBaseAddress; var semVer2RegistrationBaseAddress = config.SemVer2RegistrationBaseAddress; searcherManager.RegistrationBaseAddresses.LegacyHttp = MakeRegistrationBaseAddress("http", registrationBaseAddress); searcherManager.RegistrationBaseAddresses.LegacyHttps = MakeRegistrationBaseAddress("https", registrationBaseAddress); searcherManager.RegistrationBaseAddresses.SemVer2Http = MakeRegistrationBaseAddress("http", semVer2RegistrationBaseAddress); searcherManager.RegistrationBaseAddresses.SemVer2Https = MakeRegistrationBaseAddress("https", semVer2RegistrationBaseAddress); return(searcherManager); }
public void Ini() { if (UseAzureDirectory) { var azureBlobConnectionString = config.GetValue <string>("AzureDirectoryConnectionString"); var azureDirectoryCachePath = ReplacePathTokens(config.GetValue <string>("AzureDirectoryCachePath")); var azureDirectoryContainerName = config.GetValue <string>("AzureDirectoryContainerName"); //if empty string, ensure value is null if (string.IsNullOrEmpty(azureDirectoryContainerName)) { azureDirectoryContainerName = null; } var cloudStorageAccount = CloudStorageAccount.Parse(azureBlobConnectionString); Directory = new AzureDirectory(cloudStorageAccount, azureDirectoryCachePath, containerName: azureDirectoryContainerName); } else { if (!System.IO.Directory.Exists(INDEXPATH)) { System.IO.Directory.CreateDirectory(INDEXPATH); } Directory = FSDirectory.Open(INDEXPATH); } bool create = !DirectoryReader.IndexExists(Directory); lock (write_lock) { try { SetWriter(create); if (Writer != null && !UseAzureDirectory) { CloseWriter(); } //Writer.Optimize(); } catch (Lucene.Net.Store.LockObtainFailedException ex) { logger.Log(ex); } catch (Exception ex) { throw; //logger.Log(ex); } finally { //CloseWriter(); } } SetSearcher(); }
private void OpenIndexOnStartup() { analyzer = new LowerCaseKeywordAnalyzer(); snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); bool resetTried = false; bool recoveryTried = false; while (true) { LuceneDirectory luceneDirectory = null; try { luceneDirectory = OpenOrCreateLuceneDirectory(indexDirectory); // Skip sanity test if we are running in memory. Index will not exist anyways. if (!configuration.RunInMemory && !IsIndexStateValid(luceneDirectory)) { throw new InvalidOperationException("Sanity check on the index failed."); } directory = luceneDirectory; writer = new IndexWriter(directory, analyzer, snapshotter, IndexWriter.MaxFieldLength.UNLIMITED); writer.SetMergeScheduler(new ErrorLoggingConcurrentMergeScheduler()); currentIndexSearcherHolder.SetIndexSearcher(new IndexSearcher(directory, true)); break; } catch (Exception e) { if (resetTried) { throw new InvalidOperationException("Could not open / create index for file system '" + name + "', reset already tried", e); } if (recoveryTried == false && luceneDirectory != null) { recoveryTried = true; StartupLog.WarnException("Could not open index for file system '" + name + "'. Trying to recover index", e); StartupLog.Info("Recover functionality is still not implemented. Skipping."); } else { resetTried = true; StartupLog.WarnException("Could not open index for file system '" + name + "'. Recovery operation failed, forcibly resetting index", e); TryResettingIndex(); } } } }
private void button2_Click(object sender, EventArgs e) { if (string.IsNullOrEmpty(this.txtSearch.Text)) { MessageBox.Show("请输入搜索的文本"); } StringBuilder sb = new StringBuilder(); Stopwatch sw = new Stopwatch(); sw.Start(); //索引库目录 Lucene.Net.Store.Directory dir = FSDirectory.Open(new System.IO.DirectoryInfo("IndexDir"), new NoLockFactory()); IndexReader reader = IndexReader.Open(dir, true); IndexSearcher search = null; try { search = new IndexSearcher(reader); QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, "body", new PanGuAnalyzer()); Query query = parser.Parse(LuceneHelper.GetKeyWordSplid(this.txtSearch.Text)); //执行搜索,获取查询结果集对象 TopDocs ts = search.Search(query, null, 1000); ///获取命中的文档信息对象 ScoreDoc[] docs = ts.ScoreDocs; sw.Stop(); this.listBox1.Items.Clear(); for (int i = 0; i < docs.Length; i++) { int docId = docs[i].Doc; Document doc = search.Doc(docId); this.listBox1.Items.Add(doc.Get("number") + "\r\n"); this.listBox1.Items.Add(doc.Get("body") + "\r\n"); this.listBox1.Items.Add("------------------------\r\n"); } } catch (Exception ex) { throw; } finally { if (search != null) { search.Dispose(); } if (dir != null) { dir.Dispose(); } } this.label1.Text = "搜索用时:" + sw.ElapsedMilliseconds + "毫秒"; }
/// <summary> /// Remove facet documents from the existing index. /// </summary> /// <param name="directoryIndexInfo">The directory infomation where the index files are located.</param> /// <param name="directoryFacetInfo">The directory infomation where the facet files are to be placed.</param> /// <param name="textNames">The array of names for text data.</param> /// <param name="filePaths">The array of full paths (without root 'C:\'. e.g. 'temp/http/index.html') for file documents.</param> public void RemoveMultiFacetDocuments(DirectoryInfo directoryIndexInfo, DirectoryInfo directoryFacetInfo, string[] textNames, string[] filePaths) { Lucene.Net.Index.IndexWriter writer = null; DirectoryTaxonomyWriter facetWriter = null; Lucene.Net.Store.Directory directory = null; Lucene.Net.Store.Directory directoryFacet = null; try { // Create the analyzer. SimpleAnalyzer simpleAnalyzer = new Analyzer.SimpleAnalyzer(); StandardAnalyzer standardAnalyzer = new Analyzer.StandardAnalyzer(simpleAnalyzer); // Create the index writer. directory = FSDirectory.Open(directoryIndexInfo); IndexWriterConfig indexConfig = new IndexWriterConfig(Lucene.Net.Util.LuceneVersion.LUCENE_48, standardAnalyzer); indexConfig.SetOpenMode(IndexWriterConfig.OpenMode_e.APPEND); // Open existing or create new. writer = new IndexWriter(directory, indexConfig); // Create the facet writer. directoryFacet = FSDirectory.Open(directoryFacetInfo); facetWriter = new DirectoryTaxonomyWriter(directoryFacet, IndexWriterConfig.OpenMode_e.APPEND); // Create the delet query. FacetFilter filter = new FacetFilter(); Query[] queries = filter.RemoveDocuments(textNames, filePaths); writer.DeleteDocuments(queries); // Commit the index. writer.Commit(); facetWriter.Commit(); } catch (Exception) { throw; } finally { if (writer != null) { writer.Dispose(); } if (directory != null) { directory.Dispose(); } } }
/// <summary>. </summary> public void SyncFile(Lucene.Net.Store.Directory directory, string fileName, bool CompressBlobs) { Trace.WriteLine($"INFO Syncing file {fileName} for {_rootFolderName}"); // then we will get it fresh into local deflatedName // StreamOutput deflatedStream = new StreamOutput(CacheDirectory.CreateOutput(deflatedName)); // seek back to begininng if (ShouldCompressFile(fileName, CompressBlobs)) { using (var deflatedStream = new MemoryStream()) { #if FULLDEBUG Trace.WriteLine($"GET {fileName} RETREIVED {deflatedStream.Length} bytes"); #endif // get the deflated blob blob.DownloadTo(deflatedStream); deflatedStream.Seek(0, SeekOrigin.Begin); // open output file for uncompressed contents using (var fileStream = new StreamOutput(directory.CreateOutput(fileName))) using (var decompressor = new DeflateStream(deflatedStream, CompressionMode.Decompress)) { var bytes = new byte[65535]; var nRead = 0; do { nRead = decompressor.Read(bytes, 0, 65535); if (nRead > 0) { fileStream.Write(bytes, 0, nRead); } } while (nRead == 65535); } } } else { using (var fileStream = new StreamOutput(directory.CreateOutput(fileName))) { // get the blob blob.DownloadTo(fileStream); fileStream.Flush(); #if FULLDEBUG Trace.WriteLine($"GET {fileName} RETREIVED {fileStream.Length} bytes"); #endif } } }
public void Write(Directory d, System.String name, IState state) { IndexOutput output = d.CreateOutput(name, state); try { Write(output); } finally { output.Close(); } }
protected virtual IndexWriter CreateWriter(Directory d, Analyzer a) { try { return(CreateWriterNoTry(d, a)); } catch (Lucene.Net.Store.LockObtainFailedException) { logger.Debug("Failed to obtain lock, deleting it and retrying."); ClearLock(); return(CreateWriterNoTry(d, a)); } }
public virtual Directory GetDirectory() { lock (this) { if (!System.IO.Directory.Exists(indexPath)) { System.IO.Directory.CreateDirectory(indexPath); } var d = directory ?? (directory = new SimpleFSDirectory(new DirectoryInfo(indexPath), new SimpleFSLockFactory())); return(d); } }
private void Initialize(Directory directory, System.String segment, FieldInfos fis, int interval, bool isi, IState state) { indexInterval = interval; fieldInfos = fis; isIndex = isi; output = directory.CreateOutput(segment + (isIndex?".tii":".tis"), state); output.WriteInt(FORMAT_CURRENT); // write format output.WriteLong(0); // leave space for size output.WriteInt(indexInterval); // write indexInterval output.WriteInt(skipInterval); // write skipInterval output.WriteInt(maxSkipLevels); // write maxSkipLevels System.Diagnostics.Debug.Assert(InitUTF16Results()); }
public LocalPackageIndex(ILog logger = null) { Logger = logger; _location = Environment.ExpandEnvironmentVariables(DefaultIndexPath); if (!System.IO.Directory.Exists(_location)) { System.IO.Directory.CreateDirectory(_location); } _directory = FSDirectory.Open(_location); _engine = new PackageSearchEngine(IndexDirectory, Analyzer, Logger); }
protected virtual Lucene.Net.Store.Directory GetLuceneDirectory() { if (_nrtWriter != null) { return(_nrtWriter.GetDirectory()); } if (_luceneDirectory == null) { _luceneDirectory = new SimpleFSDirectory(LuceneIndexFolder); } return(_luceneDirectory); }
// Activity 7 public void OpenIndex(string indexPath) { /* Make sure to pass a new directory that does not exist */ if (System.IO.Directory.Exists(indexPath)) { Console.WriteLine("This directory already exists - Choose a directory that does not exist"); Console.Write("Hit any key to exit"); Console.ReadKey(); Environment.Exit(0); } luceneIndexDirectory = FSDirectory.Open(indexPath); }
public RemoteSyncDirectory(IRemoteDirectory remoteDirectory, Directory cacheDirectory, bool compressBlobs = false) { CacheDirectory = cacheDirectory; LoggingService = new TraceLoggingService(); RemoteDirectory = remoteDirectory; _lockFactory = GetLockFactory(); _remoteIndexOutputFactory = GetAzureIndexOutputFactory(); _remoteDirectoryIndexInputFactory = GetAzureIndexInputFactory(); GuardCacheDirectory(CacheDirectory); CompressBlobs = compressBlobs; }
public IndexAndTaxonomyReadyCallback(MockDirectoryWrapper indexDir, MockDirectoryWrapper taxoDir) { this.indexDir = indexDir; this.taxoDir = taxoDir; config = new FacetsConfig(); config.SetHierarchical("A", true); if (DirectoryReader.IndexExists(indexDir)) { indexReader = DirectoryReader.Open(indexDir); lastIndexGeneration = indexReader.IndexCommit.Generation; taxoReader = new DirectoryTaxonomyReader(taxoDir); } }
protected virtual Lucene.Net.Store.Directory GetLuceneDirectory() { if (_nrtWriter != null) { return(_nrtWriter.GetDirectory()); } if (_luceneDirectory == null) { _luceneDirectory = DirectoryTracker.Current.GetDirectory(LuceneIndexFolder); } return(_luceneDirectory); }
public void Dispose() { LogManager.GetCurrentClassLogger().Info("Stopping Lucene indexing services."); if (PackageFileSystemWatcher != null) { PackageFileSystemWatcher.Dispose(); } PackageIndexer.Dispose(); Provider.Dispose(); LuceneDirectory.Dispose(); }
public virtual String SegString(Directory dir, IState state) { var b = new System.Text.StringBuilder(); b.Append("MergeSpec:\n"); int count = merges.Count; for (int i = 0; i < count; i++) { b.Append(" ").Append(1 + i).Append(": ").Append(merges[i].SegString(dir, state)); } return(b.ToString()); }
public MultiIndexLockFactory(Lucene.Net.Store.Directory master, Lucene.Net.Store.Directory child) { if (master == null) { throw new ArgumentNullException("master"); } if (child == null) { throw new ArgumentNullException("child"); } _master = master; _child = child; }
public LuceneIndexingService( IEntityRepository <Package> packageSource, IEntityRepository <CuratedPackage> curatedPackageSource, Lucene.Net.Store.Directory directory, IDiagnosticsService diagnostics, IAppConfiguration config) { _packageRepository = packageSource; _curatedPackageRepository = curatedPackageSource; _directory = directory; _getShouldAutoUpdate = config == null ? new Func <bool>(() => true) : new Func <bool>(() => config.AutoUpdateSearchIndex); Trace = diagnostics.SafeGetSource("LuceneIndexingService"); }
public void Dispose() { lock (this) { if (writer != null) { writer.Close(waitForMerges: true); } writer = null; searcher = null; directory = null; } }
public SqloogleMiaSearcher(string indexPath, int resultsLimit = 50) { _resultsLimit = resultsLimit; var fields = new[] { "server", "database", "schema", "name", "equality", "inequality", "included" }; var boosts = new Dictionary<string, float> { { "server", .4F }, { "database", .3F }, { "schema", .2F }, { "name", .1F }, { "equality", .0F }, { "inequality", .0F }, { "included", .0F } }; _directory = FSDirectory.Open(new DirectoryInfo(indexPath)); _searcher = new IndexSearcher(_directory, true); _analyzer = new StandardAnalyzer(Version.LUCENE_30); _parser = new MultiFieldQueryParser(Version.LUCENE_30, fields, _analyzer, boosts) { DefaultOperator = QueryParser.Operator.AND }; _logger.Trace("Searcher is ready."); }