public IndexInputSlicerAnonymousInnerClassHelper(SimpleFSDirectory outerInstance, IOContext context, FileInfo file, FileStream descriptor) : base(outerInstance) { this.Context = context; this.File = file; this.Descriptor = descriptor; }
public void CreateIndex(Analyzer analayer) { FSDirectory fsDir = new SimpleFSDirectory(new DirectoryInfo(_indexerFolder)); IndexWriter indexWriter = new IndexWriter(fsDir, analayer, true, Lucene.Net.Index.IndexWriter.MaxFieldLength.UNLIMITED); string[] files = System.IO.Directory.GetFiles(_textFilesFolder, Config.FileSearchPattern, SearchOption.AllDirectories); foreach (string file in files) { string name = new FileInfo(file).Name; string content = File.ReadAllText(file); Document doc = new Document(); doc.Add(new Field(Config.Field_Path, file, Field.Store.YES, Field.Index.NOT_ANALYZED)); doc.Add(new Field(Config.Field_Name, name, Field.Store.YES, Field.Index.ANALYZED)); doc.Add(new Field(Config.Field_Content, content, Field.Store.NO, Field.Index.ANALYZED)); indexWriter.AddDocument(doc); Console.WriteLine("{0} - {1}", file, name); } indexWriter.Optimize(); indexWriter.Dispose(); Console.WriteLine("File count: {0}", files.Length); }
public Directory DirectoryFor(string id, bool persistent) { if (persistent) { var info = new DirectoryInfo(Path.Combine(_root.FullName, id+".index")); var directory = new SimpleFSDirectory(info); if (!info.Exists || !info.EnumerateFiles().Any()) { new IndexWriter(directory, new StandardAnalyzer(Version.LUCENE_29), true, IndexWriter.MaxFieldLength.UNLIMITED) .Dispose(); } return directory; } else { if (!_inMemoryDirectories.ContainsKey(id)) { _inMemoryDirectories[id] = new RAMDirectory(); } return _inMemoryDirectories[id]; } }
/// <summary> /// Ensures that the data has been loaded in the Lucene index /// /// SIDE EFFECT /// This method sets property _searcher /// </summary> /// <param name="absoluteLucenePath"> /// Absolute path of the directory where the Lucene files get stored. /// </param> /// <param name="onlyIfNotExists"> /// If true, the index will only be created if there is no index at all (that is, no Lucene directory). /// If false, this will always create a new index. /// </param> /// <param name="loadLuceneIndex"> /// Actually loads data into the Lucene index /// </param> protected void LoadStore(string absoluteLucenePath, bool onlyIfNotExists, Action<SimpleFSDirectory> loadLuceneIndex) { var luceneDir = new DirectoryInfo(absoluteLucenePath); bool luceneDirExists = luceneDir.Exists; if (!luceneDirExists) { luceneDir.Create(); luceneDir.Refresh(); } var directory = new SimpleFSDirectory(luceneDir); if (onlyIfNotExists && luceneDirExists) { // Make sure to always create a searcher. // There must actually be a valid Lucene index in the directory, otherwise this will throw an exception. _searcher = new IndexSearcher(directory, true); return; } loadLuceneIndex(directory); // Now that the index has been updated, need to open a new searcher. // Existing searcher will still be reading the previous index. _searcher = new IndexSearcher(directory, true); }
private Lucene.Net.Store.Directory CreateLuceneIndex(IEnumerable<PackageVersion> packages, string luceneDirectory) { Lucene.Net.Store.Directory directory; if (luceneDirectory != null) { var directoryInfo = new DirectoryInfo(luceneDirectory); directoryInfo.Create(); directory = new SimpleFSDirectory(directoryInfo); } else { directory = new RAMDirectoryWrapper(); } using (var indexWriter = DocumentCreator.CreateIndexWriter(directory, true)) { foreach (var version in packages) { var metadata = GetPackageMetadata(version); var document = DocumentCreator.CreateDocument(metadata); indexWriter.AddDocument(document); } indexWriter.Commit(); } return directory; }
public override Task<bool> Run() { FrameworksList frameworksList = new StorageFrameworksList(DataStorageAccount, DataContainerName, FrameworksList.FileName); Lucene.Net.Store.Directory directory = new SimpleFSDirectory(new DirectoryInfo(LocalIndexFolder)); PackageIndexing.RebuildIndex(PackageDatabase.ConnectionString, directory, frameworksList, Console.Out); return Task.FromResult(true); }
public void CreateIndex(Analyzer analayer) { FSDirectory fsDir = new SimpleFSDirectory(new DirectoryInfo(_indexerFolder)); IndexWriter indexWriter = new IndexWriter(fsDir, analayer, true, Lucene.Net.Index.IndexWriter.MaxFieldLength.UNLIMITED); Stopwatch stopWatch = Stopwatch.StartNew(); int analyzedCount = 0; string[] files = System.IO.Directory.GetFiles(_textFilesFolder, this._fileSearchPattern, SearchOption.AllDirectories); //统计需要索引的文件页数 int totalPages = GetTotalPages(files); WriteLog("Total pages statistics takes {0}ms", stopWatch.Elapsed.Milliseconds); stopWatch.Restart(); TextAbsorber textAbsorber = new TextAbsorber(); //开始索引 foreach (string pdfFile in files) { var fileInfo = new FileInfo(pdfFile); var fileName = fileInfo.Name; Aspose.Pdf.Document pdfDocument = new Aspose.Pdf.Document(pdfFile); WriteLog("Current file is {0}", pdfFile); //注意pdf页码从1开始 for (int i = 1;i<=pdfDocument.Pages.Count;i++) { Page page = pdfDocument.Pages[i]; page.Accept(textAbsorber); string pageContent = textAbsorber.Text; Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document(); doc.Add(new Field(LuceneConfig.Field_Path, pdfFile, Field.Store.YES, Field.Index.NOT_ANALYZED)); doc.Add(new Field(LuceneConfig.Field_FileName, fileName, Field.Store.YES, Field.Index.ANALYZED)); doc.Add(new Field(LuceneConfig.Field_PageNumber, i.ToString(), Field.Store.YES, Field.Index.ANALYZED)); doc.Add(new Field(LuceneConfig.Field_ContentByPage, pageContent, Field.Store.NO, Field.Index.ANALYZED)); indexWriter.AddDocument(doc); analyzedCount++; RaiseProgressChanged(analyzedCount * 100 / totalPages); } } indexWriter.Optimize(); indexWriter.Dispose(); stopWatch.Stop(); Console.WriteLine("All completed. It takes {0}ms", stopWatch.Elapsed); }
protected override void Context() { var dir = new SimpleFSDirectory(new DirectoryInfo(TempDirectory), new NoLockFactory()); var analyzer = new StandardAnalyzer(Version.LUCENE_20); var maxFieldLength = new IndexWriter.MaxFieldLength(200); var index = new FluentIndexWriter<TestObject>(dir, analyzer, maxFieldLength); var data = new TestObject() { Id = Guid.NewGuid(), LongId = 123, ValidProperty = "Property", IgnoredProperty = "Ignored" }; var data2 = new TestObject() { Id = Guid.Empty, LongId = 123456, ValidProperty = "Abc def ghij", IgnoredProperty = "Ignored" }; index.AddDocument(data); index.AddDocument(data2); index.Commit(); }
internal static Lucene.Net.Store.Directory GetDirectory() { if (_directorySingleton == null) { if (!System.IO.Directory.Exists(IndexDirectory)) { System.IO.Directory.CreateDirectory(IndexDirectory); } var directoryInfo = new DirectoryInfo(IndexDirectory); _directorySingleton = new SimpleFSDirectory(directoryInfo); } return _directorySingleton; }
public override bool OnStart() { var readPath = RoleEnvironment.GetLocalResource("ReadStorage").RootPath; var writePath = RoleEnvironment.GetLocalResource("WriteStorage").RootPath; var subscription = "lucene"; var account = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("Blob")); var hub = new Hub(RoleEnvironment.GetConfigurationSettingValue("ServiceBus")); var criticalWait = Task.Run(() => { string writeFolder = Path.Combine(writePath, "lucene"); if (!System.IO.Directory.Exists(writeFolder)) { System.IO.Directory.CreateDirectory(writeFolder); } string combine = Path.Combine(readPath, subscription); if (!System.IO.Directory.Exists(combine)) { System.IO.Directory.CreateDirectory(combine); } var localDirectory = new Lucene.Net.Store.SimpleFSDirectory(new DirectoryInfo(combine)); var masterDirectory = new AzureDirectory(account, subscription); var irs = new IntermediateReaderService(masterDirectory, hub, localDirectory, true); masterDirectory.Dispose(); return(new IndexInfo { IRS = irs as IDisposable, Directory = localDirectory }); }) .ContinueWith(t => _localIndexes = new[] { t.Result }); _luceneWriter = new SearchWriterService(new AzureDirectory(account, "lucene", new Lucene.Net.Store.SimpleFSDirectory(new DirectoryInfo(writePath))), true); var subscriptionName = string.Format("{0:yyyyMMddHHmmss}_{1}", DateTime.UtcNow, Guid.NewGuid().ToString().Replace("-", string.Empty)); hub.CreateSubscription("lucene", subscriptionName) .ContinueWith(t => { _luceneTempInstanceSubscription = t.Result; return(hub.SubscribeWithRoutingKey("lucene", subscriptionName, OnSearchMessage) .ContinueWith(t2 => _luceneSearchSubscription = t.Result)); }) .Unwrap(); return(base.OnStart()); }
internal static Lucene.Net.Store.Directory GetDirectory(LuceneIndexLocation location) { if (_directorySingleton == null) { var index = GetIndexLocation(location); if (!System.IO.Directory.Exists(index)) { System.IO.Directory.CreateDirectory(index); } var directoryInfo = new DirectoryInfo(index); _directorySingleton = new SimpleFSDirectory(directoryInfo, LuceneLock); } return _directorySingleton; }
public Directory CreateDirectoryFor( string indexName ) { string indexDirectory = null; configuration.DirectoryPaths.TryGetValue( indexName, out indexDirectory ); indexDirectory = indexDirectory ?? System.IO.Path.Combine( configuration.IndexPath, indexName ); var directoryInfo = System.IO.Directory.CreateDirectory( indexDirectory ); var qualifiedDirectoryPath = System.IO.Path.GetFullPath( directoryInfo.ToString() ); var simpleFsDirectory = new SimpleFSDirectory( directoryInfo, new SimpleFSLockFactory() ); simpleFsDirectory.EnsureOpen(); return simpleFsDirectory; }
public void TestFSDirectorySync() { System.IO.FileInfo path = new System.IO.FileInfo(System.IO.Path.Combine(SupportClass.AppSettings.Get("tempDir", ""), "testsync")); Lucene.Net.Store.Directory directory = new Lucene.Net.Store.SimpleFSDirectory(path, null); try { Lucene.Net.Store.IndexOutput io = directory.CreateOutput("syncfile"); io.Close(); directory.Sync("syncfile"); } finally { directory.Close(); Lucene.Net.Util._TestUtil.RmDir(path); } }
public override bool OnStart() { var readPath = RoleEnvironment.GetLocalResource("ReadStorage").RootPath; var writePath = RoleEnvironment.GetLocalResource("WriteStorage").RootPath; var subscription = "lucene" ; var account = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("Blob")); var hub = new Hub(RoleEnvironment.GetConfigurationSettingValue("ServiceBus")); var criticalWait = Task.Run(() => { string writeFolder = Path.Combine(writePath, "lucene"); if (!System.IO.Directory.Exists(writeFolder)) { System.IO.Directory.CreateDirectory(writeFolder); } string combine = Path.Combine(readPath, subscription); if (!System.IO.Directory.Exists(combine)) { System.IO.Directory.CreateDirectory(combine); } var localDirectory = new Lucene.Net.Store.SimpleFSDirectory(new DirectoryInfo(combine)); var masterDirectory = new AzureDirectory(account, subscription); var irs = new IntermediateReaderService(masterDirectory, hub, localDirectory, true); masterDirectory.Dispose(); return new IndexInfo {IRS = irs as IDisposable, Directory = localDirectory}; }) .ContinueWith(t => _localIndexes = new[] {t.Result}); _luceneWriter = new SearchWriterService(new AzureDirectory(account, "lucene", new Lucene.Net.Store.SimpleFSDirectory(new DirectoryInfo(writePath))), true); var subscriptionName = string.Format("{0:yyyyMMddHHmmss}_{1}", DateTime.UtcNow, Guid.NewGuid().ToString().Replace("-", string.Empty)); hub.CreateSubscription("lucene", subscriptionName) .ContinueWith(t => { _luceneTempInstanceSubscription = t.Result; return hub.SubscribeWithRoutingKey("lucene", subscriptionName, OnSearchMessage) .ContinueWith(t2 => _luceneSearchSubscription = t.Result); }) .Unwrap(); return base.OnStart(); }
public virtual void TestCopySubdir() { //File path = CreateTempDir("testsubdir"); var path = new DirectoryInfo(Path.Combine(AppSettings.Get("tempDir", ""), "testsubdir")); try { //path.mkdirs(); System.IO.Directory.CreateDirectory(path.FullName); //(new File(path, "subdir")).mkdirs(); System.IO.Directory.CreateDirectory(new DirectoryInfo(Path.Combine(path.FullName, "subdir")).FullName); Directory fsDir = new SimpleFSDirectory(path, null); Assert.AreEqual(0, (new RAMDirectory(fsDir, NewIOContext(Random()))).ListAll().Length); } finally { System.IO.Directory.Delete(path.FullName, true); } }
static void Main(string[] args) { string indeksYolu = @"D:\LuceneDocumentStatic"; Directory.CreateDirectory(indeksYolu); ld.Directory indexed = new ld.SimpleFSDirectory(new DirectoryInfo(indeksYolu)); // indeks'in oluşturulacağı klasörü oluşturuyoruz. // Projeyi çalıştırdıktan sonra bu adrese gidip oluşan dosyaları görebilirsiniz. // İndeksleme yapıyoruz. Projeyi bir kere çalıştırdıktan sonra tekrar çalıştırırsanız ikinci defa indeksleme yapacaktır. // Bu da aynı ürünü n defa indexlemiş olmak anlamına geliyor. o yüzden ilk çalıştırmadan sonra bu kısmı commentlemeniz gerekiyor. //Indeksleme(indexed); SqlArama("NTV"); LuceneArama(indexed, "NTV"); Console.ReadLine(); }
public override void Act() { var indexDirectory = Path.Combine(CoreConfiguration.DataDirectory, "MergedIndexes"); var directoryInfo = new DirectoryInfo(indexDirectory); if(directoryInfo.Exists) { directoryInfo.Delete(true); directoryInfo.Refresh(); } var mergedDirectory = new SimpleFSDirectory(directoryInfo); var mergedIndex = new IndexWriter(mergedDirectory, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); var directoryFactory = AutofacContainer.Resolve<IDirectoryFactory>(); mergedIndex.AddIndexes(directoryFactory.GetAllDirectories().Select(d => IndexReader.Open(d, true)).ToArray()); mergedIndex.Commit(); }
public virtual void TestDontCreate() { var parentFolder = CreateTempDir(this.GetType().Name.ToLowerInvariant()); var path = new DirectoryInfo(Path.Combine(parentFolder.FullName, "doesnotexist")); try { Assert.IsTrue(!path.Exists); Directory dir = new SimpleFSDirectory(path, null); Assert.IsTrue(!path.Exists); dir.Dispose(); } finally { if (path.Exists) { System.IO.Directory.Delete(path.FullName, true); } } }
public virtual void TestDefaultFSLockFactoryPrefix() { // Make sure we get null prefix, which wont happen if setLockFactory is ever called. DirectoryInfo dirName = CreateTempDir("TestLockFactory.10"); Directory dir = new SimpleFSDirectory(dirName); Assert.IsNull(dir.LockFactory.LockPrefix, "Default lock prefix should be null"); dir.Dispose(); dir = new MMapDirectory(dirName); Assert.IsNull(dir.LockFactory.LockPrefix, "Default lock prefix should be null"); dir.Dispose(); dir = new NIOFSDirectory(dirName); Assert.IsNull(dir.LockFactory.LockPrefix, "Default lock prefix should be null"); dir.Dispose(); System.IO.Directory.Delete(dirName.FullName, true); }
static void Main(string[] args) { if (args.Length < 2) { PrintUsage(); return; } var sqlConn = args[0]; var localPath = args[1]; var includeUnlisted = (args.Length > 2 ? bool.Parse(args[3]) : false); if (!System.IO.Directory.Exists(localPath)) { System.IO.Directory.CreateDirectory(localPath); } var directory = new SimpleFSDirectory(new DirectoryInfo(localPath)); PackageIndexing.RebuildIndex(sqlConn, directory, Console.Out, null, includeUnlisted: includeUnlisted); }
public virtual void TestNotDirectory() { System.IO.DirectoryInfo path = new System.IO.DirectoryInfo(System.IO.Path.Combine(AppSettings.Get("tempDir", ""), "testnotdir")); Directory fsDir = new SimpleFSDirectory(path, null); try { IndexOutput out_Renamed = fsDir.CreateOutput("afile", null); out_Renamed.Close(); Assert.IsTrue(fsDir.FileExists("afile", null)); Assert.Throws <NoSuchDirectoryException>( () => new SimpleFSDirectory(new System.IO.DirectoryInfo(System.IO.Path.Combine(path.FullName, "afile")), null), "did not hit expected exception"); } finally { fsDir.Close(); _TestUtil.RmDir(path); } }
public DataTable Search(Analyzer analyzer, string keyword) { if (string.IsNullOrEmpty(keyword)) { throw new ArgumentException("keyword"); } // 计时 Stopwatch watch = Stopwatch.StartNew(); // 设置 int numHits = 10; bool docsScoredInOrder = true; bool isReadOnly = true; // 创建Searcher FSDirectory fsDir = new SimpleFSDirectory(new DirectoryInfo(_indexerFolder)); IndexSearcher indexSearcher = new IndexSearcher(IndexReader.Open(fsDir, isReadOnly)); TopScoreDocCollector collector = TopScoreDocCollector.Create(numHits, docsScoredInOrder); QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, LuceneConfig.Field_ContentByPage, analyzer); Query query = parser.Parse(keyword); indexSearcher.Search(query, collector); //Console.WriteLine(collector.TotalHits); var result = collector.TopDocs().ScoreDocs; DataTable dt = ConvertToDataTable(indexSearcher, result); // dispose IndexSearcher indexSearcher.Dispose(); watch.Stop(); WriteLog("总共耗时{0}毫秒", watch.ElapsedMilliseconds); WriteLog("总共找到{0}个文件", result.Count()); return dt; }
static async Task BuildSearchIndexFromCatalog(string catalogPath, string resolverBlobPath, string dest, bool resetIndex) { Directory dir = new Lucene.Net.Store.SimpleFSDirectory(new System.IO.DirectoryInfo(dest)); if (resetIndex) { PackageIndexing.CreateNewEmptyIndex(dir); } var catalog = new SearchIndexFromCatalogCollector(dir, "{0}/{1}.json"); if (resolverBlobPath != null) { catalog.DependentCollections = new List <Uri> { new Uri(resolverBlobPath) }; } CollectorCursor cursor; //try //{ // IndexWriter writer = SearchIndexFromCatalogCollector.CreateIndexWriter(dir, false); //} //catch //{ cursor = CollectorCursor.None; //} CollectorCursor finalCursor = await catalog.Run( new NuGet.Services.Metadata.Catalog.Collecting.CollectorHttpClient(), new Uri(catalogPath), cursor); return; }
static void Main(string[] args) { SqlConnectionString = args[0]; AccountName = args[1]; KeyValue = args[2]; Path = args[3]; try { DirectoryInfo path = new DirectoryInfo(Path); if (!path.Exists) { path.Create(); } Lucene.Net.Store.Directory directory = new SimpleFSDirectory(path); PackageIndexing.RebuildIndex(SqlConnectionString, directory, Console.Out); } catch (Exception e) { Console.WriteLine(e.Message); } }
public virtual void TestDirectInstantiation() { System.IO.DirectoryInfo path = new System.IO.DirectoryInfo(AppSettings.Get("tempDir", System.IO.Path.GetTempPath())); int sz = 2; Directory[] dirs = new Directory[sz]; dirs[0] = new SimpleFSDirectory(path, null); // dirs[1] = new NIOFSDirectory(path, null); System.Console.WriteLine("Skipping NIOFSDirectory() test under Lucene.Net"); dirs[1] = new MMapDirectory(path, null); for (int i = 0; i < sz; i++) { Directory dir = dirs[i]; dir.EnsureOpen(); System.String fname = "foo." + i; System.String lockname = "foo" + i + ".lck"; IndexOutput out_Renamed = dir.CreateOutput(fname); out_Renamed.WriteByte((byte) i); out_Renamed.Close(); for (int j = 0; j < sz; j++) { Directory d2 = dirs[j]; d2.EnsureOpen(); Assert.IsTrue(d2.FileExists(fname)); Assert.AreEqual(1, d2.FileLength(fname)); // don't test read on MMapDirectory, since it can't really be // closed and will cause a failure to delete the file. if (d2 is MMapDirectory) continue; IndexInput input = d2.OpenInput(fname); Assert.AreEqual((byte) i, input.ReadByte()); input.Close(); } // delete with a different dir dirs[(i + 1) % sz].DeleteFile(fname); for (int j = 0; j < sz; j++) { Directory d2 = dirs[j]; Assert.IsFalse(d2.FileExists(fname)); } Lock lock_Renamed = dir.MakeLock(lockname); Assert.IsTrue(lock_Renamed.Obtain()); for (int j = 0; j < sz; j++) { Directory d2 = dirs[j]; Lock lock2 = d2.MakeLock(lockname); try { Assert.IsFalse(lock2.Obtain(1)); } catch (LockObtainFailedException) { // OK } } lock_Renamed.Release(); // now lock with different dir lock_Renamed = dirs[(i + 1) % sz].MakeLock(lockname); Assert.IsTrue(lock_Renamed.Obtain()); lock_Renamed.Release(); } for (int i = 0; i < sz; i++) { Directory dir = dirs[i]; dir.EnsureOpen(); dir.Close(); Assert.IsFalse(dir.isOpen_ForNUnit); } }
public virtual void TestNotDirectory() { System.IO.DirectoryInfo path = new System.IO.DirectoryInfo(System.IO.Path.Combine(AppSettings.Get("tempDir", ""), "testnotdir")); Directory fsDir = new SimpleFSDirectory(path, null); try { IndexOutput out_Renamed = fsDir.CreateOutput("afile"); out_Renamed.Close(); Assert.IsTrue(fsDir.FileExists("afile")); Assert.Throws<NoSuchDirectoryException>( () => new SimpleFSDirectory(new System.IO.DirectoryInfo(System.IO.Path.Combine(path.FullName, "afile")), null), "did not hit expected exception"); } finally { fsDir.Close(); _TestUtil.RmDir(path); } }
public virtual void TestCopySubdir() { System.IO.DirectoryInfo path = new System.IO.DirectoryInfo(System.IO.Path.Combine(AppSettings.Get("tempDir", ""), "testsubdir")); try { System.IO.Directory.CreateDirectory(path.FullName); System.IO.Directory.CreateDirectory(new System.IO.DirectoryInfo(System.IO.Path.Combine(path.FullName, "subdir")).FullName); Directory fsDir = new SimpleFSDirectory(path, null); Assert.AreEqual(0, new RAMDirectory(fsDir).ListAll().Length); } finally { _TestUtil.RmDir(path); } }
public virtual void TestDontCreate() { System.IO.DirectoryInfo path = new System.IO.DirectoryInfo(System.IO.Path.Combine(AppSettings.Get("tempDir", ""), "doesnotexist")); try { bool tmpBool; if (System.IO.File.Exists(path.FullName)) tmpBool = true; else tmpBool = System.IO.Directory.Exists(path.FullName); Assert.IsTrue(!tmpBool); Directory dir = new SimpleFSDirectory(path, null); bool tmpBool2; if (System.IO.File.Exists(path.FullName)) tmpBool2 = true; else tmpBool2 = System.IO.Directory.Exists(path.FullName); Assert.IsTrue(!tmpBool2); dir.Close(); } finally { _TestUtil.RmDir(path); } }
private Directory NewFSSwitchDirectory(DirectoryInfo aDir, DirectoryInfo bDir, ISet<string> primaryExtensions) { Directory a = new SimpleFSDirectory(aDir); Directory b = new SimpleFSDirectory(bDir); FileSwitchDirectory switchDir = new FileSwitchDirectory(primaryExtensions, a, b, true); return new MockDirectoryWrapper(Random(), switchDir); }
public virtual void TestDefaultFSLockFactoryPrefix() { // Make sure we get null prefix, which wont happen if setLockFactory is ever called. DirectoryInfo dirName = CreateTempDir("TestLockFactory.10"); Directory dir = new SimpleFSDirectory(dirName); Assert.IsNull(dir.LockFactory.LockPrefix, "Default lock prefix should be null"); dir.Dispose(); dir = new MMapDirectory(dirName); Assert.IsNull(dir.LockFactory.LockPrefix, "Default lock prefix should be null"); dir.Dispose(); dir = new NIOFSDirectory(dirName); Assert.IsNull(dir.LockFactory.LockPrefix, "Default lock prefix should be null"); dir.Dispose(); System.IO.Directory.Delete(dirName.FullName, true); }
public void Search(Analyzer analyzer, string keyword) { if (string.IsNullOrEmpty(keyword)) { throw new ArgumentException("content"); } // 计时 Stopwatch watch = new Stopwatch(); watch.Start(); // 设置 int numHits = 10; bool docsScoredInOrder = true; bool isReadOnly = true; // 创建Searcher FSDirectory fsDir = new SimpleFSDirectory(new DirectoryInfo(_indexerFolder)); IndexSearcher indexSearcher = new IndexSearcher(IndexReader.Open(fsDir, isReadOnly)); TopScoreDocCollector collector = TopScoreDocCollector.Create(numHits, docsScoredInOrder); QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, Config.Field_Content, analyzer); Query query = parser.Parse(keyword); indexSearcher.Search(query, collector); //Console.WriteLine(collector.TotalHits); var result = collector.TopDocs().ScoreDocs; watch.Stop(); Console.WriteLine("总共耗时{0}毫秒", watch.ElapsedMilliseconds); Console.WriteLine("总共找到{0}个文件", result.Count()); foreach (var docs in result) { Document doc = indexSearcher.Doc(docs.Doc); Console.WriteLine("得分:{0},文件名:{1}", docs.Score, doc.Get(Config.Field_Name)); } indexSearcher.Dispose(); //BooleanQuery booleanQuery = new BooleanQuery(); //QueryParser parser1 = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, Config.Field_Path, analyzer); //Query query1 = parser1.Parse(path); //parser1.DefaultOperator = QueryParser.Operator.AND; //booleanQuery.Add(query1, Occur.MUST); ////QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, Config.Field_Content, analyzer); ////Query query = parser.Parse(content); ////parser.DefaultOperator = QueryParser.Operator.AND; ////booleanQuery.Add(query, Occur.MUST); ////var queryParserFilePath = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, new string[] { Config.Field_FilePath }, analyzer); ////query.Add(queryParserFilePath.Parse(path), Occur.SHOULD); ////var queryParserContent = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, new string[] { Config.Field_Content}, analyzer); ////query.Add(queryParserContent.Parse(content), Occur.MUST); //FSDirectory fsDir = new SimpleFSDirectory(new DirectoryInfo(_indexerFolder)); //bool isReadOnly = true; //IndexSearcher indexSearcher = new IndexSearcher(IndexReader.Open(fsDir, isReadOnly)); ////TopDocs result = indexSearcher.Search(booleanQuery, 10); //Sort sort = new Sort(new SortField(Config.Field_Path, SortField.STRING, true)); //var result = indexSearcher.Search(booleanQuery, (Filter)null, 10 * 1, sort); }
/// <summary> /// Loads the data into the Lucene index /// </summary> /// <param name="directory"> /// Directory where the index is located. /// </param> private void LoadLuceneIndex(SimpleFSDirectory directory) { // Create an analyzer that uses UpperCaseLetterOrDigitAnalyzer for all fields, but UpperCaseKeywordAnalyzer for ProductCode // (because we want to regard product codes as 1 word). var analyzer = new PerFieldAnalyzerWrapper(new UpperCaseLetterOrDigitAnalyzer()); analyzer.AddAnalyzer("ProductCode", new UpperCaseKeywordAnalyzer()); // ----------- // Store products into Lucene. // This will create a new index. Other requests will still be able to read the existing index. // Create writer that will overwrite the existing index using (var writer = new IndexWriter(directory, analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED)) { IEnumerable<ProductSearchResult> results = _productRepository.GetAllProductSearchResults(); foreach (var result in results) { var doc = new Document(); doc.Add(new Field("ProductId", result.ProductId.ToString(CultureInfo.InvariantCulture), Field.Store.YES, Field.Index.NO)); // Store field in index so it can be searched, but don't analyse it - just store as is. var productCodeField = new Field("ProductCode", result.ProductCode, Field.Store.YES, Field.Index.ANALYZED); doc.Add(productCodeField); doc.Add(new Field("ProductDescription", result.ProductDescription, Field.Store.YES, Field.Index.ANALYZED)); writer.AddDocument(doc); } } }
private static void KeepEmailsWithGoodNames(RayCommandOptions rayCommandOptions, List<OneRawContactsListCsvRow> rows) { var names = File.ReadAllLines(rayCommandOptions.NamesFile).ToList(); var directory = Guid.NewGuid().ToString(); var simpleFsDirectory = new SimpleFSDirectory(new DirectoryInfo(directory)); if (!Directory.Exists(directory)) { var standardAnalyzer = new StandardAnalyzer(Version.LUCENE_30); var indexer = new IndexWriter(simpleFsDirectory, standardAnalyzer, true, IndexWriter.MaxFieldLength.UNLIMITED); var st = new Stopwatch(); st.Start(); var counter = 0; rows.ForEach(x => { var document = new Document(); document.Add(new Field("address", x.Email.Split('@')[0], Field.Store.YES, Field.Index.ANALYZED)); document.Add(new Field("collectionIndex", counter.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED)); indexer.AddDocument(document); counter++; }); indexer.Commit(); st.Stop(); WriteToConsole("Index took: " + st.ElapsedMilliseconds / 1000); } var reader = IndexReader.Open(simpleFsDirectory, true); var searcher = new IndexSearcher(reader); var st2 = new Stopwatch(); st2.Start(); Parallel.ForEach(names, x => { var ids = Search(searcher, "address", "*" + x + "*"); if (!ids.Any()) { // Console.WriteLine("for " + x + " there were no ids found."); return; } ids.ToList().ForEach(p => { rows[p].Mark = true; }); }); var emailsWithNames = rows.AsParallel().Where(x => x.Mark).ToList(); var emailsWithoutNames = rows.AsParallel().Where(x => !x.Mark).Select(x => x.Email).ToList(); st2.Stop(); WriteToConsole("Processing all names took: " + st2.ElapsedMilliseconds / 1000); File.WriteAllLines(rayCommandOptions.OutputFile + ".emails.without.names.txt", emailsWithoutNames); WriteCsv(rayCommandOptions, emailsWithNames); }
private static List<OneRawContactsListCsvRow> RemoveRowsByDomains(List<OneRawContactsListCsvRow> rows, List<string> removeDomains) { var directory = Guid.NewGuid().ToString(); var simpleFsDirectory = new SimpleFSDirectory(new DirectoryInfo(directory)); if (!Directory.Exists(directory)) { var standardAnalyzer = new WhitespaceAnalyzer(); var indexer = new IndexWriter(simpleFsDirectory, standardAnalyzer, true, IndexWriter.MaxFieldLength.UNLIMITED); var st = new Stopwatch(); st.Start(); var counter = 0; rows.ForEach(x => { var document = new Document(); document.Add(new Field("email", x.Email.Replace("@", " "), Field.Store.YES, Field.Index.ANALYZED)); document.Add(new Field("collectionIndex", counter.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED)); indexer.AddDocument(document); counter++; }); indexer.Commit(); st.Stop(); WriteToConsole("Index took: " + st.ElapsedMilliseconds / 1000); } var reader = IndexReader.Open(simpleFsDirectory, true); var searcher = new IndexSearcher(reader); var st2 = new Stopwatch(); st2.Start(); removeDomains.ForEach(x => { var ids = Search(searcher, "email", x); if (!ids.Any()) Console.WriteLine("for " + x + " there were no ids found."); ids.ToList().ForEach(p => { rows[p].Mark = true; }); }); var oneRawContactsListCsvRows = rows.AsParallel().Where(x => !x.Mark).ToList(); st2.Stop(); WriteToConsole("Removing took: " + st2.ElapsedMilliseconds / 1000); return oneRawContactsListCsvRows; }
public virtual void TestDirectInstantiation() { System.IO.DirectoryInfo path = new System.IO.DirectoryInfo(AppSettings.Get("tempDir", System.IO.Path.GetTempPath())); int sz = 2; Directory[] dirs = new Directory[sz]; dirs[0] = new SimpleFSDirectory(path, null); // dirs[1] = new NIOFSDirectory(path, null); System.Console.WriteLine("Skipping NIOFSDirectory() test under Lucene.Net"); dirs[1] = new MMapDirectory(path, null); for (int i = 0; i < sz; i++) { Directory dir = dirs[i]; dir.EnsureOpen(); System.String fname = "foo." + i; System.String lockname = "foo" + i + ".lck"; IndexOutput out_Renamed = dir.CreateOutput(fname, null); out_Renamed.WriteByte((byte)i); out_Renamed.Close(); for (int j = 0; j < sz; j++) { Directory d2 = dirs[j]; d2.EnsureOpen(); Assert.IsTrue(d2.FileExists(fname, null)); Assert.AreEqual(1, d2.FileLength(fname, null)); // don't test read on MMapDirectory, since it can't really be // closed and will cause a failure to delete the file. if (d2 is MMapDirectory) { continue; } IndexInput input = d2.OpenInput(fname, null); Assert.AreEqual((byte)i, input.ReadByte(null)); input.Close(); } // delete with a different dir dirs[(i + 1) % sz].DeleteFile(fname, null); for (int j = 0; j < sz; j++) { Directory d2 = dirs[j]; Assert.IsFalse(d2.FileExists(fname, null)); } Lock lock_Renamed = dir.MakeLock(lockname); Assert.IsTrue(lock_Renamed.Obtain()); for (int j = 0; j < sz; j++) { Directory d2 = dirs[j]; Lock lock2 = d2.MakeLock(lockname); try { Assert.IsFalse(lock2.Obtain(1)); } catch (LockObtainFailedException) { // OK } } lock_Renamed.Release(); // now lock with different dir lock_Renamed = dirs[(i + 1) % sz].MakeLock(lockname); Assert.IsTrue(lock_Renamed.Obtain()); lock_Renamed.Release(); } for (int i = 0; i < sz; i++) { Directory dir = dirs[i]; dir.EnsureOpen(); dir.Close(); Assert.IsFalse(dir.isOpen_ForNUnit); } }
public virtual void TestFsyncDoesntCreateNewFiles() { var path = CreateTempDir("nocreate"); Console.WriteLine(path.FullName); using (Directory fsdir = new SimpleFSDirectory(path)) { // write a file using (var o = fsdir.CreateOutput("afile", NewIOContext(Random()))) { o.WriteString("boo"); } // delete it try { File.Delete(Path.Combine(path.FullName, "afile")); } catch (Exception e) { Assert.Fail("Deletion of new Directory should never fail.\nException thrown: {0}", e); } // directory is empty Assert.AreEqual(0, fsdir.ListAll().Length); // fsync it try { fsdir.Sync(Collections.Singleton("afile")); Assert.Fail("didn't get expected exception, instead fsync created new files: " + Arrays.AsList(fsdir.ListAll())); } catch (FileNotFoundException) { // ok } // directory is still empty Assert.AreEqual(0, fsdir.ListAll().Length); } }
public virtual void TestNotDirectory() { DirectoryInfo path = CreateTempDir("testnotdir"); Directory fsDir = new SimpleFSDirectory(path, null); try { IndexOutput @out = fsDir.CreateOutput("afile", NewIOContext(Random())); @out.Dispose(); Assert.IsTrue(SlowFileExists(fsDir, "afile")); try { var d = new SimpleFSDirectory(new DirectoryInfo(Path.Combine(path.FullName, "afile")), null); Assert.Fail("did not hit expected exception"); } catch (NoSuchDirectoryException nsde) { // Expected } } finally { fsDir.Dispose(); System.IO.Directory.Delete(path.FullName, true); } }
public virtual void TestDontCreate() { var parentFolder = CreateTempDir(this.GetType().Name.ToLowerInvariant()); var path = new DirectoryInfo(Path.Combine(parentFolder.FullName, "doesnotexist")); try { Assert.IsTrue(!path.Exists); Directory dir = new SimpleFSDirectory(path, null); Assert.IsTrue(!path.Exists); dir.Dispose(); } finally { if (path.Exists) System.IO.Directory.Delete(path.FullName, true); } }