public IndexInputSlicerAnonymousInnerClassHelper(SimpleFSDirectory outerInstance, IOContext context, FileInfo file, FileStream descriptor)
: base(outerInstance)
{
this.Context = context;
this.File = file;
this.Descriptor = descriptor;
}
public void CreateIndex(Analyzer analayer)
{
FSDirectory fsDir = new SimpleFSDirectory(new DirectoryInfo(_indexerFolder));
IndexWriter indexWriter = new IndexWriter(fsDir, analayer, true, Lucene.Net.Index.IndexWriter.MaxFieldLength.UNLIMITED);
string[] files = System.IO.Directory.GetFiles(_textFilesFolder, Config.FileSearchPattern, SearchOption.AllDirectories);
foreach (string file in files)
{
string name = new FileInfo(file).Name;
string content = File.ReadAllText(file);
Document doc = new Document();
doc.Add(new Field(Config.Field_Path, file, Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.Add(new Field(Config.Field_Name, name, Field.Store.YES, Field.Index.ANALYZED));
doc.Add(new Field(Config.Field_Content, content, Field.Store.NO, Field.Index.ANALYZED));
indexWriter.AddDocument(doc);
Console.WriteLine("{0} - {1}", file, name);
}
indexWriter.Optimize();
indexWriter.Dispose();
Console.WriteLine("File count: {0}", files.Length);
}
public Directory DirectoryFor(string id, bool persistent)
{
if (persistent)
{
var info = new DirectoryInfo(Path.Combine(_root.FullName, id+".index"));
var directory = new SimpleFSDirectory(info);
if (!info.Exists || !info.EnumerateFiles().Any())
{
new IndexWriter(directory, new StandardAnalyzer(Version.LUCENE_29),
true,
IndexWriter.MaxFieldLength.UNLIMITED)
.Dispose();
}
return directory;
}
else
{
if (!_inMemoryDirectories.ContainsKey(id))
{
_inMemoryDirectories[id] = new RAMDirectory();
}
return _inMemoryDirectories[id];
}
}
/// <summary>
/// Ensures that the data has been loaded in the Lucene index
///
/// SIDE EFFECT
/// This method sets property _searcher
/// </summary>
/// <param name="absoluteLucenePath">
/// Absolute path of the directory where the Lucene files get stored.
/// </param>
/// <param name="onlyIfNotExists">
/// If true, the index will only be created if there is no index at all (that is, no Lucene directory).
/// If false, this will always create a new index.
/// </param>
/// <param name="loadLuceneIndex">
/// Actually loads data into the Lucene index
/// </param>
protected void LoadStore(string absoluteLucenePath, bool onlyIfNotExists, Action<SimpleFSDirectory> loadLuceneIndex)
{
var luceneDir = new DirectoryInfo(absoluteLucenePath);
bool luceneDirExists = luceneDir.Exists;
if (!luceneDirExists)
{
luceneDir.Create();
luceneDir.Refresh();
}
var directory = new SimpleFSDirectory(luceneDir);
if (onlyIfNotExists && luceneDirExists)
{
// Make sure to always create a searcher.
// There must actually be a valid Lucene index in the directory, otherwise this will throw an exception.
_searcher = new IndexSearcher(directory, true);
return;
}
loadLuceneIndex(directory);
// Now that the index has been updated, need to open a new searcher.
// Existing searcher will still be reading the previous index.
_searcher = new IndexSearcher(directory, true);
}
private Lucene.Net.Store.Directory CreateLuceneIndex(IEnumerable<PackageVersion> packages, string luceneDirectory)
{
Lucene.Net.Store.Directory directory;
if (luceneDirectory != null)
{
var directoryInfo = new DirectoryInfo(luceneDirectory);
directoryInfo.Create();
directory = new SimpleFSDirectory(directoryInfo);
}
else
{
directory = new RAMDirectoryWrapper();
}
using (var indexWriter = DocumentCreator.CreateIndexWriter(directory, true))
{
foreach (var version in packages)
{
var metadata = GetPackageMetadata(version);
var document = DocumentCreator.CreateDocument(metadata);
indexWriter.AddDocument(document);
}
indexWriter.Commit();
}
return directory;
}
public override Task<bool> Run()
{
FrameworksList frameworksList = new StorageFrameworksList(DataStorageAccount, DataContainerName, FrameworksList.FileName);
Lucene.Net.Store.Directory directory = new SimpleFSDirectory(new DirectoryInfo(LocalIndexFolder));
PackageIndexing.RebuildIndex(PackageDatabase.ConnectionString, directory, frameworksList, Console.Out);
return Task.FromResult(true);
}
public void CreateIndex(Analyzer analayer)
{
FSDirectory fsDir = new SimpleFSDirectory(new DirectoryInfo(_indexerFolder));
IndexWriter indexWriter = new IndexWriter(fsDir, analayer, true, Lucene.Net.Index.IndexWriter.MaxFieldLength.UNLIMITED);
Stopwatch stopWatch = Stopwatch.StartNew();
int analyzedCount = 0;
string[] files = System.IO.Directory.GetFiles(_textFilesFolder, this._fileSearchPattern, SearchOption.AllDirectories);
//统计需要索引的文件页数
int totalPages = GetTotalPages(files);
WriteLog("Total pages statistics takes {0}ms", stopWatch.Elapsed.Milliseconds);
stopWatch.Restart();
TextAbsorber textAbsorber = new TextAbsorber();
//开始索引
foreach (string pdfFile in files)
{
var fileInfo = new FileInfo(pdfFile);
var fileName = fileInfo.Name;
Aspose.Pdf.Document pdfDocument = new Aspose.Pdf.Document(pdfFile);
WriteLog("Current file is {0}", pdfFile);
//注意pdf页码从1开始
for (int i = 1;i<=pdfDocument.Pages.Count;i++)
{
Page page = pdfDocument.Pages[i];
page.Accept(textAbsorber);
string pageContent = textAbsorber.Text;
Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document();
doc.Add(new Field(LuceneConfig.Field_Path, pdfFile, Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.Add(new Field(LuceneConfig.Field_FileName, fileName, Field.Store.YES, Field.Index.ANALYZED));
doc.Add(new Field(LuceneConfig.Field_PageNumber, i.ToString(), Field.Store.YES, Field.Index.ANALYZED));
doc.Add(new Field(LuceneConfig.Field_ContentByPage, pageContent, Field.Store.NO, Field.Index.ANALYZED));
indexWriter.AddDocument(doc);
analyzedCount++;
RaiseProgressChanged(analyzedCount * 100 / totalPages);
}
}
indexWriter.Optimize();
indexWriter.Dispose();
stopWatch.Stop();
Console.WriteLine("All completed. It takes {0}ms", stopWatch.Elapsed);
}
protected override void Context()
{
var dir = new SimpleFSDirectory(new DirectoryInfo(TempDirectory), new NoLockFactory());
var analyzer = new StandardAnalyzer(Version.LUCENE_20);
var maxFieldLength = new IndexWriter.MaxFieldLength(200);
var index = new FluentIndexWriter<TestObject>(dir, analyzer, maxFieldLength);
var data = new TestObject() { Id = Guid.NewGuid(), LongId = 123, ValidProperty = "Property", IgnoredProperty = "Ignored" };
var data2 = new TestObject() { Id = Guid.Empty, LongId = 123456, ValidProperty = "Abc def ghij", IgnoredProperty = "Ignored" };
index.AddDocument(data);
index.AddDocument(data2);
index.Commit();
}
internal static Lucene.Net.Store.Directory GetDirectory()
{
if (_directorySingleton == null)
{
if (!System.IO.Directory.Exists(IndexDirectory))
{
System.IO.Directory.CreateDirectory(IndexDirectory);
}
var directoryInfo = new DirectoryInfo(IndexDirectory);
_directorySingleton = new SimpleFSDirectory(directoryInfo);
}
return _directorySingleton;
}
public override bool OnStart()
{
var readPath = RoleEnvironment.GetLocalResource("ReadStorage").RootPath;
var writePath = RoleEnvironment.GetLocalResource("WriteStorage").RootPath;
var subscription = "lucene";
var account = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("Blob"));
var hub = new Hub(RoleEnvironment.GetConfigurationSettingValue("ServiceBus"));
var criticalWait = Task.Run(() =>
{
string writeFolder = Path.Combine(writePath, "lucene");
if (!System.IO.Directory.Exists(writeFolder))
{
System.IO.Directory.CreateDirectory(writeFolder);
}
string combine = Path.Combine(readPath, subscription);
if (!System.IO.Directory.Exists(combine))
{
System.IO.Directory.CreateDirectory(combine);
}
var localDirectory = new Lucene.Net.Store.SimpleFSDirectory(new DirectoryInfo(combine));
var masterDirectory = new AzureDirectory(account, subscription);
var irs = new IntermediateReaderService(masterDirectory, hub, localDirectory, true);
masterDirectory.Dispose();
return(new IndexInfo {
IRS = irs as IDisposable, Directory = localDirectory
});
})
.ContinueWith(t => _localIndexes = new[] { t.Result });
_luceneWriter = new SearchWriterService(new AzureDirectory(account, "lucene", new Lucene.Net.Store.SimpleFSDirectory(new DirectoryInfo(writePath))), true);
var subscriptionName = string.Format("{0:yyyyMMddHHmmss}_{1}", DateTime.UtcNow, Guid.NewGuid().ToString().Replace("-", string.Empty));
hub.CreateSubscription("lucene", subscriptionName)
.ContinueWith(t =>
{
_luceneTempInstanceSubscription = t.Result;
return(hub.SubscribeWithRoutingKey("lucene", subscriptionName, OnSearchMessage)
.ContinueWith(t2 => _luceneSearchSubscription = t.Result));
})
.Unwrap();
return(base.OnStart());
}
internal static Lucene.Net.Store.Directory GetDirectory(LuceneIndexLocation location)
{
if (_directorySingleton == null)
{
var index = GetIndexLocation(location);
if (!System.IO.Directory.Exists(index))
{
System.IO.Directory.CreateDirectory(index);
}
var directoryInfo = new DirectoryInfo(index);
_directorySingleton = new SimpleFSDirectory(directoryInfo, LuceneLock);
}
return _directorySingleton;
}
public Directory CreateDirectoryFor( string indexName )
{
string indexDirectory = null;
configuration.DirectoryPaths.TryGetValue( indexName, out indexDirectory );
indexDirectory = indexDirectory ?? System.IO.Path.Combine( configuration.IndexPath, indexName );
var directoryInfo = System.IO.Directory.CreateDirectory( indexDirectory );
var qualifiedDirectoryPath = System.IO.Path.GetFullPath( directoryInfo.ToString() );
var simpleFsDirectory = new SimpleFSDirectory(
directoryInfo,
new SimpleFSLockFactory() );
simpleFsDirectory.EnsureOpen();
return simpleFsDirectory;
}
public void TestFSDirectorySync()
{
System.IO.FileInfo path = new System.IO.FileInfo(System.IO.Path.Combine(SupportClass.AppSettings.Get("tempDir", ""), "testsync"));
Lucene.Net.Store.Directory directory = new Lucene.Net.Store.SimpleFSDirectory(path, null);
try
{
Lucene.Net.Store.IndexOutput io = directory.CreateOutput("syncfile");
io.Close();
directory.Sync("syncfile");
}
finally
{
directory.Close();
Lucene.Net.Util._TestUtil.RmDir(path);
}
}
public override bool OnStart()
{
var readPath = RoleEnvironment.GetLocalResource("ReadStorage").RootPath;
var writePath = RoleEnvironment.GetLocalResource("WriteStorage").RootPath;
var subscription = "lucene" ;
var account = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("Blob"));
var hub = new Hub(RoleEnvironment.GetConfigurationSettingValue("ServiceBus"));
var criticalWait = Task.Run(() =>
{
string writeFolder = Path.Combine(writePath, "lucene");
if (!System.IO.Directory.Exists(writeFolder))
{
System.IO.Directory.CreateDirectory(writeFolder);
}
string combine = Path.Combine(readPath, subscription);
if (!System.IO.Directory.Exists(combine))
{
System.IO.Directory.CreateDirectory(combine);
}
var localDirectory = new Lucene.Net.Store.SimpleFSDirectory(new DirectoryInfo(combine));
var masterDirectory = new AzureDirectory(account, subscription);
var irs = new IntermediateReaderService(masterDirectory, hub, localDirectory, true);
masterDirectory.Dispose();
return new IndexInfo {IRS = irs as IDisposable, Directory = localDirectory};
})
.ContinueWith(t => _localIndexes = new[] {t.Result});
_luceneWriter = new SearchWriterService(new AzureDirectory(account, "lucene", new Lucene.Net.Store.SimpleFSDirectory(new DirectoryInfo(writePath))), true);
var subscriptionName = string.Format("{0:yyyyMMddHHmmss}_{1}", DateTime.UtcNow, Guid.NewGuid().ToString().Replace("-", string.Empty));
hub.CreateSubscription("lucene", subscriptionName)
.ContinueWith(t =>
{
_luceneTempInstanceSubscription = t.Result;
return hub.SubscribeWithRoutingKey("lucene", subscriptionName, OnSearchMessage)
.ContinueWith(t2 => _luceneSearchSubscription = t.Result);
})
.Unwrap();
return base.OnStart();
}
public virtual void TestCopySubdir()
{
//File path = CreateTempDir("testsubdir");
var path = new DirectoryInfo(Path.Combine(AppSettings.Get("tempDir", ""), "testsubdir"));
try
{
//path.mkdirs();
System.IO.Directory.CreateDirectory(path.FullName);
//(new File(path, "subdir")).mkdirs();
System.IO.Directory.CreateDirectory(new DirectoryInfo(Path.Combine(path.FullName, "subdir")).FullName);
Directory fsDir = new SimpleFSDirectory(path, null);
Assert.AreEqual(0, (new RAMDirectory(fsDir, NewIOContext(Random()))).ListAll().Length);
}
finally
{
System.IO.Directory.Delete(path.FullName, true);
}
}
static void Main(string[] args)
{
string indeksYolu = @"D:\LuceneDocumentStatic";
Directory.CreateDirectory(indeksYolu);
ld.Directory indexed = new ld.SimpleFSDirectory(new DirectoryInfo(indeksYolu));
// indeks'in oluşturulacağı klasörü oluşturuyoruz.
// Projeyi çalıştırdıktan sonra bu adrese gidip oluşan dosyaları görebilirsiniz. // İndeksleme yapıyoruz. Projeyi bir kere çalıştırdıktan sonra tekrar çalıştırırsanız ikinci defa indeksleme yapacaktır.
// Bu da aynı ürünü n defa indexlemiş olmak anlamına geliyor. o yüzden ilk çalıştırmadan sonra bu kısmı commentlemeniz gerekiyor.
//Indeksleme(indexed);
SqlArama("NTV");
LuceneArama(indexed, "NTV");
Console.ReadLine();
}
public override void Act()
{
var indexDirectory = Path.Combine(CoreConfiguration.DataDirectory, "MergedIndexes");
var directoryInfo = new DirectoryInfo(indexDirectory);
if(directoryInfo.Exists)
{
directoryInfo.Delete(true);
directoryInfo.Refresh();
}
var mergedDirectory = new SimpleFSDirectory(directoryInfo);
var mergedIndex = new IndexWriter(mergedDirectory, new SimpleAnalyzer(), true,
IndexWriter.MaxFieldLength.UNLIMITED);
var directoryFactory = AutofacContainer.Resolve<IDirectoryFactory>();
mergedIndex.AddIndexes(directoryFactory.GetAllDirectories().Select(d => IndexReader.Open(d, true)).ToArray());
mergedIndex.Commit();
}
public virtual void TestDontCreate()
{
var parentFolder = CreateTempDir(this.GetType().Name.ToLowerInvariant());
var path = new DirectoryInfo(Path.Combine(parentFolder.FullName, "doesnotexist"));
try
{
Assert.IsTrue(!path.Exists);
Directory dir = new SimpleFSDirectory(path, null);
Assert.IsTrue(!path.Exists);
dir.Dispose();
}
finally
{
if (path.Exists)
{
System.IO.Directory.Delete(path.FullName, true);
}
}
}
public virtual void TestDefaultFSLockFactoryPrefix()
{
// Make sure we get null prefix, which wont happen if setLockFactory is ever called.
DirectoryInfo dirName = CreateTempDir("TestLockFactory.10");
Directory dir = new SimpleFSDirectory(dirName);
Assert.IsNull(dir.LockFactory.LockPrefix, "Default lock prefix should be null");
dir.Dispose();
dir = new MMapDirectory(dirName);
Assert.IsNull(dir.LockFactory.LockPrefix, "Default lock prefix should be null");
dir.Dispose();
dir = new NIOFSDirectory(dirName);
Assert.IsNull(dir.LockFactory.LockPrefix, "Default lock prefix should be null");
dir.Dispose();
System.IO.Directory.Delete(dirName.FullName, true);
}
static void Main(string[] args)
{
if (args.Length < 2)
{
PrintUsage();
return;
}
var sqlConn = args[0];
var localPath = args[1];
var includeUnlisted = (args.Length > 2 ? bool.Parse(args[3]) : false);
if (!System.IO.Directory.Exists(localPath))
{
System.IO.Directory.CreateDirectory(localPath);
}
var directory = new SimpleFSDirectory(new DirectoryInfo(localPath));
PackageIndexing.RebuildIndex(sqlConn, directory, Console.Out, null, includeUnlisted: includeUnlisted);
}
public virtual void TestNotDirectory()
{
System.IO.DirectoryInfo path = new System.IO.DirectoryInfo(System.IO.Path.Combine(AppSettings.Get("tempDir", ""), "testnotdir"));
Directory fsDir = new SimpleFSDirectory(path, null);
try
{
IndexOutput out_Renamed = fsDir.CreateOutput("afile", null);
out_Renamed.Close();
Assert.IsTrue(fsDir.FileExists("afile", null));
Assert.Throws <NoSuchDirectoryException>(
() =>
new SimpleFSDirectory(new System.IO.DirectoryInfo(System.IO.Path.Combine(path.FullName, "afile")), null),
"did not hit expected exception");
}
finally
{
fsDir.Close();
_TestUtil.RmDir(path);
}
}
public DataTable Search(Analyzer analyzer, string keyword)
{
if (string.IsNullOrEmpty(keyword))
{
throw new ArgumentException("keyword");
}
// 计时
Stopwatch watch = Stopwatch.StartNew();
// 设置
int numHits = 10;
bool docsScoredInOrder = true;
bool isReadOnly = true;
// 创建Searcher
FSDirectory fsDir = new SimpleFSDirectory(new DirectoryInfo(_indexerFolder));
IndexSearcher indexSearcher = new IndexSearcher(IndexReader.Open(fsDir, isReadOnly));
TopScoreDocCollector collector = TopScoreDocCollector.Create(numHits, docsScoredInOrder);
QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, LuceneConfig.Field_ContentByPage, analyzer);
Query query = parser.Parse(keyword);
indexSearcher.Search(query, collector);
//Console.WriteLine(collector.TotalHits);
var result = collector.TopDocs().ScoreDocs;
DataTable dt = ConvertToDataTable(indexSearcher, result);
// dispose IndexSearcher
indexSearcher.Dispose();
watch.Stop();
WriteLog("总共耗时{0}毫秒", watch.ElapsedMilliseconds);
WriteLog("总共找到{0}个文件", result.Count());
return dt;
}
static async Task BuildSearchIndexFromCatalog(string catalogPath, string resolverBlobPath, string dest, bool resetIndex)
{
Directory dir = new Lucene.Net.Store.SimpleFSDirectory(new System.IO.DirectoryInfo(dest));
if (resetIndex)
{
PackageIndexing.CreateNewEmptyIndex(dir);
}
var catalog = new SearchIndexFromCatalogCollector(dir, "{0}/{1}.json");
if (resolverBlobPath != null)
{
catalog.DependentCollections = new List <Uri> {
new Uri(resolverBlobPath)
};
}
CollectorCursor cursor;
//try
//{
// IndexWriter writer = SearchIndexFromCatalogCollector.CreateIndexWriter(dir, false);
//}
//catch
//{
cursor = CollectorCursor.None;
//}
CollectorCursor finalCursor = await catalog.Run(
new NuGet.Services.Metadata.Catalog.Collecting.CollectorHttpClient(),
new Uri(catalogPath),
cursor);
return;
}
static void Main(string[] args)
{
SqlConnectionString = args[0];
AccountName = args[1];
KeyValue = args[2];
Path = args[3];
try
{
DirectoryInfo path = new DirectoryInfo(Path);
if (!path.Exists)
{
path.Create();
}
Lucene.Net.Store.Directory directory = new SimpleFSDirectory(path);
PackageIndexing.RebuildIndex(SqlConnectionString, directory, Console.Out);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
public virtual void TestDirectInstantiation()
{
System.IO.DirectoryInfo path = new System.IO.DirectoryInfo(AppSettings.Get("tempDir", System.IO.Path.GetTempPath()));
int sz = 2;
Directory[] dirs = new Directory[sz];
dirs[0] = new SimpleFSDirectory(path, null);
// dirs[1] = new NIOFSDirectory(path, null);
System.Console.WriteLine("Skipping NIOFSDirectory() test under Lucene.Net");
dirs[1] = new MMapDirectory(path, null);
for (int i = 0; i < sz; i++)
{
Directory dir = dirs[i];
dir.EnsureOpen();
System.String fname = "foo." + i;
System.String lockname = "foo" + i + ".lck";
IndexOutput out_Renamed = dir.CreateOutput(fname);
out_Renamed.WriteByte((byte) i);
out_Renamed.Close();
for (int j = 0; j < sz; j++)
{
Directory d2 = dirs[j];
d2.EnsureOpen();
Assert.IsTrue(d2.FileExists(fname));
Assert.AreEqual(1, d2.FileLength(fname));
// don't test read on MMapDirectory, since it can't really be
// closed and will cause a failure to delete the file.
if (d2 is MMapDirectory)
continue;
IndexInput input = d2.OpenInput(fname);
Assert.AreEqual((byte) i, input.ReadByte());
input.Close();
}
// delete with a different dir
dirs[(i + 1) % sz].DeleteFile(fname);
for (int j = 0; j < sz; j++)
{
Directory d2 = dirs[j];
Assert.IsFalse(d2.FileExists(fname));
}
Lock lock_Renamed = dir.MakeLock(lockname);
Assert.IsTrue(lock_Renamed.Obtain());
for (int j = 0; j < sz; j++)
{
Directory d2 = dirs[j];
Lock lock2 = d2.MakeLock(lockname);
try
{
Assert.IsFalse(lock2.Obtain(1));
}
catch (LockObtainFailedException)
{
// OK
}
}
lock_Renamed.Release();
// now lock with different dir
lock_Renamed = dirs[(i + 1) % sz].MakeLock(lockname);
Assert.IsTrue(lock_Renamed.Obtain());
lock_Renamed.Release();
}
for (int i = 0; i < sz; i++)
{
Directory dir = dirs[i];
dir.EnsureOpen();
dir.Close();
Assert.IsFalse(dir.isOpen_ForNUnit);
}
}
public virtual void TestNotDirectory()
{
System.IO.DirectoryInfo path = new System.IO.DirectoryInfo(System.IO.Path.Combine(AppSettings.Get("tempDir", ""), "testnotdir"));
Directory fsDir = new SimpleFSDirectory(path, null);
try
{
IndexOutput out_Renamed = fsDir.CreateOutput("afile");
out_Renamed.Close();
Assert.IsTrue(fsDir.FileExists("afile"));
Assert.Throws<NoSuchDirectoryException>(
() =>
new SimpleFSDirectory(new System.IO.DirectoryInfo(System.IO.Path.Combine(path.FullName, "afile")), null),
"did not hit expected exception");
}
finally
{
fsDir.Close();
_TestUtil.RmDir(path);
}
}
public virtual void TestCopySubdir()
{
System.IO.DirectoryInfo path = new System.IO.DirectoryInfo(System.IO.Path.Combine(AppSettings.Get("tempDir", ""), "testsubdir"));
try
{
System.IO.Directory.CreateDirectory(path.FullName);
System.IO.Directory.CreateDirectory(new System.IO.DirectoryInfo(System.IO.Path.Combine(path.FullName, "subdir")).FullName);
Directory fsDir = new SimpleFSDirectory(path, null);
Assert.AreEqual(0, new RAMDirectory(fsDir).ListAll().Length);
}
finally
{
_TestUtil.RmDir(path);
}
}
public virtual void TestDontCreate()
{
System.IO.DirectoryInfo path = new System.IO.DirectoryInfo(System.IO.Path.Combine(AppSettings.Get("tempDir", ""), "doesnotexist"));
try
{
bool tmpBool;
if (System.IO.File.Exists(path.FullName))
tmpBool = true;
else
tmpBool = System.IO.Directory.Exists(path.FullName);
Assert.IsTrue(!tmpBool);
Directory dir = new SimpleFSDirectory(path, null);
bool tmpBool2;
if (System.IO.File.Exists(path.FullName))
tmpBool2 = true;
else
tmpBool2 = System.IO.Directory.Exists(path.FullName);
Assert.IsTrue(!tmpBool2);
dir.Close();
}
finally
{
_TestUtil.RmDir(path);
}
}
private Directory NewFSSwitchDirectory(DirectoryInfo aDir, DirectoryInfo bDir, ISet<string> primaryExtensions)
{
Directory a = new SimpleFSDirectory(aDir);
Directory b = new SimpleFSDirectory(bDir);
FileSwitchDirectory switchDir = new FileSwitchDirectory(primaryExtensions, a, b, true);
return new MockDirectoryWrapper(Random(), switchDir);
}
public virtual void TestDefaultFSLockFactoryPrefix()
{
// Make sure we get null prefix, which wont happen if setLockFactory is ever called.
DirectoryInfo dirName = CreateTempDir("TestLockFactory.10");
Directory dir = new SimpleFSDirectory(dirName);
Assert.IsNull(dir.LockFactory.LockPrefix, "Default lock prefix should be null");
dir.Dispose();
dir = new MMapDirectory(dirName);
Assert.IsNull(dir.LockFactory.LockPrefix, "Default lock prefix should be null");
dir.Dispose();
dir = new NIOFSDirectory(dirName);
Assert.IsNull(dir.LockFactory.LockPrefix, "Default lock prefix should be null");
dir.Dispose();
System.IO.Directory.Delete(dirName.FullName, true);
}
public void Search(Analyzer analyzer, string keyword)
{
if (string.IsNullOrEmpty(keyword))
{
throw new ArgumentException("content");
}
// 计时
Stopwatch watch = new Stopwatch();
watch.Start();
// 设置
int numHits = 10;
bool docsScoredInOrder = true;
bool isReadOnly = true;
// 创建Searcher
FSDirectory fsDir = new SimpleFSDirectory(new DirectoryInfo(_indexerFolder));
IndexSearcher indexSearcher = new IndexSearcher(IndexReader.Open(fsDir, isReadOnly));
TopScoreDocCollector collector = TopScoreDocCollector.Create(numHits, docsScoredInOrder);
QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, Config.Field_Content, analyzer);
Query query = parser.Parse(keyword);
indexSearcher.Search(query, collector);
//Console.WriteLine(collector.TotalHits);
var result = collector.TopDocs().ScoreDocs;
watch.Stop();
Console.WriteLine("总共耗时{0}毫秒", watch.ElapsedMilliseconds);
Console.WriteLine("总共找到{0}个文件", result.Count());
foreach (var docs in result)
{
Document doc = indexSearcher.Doc(docs.Doc);
Console.WriteLine("得分：{0}，文件名：{1}", docs.Score, doc.Get(Config.Field_Name));
}
indexSearcher.Dispose();
//BooleanQuery booleanQuery = new BooleanQuery();
//QueryParser parser1 = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, Config.Field_Path, analyzer);
//Query query1 = parser1.Parse(path);
//parser1.DefaultOperator = QueryParser.Operator.AND;
//booleanQuery.Add(query1, Occur.MUST);
////QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, Config.Field_Content, analyzer);
////Query query = parser.Parse(content);
////parser.DefaultOperator = QueryParser.Operator.AND;
////booleanQuery.Add(query, Occur.MUST);
////var queryParserFilePath = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, new string[] { Config.Field_FilePath }, analyzer);
////query.Add(queryParserFilePath.Parse(path), Occur.SHOULD);
////var queryParserContent = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, new string[] { Config.Field_Content}, analyzer);
////query.Add(queryParserContent.Parse(content), Occur.MUST);
//FSDirectory fsDir = new SimpleFSDirectory(new DirectoryInfo(_indexerFolder));
//bool isReadOnly = true;
//IndexSearcher indexSearcher = new IndexSearcher(IndexReader.Open(fsDir, isReadOnly));
////TopDocs result = indexSearcher.Search(booleanQuery, 10);
//Sort sort = new Sort(new SortField(Config.Field_Path, SortField.STRING, true));
//var result = indexSearcher.Search(booleanQuery, (Filter)null, 10 * 1, sort);
}
/// <summary>
/// Loads the data into the Lucene index
/// </summary>
/// <param name="directory">
/// Directory where the index is located.
/// </param>
private void LoadLuceneIndex(SimpleFSDirectory directory)
{
// Create an analyzer that uses UpperCaseLetterOrDigitAnalyzer for all fields, but UpperCaseKeywordAnalyzer for ProductCode
// (because we want to regard product codes as 1 word).
var analyzer = new PerFieldAnalyzerWrapper(new UpperCaseLetterOrDigitAnalyzer());
analyzer.AddAnalyzer("ProductCode", new UpperCaseKeywordAnalyzer());
// -----------
// Store products into Lucene.
// This will create a new index. Other requests will still be able to read the existing index.
// Create writer that will overwrite the existing index
using (var writer = new IndexWriter(directory, analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED))
{
IEnumerable<ProductSearchResult> results = _productRepository.GetAllProductSearchResults();
foreach (var result in results)
{
var doc = new Document();
doc.Add(new Field("ProductId", result.ProductId.ToString(CultureInfo.InvariantCulture), Field.Store.YES, Field.Index.NO));
// Store field in index so it can be searched, but don't analyse it - just store as is.
var productCodeField = new Field("ProductCode", result.ProductCode, Field.Store.YES, Field.Index.ANALYZED);
doc.Add(productCodeField);
doc.Add(new Field("ProductDescription", result.ProductDescription, Field.Store.YES, Field.Index.ANALYZED));
writer.AddDocument(doc);
}
}
}
private static void KeepEmailsWithGoodNames(RayCommandOptions rayCommandOptions, List<OneRawContactsListCsvRow> rows)
{
var names = File.ReadAllLines(rayCommandOptions.NamesFile).ToList();
var directory = Guid.NewGuid().ToString();
var simpleFsDirectory = new SimpleFSDirectory(new DirectoryInfo(directory));
if (!Directory.Exists(directory))
{
var standardAnalyzer = new StandardAnalyzer(Version.LUCENE_30);
var indexer = new IndexWriter(simpleFsDirectory, standardAnalyzer, true, IndexWriter.MaxFieldLength.UNLIMITED);
var st = new Stopwatch();
st.Start();
var counter = 0;
rows.ForEach(x =>
{
var document = new Document();
document.Add(new Field("address", x.Email.Split('@')[0], Field.Store.YES, Field.Index.ANALYZED));
document.Add(new Field("collectionIndex", counter.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
indexer.AddDocument(document);
counter++;
});
indexer.Commit();
st.Stop();
WriteToConsole("Index took: " + st.ElapsedMilliseconds / 1000);
}
var reader = IndexReader.Open(simpleFsDirectory, true);
var searcher = new IndexSearcher(reader);
var st2 = new Stopwatch();
st2.Start();
Parallel.ForEach(names, x =>
{
var ids = Search(searcher, "address", "*" + x + "*");
if (!ids.Any())
{
// Console.WriteLine("for " + x + " there were no ids found.");
return;
}
ids.ToList().ForEach(p => { rows[p].Mark = true; });
});
var emailsWithNames = rows.AsParallel().Where(x => x.Mark).ToList();
var emailsWithoutNames = rows.AsParallel().Where(x => !x.Mark).Select(x => x.Email).ToList();
st2.Stop();
WriteToConsole("Processing all names took: " + st2.ElapsedMilliseconds / 1000);
File.WriteAllLines(rayCommandOptions.OutputFile + ".emails.without.names.txt", emailsWithoutNames);
WriteCsv(rayCommandOptions, emailsWithNames);
}
private static List<OneRawContactsListCsvRow> RemoveRowsByDomains(List<OneRawContactsListCsvRow> rows, List<string> removeDomains)
{
var directory = Guid.NewGuid().ToString();
var simpleFsDirectory = new SimpleFSDirectory(new DirectoryInfo(directory));
if (!Directory.Exists(directory))
{
var standardAnalyzer = new WhitespaceAnalyzer();
var indexer = new IndexWriter(simpleFsDirectory, standardAnalyzer, true, IndexWriter.MaxFieldLength.UNLIMITED);
var st = new Stopwatch();
st.Start();
var counter = 0;
rows.ForEach(x =>
{
var document = new Document();
document.Add(new Field("email", x.Email.Replace("@", " "), Field.Store.YES, Field.Index.ANALYZED));
document.Add(new Field("collectionIndex", counter.ToString(), Field.Store.YES, Field.Index.NOT_ANALYZED));
indexer.AddDocument(document);
counter++;
});
indexer.Commit();
st.Stop();
WriteToConsole("Index took: " + st.ElapsedMilliseconds / 1000);
}
var reader = IndexReader.Open(simpleFsDirectory, true);
var searcher = new IndexSearcher(reader);
var st2 = new Stopwatch();
st2.Start();
removeDomains.ForEach(x =>
{
var ids = Search(searcher, "email", x);
if (!ids.Any())
Console.WriteLine("for " + x + " there were no ids found.");
ids.ToList().ForEach(p =>
{
rows[p].Mark = true;
});
});
var oneRawContactsListCsvRows = rows.AsParallel().Where(x => !x.Mark).ToList();
st2.Stop();
WriteToConsole("Removing took: " + st2.ElapsedMilliseconds / 1000);
return oneRawContactsListCsvRows;
}
public virtual void TestDirectInstantiation()
{
System.IO.DirectoryInfo path = new System.IO.DirectoryInfo(AppSettings.Get("tempDir", System.IO.Path.GetTempPath()));
int sz = 2;
Directory[] dirs = new Directory[sz];
dirs[0] = new SimpleFSDirectory(path, null);
// dirs[1] = new NIOFSDirectory(path, null);
System.Console.WriteLine("Skipping NIOFSDirectory() test under Lucene.Net");
dirs[1] = new MMapDirectory(path, null);
for (int i = 0; i < sz; i++)
{
Directory dir = dirs[i];
dir.EnsureOpen();
System.String fname = "foo." + i;
System.String lockname = "foo" + i + ".lck";
IndexOutput out_Renamed = dir.CreateOutput(fname, null);
out_Renamed.WriteByte((byte)i);
out_Renamed.Close();
for (int j = 0; j < sz; j++)
{
Directory d2 = dirs[j];
d2.EnsureOpen();
Assert.IsTrue(d2.FileExists(fname, null));
Assert.AreEqual(1, d2.FileLength(fname, null));
// don't test read on MMapDirectory, since it can't really be
// closed and will cause a failure to delete the file.
if (d2 is MMapDirectory)
{
continue;
}
IndexInput input = d2.OpenInput(fname, null);
Assert.AreEqual((byte)i, input.ReadByte(null));
input.Close();
}
// delete with a different dir
dirs[(i + 1) % sz].DeleteFile(fname, null);
for (int j = 0; j < sz; j++)
{
Directory d2 = dirs[j];
Assert.IsFalse(d2.FileExists(fname, null));
}
Lock lock_Renamed = dir.MakeLock(lockname);
Assert.IsTrue(lock_Renamed.Obtain());
for (int j = 0; j < sz; j++)
{
Directory d2 = dirs[j];
Lock lock2 = d2.MakeLock(lockname);
try
{
Assert.IsFalse(lock2.Obtain(1));
}
catch (LockObtainFailedException)
{
// OK
}
}
lock_Renamed.Release();
// now lock with different dir
lock_Renamed = dirs[(i + 1) % sz].MakeLock(lockname);
Assert.IsTrue(lock_Renamed.Obtain());
lock_Renamed.Release();
}
for (int i = 0; i < sz; i++)
{
Directory dir = dirs[i];
dir.EnsureOpen();
dir.Close();
Assert.IsFalse(dir.isOpen_ForNUnit);
}
}
public virtual void TestFsyncDoesntCreateNewFiles()
{
var path = CreateTempDir("nocreate");
Console.WriteLine(path.FullName);
using (Directory fsdir = new SimpleFSDirectory(path))
{
// write a file
using (var o = fsdir.CreateOutput("afile", NewIOContext(Random())))
{
o.WriteString("boo");
}
// delete it
try
{
File.Delete(Path.Combine(path.FullName, "afile"));
}
catch (Exception e)
{
Assert.Fail("Deletion of new Directory should never fail.\nException thrown: {0}", e);
}
// directory is empty
Assert.AreEqual(0, fsdir.ListAll().Length);
// fsync it
try
{
fsdir.Sync(Collections.Singleton("afile"));
Assert.Fail("didn't get expected exception, instead fsync created new files: " +
Arrays.AsList(fsdir.ListAll()));
}
catch (FileNotFoundException)
{
// ok
}
// directory is still empty
Assert.AreEqual(0, fsdir.ListAll().Length);
}
}
public virtual void TestNotDirectory()
{
DirectoryInfo path = CreateTempDir("testnotdir");
Directory fsDir = new SimpleFSDirectory(path, null);
try
{
IndexOutput @out = fsDir.CreateOutput("afile", NewIOContext(Random()));
@out.Dispose();
Assert.IsTrue(SlowFileExists(fsDir, "afile"));
try
{
var d = new SimpleFSDirectory(new DirectoryInfo(Path.Combine(path.FullName, "afile")), null);
Assert.Fail("did not hit expected exception");
}
catch (NoSuchDirectoryException nsde)
{
// Expected
}
}
finally
{
fsDir.Dispose();
System.IO.Directory.Delete(path.FullName, true);
}
}
public virtual void TestDontCreate()
{
var parentFolder = CreateTempDir(this.GetType().Name.ToLowerInvariant());
var path = new DirectoryInfo(Path.Combine(parentFolder.FullName, "doesnotexist"));
try
{
Assert.IsTrue(!path.Exists);
Directory dir = new SimpleFSDirectory(path, null);
Assert.IsTrue(!path.Exists);
dir.Dispose();
}
finally
{
if (path.Exists) System.IO.Directory.Delete(path.FullName, true);
}
}
