public Directory GetDirectory(DirectoryInfo dir, bool throwIfEmpty)
{
if (throwIfEmpty)
{
Directory d;
if (!_directories.TryGetValue(dir.FullName, out d))
{
throw new NullReferenceException("No directory was added with path " + dir.FullName + ", maybe an indexer hasn't been initialized?");
}
return(d);
}
var resolved = _directories.GetOrAdd(dir.FullName, s =>
{
var simpleFsDirectory = new SimpleFSDirectory(dir);
simpleFsDirectory.SetLockFactory(DirectoryTracker.DefaultLockFactory(dir));
return(simpleFsDirectory);
});
return(resolved);
}
static void MainV2()
{
var numberOfSuggestion = 100;
var testFilePath = @"C:/lucene/test1.txt";
var testDictionaryPath = @"C:/lucene/ruStem.dict";
var testIndexPath = @"C:/lucene/indexV2";
var stopWordsPath = @"C:/lucene/stopWords.txt";
var stopWordsSet = new HashSet <string>();
using (var reader = new StreamReader(stopWordsPath))
{
while (!reader.EndOfStream)
{
stopWordsSet.Add(reader.ReadLine());
}
}
using (var reader = new StreamReader(testFilePath))
{
var directory = new SimpleFSDirectory(new DirectoryInfo(testIndexPath));
var spellChecker = new SpellChecker.Net.Search.Spell.SpellChecker(directory);
spellChecker.IndexDictionary(new PlainTextDictionary(new FileInfo(testDictionaryPath)));
StringDistance getDist = spellChecker.GetStringDistance();
var analyzer = new StemmerCompareAnalyzer(stopWordsSet, spellChecker, numberOfSuggestion);
var stream = analyzer.TokenStream(null, reader);
while (stream.IncrementToken())
{
var termAttribute = stream.GetAttribute <ITermAttribute>().Term;
var spellAttribute = stream.GetAttribute <ISpellAttribute>().Term;
var stemAttribute = stream.GetAttribute <IStemAttribute>().Term;
Console.WriteLine("{0, 20} {1, 20} {2, 20}", termAttribute, spellAttribute, stemAttribute);
}
}
}
public List <LinkItem> Search(IEnumerable <int> idArr)
{
if (!Directory.Exists(LuceneCommon.IndexOutDoorDirectory))
{
return(new List <LinkItem>());
}
var combineQuery = new BooleanQuery();
var queryList = idArr.Select(x => new TermQuery(new Term(OutDoorIndexFields.ID, x.ToString()))).ToArray();
foreach (var q in queryList)
{
combineQuery.Add(q, Occur.SHOULD);
}
#region 用户状态
var memberStatusQuery = NumericRangeQuery.NewIntRange(OutDoorIndexFields.MemberStatus, (int)MemberStatus.CompanyAuth, 99, true, true);
combineQuery.Add(memberStatusQuery, Occur.MUST);
#endregion
#region 审核状态查询构建
var verifyStatus = NumericRangeQuery.NewIntRange(OutDoorIndexFields.Status, (int)OutDoorStatus.ShowOnline, 99, true, true);
combineQuery.Add(verifyStatus, Occur.MUST);
#endregion
using (var directory = new SimpleFSDirectory(new DirectoryInfo(LuceneCommon.IndexOutDoorDirectory)))
{
var searcher = new IndexSearcher(directory, readOnly: true);
var results = searcher.Search(combineQuery, idArr.Count());
var keys = results.ScoreDocs.Skip(0)
.Select(c => GetMediaItem(searcher.Doc(c.Doc)))
.ToList();
searcher.Dispose();
return(keys);
}
}
protected override void Context()
{
var dir = new SimpleFSDirectory(new DirectoryInfo(TempDirectory), new NoLockFactory());
var analyzer = new StandardAnalyzer(Version.LUCENE_20);
var maxFieldLength = new IndexWriter.MaxFieldLength(200);
var index = new FluentIndexWriter <TestObject>(dir, analyzer, maxFieldLength);
var data = new TestObject()
{
Id = Guid.NewGuid(), LongId = 123, ValidProperty = "Property", IgnoredProperty = "Ignored"
};
var data2 = new TestObject()
{
Id = Guid.Empty, LongId = 123456, ValidProperty = "Abc def ghij", IgnoredProperty = "Ignored"
};
index.AddDocument(data);
index.AddDocument(data2);
index.Commit();
}
public override void Act()
{
var indexDirectory = Path.Combine(CoreConfiguration.DataDirectory, "MergedIndexes");
var directoryInfo = new DirectoryInfo(indexDirectory);
if (directoryInfo.Exists)
{
directoryInfo.Delete(true);
directoryInfo.Refresh();
}
var mergedDirectory = new SimpleFSDirectory(directoryInfo);
var mergedIndex = new IndexWriter(mergedDirectory, new SimpleAnalyzer(), true,
IndexWriter.MaxFieldLength.UNLIMITED);
var directoryFactory = AutofacContainer.Resolve <IDirectoryFactory>();
mergedIndex.AddIndexes(directoryFactory.GetAllDirectories().Select(d => IndexReader.Open(d, true)).ToArray());
mergedIndex.Commit();
}
public void Index(string contentPath, List <string> foldersToExclude)
{
if (foldersToExclude == null)
{
throw new ArgumentException("Must not have null collection of folders", "foldersToExclude");
}
var indexDirectory = new SimpleFSDirectory(new DirectoryInfo(_configuration.IndexPath));
Log(string.Format("Begining to index {0}. Index location: {1}", contentPath, indexDirectory.Directory.FullName));
var stopWatch = new Stopwatch();
stopWatch.Start();
var analyzer = AnalyzerBuilder.CreateAnalyzer();
using (var writer = new IndexWriter(indexDirectory, analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED))
{
IndexDirectory(writer, new DirectoryInfo(contentPath), foldersToExclude);
}
stopWatch.Stop();
Log(string.Format("Indexed {0:N0} files in {1:00}:{2:00}.{3:00}", _fileCount, stopWatch.Elapsed.Hours, stopWatch.Elapsed.Minutes, stopWatch.Elapsed.Seconds));
}
public int RemoveLibrary(string database, string id)
{
using (var analyzer = GetAnalyzer())
using (var directory = new SimpleFSDirectory(new DirectoryInfo(database)))
using (var writer = new IndexWriter(directory, analyzer, IndexWriter.MaxFieldLength.UNLIMITED))
{
var query = new TermQuery(new Term("_library_id", id));
int total = 0;
using (var searcher = new IndexSearcher(directory))
{
var docs = searcher.Search(query, 1);
total = docs.TotalHits;
}
writer.DeleteDocuments(query);
writer.Optimize(true);
return(total);
}
}
public virtual Directory CreateFileSystemLuceneDirectory(string folderName)
{
var combinedPath = Path.Combine("App_Data/TEMP", "ExamineIndexes", folderName);
var baseDirectory = AppContext.BaseDirectory;
var currentPath = Path.Combine(AppContext.BaseDirectory, combinedPath);
var dirInfo = new DirectoryInfo(currentPath);
if (!dirInfo.Exists)
{
System.IO.Directory.CreateDirectory(dirInfo.FullName);
}
string configuredDirectoryFactory =
_configuration["Novicell.Examine.LuceneDirectoryFactory"];
if (!string.IsNullOrWhiteSpace(configuredDirectoryFactory))
{
/* TODO: Check with Mikkel if is better way for TypeFinder */
var factoryType = ExamineTypeFinder.GetTypeByName(configuredDirectoryFactory);
if (factoryType == null)
{
throw new NullReferenceException("No directory type found for value: " +
configuredDirectoryFactory);
}
var directoryFactory = (IDirectoryFactory)this.Resolve <IDirectoryFactory>(factoryType);
return(directoryFactory.CreateDirectory(dirInfo, true));
}
//no dir factory, just create a normal fs directory
var luceneDir = new SimpleFSDirectory(dirInfo);
//we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the appdomain
//terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock
//which simply checks the existence of the lock file
// The full syntax of this is: new NoPrefixSimpleFsLockFactory(dirInfo)
// however, we are setting the DefaultLockFactory in startup so we'll use that instead since it can be managed globally.
luceneDir.SetLockFactory(DirectoryFactory.DefaultLockFactory(dirInfo));
return(luceneDir);
}
static LuceneProvider()
{
_appDataFolder = new DirectoryInfo(HostingEnvironment.MapPath("~/App_Data/BlogIndex"));
var directory = new SimpleFSDirectory(_appDataFolder);
IndexWriter writer;
FSDirectory indexFSDir = FSDirectory.Open(_appDataFolder, new Lucene.Net.Store.SimpleFSLockFactory(_appDataFolder));
try
{
writer = new IndexWriter(directory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), IndexWriter.MaxFieldLength.UNLIMITED);
}
catch (LockObtainFailedException ex)
{
IndexWriter.Unlock(indexFSDir);
writer = new IndexWriter(directory, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30), IndexWriter.MaxFieldLength.UNLIMITED);
}
_instance = new LuceneDataProvider(directory, writer.Analyzer, Lucene.Net.Util.Version.LUCENE_30, writer);
_readOnlyInstance = new ReadOnlyLuceneDataProvider(directory, Lucene.Net.Util.Version.LUCENE_30);
}
static public void indexing(Sentence[] sentences, string indexPath = "luceneIndex")
{
if (System.IO.Directory.Exists(indexPath))
{
System.IO.DirectoryInfo di = new System.IO.DirectoryInfo(indexPath);
di.Delete(true);
}
java.io.File indexDir = new java.io.File(indexPath);
EnglishAnalyzer luceneAnalyzer = new EnglishAnalyzer(org.apache.lucene.util.Version.LUCENE_35);
IndexWriterConfig config = new IndexWriterConfig(org.apache.lucene.util.Version.LUCENE_35, luceneAnalyzer);
FSDirectory indexFSDir = new SimpleFSDirectory(indexDir);
IndexWriter writer = new IndexWriter(indexFSDir, config);
foreach (Sentence s in sentences)
{
Document doc = new Document();
doc.add(new Field("text", s.sentnece, Field.Store.YES, Field.Index.ANALYZED));
writer.addDocument(doc);
}
writer.close();
}
public List <LinkItem> Search(QueryTerm queryTerm, SearchFilter searchFilter, out int totalHits)
{
if (!Directory.Exists(LuceneCommon.IndexOutDoorDirectory))
{
totalHits = 0;
return(new List <LinkItem>());
}
SortField sortField = GetSortField(searchFilter);
var sortFieldArry = new List <SortField>()
{
new SortField(OutDoorIndexFields.SuggestStatus, SortField.INT, reverse: true)
};
sortFieldArry.Add(sortField);
int numRecords = searchFilter.Skip + searchFilter.Take;
using (var directory = new SimpleFSDirectory(new DirectoryInfo(LuceneCommon.IndexOutDoorDirectory)))
{
var searcher = new IndexSearcher(directory, readOnly: true);
var query = ParseQuery(queryTerm, searchFilter);
//var termQuery = new TermQuery(new Term(OutDoorIndexFields.ID, "1"));
var results = searcher.Search(query, filter: null, n: numRecords, sort: new Sort(sortFieldArry.ToArray()));
var keys = results.ScoreDocs.Skip(searchFilter.Skip)
.Select(c => GetMediaItem(searcher.Doc(c.Doc)))
.ToList();
totalHits = results.TotalHits;
searcher.Dispose();
return(keys);
}
}
public DataTable Search(Analyzer analyzer, string keyword)
{
if (string.IsNullOrEmpty(keyword))
{
throw new ArgumentException("keyword");
}
// 计时
Stopwatch watch = Stopwatch.StartNew();
// 设置
int numHits = 10;
bool docsScoredInOrder = true;
bool isReadOnly = true;
// 创建Searcher
FSDirectory fsDir = new SimpleFSDirectory(new DirectoryInfo(_indexerFolder));
IndexSearcher indexSearcher = new IndexSearcher(IndexReader.Open(fsDir, isReadOnly));
TopScoreDocCollector collector = TopScoreDocCollector.Create(numHits, docsScoredInOrder);
QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, LuceneConfig.Field_ContentByPage, analyzer);
Query query = parser.Parse(keyword);
indexSearcher.Search(query, collector);
//Console.WriteLine(collector.TotalHits);
var result = collector.TopDocs().ScoreDocs;
DataTable dt = ConvertToDataTable(indexSearcher, result);
// dispose IndexSearcher
indexSearcher.Dispose();
watch.Stop();
WriteLog("总共耗时{0}毫秒", watch.ElapsedMilliseconds);
WriteLog("总共找到{0}个文件", result.Count());
return(dt);
}
static void Test6()
{
SimpleFSDirectory directory = new SimpleFSDirectory(new DirectoryInfo(@"c:\data\index20151113int"));
//using (IndexReader reader = IndexReader.Open(directory, true))
//{
//}
using (IndexWriter writer = new IndexWriter(directory, new PackageAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED))
{
using (IndexReader reader = writer.GetReader())
{
Console.WriteLine(new IndexSearcher(writer.GetReader()).Search(new MatchAllDocsQuery(), 10).TotalHits);
Document document = new Document();
document.Add(new Field("test", "test", Field.Store.NO, Field.Index.NOT_ANALYZED));
writer.AddDocument(document);
writer.Commit();
Console.WriteLine(new IndexSearcher(writer.GetReader()).Search(new MatchAllDocsQuery(), 10).TotalHits);
}
}
}
public IEnumerable <Document> Search(string expression, bool allowLeadingWildcard = false, int maxResults = 100)
{
if (expression == null)
{
throw new ArgumentNullException(nameof(expression));
}
var directory = SimpleFSDirectory.Open(indexDirectory);
using (var indexReader = IndexReader.Open(directory, true)) {
using (var indexSearcher = new IndexSearcher(indexReader)) {
using (var analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30)) {
var parser = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, defaultFields.ToArray(), analyzer);
parser.AllowLeadingWildcard = allowLeadingWildcard;
//var parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, "headline", analyzer);
var query = parser.Parse(expression);
var hits = indexSearcher.Search(query, null, maxResults).ScoreDocs;
var docs = hits.Select(e => indexSearcher.Doc(e.Doc)).ToList(); // Need ToList as indexSearcher will be disposed...
return(docs);
}
}
}
}
partial void StartIndexing(NSObject sender)
{
var appPath = Environment.GetFolderPath(Environment.SpecialFolder.Personal);
DirectoryInfo indexDirectoryInfo = new DirectoryInfo(Path.Combine(appPath, "Search"));
using (var directory = new SimpleFSDirectory(indexDirectoryInfo))
{
var standardAnalyzer = new StandardAnalyzer(Version.LUCENE_30);
using (var writer = new IndexWriter(directory, standardAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED))
{
Document doc = new Document();
doc.Add(new Field("id", "1", Field.Store.YES, Field.Index.ANALYZED));
doc.Add(new Field("title", "The quick brown fox jumps over the lazy dog.", Field.Store.YES, Field.Index.ANALYZED));
writer.AddDocument(doc);
writer.Optimize(true);
writer.Flush(true, true, true);
}
var query = new BooleanQuery();
query.Add(new TermQuery(new Term("title", "fox")), Occur.MUST);
IndexSearcher searcher = new IndexSearcher(directory);
var results = searcher.Search(query, Int16.MaxValue);
using (var reader = IndexReader.Open(directory, true))
{
foreach (var result in results.ScoreDocs)
{
var doc = reader.Document(result.Doc);
Console.WriteLine("found document: {0}", doc.Get("id"));
}
}
}
}
private string Highlight(int numId, string pattern, string html)
{
if (!string.IsNullOrWhiteSpace(pattern))
{
using (Analyzer analyzer = new StandardAnalyzer(LuceneVersion.LUCENE_48))
using (Lucene.Net.Store.Directory index = new SimpleFSDirectory(Path.ChangeExtension(_bookFile.FullName,
Convert.ToInt32(LuceneVersion.LUCENE_48).ToString())))
using (IndexReader reader = DirectoryReader.Open(index))
{
Lucene.Net.Search.IndexSearcher searcher = new Lucene.Net.Search.IndexSearcher(reader);
Lucene.Net.Search.TopDocs docs = searcher.Search(
Lucene.Net.Search.NumericRangeQuery.NewInt32Range(nameof(TabHtmlText.NumId), numId, numId, true,
true), 1);
int docId = docs.ScoreDocs.First().Doc;
QueryScorer scorer =
new QueryScorer(new QueryParser(LuceneVersion.LUCENE_48, nameof(TabHtmlText.Html), analyzer)
.Parse(pattern));
Highlighter highlighter =
new Highlighter(new SimpleHTMLFormatter("<span style=\"background-color: yellow\">", "</span>"),
scorer)
{
TextFragmenter = new NullFragmenter()
};
using (TokenStream stream =
TokenSources.GetAnyTokenStream(reader, docId, nameof(TabHtmlText.Html), analyzer))
{
return(highlighter.GetBestFragment(stream, html));
}
}
}
return(html);
}
/// <summary>
/// Creates a file system based Lucene <see cref="Lucene.Net.Store.Directory"/> with the correct locking guidelines for Umbraco
/// </summary>
/// <param name="folderName">
/// The folder name to store the index (single word, not a fully qualified folder) (i.e. Internal)
/// </param>
/// <returns></returns>
public virtual Lucene.Net.Store.Directory CreateFileSystemLuceneDirectory(string folderName)
{
var dirInfo = new DirectoryInfo(Path.Combine(IOHelper.MapPath(SystemDirectories.Data), "TEMP", "ExamineIndexes", folderName));
if (!dirInfo.Exists)
{
System.IO.Directory.CreateDirectory(dirInfo.FullName);
}
//check if there's a configured directory factory, if so create it and use that to create the lucene dir
var configuredDirectoryFactory = ConfigurationManager.AppSettings["Umbraco.Examine.LuceneDirectoryFactory"];
if (!configuredDirectoryFactory.IsNullOrWhiteSpace())
{
//this should be a fully qualified type
var factoryType = TypeFinder.GetTypeByName(configuredDirectoryFactory);
if (factoryType == null)
{
throw new NullReferenceException("No directory type found for value: " + configuredDirectoryFactory);
}
var directoryFactory = (IDirectoryFactory)Activator.CreateInstance(factoryType);
return(directoryFactory.CreateDirectory(dirInfo));
}
//no dir factory, just create a normal fs directory
var luceneDir = new SimpleFSDirectory(dirInfo);
//we want to tell examine to use a different fs lock instead of the default NativeFSFileLock which could cause problems if the appdomain
//terminates and in some rare cases would only allow unlocking of the file if IIS is forcefully terminated. Instead we'll rely on the simplefslock
//which simply checks the existence of the lock file
// The full syntax of this is: new NoPrefixSimpleFsLockFactory(dirInfo)
// however, we are setting the DefaultLockFactory in startup so we'll use that instead since it can be managed globally.
luceneDir.SetLockFactory(DirectoryFactory.DefaultLockFactory(dirInfo));
return(luceneDir);
}
private IEnumerator IndexInternal <T>(IEnumerable <T> items, int timeSlice = 13, bool processYields = true)
{
if (items == null)
{
throw new ArgumentNullException(nameof(items));
}
var type = typeof(T);
if (!indexers.ContainsKey(type))
{
throw new ArgumentOutOfRangeException(nameof(items), "At least one index must be defined using DefineIndexTerm for a type before it can be indexed.");
}
var keyIndexer = indexers[type].PrimaryKey;
var definitions = indexers[type].Indexers;
var analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30);
var stopwatch = new Stopwatch();
stopwatch.Start();
var count = 0;
var notNullItems = items.Where(e => e != null);
var total = notNullItems.Count();
OnProgress("Indexing", 0, total);
var directory = SimpleFSDirectory.Open(indexDirectory);
var create = !IndexReader.IndexExists(directory);
using (var writer = new IndexWriter(directory, analyzer, create, IndexWriter.MaxFieldLength.LIMITED)) {
foreach (var item in notNullItems)
{
var doc = new Document();
Term term = null;
if (keyIndexer != null)
{
term = new Term(keyIndexer.Name, keyIndexer.Indexer(item));
}
foreach (var definition in definitions)
{
var value = definition.Indexer(item);
var field = new Field(definition.Name, value, definition.StoreType(), definition.IndexType());
doc.Add(field);
}
if (term != null)
{
writer.UpdateDocument(term, doc);
}
else
{
writer.AddDocument(doc);
}
if (stopwatch.ElapsedMilliseconds >= timeSlice)
{
OnProgress("Indexing", count, total);
if (processYields)
{
yield return(null);
}
stopwatch.Restart();
}
++count;
}
OnProgress("Indexing", count, total);
if (processYields)
{
yield return(null);
}
OnProgress("Optimizing", 0, 1);
if (processYields)
{
yield return(null);
}
writer.Optimize();
writer.Commit();
OnProgress("Optimizing", 1, 1);
}
}
protected List <Book> Search(Query query, int take, int skip, out int count)
{
Query = query.ToString();
watch = Stopwatch.StartNew();
var fields = new SortField[]
{
new SortField("title_sort", SortField.STRING, false),
new SortField("pubdate", SortField.STRING, true),
SortField.FIELD_SCORE
};
var sort = new Sort(fields);
using (var directory = new SimpleFSDirectory(new DirectoryInfo(Settings.Instance.DatabaseIndex)))
using (var searcher = new IndexSearcher(directory))
{
var docs = searcher.Search(query, null, skip + take, sort);
count = docs.TotalHits;
var books = new List <Book>();
for (int i = skip; i < docs.TotalHits; i++)
{
if (i > (skip + take) - 1)
{
break;
}
var doc = searcher.Doc(docs.ScoreDocs[i].Doc);
var authors = doc.GetFields("author_fullname")
.Select(x => x.StringValue.Split(','))
.Select(x => new Author {
FirstName = x[0], MiddleName = x[1], LastName = x[2]
});
var genres = doc.GetFields("genre")
.Select(x => x.StringValue)
.Select(x => GenreExtensions.Construct(x));
Cover cover = null;
var coverContentType = doc.Get("_cover_type");
if (coverContentType != null)
{
cover = new Cover
{
Data = doc.GetBinaryValue("_cover_data"),
ContentType = coverContentType
};
}
var meta = new List <MetaField>();
var docFields = doc.GetFields();
foreach (var f in docFields)
{
if (!KNOWN_FIELDS.Contains(f.Name, StringComparer.OrdinalIgnoreCase))
{
meta.Add(new MetaField {
Name = f.Name, Value = f.StringValue
});
}
}
var book = new Book
{
Id = Guid.Parse(doc.Get("guid")),
LibraryId = Guid.Parse(doc.Get("_library_id")),
UpdatedFromFile = bool.Parse(doc.Get("_updated_from_file")),
UpdatedAt = DateTools.StringToDate(doc.Get("_updated_at")),
Title = doc.Get("title"),
Series = doc.Get("series"),
SeriesNo = int.Parse(doc.Get("seriesno")),
File = doc.Get("file"),
Ext = doc.Get("ext"),
Date = DateTime.Parse(doc.Get("pubdate")),
Archive = doc.Get("archive"),
Authors = authors,
Genres = genres,
Annotation = doc.Get("annotation"),
Language = doc.Get("language"),
Cover = cover,
Meta = meta
};
books.Add(book);
}
watch.Stop();
return(books);
}
}
public List <LinkItem> Search(GenerateSchemeViewModel model, out int totalHits)
{
if (!Directory.Exists(LuceneCommon.IndexOutDoorDirectory))
{
totalHits = 0;
return(new List <LinkItem>());
}
var combineQuery = new BooleanQuery();
SortField sortField = GetSortField(model.generateType);
#region 用户状态
var memberStatusQuery = NumericRangeQuery.NewIntRange(OutDoorIndexFields.MemberStatus, (int)MemberStatus.CompanyAuth, 99, true, true);
combineQuery.Add(memberStatusQuery, Occur.MUST);
#endregion
#region 审核状态查询构建
var verifyStatus = NumericRangeQuery.NewIntRange(OutDoorIndexFields.Status, (int)OutDoorStatus.ShowOnline, 99, true, true);
combineQuery.Add(verifyStatus, Occur.MUST);
#endregion
#region 媒体类别查询
if (!string.IsNullOrEmpty(model.mediaCode))
{
var mediaCodes = model.mediaCode.Split(',').Select(x => Convert.ToInt32(x));
var mediaCodeCombineQuery = new BooleanQuery();
foreach (var code in mediaCodes)
{
var maxCode = Utilities.GetMaxCode(code);
var mediaCodeQuery = NumericRangeQuery.NewLongRange(OutDoorIndexFields.MediaCateCode,
code, maxCode, true, true);
mediaCodeCombineQuery.Add(mediaCodeQuery, Occur.SHOULD);
}
combineQuery.Add(mediaCodeCombineQuery, Occur.MUST);
}
#endregion
#region 媒体档期查询
if (!string.IsNullOrEmpty(model.dq))
{
var minValue = (DateTime.Now.AddYears(-10)).Ticks;
var maxValue = Convert.ToDateTime(model.dq).Ticks;
var DeadLineQuery = NumericRangeQuery.NewLongRange(OutDoorIndexFields.DeadLine,
Convert.ToInt64(minValue), Convert.ToInt64(maxValue), true, true);
combineQuery.Add(DeadLineQuery, Occur.MUST);
}
#endregion
#region 地区查询
if (!string.IsNullOrEmpty(model.cityCode))
{
var cityCodes = model.cityCode.Split(',').Select(x => Convert.ToInt32(x));
var cityCodeCombineQuery = new BooleanQuery();
foreach (var code in cityCodes)
{
var maxCode = Utilities.GetMaxCode(code);
var cityCodeQuery = NumericRangeQuery.NewLongRange(OutDoorIndexFields.CityCateCode,
code, maxCode, true, true);
cityCodeCombineQuery.Add(cityCodeQuery, Occur.SHOULD);
}
combineQuery.Add(cityCodeCombineQuery, Occur.MUST);
}
#endregion
#region 关键字查询
if (!string.IsNullOrEmpty(model.formatCate) || !string.IsNullOrEmpty(model.crowdCate) || !string.IsNullOrEmpty(model.industryCate) || !string.IsNullOrEmpty(model.purposeCate))
{
var fields = new[] {
OutDoorIndexFields.IndustryCate,
OutDoorIndexFields.CrowdCate,
OutDoorIndexFields.PurposeCate,
OutDoorIndexFields.FormatName
};
var keywords = (string.IsNullOrEmpty(model.formatCate) ? string.Empty : model.formatCate + ",")
+ (string.IsNullOrEmpty(model.crowdCate) ? string.Empty : model.crowdCate + ",")
+ (string.IsNullOrEmpty(model.industryCate) ? string.Empty : model.industryCate + ",")
+ (string.IsNullOrEmpty(model.purposeCate) ? string.Empty : model.purposeCate);
var analyzer = new PanGuAnalyzer();
//var analyzer = new StandardAnalyzer(LuceneCommon.LuceneVersion);
var queryParser = new MultiFieldQueryParser(LuceneCommon.LuceneVersion, fields, analyzer);
//conjuction 一起选择
var conjuctionQuery = new BooleanQuery();
conjuctionQuery.Boost = 2.0f;
//disjunction 分离
var disjunctionQuery = new BooleanQuery();
disjunctionQuery.Boost = 0.1f;
//wildCard 通配符
var wildCardQuery = new BooleanQuery();
wildCardQuery.Boost = 0.5f;
var escapedSearchTerm = Escape(keywords);
foreach (var term in GetSearchTerms(keywords))
{
var termQuery = queryParser.Parse(term);
conjuctionQuery.Add(termQuery, Occur.MUST);
disjunctionQuery.Add(termQuery, Occur.SHOULD);
foreach (var field in fields)
{
var wildCardTermQuery = new WildcardQuery(new Term(field, term + "*"));
wildCardTermQuery.Boost = 0.7f;
wildCardQuery.Add(wildCardTermQuery, Occur.SHOULD);
}
}
//关键查询
var keywordsQuery =
conjuctionQuery.Combine(new Query[] { conjuctionQuery, disjunctionQuery, wildCardQuery });
combineQuery.Add(keywordsQuery, Occur.MUST);
}
#endregion
#region 媒体价格区间查询
if (model.priceCate != 0)
{
var rangeValue = EnumHelper.GetPriceValue(model.priceCate);
if (rangeValue.Max > 99999)
{
rangeValue.Max = 1000;
}
var PriceQuery = NumericRangeQuery.NewDoubleRange(OutDoorIndexFields.Price,
0, Convert.ToDouble(rangeValue.Max), true, true);
combineQuery.Add(PriceQuery, Occur.MUST);
}
#endregion
using (var directory = new SimpleFSDirectory(new DirectoryInfo(LuceneCommon.IndexOutDoorDirectory)))
{
var searcher = new IndexSearcher(directory, readOnly: true);
var results = searcher.Search(combineQuery, filter: null, n: 30, sort: new Sort(sortField));
var keys = results.ScoreDocs.Skip(0)
.Select(c => GetMediaItem(searcher.Doc(c.Doc)))
.ToList();
totalHits = results.TotalHits;
searcher.Dispose();
return(keys);
}
}
public void Open(string connection)
{
analyzer = GetAnalyzer();
directory = new SimpleFSDirectory(new System.IO.DirectoryInfo(connection));
writer = new IndexWriter(directory, analyzer, IndexWriter.MaxFieldLength.UNLIMITED);
}
//todo: make that as background task. Need input from someone how to handle that correctly as now it is as sync task to avoid issues, but need be change
public override void RebuildCache()
{
lock (RebuildLock)
{
//Needs locking
LoggingService.Log(new LogEntry(LogLevel.Info, null, $"Rebuilding cache"));
var tempDir = new DirectoryInfo(
Path.Combine(_cacheDirectoryPath,
_cacheDirectoryName, DateTimeOffset.UtcNow.ToString("yyyyMMddTHHmmssfffffff")));
if (tempDir.Exists == false)
{
tempDir.Create();
}
Lucene.Net.Store.Directory newIndex = new SimpleFSDirectory(tempDir);
foreach (string file in GetAllBlobFiles())
{
// newIndex.TouchFile(file);
if (file.EndsWith(".lock"))
{
continue;
}
var status = RemoteDirectory.SyncFile(newIndex, file, CompressBlobs);
if (!status)
{
LoggingService.Log(new LogEntry(LogLevel.Error, null, $"Rebuilding cache failed"));
newIndex.Dispose();
}
}
var oldIndex = CacheDirectory;
newIndex.Dispose();
newIndex = new SimpleFSDirectory(tempDir);
CacheDirectory = newIndex;
_lockFactory = newIndex.LockFactory;
if (oldIndex != null)
{
try
{
if (!string.IsNullOrEmpty(LockFactory.LockPrefix))
{
oldIndex.ClearLock(LockFactory.LockPrefix + "-write.lock");
}
else
{
oldIndex.ClearLock("write.lock");
}
}
catch (Exception ex)
{
LoggingService.Log(new LogEntry(LogLevel.Error, ex, $"Exception on unlocking old cache index folder"));
}
oldIndex.Dispose();
try
{
DirectoryInfo oldindexDir = new DirectoryInfo(Path.Combine(_cacheDirectoryPath,
_cacheDirectoryName, _oldIndexFolderName));
foreach (var file in oldindexDir.GetFiles())
{
file.Delete();
}
oldindexDir.Delete();
}
catch (Exception ex)
{
LoggingService.Log(new LogEntry(LogLevel.Error, ex, $"Cleaning of old directory failed."));
}
}
_oldIndexFolderName = tempDir.Name;
}
}
private InitializeDirectoryFlags InitializeLocalIndexAndDirectory(Directory baseLuceneDirectory, Analyzer analyzer, string configuredPath)
{
lock (_locker)
{
if (System.IO.Directory.Exists(TempPath) == false)
{
System.IO.Directory.CreateDirectory(TempPath);
}
//copy index if it exists, don't do anything if it's not there
if (IndexReader.IndexExists(baseLuceneDirectory) == false)
{
return(InitializeDirectoryFlags.SuccessNoIndexExists);
}
var writerAttempt = TryCreateWriterWithRetry(baseLuceneDirectory, analyzer);
if (writerAttempt.Success == false)
{
LogHelper.Error <LocalTempStorageIndexer>("Could not create index writer with snapshot policy for copying, the index cannot be used", writerAttempt.Exception);
return(InitializeDirectoryFlags.FailedLocked);
}
//Try to open the reader from the source, this will fail if the index is corrupt and we'll need to handle that
try
{
//NOTE: To date I've not seen this error occur
using (writerAttempt.Result.GetReader())
{
}
}
catch (Exception ex)
{
writerAttempt.Result.Dispose();
LogHelper.Error <LocalTempStorageIndexer>(
string.Format("Could not open an index reader, {0} is empty or corrupt... attempting to clear index files in master folder", configuredPath),
ex);
if (ClearLuceneDirFiles(baseLuceneDirectory) == false)
{
//hrm, not much we can do in this situation, but this shouldn't happen
LogHelper.Error <LocalTempStorageIndexer>("Could not open an index reader, index is corrupt.", ex);
return(InitializeDirectoryFlags.FailedCorrupt);
}
//the main index is now blank, we'll proceed as normal with a new empty index...
writerAttempt = TryCreateWriter(baseLuceneDirectory, analyzer);
if (writerAttempt.Success == false)
{
//ultra fail...
LogHelper.Error <LocalTempStorageIndexer>("Could not create index writer with snapshot policy for copying, the index cannot be used", writerAttempt.Exception);
return(InitializeDirectoryFlags.FailedLocked);
}
}
using (writerAttempt.Result)
{
try
{
var basePath = IOHelper.MapPath(configuredPath);
var commit = Snapshotter.Snapshot();
var allSnapshotFiles = commit.GetFileNames()
.Concat(new[]
{
commit.GetSegmentsFileName(),
//we need to manually include the segments.gen file
"segments.gen"
})
.Distinct()
.ToArray();
var tempDir = new DirectoryInfo(TempPath);
//Get all files in the temp storage that don't exist in the snapshot collection, we want to remove these
var toRemove = tempDir.GetFiles()
.Select(x => x.Name)
.Except(allSnapshotFiles);
using (var tempDirectory = new SimpleFSDirectory(tempDir))
{
if (TryWaitForDirectoryUnlock(tempDirectory))
{
foreach (var file in toRemove)
{
try
{
File.Delete(Path.Combine(TempPath, file));
}
catch (IOException ex)
{
if (file.InvariantEquals("write.lock"))
{
//This might happen if the writer is open
LogHelper.Warn <LocalTempStorageIndexer>("The lock file could not be deleted but should be removed when the writer is disposed");
}
LogHelper.Debug <LocalTempStorageIndexer>("Could not delete non synced index file file, index sync will continue but old index files will remain - this shouldn't affect indexing/searching operations. {0}", () => ex.ToString());
}
}
}
else
{
//quit here, this shouldn't happen with all the checks above.
LogHelper.Warn <LocalTempStorageIndexer>("Cannot sync index files from main storage, the temp file index is currently locked");
return(InitializeDirectoryFlags.FailedLocked);
}
foreach (var fileName in allSnapshotFiles.Where(f => f.IsNullOrWhiteSpace() == false))
{
var destination = Path.Combine(TempPath, Path.GetFileName(fileName));
//don't copy if it's already there, lucene is 'write once' so this file is meant to be there already
if (File.Exists(destination))
{
continue;
}
try
{
File.Copy(
Path.Combine(basePath, "Index", fileName),
destination);
}
catch (IOException ex)
{
LogHelper.Error <LocalTempStorageIndexer>("Could not copy index file, could not sync from main storage", ex);
//quit here
return(InitializeDirectoryFlags.FailedFileSync);
}
}
}
}
finally
{
Snapshotter.Release();
}
}
LogHelper.Info <LocalTempStorageIndexer>("Successfully sync'd main index to local temp storage for index: {0}", () => configuredPath);
return(InitializeDirectoryFlags.Success);
}
}
public void Search(Analyzer analyzer, string keyword)
{
if (string.IsNullOrEmpty(keyword))
{
throw new ArgumentException("content");
}
// 计时
Stopwatch watch = new Stopwatch();
watch.Start();
// 设置
int numHits = 10;
bool docsScoredInOrder = true;
bool isReadOnly = true;
// 创建Searcher
FSDirectory fsDir = new SimpleFSDirectory(new DirectoryInfo(_indexerFolder));
IndexSearcher indexSearcher = new IndexSearcher(IndexReader.Open(fsDir, isReadOnly));
TopScoreDocCollector collector = TopScoreDocCollector.Create(numHits, docsScoredInOrder);
QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, Config.Field_Content, analyzer);
Query query = parser.Parse(keyword);
indexSearcher.Search(query, collector);
//Console.WriteLine(collector.TotalHits);
var result = collector.TopDocs().ScoreDocs;
watch.Stop();
Console.WriteLine("总共耗时{0}毫秒", watch.ElapsedMilliseconds);
Console.WriteLine("总共找到{0}个文件", result.Count());
foreach (var docs in result)
{
Document doc = indexSearcher.Doc(docs.Doc);
Console.WriteLine("得分：{0}，文件名：{1}", docs.Score, doc.Get(Config.Field_Name));
}
indexSearcher.Dispose();
//BooleanQuery booleanQuery = new BooleanQuery();
//QueryParser parser1 = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, Config.Field_Path, analyzer);
//Query query1 = parser1.Parse(path);
//parser1.DefaultOperator = QueryParser.Operator.AND;
//booleanQuery.Add(query1, Occur.MUST);
////QueryParser parser = new QueryParser(Lucene.Net.Util.Version.LUCENE_30, Config.Field_Content, analyzer);
////Query query = parser.Parse(content);
////parser.DefaultOperator = QueryParser.Operator.AND;
////booleanQuery.Add(query, Occur.MUST);
////var queryParserFilePath = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, new string[] { Config.Field_FilePath }, analyzer);
////query.Add(queryParserFilePath.Parse(path), Occur.SHOULD);
////var queryParserContent = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, new string[] { Config.Field_Content}, analyzer);
////query.Add(queryParserContent.Parse(content), Occur.MUST);
//FSDirectory fsDir = new SimpleFSDirectory(new DirectoryInfo(_indexerFolder));
//bool isReadOnly = true;
//IndexSearcher indexSearcher = new IndexSearcher(IndexReader.Open(fsDir, isReadOnly));
////TopDocs result = indexSearcher.Search(booleanQuery, 10);
//Sort sort = new Sort(new SortField(Config.Field_Path, SortField.STRING, true));
//var result = indexSearcher.Search(booleanQuery, (Filter)null, 10 * 1, sort);
}
public bool Copy(SimpleFSDirectory sourceLuceneDirectory, Analyzer analyzer, DirectoryInfo targetPath)
{
if (targetPath.Exists == false)
{
Directory.CreateDirectory(targetPath.FullName);
}
//copy index if it exists, don't do anything if it's not there
if (IndexReader.IndexExists(sourceLuceneDirectory) == false)
{
return(true);
}
using (new IndexWriter(
//read from the underlying/default directory, not the temp codegen dir
sourceLuceneDirectory,
analyzer,
_snapshotter,
IndexWriter.MaxFieldLength.UNLIMITED))
{
try
{
//var basePath = IOHelper.MapPath(configuredPath);
var commit = _snapshotter.Snapshot();
var allSnapshotFiles = commit.GetFileNames()
.Concat(new[]
{
commit.GetSegmentsFileName(),
//we need to manually include the segments.gen file
"segments.gen"
})
.Distinct()
.ToArray();
//Get all files in the temp storage that don't exist in the snapshot collection, we want to remove these
var toRemove = targetPath.GetFiles()
.Select(x => x.Name)
.Except(allSnapshotFiles);
//using (var tempDirectory = new SimpleFSDirectory(tempDir))
//{
//TODO: We're ignoring if it is locked right now, it shouldn't be unless for some strange reason the
// last process hasn't fully shut down, in that case we're not going to worry about it.
//if (IndexWriter.IsLocked(tempDirectory) == false)
//{
foreach (var file in toRemove)
{
try
{
File.Delete(Path.Combine(targetPath.FullName, file));
}
catch (IOException ex)
{
//TODO: we're ignoring this, as old files shouldn't affect the index/search operations, lucene files are 'write once'
//TODO: Return some info?
//quit here
//return false;
}
}
//}
//else
//{
// LogHelper.Warn<LocalTempStorageIndexer>("Cannot sync index files from main storage, the index is currently locked");
// //quit here
// return false;
//}
foreach (var fileName in allSnapshotFiles.Where(f => string.IsNullOrWhiteSpace(f) == false))
{
var destination = Path.Combine(targetPath.FullName, Path.GetFileName(fileName));
//don't copy if it's already there, lucene is 'write once' so this file is meant to be there already
if (File.Exists(destination))
{
continue;
}
try
{
File.Copy(
Path.Combine(sourceLuceneDirectory.GetDirectory().FullName, "Index", fileName),
destination);
}
catch (IOException ex)
{
//TODO: Return some info?
//quit here
return(false);
}
}
//}
}
finally
{
_snapshotter.Release();
}
}
return(true);
}
public List <ResultData> GetResults(Query query, List <RequiredHighlight> requiredHighlights)
{
try
{
Directory indexDir = new SimpleFSDirectory(ConfigurationManager.IndexDir);
DirectoryReader reader = DirectoryReader.Open(indexDir);
IndexSearcher isr = new IndexSearcher(reader);
TopScoreDocCollector collector = TopScoreDocCollector.Create(
10, true);
List <ResultData> results = new List <ResultData>();
isr.Search(query, collector);
ScoreDoc[] hits = collector.GetTopDocs().ScoreDocs;
ResultData re;
Document doc;
Highlighter hi;
foreach (ScoreDoc sd in hits)
{
doc = isr.Doc(sd.Doc);
string[] allKeywords = doc.GetValues("keyword");
string keywords = "";
foreach (string keyword in allKeywords)
{
keywords += keyword.Trim() + " ";
}
keywords = keywords.Trim();
string title = doc.Get("title");
string location = doc.Get("filename");
string author = doc.Get("author");
int category = Int32.Parse(doc.Get("category"));
string language = doc.Get("language");
string highlight = "";
string text = GetDocumentText(location);
foreach (var item in requiredHighlights)
{
hi = new Highlighter(new QueryScorer(query, reader, item.FieldName));
try
{
highlight += hi.GetBestFragment(analyzer, item.FieldName, text);
}
catch (Exception e)
{
Console.WriteLine(e.Message + " on" + e.StackTrace);
}
}
re = new ResultData()
{
Title = title,
Filename = location,
Keywords = keywords,
CategoryId = category,
Author = author,
Highlight = highlight,
};
results.Add(re);
}
reader.Dispose();
return(results);
}
catch (Exception e)
{
Console.WriteLine(e.Source);
return(new List <ResultData>());
}
}
static string IndexBatch(string path, string connectionString, IDictionary <int, List <string> > packageFrameworks, int beginKey, int endKey)
{
string folder = string.Format(@"{0}\index_{1}_{2}", path, beginKey, endKey);
DirectoryInfo directoryInfo = new DirectoryInfo(folder);
directoryInfo.Create();
using (SqlConnection connection = new SqlConnection(connectionString))
{
connection.Open();
string cmdText = @"
SELECT
Packages.[Key] 'key',
PackageRegistrations.Id 'id',
Packages.[Version] 'originalVersion',
Packages.NormalizedVersion 'version',
Packages.Title 'title',
Packages.Tags 'tags',
Packages.[Description] 'description',
Packages.FlattenedAuthors 'flattenedAuthors',
Packages.Summary 'summary',
Packages.IconUrl 'iconUrl',
Packages.ProjectUrl 'projectUrl',
Packages.MinClientVersion 'minClientVersion',
Packages.ReleaseNotes 'releaseNotes',
Packages.Copyright 'copyright',
Packages.[Language] 'language',
Packages.LicenseUrl 'licenseUrl',
Packages.RequiresLicenseAcceptance 'requiresLicenseAcceptance',
Packages.[Hash] 'packageHash',
Packages.HashAlgorithm 'packageHashAlgorithm',
Packages.PackageFileSize 'packageSize',
Packages.FlattenedDependencies 'flattenedDependencies',
Packages.LicenseNames 'licenseNames',
Packages.LicenseReportUrl 'licenseReportUrl',
Packages.Created 'created',
Packages.LastEdited 'lastEdited',
Packages.Published 'published',
Packages.Listed 'listed'
FROM Packages
INNER JOIN PackageRegistrations ON Packages.PackageRegistrationKey = PackageRegistrations.[Key]
WHERE Packages.Listed = 1
AND Packages.[Key] >= @BeginKey
AND Packages.[Key] < @EndKey
ORDER BY Packages.[Key]
";
SqlCommand command = new SqlCommand(cmdText, connection);
command.Parameters.AddWithValue("BeginKey", beginKey);
command.Parameters.AddWithValue("EndKey", endKey);
SqlDataReader reader = command.ExecuteReader();
int batch = 0;
SimpleFSDirectory directory = new SimpleFSDirectory(directoryInfo);
using (IndexWriter writer = new IndexWriter(directory, new PackageAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED))
{
while (reader.Read())
{
Document document = CreateDocument(reader, packageFrameworks);
writer.AddDocument(document);
if (batch++ == 1000)
{
writer.Commit();
batch = 0;
}
}
if (batch > 0)
{
writer.Commit();
}
}
}
return(folder);
}
public async Task DoesNotSkipPackagesWhenExceptionOccurs(string catalogUri, string expectedCursorBeforeRetry)
{
// Arrange
var storage = new MemoryStorage();
var storageFactory = new TestStorageFactory(name => storage.WithName(name));
MockServerHttpClientHandler mockServer;
mockServer = new MockServerHttpClientHandler();
mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)));
var catalogStorage = Catalogs.CreateTestCatalogWithCommitThenTwoPackageCommit();
await mockServer.AddStorageAsync(catalogStorage);
// Make the first request for a catalog leaf node fail. This will cause the registration collector
// to fail the first time but pass the second time.
FailFirstRequest(mockServer, catalogUri);
expectedCursorBeforeRetry = expectedCursorBeforeRetry ?? MemoryCursor.MinValue.ToString("O");
ReadWriteCursor front = new DurableCursor(
storage.ResolveUri("cursor.json"),
storage,
MemoryCursor.MinValue);
ReadCursor back = MemoryCursor.CreateMax();
using (var testDirectory = TestDirectory.Create())
{
var luceneDirectory = new SimpleFSDirectory(new DirectoryInfo(testDirectory));
using (var indexWriter = Catalog2LuceneJob.CreateIndexWriter(luceneDirectory))
{
var target = new SearchIndexFromCatalogCollector(
new Uri("http://tempuri.org/index.json"),
indexWriter,
commitEachBatch: true,
baseAddress: null,
telemetryService: new Mock <ITelemetryService>().Object,
logger: new TestLogger(),
handlerFunc: () => mockServer);
// Act
await Assert.ThrowsAsync <Exception>(() => target.RunAsync(front, back, CancellationToken.None));
var cursorBeforeRetry = front.Value;
await target.RunAsync(front, back, CancellationToken.None);
var cursorAfterRetry = front.Value;
// Assert
var reader = indexWriter.GetReader();
var documents = Enumerable
.Range(0, reader.NumDeletedDocs + reader.NumDocs())
.Where(i => !reader.IsDeleted(i))
.Select(i => reader.Document(i))
.ToList();
Assert.Equal(4, documents.Count);
var documentsByType = documents
.ToLookup(doc => doc
.fields_ForNUnit
.FirstOrDefault(f => f.Name == "@type")?
.StringValue);
var commitDocuments = documentsByType[Schema.DataTypes.CatalogInfastructure.AbsoluteUri.ToString()].ToList();
var packageDocuments = documentsByType[null].ToList();
Assert.Equal(1, commitDocuments.Count);
Assert.Equal(3, packageDocuments.Count);
Assert.Equal(
"UnlistedPackage",
packageDocuments[0].fields_ForNUnit.FirstOrDefault(x => x.Name == Constants.LucenePropertyId)?.StringValue);
Assert.Equal(
"ListedPackage",
packageDocuments[1].fields_ForNUnit.FirstOrDefault(x => x.Name == Constants.LucenePropertyId)?.StringValue);
Assert.Equal(
"AnotherPackage",
packageDocuments[2].fields_ForNUnit.FirstOrDefault(x => x.Name == Constants.LucenePropertyId)?.StringValue);
Assert.Equal(DateTime.Parse(expectedCursorBeforeRetry).ToUniversalTime(), cursorBeforeRetry);
Assert.Equal(DateTime.Parse("2015-10-12T10:08:55.3335317Z").ToUniversalTime(), cursorAfterRetry);
}
}
}
public static void TestFlexLuceneFS(string[] args)
{
string ixdir = string.Empty;
if (IsLinux)
{
ixdir = "/home/master/flexlucene/indexes";
}
else
{
ixdir = @"c:\temp\flexlucene\indexes";
}
StandardAnalyzer analyzer = new StandardAnalyzer();
//FlexLucene.Store.Directory index = SimpleFSDirectory.Open(java.nio.file.Paths.get(ixdir),SimpleFSLockFactory.INSTANCE);
FlexLucene.Store.Directory index = SimpleFSDirectory.Open(java.nio.file.Paths.get(ixdir));
// FlexLucene.Store.Directory index = NIOFSDirectory.Open(java.nio.file.Paths.get(ixdir));
config = new IndexWriterConfig(analyzer);
cnf = new FacetsConfig();
cnf.SetIndexFieldName("title", "facet_title");
cnf.SetIndexFieldName("isbn", "facet_isbn");
LuceneTest.taxoDir = (FlexLucene.Store.Directory) new RAMDirectory();
LuceneTest.taxoWriter = (TaxonomyWriter) new FlexLucene.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter(LuceneTest.taxoDir, IndexWriterConfigOpenMode.CREATE);
IndexWriter w = new IndexWriter(index, LuceneTest.config);
addDoc(w, "Lucene in Action", "9900001");
addDoc(w, "Lucene for Dummies", "9900002");
addDoc(w, "Lucene for Dummies 2", "9900003");
w.close();
String querystr = "isbn:99*";
Query q = new QueryParser("title", (Analyzer)analyzer).Parse(querystr);
int hitsPerPage = 10;
IndexReader reader = (IndexReader)DirectoryReader.Open(index);
IndexSearcher searcher = new IndexSearcher(reader);
TopScoreDocCollector collector = TopScoreDocCollector.Create(hitsPerPage);
searcher.Search(q, (Collector)collector);
ScoreDoc[] hits = collector.TopDocs().ScoreDocs;
Console.WriteLine("Found " + hits.Length + " hits.");
for (int i = 0; i < hits.Length; ++i)
{
int docId = hits [i].Doc;
Document d = searcher.Doc(docId);
Console.WriteLine(i + 1 + ". " + d.Get("isbn") + "\t" + d.Get("title"));
}
SortedSetDocValuesReaderState state = (SortedSetDocValuesReaderState) new DefaultSortedSetDocValuesReaderState(reader, "facet_isbn");
FacetsCollector fc = new FacetsCollector();
FacetsCollector.Search(searcher, q, 10, (Collector)fc);
Facets facets = (Facets) new SortedSetDocValuesFacetCounts(state, fc);
FacetResult result = facets.GetTopChildren(10, "isbn", new String[0]);
for (int j = 0; j < result.ChildCount; ++j)
{
LabelAndValue lv = result.LabelValues [j];
Console.WriteLine(String.Format("Label={0}, Value={1}", lv.Label, lv.Value));
}
reader.close();
}
public async Task CreateIndexFileAsync(bool forceCreate)
{
SpinWait.SpinUntil(() => 0 == Interlocked.Read(ref this.ActiveSearchCount));
Interlocked.Exchange(ref this.IsSearcherReady, 0);
this.OnIndexCreating();
DirectoryInfo indexDirectoryInfo = new DirectoryInfo(this.IndexPath);
if (!forceCreate && this.CanReuseIndex(indexDirectoryInfo))
{
return;
}
this.Dispose(true);
var files = indexDirectoryInfo.GetFiles();
Array.ForEach(files, file =>
{
if (!file.Name.Equals("placeholder.txt", StringComparison.OrdinalIgnoreCase) &&
!file.Name.Equals("work.lock", StringComparison.OrdinalIgnoreCase))
{
file.Delete();
}
});
using (SimpleFSDirectory fsDirectory = new SimpleFSDirectory(indexDirectoryInfo))
{
using (var analyzer = new AdrivaAnalyzer(LuceneVersion.LUCENE_48))
{
IndexWriterConfig indexWriterConfig = new IndexWriterConfig(LuceneVersion.LUCENE_48, analyzer)
{
};
using (IndexWriter writer = new IndexWriter(fsDirectory, indexWriterConfig))
{
RawDataResult result = new RawDataResult
{
HasMore = false
};
do
{
result = await this.GetRawDataAsync(result.LastRowId);
var rawDataEnumerator = result.Items.GetEnumerator();
while (rawDataEnumerator.MoveNext())
{
Document document = this.ResolveDocument(rawDataEnumerator.Current);
if (null != document)
{
writer.AddDocument(document, analyzer);
}
}
} while (result.HasMore);
writer.Commit();
}
}
}
this.OnIndexCreated();
}
