public void Can_Add_Doc_With_Fields()
{
using (var luceneDir = new RandomIdRAMDirectory())
using (var indexer = new TestIndex(luceneDir, new StandardAnalyzer(Version.LUCENE_30)))
{
indexer.IndexItem(new ValueSet(1.ToString(), "content", "test",
new Dictionary <string, IEnumerable <object> >
{
{ "item1", new List <object>(new[] { "value1" }) },
{ "item2", new List <object>(new[] { "value2" }) }
}));
using (var s = (LuceneSearcher)indexer.GetSearcher())
{
var luceneSearcher = s.GetLuceneSearcher();
var fields = luceneSearcher.Doc(0).GetFields().ToArray();
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == "item1"));
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == "item2"));
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == LuceneIndex.ItemTypeFieldName));
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == LuceneIndex.ItemIdFieldName));
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == LuceneIndex.CategoryFieldName));
Assert.AreEqual("value1", fields.Single(x => x.Name == "item1").StringValue);
Assert.AreEqual("value2", fields.Single(x => x.Name == "item2").StringValue);
Assert.AreEqual("test", fields.Single(x => x.Name == LuceneIndex.ItemTypeFieldName).StringValue);
Assert.AreEqual("1", fields.Single(x => x.Name == LuceneIndex.ItemIdFieldName).StringValue);
Assert.AreEqual("content", fields.Single(x => x.Name == LuceneIndex.CategoryFieldName).StringValue);
}
}
}
public void Number_Field()
{
using (var luceneDir = new RandomIdRAMDirectory())
using (var indexer = new TestIndex(
new FieldDefinitionCollection(new FieldDefinition("item2", "number")),
luceneDir,
new StandardAnalyzer(Version.LUCENE_30)))
{
indexer.IndexItem(new ValueSet(1.ToString(), "content",
new Dictionary <string, IEnumerable <object> >
{
{ "item1", new List <object>(new[] { "value1" }) },
{ "item2", new List <object>(new object[] { 123456 }) }
}));
using (var s = (LuceneSearcher)indexer.GetSearcher())
{
var luceneSearcher = s.GetLuceneSearcher();
var fields = luceneSearcher.Doc(luceneSearcher.MaxDoc - 1).GetFields().ToArray();
var valType = indexer.FieldValueTypeCollection.GetValueType("item2");
Assert.AreEqual(typeof(Int32Type), valType.GetType());
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == "item2"));
}
}
}
private void DoSpatialSearch(
SpatialContext ctx, SpatialStrategy strategy,
TestIndex indexer, double searchRadius, string idToMatch, Func <SpatialArgs, Query> createQuery, int lat,
int lng)
{
var searcher = (LuceneSearcher)indexer.GetSearcher();
var luceneSearcher = searcher.GetLuceneSearcher();
GetXYFromCoords(lat, lng, out var x, out var y);
// Make a circle around the search point
var args = new SpatialArgs(
SpatialOperation.Intersects,
ctx.MakeCircle(x, y, DistanceUtils.Dist2Degrees(searchRadius, DistanceUtils.EARTH_MEAN_RADIUS_KM)));
var filter = strategy.MakeFilter(args);
var query = createQuery(args);
// TODO: It doesn't make a whole lot of sense to sort by score when searching on only geo-coords,
// typically you would sort by closest distance
// Which can be done, see https://github.com/apache/lucene-solr/blob/branch_4x/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialExample.java#L169
TopDocs docs = luceneSearcher.Search(query, filter, MaxResultDocs, new Sort(new SortField(null, SortField.SCORE)));
AssertDocMatchedIds(luceneSearcher, docs, idToMatch);
// TODO: We should make this possible and allow passing in a Lucene Filter
// to the LuceneSearchQuery along with the Lucene Query, then we
// don't need to manually perform the Lucene Search
//var criteria = (LuceneSearchQuery)searcher.CreateQuery();
//criteria.LuceneQuery(q);
//var results = criteria.Execute();
}
public void Can_Add_Doc_With_Easy_Fields()
{
using (var luceneDir = new RandomIdRAMDirectory())
using (var indexer = new TestIndex(luceneDir, new StandardAnalyzer(Version.LUCENE_30)))
{
indexer.IndexItem(ValueSet.FromObject(1.ToString(), "content",
new { item1 = "value1", item2 = "value2" }));
using (var s = (LuceneSearcher)indexer.GetSearcher())
{
var luceneSearcher = s.GetLuceneSearcher();
var fields = luceneSearcher.Doc(0).GetFields().ToArray();
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == "item1"));
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == "item2"));
Assert.AreEqual("value1", fields.Single(x => x.Name == "item1").StringValue);
Assert.AreEqual("value2", fields.Single(x => x.Name == "item2").StringValue);
}
}
}
public void Track_Readers()
{
var analyzer = new StandardAnalyzer(Version.LUCENE_30);
using (var luceneDir = new RandomIdRAMDirectory())
using (var indexer = new TestIndex(luceneDir, analyzer))
{
indexer.IndexItems(new[] {
ValueSet.FromObject(1.ToString(), "content",
new { nodeName = "umbraco", headerText = "world", writerName = "administrator" }),
ValueSet.FromObject(2.ToString(), "content",
new { nodeName = "umbraco", headerText = "umbraco", writerName = "administrator" }),
ValueSet.FromObject(3.ToString(), "content",
new { nodeName = "umbraco", headerText = "umbraco", writerName = "administrator" }),
ValueSet.FromObject(4.ToString(), "content",
new { nodeName = "hello", headerText = "world", writerName = "blah" })
});
LuceneSearcher searcher = (LuceneSearcher)indexer.GetSearcher();
IndexSearcher luceneSearcher = (IndexSearcher)searcher.GetLuceneSearcher();
//Arrange
var sc = searcher.CreateQuery("content").Field("writerName", "administrator");
//Act
var results = sc.Execute();
using (var e1 = results.GetEnumerator())
{
Assert.AreEqual(2, luceneSearcher.IndexReader.RefCount);
using (var e2 = results.Skip(2).GetEnumerator())
{
Assert.AreEqual(3, luceneSearcher.IndexReader.RefCount);
}
Assert.AreEqual(2, luceneSearcher.IndexReader.RefCount);
}
Assert.AreEqual(1, luceneSearcher.IndexReader.RefCount);
}
}
public void Can_Have_Multiple_Values_In_Fields()
{
using (var luceneDir = new RandomIdRAMDirectory())
using (var indexer = new TestIndex(luceneDir, new StandardAnalyzer(Version.LUCENE_30)))
{
indexer.IndexItem(new ValueSet(1.ToString(), "content",
new Dictionary <string, IEnumerable <object> >
{
{
"item1", new List <object> {
"subval1", "subval2"
}
},
{
"item2", new List <object> {
"subval1", "subval2", "subval3"
}
}
}));
using (var s = (LuceneSearcher)indexer.GetSearcher())
{
var luceneSearcher = s.GetLuceneSearcher();
var fields = luceneSearcher.Doc(0).GetFields().ToArray();
Assert.AreEqual(2, fields.Count(x => x.Name == "item1"));
Assert.AreEqual(3, fields.Count(x => x.Name == "item2"));
Assert.AreEqual("subval1", fields.Where(x => x.Name == "item1").ElementAt(0).StringValue);
Assert.AreEqual("subval2", fields.Where(x => x.Name == "item1").ElementAt(1).StringValue);
Assert.AreEqual("subval1", fields.Where(x => x.Name == "item2").ElementAt(0).StringValue);
Assert.AreEqual("subval2", fields.Where(x => x.Name == "item2").ElementAt(1).StringValue);
Assert.AreEqual("subval3", fields.Where(x => x.Name == "item2").ElementAt(2).StringValue);
}
}
}
public void Index_Read_And_Write_Ensure_No_Errors_In_Async()
{
using (var d = new RandomIdRAMDirectory())
using (var writer = new IndexWriter(d, new CultureInvariantStandardAnalyzer(Version.LUCENE_30), IndexWriter.MaxFieldLength.LIMITED))
using (var customIndexer = new TestIndex(writer))
using (var customSearcher = (LuceneSearcher)customIndexer.GetSearcher())
{
var waitHandle = new ManualResetEvent(false);
void OperationComplete(object sender, IndexOperationEventArgs e)
{
//signal that we are done
waitHandle.Set();
}
//add the handler for optimized since we know it will be optimized last based on the commit count
customIndexer.IndexOperationComplete += OperationComplete;
//remove the normal indexing error handler
customIndexer.IndexingError -= IndexInitializer.IndexingError;
//run in async mode
customIndexer.RunAsync = false;
//get a node from the data repo
var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]")
.Root
.Elements()
.First();
var idQueue = new ConcurrentQueue <int>(Enumerable.Range(1, 10));
var searchThreadCount = 500;
var indexThreadCount = 10;
var searchCount = 10700;
var indexCount = 100;
var searchCountPerThread = Convert.ToInt32(searchCount / searchThreadCount);
var indexCountPerThread = Convert.ToInt32(indexCount / indexThreadCount);
//spawn a bunch of threads to perform some reading
var tasks = new List <Task>();
Action <ISearcher> doSearch = (s) =>
{
try
{
for (var counter = 0; counter < searchCountPerThread; counter++)
{
//get next id and put it to the back of the list
int docId;
if (idQueue.TryDequeue(out docId))
{
idQueue.Enqueue(docId);
var r = s.CreateQuery().Id(docId.ToString()).Execute();
Debug.WriteLine("searching thread: {0}, id: {1}, found: {2}", Thread.CurrentThread.ManagedThreadId, docId, r.Count());
Thread.Sleep(50);
}
}
}
catch (Exception ex)
{
Debug.WriteLine("ERROR!! {0}", ex);
throw;
}
};
Action <IIndex> doIndex = (ind) =>
{
try
{
//reindex the same node a bunch of times
for (var i = 0; i < indexCountPerThread; i++)
{
//get next id and put it to the back of the list
int docId;
if (idQueue.TryDequeue(out docId))
{
idQueue.Enqueue(docId);
var cloned = new XElement(node);
cloned.Attribute("id").Value = docId.ToString(CultureInfo.InvariantCulture);
Debug.WriteLine("Indexing {0}", docId);
ind.IndexItems(new[] { cloned.ConvertToValueSet(IndexTypes.Content) });
Thread.Sleep(100);
}
}
}
catch (Exception ex)
{
Debug.WriteLine("ERROR!! {0}", ex);
throw;
}
};
//indexing threads
for (var i = 0; i < indexThreadCount; i++)
{
var indexer = customIndexer;
tasks.Add(Task.Factory.StartNew(() => doIndex(indexer), TaskCreationOptions.LongRunning));
}
//searching threads
for (var i = 0; i < searchThreadCount; i++)
{
var searcher = customSearcher;
tasks.Add(Task.Factory.StartNew(() => doSearch(searcher), TaskCreationOptions.LongRunning));
}
try
{
Task.WaitAll(tasks.ToArray());
}
catch (AggregateException e)
{
var sb = new StringBuilder();
sb.Append(e.Message + ": ");
foreach (var v in e.InnerExceptions)
{
sb.Append(v.Message + "; ");
}
Assert.Fail(sb.ToString());
}
//reset the async mode and remove event handler
customIndexer.IndexingError += IndexInitializer.IndexingError;
customIndexer.RunAsync = false;
//wait until we are done
waitHandle.WaitOne();
writer.WaitForMerges();
var results = customSearcher.CreateQuery().Field("nodeName", (IExamineValue) new ExamineValue(Examineness.Explicit, "Home")).Execute();
Assert.AreEqual(10, results.Count());
}
}
public void Index_Ensure_No_Duplicates_In_Async()
{
using (var d = new RandomIdRAMDirectory())
using (var writer = new IndexWriter(d, new CultureInvariantStandardAnalyzer(Version.LUCENE_30), IndexWriter.MaxFieldLength.LIMITED))
using (var customIndexer = new TestIndex(writer))
//using (var customSearcher = (LuceneSearcher)customIndexer.GetSearcher())
{
var waitHandle = new ManualResetEvent(false);
void OperationComplete(object sender, IndexOperationEventArgs e)
{
//signal that we are done
waitHandle.Set();
}
//add the handler for optimized since we know it will be optimized last based on the commit count
customIndexer.IndexOperationComplete += OperationComplete;
//remove the normal indexing error handler
customIndexer.IndexingError -= IndexInitializer.IndexingError;
//run in async mode
customIndexer.RunAsync = true;
//get a node from the data repo
var idQueue = new ConcurrentQueue <int>(Enumerable.Range(1, 3));
var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]")
.Root
.Elements()
.First();
//reindex the same nodes a bunch of times
for (var i = 0; i < idQueue.Count * 20; i++)
{
//get next id and put it to the back of the list
int docId;
if (idQueue.TryDequeue(out docId))
{
idQueue.Enqueue(docId);
var cloned = new XElement(node);
cloned.Attribute("id").Value = docId.ToString(CultureInfo.InvariantCulture);
Debug.WriteLine("Indexing {0}", docId);
customIndexer.IndexItems(new[] { cloned.ConvertToValueSet(IndexTypes.Content) });
Thread.Sleep(100);
}
}
//reset the async mode and remove event handler
customIndexer.IndexingError += IndexInitializer.IndexingError;
customIndexer.RunAsync = false;
//wait until we are done
waitHandle.WaitOne();
writer.WaitForMerges();
//ensure no duplicates
var customSearcher = (LuceneSearcher)customIndexer.GetSearcher();
var results = customSearcher.CreateQuery().Field("nodeName", (IExamineValue) new ExamineValue(Examineness.Explicit, "Home")).Execute();
Assert.AreEqual(3, results.Count());
}
}
public void Can_Overwrite_Index_During_Indexing_Operation()
{
const int ThreadCount = 1000;
using (var d = new RandomIdRAMDirectory())
using (var writer = new IndexWriter(d, new CultureInvariantStandardAnalyzer(Version.LUCENE_30), IndexWriter.MaxFieldLength.LIMITED))
using (var customIndexer = new TestIndex(writer))
using (var customSearcher = (LuceneSearcher)customIndexer.GetSearcher())
{
var waitHandle = new ManualResetEvent(false);
var opCompleteCount = 0;
void OperationComplete(object sender, IndexOperationEventArgs e)
{
Interlocked.Increment(ref opCompleteCount);
Console.WriteLine($"OperationComplete: {opCompleteCount}");
if (opCompleteCount == ThreadCount)
{
//signal that we are done
waitHandle.Set();
}
}
//add the handler for optimized since we know it will be optimized last based on the commit count
customIndexer.IndexOperationComplete += OperationComplete;
//remove the normal indexing error handler
customIndexer.IndexingError -= IndexInitializer.IndexingError;
//run in async mode
customIndexer.RunAsync = true;
//get a node from the data repo
var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]")
.Root
.Elements()
.First();
//get the id for th node we're re-indexing.
var id = (int)node.Attribute("id");
//spawn a bunch of threads to perform some reading
var tasks = new List <Task>();
//reindex the same node a bunch of times - then while this is running we'll overwrite below
for (var i = 0; i < ThreadCount; i++)
{
var indexer = customIndexer;
tasks.Add(Task.Factory.StartNew(() =>
{
//get next id and put it to the back of the list
int docId = i;
var cloned = new XElement(node);
Debug.WriteLine("Indexing {0}", docId);
indexer.IndexItem(cloned.ConvertToValueSet(IndexTypes.Content));
}, TaskCreationOptions.LongRunning));
}
Thread.Sleep(100);
//overwrite!
customIndexer.EnsureIndex(true);
try
{
Task.WaitAll(tasks.ToArray());
}
catch (AggregateException e)
{
var sb = new StringBuilder();
sb.Append(e.Message + ": ");
foreach (var v in e.InnerExceptions)
{
sb.Append(v.Message + "; ");
}
Assert.Fail(sb.ToString());
}
//reset the async mode and remove event handler
customIndexer.IndexingError += IndexInitializer.IndexingError;
customIndexer.RunAsync = false;
//wait until we are done
waitHandle.WaitOne(TimeSpan.FromMinutes(2));
writer.WaitForMerges();
//ensure no data since it's a new index
var results = customSearcher.CreateQuery().Field("nodeName", (IExamineValue) new ExamineValue(Examineness.Explicit, "Home")).Execute();
//the total times that OperationComplete event should be fired is 1000
Assert.AreEqual(1000, opCompleteCount);
//should be less than the total inserted because we overwrote it in the middle of processing
Debug.WriteLine("TOTAL RESULTS: " + results.TotalItemCount);
Assert.Less(results.Count(), 1000);
}
}
public void Index_Read_And_Write_Ensure_No_Errors_In_Async(
int indexCount,
int searchCount,
int indexThreadCount,
int searchThreadCount,
int indexThreadWait,
int searchThreadWait,
bool inMemory)
{
// TODO: In this test can we ensure all readers are tracked and closed?
// TODO: In the search part, we should be searching in various ways and also with skip
DirectoryInfo temp = null;
Lucene.Net.Store.Directory directory;
if (inMemory)
{
directory = new RandomIdRAMDirectory();
}
else
{
// try to clear out old files
var tempBasePath = Path.Combine(Path.GetTempPath(), "ExamineTests");
if (System.IO.Directory.Exists(tempBasePath))
{
try
{
System.IO.Directory.Delete(tempBasePath, true);
}
catch
{
}
}
var tempPath = Path.Combine(tempBasePath, Guid.NewGuid().ToString());
System.IO.Directory.CreateDirectory(tempPath);
temp = new DirectoryInfo(tempPath);
directory = new SimpleFSDirectory(temp);
}
try
{
using (var d = directory)
using (var writer = new IndexWriter(d, new CultureInvariantStandardAnalyzer(Version.LUCENE_30), IndexWriter.MaxFieldLength.LIMITED))
using (var customIndexer = new TestIndex(writer))
using (var customSearcher = (LuceneSearcher)customIndexer.GetSearcher())
{
var waitHandle = new ManualResetEvent(false);
void OperationComplete(object sender, IndexOperationEventArgs e)
{
//signal that we are done
waitHandle.Set();
}
//add the handler for optimized since we know it will be optimized last based on the commit count
customIndexer.IndexOperationComplete += OperationComplete;
//remove the normal indexing error handler
//customIndexer.IndexingError -= IndexInitializer.IndexingError;
//run in async mode
customIndexer.RunAsync = true;
//get all nodes
var nodes = _contentService.GetPublishedContentByXPath("//*[@isDoc]")
.Root
.Elements()
.ToList();
Func <int, XElement> getNode = (index) =>
{
// clone it
return(new XElement(nodes[index]));
};
// we know there are 20 documents available, this is important for the getNode call
var idQueue = new ConcurrentQueue <int>(Enumerable.Range(1, 20));
var searchCountPerThread = Convert.ToInt32(searchCount / searchThreadCount);
var indexCountPerThread = Convert.ToInt32(indexCount / indexThreadCount);
//spawn a bunch of threads to perform some reading
var tasks = new List <Task>();
Action <ISearcher> doSearch = (s) =>
{
try
{
for (var counter = 0; counter < searchCountPerThread; counter++)
{
//get next id and put it to the back of the list
int docId;
if (idQueue.TryDequeue(out docId))
{
idQueue.Enqueue(docId);
var r = s.CreateQuery().Id(docId.ToString()).Execute();
Console.WriteLine("searching thread: {0}, id: {1}, found: {2}", Thread.CurrentThread.ManagedThreadId, docId, r.Count());
Thread.Sleep(searchThreadWait);
}
}
}
catch (Exception ex)
{
Console.WriteLine("Search ERROR!! {0}", ex);
throw;
}
};
Action <IIndex> doIndex = (ind) =>
{
try
{
//reindex a nodes a bunch of times
for (var i = 0; i < indexCountPerThread; i++)
{
//get next id and put it to the back of the list
int docId;
if (idQueue.TryDequeue(out docId))
{
idQueue.Enqueue(docId);
var node = getNode(docId - 1);
node.Attribute("id").Value = docId.ToString(CultureInfo.InvariantCulture);
Console.WriteLine("Indexing {0}", docId);
ind.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Content) });
Thread.Sleep(indexThreadWait);
}
}
}
catch (Exception ex)
{
Console.WriteLine("Index ERROR!! {0}", ex);
throw;
}
};
//indexing threads
for (var i = 0; i < indexThreadCount; i++)
{
var indexer = customIndexer;
tasks.Add(Task.Run(() => doIndex(indexer)));
}
//searching threads
for (var i = 0; i < searchThreadCount; i++)
{
var searcher = customSearcher;
tasks.Add(Task.Run(() => doSearch(searcher)));
}
try
{
Task.WaitAll(tasks.ToArray());
}
catch (AggregateException e)
{
var sb = new StringBuilder();
sb.Append(e.Message + ": ");
foreach (var v in e.InnerExceptions)
{
sb.Append(v.Message + "; ");
}
Assert.Fail(sb.ToString());
}
var results = customSearcher.CreateQuery().All().Execute();
Assert.AreEqual(20, results.Count());
//reset the async mode and remove event handler
//customIndexer.IndexingError += IndexInitializer.IndexingError;
//customIndexer.RunAsync = false;
//wait until we are done
waitHandle.WaitOne();
writer.WaitForMerges();
writer.Dispose(true);
results = customSearcher.CreateQuery().All().Execute();
Assert.AreEqual(20, results.Count());
}
}
finally
{
if (temp != null)
{
try
{
temp.Delete(true);
}
catch (Exception ex)
{
Console.WriteLine("Could not delete temp folder {0}", ex);
}
}
}
}