public void MultiIndex_Field_Count()
{
using (var cwsDir = new RandomIdRAMDirectory())
using (var pdfDir = new RandomIdRAMDirectory())
using (var simpleDir = new RandomIdRAMDirectory())
using (var conventionDir = new RandomIdRAMDirectory())
{
//get all of the indexers and rebuild them all first
var indexers = new IIndexer[]
{
IndexInitializer.GetUmbracoIndexer(cwsDir),
IndexInitializer.GetSimpleIndexer(simpleDir),
IndexInitializer.GetUmbracoIndexer(conventionDir)
};
foreach (var i in indexers)
{
i.RebuildIndex();
}
//now get the multi index searcher for all indexes
using (var searcher = IndexInitializer.GetMultiSearcher(pdfDir, simpleDir, conventionDir, cwsDir))
{
var result = searcher.GetSearchFields();
Assert.AreEqual(26, result.Count(), "The total number for fields between all of the indexes should be ");
}
}
}
private void RunTest(SpatialContext ctx, SpatialStrategy strategy, Func <SpatialArgs, Query> createQuery)
{
var analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30);
using (var luceneDir = new RandomIdRAMDirectory())
{
string id1 = 1.ToString();
string id2 = 2.ToString();
string id3 = 3.ToString();
string id4 = 4.ToString();
using (var indexer = new TestIndex(luceneDir, analyzer))
{
indexer.DocumentWriting += (sender, args) => Indexer_DocumentWriting(args, ctx, strategy);
indexer.IndexItems(new[] {
ValueSet.FromObject(id1, "content",
new { nodeName = "location 1", bodyText = "Zanzibar is in Africa", lat = -6.1357, lng = 39.3621 }),
ValueSet.FromObject(id2, "content",
new { nodeName = "location 2", bodyText = "In Canada there is a town called Sydney in Nova Scotia", lat = 46.1368, lng = -60.1942 }),
ValueSet.FromObject(id3, "content",
new { nodeName = "location 3", bodyText = "Sydney is the capital of NSW in Australia", lat = -33.8688, lng = 151.2093 }),
ValueSet.FromObject(id4, "content",
new { nodeName = "location 4", bodyText = "Somewhere unknown", lat = 50, lng = 50 })
});
DoSpatialSearch(ctx, strategy, indexer, SearchRadius, id3, createQuery, lat: -33, lng: 151);
DoSpatialSearch(ctx, strategy, indexer, SearchRadius, id2, createQuery, lat: 46, lng: -60);
DoSpatialSearch(ctx, strategy, indexer, SearchRadius, id1, createQuery, lat: -6, lng: 39);
DoSpatialSearch(ctx, strategy, indexer, SearchRadius, id4, createQuery, lat: 50, lng: 50);
}
}
}
public void Ensure_Result_Has_All_Values()
{
using (var luceneDir = new RandomIdRAMDirectory())
using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
{
//var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
indexer.RebuildIndex();
//var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
var result = searcher.Search(searcher.CreateSearchCriteria().Id(1111).Compile());
Assert.IsNotNull(result);
Assert.AreEqual(1, result.TotalItemCount);
var searchItem = result.First();
var backedMedia = new ExamineBackedMedia(searchItem, indexer, searcher);
Assert.AreEqual(searchItem.Id, backedMedia.Id);
Assert.AreEqual(searchItem.Fields["sortOrder"], backedMedia.SortOrder.ToString());
Assert.AreEqual(searchItem.Fields["urlName"], backedMedia.UrlName);
Assert.AreEqual(DateTools.StringToDate(searchItem.Fields["createDate"]), backedMedia.CreateDate);
Assert.AreEqual(DateTools.StringToDate(searchItem.Fields["updateDate"]), backedMedia.UpdateDate);
Assert.AreEqual(Guid.Parse(searchItem.Fields["version"]), backedMedia.Version);
Assert.AreEqual(searchItem.Fields["level"], backedMedia.Level.ToString());
Assert.AreEqual(searchItem.Fields["writerID"], backedMedia.WriterID.ToString());
Assert.AreEqual(searchItem.Fields["writerID"], backedMedia.CreatorID.ToString()); //there's only writerId in the xml
Assert.AreEqual(searchItem.Fields["writerName"], backedMedia.CreatorName);
Assert.AreEqual(searchItem.Fields["writerName"], backedMedia.WriterName); //tehre's only writer name in the xml
}
}
public void Can_Add_Multiple_Values_To_Single_Index_Field()
{
using (var d = new RandomIdRAMDirectory())
using (var customIndexer = IndexInitializer.GetUmbracoIndexer(d))
{
EventHandler <DocumentWritingEventArgs> handler = (sender, args) =>
{
args.Document.Add(new Field("headerText", "another value", Field.Store.YES, Field.Index.ANALYZED));
};
customIndexer.DocumentWriting += handler;
customIndexer.RebuildIndex();
customIndexer.DocumentWriting += handler;
var customSearcher = IndexInitializer.GetLuceneSearcher(d);
var results = customSearcher.Search(customSearcher.CreateSearchCriteria().NodeName("home").Compile());
Assert.Greater(results.TotalItemCount, 0);
foreach (var result in results)
{
var vals = result.GetValues("headerText");
Assert.AreEqual(2, vals.Count());
Assert.AreEqual("another value", vals.ElementAt(1));
}
}
}
public void Number_Field()
{
using (var luceneDir = new RandomIdRAMDirectory())
using (var indexer = new TestIndex(
new FieldDefinitionCollection(new FieldDefinition("item2", "number")),
luceneDir,
new StandardAnalyzer(Version.LUCENE_30)))
{
indexer.IndexItem(new ValueSet(1.ToString(), "content",
new Dictionary <string, IEnumerable <object> >
{
{ "item1", new List <object>(new[] { "value1" }) },
{ "item2", new List <object>(new object[] { 123456 }) }
}));
using (var s = (LuceneSearcher)indexer.GetSearcher())
{
var luceneSearcher = s.GetLuceneSearcher();
var fields = luceneSearcher.Doc(luceneSearcher.MaxDoc - 1).GetFields().ToArray();
var valType = indexer.FieldValueTypeCollection.GetValueType("item2");
Assert.AreEqual(typeof(Int32Type), valType.GetType());
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == "item2"));
}
}
}
public void Index_Rebuild_Index()
{
using (var d = new RandomIdRAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(d))
{
indexer.RebuildIndex();
var searcher = IndexInitializer.GetUmbracoSearcher(d);
//get searcher and reader to get stats
var r = ((IndexSearcher)searcher.GetSearcher()).GetIndexReader();
//there's 16 fields in the index, but 3 sorted fields
var fields = r.GetFieldNames(IndexReader.FieldOption.ALL);
Assert.AreEqual(21, fields.Count());
//ensure there's 3 sorting fields
Assert.AreEqual(4, fields.Count(x => x.StartsWith(LuceneIndexer.SortedFieldNamePrefix)));
//there should be 11 documents (10 content, 1 media)
Assert.AreEqual(10, r.NumDocs());
//test for the special fields to ensure they are there:
Assert.AreEqual(1, fields.Where(x => x == LuceneIndexer.IndexNodeIdFieldName).Count());
Assert.AreEqual(1, fields.Where(x => x == LuceneIndexer.IndexTypeFieldName).Count());
Assert.AreEqual(1, fields.Where(x => x == UmbracoContentIndexer.IndexPathFieldName).Count());
Assert.AreEqual(1, fields.Where(x => x == UmbracoContentIndexer.NodeTypeAliasFieldName).Count());
}
}
public void Index_Ensure_No_Duplicates_In_Non_Async()
{
using (var d = new RandomIdRAMDirectory())
using (var indexer = IndexInitializer.GetUmbracoIndexer(d))
{
indexer.RebuildIndex();
var searcher = IndexInitializer.GetUmbracoSearcher(d);
//set to 5 so optmization occurs frequently
indexer.OptimizationCommitThreshold = 5;
//get a node from the data repo
var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]")
.Root
.Elements()
.First();
//get the id for th node we're re-indexing.
var id = (int)node.Attribute("id");
//reindex the same node a bunch of times
for (var i = 0; i < 29; i++)
{
indexer.ReIndexNode(node, IndexTypes.Content);
}
//ensure no duplicates
var results = searcher.Search(searcher.CreateSearchCriteria().Id(id).Compile());
Assert.AreEqual(1, results.Count());
}
}
public void MultiIndex_Simple_Search()
{
var analyzer = new StandardAnalyzer(Version.LUCENE_30);
using (var luceneDir1 = new RandomIdRAMDirectory())
using (var luceneDir2 = new RandomIdRAMDirectory())
using (var luceneDir3 = new RandomIdRAMDirectory())
using (var luceneDir4 = new RandomIdRAMDirectory())
using (var indexer1 = new TestIndex(luceneDir1, analyzer))
using (var indexer2 = new TestIndex(luceneDir2, analyzer))
using (var indexer3 = new TestIndex(luceneDir3, analyzer))
using (var indexer4 = new TestIndex(luceneDir4, analyzer))
{
indexer1.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "value1", item2 = "The agitated zebras gallop back and forth in short, panicky dashes, then skitter off into the absolute darkness." }));
indexer2.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "value2", item2 = "The festival lasts five days and celebrates the victory of good over evil, light over darkness, and knowledge over ignorance." }));
indexer3.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "value3", item2 = "They are expected to confront the darkness and show evidence that they have done so in their papers" }));
indexer4.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "value4", item2 = "Scientists believe the lake could be home to cold-loving microbial life adapted to living in total darkness." }));
indexer3.IndexItem(ValueSet.FromObject(2.ToString(), "content", new { item1 = "value3", item2 = "Scotch scotch scotch, i love scotch" }));
indexer4.IndexItem(ValueSet.FromObject(2.ToString(), "content", new { item1 = "value4", item2 = "60% of the time, it works everytime" }));
using (var searcher = new MultiIndexSearcher("testSearcher",
new[] { indexer1, indexer2, indexer3, indexer4 },
analyzer))
{
var result = searcher.Search("darkness");
Assert.AreEqual(4, result.TotalItemCount);
foreach (var r in result)
{
Console.WriteLine("Score = " + r.Score);
}
}
}
}
public void Can_Add_Doc_With_Fields()
{
using (var luceneDir = new RandomIdRAMDirectory())
using (var indexer = new TestIndex(luceneDir, new StandardAnalyzer(Version.LUCENE_30)))
{
indexer.IndexItem(new ValueSet(1.ToString(), "content", "test",
new Dictionary <string, IEnumerable <object> >
{
{ "item1", new List <object>(new[] { "value1" }) },
{ "item2", new List <object>(new[] { "value2" }) }
}));
using (var s = (LuceneSearcher)indexer.GetSearcher())
{
var luceneSearcher = s.GetLuceneSearcher();
var fields = luceneSearcher.Doc(0).GetFields().ToArray();
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == "item1"));
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == "item2"));
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == LuceneIndex.ItemTypeFieldName));
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == LuceneIndex.ItemIdFieldName));
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == LuceneIndex.CategoryFieldName));
Assert.AreEqual("value1", fields.Single(x => x.Name == "item1").StringValue);
Assert.AreEqual("value2", fields.Single(x => x.Name == "item2").StringValue);
Assert.AreEqual("test", fields.Single(x => x.Name == LuceneIndex.ItemTypeFieldName).StringValue);
Assert.AreEqual("1", fields.Single(x => x.Name == LuceneIndex.ItemIdFieldName).StringValue);
Assert.AreEqual("content", fields.Single(x => x.Name == LuceneIndex.CategoryFieldName).StringValue);
}
}
}
public void Ensure_Children_Are_Sorted()
{
using (var luceneDir = new RandomIdRAMDirectory())
using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
{
//var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
indexer.RebuildIndex();
//var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
var result = searcher.Search(searcher.CreateSearchCriteria().Id(1111).Compile());
Assert.IsNotNull(result);
Assert.AreEqual(1, result.TotalItemCount);
var searchItem = result.First();
var backedMedia = new ExamineBackedMedia(searchItem, indexer, searcher);
var children = backedMedia.ChildrenAsList.Value;
var currSort = 0;
Assert.Greater(children.Count, 0);
for (var i = 0; i < children.Count; i++)
{
Assert.GreaterOrEqual(children[i].SortOrder, currSort);
currSort = children[i].SortOrder;
}
}
}
public void Index_Exists()
{
using (var luceneDir = new RandomIdRAMDirectory())
using (var indexer = new TestIndex(luceneDir, new StandardAnalyzer(Version.LUCENE_30)))
{
indexer.EnsureIndex(true);
Assert.IsTrue(indexer.IndexExists());
}
}
public void Rebuild_Index()
{
using (var d = new RandomIdRAMDirectory())
using (var indexer = new TestIndex(d, new StandardAnalyzer(Version.LUCENE_30)))
{
indexer.CreateIndex();
indexer.IndexItems(indexer.AllData());
var indexWriter = indexer.GetIndexWriter();
var reader = indexWriter.GetReader();
Assert.AreEqual(100, reader.NumDocs());
}
}
public void MultiIndex_Field_Count()
{
var analyzer = new StandardAnalyzer(Version.LUCENE_30);
using (var luceneDir1 = new RandomIdRAMDirectory())
using (var luceneDir2 = new RandomIdRAMDirectory())
using (var luceneDir3 = new RandomIdRAMDirectory())
using (var luceneDir4 = new RandomIdRAMDirectory())
using (var indexer1 = new TestIndex(luceneDir1, analyzer)
{
RunAsync = false
})
using (var indexer2 = new TestIndex(luceneDir2, analyzer)
{
RunAsync = false
})
using (var indexer3 = new TestIndex(luceneDir3, analyzer)
{
RunAsync = false
})
using (var indexer4 = new TestIndex(luceneDir4, analyzer)
{
RunAsync = false
})
{
indexer1.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "hello", item2 = "The agitated zebras gallop back and forth in short, panicky dashes, then skitter off into the absolute darkness." }));
indexer2.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "world", item2 = "The festival lasts five days and celebrates the victory of good over evil, light over darkness, and knowledge over ignorance." }));
indexer3.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "here", item2 = "They are expected to confront the darkness and show evidence that they have done so in their papers" }));
indexer4.IndexItem(ValueSet.FromObject(1.ToString(), "content", new { item1 = "are", item2 = "Scientists believe the lake could be home to cold-loving microbial life adapted to living in total darkness." }));
indexer3.IndexItem(ValueSet.FromObject(2.ToString(), "content", new { item3 = "some", item2 = "Scotch scotch scotch, i love scotch" }));
indexer4.IndexItem(ValueSet.FromObject(2.ToString(), "content", new { item4 = "values", item2 = "60% of the time, it works everytime" }));
using (var searcher = new MultiIndexSearcher("testSearcher",
new[] { indexer1, indexer2, indexer3, indexer4 },
analyzer))
{
var result = searcher.GetAllIndexedFields();
//will be item1 , item2, item3, and item4
Assert.AreEqual(4, result.Count());
foreach (var s in new[] { "item1", "item2", "item3", "item4" })
{
Assert.IsTrue(result.Contains(s));
}
}
}
}
public string Search(string culture)
{
using (var luceneDir = new RandomIdRAMDirectory())
{
var fieldNames = new[] { "title", "title_en-us", "title_fr-fr" };
using (var indexer = CreateTestIndex(luceneDir, fieldNames))
{
var pcq = CreatePublishedContentQuery(indexer);
var results = pcq.Search("Products", culture, "TestIndex");
var ids = results.Select(x => x.Content.Id).ToArray();
return(string.Join(", ", ids));
}
}
}
public void Can_Add_One_Document()
{
using (var luceneDir = new RandomIdRAMDirectory())
using (var indexer = new TestIndex(luceneDir, new StandardAnalyzer(Version.LUCENE_30)))
{
indexer.IndexItem(new ValueSet(1.ToString(), "content",
new Dictionary <string, IEnumerable <object> >
{
{ "item1", new List <object>(new[] { "value1" }) },
{ "item2", new List <object>(new[] { "value2" }) }
}));
var indexWriter = indexer.GetIndexWriter();
var reader = indexWriter.GetReader();
Assert.AreEqual(1, reader.NumDocs());
}
}
public void AncestorsOrSelf_With_Examine()
{
using (var luceneDir = new RandomIdRAMDirectory())
using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
{
//var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
indexer.RebuildIndex();
var ctx = GetUmbracoContext("/test", 1234);
//var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
//we are using the media.xml media to test the examine results implementation, see the media.xml file in the ExamineHelpers namespace
var publishedMedia = cache.GetById(3113);
var ancestors = publishedMedia.AncestorsOrSelf();
Assert.IsTrue(ancestors.Select(x => x.Id).ContainsAll(new[] { 3113, 2112, 2222, 1111 }));
}
}
public void Can_Add_Doc_With_Easy_Fields()
{
using (var luceneDir = new RandomIdRAMDirectory())
using (var indexer = new TestIndex(luceneDir, new StandardAnalyzer(Version.LUCENE_30)))
{
indexer.IndexItem(ValueSet.FromObject(1.ToString(), "content",
new { item1 = "value1", item2 = "value2" }));
using (var s = (LuceneSearcher)indexer.GetSearcher())
{
var luceneSearcher = s.GetLuceneSearcher();
var fields = luceneSearcher.Doc(0).GetFields().ToArray();
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == "item1"));
Assert.IsNotNull(fields.SingleOrDefault(x => x.Name == "item2"));
Assert.AreEqual("value1", fields.Single(x => x.Name == "item1").StringValue);
Assert.AreEqual("value2", fields.Single(x => x.Name == "item2").StringValue);
}
}
}
public void Do_Not_Find_In_Recycle_Bin()
{
using (var luceneDir = new RandomIdRAMDirectory())
using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
{
//var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
indexer.RebuildIndex();
//var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
var ctx = GetUmbracoContext("/test", 1234);
var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
//ensure it is found
var publishedMedia = cache.GetById(3113);
Assert.IsNotNull(publishedMedia);
//move item to recycle bin
var newXml = XElement.Parse(@"<node id='3113' version='5b3e46ab-3e37-4cfa-ab70-014234b5bd33' parentID='-21' level='1' writerID='0' nodeType='1032' template='0' sortOrder='2' createDate='2010-05-19T17:32:46' updateDate='2010-05-19T17:32:46' nodeName='Another Umbraco Image' urlName='acnestressscrub' writerName='Administrator' nodeTypeAlias='Image' path='-1,-21,3113'>
<data alias='umbracoFile'><![CDATA[/media/1234/blah.pdf]]></data>
<data alias='umbracoWidth'>115</data>
<data alias='umbracoHeight'>268</data>
<data alias='umbracoBytes'>10726</data>
<data alias='umbracoExtension'>jpg</data>
</node>" );
indexer.ReIndexNode(newXml, "media");
//ensure it still exists in the index (raw examine search)
var criteria = searcher.CreateSearchCriteria();
var filter = criteria.Id(3113);
var found = searcher.Search(filter.Compile());
Assert.IsNotNull(found);
Assert.AreEqual(1, found.TotalItemCount);
//ensure it does not show up in the published media store
var recycledMedia = cache.GetById(3113);
Assert.IsNull(recycledMedia);
}
}
public void Track_Readers()
{
var analyzer = new StandardAnalyzer(Version.LUCENE_30);
using (var luceneDir = new RandomIdRAMDirectory())
using (var indexer = new TestIndex(luceneDir, analyzer))
{
indexer.IndexItems(new[] {
ValueSet.FromObject(1.ToString(), "content",
new { nodeName = "umbraco", headerText = "world", writerName = "administrator" }),
ValueSet.FromObject(2.ToString(), "content",
new { nodeName = "umbraco", headerText = "umbraco", writerName = "administrator" }),
ValueSet.FromObject(3.ToString(), "content",
new { nodeName = "umbraco", headerText = "umbraco", writerName = "administrator" }),
ValueSet.FromObject(4.ToString(), "content",
new { nodeName = "hello", headerText = "world", writerName = "blah" })
});
LuceneSearcher searcher = (LuceneSearcher)indexer.GetSearcher();
IndexSearcher luceneSearcher = (IndexSearcher)searcher.GetLuceneSearcher();
//Arrange
var sc = searcher.CreateQuery("content").Field("writerName", "administrator");
//Act
var results = sc.Execute();
using (var e1 = results.GetEnumerator())
{
Assert.AreEqual(2, luceneSearcher.IndexReader.RefCount);
using (var e2 = results.Skip(2).GetEnumerator())
{
Assert.AreEqual(3, luceneSearcher.IndexReader.RefCount);
}
Assert.AreEqual(2, luceneSearcher.IndexReader.RefCount);
}
Assert.AreEqual(1, luceneSearcher.IndexReader.RefCount);
}
}
public void MultiIndex_Simple_Search()
{
using (var cwsDir = new RandomIdRAMDirectory())
using (var pdfDir = new RandomIdRAMDirectory())
using (var simpleDir = new RandomIdRAMDirectory())
using (var conventionDir = new RandomIdRAMDirectory())
{
//get all of the indexers and rebuild them all first
var indexers = new IIndexer[]
{
IndexInitializer.GetUmbracoIndexer(cwsDir),
IndexInitializer.GetSimpleIndexer(simpleDir),
IndexInitializer.GetUmbracoIndexer(conventionDir)
};
foreach (var i in indexers)
{
i.RebuildIndex();
}
using (var cwsIndexer = IndexInitializer.GetUmbracoIndexer(cwsDir))
{
cwsIndexer.RebuildIndex();
//now get the multi index searcher for all indexes
using (var searcher = IndexInitializer.GetMultiSearcher(pdfDir, simpleDir, conventionDir, cwsDir))
using (var cwsSearcher = IndexInitializer.GetUmbracoSearcher(cwsDir))
{
var cwsResult = cwsSearcher.Search("sam", false);
var result = searcher.Search("sam", false);
//ensure there's more results than just the one index
Assert.IsTrue(cwsResult.Count() < result.Count());
//there should be 8
Assert.AreEqual(8, result.Count(), "Results returned for 'sam' should be equal to 5 with the StandardAnalyzer");
}
};
}
}
public void Can_Have_Multiple_Values_In_Fields()
{
using (var luceneDir = new RandomIdRAMDirectory())
using (var indexer = new TestIndex(luceneDir, new StandardAnalyzer(Version.LUCENE_30)))
{
indexer.IndexItem(new ValueSet(1.ToString(), "content",
new Dictionary <string, IEnumerable <object> >
{
{
"item1", new List <object> {
"subval1", "subval2"
}
},
{
"item2", new List <object> {
"subval1", "subval2", "subval3"
}
}
}));
using (var s = (LuceneSearcher)indexer.GetSearcher())
{
var luceneSearcher = s.GetLuceneSearcher();
var fields = luceneSearcher.Doc(0).GetFields().ToArray();
Assert.AreEqual(2, fields.Count(x => x.Name == "item1"));
Assert.AreEqual(3, fields.Count(x => x.Name == "item2"));
Assert.AreEqual("subval1", fields.Where(x => x.Name == "item1").ElementAt(0).StringValue);
Assert.AreEqual("subval2", fields.Where(x => x.Name == "item1").ElementAt(1).StringValue);
Assert.AreEqual("subval1", fields.Where(x => x.Name == "item2").ElementAt(0).StringValue);
Assert.AreEqual("subval2", fields.Where(x => x.Name == "item2").ElementAt(1).StringValue);
Assert.AreEqual("subval3", fields.Where(x => x.Name == "item2").ElementAt(2).StringValue);
}
}
}
public void Dont_Initialize_Searchers_On_Dispose_If_Not_Already_Initialized()
{
var analyzer = new StandardAnalyzer(Version.LUCENE_30);
using (var luceneDir1 = new RandomIdRAMDirectory())
using (var luceneDir2 = new RandomIdRAMDirectory())
using (var luceneDir3 = new RandomIdRAMDirectory())
using (var luceneDir4 = new RandomIdRAMDirectory())
using (var indexer1 = new TestIndex(luceneDir1, analyzer))
using (var indexer2 = new TestIndex(luceneDir2, analyzer))
using (var indexer3 = new TestIndex(luceneDir3, analyzer))
using (var indexer4 = new TestIndex(luceneDir4, analyzer))
{
var searcher = new MultiIndexSearcher("testSearcher",
new[] { indexer1, indexer2, indexer3, indexer4 },
analyzer);
Assert.IsFalse(searcher.SearchersInitialized);
searcher.Dispose();
Assert.IsFalse(searcher.SearchersInitialized);
}
}
public void Ensure_Children_Sorted_With_Examine()
{
using (var luceneDir = new RandomIdRAMDirectory())
using (var writer = new IndexWriter(luceneDir, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
using (var indexer = IndexInitializer.GetUmbracoIndexer(writer))
using (var searcher = IndexInitializer.GetUmbracoSearcher(writer))
{
//var indexer = IndexInitializer.GetUmbracoIndexer(luceneDir);
indexer.RebuildIndex();
//var searcher = IndexInitializer.GetUmbracoSearcher(luceneDir);
var ctx = GetUmbracoContext("/test", 1234);
var cache = new ContextualPublishedMediaCache(new PublishedMediaCache(ctx.Application, searcher, indexer), ctx);
//we are using the media.xml media to test the examine results implementation, see the media.xml file in the ExamineHelpers namespace
var publishedMedia = cache.GetById(1111);
var rootChildren = publishedMedia.Children().ToArray();
var currSort = 0;
for (var i = 0; i < rootChildren.Count(); i++)
{
Assert.GreaterOrEqual(rootChildren[i].SortOrder, currSort);
currSort = rootChildren[i].SortOrder;
}
}
}
public void Index_Read_And_Write_Ensure_No_Errors_In_Async(
int indexCount,
int searchCount,
int indexThreadCount,
int searchThreadCount,
int indexThreadWait,
int searchThreadWait,
bool inMemory)
{
// TODO: In this test can we ensure all readers are tracked and closed?
// TODO: In the search part, we should be searching in various ways and also with skip
DirectoryInfo temp = null;
Lucene.Net.Store.Directory directory;
if (inMemory)
{
directory = new RandomIdRAMDirectory();
}
else
{
// try to clear out old files
var tempBasePath = Path.Combine(Path.GetTempPath(), "ExamineTests");
if (System.IO.Directory.Exists(tempBasePath))
{
try
{
System.IO.Directory.Delete(tempBasePath, true);
}
catch
{
}
}
var tempPath = Path.Combine(tempBasePath, Guid.NewGuid().ToString());
System.IO.Directory.CreateDirectory(tempPath);
temp = new DirectoryInfo(tempPath);
directory = new SimpleFSDirectory(temp);
}
try
{
using (var d = directory)
using (var writer = new IndexWriter(d, new CultureInvariantStandardAnalyzer(Version.LUCENE_30), IndexWriter.MaxFieldLength.LIMITED))
using (var customIndexer = new TestIndex(writer))
using (var customSearcher = (LuceneSearcher)customIndexer.GetSearcher())
{
var waitHandle = new ManualResetEvent(false);
void OperationComplete(object sender, IndexOperationEventArgs e)
{
//signal that we are done
waitHandle.Set();
}
//add the handler for optimized since we know it will be optimized last based on the commit count
customIndexer.IndexOperationComplete += OperationComplete;
//remove the normal indexing error handler
//customIndexer.IndexingError -= IndexInitializer.IndexingError;
//run in async mode
customIndexer.RunAsync = true;
//get all nodes
var nodes = _contentService.GetPublishedContentByXPath("//*[@isDoc]")
.Root
.Elements()
.ToList();
Func <int, XElement> getNode = (index) =>
{
// clone it
return(new XElement(nodes[index]));
};
// we know there are 20 documents available, this is important for the getNode call
var idQueue = new ConcurrentQueue <int>(Enumerable.Range(1, 20));
var searchCountPerThread = Convert.ToInt32(searchCount / searchThreadCount);
var indexCountPerThread = Convert.ToInt32(indexCount / indexThreadCount);
//spawn a bunch of threads to perform some reading
var tasks = new List <Task>();
Action <ISearcher> doSearch = (s) =>
{
try
{
for (var counter = 0; counter < searchCountPerThread; counter++)
{
//get next id and put it to the back of the list
int docId;
if (idQueue.TryDequeue(out docId))
{
idQueue.Enqueue(docId);
var r = s.CreateQuery().Id(docId.ToString()).Execute();
Console.WriteLine("searching thread: {0}, id: {1}, found: {2}", Thread.CurrentThread.ManagedThreadId, docId, r.Count());
Thread.Sleep(searchThreadWait);
}
}
}
catch (Exception ex)
{
Console.WriteLine("Search ERROR!! {0}", ex);
throw;
}
};
Action <IIndex> doIndex = (ind) =>
{
try
{
//reindex a nodes a bunch of times
for (var i = 0; i < indexCountPerThread; i++)
{
//get next id and put it to the back of the list
int docId;
if (idQueue.TryDequeue(out docId))
{
idQueue.Enqueue(docId);
var node = getNode(docId - 1);
node.Attribute("id").Value = docId.ToString(CultureInfo.InvariantCulture);
Console.WriteLine("Indexing {0}", docId);
ind.IndexItems(new[] { node.ConvertToValueSet(IndexTypes.Content) });
Thread.Sleep(indexThreadWait);
}
}
}
catch (Exception ex)
{
Console.WriteLine("Index ERROR!! {0}", ex);
throw;
}
};
//indexing threads
for (var i = 0; i < indexThreadCount; i++)
{
var indexer = customIndexer;
tasks.Add(Task.Run(() => doIndex(indexer)));
}
//searching threads
for (var i = 0; i < searchThreadCount; i++)
{
var searcher = customSearcher;
tasks.Add(Task.Run(() => doSearch(searcher)));
}
try
{
Task.WaitAll(tasks.ToArray());
}
catch (AggregateException e)
{
var sb = new StringBuilder();
sb.Append(e.Message + ": ");
foreach (var v in e.InnerExceptions)
{
sb.Append(v.Message + "; ");
}
Assert.Fail(sb.ToString());
}
var results = customSearcher.CreateQuery().All().Execute();
Assert.AreEqual(20, results.Count());
//reset the async mode and remove event handler
//customIndexer.IndexingError += IndexInitializer.IndexingError;
//customIndexer.RunAsync = false;
//wait until we are done
waitHandle.WaitOne();
writer.WaitForMerges();
writer.Dispose(true);
results = customSearcher.CreateQuery().All().Execute();
Assert.AreEqual(20, results.Count());
}
}
finally
{
if (temp != null)
{
try
{
temp.Delete(true);
}
catch (Exception ex)
{
Console.WriteLine("Could not delete temp folder {0}", ex);
}
}
}
}
public void Can_Overwrite_Index_During_Indexing_Operation()
{
using (var d = new RandomIdRAMDirectory())
using (var writer = new IndexWriter(d, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29), IndexWriter.MaxFieldLength.LIMITED))
using (var customIndexer = IndexInitializer.GetUmbracoIndexer(writer))
using (var customSearcher = IndexInitializer.GetUmbracoSearcher(writer))
{
var waitHandle = new ManualResetEvent(false);
EventHandler operationComplete = (sender, e) =>
{
//signal that we are done
waitHandle.Set();
};
//add the handler for optimized since we know it will be optimized last based on the commit count
customIndexer.IndexOperationComplete += operationComplete;
//remove the normal indexing error handler
customIndexer.IndexingError -= IndexInitializer.IndexingError;
//run in async mode
customIndexer.RunAsync = true;
//get a node from the data repo
var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]")
.Root
.Elements()
.First();
//get the id for th node we're re-indexing.
var id = (int)node.Attribute("id");
//spawn a bunch of threads to perform some reading
var tasks = new List <Task>();
//reindex the same node a bunch of times - then while this is running we'll overwrite below
for (var i = 0; i < 1000; i++)
{
var indexer = customIndexer;
tasks.Add(Task.Factory.StartNew(() =>
{
//get next id and put it to the back of the list
int docId = i;
var cloned = new XElement(node);
cloned.Attribute("id").Value = docId.ToString(CultureInfo.InvariantCulture);
Debug.WriteLine("Indexing {0}", docId);
indexer.ReIndexNode(cloned, IndexTypes.Content);
}, TaskCreationOptions.LongRunning));
}
Thread.Sleep(100);
//overwrite!
customIndexer.EnsureIndex(true);
try
{
Task.WaitAll(tasks.ToArray());
}
catch (AggregateException e)
{
var sb = new StringBuilder();
sb.Append(e.Message + ": ");
foreach (var v in e.InnerExceptions)
{
sb.Append(v.Message + "; ");
}
Assert.Fail(sb.ToString());
}
//reset the async mode and remove event handler
customIndexer.IndexingError += IndexInitializer.IndexingError;
customIndexer.RunAsync = false;
//wait until we are done
waitHandle.WaitOne();
writer.WaitForMerges();
//ensure no data since it's a new index
var results = customSearcher.Search(customSearcher.CreateSearchCriteria().NodeName("Home").Compile());
//should be less than the total inserted because we overwrote it in the middle of processing
Debug.WriteLine("TOTAL RESULTS: " + results.TotalItemCount);
Assert.Less(results.Count(), 1000);
}
}
public void Index_Read_And_Write_Ensure_No_Errors_In_Async()
{
using (var d = new RandomIdRAMDirectory())
using (var writer = new IndexWriter(d, new CultureInvariantStandardAnalyzer(Version.LUCENE_30), IndexWriter.MaxFieldLength.LIMITED))
using (var customIndexer = new TestIndex(writer))
using (var customSearcher = (LuceneSearcher)customIndexer.GetSearcher())
{
var waitHandle = new ManualResetEvent(false);
void OperationComplete(object sender, IndexOperationEventArgs e)
{
//signal that we are done
waitHandle.Set();
}
//add the handler for optimized since we know it will be optimized last based on the commit count
customIndexer.IndexOperationComplete += OperationComplete;
//remove the normal indexing error handler
customIndexer.IndexingError -= IndexInitializer.IndexingError;
//run in async mode
customIndexer.RunAsync = false;
//get a node from the data repo
var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]")
.Root
.Elements()
.First();
var idQueue = new ConcurrentQueue <int>(Enumerable.Range(1, 10));
var searchThreadCount = 500;
var indexThreadCount = 10;
var searchCount = 10700;
var indexCount = 100;
var searchCountPerThread = Convert.ToInt32(searchCount / searchThreadCount);
var indexCountPerThread = Convert.ToInt32(indexCount / indexThreadCount);
//spawn a bunch of threads to perform some reading
var tasks = new List <Task>();
Action <ISearcher> doSearch = (s) =>
{
try
{
for (var counter = 0; counter < searchCountPerThread; counter++)
{
//get next id and put it to the back of the list
int docId;
if (idQueue.TryDequeue(out docId))
{
idQueue.Enqueue(docId);
var r = s.CreateQuery().Id(docId.ToString()).Execute();
Debug.WriteLine("searching thread: {0}, id: {1}, found: {2}", Thread.CurrentThread.ManagedThreadId, docId, r.Count());
Thread.Sleep(50);
}
}
}
catch (Exception ex)
{
Debug.WriteLine("ERROR!! {0}", ex);
throw;
}
};
Action <IIndex> doIndex = (ind) =>
{
try
{
//reindex the same node a bunch of times
for (var i = 0; i < indexCountPerThread; i++)
{
//get next id and put it to the back of the list
int docId;
if (idQueue.TryDequeue(out docId))
{
idQueue.Enqueue(docId);
var cloned = new XElement(node);
cloned.Attribute("id").Value = docId.ToString(CultureInfo.InvariantCulture);
Debug.WriteLine("Indexing {0}", docId);
ind.IndexItems(new[] { cloned.ConvertToValueSet(IndexTypes.Content) });
Thread.Sleep(100);
}
}
}
catch (Exception ex)
{
Debug.WriteLine("ERROR!! {0}", ex);
throw;
}
};
//indexing threads
for (var i = 0; i < indexThreadCount; i++)
{
var indexer = customIndexer;
tasks.Add(Task.Factory.StartNew(() => doIndex(indexer), TaskCreationOptions.LongRunning));
}
//searching threads
for (var i = 0; i < searchThreadCount; i++)
{
var searcher = customSearcher;
tasks.Add(Task.Factory.StartNew(() => doSearch(searcher), TaskCreationOptions.LongRunning));
}
try
{
Task.WaitAll(tasks.ToArray());
}
catch (AggregateException e)
{
var sb = new StringBuilder();
sb.Append(e.Message + ": ");
foreach (var v in e.InnerExceptions)
{
sb.Append(v.Message + "; ");
}
Assert.Fail(sb.ToString());
}
//reset the async mode and remove event handler
customIndexer.IndexingError += IndexInitializer.IndexingError;
customIndexer.RunAsync = false;
//wait until we are done
waitHandle.WaitOne();
writer.WaitForMerges();
var results = customSearcher.CreateQuery().Field("nodeName", (IExamineValue) new ExamineValue(Examineness.Explicit, "Home")).Execute();
Assert.AreEqual(10, results.Count());
}
}
public void Index_Ensure_No_Duplicates_In_Async()
{
using (var d = new RandomIdRAMDirectory())
using (var writer = new IndexWriter(d, new CultureInvariantStandardAnalyzer(Version.LUCENE_30), IndexWriter.MaxFieldLength.LIMITED))
using (var customIndexer = new TestIndex(writer))
//using (var customSearcher = (LuceneSearcher)customIndexer.GetSearcher())
{
var waitHandle = new ManualResetEvent(false);
void OperationComplete(object sender, IndexOperationEventArgs e)
{
//signal that we are done
waitHandle.Set();
}
//add the handler for optimized since we know it will be optimized last based on the commit count
customIndexer.IndexOperationComplete += OperationComplete;
//remove the normal indexing error handler
customIndexer.IndexingError -= IndexInitializer.IndexingError;
//run in async mode
customIndexer.RunAsync = true;
//get a node from the data repo
var idQueue = new ConcurrentQueue <int>(Enumerable.Range(1, 3));
var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]")
.Root
.Elements()
.First();
//reindex the same nodes a bunch of times
for (var i = 0; i < idQueue.Count * 20; i++)
{
//get next id and put it to the back of the list
int docId;
if (idQueue.TryDequeue(out docId))
{
idQueue.Enqueue(docId);
var cloned = new XElement(node);
cloned.Attribute("id").Value = docId.ToString(CultureInfo.InvariantCulture);
Debug.WriteLine("Indexing {0}", docId);
customIndexer.IndexItems(new[] { cloned.ConvertToValueSet(IndexTypes.Content) });
Thread.Sleep(100);
}
}
//reset the async mode and remove event handler
customIndexer.IndexingError += IndexInitializer.IndexingError;
customIndexer.RunAsync = false;
//wait until we are done
waitHandle.WaitOne();
writer.WaitForMerges();
//ensure no duplicates
var customSearcher = (LuceneSearcher)customIndexer.GetSearcher();
var results = customSearcher.CreateQuery().Field("nodeName", (IExamineValue) new ExamineValue(Examineness.Explicit, "Home")).Execute();
Assert.AreEqual(3, results.Count());
}
}
/// <summary>
/// Used to create and manage a testable index
/// </summary>
/// <param name="publishedValuesOnly"></param>
/// <param name="index"></param>
/// <param name="contentRebuilder"></param>
/// <param name="contentValueSetBuilder"></param>
/// <param name="parentId"></param>
/// <param name="contentService"></param>
/// <returns></returns>
protected IDisposable GetSynchronousContentIndex(
bool publishedValuesOnly,
out UmbracoContentIndex index,
out ContentIndexPopulator contentRebuilder,
out ContentValueSetBuilder contentValueSetBuilder,
int?parentId = null,
IContentService contentService = null)
{
contentValueSetBuilder = IndexInitializer.GetContentValueSetBuilder(publishedValuesOnly);
var sqlContext = Mock.Of <ISqlContext>(x => x.Query <IContent>() == Mock.Of <IQuery <IContent> >());
var dbFactory = Mock.Of <IUmbracoDatabaseFactory>(x => x.SqlContext == sqlContext);
if (contentService == null)
{
contentService = IndexInitializer.GetMockContentService();
}
contentRebuilder = IndexInitializer.GetContentIndexRebuilder(contentService, publishedValuesOnly, dbFactory);
var luceneDir = new RandomIdRAMDirectory();
ContentValueSetValidator validator;
// if only published values then we'll change the validator for tests to
// ensure we don't support protected nodes and that we
// mock the public access service for the special protected node.
if (publishedValuesOnly)
{
var publicAccessServiceMock = new Mock <IPublicAccessService>();
publicAccessServiceMock.Setup(x => x.IsProtected(It.IsAny <string>()))
.Returns((string path) =>
{
if (path.EndsWith("," + ExamineDemoDataContentService.ProtectedNode))
{
return(Attempt <PublicAccessEntry> .Succeed());
}
return(Attempt <PublicAccessEntry> .Fail());
});
var scopeProviderMock = new Mock <IScopeProvider>();
scopeProviderMock.Setup(x => x.CreateScope(
It.IsAny <IsolationLevel>(),
It.IsAny <RepositoryCacheMode>(),
It.IsAny <IEventDispatcher>(),
It.IsAny <IScopedNotificationPublisher>(),
It.IsAny <bool?>(),
It.IsAny <bool>(),
It.IsAny <bool>()))
.Returns(Mock.Of <IScope>);
validator = new ContentValueSetValidator(
publishedValuesOnly,
false,
publicAccessServiceMock.Object,
scopeProviderMock.Object,
parentId);
}
else
{
validator = new ContentValueSetValidator(publishedValuesOnly, parentId);
}
index = IndexInitializer.GetUmbracoIndexer(
HostingEnvironment,
RunningRuntimeState,
luceneDir,
validator: validator);
var syncMode = index.WithThreadingMode(IndexThreadingMode.Synchronous);
return(new DisposableWrapper(syncMode, index, luceneDir));
}
public void Can_Overwrite_Index_During_Indexing_Operation()
{
const int ThreadCount = 1000;
using (var d = new RandomIdRAMDirectory())
using (var writer = new IndexWriter(d, new CultureInvariantStandardAnalyzer(Version.LUCENE_30), IndexWriter.MaxFieldLength.LIMITED))
using (var customIndexer = new TestIndex(writer))
using (var customSearcher = (LuceneSearcher)customIndexer.GetSearcher())
{
var waitHandle = new ManualResetEvent(false);
var opCompleteCount = 0;
void OperationComplete(object sender, IndexOperationEventArgs e)
{
Interlocked.Increment(ref opCompleteCount);
Console.WriteLine($"OperationComplete: {opCompleteCount}");
if (opCompleteCount == ThreadCount)
{
//signal that we are done
waitHandle.Set();
}
}
//add the handler for optimized since we know it will be optimized last based on the commit count
customIndexer.IndexOperationComplete += OperationComplete;
//remove the normal indexing error handler
customIndexer.IndexingError -= IndexInitializer.IndexingError;
//run in async mode
customIndexer.RunAsync = true;
//get a node from the data repo
var node = _contentService.GetPublishedContentByXPath("//*[string-length(@id)>0 and number(@id)>0]")
.Root
.Elements()
.First();
//get the id for th node we're re-indexing.
var id = (int)node.Attribute("id");
//spawn a bunch of threads to perform some reading
var tasks = new List <Task>();
//reindex the same node a bunch of times - then while this is running we'll overwrite below
for (var i = 0; i < ThreadCount; i++)
{
var indexer = customIndexer;
tasks.Add(Task.Factory.StartNew(() =>
{
//get next id and put it to the back of the list
int docId = i;
var cloned = new XElement(node);
Debug.WriteLine("Indexing {0}", docId);
indexer.IndexItem(cloned.ConvertToValueSet(IndexTypes.Content));
}, TaskCreationOptions.LongRunning));
}
Thread.Sleep(100);
//overwrite!
customIndexer.EnsureIndex(true);
try
{
Task.WaitAll(tasks.ToArray());
}
catch (AggregateException e)
{
var sb = new StringBuilder();
sb.Append(e.Message + ": ");
foreach (var v in e.InnerExceptions)
{
sb.Append(v.Message + "; ");
}
Assert.Fail(sb.ToString());
}
//reset the async mode and remove event handler
customIndexer.IndexingError += IndexInitializer.IndexingError;
customIndexer.RunAsync = false;
//wait until we are done
waitHandle.WaitOne(TimeSpan.FromMinutes(2));
writer.WaitForMerges();
//ensure no data since it's a new index
var results = customSearcher.CreateQuery().Field("nodeName", (IExamineValue) new ExamineValue(Examineness.Explicit, "Home")).Execute();
//the total times that OperationComplete event should be fired is 1000
Assert.AreEqual(1000, opCompleteCount);
//should be less than the total inserted because we overwrote it in the middle of processing
Debug.WriteLine("TOTAL RESULTS: " + results.TotalItemCount);
Assert.Less(results.Count(), 1000);
}
}
