private void DoAssert(Document doc, bool fromIndex)
{
System.String[] keywordFieldValues = doc.GetValues("keyword");
System.String[] textFieldValues = doc.GetValues("text");
System.String[] unindexedFieldValues = doc.GetValues("unindexed");
System.String[] unstoredFieldValues = doc.GetValues("unstored");
Assert.IsTrue(keywordFieldValues.Length == 2);
Assert.IsTrue(textFieldValues.Length == 2);
Assert.IsTrue(unindexedFieldValues.Length == 2);
// this test cannot work for documents retrieved from the index
// since unstored fields will obviously not be returned
if (!fromIndex)
{
Assert.IsTrue(unstoredFieldValues.Length == 2);
}
Assert.IsTrue(keywordFieldValues[0].Equals("test1"));
Assert.IsTrue(keywordFieldValues[1].Equals("test2"));
Assert.IsTrue(textFieldValues[0].Equals("test1"));
Assert.IsTrue(textFieldValues[1].Equals("test2"));
Assert.IsTrue(unindexedFieldValues[0].Equals("test1"));
Assert.IsTrue(unindexedFieldValues[1].Equals("test2"));
// this test cannot work for documents retrieved from the index
// since unstored fields will obviously not be returned
if (!fromIndex)
{
Assert.IsTrue(unstoredFieldValues[0].Equals("test1"));
Assert.IsTrue(unstoredFieldValues[1].Equals("test2"));
}
}
public static MarcellDocument ToDocument(this LuceneDocument source)
{
MarcellDocument result = new MarcellDocument
{
Id = source.GetValues("Id")?.FirstOrDefault(),
InternalId = Guid.Parse(source.GetValues("InternalId").First()),
ApprovalDate = source.GetDate("ApprovalDate"),
DocumentDate = source.GetDate("DocumentDate"),
EffectiveDate = source.GetDate("EffectiveDate"),
DocumentSimilarityData = new SimilarityData
{
ConsolidatedTokens = source.GetStringList("DocumentToken"),
ConsolidatedTopics = source.GetStringList("DocumentTopic"),
},
TokenCount = source.GetInt("TokenCount"),
DocumentType = source.GetValues("DocumentType")?.FirstOrDefault(),
OriginalType = source.GetValues("OriginalType")?.FirstOrDefault(),
Issuer = source.GetValues("Issuer")?.FirstOrDefault(),
Language = source.GetValues("Language")?.FirstOrDefault(),
Url = source.GetValues("Url")?.FirstOrDefault(),
RecognitionQuality = source.GetDouble("RecognitionQuality"),
IsStructured = source.GetBool("IsStructured"),
FileName = source.GetValues("FileName")?.FirstOrDefault(),
};
return(result);
}
public static Section ToSection(this LuceneDocument source)
{
SectionType secType;
Enum.TryParse(source.GetValues("Type").FirstOrDefault(), true, out secType);
Section result = new Section
{
Id = source.GetValues("Id").FirstOrDefault(),
InternalId = Guid.Parse(source.GetValues("InternalId").First()),
DocumentSimilarityData = new SimilarityData
{
ConsolidatedTokens = source.GetStringList("DocumentToken"),
ConsolidatedTopics = source.GetStringList("DocumentTopic"),
},
SectionSimilarityData = new SimilarityData
{
ConsolidatedTokens = source.GetStringList("SectionToken"),
ConsolidatedTopics = source.GetStringList("SectionTopic"),
},
TokenCount = source.GetInt("TokenCount"),
Language = source.GetValues("Language")?.FirstOrDefault(),
RecognitionQuality = source.GetDouble("RecognitionQuality"),
Type = secType,
Text = source.GetValues("Text")?.FirstOrDefault(),
ParentId = Guid.Parse(source.GetValues("ParentDocumentId").First())
};
return(result);
}
public static Sentence ToSentence(this LuceneDocument source)
{
Sentence result = new Sentence
{
Id = source.GetValues("Id").FirstOrDefault(),
InternalId = Guid.Parse(source.GetValues("InternalId").First()),
DocumentSimilarityData = new SimilarityData
{
ConsolidatedTokens = source.GetStringList("DocumentToken"),
ConsolidatedTopics = source.GetStringList("DocumentTopic"),
},
SectionSimilarityData = new SimilarityData
{
ConsolidatedTokens = source.GetStringList("SectionToken"),
ConsolidatedTopics = source.GetStringList("SectionTopic"),
},
ParagraphSimilarityData = new SimilarityData
{
ConsolidatedTokens = source.GetStringList("ParagraphToken"),
ConsolidatedTopics = source.GetStringList("ParagraphTopic"),
},
SentenceSimilarityData = new SimilarityData
{
ConsolidatedTokens = source.GetStringList("SentenceToken"),
ConsolidatedTopics = source.GetStringList("SentenceTopic"),
},
TokenCount = source.GetInt("TokenCount"),
Language = source.GetValues("Language")?.FirstOrDefault(),
RecognitionQuality = source.GetDouble("RecognitionQuality"),
Order = source.GetInt("Order"),
Text = source.GetValues("Text")?.FirstOrDefault(),
ParentId = Guid.Parse(source.GetValues("ParentParagraphId").First())
};
return(result);
}
public BasicAuditEntry(Document doc, int luceneId)
{
User = doc.Get("user");
Role = doc.GetValues("role").ToList();
Id = new ID(doc.Get("id"));
Path = doc.Get("path");
DateTime tmp;
DateTime.TryParse(doc.Get("timestamp"), out tmp);
TimeStamp = tmp;
EventId = doc.Get("event");
Note = doc.Get("note");
Label = doc.Get("label");
Color = doc.Get("color");
Database = doc.Get("database");
Uid = luceneId.ToString();
}
public static void PopulateDocument(Document doc, IEnumerable<IFacetHandler> handlers)
{
StringBuilder tokenBuffer = new StringBuilder();
foreach (var handler in handlers)
{
string name = handler.Name;
string[] values = doc.GetValues(name);
if (values != null)
{
doc.RemoveFields(name);
foreach (string value in values)
{
doc.Add(new Field(name, value, Field.Store.NO, Field.Index.NOT_ANALYZED));
tokenBuffer.Append(' ').Append(value);
}
}
}
}
public static void PopulateDocument(Document doc, FieldConfiguration fConf)
{
var fields = fConf.GetFieldNames();
var tokenBuffer = new StringBuilder();
for (int i = 0; i < fields.Length; ++i)
{
var values = doc.GetValues(fields[i]);
if (values != null)
{
doc.RemoveFields(fields[i]);
for (int k = 0; k < values.Length; ++k)
{
doc.Add(new Field(fields[i], values[i], Field.Store.NO, Field.Index.NOT_ANALYZED));
tokenBuffer.Append(" ").Append(values[i]);
}
}
}
}
public static Paragraph ToParagraph(this LuceneDocument source)
{
ParagraphType pType;
Enum.TryParse(source.GetValues("Type")?.FirstOrDefault(), true, out pType);
Paragraph result = new Paragraph
{
Id = source.GetValues("Id").FirstOrDefault(),
InternalId = Guid.Parse(source.GetValues("InternalId").First()),
DocumentSimilarityData = new SimilarityData
{
ConsolidatedTokens = source.GetStringList("DocumentToken"),
ConsolidatedTopics = source.GetStringList("DocumentTopic"),
},
SectionSimilarityData = new SimilarityData
{
ConsolidatedTokens = source.GetStringList("SectionToken"),
ConsolidatedTopics = source.GetStringList("SectionTopic"),
},
ParagraphSimilarityData = new SimilarityData
{
ConsolidatedTokens = source.GetStringList("ParagraphToken"),
ConsolidatedTopics = source.GetStringList("ParagraphTopic"),
},
TokenCount = source.GetInt("TokenCount"),
Language = source.GetValues("Language")?.FirstOrDefault(),
RecognitionQuality = source.GetDouble("RecognitionQuality"),
Order = source.GetInt("Order"),
Text = source.GetValues("Text")?.FirstOrDefault(),
ParagraphNumber = source.GetValues("ParagraphNumber")?.FirstOrDefault(),
PointNumber = source.GetValues("PointNumber")?.FirstOrDefault(),
ParentId = Guid.Parse(source.GetValues("ParentSectionId").First())
};
return(result);
}
private void DoAssert(Document doc, bool fromIndex)
{
System.String[] keywordFieldValues = doc.GetValues("keyword");
System.String[] textFieldValues = doc.GetValues("text");
System.String[] unindexedFieldValues = doc.GetValues("unindexed");
System.String[] unstoredFieldValues = doc.GetValues("unstored");
Assert.IsTrue(keywordFieldValues.Length == 2);
Assert.IsTrue(textFieldValues.Length == 2);
Assert.IsTrue(unindexedFieldValues.Length == 2);
// this test cannot work for documents retrieved from the index
// since unstored fields will obviously not be returned
if (!fromIndex)
{
Assert.IsTrue(unstoredFieldValues.Length == 2);
}
Assert.IsTrue(keywordFieldValues[0].Equals("test1"));
Assert.IsTrue(keywordFieldValues[1].Equals("test2"));
Assert.IsTrue(textFieldValues[0].Equals("test1"));
Assert.IsTrue(textFieldValues[1].Equals("test2"));
Assert.IsTrue(unindexedFieldValues[0].Equals("test1"));
Assert.IsTrue(unindexedFieldValues[1].Equals("test2"));
// this test cannot work for documents retrieved from the index
// since unstored fields will obviously not be returned
if (!fromIndex)
{
Assert.IsTrue(unstoredFieldValues[0].Equals("test1"));
Assert.IsTrue(unstoredFieldValues[1].Equals("test2"));
}
}
/// <summary>
/// Gets an item from mthe document
/// </summary>
/// <param name="document">The lucene document to use</param>
/// <returns></returns>
protected virtual Item GetItem(Document document)
{
string itemShortId = document.GetValues(SitecoreFields.Id).FirstOrDefault();
if (String.IsNullOrEmpty(itemShortId))
{
return null;
}
ID itemId = new ID(itemShortId);
string language = document.GetValues(SitecoreFields.Language).FirstOrDefault();
if (String.IsNullOrEmpty(language))
{
throw new Exception("The language could not be retrieved from the lucene return");
}
Language itemLanguage = Language.Parse(language);
Item item = DatabaseHelper.GetItem(itemId, itemLanguage);
return item.Versions.Count > 0 ? item : null;
}
private static List <string> GetStringList(this LuceneDocument doc, string name)
{
return(doc.GetValues(name)?.FirstOrDefault()?.Split(' ').ToList());
}
private string GetSecondarySortString(Document document)
{
return String.Format("{0}_{1}", document.GetValues(BBCFields.SecondarySort).FirstOrDefault(), document.GetValues(BBCFields.Sortable).FirstOrDefault());
}
private Offer MapOffer(Document doc)
{
int attractivenes;
var strAttractivenes = doc.Get("Attractivenes");
int.TryParse(strAttractivenes, out attractivenes);
return new Offer
{
Id = doc.Get("Id"),
Title = doc.Get("Title"),
Url = doc.Get("Url"),
Price = doc.Get("Price"),
Description = TextHelper.CleanText(doc.Get("Description")),
Pictures = doc.GetValues("Pictures").ToList(),
Date = DateTools.StringToDate(doc.Get("Date")),
Teaser = doc.Get("Teaser")=="1",
PrivateOffer = doc.Get("PrivateOffer") == "1",
Attractivenes = attractivenes,
HaveSeen = doc.Get("HaveSeen") == "1",
Hide = doc.Get("Hide") == "1",
Notes = doc.Get("Notes")
};
}
protected SearchResult CreateSearchResult(Document doc, float score)
{
string id = doc.Get("id");
if (string.IsNullOrEmpty(id))
{
id = doc.Get(LuceneIndexer.IndexNodeIdFieldName);
}
var sr = new SearchResult()
{
Id = int.Parse(id),
Score = score
};
//we can use lucene to find out the fields which have been stored for this particular document
//I'm not sure if it'll return fields that have null values though
var fields = doc.GetFields();
//ignore our internal fields though
foreach (var field in fields.Cast<Field>())
{
var fieldName = field.Name();
var values = doc.GetValues(fieldName);
if (values.Length > 1)
{
sr.MultiValueFields[fieldName] = values.ToList();
//ensure the first value is added to the normal fields
sr.Fields[fieldName] = values[0];
}
else if (values.Length > 0)
{
sr.Fields[fieldName] = values[0];
}
}
return sr;
}
