public SampleModel(Document document)
{
Id = new Guid(document.GetField("Id").StringValue);
Description = document.GetField("Description").StringValue;
CreatedDate = DateTools.StringToDate(document.GetField("CreatedDate").StringValue);
ViewCount = Convert.ToInt32(document.GetField("ViewCount").StringValue);
}
internal static FieldType GetFieldType(string field, Lucene.Net.Documents.Document indexDocument)
{
return(new FieldType
{
IsArray = indexDocument.GetField(field + LuceneDocumentConverterBase.IsArrayFieldSuffix) != null,
IsJson = indexDocument.GetField(field + LuceneDocumentConverterBase.ConvertToJsonSuffix) != null
});
}
private static FieldType GetFieldType(IFieldable field, Lucene.Net.Documents.Document indexDocument)
{
return(new FieldType
{
IsArray = indexDocument.GetField(field.Name + LuceneDocumentConverterBase.IsArrayFieldSuffix) != null,
IsJson = indexDocument.GetField(field.Name + LuceneDocumentConverterBase.ConvertToJsonSuffix) != null,
});
}
public override object[] GetParametersForConstructor(string name, ProgramElementType programElementType, string fullFilePath, int definitionLineNumber, int definitionColumnNumber, string snippet, Document document)
{
string namespaceName = document.GetField(SandoField.Namespace.ToString()).StringValue();
AccessLevel accessLevel = (AccessLevel)Enum.Parse(typeof(AccessLevel), document.GetField(SandoField.AccessLevel.ToString()).StringValue(), true);
string body = "not stored in index";//document.GetField(SandoField.Body.ToString()).StringValue();
if(name == String.Empty) name = ProgramElement.UndefinedName;
return new object[] { name, definitionLineNumber, definitionColumnNumber, fullFilePath, snippet, accessLevel, namespaceName, body };
}
public override object[] GetParametersForConstructor(string name, ProgramElementType programElementType, string fullFilePath, int definitionLineNumber, int definitionColumnNumber, string snippet, Document document)
{
AccessLevel accessLevel = (AccessLevel)Enum.Parse(typeof(AccessLevel), document.GetField(SandoField.AccessLevel.ToString()).StringValue(), true);
string arguments = document.GetField(SandoField.Arguments.ToString()).StringValue();
string returnType = document.GetField(SandoField.ReturnType.ToString()).StringValue();
bool isConstructor = bool.Parse(document.GetField(SandoField.IsConstructor.ToString()).StringValue());
return new object[] { name, definitionLineNumber, definitionColumnNumber, returnType, accessLevel, arguments, fullFilePath, snippet, isConstructor };
}
public override object[] GetParametersForConstructor(string name, ProgramElementType programElementType, string fullFilePath, int definitionLineNumber, int definitionColumnNumber, string snippet, Document document)
{
string namespaceName = document.GetField(SandoField.Namespace.ToString()).StringValue();
AccessLevel accessLevel = (AccessLevel)Enum.Parse(typeof(AccessLevel), document.GetField(SandoField.AccessLevel.ToString()).StringValue(), true);
string extendedClasses = document.GetField(SandoField.ExtendedClasses.ToString()).StringValue();
string implementedInterfaces = document.GetField(SandoField.ImplementedInterfaces.ToString()).StringValue();
string modifiers = document.GetField(SandoField.Modifiers.ToString()).StringValue();
return new object[] { name, definitionLineNumber, definitionColumnNumber, fullFilePath, snippet, accessLevel, namespaceName, extendedClasses, implementedInterfaces, modifiers, "" };
}
public override object[] GetParametersForConstructor(string name, ProgramElementType programElementType, string fullFilePath, int definitionLineNumber, int definitionColumnNumber, string snippet, Document document)
{
AccessLevel accessLevel = (AccessLevel)Enum.Parse(typeof(AccessLevel), document.GetField(SandoField.AccessLevel.ToString()).StringValue(), true);
string propertyType = document.GetField(SandoField.DataType.ToString()).StringValue();
string body = "not stored in index";//document.GetField(SandoField.Body.ToString()).StringValue();
Guid classId = new Guid(document.GetField(SandoField.ClassId.ToString()).StringValue());
string className = document.GetField(SandoField.ClassName.ToString()).StringValue();
string modifiers = document.GetField(SandoField.Modifiers.ToString()).StringValue();
return new object[] { name, definitionLineNumber, definitionColumnNumber, fullFilePath, snippet, accessLevel, propertyType, body, classId, className, modifiers };
}
/// <summary>
/// Directory Članci
/// </summary>
/// <param name="postVrstaId"></param>
/// <returns></returns>
public static Directory GetDirectoryClanci()
{
using (TriglavBL temp = new TriglavBL())
{
Directory directoryClanci = FSDirectory.Open(new DirectoryInfo(AppDomain.CurrentDomain.BaseDirectory +"Clanci"));
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_30);
var writer = new IndexWriter(directoryClanci, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
List<Post> sviPostovi = temp.getAllClanci();
foreach (var post in sviPostovi)
{
//Add & boost
var clanak = new Document();
clanak.Add(new Field("id", Convert.ToString(post.id), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO));
clanak.Add(new Field("PostVrsta", Convert.ToString(post.PostVrsta), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("Naslov", post.Naslov, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
clanak.GetField("Naslov").Boost = (2.0F);
clanak.Add(new Field("Sadrzaj", post.Sadrzaj, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
clanak.GetField("Sadrzaj").Boost = (1.2F);
if (post.Sazetak != null)
{
clanak.Add(new Field("Sazetak", post.Sazetak, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
clanak.GetField("Sazetak").Boost = (1.5F);
}
clanak.Add(new Field("Tagovi", post.Tagovi, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
clanak.GetField("Tagovi").Boost = (2.5F);
clanak.Add(new Field("VlasnikId", Convert.ToString(post.VlasnikID), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("VlasnikNadimak", Convert.ToString(post.VlasnikNadimak), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
clanak.Add(new Field("DatumKreiranja", Convert.ToString(post.DatumKreiranja), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("DatumZadnjeIzmjene", Convert.ToString(post.DatumZadnjeIzmjene), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("DatumZadnjeAktivnosti", Convert.ToString(post.DatumZadnjeAktivnosti), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("PrihvaceniOdgovori", Convert.ToString(post.PrihvaceniOdgovori), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("BrojOdgovora", Convert.ToString(post.BrojOdgovora), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("BrojKomentara", Convert.ToString(post.BrojKomentara), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("BrojOmiljenih", Convert.ToString(post.BrojOmiljenih), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("BrojPregleda", Convert.ToString(post.BrojPregleda), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("BrojPoena", Convert.ToString(post.BrojPoena), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("Likes", Convert.ToString(post.Likes), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("Unlikes", Convert.ToString(post.Unlikes), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("PromijenioID", Convert.ToString(post.PromijenioID), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("RoditeljskiPostID", Convert.ToString(post.RoditeljskiPostID), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("BrojRangiranja", Convert.ToString(post.BrojRangiranja), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("PrihvacenaIzmjena", Convert.ToString(post.PrihvacenaIzmjena), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
clanak.Add(new Field("Broj_Razgovora", Convert.ToString(post.Broj_Razgovora), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
writer.AddDocument(clanak);
}
writer.Optimize();
writer.Dispose();
return directoryClanci;
}
}
internal LuceneEntry(Document myLuceneDocument, float? myScore = null, int? myDocNum = null)
{
this.Id = myLuceneDocument.GetField(LuceneIndex.FieldNames[LuceneIndex.Fields.ID]).StringValue();
this.IndexId = myLuceneDocument.GetField(LuceneIndex.FieldNames[LuceneIndex.Fields.INDEX_ID]).StringValue();
this.VertexId = Convert.ToInt64(myLuceneDocument.GetField(LuceneIndex.FieldNames[LuceneIndex.Fields.VERTEX_ID]).StringValue());
this.Text = myLuceneDocument.GetField(LuceneIndex.FieldNames[LuceneIndex.Fields.TEXT]).StringValue();
this.Score = myScore;
this.DocNum = myDocNum;
var propertyIdField = myLuceneDocument.GetField(LuceneIndex.FieldNames[LuceneIndex.Fields.PROPERTY_ID]);
if (propertyIdField != null)
{
this.PropertyId = Convert.ToInt64(propertyIdField.StringValue());
}
}
public Object Get(String name, Document document)
{
Field field = document.GetField(name);
if (field == null)
return null;
return stringBridge.StringToObject(field.StringValue());
}
// Activity 7
/// <summary>
/// Outputs results to the screen
/// </summary>
/// <param name="results">Search results</param>
public void DisplayResults(TopDocs results)
{
int rank = 0;
foreach (ScoreDoc scoreDoc in results.ScoreDocs)
{
rank++;
// retrieve the document from the 'ScoreDoc' object
Lucene.Net.Documents.Document doc = searcher.Doc(scoreDoc.Doc);
if (doc.GetField("passage_text") != null)
{
string Result_passage_text = doc.Get("passage_text").ToString();
string Result_url = doc.Get("url").ToString();
Console.WriteLine("\n\nRank: " + rank + "\nPassage text: " + Result_passage_text);
Console.WriteLine("url: " + Result_url);
}
else
{
string Result_query = doc.Get("query").ToString();
string Result_answers = doc.Get("answers").ToString();
Console.WriteLine("\n\nRank: " + rank + "\nQuery: " + Result_query);
Console.WriteLine("answers: " + Result_answers);
}
//Console.WriteLine("Rank " + rank + " score " + scoreDoc.Score + " text " + myFieldValue);
}
}
public override void OnIndexEntryCreated(string entryKey, Document document)
{
var resultDocId = document.GetField(setupDoc.DocumentKey);
if (resultDocId == null)
{
log.Warn("Could not find document id property '{0}' in '{1}' for index '{2}'", setupDoc.DocumentKey, entryKey, index);
return;
}
var documentId = resultDocId.StringValue;
itemsToRemove.Remove(documentId);
var resultDoc = database.Get(documentId, null);
if (resultDoc == null)
{
log.Warn("Could not find a document with the id '{0}' for index '{1}'", documentId, index);
return;
}
var entityName = resultDoc.Metadata.Value<string>(Constants.RavenEntityName);
if(entityName != null && viewGenerator.ForEntityNames.Contains(entityName))
{
log.Warn(
"Rejected update for a potentially recursive update on document '{0}' because the index '{1}' includes documents with entity name of '{2}'",
documentId, index, entityName);
return;
}
if(viewGenerator.ForEntityNames.Count == 0)
{
log.Warn(
"Rejected update for a potentially recursive update on document '{0}' because the index '{1}' includes all documents",
documentId, index);
return;
}
var changesMade = false;
foreach (var mapping in setupDoc.FieldNameMappings)
{
var field =
document.GetFieldable(mapping.Key + "_Range") ??
document.GetFieldable(mapping.Key);
if (field == null)
continue;
var numericField = field as NumericField;
if (numericField != null)
{
resultDoc.DataAsJson[mapping.Value] = new RavenJValue(numericField.NumericValue);
}
else
{
resultDoc.DataAsJson[mapping.Value] = field.StringValue;
}
changesMade = true;
}
if (changesMade)
database.Put(documentId, resultDoc.Etag, resultDoc.DataAsJson, resultDoc.Metadata, null);
}
public static IndexDocumentData FromDocument(Document doc)
{
return new IndexDocumentData()
{
Package = PackageJson.FromJson(JObject.Parse(doc.GetField("Data").StringValue)),
Checksum = Int32.Parse(doc.GetFieldable("Checksum").StringValue),
Feeds = doc.GetFields("CuratedFeed").Select(f => f.StringValue).ToList()
};
}
//--- Constructors ---
public LuceneResult(Document document, float score) {
Document = document;
Score = score;
var field = document.GetField("id.page");
if(field == null) {
return;
}
ulong pageId;
PageId = ulong.TryParse(field.StringValue(), out pageId) ? pageId : (ulong?)null;
}
internal static Type GetMyType(Document luceneDocument)
{
try
{
string typeId = luceneDocument.GetField(ProgramElement.CustomTypeTag).StringValue();
return Type.GetType(typeId);
}
catch
{
return typeof (ProgramElement);
}
}
internal static void SetCustomFields(object myElement, Lucene.Net.Documents.Document luceneDocument)
{
foreach (var property in (myElement as ProgramElement).GetCustomProperties())
{
if (!property.Name.Equals(ProgramElement.CustomTypeTag))
{
Field field = luceneDocument.GetField(property.Name);
Contract.Assert(field != null, "Field " + property.Name + " was not populated");
property.SetValue(myElement, field.StringValue(), null);
}
}
}
public override void OnIndexEntryCreated(string entryKey, Document document)
{
var conflitsIndex = entryKey.IndexOf("/conflicts/", StringComparison.Ordinal);
if (conflitsIndex <= 0)
return;
var documentIdField = document.GetField(Constants.DocumentIdFieldName);
if(documentIdField == null)
return;
documentIdField.SetValue(entryKey.Substring(0, conflitsIndex));
}
public void ThenIWantAnAnalyzedNormsFieldAdded()
{
// Arrange
var document = new Document();
// Act
document.Add("Bar").Indexed().Analyzed().As("Foo");
// Assert
var field = document.GetField("Foo");
Assert.True(field.IsIndexed);
Assert.True(field.IsTokenized);
Assert.False(field.OmitNorms);
}
public ProgramElement Convert()
{
Contract.Ensures(Contract.Result <ProgramElement>() != null, "ConverterFromHitToProgramElement:ReadProgramElementFromDocument - an object must be returned from this method!");
//Get standard field values
string name = luceneDocument.GetField(SandoField.Name.ToString()).StringValue().ToSandoDisplayable();
ProgramElementType type = (ProgramElementType)Enum.Parse(typeof(ProgramElementType), luceneDocument.GetField(SandoField.ProgramElementType.ToString()).StringValue(), true);
string fullFilePath = luceneDocument.GetField(SandoField.FullFilePath.ToString()).StringValue();
int definitionLineNumber = int.Parse(luceneDocument.GetField(SandoField.DefinitionLineNumber.ToString()).StringValue());
int definitionColumnNumber = int.Parse(luceneDocument.GetField(SandoField.DefinitionColumnNumber.ToString()).StringValue());
string snippet = luceneDocument.GetField(SandoField.Source.ToString()).StringValue();
//Add values that vary according to element type (e.g., a textline doesn't have a parent class whereas a method often does)
//Note: Parameters must match the parameter list for corresponding program element. See MethodDocument.GetParametersForConstructor and the MethodElement constructor
var parameters = sandoDocument.GetParametersForConstructor(name, type, fullFilePath, definitionLineNumber, definitionColumnNumber, snippet, luceneDocument);
//Create type from the collected values
var myClassType = GetMyType(luceneDocument);
var myElement = Activator.CreateInstance(myClassType, parameters);
//Populate any custom fields from user-defined types, only happens when a third party extends Sando
SetCustomFields(myElement, luceneDocument);
return(myElement as ProgramElement);
}
public void ThenIWantItToBeIndexedAndBoosted()
{
// Arrange
var document = new Document();
var boost = 2.0f;
// Act
document.Add("Bar").Indexed().BoostBy(boost).As("Foo");
// Assert
var field = document.GetField("Foo");
Assert.True(field.IsIndexed);
Assert.False(field.IsTokenized);
Assert.False(field.OmitNorms);
Assert.Equal(boost, field.Boost);
}
public SearchResultViewModel(Document document, ScoreDoc scoreDoc)
{
if (document == null)
throw new ArgumentNullException("document");
if (scoreDoc == null)
throw new ArgumentNullException("scoreDoc");
EnsureFieldsExist(document);
Id = int.Parse(document.GetField("id").StringValue);
Title = document.GetField("title").StringValue;
ContentSummary = document.GetField("contentsummary").StringValue;
Tags = document.GetField("tags").StringValue;
CreatedBy = document.GetField("createdby").StringValue;
ContentLength = int.Parse(document.GetField("contentlength").StringValue);
Score = scoreDoc.Score;
DateTime createdOn = DateTime.UtcNow;
if (!DateTime.TryParse(document.GetField("createdon").StringValue, out createdOn))
createdOn = DateTime.UtcNow;
CreatedOn = createdOn;
}
private static KeyValuePair<string, RavenJToken> CreateProperty(Field fld, Document document)
{
var stringValue = fld.StringValue();
if (document.GetField(fld.Name() + "_ConvertToJson") != null)
{
var val = RavenJToken.Parse(fld.StringValue()) as RavenJObject;
return new KeyValuePair<string, RavenJToken>(fld.Name(), val);
}
if (stringValue == Constants.NullValue)
stringValue = null;
return new KeyValuePair<string, RavenJToken>(fld.Name(), stringValue);
}
public virtual void searchIndex(System.String dirName, System.String oldName)
{
//QueryParser parser = new QueryParser("contents", new WhitespaceAnalyzer());
//Query query = parser.parse("handle:1");
dirName = FullDir(dirName);
Directory dir = FSDirectory.Open(new System.IO.DirectoryInfo(dirName));
IndexSearcher searcher = new IndexSearcher(dir, true);
IndexReader reader = searcher.IndexReader;
_TestUtil.CheckIndex(dir);
for (int i = 0; i < 35; i++)
{
if (!reader.IsDeleted(i))
{
Document d = reader.Document(i);
var fields = d.GetFields();
if (!oldName.StartsWith("19.") && !oldName.StartsWith("20.") && !oldName.StartsWith("21.") && !oldName.StartsWith("22."))
{
if (d.GetField("content3") == null)
{
int numFields = oldName.StartsWith("29.") ? 7 : 5;
Assert.AreEqual(numFields, fields.Count);
Field f = d.GetField("id");
Assert.AreEqual("" + i, f.StringValue);
f = (Field)d.GetField("utf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.StringValue);
f = (Field)d.GetField("autf8");
Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.StringValue);
f = (Field)d.GetField("content2");
Assert.AreEqual("here is more content with aaa aaa aaa", f.StringValue);
f = (Field)d.GetField("fie\u2C77ld");
Assert.AreEqual("field with non-ascii name", f.StringValue);
}
}
}
// Only ID 7 is deleted
else
{
Assert.AreEqual(7, i);
}
}
ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs;
// First document should be #21 since it's norm was
// increased:
Document d2 = searcher.Doc(hits[0].Doc);
Assert.AreEqual("21", d2.Get("id"), "didn't get the right document first");
TestHits(hits, 34, searcher.IndexReader);
if (!oldName.StartsWith("19.") && !oldName.StartsWith("20.") && !oldName.StartsWith("21.") && !oldName.StartsWith("22."))
{
// Test on indices >= 2.3
hits = searcher.Search(new TermQuery(new Term("utf8", "\u0000")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
hits = searcher.Search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), null, 1000).ScoreDocs;
Assert.AreEqual(34, hits.Length);
}
searcher.Close();
dir.Close();
}
private static int GetInt(this LuceneDocument doc, string name)
{
var value = doc.GetField(name)?.GetInt32ValueOrDefault();
return(value ?? 0);
}
private static KeyValuePair<string, RavenJToken> CreateProperty(Field fld, Document document)
{
if (fld.IsBinary)
return new KeyValuePair<string, RavenJToken>(fld.Name, fld.GetBinaryValue());
var stringValue = fld.StringValue;
if (document.GetField(fld.Name + "_ConvertToJson") != null)
{
var val = RavenJToken.Parse(fld.StringValue) as RavenJObject;
return new KeyValuePair<string, RavenJToken>(fld.Name, val);
}
if (stringValue == Constants.NullValue)
stringValue = null;
if (stringValue == Constants.EmptyString)
stringValue = string.Empty;
return new KeyValuePair<string, RavenJToken>(fld.Name, stringValue);
}
protected bool TryGetValue(FieldsToFetch.FieldToFetch fieldToFetch, Document document, Lucene.Net.Documents.Document luceneDoc, IState state, Dictionary <string, IndexField> indexFields, bool?anyDynamicIndexFields, out string key, out object value)
{
key = fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value;
if (fieldToFetch.QueryField == null)
{
return(TryGetFieldValueFromDocument(document, fieldToFetch, out value));
}
if (fieldToFetch.QueryField.Function != null)
{
var args = new object[fieldToFetch.QueryField.FunctionArgs.Length + 1];
for (int i = 0; i < fieldToFetch.FunctionArgs.Length; i++)
{
TryGetValue(fieldToFetch.FunctionArgs[i], document, luceneDoc, state, indexFields, anyDynamicIndexFields, out _, out args[i]);
if (ReferenceEquals(args[i], document))
{
args[i] = Tuple.Create(document, luceneDoc, state, indexFields, anyDynamicIndexFields);
}
}
value = GetFunctionValue(fieldToFetch, document.Id, args);
return(true);
}
if (fieldToFetch.QueryField.IsCounter)
{
string name;
string id = document.Id;
if (fieldToFetch.QueryField.IsParameter)
{
if (_query.QueryParameters == null)
{
throw new InvalidQueryException("The query is parametrized but the actual values of parameters were not provided", _query.Query, null);
}
if (_query.QueryParameters.TryGetMember(fieldToFetch.QueryField.Name, out var nameObject) == false)
{
throw new InvalidQueryException($"Value of parameter '{fieldToFetch.QueryField.Name}' was not provided", _query.Query, _query.QueryParameters);
}
name = nameObject.ToString();
key = fieldToFetch.QueryField.Alias ?? name;
}
else
{
name = fieldToFetch.Name.Value;
}
if (fieldToFetch.QueryField.SourceAlias != null &&
BlittableJsonTraverser.Default.TryRead(document.Data, fieldToFetch.QueryField.SourceAlias, out var sourceId, out _))
{
id = sourceId.ToString();
}
if (fieldToFetch.QueryField.FunctionArgs != null)
{
value = GetCounterRaw(id, name);
}
else
{
value = GetCounter(id, name);
}
return(true);
}
if (fieldToFetch.QueryField.ValueTokenType != null)
{
var val = fieldToFetch.QueryField.Value;
if (fieldToFetch.QueryField.ValueTokenType.Value == ValueTokenType.Parameter)
{
if (_query == null)
{
value = null;
return(false); // only happens for debug endpoints and more like this
}
_query.QueryParameters.TryGet((string)val, out val);
}
value = val;
return(true);
}
if (fieldToFetch.QueryField.HasSourceAlias == false)
{
return(TryGetFieldValueFromDocument(document, fieldToFetch, out value));
}
if (_loadedDocumentIds == null)
{
_loadedDocumentIds = new HashSet <string>();
_loadedDocuments = new Dictionary <string, Document>();
_loadedDocumentsByAliasName = new Dictionary <string, Document>();
}
_loadedDocumentIds.Clear();
//_loadedDocuments.Clear(); - explicitly not clearing this, we want to cache this for the duration of the query
_loadedDocuments[document.Id ?? string.Empty] = document;
if (fieldToFetch.QueryField.SourceAlias != null)
{
if (fieldToFetch.QueryField.IsQuoted)
{
_loadedDocumentIds.Add(fieldToFetch.QueryField.SourceAlias);
}
else if (fieldToFetch.QueryField.IsParameter)
{
if (_query.QueryParameters == null)
{
throw new InvalidQueryException("The query is parametrized but the actual values of parameters were not provided", _query.Query, (BlittableJsonReaderObject)null);
}
if (_query.QueryParameters.TryGetMember(fieldToFetch.QueryField.SourceAlias, out var id) == false)
{
throw new InvalidQueryException($"Value of parameter '{fieldToFetch.QueryField.SourceAlias}' was not provided", _query.Query, _query.QueryParameters);
}
_loadedDocumentIds.Add(id.ToString());
}
else if (fieldToFetch.QueryField.LoadFromAlias != null)
{
if (_loadedDocumentsByAliasName.TryGetValue(fieldToFetch.QueryField.LoadFromAlias, out var loadedDoc))
{
IncludeUtil.GetDocIdFromInclude(loadedDoc.Data, fieldToFetch.QueryField.SourceAlias, _loadedDocumentIds, _database.IdentityPartsSeparator);
}
}
else if (fieldToFetch.CanExtractFromIndex)
{
if (luceneDoc != null)
{
var field = luceneDoc.GetField(fieldToFetch.QueryField.SourceAlias);
if (field != null)
{
var fieldValue = ConvertType(_context, field, GetFieldType(field.Name, luceneDoc), state);
_loadedDocumentIds.Add(fieldValue.ToString());
}
}
}
else
{
IncludeUtil.GetDocIdFromInclude(document.Data, fieldToFetch.QueryField.SourceAlias, _loadedDocumentIds, _database.IdentityPartsSeparator);
}
}
else
{
_loadedDocumentIds.Add(document.Id ?? string.Empty); // null source alias is the root doc
_loadedDocumentsByAliasName.Clear();
}
if (_loadedDocumentIds.Count == 0)
{
if (fieldToFetch.QueryField.SourceIsArray)
{
value = new List <object>();
return(true);
}
value = null;
return(false);
}
var buffer = new List <object>();
foreach (var docId in _loadedDocumentIds)
{
if (docId == null)
{
continue;
}
if (_loadedDocuments.TryGetValue(docId, out var doc) == false)
{
using (_loadScope = _loadScope?.Start() ?? _projectionScope?.For(nameof(QueryTimingsScope.Names.Load)))
_loadedDocuments[docId] = doc = LoadDocument(docId);
}
if (doc == null)
{
continue;
}
if (fieldToFetch.QueryField.Alias != null)
{
_loadedDocumentsByAliasName[fieldToFetch.QueryField.Alias] = doc;
}
if (string.IsNullOrEmpty(fieldToFetch.Name.Value)) // we need the whole document here
{
buffer.Add(doc);
continue;
}
if (TryGetFieldValueFromDocument(doc, fieldToFetch, out var val))
{
if (val is string == false && val is LazyStringValue == false && val is System.Collections.IEnumerable items)
{
// we flatten arrays in projections
foreach (var item in items)
{
buffer.Add(item);
}
fieldToFetch.QueryField.SourceIsArray = true;
}
else
{
buffer.Add(val);
}
}
}
if (fieldToFetch.QueryField.SourceIsArray)
{
value = buffer;
return(true);
}
if (buffer.Count > 0)
{
if (buffer.Count > 1)
{
ThrowOnlyArrayFieldCanHaveMultipleValues(fieldToFetch);
}
value = buffer[0];
return(true);
}
value = null;
return(false);
}
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score)
{
fieldsToFetch.EnsureHasField(Constants.ReduceKeyFieldName);
if (fieldsToFetch.HasExplicitFieldsToFetch)
{
return base.RetrieveDocument(document, fieldsToFetch, score);
}
var field = document.GetField(Constants.ReduceValueFieldName);
if (field == null)
{
fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().Select(x => x.Name).ToArray());
return base.RetrieveDocument(document, fieldsToFetch, score);
}
var projection = RavenJObject.Parse(field.StringValue);
if (fieldsToFetch.FetchAllStoredFields)
{
var fields = new HashSet<string>(document.GetFields().Select(x => x.Name));
fields.Remove(Constants.ReduceKeyFieldName);
var documentFromFields = new RavenJObject();
AddFieldsToDocument(document, fields, documentFromFields);
foreach (var kvp in projection)
{
documentFromFields[kvp.Key] = kvp.Value;
}
projection = documentFromFields;
}
return new IndexQueryResult
{
Projection = projection,
Score = score.Score,
ReduceVal = field.StringValue
};
}
private static DateTime GetDate(this LuceneDocument doc, string name)
{
var ticks = doc.GetField(name)?.GetInt64ValueOrDefault();
return(ticks.HasValue ? new DateTime(ticks.Value) : DateTime.MinValue);
}
private static bool GetBool(this LuceneDocument doc, string name)
{
var value = doc.GetField(name)?.GetInt32ValueOrDefault();
return(value == 1);
}
private string GetStringFromField(Document doc, SortField sortField)
{
var field = doc.GetField(sortField.Field);
return field == null ? "" : field.StringValue;
}
/// <summary>
/// Returns a field with the given name if any exist in this document cast to type <typeparam name="T"/>, or
/// <c>null</c>. If multiple fields exists with this name, this method returns the
/// first value added.
/// <para/>
/// LUCENENET specific
/// </summary>
/// <exception cref="InvalidCastException">If the field type cannot be cast to <typeparam name="T"/>.</exception>
public static T GetField <T>(this Document document, string name) where T : IIndexableField
{
return((T)document.GetField(name));
}
public override void OnIndexEntryCreated(string entryKey, Document document)
{
var resultDocId = document.GetField(setupDoc.DocumentKey);
if (resultDocId == null)
{
log.Warn("Could not find document id property '{0}' in '{1}' for index '{2}'", setupDoc.DocumentKey, entryKey, index);
return;
}
var documentId = resultDocId.StringValue;
itemsToRemove.TryRemove(documentId);
var resultDoc = database.Get(documentId, null);
if (resultDoc == null)
{
log.Warn("Could not find a document with the id '{0}' for index '{1}'", documentId, index);
return;
}
var entityName = resultDoc.Metadata.Value <string>(Constants.RavenEntityName);
if (entityName != null && viewGenerator.ForEntityNames.Contains(entityName))
{
log.Warn(
"Rejected update for a potentially recursive update on document '{0}' because the index '{1}' includes documents with entity name of '{2}'",
documentId, index, entityName);
return;
}
if (viewGenerator.ForEntityNames.Count == 0)
{
log.Warn(
"Rejected update for a potentially recursive update on document '{0}' because the index '{1}' includes all documents",
documentId, index);
return;
}
var changesMade = false;
foreach (var mapping in setupDoc.FieldNameMappings)
{
var field =
document.GetFieldable(mapping.Key + "_Range") ??
document.GetFieldable(mapping.Key);
if (field == null)
{
continue;
}
var numericField = field as NumericField;
if (numericField != null)
{
resultDoc.DataAsJson[mapping.Value] = new RavenJValue(numericField.NumericValue);
}
else
{
resultDoc.DataAsJson[mapping.Value] = field.StringValue;
}
changesMade = true;
}
if (changesMade)
{
database.Put(documentId, resultDoc.Etag, resultDoc.DataAsJson, resultDoc.Metadata, null);
}
}
private static SearchResult GetPartialSearchResult(Document doc)
{
var result = new SearchResult();
var localeField = doc.GetField(Constants.LocaleTag);
if (localeField != null)
{
int id;
result.CultureCode = int.TryParse(localeField.StringValue, out id) && id >= 0
? LocaleController.Instance.GetLocale(id).Code : Null.NullString;
}
FillTagsValues(doc, result);
return result;
}
private static RavenJObject CreateDocumentFromFields(Document document, IEnumerable<string> fieldsToFetch)
{
var documentFromFields = new RavenJObject();
var q = fieldsToFetch
.SelectMany(name => document.GetFields(name) ?? new Field[0])
.Where(x => x != null)
.Where(
x =>
x.Name().EndsWith("_IsArray") == false &&
x.Name().EndsWith("_Range") == false &&
x.Name().EndsWith("_ConvertToJson") == false)
.Select(fld => CreateProperty(fld, document))
.GroupBy(x => x.Key)
.Select(g =>
{
if (g.Count() == 1 && document.GetField(g.Key + "_IsArray") == null)
{
return g.First();
}
return new KeyValuePair<string, RavenJToken>(g.Key, new RavenJArray(g.Select(x => x.Value)));
});
foreach (var keyValuePair in q)
{
documentFromFields.Add(keyValuePair.Key, keyValuePair.Value);
}
return documentFromFields;
}
private static RavenJObject CreateJsonDocumentFromLuceneDocument(Document document)
{
var field = document.GetField(Constants.ReduceValueFieldName);
if (field != null)
return RavenJObject.Parse(field.StringValue);
var ravenJObject = new RavenJObject();
var fields = document.GetFields();
var arrayMarkers = fields
.Where(x => x.Name.EndsWith("_IsArray"))
.Select(x => x.Name)
.ToList();
foreach (var fieldable in fields)
{
var stringValue = GetStringValue(fieldable);
var isArrayMarker = fieldable.Name.EndsWith("_IsArray");
var isArray = !isArrayMarker && arrayMarkers.Contains(fieldable.Name + "_IsArray");
RavenJToken token;
var isJson = RavenJToken.TryParse(stringValue, out token);
RavenJToken value;
if (ravenJObject.TryGetValue(fieldable.Name, out value) == false)
{
if (isArray)
ravenJObject[fieldable.Name] = new RavenJArray { isJson ? token : stringValue };
else if (isArrayMarker)
{
var fieldName = fieldable.Name.Substring(0, fieldable.Name.Length - 8);
ravenJObject[fieldable.Name] = isJson ? token : stringValue;
ravenJObject[fieldName] = new RavenJArray();
}
else
ravenJObject[fieldable.Name] = isJson ? token : stringValue;
}
else
{
var ravenJArray = value as RavenJArray;
if (ravenJArray != null)
ravenJArray.Add(isJson ? token : stringValue);
else
{
ravenJArray = new RavenJArray { value, isJson ? token : stringValue };
ravenJObject[fieldable.Name] = ravenJArray;
}
}
}
return ravenJObject;
}
//TODO: The last param, Cache isn't used. Remove it and update dependant projects. (Version 1.3)
public static SearchResults <Document> GetDocuments(QueryParser defaultQueryParser, QueryParser customQueryParser, IndexSearcher indexSearcher, string query, int pageNumber, int pageSize, bool shouldDocumentsBeClustered, string sort, int maximumNumberOfDocumentsToScore, Cache cache)
{
Query query2 = customQueryParser.Parse(query);
Hits hits = null;
if (!string.IsNullOrEmpty(sort))
{
string[] sorts = sort.ToLower().Split(",".ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
List <SortField> sortFields = new List <SortField>(sorts.Length / 2);
for (int i = 0; i < sorts.Length; i++)
{
if (sorts[i].Split(' ')[1] == "asc")
{
sortFields.Add(new SortField(sorts[i].Split(' ')[0], false));
}
else
{
sortFields.Add(new SortField(sorts[i].Split(' ')[0], true));
}
}
hits = indexSearcher.Search(query2, new Sort(sortFields.ToArray()));
}
else
{
hits = indexSearcher.Search(query2);
}
SearchResults <Document> searchResults = new SearchResults <Document>();
searchResults.Documents = new List <Document>();
searchResults.Query = query2;
if (hits.Length() != 0)
{
Dictionary <string, string> domains = new Dictionary <string, string>();
PriorityQueue <Document> priorityQueue = new PriorityQueue <Document>();
//Get the Hits!!!
//TODO: Optimize this!!! (Version 1.3)
for (int j = 0; j < hits.Length() && searchResults.Documents.Count < maximumNumberOfDocumentsToScore && priorityQueue.Count < maximumNumberOfDocumentsToScore; j++)
{
Document document = hits.Doc(j);
float score = hits.Score(j);
document.Add(new Field("documentid", j.ToString(), Field.Store.YES, Field.Index.NO));
document.Add(new Field("relevancyscore", score.ToString(), Field.Store.YES, Field.Index.NO));
if (!string.IsNullOrEmpty(sort))
{
if (shouldDocumentsBeClustered)
{
if (document.GetField("domain") != null)
{
string domain = document.GetField("domain").StringValue();
if (!domains.ContainsKey(domain))
{
domains.Add(domain, null);
if (searchResults.Documents.Count < pageSize && j >= (pageNumber * pageSize) - pageSize)
{
searchResults.Documents.Add(document);
}
}
}
}
else
{
if (searchResults.Documents.Count < pageSize && j >= (pageNumber * pageSize) - pageSize)
{
searchResults.Documents.Add(document);
}
}
}
else
{
priorityQueue.Enqueue(document, score * double.Parse(document.GetField("strength").StringValue()));
}
}
if (string.IsNullOrEmpty(sort))
{
for (int i = 0; i < hits.Length() && priorityQueue.Count != 0; i++)
{
Document document = priorityQueue.Dequeue();
if (shouldDocumentsBeClustered)
{
if (document.GetField("domain") != null)
{
string domain = document.GetField("domain").StringValue();
if (!domains.ContainsKey(domain))
{
domains.Add(domain, null);
if (searchResults.Documents.Count < pageSize && i >= (pageNumber * pageSize) - pageSize)
{
searchResults.Documents.Add(document);
}
}
else
{
i--;
}
}
}
else
{
if (searchResults.Documents.Count < pageSize && i >= (pageNumber * pageSize) - pageSize)
{
searchResults.Documents.Add(document);
}
}
}
}
if (shouldDocumentsBeClustered)
{
searchResults.TotalNumberOfHits = domains.Count;
}
else
{
searchResults.TotalNumberOfHits = hits.Length();
}
}
return(searchResults);
}
private LogEntry DocumentToLogEntry(Document document)
{
return new LogEntry()
{
Id = Guid.Parse(document.GetField("id").StringValue),
ApplicationName = document.GetField("application").StringValue,
Server = document.GetField("server").StringValue,
Environment = document.GetField("environment").StringValue,
DateTime = DateTime.Parse(document.GetField("date").StringValue),
ExceptionMessage = document.GetField("exceptionMessage").StringValue,
ExceptionType = document.GetField("exceptionType").StringValue,
Message = document.GetField("truncatedMessage").StringValue,
};
}
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score)
{
fieldsToFetch.EnsureHasField(Constants.ReduceKeyFieldName);
if (fieldsToFetch.HasExplicitFieldsToFetch)
{
return base.RetrieveDocument(document, fieldsToFetch, score);
}
var field = document.GetField(Constants.ReduceValueFieldName);
if (field == null)
{
fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().Select(x => x.Name).ToArray());
return base.RetrieveDocument(document, fieldsToFetch, score);
}
return new IndexQueryResult
{
Projection = RavenJObject.Parse(field.StringValue),
Score = score.Score
};
}
public int GetInt(string name)
{
var field = _doc.GetField(name);
return(field == null ? 0 : Int32.Parse(field.StringValue));
}
public static RavenJObject CreateDocumentFromFields(Document document, FieldsToFetch fieldsToFetch)
{
var documentFromFields = new RavenJObject();
var fields = fieldsToFetch.Fields;
if (fieldsToFetch.FetchAllStoredFields)
fields = fields.Concat(document.GetFields().Select(x => x.Name));
var q = fields
.Distinct()
.SelectMany(name => document.GetFields(name) ?? new Field[0])
.Where(x => x != null)
.Where(
x =>
x.Name.EndsWith("_IsArray") == false &&
x.Name.EndsWith("_Range") == false &&
x.Name.EndsWith("_ConvertToJson") == false)
.Select(fld => CreateProperty(fld, document))
.GroupBy(x => x.Key)
.Select(g =>
{
if (g.Count() == 1 && document.GetField(g.Key + "_IsArray") == null)
{
return g.First();
}
var ravenJTokens = g.Select(x => x.Value).ToArray();
return new KeyValuePair<string, RavenJToken>(g.Key, new RavenJArray((IEnumerable)ravenJTokens));
});
foreach (var keyValuePair in q)
{
documentFromFields.Add(keyValuePair.Key, keyValuePair.Value);
}
return documentFromFields;
}
public static object ToModel(Document document, Type modelType)
{
var model = Activator.CreateInstance(modelType);
foreach (var prop in modelType.GetProperties(BindingFlags.Public | BindingFlags.Instance))
{
if (TypeHelper.IsSimpleType(prop.PropertyType))
{
var field = document.GetField(prop.Name);
if (field != null)
{
var propValue = LuceneUtility.FromFieldStringValue(field.StringValue, prop.PropertyType);
prop.SetValue(model, propValue, null);
}
}
else
{
var propTypeInfo = ModelTypeInfo.GetTypeInfo(prop.PropertyType);
if (propTypeInfo.IsCollection)
{
if (propTypeInfo.IsDictionary)
{
var propValue = prop.GetValue(model, null);
if (propValue == null)
{
propValue = Activator.CreateInstance(typeof(Dictionary<,>).MakeGenericType(propTypeInfo.DictionaryKeyType, propTypeInfo.DictionaryValueType));
prop.SetValue(model, propValue, null);
}
var dic = propValue as IDictionary;
var fields = document.GetFields().Where(f => f.Name.StartsWith(prop.Name));
// Property type is IDictionary<TKey, TValue>
if (TypeHelper.IsSimpleType(propTypeInfo.DictionaryValueType))
{
foreach (var field in fields)
{
string propName;
string dicKey;
if (TryParseDictionaryFieldName(field.Name, out propName, out dicKey))
{
var fieldValue = LuceneUtility.FromFieldStringValue(field.StringValue, propTypeInfo.DictionaryValueType);
dic.Add(dicKey, fieldValue);
}
}
}
else // Property type is IDictionary<TKey, IList<TValue>> or IDictionary<TKey, ISet<TValue>>
{
var dicValueTypeInfo = ModelTypeInfo.GetTypeInfo(propTypeInfo.DictionaryValueType);
if (dicValueTypeInfo.IsCollection && TypeHelper.IsSimpleType(dicValueTypeInfo.ElementType))
{
Type newDicValueType = null;
MethodInfo hashsetAddMethod = null;
if (dicValueTypeInfo.IsSet)
{
newDicValueType = typeof(HashSet<>).MakeGenericType(dicValueTypeInfo.ElementType);
hashsetAddMethod = GetAddMethod(newDicValueType, dicValueTypeInfo.ElementType);
}
else
{
newDicValueType = typeof(List<>).MakeGenericType(dicValueTypeInfo.ElementType);
}
foreach (var field in fields)
{
string propName;
string dicKey;
if (TryParseDictionaryFieldName(field.Name, out propName, out dicKey))
{
var fieldValue = LuceneUtility.FromFieldStringValue(field.StringValue, dicValueTypeInfo.ElementType);
if (!dic.Contains(dicKey))
{
dic.Add(dicKey, Activator.CreateInstance(newDicValueType));
}
var list = dic[dicKey];
if (dicValueTypeInfo.IsSet) // is HashSet<>
{
hashsetAddMethod.Invoke(list, new[] { fieldValue });
}
else // is IList<>
{
(list as IList).Add(fieldValue);
}
}
}
}
}
}
else // Property is collection but not dictionary
{
var fields = document.GetFields(prop.Name);
if (fields.Length == 0)
{
continue;
}
var list = Activator.CreateInstance(typeof(List<>).MakeGenericType(propTypeInfo.ElementType)) as IList;
foreach (var field in fields)
{
var fieldValue = LuceneUtility.FromFieldStringValue(field.StringValue, propTypeInfo.ElementType);
list.Add(fieldValue);
}
if (prop.PropertyType.IsArray)
{
prop.SetValue(model, list.OfType<object>().ToArray(), null);
}
else
{
prop.SetValue(model, list, null);
}
}
}
}
}
return model;
}
public Object Get(String name, Document document)
{
Field field = document.GetField(name);
return field.StringValue();
}
private static double GetDouble(this LuceneDocument doc, string name)
{
var value = doc.GetField(name)?.GetDoubleValueOrDefault();
return(value ?? 0.0);
}
private long GetLongFromField(Document doc, SortField sortField)
{
var field = doc.GetField(sortField.Field);
if (field == null) return 0;
long data;
if (long.TryParse(field.StringValue, out data) && data >= 0) return data;
return 0;
}
public override void OnIndexEntryCreated(string entryKey, Document document)
{
if (indexName != "Aggregates/ShoppingCart")
return;
var shoppingCart = RavenJObject.Parse(document.GetField("Aggregate").StringValue);
var shoppingCartId = document.GetField("Id").StringValue;
var result = database.Put("shoppingcarts/" + shoppingCartId + "/snapshots/", null, shoppingCart, new RavenJObject(), null);
document.Add(new Field("Snapshot", result.Key, Field.Store.YES, Field.Index.NOT_ANALYZED));
}
