public SampleModel(Document document) { Id = new Guid(document.GetField("Id").StringValue); Description = document.GetField("Description").StringValue; CreatedDate = DateTools.StringToDate(document.GetField("CreatedDate").StringValue); ViewCount = Convert.ToInt32(document.GetField("ViewCount").StringValue); }
internal static FieldType GetFieldType(string field, Lucene.Net.Documents.Document indexDocument) { return(new FieldType { IsArray = indexDocument.GetField(field + LuceneDocumentConverterBase.IsArrayFieldSuffix) != null, IsJson = indexDocument.GetField(field + LuceneDocumentConverterBase.ConvertToJsonSuffix) != null }); }
private static FieldType GetFieldType(IFieldable field, Lucene.Net.Documents.Document indexDocument) { return(new FieldType { IsArray = indexDocument.GetField(field.Name + LuceneDocumentConverterBase.IsArrayFieldSuffix) != null, IsJson = indexDocument.GetField(field.Name + LuceneDocumentConverterBase.ConvertToJsonSuffix) != null, }); }
public override object[] GetParametersForConstructor(string name, ProgramElementType programElementType, string fullFilePath, int definitionLineNumber, int definitionColumnNumber, string snippet, Document document) { string namespaceName = document.GetField(SandoField.Namespace.ToString()).StringValue(); AccessLevel accessLevel = (AccessLevel)Enum.Parse(typeof(AccessLevel), document.GetField(SandoField.AccessLevel.ToString()).StringValue(), true); string body = "not stored in index";//document.GetField(SandoField.Body.ToString()).StringValue(); if(name == String.Empty) name = ProgramElement.UndefinedName; return new object[] { name, definitionLineNumber, definitionColumnNumber, fullFilePath, snippet, accessLevel, namespaceName, body }; }
public override object[] GetParametersForConstructor(string name, ProgramElementType programElementType, string fullFilePath, int definitionLineNumber, int definitionColumnNumber, string snippet, Document document) { AccessLevel accessLevel = (AccessLevel)Enum.Parse(typeof(AccessLevel), document.GetField(SandoField.AccessLevel.ToString()).StringValue(), true); string arguments = document.GetField(SandoField.Arguments.ToString()).StringValue(); string returnType = document.GetField(SandoField.ReturnType.ToString()).StringValue(); bool isConstructor = bool.Parse(document.GetField(SandoField.IsConstructor.ToString()).StringValue()); return new object[] { name, definitionLineNumber, definitionColumnNumber, returnType, accessLevel, arguments, fullFilePath, snippet, isConstructor }; }
public override object[] GetParametersForConstructor(string name, ProgramElementType programElementType, string fullFilePath, int definitionLineNumber, int definitionColumnNumber, string snippet, Document document) { string namespaceName = document.GetField(SandoField.Namespace.ToString()).StringValue(); AccessLevel accessLevel = (AccessLevel)Enum.Parse(typeof(AccessLevel), document.GetField(SandoField.AccessLevel.ToString()).StringValue(), true); string extendedClasses = document.GetField(SandoField.ExtendedClasses.ToString()).StringValue(); string implementedInterfaces = document.GetField(SandoField.ImplementedInterfaces.ToString()).StringValue(); string modifiers = document.GetField(SandoField.Modifiers.ToString()).StringValue(); return new object[] { name, definitionLineNumber, definitionColumnNumber, fullFilePath, snippet, accessLevel, namespaceName, extendedClasses, implementedInterfaces, modifiers, "" }; }
public override object[] GetParametersForConstructor(string name, ProgramElementType programElementType, string fullFilePath, int definitionLineNumber, int definitionColumnNumber, string snippet, Document document) { AccessLevel accessLevel = (AccessLevel)Enum.Parse(typeof(AccessLevel), document.GetField(SandoField.AccessLevel.ToString()).StringValue(), true); string propertyType = document.GetField(SandoField.DataType.ToString()).StringValue(); string body = "not stored in index";//document.GetField(SandoField.Body.ToString()).StringValue(); Guid classId = new Guid(document.GetField(SandoField.ClassId.ToString()).StringValue()); string className = document.GetField(SandoField.ClassName.ToString()).StringValue(); string modifiers = document.GetField(SandoField.Modifiers.ToString()).StringValue(); return new object[] { name, definitionLineNumber, definitionColumnNumber, fullFilePath, snippet, accessLevel, propertyType, body, classId, className, modifiers }; }
/// <summary> /// Directory Članci /// </summary> /// <param name="postVrstaId"></param> /// <returns></returns> public static Directory GetDirectoryClanci() { using (TriglavBL temp = new TriglavBL()) { Directory directoryClanci = FSDirectory.Open(new DirectoryInfo(AppDomain.CurrentDomain.BaseDirectory +"Clanci")); Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_30); var writer = new IndexWriter(directoryClanci, analyzer, true, IndexWriter.MaxFieldLength.LIMITED); List<Post> sviPostovi = temp.getAllClanci(); foreach (var post in sviPostovi) { //Add & boost var clanak = new Document(); clanak.Add(new Field("id", Convert.ToString(post.id), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS, Field.TermVector.NO)); clanak.Add(new Field("PostVrsta", Convert.ToString(post.PostVrsta), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("Naslov", post.Naslov, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); clanak.GetField("Naslov").Boost = (2.0F); clanak.Add(new Field("Sadrzaj", post.Sadrzaj, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); clanak.GetField("Sadrzaj").Boost = (1.2F); if (post.Sazetak != null) { clanak.Add(new Field("Sazetak", post.Sazetak, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); clanak.GetField("Sazetak").Boost = (1.5F); } clanak.Add(new Field("Tagovi", post.Tagovi, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); clanak.GetField("Tagovi").Boost = (2.5F); clanak.Add(new Field("VlasnikId", Convert.ToString(post.VlasnikID), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("VlasnikNadimak", Convert.ToString(post.VlasnikNadimak), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); clanak.Add(new Field("DatumKreiranja", Convert.ToString(post.DatumKreiranja), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("DatumZadnjeIzmjene", Convert.ToString(post.DatumZadnjeIzmjene), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("DatumZadnjeAktivnosti", Convert.ToString(post.DatumZadnjeAktivnosti), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("PrihvaceniOdgovori", Convert.ToString(post.PrihvaceniOdgovori), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("BrojOdgovora", Convert.ToString(post.BrojOdgovora), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("BrojKomentara", Convert.ToString(post.BrojKomentara), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("BrojOmiljenih", Convert.ToString(post.BrojOmiljenih), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("BrojPregleda", Convert.ToString(post.BrojPregleda), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("BrojPoena", Convert.ToString(post.BrojPoena), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("Likes", Convert.ToString(post.Likes), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("Unlikes", Convert.ToString(post.Unlikes), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("PromijenioID", Convert.ToString(post.PromijenioID), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("RoditeljskiPostID", Convert.ToString(post.RoditeljskiPostID), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("BrojRangiranja", Convert.ToString(post.BrojRangiranja), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("PrihvacenaIzmjena", Convert.ToString(post.PrihvacenaIzmjena), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); clanak.Add(new Field("Broj_Razgovora", Convert.ToString(post.Broj_Razgovora), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); writer.AddDocument(clanak); } writer.Optimize(); writer.Dispose(); return directoryClanci; } }
internal LuceneEntry(Document myLuceneDocument, float? myScore = null, int? myDocNum = null) { this.Id = myLuceneDocument.GetField(LuceneIndex.FieldNames[LuceneIndex.Fields.ID]).StringValue(); this.IndexId = myLuceneDocument.GetField(LuceneIndex.FieldNames[LuceneIndex.Fields.INDEX_ID]).StringValue(); this.VertexId = Convert.ToInt64(myLuceneDocument.GetField(LuceneIndex.FieldNames[LuceneIndex.Fields.VERTEX_ID]).StringValue()); this.Text = myLuceneDocument.GetField(LuceneIndex.FieldNames[LuceneIndex.Fields.TEXT]).StringValue(); this.Score = myScore; this.DocNum = myDocNum; var propertyIdField = myLuceneDocument.GetField(LuceneIndex.FieldNames[LuceneIndex.Fields.PROPERTY_ID]); if (propertyIdField != null) { this.PropertyId = Convert.ToInt64(propertyIdField.StringValue()); } }
public Object Get(String name, Document document) { Field field = document.GetField(name); if (field == null) return null; return stringBridge.StringToObject(field.StringValue()); }
// Activity 7 /// <summary> /// Outputs results to the screen /// </summary> /// <param name="results">Search results</param> public void DisplayResults(TopDocs results) { int rank = 0; foreach (ScoreDoc scoreDoc in results.ScoreDocs) { rank++; // retrieve the document from the 'ScoreDoc' object Lucene.Net.Documents.Document doc = searcher.Doc(scoreDoc.Doc); if (doc.GetField("passage_text") != null) { string Result_passage_text = doc.Get("passage_text").ToString(); string Result_url = doc.Get("url").ToString(); Console.WriteLine("\n\nRank: " + rank + "\nPassage text: " + Result_passage_text); Console.WriteLine("url: " + Result_url); } else { string Result_query = doc.Get("query").ToString(); string Result_answers = doc.Get("answers").ToString(); Console.WriteLine("\n\nRank: " + rank + "\nQuery: " + Result_query); Console.WriteLine("answers: " + Result_answers); } //Console.WriteLine("Rank " + rank + " score " + scoreDoc.Score + " text " + myFieldValue); } }
public override void OnIndexEntryCreated(string entryKey, Document document) { var resultDocId = document.GetField(setupDoc.DocumentKey); if (resultDocId == null) { log.Warn("Could not find document id property '{0}' in '{1}' for index '{2}'", setupDoc.DocumentKey, entryKey, index); return; } var documentId = resultDocId.StringValue; itemsToRemove.Remove(documentId); var resultDoc = database.Get(documentId, null); if (resultDoc == null) { log.Warn("Could not find a document with the id '{0}' for index '{1}'", documentId, index); return; } var entityName = resultDoc.Metadata.Value<string>(Constants.RavenEntityName); if(entityName != null && viewGenerator.ForEntityNames.Contains(entityName)) { log.Warn( "Rejected update for a potentially recursive update on document '{0}' because the index '{1}' includes documents with entity name of '{2}'", documentId, index, entityName); return; } if(viewGenerator.ForEntityNames.Count == 0) { log.Warn( "Rejected update for a potentially recursive update on document '{0}' because the index '{1}' includes all documents", documentId, index); return; } var changesMade = false; foreach (var mapping in setupDoc.FieldNameMappings) { var field = document.GetFieldable(mapping.Key + "_Range") ?? document.GetFieldable(mapping.Key); if (field == null) continue; var numericField = field as NumericField; if (numericField != null) { resultDoc.DataAsJson[mapping.Value] = new RavenJValue(numericField.NumericValue); } else { resultDoc.DataAsJson[mapping.Value] = field.StringValue; } changesMade = true; } if (changesMade) database.Put(documentId, resultDoc.Etag, resultDoc.DataAsJson, resultDoc.Metadata, null); }
public static IndexDocumentData FromDocument(Document doc) { return new IndexDocumentData() { Package = PackageJson.FromJson(JObject.Parse(doc.GetField("Data").StringValue)), Checksum = Int32.Parse(doc.GetFieldable("Checksum").StringValue), Feeds = doc.GetFields("CuratedFeed").Select(f => f.StringValue).ToList() }; }
//--- Constructors --- public LuceneResult(Document document, float score) { Document = document; Score = score; var field = document.GetField("id.page"); if(field == null) { return; } ulong pageId; PageId = ulong.TryParse(field.StringValue(), out pageId) ? pageId : (ulong?)null; }
internal static Type GetMyType(Document luceneDocument) { try { string typeId = luceneDocument.GetField(ProgramElement.CustomTypeTag).StringValue(); return Type.GetType(typeId); } catch { return typeof (ProgramElement); } }
internal static void SetCustomFields(object myElement, Lucene.Net.Documents.Document luceneDocument) { foreach (var property in (myElement as ProgramElement).GetCustomProperties()) { if (!property.Name.Equals(ProgramElement.CustomTypeTag)) { Field field = luceneDocument.GetField(property.Name); Contract.Assert(field != null, "Field " + property.Name + " was not populated"); property.SetValue(myElement, field.StringValue(), null); } } }
public override void OnIndexEntryCreated(string entryKey, Document document) { var conflitsIndex = entryKey.IndexOf("/conflicts/", StringComparison.Ordinal); if (conflitsIndex <= 0) return; var documentIdField = document.GetField(Constants.DocumentIdFieldName); if(documentIdField == null) return; documentIdField.SetValue(entryKey.Substring(0, conflitsIndex)); }
public void ThenIWantAnAnalyzedNormsFieldAdded() { // Arrange var document = new Document(); // Act document.Add("Bar").Indexed().Analyzed().As("Foo"); // Assert var field = document.GetField("Foo"); Assert.True(field.IsIndexed); Assert.True(field.IsTokenized); Assert.False(field.OmitNorms); }
public ProgramElement Convert() { Contract.Ensures(Contract.Result <ProgramElement>() != null, "ConverterFromHitToProgramElement:ReadProgramElementFromDocument - an object must be returned from this method!"); //Get standard field values string name = luceneDocument.GetField(SandoField.Name.ToString()).StringValue().ToSandoDisplayable(); ProgramElementType type = (ProgramElementType)Enum.Parse(typeof(ProgramElementType), luceneDocument.GetField(SandoField.ProgramElementType.ToString()).StringValue(), true); string fullFilePath = luceneDocument.GetField(SandoField.FullFilePath.ToString()).StringValue(); int definitionLineNumber = int.Parse(luceneDocument.GetField(SandoField.DefinitionLineNumber.ToString()).StringValue()); int definitionColumnNumber = int.Parse(luceneDocument.GetField(SandoField.DefinitionColumnNumber.ToString()).StringValue()); string snippet = luceneDocument.GetField(SandoField.Source.ToString()).StringValue(); //Add values that vary according to element type (e.g., a textline doesn't have a parent class whereas a method often does) //Note: Parameters must match the parameter list for corresponding program element. See MethodDocument.GetParametersForConstructor and the MethodElement constructor var parameters = sandoDocument.GetParametersForConstructor(name, type, fullFilePath, definitionLineNumber, definitionColumnNumber, snippet, luceneDocument); //Create type from the collected values var myClassType = GetMyType(luceneDocument); var myElement = Activator.CreateInstance(myClassType, parameters); //Populate any custom fields from user-defined types, only happens when a third party extends Sando SetCustomFields(myElement, luceneDocument); return(myElement as ProgramElement); }
public void ThenIWantItToBeIndexedAndBoosted() { // Arrange var document = new Document(); var boost = 2.0f; // Act document.Add("Bar").Indexed().BoostBy(boost).As("Foo"); // Assert var field = document.GetField("Foo"); Assert.True(field.IsIndexed); Assert.False(field.IsTokenized); Assert.False(field.OmitNorms); Assert.Equal(boost, field.Boost); }
public SearchResultViewModel(Document document, ScoreDoc scoreDoc) { if (document == null) throw new ArgumentNullException("document"); if (scoreDoc == null) throw new ArgumentNullException("scoreDoc"); EnsureFieldsExist(document); Id = int.Parse(document.GetField("id").StringValue); Title = document.GetField("title").StringValue; ContentSummary = document.GetField("contentsummary").StringValue; Tags = document.GetField("tags").StringValue; CreatedBy = document.GetField("createdby").StringValue; ContentLength = int.Parse(document.GetField("contentlength").StringValue); Score = scoreDoc.Score; DateTime createdOn = DateTime.UtcNow; if (!DateTime.TryParse(document.GetField("createdon").StringValue, out createdOn)) createdOn = DateTime.UtcNow; CreatedOn = createdOn; }
private static KeyValuePair<string, RavenJToken> CreateProperty(Field fld, Document document) { var stringValue = fld.StringValue(); if (document.GetField(fld.Name() + "_ConvertToJson") != null) { var val = RavenJToken.Parse(fld.StringValue()) as RavenJObject; return new KeyValuePair<string, RavenJToken>(fld.Name(), val); } if (stringValue == Constants.NullValue) stringValue = null; return new KeyValuePair<string, RavenJToken>(fld.Name(), stringValue); }
public virtual void searchIndex(System.String dirName, System.String oldName) { //QueryParser parser = new QueryParser("contents", new WhitespaceAnalyzer()); //Query query = parser.parse("handle:1"); dirName = FullDir(dirName); Directory dir = FSDirectory.Open(new System.IO.DirectoryInfo(dirName)); IndexSearcher searcher = new IndexSearcher(dir, true); IndexReader reader = searcher.IndexReader; _TestUtil.CheckIndex(dir); for (int i = 0; i < 35; i++) { if (!reader.IsDeleted(i)) { Document d = reader.Document(i); var fields = d.GetFields(); if (!oldName.StartsWith("19.") && !oldName.StartsWith("20.") && !oldName.StartsWith("21.") && !oldName.StartsWith("22.")) { if (d.GetField("content3") == null) { int numFields = oldName.StartsWith("29.") ? 7 : 5; Assert.AreEqual(numFields, fields.Count); Field f = d.GetField("id"); Assert.AreEqual("" + i, f.StringValue); f = (Field)d.GetField("utf8"); Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.StringValue); f = (Field)d.GetField("autf8"); Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.StringValue); f = (Field)d.GetField("content2"); Assert.AreEqual("here is more content with aaa aaa aaa", f.StringValue); f = (Field)d.GetField("fie\u2C77ld"); Assert.AreEqual("field with non-ascii name", f.StringValue); } } } // Only ID 7 is deleted else { Assert.AreEqual(7, i); } } ScoreDoc[] hits = searcher.Search(new TermQuery(new Term("content", "aaa")), null, 1000).ScoreDocs; // First document should be #21 since it's norm was // increased: Document d2 = searcher.Doc(hits[0].Doc); Assert.AreEqual("21", d2.Get("id"), "didn't get the right document first"); TestHits(hits, 34, searcher.IndexReader); if (!oldName.StartsWith("19.") && !oldName.StartsWith("20.") && !oldName.StartsWith("21.") && !oldName.StartsWith("22.")) { // Test on indices >= 2.3 hits = searcher.Search(new TermQuery(new Term("utf8", "\u0000")), null, 1000).ScoreDocs; Assert.AreEqual(34, hits.Length); hits = searcher.Search(new TermQuery(new Term("utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne")), null, 1000).ScoreDocs; Assert.AreEqual(34, hits.Length); hits = searcher.Search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), null, 1000).ScoreDocs; Assert.AreEqual(34, hits.Length); } searcher.Close(); dir.Close(); }
private static int GetInt(this LuceneDocument doc, string name) { var value = doc.GetField(name)?.GetInt32ValueOrDefault(); return(value ?? 0); }
private static KeyValuePair<string, RavenJToken> CreateProperty(Field fld, Document document) { if (fld.IsBinary) return new KeyValuePair<string, RavenJToken>(fld.Name, fld.GetBinaryValue()); var stringValue = fld.StringValue; if (document.GetField(fld.Name + "_ConvertToJson") != null) { var val = RavenJToken.Parse(fld.StringValue) as RavenJObject; return new KeyValuePair<string, RavenJToken>(fld.Name, val); } if (stringValue == Constants.NullValue) stringValue = null; if (stringValue == Constants.EmptyString) stringValue = string.Empty; return new KeyValuePair<string, RavenJToken>(fld.Name, stringValue); }
protected bool TryGetValue(FieldsToFetch.FieldToFetch fieldToFetch, Document document, Lucene.Net.Documents.Document luceneDoc, IState state, Dictionary <string, IndexField> indexFields, bool?anyDynamicIndexFields, out string key, out object value) { key = fieldToFetch.ProjectedName ?? fieldToFetch.Name.Value; if (fieldToFetch.QueryField == null) { return(TryGetFieldValueFromDocument(document, fieldToFetch, out value)); } if (fieldToFetch.QueryField.Function != null) { var args = new object[fieldToFetch.QueryField.FunctionArgs.Length + 1]; for (int i = 0; i < fieldToFetch.FunctionArgs.Length; i++) { TryGetValue(fieldToFetch.FunctionArgs[i], document, luceneDoc, state, indexFields, anyDynamicIndexFields, out _, out args[i]); if (ReferenceEquals(args[i], document)) { args[i] = Tuple.Create(document, luceneDoc, state, indexFields, anyDynamicIndexFields); } } value = GetFunctionValue(fieldToFetch, document.Id, args); return(true); } if (fieldToFetch.QueryField.IsCounter) { string name; string id = document.Id; if (fieldToFetch.QueryField.IsParameter) { if (_query.QueryParameters == null) { throw new InvalidQueryException("The query is parametrized but the actual values of parameters were not provided", _query.Query, null); } if (_query.QueryParameters.TryGetMember(fieldToFetch.QueryField.Name, out var nameObject) == false) { throw new InvalidQueryException($"Value of parameter '{fieldToFetch.QueryField.Name}' was not provided", _query.Query, _query.QueryParameters); } name = nameObject.ToString(); key = fieldToFetch.QueryField.Alias ?? name; } else { name = fieldToFetch.Name.Value; } if (fieldToFetch.QueryField.SourceAlias != null && BlittableJsonTraverser.Default.TryRead(document.Data, fieldToFetch.QueryField.SourceAlias, out var sourceId, out _)) { id = sourceId.ToString(); } if (fieldToFetch.QueryField.FunctionArgs != null) { value = GetCounterRaw(id, name); } else { value = GetCounter(id, name); } return(true); } if (fieldToFetch.QueryField.ValueTokenType != null) { var val = fieldToFetch.QueryField.Value; if (fieldToFetch.QueryField.ValueTokenType.Value == ValueTokenType.Parameter) { if (_query == null) { value = null; return(false); // only happens for debug endpoints and more like this } _query.QueryParameters.TryGet((string)val, out val); } value = val; return(true); } if (fieldToFetch.QueryField.HasSourceAlias == false) { return(TryGetFieldValueFromDocument(document, fieldToFetch, out value)); } if (_loadedDocumentIds == null) { _loadedDocumentIds = new HashSet <string>(); _loadedDocuments = new Dictionary <string, Document>(); _loadedDocumentsByAliasName = new Dictionary <string, Document>(); } _loadedDocumentIds.Clear(); //_loadedDocuments.Clear(); - explicitly not clearing this, we want to cache this for the duration of the query _loadedDocuments[document.Id ?? string.Empty] = document; if (fieldToFetch.QueryField.SourceAlias != null) { if (fieldToFetch.QueryField.IsQuoted) { _loadedDocumentIds.Add(fieldToFetch.QueryField.SourceAlias); } else if (fieldToFetch.QueryField.IsParameter) { if (_query.QueryParameters == null) { throw new InvalidQueryException("The query is parametrized but the actual values of parameters were not provided", _query.Query, (BlittableJsonReaderObject)null); } if (_query.QueryParameters.TryGetMember(fieldToFetch.QueryField.SourceAlias, out var id) == false) { throw new InvalidQueryException($"Value of parameter '{fieldToFetch.QueryField.SourceAlias}' was not provided", _query.Query, _query.QueryParameters); } _loadedDocumentIds.Add(id.ToString()); } else if (fieldToFetch.QueryField.LoadFromAlias != null) { if (_loadedDocumentsByAliasName.TryGetValue(fieldToFetch.QueryField.LoadFromAlias, out var loadedDoc)) { IncludeUtil.GetDocIdFromInclude(loadedDoc.Data, fieldToFetch.QueryField.SourceAlias, _loadedDocumentIds, _database.IdentityPartsSeparator); } } else if (fieldToFetch.CanExtractFromIndex) { if (luceneDoc != null) { var field = luceneDoc.GetField(fieldToFetch.QueryField.SourceAlias); if (field != null) { var fieldValue = ConvertType(_context, field, GetFieldType(field.Name, luceneDoc), state); _loadedDocumentIds.Add(fieldValue.ToString()); } } } else { IncludeUtil.GetDocIdFromInclude(document.Data, fieldToFetch.QueryField.SourceAlias, _loadedDocumentIds, _database.IdentityPartsSeparator); } } else { _loadedDocumentIds.Add(document.Id ?? string.Empty); // null source alias is the root doc _loadedDocumentsByAliasName.Clear(); } if (_loadedDocumentIds.Count == 0) { if (fieldToFetch.QueryField.SourceIsArray) { value = new List <object>(); return(true); } value = null; return(false); } var buffer = new List <object>(); foreach (var docId in _loadedDocumentIds) { if (docId == null) { continue; } if (_loadedDocuments.TryGetValue(docId, out var doc) == false) { using (_loadScope = _loadScope?.Start() ?? _projectionScope?.For(nameof(QueryTimingsScope.Names.Load))) _loadedDocuments[docId] = doc = LoadDocument(docId); } if (doc == null) { continue; } if (fieldToFetch.QueryField.Alias != null) { _loadedDocumentsByAliasName[fieldToFetch.QueryField.Alias] = doc; } if (string.IsNullOrEmpty(fieldToFetch.Name.Value)) // we need the whole document here { buffer.Add(doc); continue; } if (TryGetFieldValueFromDocument(doc, fieldToFetch, out var val)) { if (val is string == false && val is LazyStringValue == false && val is System.Collections.IEnumerable items) { // we flatten arrays in projections foreach (var item in items) { buffer.Add(item); } fieldToFetch.QueryField.SourceIsArray = true; } else { buffer.Add(val); } } } if (fieldToFetch.QueryField.SourceIsArray) { value = buffer; return(true); } if (buffer.Count > 0) { if (buffer.Count > 1) { ThrowOnlyArrayFieldCanHaveMultipleValues(fieldToFetch); } value = buffer[0]; return(true); } value = null; return(false); }
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score) { fieldsToFetch.EnsureHasField(Constants.ReduceKeyFieldName); if (fieldsToFetch.HasExplicitFieldsToFetch) { return base.RetrieveDocument(document, fieldsToFetch, score); } var field = document.GetField(Constants.ReduceValueFieldName); if (field == null) { fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().Select(x => x.Name).ToArray()); return base.RetrieveDocument(document, fieldsToFetch, score); } var projection = RavenJObject.Parse(field.StringValue); if (fieldsToFetch.FetchAllStoredFields) { var fields = new HashSet<string>(document.GetFields().Select(x => x.Name)); fields.Remove(Constants.ReduceKeyFieldName); var documentFromFields = new RavenJObject(); AddFieldsToDocument(document, fields, documentFromFields); foreach (var kvp in projection) { documentFromFields[kvp.Key] = kvp.Value; } projection = documentFromFields; } return new IndexQueryResult { Projection = projection, Score = score.Score, ReduceVal = field.StringValue }; }
private static DateTime GetDate(this LuceneDocument doc, string name) { var ticks = doc.GetField(name)?.GetInt64ValueOrDefault(); return(ticks.HasValue ? new DateTime(ticks.Value) : DateTime.MinValue); }
private static bool GetBool(this LuceneDocument doc, string name) { var value = doc.GetField(name)?.GetInt32ValueOrDefault(); return(value == 1); }
private string GetStringFromField(Document doc, SortField sortField) { var field = doc.GetField(sortField.Field); return field == null ? "" : field.StringValue; }
/// <summary> /// Returns a field with the given name if any exist in this document cast to type <typeparam name="T"/>, or /// <c>null</c>. If multiple fields exists with this name, this method returns the /// first value added. /// <para/> /// LUCENENET specific /// </summary> /// <exception cref="InvalidCastException">If the field type cannot be cast to <typeparam name="T"/>.</exception> public static T GetField <T>(this Document document, string name) where T : IIndexableField { return((T)document.GetField(name)); }
public override void OnIndexEntryCreated(string entryKey, Document document) { var resultDocId = document.GetField(setupDoc.DocumentKey); if (resultDocId == null) { log.Warn("Could not find document id property '{0}' in '{1}' for index '{2}'", setupDoc.DocumentKey, entryKey, index); return; } var documentId = resultDocId.StringValue; itemsToRemove.TryRemove(documentId); var resultDoc = database.Get(documentId, null); if (resultDoc == null) { log.Warn("Could not find a document with the id '{0}' for index '{1}'", documentId, index); return; } var entityName = resultDoc.Metadata.Value <string>(Constants.RavenEntityName); if (entityName != null && viewGenerator.ForEntityNames.Contains(entityName)) { log.Warn( "Rejected update for a potentially recursive update on document '{0}' because the index '{1}' includes documents with entity name of '{2}'", documentId, index, entityName); return; } if (viewGenerator.ForEntityNames.Count == 0) { log.Warn( "Rejected update for a potentially recursive update on document '{0}' because the index '{1}' includes all documents", documentId, index); return; } var changesMade = false; foreach (var mapping in setupDoc.FieldNameMappings) { var field = document.GetFieldable(mapping.Key + "_Range") ?? document.GetFieldable(mapping.Key); if (field == null) { continue; } var numericField = field as NumericField; if (numericField != null) { resultDoc.DataAsJson[mapping.Value] = new RavenJValue(numericField.NumericValue); } else { resultDoc.DataAsJson[mapping.Value] = field.StringValue; } changesMade = true; } if (changesMade) { database.Put(documentId, resultDoc.Etag, resultDoc.DataAsJson, resultDoc.Metadata, null); } }
private static SearchResult GetPartialSearchResult(Document doc) { var result = new SearchResult(); var localeField = doc.GetField(Constants.LocaleTag); if (localeField != null) { int id; result.CultureCode = int.TryParse(localeField.StringValue, out id) && id >= 0 ? LocaleController.Instance.GetLocale(id).Code : Null.NullString; } FillTagsValues(doc, result); return result; }
private static RavenJObject CreateDocumentFromFields(Document document, IEnumerable<string> fieldsToFetch) { var documentFromFields = new RavenJObject(); var q = fieldsToFetch .SelectMany(name => document.GetFields(name) ?? new Field[0]) .Where(x => x != null) .Where( x => x.Name().EndsWith("_IsArray") == false && x.Name().EndsWith("_Range") == false && x.Name().EndsWith("_ConvertToJson") == false) .Select(fld => CreateProperty(fld, document)) .GroupBy(x => x.Key) .Select(g => { if (g.Count() == 1 && document.GetField(g.Key + "_IsArray") == null) { return g.First(); } return new KeyValuePair<string, RavenJToken>(g.Key, new RavenJArray(g.Select(x => x.Value))); }); foreach (var keyValuePair in q) { documentFromFields.Add(keyValuePair.Key, keyValuePair.Value); } return documentFromFields; }
private static RavenJObject CreateJsonDocumentFromLuceneDocument(Document document) { var field = document.GetField(Constants.ReduceValueFieldName); if (field != null) return RavenJObject.Parse(field.StringValue); var ravenJObject = new RavenJObject(); var fields = document.GetFields(); var arrayMarkers = fields .Where(x => x.Name.EndsWith("_IsArray")) .Select(x => x.Name) .ToList(); foreach (var fieldable in fields) { var stringValue = GetStringValue(fieldable); var isArrayMarker = fieldable.Name.EndsWith("_IsArray"); var isArray = !isArrayMarker && arrayMarkers.Contains(fieldable.Name + "_IsArray"); RavenJToken token; var isJson = RavenJToken.TryParse(stringValue, out token); RavenJToken value; if (ravenJObject.TryGetValue(fieldable.Name, out value) == false) { if (isArray) ravenJObject[fieldable.Name] = new RavenJArray { isJson ? token : stringValue }; else if (isArrayMarker) { var fieldName = fieldable.Name.Substring(0, fieldable.Name.Length - 8); ravenJObject[fieldable.Name] = isJson ? token : stringValue; ravenJObject[fieldName] = new RavenJArray(); } else ravenJObject[fieldable.Name] = isJson ? token : stringValue; } else { var ravenJArray = value as RavenJArray; if (ravenJArray != null) ravenJArray.Add(isJson ? token : stringValue); else { ravenJArray = new RavenJArray { value, isJson ? token : stringValue }; ravenJObject[fieldable.Name] = ravenJArray; } } } return ravenJObject; }
//TODO: The last param, Cache isn't used. Remove it and update dependant projects. (Version 1.3) public static SearchResults <Document> GetDocuments(QueryParser defaultQueryParser, QueryParser customQueryParser, IndexSearcher indexSearcher, string query, int pageNumber, int pageSize, bool shouldDocumentsBeClustered, string sort, int maximumNumberOfDocumentsToScore, Cache cache) { Query query2 = customQueryParser.Parse(query); Hits hits = null; if (!string.IsNullOrEmpty(sort)) { string[] sorts = sort.ToLower().Split(",".ToCharArray(), StringSplitOptions.RemoveEmptyEntries); List <SortField> sortFields = new List <SortField>(sorts.Length / 2); for (int i = 0; i < sorts.Length; i++) { if (sorts[i].Split(' ')[1] == "asc") { sortFields.Add(new SortField(sorts[i].Split(' ')[0], false)); } else { sortFields.Add(new SortField(sorts[i].Split(' ')[0], true)); } } hits = indexSearcher.Search(query2, new Sort(sortFields.ToArray())); } else { hits = indexSearcher.Search(query2); } SearchResults <Document> searchResults = new SearchResults <Document>(); searchResults.Documents = new List <Document>(); searchResults.Query = query2; if (hits.Length() != 0) { Dictionary <string, string> domains = new Dictionary <string, string>(); PriorityQueue <Document> priorityQueue = new PriorityQueue <Document>(); //Get the Hits!!! //TODO: Optimize this!!! (Version 1.3) for (int j = 0; j < hits.Length() && searchResults.Documents.Count < maximumNumberOfDocumentsToScore && priorityQueue.Count < maximumNumberOfDocumentsToScore; j++) { Document document = hits.Doc(j); float score = hits.Score(j); document.Add(new Field("documentid", j.ToString(), Field.Store.YES, Field.Index.NO)); document.Add(new Field("relevancyscore", score.ToString(), Field.Store.YES, Field.Index.NO)); if (!string.IsNullOrEmpty(sort)) { if (shouldDocumentsBeClustered) { if (document.GetField("domain") != null) { string domain = document.GetField("domain").StringValue(); if (!domains.ContainsKey(domain)) { domains.Add(domain, null); if (searchResults.Documents.Count < pageSize && j >= (pageNumber * pageSize) - pageSize) { searchResults.Documents.Add(document); } } } } else { if (searchResults.Documents.Count < pageSize && j >= (pageNumber * pageSize) - pageSize) { searchResults.Documents.Add(document); } } } else { priorityQueue.Enqueue(document, score * double.Parse(document.GetField("strength").StringValue())); } } if (string.IsNullOrEmpty(sort)) { for (int i = 0; i < hits.Length() && priorityQueue.Count != 0; i++) { Document document = priorityQueue.Dequeue(); if (shouldDocumentsBeClustered) { if (document.GetField("domain") != null) { string domain = document.GetField("domain").StringValue(); if (!domains.ContainsKey(domain)) { domains.Add(domain, null); if (searchResults.Documents.Count < pageSize && i >= (pageNumber * pageSize) - pageSize) { searchResults.Documents.Add(document); } } else { i--; } } } else { if (searchResults.Documents.Count < pageSize && i >= (pageNumber * pageSize) - pageSize) { searchResults.Documents.Add(document); } } } } if (shouldDocumentsBeClustered) { searchResults.TotalNumberOfHits = domains.Count; } else { searchResults.TotalNumberOfHits = hits.Length(); } } return(searchResults); }
private LogEntry DocumentToLogEntry(Document document) { return new LogEntry() { Id = Guid.Parse(document.GetField("id").StringValue), ApplicationName = document.GetField("application").StringValue, Server = document.GetField("server").StringValue, Environment = document.GetField("environment").StringValue, DateTime = DateTime.Parse(document.GetField("date").StringValue), ExceptionMessage = document.GetField("exceptionMessage").StringValue, ExceptionType = document.GetField("exceptionType").StringValue, Message = document.GetField("truncatedMessage").StringValue, }; }
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score) { fieldsToFetch.EnsureHasField(Constants.ReduceKeyFieldName); if (fieldsToFetch.HasExplicitFieldsToFetch) { return base.RetrieveDocument(document, fieldsToFetch, score); } var field = document.GetField(Constants.ReduceValueFieldName); if (field == null) { fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().Select(x => x.Name).ToArray()); return base.RetrieveDocument(document, fieldsToFetch, score); } return new IndexQueryResult { Projection = RavenJObject.Parse(field.StringValue), Score = score.Score }; }
public int GetInt(string name) { var field = _doc.GetField(name); return(field == null ? 0 : Int32.Parse(field.StringValue)); }
public static RavenJObject CreateDocumentFromFields(Document document, FieldsToFetch fieldsToFetch) { var documentFromFields = new RavenJObject(); var fields = fieldsToFetch.Fields; if (fieldsToFetch.FetchAllStoredFields) fields = fields.Concat(document.GetFields().Select(x => x.Name)); var q = fields .Distinct() .SelectMany(name => document.GetFields(name) ?? new Field[0]) .Where(x => x != null) .Where( x => x.Name.EndsWith("_IsArray") == false && x.Name.EndsWith("_Range") == false && x.Name.EndsWith("_ConvertToJson") == false) .Select(fld => CreateProperty(fld, document)) .GroupBy(x => x.Key) .Select(g => { if (g.Count() == 1 && document.GetField(g.Key + "_IsArray") == null) { return g.First(); } var ravenJTokens = g.Select(x => x.Value).ToArray(); return new KeyValuePair<string, RavenJToken>(g.Key, new RavenJArray((IEnumerable)ravenJTokens)); }); foreach (var keyValuePair in q) { documentFromFields.Add(keyValuePair.Key, keyValuePair.Value); } return documentFromFields; }
public static object ToModel(Document document, Type modelType) { var model = Activator.CreateInstance(modelType); foreach (var prop in modelType.GetProperties(BindingFlags.Public | BindingFlags.Instance)) { if (TypeHelper.IsSimpleType(prop.PropertyType)) { var field = document.GetField(prop.Name); if (field != null) { var propValue = LuceneUtility.FromFieldStringValue(field.StringValue, prop.PropertyType); prop.SetValue(model, propValue, null); } } else { var propTypeInfo = ModelTypeInfo.GetTypeInfo(prop.PropertyType); if (propTypeInfo.IsCollection) { if (propTypeInfo.IsDictionary) { var propValue = prop.GetValue(model, null); if (propValue == null) { propValue = Activator.CreateInstance(typeof(Dictionary<,>).MakeGenericType(propTypeInfo.DictionaryKeyType, propTypeInfo.DictionaryValueType)); prop.SetValue(model, propValue, null); } var dic = propValue as IDictionary; var fields = document.GetFields().Where(f => f.Name.StartsWith(prop.Name)); // Property type is IDictionary<TKey, TValue> if (TypeHelper.IsSimpleType(propTypeInfo.DictionaryValueType)) { foreach (var field in fields) { string propName; string dicKey; if (TryParseDictionaryFieldName(field.Name, out propName, out dicKey)) { var fieldValue = LuceneUtility.FromFieldStringValue(field.StringValue, propTypeInfo.DictionaryValueType); dic.Add(dicKey, fieldValue); } } } else // Property type is IDictionary<TKey, IList<TValue>> or IDictionary<TKey, ISet<TValue>> { var dicValueTypeInfo = ModelTypeInfo.GetTypeInfo(propTypeInfo.DictionaryValueType); if (dicValueTypeInfo.IsCollection && TypeHelper.IsSimpleType(dicValueTypeInfo.ElementType)) { Type newDicValueType = null; MethodInfo hashsetAddMethod = null; if (dicValueTypeInfo.IsSet) { newDicValueType = typeof(HashSet<>).MakeGenericType(dicValueTypeInfo.ElementType); hashsetAddMethod = GetAddMethod(newDicValueType, dicValueTypeInfo.ElementType); } else { newDicValueType = typeof(List<>).MakeGenericType(dicValueTypeInfo.ElementType); } foreach (var field in fields) { string propName; string dicKey; if (TryParseDictionaryFieldName(field.Name, out propName, out dicKey)) { var fieldValue = LuceneUtility.FromFieldStringValue(field.StringValue, dicValueTypeInfo.ElementType); if (!dic.Contains(dicKey)) { dic.Add(dicKey, Activator.CreateInstance(newDicValueType)); } var list = dic[dicKey]; if (dicValueTypeInfo.IsSet) // is HashSet<> { hashsetAddMethod.Invoke(list, new[] { fieldValue }); } else // is IList<> { (list as IList).Add(fieldValue); } } } } } } else // Property is collection but not dictionary { var fields = document.GetFields(prop.Name); if (fields.Length == 0) { continue; } var list = Activator.CreateInstance(typeof(List<>).MakeGenericType(propTypeInfo.ElementType)) as IList; foreach (var field in fields) { var fieldValue = LuceneUtility.FromFieldStringValue(field.StringValue, propTypeInfo.ElementType); list.Add(fieldValue); } if (prop.PropertyType.IsArray) { prop.SetValue(model, list.OfType<object>().ToArray(), null); } else { prop.SetValue(model, list, null); } } } } } return model; }
public Object Get(String name, Document document) { Field field = document.GetField(name); return field.StringValue(); }
private static double GetDouble(this LuceneDocument doc, string name) { var value = doc.GetField(name)?.GetDoubleValueOrDefault(); return(value ?? 0.0); }
private long GetLongFromField(Document doc, SortField sortField) { var field = doc.GetField(sortField.Field); if (field == null) return 0; long data; if (long.TryParse(field.StringValue, out data) && data >= 0) return data; return 0; }
public override void OnIndexEntryCreated(string entryKey, Document document) { if (indexName != "Aggregates/ShoppingCart") return; var shoppingCart = RavenJObject.Parse(document.GetField("Aggregate").StringValue); var shoppingCartId = document.GetField("Id").StringValue; var result = database.Put("shoppingcarts/" + shoppingCartId + "/snapshots/", null, shoppingCart, new RavenJObject(), null); document.Add(new Field("Snapshot", result.Key, Field.Store.YES, Field.Index.NOT_ANALYZED)); }