public MoreLikeThisQueryResultServerSide ExecuteMoreLikeThisQuery(MoreLikeThisQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token)
{
if (query == null)
{
throw new ArgumentNullException(nameof(query));
}
if (string.IsNullOrWhiteSpace(query.DocumentId) &&
query.MapGroupFields.Count == 0 &&
string.IsNullOrWhiteSpace(query.Document))
{
throw new InvalidOperationException("The document id, document (artificial document property) or map group fields are mandatory");
}
var sw = Stopwatch.StartNew();
var index = GetIndex(query.Metadata.IndexName);
if (existingResultEtag.HasValue)
{
var etag = index.GetIndexEtag();
if (etag == existingResultEtag.Value)
{
return(MoreLikeThisQueryResultServerSide.NotModifiedResult);
}
}
context.OpenReadTransaction();
var result = index.MoreLikeThisQuery(query, context, token);
result.DurationInMs = (int)sw.Elapsed.TotalMilliseconds;
return(result);
}
private MoreLikeThisQueryServerSide GetMoreLikeThisQuery(JsonOperationContext context, HttpMethod method)
{
if (method == HttpMethod.Get)
{
//MoreLikeThisQueryServerSide.Create(HttpContext, GetPageSize(), context);
throw new NotImplementedException();
}
var json = context.ReadForMemory(RequestBodyStream(), "morelikethis/query");
return(MoreLikeThisQueryServerSide.Create(json));
}
private static void AssignParameters(MoreLikeThis mlt, MoreLikeThisQueryServerSide parameters)
{
if (parameters.Boost != null)
{
mlt.Boost = parameters.Boost.Value;
}
if (parameters.BoostFactor != null)
{
mlt.BoostFactor = parameters.BoostFactor.Value;
}
if (parameters.MaximumNumberOfTokensParsed != null)
{
mlt.MaxNumTokensParsed = parameters.MaximumNumberOfTokensParsed.Value;
}
if (parameters.MaximumQueryTerms != null)
{
mlt.MaxQueryTerms = parameters.MaximumQueryTerms.Value;
}
if (parameters.MinimumWordLength != null)
{
mlt.MinWordLen = parameters.MinimumWordLength.Value;
}
if (parameters.MaximumWordLength != null)
{
mlt.MaxWordLen = parameters.MaximumWordLength.Value;
}
if (parameters.MinimumTermFrequency != null)
{
mlt.MinTermFreq = parameters.MinimumTermFrequency.Value;
}
if (parameters.MinimumDocumentFrequency != null)
{
mlt.MinDocFreq = parameters.MinimumDocumentFrequency.Value;
}
if (parameters.MaximumDocumentFrequency != null)
{
mlt.MaxDocFreq = parameters.MaximumDocumentFrequency.Value;
}
if (parameters.MaximumDocumentFrequencyPercentage != null)
{
mlt.SetMaxDocFreqPct(parameters.MaximumDocumentFrequencyPercentage.Value);
}
}
private void MoreLikeThis(DocumentsOperationContext context, string indexName, OperationCancelToken token)
{
var existingResultEtag = GetLongFromHeaders("If-None-Match");
var query = MoreLikeThisQueryServerSide.Create(HttpContext, GetPageSize(Database.Configuration.Core.MaxPageSize), context);
var runner = new QueryRunner(Database, context);
var result = runner.ExecuteMoreLikeThisQuery(indexName, query, context, existingResultEtag, token);
if (result.NotModified)
{
HttpContext.Response.StatusCode = 304;
return;
}
HttpContext.Response.Headers[Constants.MetadataEtagField] = result.ResultEtag.ToInvariantString();
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
writer.WriteQueryResult(context, result, metadataOnly: false);
}
}
public static void WriteMoreLikeThisQuery(this BlittableJsonTextWriter writer, JsonOperationContext context, MoreLikeThisQueryServerSide query)
{
var djv = (DynamicJsonValue)TypeConverter.ToBlittableSupportedType(query);
var json = context.ReadObject(djv, "more-like-this-query");
writer.WriteObject(json);
}
public override MoreLikeThisQueryResultServerSide MoreLikeThisQuery(MoreLikeThisQueryServerSide query, DocumentsOperationContext documentsContext, OperationCancelToken token)
{
throw new NotSupportedException($"Index with id {IndexId} is in-memory implementation of a faulty index", _e);
}
public IEnumerable <Document> MoreLikeThis(
MoreLikeThisQueryServerSide query,
HashSet <string> stopWords,
Func <SelectField[], IQueryResultRetriever> createRetriever,
JsonOperationContext context,
Func <string, SpatialField> getSpatialField, CancellationToken token)
{
int?baseDocId = null;
if (string.IsNullOrWhiteSpace(query.DocumentId) == false || query.MapGroupFields.Count > 0)
{
var documentQuery = new BooleanQuery();
if (string.IsNullOrWhiteSpace(query.DocumentId) == false)
{
documentQuery.Add(new TermQuery(new Term(Constants.Documents.Indexing.Fields.DocumentIdFieldName, query.DocumentId.ToLowerInvariant())), Occur.MUST);
}
foreach (var key in query.MapGroupFields.Keys)
{
documentQuery.Add(new TermQuery(new Term(key, query.MapGroupFields[key])), Occur.MUST);
}
var td = _searcher.Search(documentQuery, 1, _state);
// get the current Lucene docid for the given RavenDB doc ID
if (td.ScoreDocs.Length == 0)
{
throw new InvalidOperationException("Document " + query.DocumentId + " could not be found");
}
baseDocId = td.ScoreDocs[0].Doc;
}
var ir = _searcher.IndexReader;
var mlt = new RavenMoreLikeThis(ir, query, _state);
AssignParameters(mlt, query);
if (stopWords != null)
{
mlt.SetStopWords(stopWords);
}
string[] fieldNames;
if (query.Fields != null && query.Fields.Length > 0)
{
fieldNames = query.Fields;
}
else
{
fieldNames = ir.GetFieldNames(IndexReader.FieldOption.INDEXED)
.Where(x => x != Constants.Documents.Indexing.Fields.DocumentIdFieldName && x != Constants.Documents.Indexing.Fields.ReduceKeyHashFieldName)
.ToArray();
}
mlt.SetFieldNames(fieldNames);
mlt.Analyzer = _analyzer;
var pageSize = GetPageSize(_searcher, query.PageSize);
Query mltQuery;
if (baseDocId.HasValue)
{
mltQuery = mlt.Like(baseDocId.Value);
}
else
{
using (var blittableJson = ParseJsonStringIntoBlittable(query.Document, context))
mltQuery = mlt.Like(blittableJson);
}
var tsdc = TopScoreDocCollector.Create(pageSize, true);
if (query.Metadata.WhereFields.Count > 0)
{
var additionalQuery = QueryBuilder.BuildQuery(context, query.Metadata, query.Metadata.Query.Where, null, _analyzer, getSpatialField);
mltQuery = new BooleanQuery
{
{ mltQuery, Occur.MUST },
{ additionalQuery, Occur.MUST }
};
}
_searcher.Search(mltQuery, tsdc, _state);
var hits = tsdc.TopDocs().ScoreDocs;
var ids = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
var retriever = createRetriever(null);
foreach (var hit in hits)
{
if (hit.Doc == baseDocId)
{
continue;
}
var doc = _searcher.Doc(hit.Doc, _state);
var id = doc.Get(Constants.Documents.Indexing.Fields.DocumentIdFieldName, _state) ?? doc.Get(Constants.Documents.Indexing.Fields.ReduceKeyHashFieldName, _state);
if (id == null)
{
continue;
}
if (ids.Add(id) == false)
{
continue;
}
yield return(retriever.Get(doc, hit.Score, _state));
}
}
public async Task QueriesRunning()
{
using (var store = GetDocumentStore())
{
IndexQuery q;
using (var session = store.OpenSession())
{
var people = session.Query <Person>()
.Where(x => x.Name == "John")
.ToList(); // create index
q = session.Advanced.DocumentQuery <Person>()
.WhereEquals(x => x.Name, "John")
.Take(20)
.GetIndexQuery();
}
using (var context = JsonOperationContext.ShortTermSingleUse())
{
var query1 = new IndexQueryServerSide(q.Query, context.ReadObject(new DynamicJsonValue
{
["p0"] = q.QueryParameters["p0"]
}, "query/parameters"))
{
Start = q.Start,
CutoffEtag = q.CutoffEtag,
ExplainScores = q.ExplainScores,
PageSize = q.PageSize,
ShowTimings = q.ShowTimings,
SkipDuplicateChecking = q.SkipDuplicateChecking,
WaitForNonStaleResults = q.WaitForNonStaleResults,
WaitForNonStaleResultsTimeout = q.WaitForNonStaleResultsTimeout
};
var query2 = new MoreLikeThisQueryServerSide
{
DocumentId = "docs/1"
};
var query3 = new FacetQuery
{
FacetSetupDoc = "setup/1"
};
var database = await Server
.ServerStore
.DatabasesLandlord
.TryGetOrCreateResourceStore(new StringSegment(store.Database));
var index = database.IndexStore.GetIndexes().First();
var now = SystemTime.UtcNow;
index.CurrentlyRunningQueries.TryAdd(new ExecutingQueryInfo(now, query1, 10, OperationCancelToken.None));
index.CurrentlyRunningQueries.TryAdd(new ExecutingQueryInfo(now, query2, 11, OperationCancelToken.None));
index.CurrentlyRunningQueries.TryAdd(new ExecutingQueryInfo(now, query3, 12, OperationCancelToken.None));
var conventions = new DocumentConventions();
using (var commands = store.Commands())
{
var json = commands.RawGetJson <BlittableJsonReaderObject>("/debug/queries/running");
Assert.True(json.TryGet(index.Name, out BlittableJsonReaderArray array));
Assert.Equal(3, array.Length);
foreach (BlittableJsonReaderObject info in array)
{
int queryId;
Assert.True(info.TryGet(nameof(ExecutingQueryInfo.QueryId), out queryId));
string duration;
Assert.True(info.TryGet(nameof(ExecutingQueryInfo.Duration), out duration));
Assert.NotNull(duration);
string startTimeAsString;
Assert.True(info.TryGet(nameof(ExecutingQueryInfo.StartTime), out startTimeAsString));
Assert.Equal(now, DateTime.Parse(startTimeAsString).ToUniversalTime());
object token;
Assert.False(info.TryGetMember(nameof(ExecutingQueryInfo.Token), out token));
Assert.Null(token);
if (queryId == 10)
{
BlittableJsonReaderObject queryInfo;
Assert.True(info.TryGet(nameof(ExecutingQueryInfo.QueryInfo), out queryInfo));
var query = (IndexQuery)conventions.DeserializeEntityFromBlittable(typeof(IndexQuery), queryInfo);
Assert.True(q.Equals(query));
continue;
}
if (queryId == 11)
{
BlittableJsonReaderObject queryInfo;
Assert.True(info.TryGet(nameof(ExecutingQueryInfo.QueryInfo), out queryInfo));
var query = (MoreLikeThisQuery)conventions.DeserializeEntityFromBlittable(typeof(MoreLikeThisQuery), queryInfo);
Assert.Equal(query2.DocumentId, query.DocumentId);
continue;
}
if (queryId == 12)
{
BlittableJsonReaderObject queryInfo;
Assert.True(info.TryGet(nameof(ExecutingQueryInfo.QueryInfo), out queryInfo));
var query = (FacetQuery)conventions.DeserializeEntityFromBlittable(typeof(FacetQuery), queryInfo);
Assert.Equal(query3.FacetSetupDoc, query.FacetSetupDoc);
continue;
}
throw new NotSupportedException("Should not happen.");
}
}
}
}
}
private MoreLikeThisQueryServerSide GetMoreLikeThisQuery(JsonOperationContext context)
{
var json = context.ReadForMemory(RequestBodyStream(), "morelikethis/query");
return(MoreLikeThisQueryServerSide.Create(json));
}
public MoreLikeThisQueryResultServerSide ExecuteMoreLikeThisQuery(string indexName, MoreLikeThisQueryServerSide query, DocumentsOperationContext context, long?existingResultEtag, OperationCancelToken token)
{
if (query == null)
{
throw new ArgumentNullException(nameof(query));
}
if (string.IsNullOrEmpty(query.DocumentId) && query.MapGroupFields.Count == 0)
{
throw new InvalidOperationException("The document id or map group fields are mandatory");
}
var index = GetIndex(indexName);
var etag = index.GetIndexEtag();
if (etag == existingResultEtag)
{
return(MoreLikeThisQueryResultServerSide.NotModifiedResult);
}
context.OpenReadTransaction();
return(index.MoreLikeThisQuery(query, context, token));
}
public IEnumerable <Document> MoreLikeThis(MoreLikeThisQueryServerSide query, HashSet <string> stopWords, Func <string[], IQueryResultRetriever> createRetriever, CancellationToken token)
{
var documentQuery = new BooleanQuery();
if (string.IsNullOrWhiteSpace(query.DocumentId) == false)
{
documentQuery.Add(new TermQuery(new Term(Constants.Indexing.Fields.DocumentIdFieldName, query.DocumentId.ToLowerInvariant())), Occur.MUST);
}
foreach (var key in query.MapGroupFields.Keys)
{
documentQuery.Add(new TermQuery(new Term(key, query.MapGroupFields[key])), Occur.MUST);
}
var td = _searcher.Search(documentQuery, 1);
// get the current Lucene docid for the given RavenDB doc ID
if (td.ScoreDocs.Length == 0)
{
throw new InvalidOperationException("Document " + query.DocumentId + " could not be found");
}
var ir = _searcher.IndexReader;
var mlt = new RavenMoreLikeThis(ir, query);
if (stopWords != null)
{
mlt.SetStopWords(stopWords);
}
var fieldNames = query.Fields ?? ir.GetFieldNames(IndexReader.FieldOption.INDEXED)
.Where(x => x != Constants.Indexing.Fields.DocumentIdFieldName && x != Constants.Indexing.Fields.ReduceKeyFieldName)
.ToArray();
mlt.SetFieldNames(fieldNames);
mlt.Analyzer = _analyzer;
var mltQuery = mlt.Like(td.ScoreDocs[0].Doc);
var tsdc = TopScoreDocCollector.Create(query.PageSize, true);
if (string.IsNullOrWhiteSpace(query.AdditionalQuery) == false)
{
var additionalQuery = QueryBuilder.BuildQuery(query.AdditionalQuery, _analyzer);
mltQuery = new BooleanQuery
{
{ mltQuery, Occur.MUST },
{ additionalQuery, Occur.MUST },
};
}
_searcher.Search(mltQuery, tsdc);
var hits = tsdc.TopDocs().ScoreDocs;
var baseDocId = td.ScoreDocs[0].Doc;
var ids = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
var fieldsToFetch = string.IsNullOrWhiteSpace(query.DocumentId)
? _searcher.Doc(baseDocId).GetFields().Cast <AbstractField>().Select(x => x.Name).Distinct().ToArray()
: null;
var retriever = createRetriever(fieldsToFetch);
foreach (var hit in hits)
{
if (hit.Doc == baseDocId)
{
continue;
}
var doc = _searcher.Doc(hit.Doc);
var id = doc.Get(Constants.Indexing.Fields.DocumentIdFieldName) ?? doc.Get(Constants.Indexing.Fields.ReduceKeyFieldName);
if (id == null)
{
continue;
}
if (ids.Add(id) == false)
{
continue;
}
yield return(retriever.Get(doc, hit.Score));
}
}
public async Task QueriesRunning()
{
using (var store = GetDocumentStore())
{
IndexQuery q;
using (var session = store.OpenSession())
{
var people = session.Query <Person>()
.Where(x => x.Name == "John")
.ToList(); // create index
q = session.Advanced.DocumentQuery <Person>()
.WhereEquals(x => x.Name, "John")
.Take(20)
.GetIndexQuery(isAsync: false);
}
var query1 = new IndexQueryServerSide
{
Transformer = q.Transformer,
Start = q.Start,
AllowMultipleIndexEntriesForSameDocumentToResultTransformer = q.AllowMultipleIndexEntriesForSameDocumentToResultTransformer,
CutoffEtag = q.CutoffEtag,
DebugOptionGetIndexEntries = q.DebugOptionGetIndexEntries,
DefaultField = q.DefaultField,
DefaultOperator = q.DefaultOperator,
DisableCaching = q.DisableCaching,
DynamicMapReduceFields = q.DynamicMapReduceFields,
ExplainScores = q.ExplainScores,
FieldsToFetch = q.FieldsToFetch,
HighlightedFields = q.HighlightedFields,
HighlighterKeyName = q.HighlighterKeyName,
HighlighterPostTags = q.HighlighterPostTags,
HighlighterPreTags = q.HighlighterPreTags,
Includes = q.Includes,
IsDistinct = q.IsDistinct,
PageSize = q.PageSize,
Query = q.Query,
ShowTimings = q.ShowTimings,
SkipDuplicateChecking = q.SkipDuplicateChecking,
SortedFields = q.SortedFields,
WaitForNonStaleResults = q.WaitForNonStaleResults,
WaitForNonStaleResultsAsOfNow = q.WaitForNonStaleResultsAsOfNow,
WaitForNonStaleResultsTimeout = q.WaitForNonStaleResultsTimeout
};
var query2 = new MoreLikeThisQueryServerSide
{
DocumentId = "docs/1"
};
var query3 = new FacetQuery
{
FacetSetupDoc = "setup/1"
};
var database = await Server
.ServerStore
.DatabasesLandlord
.TryGetOrCreateResourceStore(new StringSegment(store.DefaultDatabase, 0));
var index = database.IndexStore.GetIndex(1);
var now = SystemTime.UtcNow;
index.CurrentlyRunningQueries.TryAdd(new ExecutingQueryInfo(now, query1, 10, OperationCancelToken.None));
index.CurrentlyRunningQueries.TryAdd(new ExecutingQueryInfo(now, query2, 11, OperationCancelToken.None));
index.CurrentlyRunningQueries.TryAdd(new ExecutingQueryInfo(now, query3, 12, OperationCancelToken.None));
string jsonString;
using (var client = new HttpClient())
{
jsonString = await client.GetStringAsync($"{store.Url.ForDatabase(store.DefaultDatabase)}/debug/queries/running");
}
var json = RavenJObject.Parse(jsonString);
var array = json.Value <RavenJArray>(index.Name);
Assert.Equal(3, array.Length);
foreach (var info in array)
{
var queryId = info.Value <int>(nameof(ExecutingQueryInfo.QueryId));
Assert.NotNull(array[0].Value <string>(nameof(ExecutingQueryInfo.Duration)));
Assert.Equal(now, info.Value <DateTime>(nameof(ExecutingQueryInfo.StartTime)));
Assert.Null(info.Value <OperationCancelToken>(nameof(ExecutingQueryInfo.Token)));
if (queryId == 10)
{
var query = info
.Value <RavenJObject>(nameof(ExecutingQueryInfo.QueryInfo))
.JsonDeserialization <IndexQuery>();
Assert.True(q.Equals(query));
continue;
}
if (queryId == 11)
{
var query = info
.Value <RavenJObject>(nameof(ExecutingQueryInfo.QueryInfo))
.JsonDeserialization <MoreLikeThisQuery>();
Assert.Equal(query2.DocumentId, query.DocumentId);
continue;
}
if (queryId == 12)
{
var query = info
.Value <RavenJObject>(nameof(ExecutingQueryInfo.QueryInfo))
.JsonDeserialization <FacetQuery>();
Assert.Equal(query3.FacetSetupDoc, query.FacetSetupDoc);
continue;
}
throw new NotSupportedException("Should not happen.");
}
}
}
