private Etag GetFacetsEtag(string index, byte[] additionalEtagBytes)
{
var bytes = Database.Indexes.GetIndexEtag(index, null).ToByteArray().Concat(additionalEtagBytes).ToArray();
return(Etag.FromHash(Hashing.Metro128.Calculate(bytes)));
}
public MoreLikeThisQueryResult ExecuteMoreLikeThisQuery(MoreLikeThisQuery query, int pageSize = 25)
{
if (query == null)
{
throw new ArgumentNullException("query");
}
var index = database.IndexStorage.GetIndexInstance(query.IndexName);
if (index == null)
{
throw new InvalidOperationException("The index " + query.IndexName + " cannot be found");
}
if (string.IsNullOrEmpty(query.DocumentId) && query.MapGroupFields.Count == 0)
{
throw new InvalidOperationException("The document id or map group fields are mandatory");
}
IndexSearcher searcher;
using (database.IndexStorage.GetCurrentIndexSearcher(index.indexId, out searcher))
{
var documentQuery = new BooleanQuery();
if (string.IsNullOrEmpty(query.DocumentId) == false)
{
documentQuery.Add(new TermQuery(new Term(Constants.DocumentIdFieldName, query.DocumentId.ToLowerInvariant())), Occur.MUST);
}
foreach (string key in query.MapGroupFields.Keys)
{
documentQuery.Add(new TermQuery(new Term(key, query.MapGroupFields[key])), Occur.MUST);
}
var td = searcher.Search(documentQuery, 1);
// get the current Lucene docid for the given RavenDB doc ID
if (td.ScoreDocs.Length == 0)
{
throw new InvalidOperationException("Document " + query.DocumentId + " could not be found");
}
var ir = searcher.IndexReader;
var mlt = new RavenMoreLikeThis(ir);
AssignParameters(mlt, query);
if (string.IsNullOrWhiteSpace(query.StopWordsDocumentId) == false)
{
var stopWordsDoc = database.Documents.Get(query.StopWordsDocumentId);
if (stopWordsDoc == null)
{
throw new InvalidOperationException("Stop words document " + query.StopWordsDocumentId + " could not be found");
}
var stopWordsSetup = stopWordsDoc.DataAsJson.JsonDeserialization <StopWordsSetup>();
if (stopWordsSetup.StopWords != null)
{
var stopWords = stopWordsSetup.StopWords;
var ht = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase);
foreach (var stopWord in stopWords)
{
ht.Add(stopWord);
}
mlt.SetStopWords(ht);
}
}
var fieldNames = query.Fields ?? GetFieldNames(ir);
mlt.SetFieldNames(fieldNames);
var toDispose = new List <Action>();
RavenPerFieldAnalyzerWrapper perFieldAnalyzerWrapper = null;
try
{
perFieldAnalyzerWrapper = index.CreateAnalyzer(new LowerCaseKeywordAnalyzer(), toDispose, true);
mlt.Analyzer = perFieldAnalyzerWrapper;
var mltQuery = mlt.Like(td.ScoreDocs[0].Doc);
var tsdc = TopScoreDocCollector.Create(pageSize, true);
if (string.IsNullOrWhiteSpace(query.AdditionalQuery) == false)
{
var additionalQuery = QueryBuilder.BuildQuery(query.AdditionalQuery, perFieldAnalyzerWrapper);
mltQuery = new BooleanQuery
{
{ mltQuery, Occur.MUST },
{ additionalQuery, Occur.MUST },
};
}
searcher.Search(mltQuery, tsdc);
var hits = tsdc.TopDocs().ScoreDocs;
var jsonDocuments = GetJsonDocuments(query, searcher, index, query.IndexName, hits, td.ScoreDocs[0].Doc);
var result = new MultiLoadResult();
var includedEtags = new List <byte>(jsonDocuments.SelectMany(x => x.Etag.ToByteArray()));
includedEtags.AddRange(database.Indexes.GetIndexEtag(query.IndexName, null).ToByteArray());
var loadedIds = new HashSet <string>(jsonDocuments.Select(x => x.Key));
var addIncludesCommand = new AddIncludesCommand(database, (etag, includedDoc) =>
{
includedEtags.AddRange(etag.ToByteArray());
result.Includes.Add(includedDoc);
}, query.Includes ?? new string[0], loadedIds);
idsToLoad = new HashSet <string>();
database.TransactionalStorage.Batch(actions =>
{
documentRetriever = new DocumentRetriever(database.Configuration, actions, database.ReadTriggers, query.TransformerParameters, idsToLoad);
using (new CurrentTransformationScope(database, documentRetriever))
{
foreach (var document in ProcessResults(query, jsonDocuments, database.WorkContext.CancellationToken))
{
result.Results.Add(document);
addIncludesCommand.Execute(document);
}
}
});
addIncludesCommand.AlsoInclude(idsToLoad);
Etag computedEtag = Etag.FromHash(Hashing.Metro128.Calculate(includedEtags.ToArray()));
return(new MoreLikeThisQueryResult
{
Etag = computedEtag,
Result = result,
});
}
finally
{
if (perFieldAnalyzerWrapper != null)
{
perFieldAnalyzerWrapper.Close();
}
foreach (var action in toDispose)
{
action();
}
}
}
}
public Etag GetIndexEtag(string indexName, Etag previousEtag, string resultTransformer = null)
{
Etag lastDocEtag = Etag.Empty;
Etag lastIndexedEtag = null;
Etag lastReducedEtag = null;
bool isStale = false;
int touchCount = 0;
TransactionalStorage.Batch(accessor =>
{
var indexInstance = Database.IndexStorage.GetIndexInstance(indexName);
if (indexInstance == null)
{
return;
}
isStale = (indexInstance.IsMapIndexingInProgress) ||
accessor.Staleness.IsIndexStale(indexInstance.indexId, null, null);
lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag();
var indexStats = accessor.Indexing.GetIndexStats(indexInstance.indexId);
if (indexStats != null)
{
lastReducedEtag = indexStats.LastReducedEtag;
lastIndexedEtag = indexStats.LastIndexedEtag;
}
touchCount = accessor.Staleness.GetIndexTouchCount(indexInstance.indexId);
});
var indexDefinition = GetIndexDefinition(indexName);
if (indexDefinition == null)
{
return(Etag.Empty); // this ensures that we will get the normal reaction of IndexNotFound later on.
}
var list = new List <byte>();
list.AddRange(indexDefinition.GetIndexHash());
list.AddRange(Encoding.Unicode.GetBytes(indexName));
if (string.IsNullOrWhiteSpace(resultTransformer) == false)
{
var abstractTransformer = IndexDefinitionStorage.GetTransformer(resultTransformer);
if (abstractTransformer == null)
{
throw new InvalidOperationException("The result transformer: " + resultTransformer + " was not found");
}
list.AddRange(abstractTransformer.GetHashCodeBytes());
}
list.AddRange(lastDocEtag.ToByteArray());
list.AddRange(BitConverter.GetBytes(touchCount));
list.AddRange(BitConverter.GetBytes(isStale));
if (lastReducedEtag != null)
{
list.AddRange(lastReducedEtag.ToByteArray());
}
if (lastIndexedEtag != null)
{
list.AddRange(lastIndexedEtag.ToByteArray());
}
list.AddRange(BitConverter.GetBytes(UuidGenerator.LastDocumentTransactionEtag));
var indexEtag = Etag.FromHash(Hashing.Metro128.Calculate(list.ToArray()));
if (previousEtag != null && previousEtag != indexEtag)
{
// the index changed between the time when we got it and the time
// we actually call this, we need to return something random so that
// the next time we won't get 304
return(Etag.InvalidEtag);
}
return(indexEtag);
}
private async Task <HttpResponseMessage> GetQueriesResponse(bool isGet)
{
RavenJArray itemsToLoad;
if (isGet == false)
{
try
{
itemsToLoad = await ReadJsonArrayAsync().ConfigureAwait(false);
}
catch (InvalidOperationException e)
{
if (Log.IsDebugEnabled)
{
Log.DebugException("Failed to deserialize query request.", e);
}
return(GetMessageWithObject(new
{
Message = "Could not understand json, please check its validity."
}, (HttpStatusCode)422)); //http code 422 - Unprocessable entity
}
catch (InvalidDataException e)
{
if (Log.IsDebugEnabled)
{
Log.DebugException("Failed to deserialize query request.", e);
}
return(GetMessageWithObject(new
{
e.Message
}, (HttpStatusCode)422)); //http code 422 - Unprocessable entity
}
AddRequestTraceInfo(sb =>
{
foreach (var item in itemsToLoad)
{
sb.Append("\t").Append(item).AppendLine();
}
});
}
else
{
itemsToLoad = new RavenJArray(GetQueryStringValues("id").Cast <object>());
}
var result = new MultiLoadResult();
var loadedIds = new HashSet <string>();
var includedIds = new HashSet <string>();
var includes = GetQueryStringValues("include") ?? new string[0];
var transformer = GetQueryStringValue("transformer") ?? GetQueryStringValue("resultTransformer");
var transformerParameters = this.ExtractTransformerParameters();
var includedEtags = new List <byte>();
if (string.IsNullOrEmpty(transformer) == false)
{
var transformerDef = Database.IndexDefinitionStorage.GetTransformer(transformer);
if (transformerDef == null)
{
return(GetMessageWithObject(new { Error = "No such transformer: " + transformer }, HttpStatusCode.BadRequest));
}
includedEtags.AddRange(transformerDef.GetHashCodeBytes());
}
Database.TransactionalStorage.Batch(actions =>
{
foreach (RavenJToken item in itemsToLoad)
{
var value = item.Value <string>();
if (loadedIds.Add(value) == false)
{
continue;
}
var documentByKey = string.IsNullOrEmpty(transformer)
? Database.Documents.Get(value)
: Database.Documents.GetWithTransformer(value, transformer, transformerParameters, out includedIds);
if (documentByKey == null)
{
if (ClientIsV3OrHigher(Request))
{
result.Results.Add(null);
}
continue;
}
result.Results.Add(documentByKey.ToJson());
if (documentByKey.Etag != null)
{
includedEtags.AddRange(documentByKey.Etag.ToByteArray());
}
// TODO: Revise this.
includedEtags.Add((false) ? (byte)0 : (byte)1);
}
var addIncludesCommand = new AddIncludesCommand(Database, (etag, includedDoc) =>
{
includedEtags.AddRange(etag.ToByteArray());
result.Includes.Add(includedDoc);
}, includes, loadedIds);
foreach (var item in result.Results.Where(item => item != null))
{
addIncludesCommand.Execute(item);
}
});
foreach (var includedId in includedIds)
{
var doc = Database.Documents.Get(includedId);
if (doc == null)
{
continue;
}
includedEtags.AddRange(doc.Etag.ToByteArray());
result.Includes.Add(doc.ToJson());
}
var computeHash = Hashing.Metro128.Calculate(includedEtags.ToArray());
Etag computedEtag = Etag.FromHash(computeHash);
if (MatchEtag(computedEtag))
{
return(GetEmptyMessage(HttpStatusCode.NotModified));
}
var msg = GetMessageWithObject(result);
WriteETag(computedEtag, msg);
AddRequestTraceInfo(sb => sb.Append("Results count: {0}, includes count: {1}", result.Results.Count, result.Includes.Count).AppendLine());
return(msg);
}
