private Etag GetFacetsEtag(string index, byte[] additionalEtagBytes) { var bytes = Database.Indexes.GetIndexEtag(index, null).ToByteArray().Concat(additionalEtagBytes).ToArray(); return(Etag.FromHash(Hashing.Metro128.Calculate(bytes))); }
public MoreLikeThisQueryResult ExecuteMoreLikeThisQuery(MoreLikeThisQuery query, int pageSize = 25) { if (query == null) { throw new ArgumentNullException("query"); } var index = database.IndexStorage.GetIndexInstance(query.IndexName); if (index == null) { throw new InvalidOperationException("The index " + query.IndexName + " cannot be found"); } if (string.IsNullOrEmpty(query.DocumentId) && query.MapGroupFields.Count == 0) { throw new InvalidOperationException("The document id or map group fields are mandatory"); } IndexSearcher searcher; using (database.IndexStorage.GetCurrentIndexSearcher(index.indexId, out searcher)) { var documentQuery = new BooleanQuery(); if (string.IsNullOrEmpty(query.DocumentId) == false) { documentQuery.Add(new TermQuery(new Term(Constants.DocumentIdFieldName, query.DocumentId.ToLowerInvariant())), Occur.MUST); } foreach (string key in query.MapGroupFields.Keys) { documentQuery.Add(new TermQuery(new Term(key, query.MapGroupFields[key])), Occur.MUST); } var td = searcher.Search(documentQuery, 1); // get the current Lucene docid for the given RavenDB doc ID if (td.ScoreDocs.Length == 0) { throw new InvalidOperationException("Document " + query.DocumentId + " could not be found"); } var ir = searcher.IndexReader; var mlt = new RavenMoreLikeThis(ir); AssignParameters(mlt, query); if (string.IsNullOrWhiteSpace(query.StopWordsDocumentId) == false) { var stopWordsDoc = database.Documents.Get(query.StopWordsDocumentId); if (stopWordsDoc == null) { throw new InvalidOperationException("Stop words document " + query.StopWordsDocumentId + " could not be found"); } var stopWordsSetup = stopWordsDoc.DataAsJson.JsonDeserialization <StopWordsSetup>(); if (stopWordsSetup.StopWords != null) { var stopWords = stopWordsSetup.StopWords; var ht = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase); foreach (var stopWord in stopWords) { ht.Add(stopWord); } mlt.SetStopWords(ht); } } var fieldNames = query.Fields ?? GetFieldNames(ir); mlt.SetFieldNames(fieldNames); var toDispose = new List <Action>(); RavenPerFieldAnalyzerWrapper perFieldAnalyzerWrapper = null; try { perFieldAnalyzerWrapper = index.CreateAnalyzer(new LowerCaseKeywordAnalyzer(), toDispose, true); mlt.Analyzer = perFieldAnalyzerWrapper; var mltQuery = mlt.Like(td.ScoreDocs[0].Doc); var tsdc = TopScoreDocCollector.Create(pageSize, true); if (string.IsNullOrWhiteSpace(query.AdditionalQuery) == false) { var additionalQuery = QueryBuilder.BuildQuery(query.AdditionalQuery, perFieldAnalyzerWrapper); mltQuery = new BooleanQuery { { mltQuery, Occur.MUST }, { additionalQuery, Occur.MUST }, }; } searcher.Search(mltQuery, tsdc); var hits = tsdc.TopDocs().ScoreDocs; var jsonDocuments = GetJsonDocuments(query, searcher, index, query.IndexName, hits, td.ScoreDocs[0].Doc); var result = new MultiLoadResult(); var includedEtags = new List <byte>(jsonDocuments.SelectMany(x => x.Etag.ToByteArray())); includedEtags.AddRange(database.Indexes.GetIndexEtag(query.IndexName, null).ToByteArray()); var loadedIds = new HashSet <string>(jsonDocuments.Select(x => x.Key)); var addIncludesCommand = new AddIncludesCommand(database, (etag, includedDoc) => { includedEtags.AddRange(etag.ToByteArray()); result.Includes.Add(includedDoc); }, query.Includes ?? new string[0], loadedIds); idsToLoad = new HashSet <string>(); database.TransactionalStorage.Batch(actions => { documentRetriever = new DocumentRetriever(database.Configuration, actions, database.ReadTriggers, query.TransformerParameters, idsToLoad); using (new CurrentTransformationScope(database, documentRetriever)) { foreach (var document in ProcessResults(query, jsonDocuments, database.WorkContext.CancellationToken)) { result.Results.Add(document); addIncludesCommand.Execute(document); } } }); addIncludesCommand.AlsoInclude(idsToLoad); Etag computedEtag = Etag.FromHash(Hashing.Metro128.Calculate(includedEtags.ToArray())); return(new MoreLikeThisQueryResult { Etag = computedEtag, Result = result, }); } finally { if (perFieldAnalyzerWrapper != null) { perFieldAnalyzerWrapper.Close(); } foreach (var action in toDispose) { action(); } } } }
public Etag GetIndexEtag(string indexName, Etag previousEtag, string resultTransformer = null) { Etag lastDocEtag = Etag.Empty; Etag lastIndexedEtag = null; Etag lastReducedEtag = null; bool isStale = false; int touchCount = 0; TransactionalStorage.Batch(accessor => { var indexInstance = Database.IndexStorage.GetIndexInstance(indexName); if (indexInstance == null) { return; } isStale = (indexInstance.IsMapIndexingInProgress) || accessor.Staleness.IsIndexStale(indexInstance.indexId, null, null); lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag(); var indexStats = accessor.Indexing.GetIndexStats(indexInstance.indexId); if (indexStats != null) { lastReducedEtag = indexStats.LastReducedEtag; lastIndexedEtag = indexStats.LastIndexedEtag; } touchCount = accessor.Staleness.GetIndexTouchCount(indexInstance.indexId); }); var indexDefinition = GetIndexDefinition(indexName); if (indexDefinition == null) { return(Etag.Empty); // this ensures that we will get the normal reaction of IndexNotFound later on. } var list = new List <byte>(); list.AddRange(indexDefinition.GetIndexHash()); list.AddRange(Encoding.Unicode.GetBytes(indexName)); if (string.IsNullOrWhiteSpace(resultTransformer) == false) { var abstractTransformer = IndexDefinitionStorage.GetTransformer(resultTransformer); if (abstractTransformer == null) { throw new InvalidOperationException("The result transformer: " + resultTransformer + " was not found"); } list.AddRange(abstractTransformer.GetHashCodeBytes()); } list.AddRange(lastDocEtag.ToByteArray()); list.AddRange(BitConverter.GetBytes(touchCount)); list.AddRange(BitConverter.GetBytes(isStale)); if (lastReducedEtag != null) { list.AddRange(lastReducedEtag.ToByteArray()); } if (lastIndexedEtag != null) { list.AddRange(lastIndexedEtag.ToByteArray()); } list.AddRange(BitConverter.GetBytes(UuidGenerator.LastDocumentTransactionEtag)); var indexEtag = Etag.FromHash(Hashing.Metro128.Calculate(list.ToArray())); if (previousEtag != null && previousEtag != indexEtag) { // the index changed between the time when we got it and the time // we actually call this, we need to return something random so that // the next time we won't get 304 return(Etag.InvalidEtag); } return(indexEtag); }
private async Task <HttpResponseMessage> GetQueriesResponse(bool isGet) { RavenJArray itemsToLoad; if (isGet == false) { try { itemsToLoad = await ReadJsonArrayAsync().ConfigureAwait(false); } catch (InvalidOperationException e) { if (Log.IsDebugEnabled) { Log.DebugException("Failed to deserialize query request.", e); } return(GetMessageWithObject(new { Message = "Could not understand json, please check its validity." }, (HttpStatusCode)422)); //http code 422 - Unprocessable entity } catch (InvalidDataException e) { if (Log.IsDebugEnabled) { Log.DebugException("Failed to deserialize query request.", e); } return(GetMessageWithObject(new { e.Message }, (HttpStatusCode)422)); //http code 422 - Unprocessable entity } AddRequestTraceInfo(sb => { foreach (var item in itemsToLoad) { sb.Append("\t").Append(item).AppendLine(); } }); } else { itemsToLoad = new RavenJArray(GetQueryStringValues("id").Cast <object>()); } var result = new MultiLoadResult(); var loadedIds = new HashSet <string>(); var includedIds = new HashSet <string>(); var includes = GetQueryStringValues("include") ?? new string[0]; var transformer = GetQueryStringValue("transformer") ?? GetQueryStringValue("resultTransformer"); var transformerParameters = this.ExtractTransformerParameters(); var includedEtags = new List <byte>(); if (string.IsNullOrEmpty(transformer) == false) { var transformerDef = Database.IndexDefinitionStorage.GetTransformer(transformer); if (transformerDef == null) { return(GetMessageWithObject(new { Error = "No such transformer: " + transformer }, HttpStatusCode.BadRequest)); } includedEtags.AddRange(transformerDef.GetHashCodeBytes()); } Database.TransactionalStorage.Batch(actions => { foreach (RavenJToken item in itemsToLoad) { var value = item.Value <string>(); if (loadedIds.Add(value) == false) { continue; } var documentByKey = string.IsNullOrEmpty(transformer) ? Database.Documents.Get(value) : Database.Documents.GetWithTransformer(value, transformer, transformerParameters, out includedIds); if (documentByKey == null) { if (ClientIsV3OrHigher(Request)) { result.Results.Add(null); } continue; } result.Results.Add(documentByKey.ToJson()); if (documentByKey.Etag != null) { includedEtags.AddRange(documentByKey.Etag.ToByteArray()); } // TODO: Revise this. includedEtags.Add((false) ? (byte)0 : (byte)1); } var addIncludesCommand = new AddIncludesCommand(Database, (etag, includedDoc) => { includedEtags.AddRange(etag.ToByteArray()); result.Includes.Add(includedDoc); }, includes, loadedIds); foreach (var item in result.Results.Where(item => item != null)) { addIncludesCommand.Execute(item); } }); foreach (var includedId in includedIds) { var doc = Database.Documents.Get(includedId); if (doc == null) { continue; } includedEtags.AddRange(doc.Etag.ToByteArray()); result.Includes.Add(doc.ToJson()); } var computeHash = Hashing.Metro128.Calculate(includedEtags.ToArray()); Etag computedEtag = Etag.FromHash(computeHash); if (MatchEtag(computedEtag)) { return(GetEmptyMessage(HttpStatusCode.NotModified)); } var msg = GetMessageWithObject(result); WriteETag(computedEtag, msg); AddRequestTraceInfo(sb => sb.Append("Results count: {0}, includes count: {1}", result.Results.Count, result.Includes.Count).AppendLine()); return(msg); }