public CurrentTransformationScope(DocumentDatabase database, DocumentRetriever documentRetriever)
{
this.database = database;
retriever = documentRetriever;
old = current;
current = this;
}
public bool IndexDocuments(IStorageActionsAccessor actions, string index, Guid etagToIndexFrom)
{
log.DebugFormat("Indexing documents for {0}, etag to index from: {1}", index, etagToIndexFrom);
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
return false; // index was deleted, probably
var jsonDocs = actions.Documents.GetDocumentsAfter(etagToIndexFrom)
.Where(x => x != null)
.Take(10000) // ensure that we won't go overboard with reading and blow up with OOM
.ToArray();
if(jsonDocs.Length == 0)
return false;
var dateTime = jsonDocs.Select(x=>x.LastModified).Min();
var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
try
{
log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index);
context.IndexStorage.Index(index, viewGenerator,
jsonDocs
.Select(doc => documentRetriever.ProcessReadVetoes(doc, null, ReadOperation.Index))
.Where(doc => doc != null)
.Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime);
return true;
}
catch (Exception e)
{
log.WarnFormat(e, "Failed to index documents for index: {0}", index);
return false;
}
finally
{
// whatever we succeeded in indexing or not, we have to update this
// because otherwise we keep trying to re-index failed documents
var last = jsonDocs.Last();
actions.Indexing.UpdateLastIndexed(index, last.Etag, last.LastModified);
}
}
public void Init()
{
highlightings = new Dictionary<string, Dictionary<string, string[]>>();
scoreExplanations = new Dictionary<string, string>();
Func<IndexQueryResult, object> tryRecordHighlightingAndScoreExplanation = queryResult =>
{
if (queryResult.Highligtings != null && (queryResult.Key != null || queryResult.HighlighterKey != null))
highlightings.Add(queryResult.Key ?? queryResult.HighlighterKey, queryResult.Highligtings);
if ((queryResult.Key != null || queryResult.ReduceVal != null) && queryResult.ScoreExplanation != null)
scoreExplanations.Add(queryResult.Key ?? queryResult.ReduceVal, queryResult.ScoreExplanation);
return null;
};
stale = false;
indexTimestamp = Tuple.Create(DateTime.MinValue, Etag.Empty);
resultEtag = Etag.Empty;
nonAuthoritativeInformation = false;
if (string.IsNullOrEmpty(query.ResultsTransformer) == false &&
(query.FieldsToFetch == null || query.FieldsToFetch.Length == 0))
{
query.FieldsToFetch = new[] { Constants.AllFields };
}
duration = Stopwatch.StartNew();
idsToLoad = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var viewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName);
var index = database.IndexDefinitionStorage.GetIndexDefinition(indexName);
if (viewGenerator == null)
throw new IndexDoesNotExistsException("Could not find index named: " + indexName);
resultEtag = database.Indexes.GetIndexEtag(index.Name, null, query.ResultsTransformer);
stale = actions.Staleness.IsIndexStale(index.IndexId, query.Cutoff, query.CutoffEtag);
if (stale == false && query.Cutoff == null && query.CutoffEtag == null)
{
var indexInstance = database.IndexStorage.GetIndexInstance(indexName);
stale = stale || (indexInstance != null && indexInstance.IsMapIndexingInProgress);
}
if (stale &&
actions.Staleness.IsIndexStaleByTask(index.IndexId, query.Cutoff) == false &&
actions.Staleness.IsReduceStale(index.IndexId) == false)
{
var forEntityNames = viewGenerator.ForEntityNames.ToList();
var lastIndexedEtag = actions.Indexing.GetIndexStats(index.IndexId).LastIndexedEtag;
if (database.LastCollectionEtags.HasEtagGreaterThan(forEntityNames, lastIndexedEtag) == false)
stale = false;
}
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index.IndexId);
var indexFailureInformation = actions.Indexing.GetFailureRate(index.IndexId);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
docRetriever = new DocumentRetriever(database.Configuration, actions, database.ReadTriggers, database.InFlightTransactionalState, query.TransformerParameters, idsToLoad);
var fieldsToFetch = new FieldsToFetch(query,
viewGenerator.ReduceDefinition == null
? Constants.DocumentIdFieldName
: Constants.ReduceKeyFieldName);
Func<IndexQueryResult, bool> shouldIncludeInResults =
result => docRetriever.ShouldIncludeResultInQuery(result, index, fieldsToFetch, ShouldSkipDuplicateChecking);
var indexQueryResults = database.IndexStorage.Query(indexName, query, shouldIncludeInResults, fieldsToFetch, database.IndexQueryTriggers, cancellationToken);
indexQueryResults = new ActiveEnumerable<IndexQueryResult>(indexQueryResults);
if (query.ShowTimings)
indexQueryResults = new TimedEnumerable<IndexQueryResult>(indexQueryResults, timeInMilliseconds => executionTimes[QueryTimings.Lucene] = timeInMilliseconds);
var docs = from queryResult in indexQueryResults
let doc = docRetriever.RetrieveDocumentForQuery(queryResult, index, fieldsToFetch, ShouldSkipDuplicateChecking)
where doc != null
let _ = nonAuthoritativeInformation |= (doc.NonAuthoritativeInformation ?? false)
let __ = tryRecordHighlightingAndScoreExplanation(queryResult)
select doc;
transformerErrors = new List<string>();
results = database
.Queries
.GetQueryResults(query, viewGenerator, docRetriever, docs, transformerErrors,
timeInMilliseconds => executionTimes[QueryTimings.LoadDocuments] = timeInMilliseconds,
timeInMilliseconds => executionTimes[QueryTimings.TransformResults] = timeInMilliseconds,
query.ShowTimings, cancellationToken);
Header = new QueryHeaderInformation
{
Index = indexName,
IsStale = stale,
ResultEtag = resultEtag,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2,
TotalResults = query.TotalSize.Value
};
}
private IEnumerable<Tuple<IndexToWorkOn, IndexingBatch>> FilterIndexes(IList<IndexToWorkOn> indexesToWorkOn, JsonDocument[] jsonDocs)
{
var last = jsonDocs.Last();
Debug.Assert(last.Etag != null);
Debug.Assert(last.LastModified != null);
var lastEtag = last.Etag.Value;
var lastModified = last.LastModified.Value;
var lastIndexedEtag = new ComparableByteArray(lastEtag.ToByteArray());
var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
var filteredDocs =
BackgroundTaskExecuter.Instance.Apply(jsonDocs, doc =>
{
doc = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Index);
return doc == null ? null : new {Doc = doc, Json = JsonToExpando.Convert(doc.ToJson())};
});
log.Debug("After read triggers executed, {0} documents remained", filteredDocs.Count);
var results = new Tuple<IndexToWorkOn, IndexingBatch>[indexesToWorkOn.Count];
var actions = new Action<IStorageActionsAccessor>[indexesToWorkOn.Count];
BackgroundTaskExecuter.Instance.ExecuteAll(context.Configuration, scheduler, indexesToWorkOn, (indexToWorkOn, i) =>
{
var indexLastInedexEtag = new ComparableByteArray(indexToWorkOn.LastIndexedEtag.ToByteArray());
if (indexLastInedexEtag.CompareTo(lastIndexedEtag) >= 0)
return;
var indexName = indexToWorkOn.IndexName;
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName);
if (viewGenerator == null)
return; // probably deleted
var batch = new IndexingBatch();
foreach (var item in filteredDocs)
{
// did we already indexed this document in this index?
if (indexLastInedexEtag.CompareTo(new ComparableByteArray(item.Doc.Etag.Value.ToByteArray())) >= 0)
continue;
// is the Raven-Entity-Name a match for the things the index executes on?
if (viewGenerator.ForEntityNames.Count != 0 &&
viewGenerator.ForEntityNames.Contains(item.Doc.Metadata.Value<string>(Constants.RavenEntityName)) == false)
{
continue;
}
batch.Add(item.Doc, item.Json);
if (batch.DateTime == null)
batch.DateTime = item.Doc.LastModified;
else
batch.DateTime = batch.DateTime > item.Doc.LastModified
? item.Doc.LastModified
: batch.DateTime;
}
if (batch.Docs.Count == 0)
{
log.Debug("All documents have been filtered for {0}, no indexing will be performed, updating to {1}, {2}", indexName,
lastEtag, lastModified);
// we use it this way to batch all the updates together
actions[i] = accessor => accessor.Indexing.UpdateLastIndexed(indexName, lastEtag, lastModified);
return;
}
log.Debug("Going to index {0} documents in {1}", batch.Ids.Count, indexToWorkOn);
results[i] = Tuple.Create(indexToWorkOn, batch);
});
transactionalStorage.Batch(actionsAccessor =>
{
foreach (var action in actions)
{
if (action != null)
action(actionsAccessor);
}
});
return results.Where(x => x != null);
}
private void IndexDocuments(IStorageActionsAccessor actions, string index, JsonDocument[] jsonDocs)
{
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
return; // index was deleted, probably
var dateTime = jsonDocs.Min(x => x.LastModified) ?? DateTime.MinValue;
var documentRetriever = new DocumentRetriever(null, context.ReadTriggers);
try
{
log.DebugFormat("Indexing {0} documents for index: {1}", jsonDocs.Length, index);
context.IndexStorage.Index(index, viewGenerator,
jsonDocs
.Select(doc => documentRetriever
.ExecuteReadTriggers(doc, null, ReadOperation.Index))
.Where(doc => doc != null)
.Select(x => JsonToExpando.Convert(x.ToJson())), context, actions, dateTime);
}
catch (Exception e)
{
if (actions.IsWriteConflict(e))
return;
log.WarnFormat(e, "Failed to index documents for index: {0}", index);
}
}
public void GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Action<RavenJObject> addDocument)
{
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize, WorkContext.CancellationToken);
var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
if (etag != null)
etag = doc.Etag;
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever
.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
continue;
addDocument(document.ToJson());
returnedDocs = true;
}
if (returnedDocs || docCount == 0)
break;
start += docCount;
}
});
}
public JsonDocument GetWithTransformer(string key, string transformer, TransactionInformation transactionInformation, Dictionary<string, RavenJToken> transformerParameters, out HashSet<string> itemsToInclude)
{
JsonDocument result = null;
DocumentRetriever docRetriever = null;
TransactionalStorage.Batch(
actions =>
{
docRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState, transformerParameters);
using (new CurrentTransformationScope(Database, docRetriever))
{
var document = Get(key, transactionInformation);
if (document == null)
return;
if (document.Metadata.ContainsKey("Raven-Read-Veto"))
{
result = document;
return;
}
var storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
throw new InvalidOperationException("No transformer with the name: " + transformer);
var transformed = storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) })
.Select(x => JsonExtensions.ToJObject(x))
.ToArray();
if (transformed.Length == 0)
return;
result = new JsonDocument
{
Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(docRetriever.Etag),
NonAuthoritativeInformation = document.NonAuthoritativeInformation,
LastModified = document.LastModified,
DataAsJson = new RavenJObject { { "$values", new RavenJArray(transformed) } },
};
}
});
itemsToInclude = docRetriever.ItemsToInclude;
return result;
}
public CurrentTransformationScope(DocumentRetriever documentRetriever)
{
current = documentRetriever;
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize,
CancellationToken token, ref int nextStart, Action<RavenJObject> addDoc,
string transformer = null, Dictionary<string, RavenJToken> transformerParameters = null,
string skipAfter = null)
{
if (idPrefix == null)
throw new ArgumentNullException("idPrefix");
idPrefix = idPrefix.Trim();
var canPerformRapidPagination = nextStart > 0 && start == nextStart;
var actualStart = canPerformRapidPagination ? start : 0;
var addedDocs = 0;
var matchedDocs = 0;
TransactionalStorage.Batch(
actions =>
{
var docsToSkip = canPerformRapidPagination ? 0 : start;
int docCount;
AbstractTransformer storedTransformer = null;
if (transformer != null)
{
storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
do
{
docCount = 0;
var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter);
var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState, transformerParameters);
foreach (var doc in docs)
{
token.ThrowIfCancellationRequested();
docCount++;
var keyTest = doc.Key.Substring(idPrefix.Length);
if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
continue;
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc;
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
continue;
matchedDocs++;
if (matchedDocs <= docsToSkip)
continue;
token.ThrowIfCancellationRequested();
if (storedTransformer != null)
{
using (new CurrentTransformationScope(Database, documentRetriever))
{
var transformed =
storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) })
.Select(x => JsonExtensions.ToJObject(x))
.ToArray();
if (transformed.Length == 0)
{
throw new InvalidOperationException("The transform results function failed on a document: " + document.Key);
}
var transformedJsonDocument = new JsonDocument
{
Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(documentRetriever.Etag),
NonAuthoritativeInformation = document.NonAuthoritativeInformation,
LastModified = document.LastModified,
DataAsJson = new RavenJObject { { "$values", new RavenJArray(transformed) } },
};
addDoc(transformedJsonDocument.ToJson());
}
}
else
{
addDoc(document.ToJson());
}
addedDocs++;
if (addedDocs >= pageSize)
break;
}
actualStart += pageSize;
}
while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue);
});
if (addedDocs != pageSize)
nextStart = start; // will mark as last page
else if (canPerformRapidPagination)
nextStart = start + matchedDocs;
else
nextStart = actualStart;
}
public Etag GetDocumentsWithIdStartingWith(string idPrefix, int pageSize, Etag etag, CancellationToken token, Func<JsonDocument, bool> addDocument)
{
Etag lastDocumentReadEtag = null;
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = actions.Documents.GetDocumentsAfterWithIdStartingWith(etag, idPrefix, pageSize, token, timeout: TimeSpan.FromSeconds(2), lastProcessedDocument: x => lastDocumentReadEtag = x );
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
etag = doc.Etag;
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
continue;
returnedDocs = true;
Database.WorkContext.UpdateFoundWork();
bool canContinue = addDocument(document);
if (!canContinue)
break;
}
if (returnedDocs)
break;
// No document was found that matches the requested criteria
if ( docCount == 0 )
{
// If we had a failure happen, we update the etag as we don't need to process those documents again (no matches there anyways).
if (lastDocumentReadEtag != null)
etag = lastDocumentReadEtag;
break;
}
}
});
return etag;
}
public Etag GetDocuments(int start, int pageSize, Etag etag, CancellationToken token,
Func<JsonDocument, bool> addDocument, long? maxSize = null, TimeSpan? timeout = null)
{
Etag lastDocumentReadEtag = null;
using (DocumentCacher.SkipSetDocumentsInDocumentCache())
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize, token, maxSize: maxSize, timeout: timeout);
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers);
var docCount = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
if (etag != null)
etag = doc.Etag;
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
continue;
returnedDocs = true;
Database.WorkContext.UpdateFoundWork();
bool canContinue = addDocument(document);
if (!canContinue)
break;
lastDocumentReadEtag = etag;
}
if (returnedDocs || docCount == 0)
break;
// No document was found that matches the requested criteria
// If we had a failure happen, we update the etag as we don't need to process those documents again (no matches there anyways).
if (lastDocumentReadEtag != null)
etag = lastDocumentReadEtag;
start += docCount;
}
});
return lastDocumentReadEtag;
}
public MoreLikeThisQueryResult ExecuteMoreLikeThisQuery(MoreLikeThisQuery query, TransactionInformation transactionInformation, int pageSize = 25)
{
if (query == null) throw new ArgumentNullException("query");
var index = database.IndexStorage.GetIndexInstance(query.IndexName);
if (index == null)
throw new InvalidOperationException("The index " + query.IndexName + " cannot be found");
if (string.IsNullOrEmpty(query.DocumentId) && query.MapGroupFields.Count == 0)
throw new InvalidOperationException("The document id or map group fields are mandatory");
IndexSearcher searcher;
using (database.IndexStorage.GetCurrentIndexSearcher(index.indexId, out searcher))
{
var documentQuery = new BooleanQuery();
if (string.IsNullOrEmpty(query.DocumentId) == false)
{
documentQuery.Add(new TermQuery(new Term(Constants.DocumentIdFieldName, query.DocumentId.ToLowerInvariant())), Occur.MUST);
}
foreach (string key in query.MapGroupFields.Keys)
{
documentQuery.Add(new TermQuery(new Term(key, query.MapGroupFields[key])), Occur.MUST);
}
var td = searcher.Search(documentQuery, 1);
// get the current Lucene docid for the given RavenDB doc ID
if (td.ScoreDocs.Length == 0)
throw new InvalidOperationException("Document " + query.DocumentId + " could not be found");
var ir = searcher.IndexReader;
var mlt = new RavenMoreLikeThis(ir);
AssignParameters(mlt, query);
if (string.IsNullOrWhiteSpace(query.StopWordsDocumentId) == false)
{
var stopWordsDoc = database.Documents.Get(query.StopWordsDocumentId, null);
if (stopWordsDoc == null)
throw new InvalidOperationException("Stop words document " + query.StopWordsDocumentId + " could not be found");
var stopWordsSetup = stopWordsDoc.DataAsJson.JsonDeserialization<StopWordsSetup>();
if (stopWordsSetup.StopWords != null)
{
var stopWords = stopWordsSetup.StopWords;
var ht = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);
foreach (var stopWord in stopWords)
{
ht.Add(stopWord);
}
mlt.SetStopWords(ht);
}
}
var fieldNames = query.Fields ?? GetFieldNames(ir);
mlt.SetFieldNames(fieldNames);
var toDispose = new List<Action>();
RavenPerFieldAnalyzerWrapper perFieldAnalyzerWrapper = null;
try
{
perFieldAnalyzerWrapper = index.CreateAnalyzer(new LowerCaseKeywordAnalyzer(), toDispose, true);
mlt.Analyzer = perFieldAnalyzerWrapper;
var mltQuery = mlt.Like(td.ScoreDocs[0].Doc);
var tsdc = TopScoreDocCollector.Create(pageSize, true);
if (string.IsNullOrWhiteSpace(query.AdditionalQuery) == false)
{
var additionalQuery = QueryBuilder.BuildQuery(query.AdditionalQuery, perFieldAnalyzerWrapper);
mltQuery = new BooleanQuery
{
{mltQuery, Occur.MUST},
{additionalQuery, Occur.MUST},
};
}
searcher.Search(mltQuery, tsdc);
var hits = tsdc.TopDocs().ScoreDocs;
var jsonDocuments = GetJsonDocuments(query, searcher, index, query.IndexName, hits, td.ScoreDocs[0].Doc);
var result = new MultiLoadResult();
var includedEtags = new List<byte>(jsonDocuments.SelectMany(x => x.Etag.ToByteArray()));
includedEtags.AddRange(database.Indexes.GetIndexEtag(query.IndexName, null).ToByteArray());
var loadedIds = new HashSet<string>(jsonDocuments.Select(x => x.Key));
var addIncludesCommand = new AddIncludesCommand(database, transactionInformation, (etag, includedDoc) =>
{
includedEtags.AddRange(etag.ToByteArray());
result.Includes.Add(includedDoc);
}, query.Includes ?? new string[0], loadedIds);
idsToLoad = new HashSet<string>();
database.TransactionalStorage.Batch(actions =>
{
documentRetriever = new DocumentRetriever(database.Configuration, actions, database.ReadTriggers, database.InFlightTransactionalState, query.TransformerParameters, idsToLoad);
using (new CurrentTransformationScope(database, documentRetriever))
{
foreach (var document in ProcessResults(query, jsonDocuments, database.WorkContext.CancellationToken))
{
result.Results.Add(document);
addIncludesCommand.Execute(document);
}
}
});
addIncludesCommand.AlsoInclude(idsToLoad);
var computeHash = Encryptor.Current.Hash.Compute16(includedEtags.ToArray());
Etag computedEtag = Etag.Parse(computeHash);
return new MoreLikeThisQueryResult
{
Etag = computedEtag,
Result = result,
};
}
finally
{
if (perFieldAnalyzerWrapper != null)
perFieldAnalyzerWrapper.Close();
foreach (var action in toDispose)
{
action();
}
}
}
}
public void CanExecuteTheTransformResultFunction()
{
var answersIndex = new Answers_ByAnswerEntity2 { Conventions = new DocumentConvention() };
IndexDefinition indexDefinition = answersIndex.CreateIndexDefinition();
var dynamicViewCompiler = new DynamicViewCompiler("test", indexDefinition, ".");
AbstractViewGenerator abstractViewGenerator = dynamicViewCompiler.GenerateInstance();
using (var documentStore = NewDocumentStore())
{
Guid questionId = Guid.NewGuid();
Guid answerId = Guid.NewGuid();
using (IDocumentSession session = documentStore.OpenSession())
{
var user = new User { Id = @"user\222", DisplayName = "John Doe" };
session.Store(user);
var question = new Question2
{
Id = questionId,
Title = "How to do this in RavenDb?",
Content = "I'm trying to find how to model documents for better DDD support.",
UserId = @"user\222"
};
session.Store(question);
var answer = new AnswerEntity2
{
Id = answerId,
Question = question,
Content = "This is doable",
UserId = user.Id
};
answer.Votes = new[]
{
new AnswerVoteEntity2
{
Id = Guid.NewGuid(),
QuestionId = questionId,
Answer = answer,
Delta = 2
}
};
session.Store(new Answer2
{
Id = answerId,
UserId = user.Id,
QuestionId = question.Id,
Content = "This is doable",
Votes = new[]
{
new AnswerVote2
{
Id = Guid.NewGuid(),
QuestionId = questionId,
AnswerId = answerId,
Delta = 2
}
}
});
session.SaveChanges();
}
documentStore.DocumentDatabase.TransactionalStorage.Batch(accessor =>
{
var documentRetriever = new DocumentRetriever(accessor, new OrderedPartCollection<AbstractReadTrigger>());
var dynamicJsonObjects = new[] { new DynamicJsonObject(accessor.Documents.DocumentByKey("answer2s/" + answerId.ToString(), null).ToJson()), };
var transformResultsDefinition = abstractViewGenerator.TransformResultsDefinition(documentRetriever,
dynamicJsonObjects
);
transformResultsDefinition.ToArray();
});
}
}
public Etag GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Func<JsonDocument, bool> addDocument)
{
Etag lastDocumentReadEtag = null;
TransactionalStorage.Batch(actions =>
{
bool returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize, token);
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, Database.InFlightTransactionalState);
int docCount = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
if (etag != null)
etag = doc.Etag;
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior<JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
continue;
returnedDocs = true;
Database.WorkContext.UpdateFoundWork();
bool canContinue = addDocument(document);
if (!canContinue)
break;
lastDocumentReadEtag = etag;
}
if (returnedDocs || docCount == 0)
break;
start += docCount;
}
});
return lastDocumentReadEtag;
}
private IEnumerable<RavenJObject> GetQueryResults(IndexQuery query,
AbstractViewGenerator viewGenerator,
DocumentRetriever docRetriever,
IEnumerable<JsonDocument> results,
List<string> transformerErrors,
CancellationToken token)
{
if (query.PageSize <= 0) // maybe they just want the stats?
{
return Enumerable.Empty<RavenJObject>();
}
IndexingFunc transformFunc = null;
// Check an explicitly declared one first
if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
{
var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer);
if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null)
transformFunc = transformGenerator.TransformResultsDefinition;
else
throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found");
}
else if (query.SkipTransformResults == false && viewGenerator.TransformResultsDefinition != null)
{
transformFunc = source => viewGenerator.TransformResultsDefinition(docRetriever, source);
}
if (transformFunc == null)
return results.Select(x => x.ToJson());
var dynamicJsonObjects = results.Select(x => new DynamicLuceneOrParentDocumntObject(docRetriever, x.ToJson()));
var robustEnumerator = new RobustEnumerator(token, 100)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
return robustEnumerator.RobustEnumeration(
dynamicJsonObjects.Cast<object>().GetEnumerator(),
transformFunc)
.Select(JsonExtensions.ToJObject);
}
public void Dispose()
{
current = null;
}
private IEnumerable<RavenJObject> GetQueryResults(IndexQuery query, AbstractViewGenerator viewGenerator, DocumentRetriever docRetriever, IEnumerable<JsonDocument> results, List<string> transformerErrors, Action<double> loadingDocumentsFinish, Action<double> transformerFinish, bool showTimings, CancellationToken token)
{
if (query.PageSize <= 0) // maybe they just want the stats?
{
return Enumerable.Empty<RavenJObject>();
}
IndexingFunc transformFunc = null;
// Check an explicitly declared one first
if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
{
var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer);
if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null)
transformFunc = transformGenerator.TransformResultsDefinition;
else
throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found");
}
if (transformFunc == null)
{
var resultsWithoutTransformer = results.Select(x =>
{
var ravenJObject = x.ToJson();
if (query.IsDistinct)
{
ravenJObject[Constants.DocumentIdFieldName] = x.Key;
}
return ravenJObject;
});
return showTimings ? new TimedEnumerable<RavenJObject>(resultsWithoutTransformer, loadingDocumentsFinish) : resultsWithoutTransformer;
}
var dynamicJsonObjects = results.Select(x =>
{
var ravenJObject = x.ToJson();
if (query.IsDistinct)
{
ravenJObject[Constants.DocumentIdFieldName] = x.Key;
}
return new DynamicLuceneOrParentDocumntObject(docRetriever, ravenJObject);
});
var robustEnumerator = new RobustEnumerator(token, 100)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
var resultsWithTransformer = robustEnumerator
.RobustEnumeration(dynamicJsonObjects.Cast<object>().GetEnumerator(), transformFunc)
.Select(JsonExtensions.ToJObject);
return showTimings ? new TimedEnumerable<RavenJObject>(resultsWithTransformer, transformerFinish) : resultsWithTransformer;
}
public DynamicLuceneOrParentDocumntObject(DocumentRetriever retriever,RavenJObject inner) : base(inner)
{
this.retriever = retriever;
}
private IEnumerable<IndexingBatchForIndex> FilterIndexes(IList<IndexToWorkOn> indexesToWorkOn, List<JsonDocument> jsonDocs, Etag highestETagInBatch)
{
var last = jsonDocs.Last();
Debug.Assert(last.Etag != null);
Debug.Assert(last.LastModified != null);
var lastEtag = last.Etag;
var lastModified = last.LastModified.Value;
var documentRetriever = new DocumentRetriever(null, context.ReadTriggers, context.Database.InFlightTransactionalState);
var filteredDocs =
BackgroundTaskExecuter.Instance.Apply(context, jsonDocs, doc =>
{
var filteredDoc = documentRetriever.ExecuteReadTriggers(doc, null, ReadOperation.Index);
return filteredDoc == null ? new
{
Doc = doc,
Json = (object)new FilteredDocument(doc)
} : new
{
Doc = filteredDoc,
Json = JsonToExpando.Convert(doc.ToJson())
};
});
Log.Debug("After read triggers executed, {0} documents remained", filteredDocs.Count);
var results = new IndexingBatchForIndex[indexesToWorkOn.Count];
var actions = new Action<IStorageActionsAccessor>[indexesToWorkOn.Count];
BackgroundTaskExecuter.Instance.ExecuteAll(context, indexesToWorkOn, (indexToWorkOn, i) =>
{
var indexName = indexToWorkOn.IndexName;
var viewGenerator = context.IndexDefinitionStorage.GetViewGenerator(indexName);
if (viewGenerator == null)
return; // probably deleted
var batch = new IndexingBatch(highestETagInBatch);
foreach (var item in filteredDocs)
{
if (prefetchingBehavior.FilterDocuments(item.Doc) == false)
continue;
// did we already indexed this document in this index?
var etag = item.Doc.Etag;
if (etag == null)
continue;
// is the Raven-Entity-Name a match for the things the index executes on?
if (viewGenerator.ForEntityNames.Count != 0 &&
viewGenerator.ForEntityNames.Contains(item.Doc.Metadata.Value<string>(Constants.RavenEntityName)) == false)
{
continue;
}
batch.Add(item.Doc, item.Json, prefetchingBehavior.ShouldSkipDeleteFromIndex(item.Doc));
if (batch.DateTime == null)
batch.DateTime = item.Doc.LastModified;
else
batch.DateTime = batch.DateTime > item.Doc.LastModified
? item.Doc.LastModified
: batch.DateTime;
}
if (batch.Docs.Count == 0)
{
Log.Debug("All documents have been filtered for {0}, no indexing will be performed, updating to {1}, {2}", indexName,
lastEtag, lastModified);
// we use it this way to batch all the updates together
actions[i] = accessor => accessor.Indexing.UpdateLastIndexed(indexName, lastEtag, lastModified);
return;
}
if (Log.IsDebugEnabled)
{
Log.Debug("Going to index {0} documents in {1}: ({2})", batch.Ids.Count, indexToWorkOn, string.Join(", ", batch.Ids));
}
results[i] = new IndexingBatchForIndex
{
Batch = batch,
IndexName = indexToWorkOn.IndexName,
Index = indexToWorkOn.Index,
LastIndexedEtag = indexToWorkOn.LastIndexedEtag
};
});
transactionalStorage.Batch(actionsAccessor =>
{
foreach (var action in actions)
{
if (action != null)
action(actionsAccessor);
}
});
return results.Where(x => x != null);
}
public RemoteQueryResults Query(LinearQuery query)
{
var viewGenerator = queryCache.GetOrAdd(query.Query,
s =>
new DynamicViewCompiler("query", new IndexDefinition { Map = query.Query, },
new AbstractDynamicCompilationExtension[0])
{
RequiresSelectNewAnonymousType = false
}.GenerateInstance());
var results = new List<string>();
var errors = new List<string>();
int lastResult = 0;
int finalResult = 0;
remoteStorage.Batch(actions =>
{
var firstAndLastDocumentIds = actions.Documents.FirstAndLastDocumentIds();
finalResult = firstAndLastDocumentIds.Item2;
var start = Math.Max(firstAndLastDocumentIds.Item1, query.Start);
var matchingDocs = actions.Documents.DocumentsById(start, firstAndLastDocumentIds.Item2);
if (string.IsNullOrEmpty(viewGenerator.ForEntityName) == false) //optimization
{
matchingDocs =
matchingDocs.Where(x => x.Item1.Metadata.Value<string>("Raven-Entity-Name") == viewGenerator.ForEntityName);
}
var documentRetriever = new DocumentRetriever(actions, new AbstractReadTrigger[0]);
var docs = matchingDocs
.Select(x=>
{
documentRetriever.EnsureIdInMetadata(x.Item1);
return x;
})
.Select(x =>
{
lastResult = x.Item2;
return new DynamicJsonObject(x.Item1.ToJson());
});
var robustEnumerator = new RobustEnumerator
{
OnError =
(exception, o) =>
errors.Add(String.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
results.AddRange(
robustEnumerator.RobustEnumeration(docs, viewGenerator.MapDefinition)
.Take(query.PageSize)
.Select(result => JsonExtensions.ToJObject(result).ToString())
);
});
return new RemoteQueryResults
{
LastScannedResult = lastResult,
TotalResults = finalResult,
Errors = errors.ToArray(),
QueryCacheSize = queryCache.Count,
Results = results.ToArray()
};
}
