private IEnumerable <RavenJObject> GetQueryResults(IndexQuery query, AbstractViewGenerator viewGenerator, DocumentRetriever docRetriever, IEnumerable <JsonDocument> results, List <string> transformerErrors, Action <double> loadingDocumentsFinish, Action <double> transformerFinish, bool showTimings, CancellationToken token)
{
if (query.PageSize <= 0) // maybe they just want the stats?
{
return(Enumerable.Empty <RavenJObject>());
}
IndexingFunc transformFunc = null;
// Check an explicitly declared one first
if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
{
var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer);
if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null)
{
transformFunc = transformGenerator.TransformResultsDefinition;
}
else
{
throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found");
}
}
if (transformFunc == null)
{
var resultsWithoutTransformer = results.Select(x =>
{
var ravenJObject = x.ToJson();
if (query.IsDistinct)
{
ravenJObject[Constants.DocumentIdFieldName] = x.Key;
}
return(ravenJObject);
});
return(showTimings ? new TimedEnumerable <RavenJObject>(resultsWithoutTransformer, loadingDocumentsFinish) : resultsWithoutTransformer);
}
var dynamicJsonObjects = results.Select(x =>
{
var ravenJObject = x.ToJson();
if (query.IsDistinct)
{
ravenJObject[Constants.DocumentIdFieldName] = x.Key;
}
return(new DynamicLuceneOrParentDocumntObject(docRetriever, ravenJObject));
});
var robustEnumerator = new RobustEnumerator(token, 100,
onError: (exception, o) => transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o), exception.Message)));
var resultsWithTransformer = robustEnumerator
.RobustEnumeration(dynamicJsonObjects.Cast <object>().GetEnumerator(), transformFunc)
.Select(JsonExtensions.ToJObject);
return(showTimings ? new TimedEnumerable <RavenJObject>(resultsWithTransformer, transformerFinish) : resultsWithTransformer);
}
public JsonDocument GetWithTransformer(string key, string transformer, TransactionInformation transactionInformation, Dictionary <string, RavenJToken> transformerParameters, out HashSet <string> itemsToInclude)
{
JsonDocument result = null;
DocumentRetriever docRetriever = null;
TransactionalStorage.Batch(
actions =>
{
docRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState, transformerParameters);
using (new CurrentTransformationScope(Database, docRetriever))
{
var document = Get(key, transactionInformation);
if (document == null)
{
return;
}
if (document.Metadata.ContainsKey("Raven-Read-Veto"))
{
result = document;
return;
}
var storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
{
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
var transformed = storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) })
.Select(x => JsonExtensions.ToJObject(x))
.ToArray();
if (transformed.Length == 0)
{
return;
}
result = new JsonDocument
{
Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(docRetriever.Etag),
NonAuthoritativeInformation = document.NonAuthoritativeInformation,
LastModified = document.LastModified,
DataAsJson = new RavenJObject {
{ "$values", new RavenJArray(transformed) }
},
};
}
});
itemsToInclude = docRetriever.ItemsToInclude;
return(result);
}
private IEnumerable <RavenJObject> GetQueryResults(IndexQuery query,
AbstractViewGenerator viewGenerator,
DocumentRetriever docRetriever,
IEnumerable <JsonDocument> results,
List <string> transformerErrors,
CancellationToken token)
{
if (query.PageSize <= 0) // maybe they just want the stats?
{
return(Enumerable.Empty <RavenJObject>());
}
IndexingFunc transformFunc = null;
// Check an explicitly declared one first
if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
{
var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer);
if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null)
{
transformFunc = transformGenerator.TransformResultsDefinition;
}
else
{
throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found");
}
}
else if (query.SkipTransformResults == false && viewGenerator.TransformResultsDefinition != null)
{
transformFunc = source => viewGenerator.TransformResultsDefinition(docRetriever, source);
}
if (transformFunc == null)
{
return(results.Select(x => x.ToJson()));
}
var dynamicJsonObjects = results.Select(x => new DynamicLuceneOrParentDocumntObject(docRetriever, x.ToJson()));
var robustEnumerator = new RobustEnumerator(token, 100)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
return(robustEnumerator.RobustEnumeration(
dynamicJsonObjects.Cast <object>().GetEnumerator(),
transformFunc)
.Select(JsonExtensions.ToJObject));
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize,
CancellationToken token, ref int nextStart, Action <RavenJObject> addDoc,
string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null,
string skipAfter = null)
{
if (idPrefix == null)
{
throw new ArgumentNullException("idPrefix");
}
idPrefix = idPrefix.Trim();
var canPerformRapidPagination = nextStart > 0 && start == nextStart;
var actualStart = canPerformRapidPagination ? start : 0;
var addedDocs = 0;
var matchedDocs = 0;
TransactionalStorage.Batch(
actions =>
{
var docsToSkip = canPerformRapidPagination ? 0 : start;
int docCount;
AbstractTransformer storedTransformer = null;
if (transformer != null)
{
storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
{
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
}
do
{
docCount = 0;
var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter);
var documentRetriever = new DocumentRetriever(actions, Database.ReadTriggers, Database.InFlightTransactionalState, transformerParameters);
foreach (var doc in docs)
{
token.ThrowIfCancellationRequested();
docCount++;
var keyTest = doc.Key.Substring(idPrefix.Length);
if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
{
continue;
}
DocumentRetriever.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = Database.InFlightTransactionalState.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc;
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
matchedDocs++;
if (matchedDocs <= docsToSkip)
{
continue;
}
token.ThrowIfCancellationRequested();
if (storedTransformer != null)
{
using (new CurrentTransformationScope(Database, documentRetriever))
{
var transformed =
storedTransformer.TransformResultsDefinition(new[] { new DynamicJsonObject(document.ToJson()) })
.Select(x => JsonExtensions.ToJObject(x))
.ToArray();
if (transformed.Length == 0)
{
throw new InvalidOperationException("The transform results function failed on a document: " + document.Key);
}
var transformedJsonDocument = new JsonDocument
{
Etag = document.Etag.HashWith(storedTransformer.GetHashCodeBytes()).HashWith(documentRetriever.Etag),
NonAuthoritativeInformation = document.NonAuthoritativeInformation,
LastModified = document.LastModified,
DataAsJson = new RavenJObject {
{ "$values", new RavenJArray(transformed) }
},
};
addDoc(transformedJsonDocument.ToJson());
}
}
else
{
addDoc(document.ToJson());
}
addedDocs++;
if (addedDocs >= pageSize)
{
break;
}
}
actualStart += pageSize;
}while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue);
});
if (addedDocs != pageSize)
{
nextStart = start; // will mark as last page
}
else if (canPerformRapidPagination)
{
nextStart = start + matchedDocs;
}
else
{
nextStart = actualStart;
}
}
public Etag GetDocuments(int start, int pageSize, Etag etag, CancellationToken token, Func <JsonDocument, bool> addDocument,
string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null, long?maxSize = null, TimeSpan?timeout = null)
{
Etag lastDocumentReadEtag = null;
using (DocumentCacher.SkipSetDocumentsInDocumentCache())
TransactionalStorage.Batch(actions =>
{
AbstractTransformer storedTransformer = null;
if (transformer != null)
{
storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
{
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
}
var returnedDocs = false;
while (true)
{
var documents = etag == null
? actions.Documents.GetDocumentsByReverseUpdateOrder(start, pageSize)
: actions.Documents.GetDocumentsAfter(etag, pageSize, token, maxSize: maxSize, timeout: timeout);
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, transformerParameters);
var docCount = 0;
var docCountOnLastAdd = 0;
foreach (var doc in documents)
{
docCount++;
token.ThrowIfCancellationRequested();
if (docCount - docCountOnLastAdd > 1000)
{
addDocument(null); // heartbeat
}
if (etag != null)
{
etag = doc.Etag;
}
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior == null ? doc : nonAuthoritativeInformationBehavior(doc);
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
returnedDocs = true;
Database.WorkContext.UpdateFoundWork();
document = TransformDocumentIfNeeded(document, storedTransformer, documentRetriever);
var canContinue = addDocument(document);
if (!canContinue)
{
break;
}
lastDocumentReadEtag = etag;
docCountOnLastAdd = docCount;
}
if (returnedDocs || docCount == 0)
{
break;
}
// No document was found that matches the requested criteria
// If we had a failure happen, we update the etag as we don't need to process those documents again (no matches there anyways).
if (lastDocumentReadEtag != null)
{
etag = lastDocumentReadEtag;
}
start += docCount;
}
});
return(lastDocumentReadEtag);
}
public void GetDocumentsWithIdStartingWith(string idPrefix, string matches, string exclude, int start, int pageSize,
CancellationToken token, ref int nextStart, Action <JsonDocument> addDoc,
string transformer = null, Dictionary <string, RavenJToken> transformerParameters = null,
string skipAfter = null)
{
if (idPrefix == null)
{
throw new ArgumentNullException("idPrefix");
}
idPrefix = idPrefix.Trim();
var canPerformRapidPagination = nextStart > 0 && start == nextStart;
var actualStart = canPerformRapidPagination ? start : 0;
var addedDocs = 0;
var docCountOnLastAdd = 0;
var matchedDocs = 0;
TransactionalStorage.Batch(
actions =>
{
var docsToSkip = canPerformRapidPagination ? 0 : start;
int docCount;
AbstractTransformer storedTransformer = null;
if (transformer != null)
{
storedTransformer = IndexDefinitionStorage.GetTransformer(transformer);
if (storedTransformer == null)
{
throw new InvalidOperationException("No transformer with the name: " + transformer);
}
}
do
{
Database.WorkContext.UpdateFoundWork();
docCount = 0;
var docs = actions.Documents.GetDocumentsWithIdStartingWith(idPrefix, actualStart, pageSize, string.IsNullOrEmpty(skipAfter) ? null : skipAfter);
var documentRetriever = new DocumentRetriever(Database.Configuration, actions, Database.ReadTriggers, transformerParameters);
foreach (var doc in docs)
{
token.ThrowIfCancellationRequested();
docCount++;
if (docCount - docCountOnLastAdd > 1000)
{
addDoc(null); // heartbeat
}
var keyTest = doc.Key.Substring(idPrefix.Length);
if (!WildcardMatcher.Matches(matches, keyTest) || WildcardMatcher.MatchesExclusion(exclude, keyTest))
{
continue;
}
JsonDocument.EnsureIdInMetadata(doc);
var nonAuthoritativeInformationBehavior = actions.InFlightStateSnapshot.GetNonAuthoritativeInformationBehavior <JsonDocument>(null, doc.Key);
var document = nonAuthoritativeInformationBehavior != null ? nonAuthoritativeInformationBehavior(doc) : doc;
document = documentRetriever.ExecuteReadTriggers(document, null, ReadOperation.Load);
if (document == null)
{
continue;
}
matchedDocs++;
if (matchedDocs <= docsToSkip)
{
continue;
}
token.ThrowIfCancellationRequested();
document = TransformDocumentIfNeeded(document, storedTransformer, documentRetriever);
addDoc(document);
addedDocs++;
docCountOnLastAdd = docCount;
if (addedDocs >= pageSize)
{
break;
}
}
actualStart += pageSize;
} while (docCount > 0 && addedDocs < pageSize && actualStart > 0 && actualStart < int.MaxValue);
});
if (addedDocs != pageSize)
{
nextStart = start; // will mark as last page
}
else if (canPerformRapidPagination)
{
nextStart = start + matchedDocs;
}
else
{
nextStart = actualStart;
}
}
public Etag GetIndexEtag(string indexName, Etag previousEtag, string resultTransformer = null)
{
Etag lastDocEtag = Etag.Empty;
Etag lastIndexedEtag = null;
Etag lastReducedEtag = null;
bool isStale = false;
int touchCount = 0;
TransactionalStorage.Batch(accessor =>
{
var indexInstance = Database.IndexStorage.GetIndexInstance(indexName);
if (indexInstance == null)
{
return;
}
isStale = (indexInstance.IsMapIndexingInProgress) ||
accessor.Staleness.IsIndexStale(indexInstance.indexId, null, null);
lastDocEtag = accessor.Staleness.GetMostRecentDocumentEtag();
var indexStats = accessor.Indexing.GetIndexStats(indexInstance.indexId);
if (indexStats != null)
{
lastReducedEtag = indexStats.LastReducedEtag;
lastIndexedEtag = indexStats.LastIndexedEtag;
}
touchCount = accessor.Staleness.GetIndexTouchCount(indexInstance.indexId);
});
var indexDefinition = GetIndexDefinition(indexName);
if (indexDefinition == null)
{
return(Etag.Empty); // this ensures that we will get the normal reaction of IndexNotFound later on.
}
var list = new List <byte>();
list.AddRange(indexDefinition.GetIndexHash());
list.AddRange(Encoding.Unicode.GetBytes(indexName));
if (string.IsNullOrWhiteSpace(resultTransformer) == false)
{
var abstractTransformer = IndexDefinitionStorage.GetTransformer(resultTransformer);
if (abstractTransformer == null)
{
throw new InvalidOperationException("The result transformer: " + resultTransformer + " was not found");
}
list.AddRange(abstractTransformer.GetHashCodeBytes());
}
list.AddRange(lastDocEtag.ToByteArray());
list.AddRange(BitConverter.GetBytes(touchCount));
list.AddRange(BitConverter.GetBytes(isStale));
if (lastReducedEtag != null)
{
list.AddRange(lastReducedEtag.ToByteArray());
}
if (lastIndexedEtag != null)
{
list.AddRange(lastIndexedEtag.ToByteArray());
}
list.AddRange(BitConverter.GetBytes(UuidGenerator.LastDocumentTransactionEtag));
var indexEtag = Etag.Parse(Encryptor.Current.Hash.Compute16(list.ToArray()));
if (previousEtag != null && previousEtag != indexEtag)
{
// the index changed between the time when we got it and the time
// we actually call this, we need to return something random so that
// the next time we won't get 304
return(Etag.InvalidEtag);
}
return(indexEtag);
}