public RemoteQueryResults Query(LinearQuery query)
{
var viewGenerator = queryCache.GetOrAdd(query.Query,
s =>
new DynamicViewCompiler("query", new IndexDefinition { Map = query.Query, },
new AbstractDynamicCompilationExtension[0])
{
RequiresSelectNewAnonymousType = false
}.GenerateInstance());
var results = new List<string>();
var errors = new List<string>();
int lastResult = 0;
int finalResult = 0;
remoteStorage.Batch(actions =>
{
var firstAndLastDocumentIds = actions.Documents.FirstAndLastDocumentIds();
finalResult = firstAndLastDocumentIds.Item2;
var start = Math.Max(firstAndLastDocumentIds.Item1, query.Start);
var matchingDocs = actions.Documents.DocumentsById(start, firstAndLastDocumentIds.Item2);
if (string.IsNullOrEmpty(viewGenerator.ForEntityName) == false) //optimization
{
matchingDocs =
matchingDocs.Where(x => x.Item1.Metadata.Value<string>("Raven-Entity-Name") == viewGenerator.ForEntityName);
}
var docs = matchingDocs
.Select(x =>
{
lastResult = x.Item2;
return new DynamicJsonObject(x.Item1.ToJson());
});
var robustEnumerator = new RobustEnumerator
{
OnError =
(exception, o) =>
errors.Add(String.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
results.AddRange(
robustEnumerator.RobustEnumeration(docs, viewGenerator.MapDefinition)
.Take(query.PageSize)
.Select(result => JsonExtensions.ToJObject(result).ToString())
);
});
return new RemoteQueryResults
{
LastScannedResult = lastResult,
TotalResults = finalResult,
Errors = errors.ToArray(),
QueryCacheSize = queryCache.Count,
Results = results.ToArray()
};
}
private IEnumerable<RavenJObject> GetQueryResults(IndexQuery query, AbstractViewGenerator viewGenerator, DocumentRetriever docRetriever, IEnumerable<JsonDocument> results, List<string> transformerErrors, Action<double> loadingDocumentsFinish, Action<double> transformerFinish, bool showTimings, CancellationToken token)
{
if (query.PageSize <= 0) // maybe they just want the stats?
{
return Enumerable.Empty<RavenJObject>();
}
IndexingFunc transformFunc = null;
// Check an explicitly declared one first
if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
{
var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer);
if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null)
transformFunc = transformGenerator.TransformResultsDefinition;
else
throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found");
}
if (transformFunc == null)
{
var resultsWithoutTransformer = results.Select(x =>
{
var ravenJObject = x.ToJson();
if (query.IsDistinct)
{
ravenJObject[Constants.DocumentIdFieldName] = x.Key;
}
return ravenJObject;
});
return showTimings ? new TimedEnumerable<RavenJObject>(resultsWithoutTransformer, loadingDocumentsFinish) : resultsWithoutTransformer;
}
var dynamicJsonObjects = results.Select(x =>
{
var ravenJObject = x.ToJson();
if (query.IsDistinct)
{
ravenJObject[Constants.DocumentIdFieldName] = x.Key;
}
return new DynamicLuceneOrParentDocumntObject(docRetriever, ravenJObject);
});
var robustEnumerator = new RobustEnumerator(token, 100)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
var resultsWithTransformer = robustEnumerator
.RobustEnumeration(dynamicJsonObjects.Cast<object>().GetEnumerator(), transformFunc)
.Select(JsonExtensions.ToJObject);
return showTimings ? new TimedEnumerable<RavenJObject>(resultsWithTransformer, transformerFinish) : resultsWithTransformer;
}
private IEnumerable<RavenJObject> GetQueryResults(IndexQuery query,
AbstractViewGenerator viewGenerator,
DocumentRetriever docRetriever,
IEnumerable<JsonDocument> results,
List<string> transformerErrors)
{
if (query.PageSize <= 0) // maybe they just want the stats?
{
return Enumerable.Empty<RavenJObject>();
}
IndexingFunc transformFunc = null;
// Check an explicitly declared one first
if (query.ResultsTransformer != null)
{
var transformGenerator = IndexDefinitionStorage.GetTransfomer(query.ResultsTransformer);
if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null)
transformFunc = transformGenerator.TransformResultsDefinition;
}
else if (query.SkipTransformResults == false && viewGenerator.TransformResultsDefinition != null)
{
transformFunc = source => viewGenerator.TransformResultsDefinition(docRetriever, source);
}
if (transformFunc == null)
return results.Select(x => x.ToJson());
var dynamicJsonObjects = results.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
var robustEnumerator = new RobustEnumerator(workContext, dynamicJsonObjects.Length)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
return robustEnumerator.RobustEnumeration(
dynamicJsonObjects.Cast<object>().GetEnumerator(),
transformFunc)
.Select(JsonExtensions.ToJObject);
}
public QueryResult Query(string index, IndexQuery query)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var list = new List<RavenJObject>();
var stale = false;
Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty);
TransactionalStorage.Batch(
actions =>
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
throw new InvalidOperationException("Could not find index named: " + index);
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers);
var indexDefinition = GetIndexDefinition(index);
var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
viewGenerator.ReduceDefinition == null
? Constants.DocumentIdFieldName
: Constants.ReduceKeyFieldName);
var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch), fieldsToFetch, IndexQueryTriggers)
select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch)
into doc
where doc != null
select doc;
var transformerErrors = new List<string>();
IEnumerable<RavenJObject> results;
if (query.SkipTransformResults == false &&
query.PageSize > 0 && // maybe they just want the stats?
viewGenerator.TransformResultsDefinition != null)
{
var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
var robustEnumerator = new RobustEnumerator(dynamicJsonObjects.Length)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
results =
robustEnumerator.RobustEnumeration(
dynamicJsonObjects,
source => viewGenerator.TransformResultsDefinition(docRetriever, source))
.Select(JsonExtensions.ToJObject);
}
else
{
results = collection.Select(x => x.ToJson());
}
if (query.PageSize > 0) // maybe they just want the query stats?
list.AddRange(results);
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
});
return new QueryResult
{
IndexName = index,
Results = list,
IsStale = stale,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2
};
}
public QueryResultWithIncludes Query(string index, IndexQuery query)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var list = new List<RavenJObject>();
var highlightings = new Dictionary<string, Dictionary<string, string[]>>();
var stale = false;
Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty);
Guid resultEtag = Guid.Empty;
var nonAuthoritativeInformation = false;
var idsToLoad = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);
TransactionalStorage.Batch(
actions =>
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
throw new IndexDoesNotExistsException("Could not find index named: " + index);
resultEtag = GetIndexEtag(index, null);
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers, idsToLoad);
var indexDefinition = GetIndexDefinition(index);
var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
viewGenerator.ReduceDefinition == null
? Constants.DocumentIdFieldName
: Constants.ReduceKeyFieldName);
Func<IndexQueryResult, bool> shouldIncludeInResults =
result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch);
var collection =
from queryResult in
IndexStorage.Query(index, query, shouldIncludeInResults, fieldsToFetch, IndexQueryTriggers)
select new
{
Document = docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch),
Fragments = queryResult.Highligtings
}
into docWithFragments
where docWithFragments.Document != null
let _ = nonAuthoritativeInformation |= (docWithFragments.Document.NonAuthoritativeInformation ?? false)
select docWithFragments;
var transformerErrors = new List<string>();
IEnumerable<RavenJObject> results;
if (query.SkipTransformResults == false &&
query.PageSize > 0 && // maybe they just want the stats?
viewGenerator.TransformResultsDefinition != null)
{
var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.Document.ToJson())).ToArray();
var robustEnumerator = new RobustEnumerator(workContext, dynamicJsonObjects.Length)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
results =
robustEnumerator.RobustEnumeration(
dynamicJsonObjects.Cast<object>().GetEnumerator(),
source => viewGenerator.TransformResultsDefinition(docRetriever, source))
.Select(JsonExtensions.ToJObject);
}
else
{
var resultList = new List<RavenJObject>();
foreach (var docWithFragments in collection)
{
resultList.Add(docWithFragments.Document.ToJson());
if (docWithFragments.Fragments != null && docWithFragments.Document.Key != null)
highlightings.Add(docWithFragments.Document.Key, docWithFragments.Fragments);
}
results = resultList;
}
list.AddRange(results);
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
});
return new QueryResultWithIncludes
{
IndexName = index,
Results = list,
IsStale = stale,
NonAuthoritativeInformation = nonAuthoritativeInformation,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2,
ResultEtag = resultEtag,
IdsToInclude = idsToLoad,
LastQueryTime = SystemTime.UtcNow,
Highlightings = highlightings
};
}
private IEnumerable<RavenJObject> ProcessResults(MoreLikeThisQuery query, IEnumerable<JsonDocument> documents, CancellationToken token)
{
IndexingFunc transformFunc = null;
if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
{
var transformGenerator = database.IndexDefinitionStorage.GetTransformer(query.ResultsTransformer);
if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null)
transformFunc = transformGenerator.TransformResultsDefinition;
else
throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found");
}
IEnumerable<RavenJObject> results;
var transformerErrors = new List<string>();
if (transformFunc == null)
results = documents.Select(x => x.ToJson());
else
{
var robustEnumerator = new RobustEnumerator(token, 100)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
results = robustEnumerator
.RobustEnumeration(documents.Select(x => new DynamicJsonObject(x.ToJson())).GetEnumerator(), transformFunc)
.Select(JsonExtensions.ToJObject);
}
return results;
}
private IEnumerable<RavenJObject> GetQueryResults(IndexQuery query,
AbstractViewGenerator viewGenerator,
DocumentRetriever docRetriever,
IEnumerable<JsonDocument> results,
List<string> transformerErrors,
CancellationToken token)
{
if (query.PageSize <= 0) // maybe they just want the stats?
{
return Enumerable.Empty<RavenJObject>();
}
IndexingFunc transformFunc = null;
// Check an explicitly declared one first
if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
{
var transformGenerator = IndexDefinitionStorage.GetTransformer(query.ResultsTransformer);
if (transformGenerator != null && transformGenerator.TransformResultsDefinition != null)
transformFunc = transformGenerator.TransformResultsDefinition;
else
throw new InvalidOperationException("The transformer " + query.ResultsTransformer + " was not found");
}
else if (query.SkipTransformResults == false && viewGenerator.TransformResultsDefinition != null)
{
transformFunc = source => viewGenerator.TransformResultsDefinition(docRetriever, source);
}
if (transformFunc == null)
return results.Select(x => x.ToJson());
var dynamicJsonObjects = results.Select(x => new DynamicLuceneOrParentDocumntObject(docRetriever, x.ToJson()));
var robustEnumerator = new RobustEnumerator(token, 100)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
return robustEnumerator.RobustEnumeration(
dynamicJsonObjects.Cast<object>().GetEnumerator(),
transformFunc)
.Select(JsonExtensions.ToJObject);
}
public QueryResult Query(string index, IndexQuery query)
{
var list = new List<JObject>();
var stale = false;
DateTime indexTimestamp = DateTime.MinValue;
TransactionalStorage.Batch(
actions =>
{
string entityName = null;
var abstractViewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (abstractViewGenerator != null)
entityName = abstractViewGenerator.ForEntityName;
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, entityName);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers);
var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, query.FieldsToFetch))
select docRetriever.RetrieveDocumentForQuery(queryResult, query.FieldsToFetch)
into doc
where doc != null
select doc;
var transformerErrors = new List<string>();
IEnumerable<JObject> results;
if (abstractViewGenerator != null &&
abstractViewGenerator.TransformResultsDefinition != null)
{
var robustEnumerator = new RobustEnumerator
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
results =
robustEnumerator.RobustEnumeration(
collection.Select(x => new DynamicJsonObject(x.ToJson())),
source => abstractViewGenerator.TransformResultsDefinition(docRetriever, source))
.Select(JsonExtensions.ToJObject);
}
else
{
results = collection.Select(x => x.ToJson());
}
list.AddRange(results);
if(transformerErrors.Count>0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
});
return new QueryResult
{
Results = list,
IsStale = stale,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp
};
}
