public QueryResult Query(string index, IndexQuery query)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var list = new List <JObject>();
var stale = false;
Tuple <DateTime, Guid> indexTimestamp = null;
TransactionalStorage.Batch(
actions =>
{
string entityName = null;
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
{
throw new InvalidOperationException("Could not find index named: " + index);
}
entityName = viewGenerator.ForEntityName;
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, entityName);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers);
var indexDefinition = GetIndexDefinition(index);
var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
viewGenerator.ReduceDefinition == null
? Abstractions.Data.Constants.DocumentIdFieldName
: Abstractions.Data.Constants.ReduceKeyFieldName);
var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch), fieldsToFetch)
select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch)
into doc
where doc != null
select doc;
var transformerErrors = new List <string>();
IEnumerable <JObject> results;
if (viewGenerator != null &&
query.SkipTransformResults == false &&
viewGenerator.TransformResultsDefinition != null)
{
var robustEnumerator = new RobustEnumerator
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
results =
robustEnumerator.RobustEnumeration(
dynamicJsonObjects,
source => viewGenerator.TransformResultsDefinition(docRetriever, source))
.Select(JsonExtensions.ToJObject);
}
else
{
results = collection.Select(x => x.ToJson());
}
list.AddRange(results);
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
});
return(new QueryResult
{
IndexName = index,
Results = list,
IsStale = stale,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2
});
}
public QueryResultWithIncludes Query(string index, IndexQuery query, Action<QueryHeaderInformation> headerInfo, Action<RavenJObject> onResult)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var highlightings = new Dictionary<string, Dictionary<string, string[]>>();
Func<IndexQueryResult, object> tryRecordHighlighting = queryResult =>
{
if (queryResult.Highligtings != null && queryResult.Key != null)
highlightings.Add(queryResult.Key, queryResult.Highligtings);
return null;
};
var stale = false;
Tuple<DateTime, Etag> indexTimestamp = Tuple.Create(DateTime.MinValue, Etag.Empty);
Etag resultEtag = Etag.Empty;
var nonAuthoritativeInformation = false;
if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
{
query.FieldsToFetch = new[] { Constants.AllFields };
}
var idsToLoad = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
TransactionalStorage.Batch(
actions =>
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
throw new IndexDoesNotExistsException("Could not find index named: " + index);
resultEtag = GetIndexEtag(index, null);
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers, query.QueryInputs, idsToLoad);
var indexDefinition = GetIndexDefinition(index);
var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
viewGenerator.ReduceDefinition == null
? Constants.DocumentIdFieldName
: Constants.ReduceKeyFieldName);
Func<IndexQueryResult, bool> shouldIncludeInResults =
result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch);
var indexQueryResults = IndexStorage.Query(index, query, shouldIncludeInResults, fieldsToFetch, IndexQueryTriggers);
indexQueryResults = new ActiveEnumerable<IndexQueryResult>(indexQueryResults);
var transformerErrors = new List<string>();
var results = GetQueryResults(query, viewGenerator, docRetriever,
from queryResult in indexQueryResults
let doc = docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch)
where doc != null
let _ = nonAuthoritativeInformation |= (doc.NonAuthoritativeInformation ?? false)
let __ = tryRecordHighlighting(queryResult)
select doc, transformerErrors);
if (headerInfo != null)
{
headerInfo(new QueryHeaderInformation
{
Index = index,
IsStable = stale,
ResultEtag = resultEtag,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2,
TotalResults = query.TotalSize.Value
});
}
using (new CurrentTransformationScope(docRetriever))
{
foreach (var result in results)
{
onResult(result);
}
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
}
});
return new QueryResultWithIncludes
{
IndexName = index,
IsStale = stale,
NonAuthoritativeInformation = nonAuthoritativeInformation,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2,
ResultEtag = resultEtag,
IdsToInclude = idsToLoad,
LastQueryTime = SystemTime.UtcNow,
Highlightings = highlightings
};
}
public QueryResult Query(string index, IndexQuery query)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var list = new List<RavenJObject>();
var stale = false;
Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty);
TransactionalStorage.Batch(
actions =>
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
throw new InvalidOperationException("Could not find index named: " + index);
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers);
var indexDefinition = GetIndexDefinition(index);
var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
viewGenerator.ReduceDefinition == null
? Constants.DocumentIdFieldName
: Constants.ReduceKeyFieldName);
var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch), fieldsToFetch, IndexQueryTriggers)
select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch)
into doc
where doc != null
select doc;
var transformerErrors = new List<string>();
IEnumerable<RavenJObject> results;
if (query.SkipTransformResults == false &&
query.PageSize > 0 && // maybe they just want the stats?
viewGenerator.TransformResultsDefinition != null)
{
var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
var robustEnumerator = new RobustEnumerator(dynamicJsonObjects.Length)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
results =
robustEnumerator.RobustEnumeration(
dynamicJsonObjects,
source => viewGenerator.TransformResultsDefinition(docRetriever, source))
.Select(JsonExtensions.ToJObject);
}
else
{
results = collection.Select(x => x.ToJson());
}
if (query.PageSize > 0) // maybe they just want the query stats?
list.AddRange(results);
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
});
return new QueryResult
{
IndexName = index,
Results = list,
IsStale = stale,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2
};
}
public void Init()
{
highlightings = new Dictionary <string, Dictionary <string, string[]> >();
scoreExplanations = new Dictionary <string, string>();
Func <IndexQueryResult, object> tryRecordHighlightingAndScoreExplanation = queryResult =>
{
if (queryResult.Highligtings != null && (queryResult.Key != null || queryResult.HighlighterKey != null))
{
highlightings.Add(queryResult.Key ?? queryResult.HighlighterKey, queryResult.Highligtings);
}
if ((queryResult.Key != null || queryResult.ReduceVal != null) && queryResult.ScoreExplanation != null)
{
scoreExplanations.Add(queryResult.Key ?? queryResult.ReduceVal, queryResult.ScoreExplanation);
}
return(null);
};
stale = false;
indexTimestamp = Tuple.Create(DateTime.MinValue, Etag.Empty);
resultEtag = Etag.Empty;
nonAuthoritativeInformation = false;
if (string.IsNullOrEmpty(query.ResultsTransformer) == false &&
(query.FieldsToFetch == null || query.FieldsToFetch.Length == 0))
{
query.FieldsToFetch = new[] { Constants.AllFields };
}
duration = Stopwatch.StartNew();
idsToLoad = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
var viewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName);
var index = database.IndexDefinitionStorage.GetIndexDefinition(indexName);
if (viewGenerator == null)
{
throw new IndexDoesNotExistsException("Could not find index named: " + indexName);
}
resultEtag = database.Indexes.GetIndexEtag(index.Name, null, query.ResultsTransformer);
stale = actions.Staleness.IsIndexStale(index.IndexId, query.Cutoff, query.CutoffEtag);
if (stale == false && query.Cutoff == null && query.CutoffEtag == null)
{
var indexInstance = database.IndexStorage.GetIndexInstance(indexName);
stale = stale || (indexInstance != null && indexInstance.IsMapIndexingInProgress);
}
if (stale &&
actions.Staleness.IsIndexStaleByTask(index.IndexId, query.Cutoff) == false &&
actions.Staleness.IsReduceStale(index.IndexId) == false)
{
var forEntityNames = viewGenerator.ForEntityNames.ToList();
var lastIndexedEtag = actions.Indexing.GetIndexStats(index.IndexId).LastIndexedEtag;
if (database.LastCollectionEtags.HasEtagGreaterThan(forEntityNames, lastIndexedEtag) == false)
{
stale = false;
}
}
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index.IndexId);
var indexFailureInformation = actions.Indexing.GetFailureRate(index.IndexId);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
docRetriever = new DocumentRetriever(database.Configuration, actions, database.ReadTriggers, database.InFlightTransactionalState, query.TransformerParameters, idsToLoad);
var fieldsToFetch = new FieldsToFetch(query,
viewGenerator.ReduceDefinition == null
? Constants.DocumentIdFieldName
: Constants.ReduceKeyFieldName);
Func <IndexQueryResult, bool> shouldIncludeInResults =
result => docRetriever.ShouldIncludeResultInQuery(result, index, fieldsToFetch, ShouldSkipDuplicateChecking);
var indexQueryResults = database.IndexStorage.Query(indexName, query, shouldIncludeInResults, fieldsToFetch, database.IndexQueryTriggers, cancellationToken);
indexQueryResults = new ActiveEnumerable <IndexQueryResult>(indexQueryResults);
if (query.ShowTimings)
{
indexQueryResults = new TimedEnumerable <IndexQueryResult>(indexQueryResults, timeInMilliseconds => executionTimes[QueryTimings.Lucene] = timeInMilliseconds);
}
var docs = from queryResult in indexQueryResults
let doc = docRetriever.RetrieveDocumentForQuery(queryResult, index, fieldsToFetch, ShouldSkipDuplicateChecking)
where doc != null
let _ = nonAuthoritativeInformation |= (doc.NonAuthoritativeInformation ?? false)
let __ = tryRecordHighlightingAndScoreExplanation(queryResult)
select doc;
transformerErrors = new List <string>();
results = database
.Queries
.GetQueryResults(query, viewGenerator, docRetriever, docs, transformerErrors,
timeInMilliseconds => executionTimes[QueryTimings.LoadDocuments] = timeInMilliseconds,
timeInMilliseconds => executionTimes[QueryTimings.TransformResults] = timeInMilliseconds,
query.ShowTimings, cancellationToken);
Header = new QueryHeaderInformation
{
Index = indexName,
IsStale = stale,
ResultEtag = resultEtag,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2,
TotalResults = query.TotalSize.Value
};
}
public QueryResultWithIncludes Query(string index, IndexQuery query)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var list = new List<RavenJObject>();
var highlightings = new Dictionary<string, Dictionary<string, string[]>>();
var stale = false;
Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty);
Guid resultEtag = Guid.Empty;
var nonAuthoritativeInformation = false;
var idsToLoad = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);
TransactionalStorage.Batch(
actions =>
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
throw new IndexDoesNotExistsException("Could not find index named: " + index);
resultEtag = GetIndexEtag(index, null);
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers, idsToLoad);
var indexDefinition = GetIndexDefinition(index);
var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
viewGenerator.ReduceDefinition == null
? Constants.DocumentIdFieldName
: Constants.ReduceKeyFieldName);
Func<IndexQueryResult, bool> shouldIncludeInResults =
result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch);
var collection =
from queryResult in
IndexStorage.Query(index, query, shouldIncludeInResults, fieldsToFetch, IndexQueryTriggers)
select new
{
Document = docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch),
Fragments = queryResult.Highligtings
}
into docWithFragments
where docWithFragments.Document != null
let _ = nonAuthoritativeInformation |= (docWithFragments.Document.NonAuthoritativeInformation ?? false)
select docWithFragments;
var transformerErrors = new List<string>();
IEnumerable<RavenJObject> results;
if (query.SkipTransformResults == false &&
query.PageSize > 0 && // maybe they just want the stats?
viewGenerator.TransformResultsDefinition != null)
{
var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.Document.ToJson())).ToArray();
var robustEnumerator = new RobustEnumerator(workContext, dynamicJsonObjects.Length)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
results =
robustEnumerator.RobustEnumeration(
dynamicJsonObjects.Cast<object>().GetEnumerator(),
source => viewGenerator.TransformResultsDefinition(docRetriever, source))
.Select(JsonExtensions.ToJObject);
}
else
{
var resultList = new List<RavenJObject>();
foreach (var docWithFragments in collection)
{
resultList.Add(docWithFragments.Document.ToJson());
if (docWithFragments.Fragments != null && docWithFragments.Document.Key != null)
highlightings.Add(docWithFragments.Document.Key, docWithFragments.Fragments);
}
results = resultList;
}
list.AddRange(results);
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
});
return new QueryResultWithIncludes
{
IndexName = index,
Results = list,
IsStale = stale,
NonAuthoritativeInformation = nonAuthoritativeInformation,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2,
ResultEtag = resultEtag,
IdsToInclude = idsToLoad,
LastQueryTime = SystemTime.UtcNow,
Highlightings = highlightings
};
}
public QueryResult Query(string index, IndexQuery query)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var list = new List<JObject>();
var stale = false;
Tuple<DateTime, Guid> indexTimestamp = null;
TransactionalStorage.Batch(
actions =>
{
string entityName = null;
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator != null)
entityName = viewGenerator.ForEntityName;
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, entityName);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers);
var indexDefinition = GetIndexDefinition(index);
var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, query.FieldsToFetch,query.AggregationOperation))
select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, query.FieldsToFetch, query.AggregationOperation)
into doc
where doc != null
select doc;
var transformerErrors = new List<string>();
IEnumerable<JObject> results;
if (viewGenerator != null &&
query.SkipTransformResults == false &&
viewGenerator.TransformResultsDefinition != null)
{
var robustEnumerator = new RobustEnumerator
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
results =
robustEnumerator.RobustEnumeration(
dynamicJsonObjects,
source => viewGenerator.TransformResultsDefinition(docRetriever, source))
.Select(JsonExtensions.ToJObject);
}
else
{
results = collection.Select(x => x.ToJson());
}
list.AddRange(results);
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
});
return new QueryResult
{
IndexName = index,
Results = list,
IsStale = stale,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2
};
}