protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score)
{
fieldsToFetch.EnsureHasField(Constants.ReduceKeyFieldName);
if (fieldsToFetch.HasExplicitFieldsToFetch)
{
return(base.RetrieveDocument(document, fieldsToFetch, score));
}
var field = document.GetField(Constants.ReduceValueFieldName);
if (field == null)
{
fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().Select(x => x.Name).ToArray());
return(base.RetrieveDocument(document, fieldsToFetch, score));
}
var projection = RavenJObject.Parse(field.StringValue);
if (fieldsToFetch.FetchAllStoredFields)
{
var fields = new HashSet <string>(document.GetFields().Select(x => x.Name));
fields.Remove(Constants.ReduceKeyFieldName);
var documentFromFields = new RavenJObject();
AddFieldsToDocument(document, fields, documentFromFields);
foreach (var kvp in projection)
{
documentFromFields[kvp.Key] = kvp.Value;
}
projection = documentFromFields;
}
return(new IndexQueryResult
{
Projection = projection,
Score = score.Score,
ReduceVal = field.StringValue
});
}
protected virtual IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch)
{
return(new IndexQueryResult
{
Key = document.Get(Constants.DocumentIdFieldName),
Projection = fieldsToFetch.IsProjection ? CreateDocumentFromFields(document, fieldsToFetch) : null
});
}
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch)
{
if (fieldsToFetch.IsProjection == false)
{
fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().OfType <Fieldable>().Select(x => x.Name()).ToArray());
}
fieldsToFetch.EnsureHasField(Raven.Abstractions.Data.Constants.ReduceKeyFieldName);
return(base.RetrieveDocument(document, fieldsToFetch));
}
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score)
{
if (fieldsToFetch.IsProjection == false)
{
fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().Select(x => x.Name).ToArray());
}
fieldsToFetch.EnsureHasField(Constants.ReduceKeyFieldName);
return(base.RetrieveDocument(document, fieldsToFetch, score));
}
public IndexQueryOperation(Index parent, IndexQuery indexQuery, Func <IndexQueryResult, bool> shouldIncludeInResults, FieldsToFetch fieldsToFetch, OrderedPartCollection <AbstractIndexQueryTrigger> indexQueryTriggers)
{
this.parent = parent;
this.indexQuery = indexQuery;
this.shouldIncludeInResults = shouldIncludeInResults;
this.fieldsToFetch = fieldsToFetch;
this.indexQueryTriggers = indexQueryTriggers;
if (fieldsToFetch.IsDistinctQuery)
{
alreadyReturned = new HashSet <RavenJObject>(new RavenJTokenEqualityComparer());
}
}
public Query GetLuceneQuery(string index, IndexQuery query, OrderedPartCollection <AbstractIndexQueryTrigger> indexQueryTriggers)
{
Index value;
if (indexes.TryGetValue(index, out value) == false)
{
log.Debug("Query on non existing index {0}", index);
throw new InvalidOperationException("Index '" + index + "' does not exists");
}
var fieldsToFetch = new FieldsToFetch(new string[0], AggregationOperation.None, null);
return(new Index.IndexQueryOperation(value, query, _ => false, fieldsToFetch, indexQueryTriggers).GetLuceneQuery());
}
public IEnumerable <IndexQueryResult> Query(
string index,
IndexQuery query,
Func <IndexQueryResult, bool> shouldIncludeInResults,
FieldsToFetch fieldsToFetch)
{
Index value;
if (indexes.TryGetValue(index, out value) == false)
{
log.DebugFormat("Query on non existing index {0}", index);
throw new InvalidOperationException("Index " + index + " does not exists");
}
return(new Index.IndexQueryOperation(value, query, shouldIncludeInResults, fieldsToFetch).Query());
}
public IndexQueryOperation(
Index parent,
IndexQuery indexQuery,
Func <IndexQueryResult, bool> shouldIncludeInResults,
FieldsToFetch fieldsToFetch)
{
this.parent = parent;
this.indexQuery = indexQuery;
this.shouldIncludeInResults = shouldIncludeInResults;
this.fieldsToFetch = fieldsToFetch;
if (fieldsToFetch.IsDistinctQuery)
{
alreadyReturned = new HashSet <JObject>(new JTokenEqualityComparer());
}
}
public IEnumerable <IndexQueryResult> Query(
string index,
IndexQuery query,
Func <IndexQueryResult, bool> shouldIncludeInResults,
FieldsToFetch fieldsToFetch,
OrderedPartCollection <AbstractIndexQueryTrigger> indexQueryTriggers)
{
Index value;
if (indexes.TryGetValue(index, out value) == false)
{
log.Debug("Query on non existing index '{0}'", index);
throw new InvalidOperationException("Index '" + index + "' does not exists");
}
return(new Index.IndexQueryOperation(value, query, shouldIncludeInResults, fieldsToFetch, indexQueryTriggers).Query());
}
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score)
{
fieldsToFetch.EnsureHasField(Constants.ReduceKeyFieldName);
if (fieldsToFetch.IsProjection)
{
return(base.RetrieveDocument(document, fieldsToFetch, score));
}
var field = document.GetField(Constants.ReduceValueFieldName);
if (field == null)
{
fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().Select(x => x.Name).ToArray());
return(base.RetrieveDocument(document, fieldsToFetch, score));
}
return(new IndexQueryResult
{
Projection = RavenJObject.Parse(field.StringValue),
Score = score.Score
});
}
public static RavenJObject CreateDocumentFromFields(Document document, FieldsToFetch fieldsToFetch)
{
var documentFromFields = new RavenJObject();
var fields = fieldsToFetch.Fields;
if (fieldsToFetch.FetchAllStoredFields)
fields = fields.Concat(document.GetFields().Select(x => x.Name));
var q = fields
.Distinct()
.SelectMany(name => document.GetFields(name) ?? new Field[0])
.Where(x => x != null)
.Where(
x =>
x.Name.EndsWith("_IsArray") == false &&
x.Name.EndsWith("_Range") == false &&
x.Name.EndsWith("_ConvertToJson") == false)
.Select(fld => CreateProperty(fld, document))
.GroupBy(x => x.Key)
.Select(g =>
{
if (g.Count() == 1 && document.GetField(g.Key + "_IsArray") == null)
{
return g.First();
}
var ravenJTokens = g.Select(x => x.Value).ToArray();
return new KeyValuePair<string, RavenJToken>(g.Key, new RavenJArray((IEnumerable)ravenJTokens));
});
foreach (var keyValuePair in q)
{
documentFromFields.Add(keyValuePair.Key, keyValuePair.Value);
}
return documentFromFields;
}
protected virtual IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score)
{
return new IndexQueryResult
{
Score = score.Score,
Key = document.Get(Constants.DocumentIdFieldName),
Projection = (fieldsToFetch.IsProjection || fieldsToFetch.FetchAllStoredFields) ? CreateDocumentFromFields(document, fieldsToFetch) : null
};
}
public DuplicateDocumentRecorder(Searchable indexSearcher,
Index parent,
HashSet<string> documentsAlreadySeenInPreviousPage,
HashSet<RavenJObject> alreadyReturned,
FieldsToFetch fieldsToFetch,
bool isProjectionOrMapReduce)
{
this.indexSearcher = indexSearcher;
this.parent = parent;
this.isProjectionOrMapReduce = isProjectionOrMapReduce;
this.alreadyReturned = alreadyReturned;
this.fieldsToFetch = fieldsToFetch;
this.documentsAlreadySeenInPreviousPage = documentsAlreadySeenInPreviousPage;
}
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score)
{
fieldsToFetch.EnsureHasField(Constants.ReduceKeyFieldName);
if (fieldsToFetch.HasExplicitFieldsToFetch)
{
return base.RetrieveDocument(document, fieldsToFetch, score);
}
var field = document.GetField(Constants.ReduceValueFieldName);
if (field == null)
{
fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().Select(x => x.Name).ToArray());
return base.RetrieveDocument(document, fieldsToFetch, score);
}
return new IndexQueryResult
{
Projection = RavenJObject.Parse(field.StringValue),
Score = score.Score
};
}
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, ScoreDoc score)
{
fieldsToFetch.EnsureHasField(Constants.ReduceKeyFieldName);
if (fieldsToFetch.HasExplicitFieldsToFetch)
{
return base.RetrieveDocument(document, fieldsToFetch, score);
}
var field = document.GetField(Constants.ReduceValueFieldName);
if (field == null)
{
fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().Select(x => x.Name).ToArray());
return base.RetrieveDocument(document, fieldsToFetch, score);
}
var projection = RavenJObject.Parse(field.StringValue);
if (fieldsToFetch.FetchAllStoredFields)
{
var fields = new HashSet<string>(document.GetFields().Select(x => x.Name));
fields.Remove(Constants.ReduceKeyFieldName);
var documentFromFields = new RavenJObject();
AddFieldsToDocument(document, fields, documentFromFields);
foreach (var kvp in projection)
{
documentFromFields[kvp.Key] = kvp.Value;
}
projection = documentFromFields;
}
return new IndexQueryResult
{
Projection = projection,
Score = score.Score,
ReduceVal = field.StringValue
};
}
protected override IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch, float score)
{
if (fieldsToFetch.IsProjection == false)
fieldsToFetch = fieldsToFetch.CloneWith(document.GetFields().OfType<Fieldable>().Select(x => x.Name()).ToArray());
fieldsToFetch.EnsureHasField(Abstractions.Data.Constants.ReduceKeyFieldName);
return base.RetrieveDocument(document, fieldsToFetch, score);
}
public IEnumerable<IndexQueryResult> Query(string index, IndexQuery query, Func<IndexQueryResult, bool> shouldIncludeInResults, FieldsToFetch fieldsToFetch, OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers, CancellationToken token)
{
Index value = TryIndexByName(index);
if (value == null)
{
log.Debug("Query on non existing index '{0}'", index);
throw new InvalidOperationException("Index '" + index + "' does not exists");
}
if ((value.Priority.HasFlag(IndexingPriority.Idle) || value.Priority.HasFlag(IndexingPriority.Abandoned)) &&
value.Priority.HasFlag(IndexingPriority.Forced) == false)
{
documentDatabase.TransactionalStorage.Batch(accessor =>
{
value.Priority = IndexingPriority.Normal;
try
{
accessor.Indexing.SetIndexPriority(value.indexId, IndexingPriority.Normal);
}
catch (Exception e)
{
if (accessor.IsWriteConflict(e) == false)
throw;
// we explciitly ignore write conflicts here, it is okay if we got set twice (two concurrent queries, or setting while indexing).
}
documentDatabase.WorkContext.ShouldNotifyAboutWork(() => "Idle index queried");
documentDatabase.Notifications.RaiseNotifications(new IndexChangeNotification()
{
Name = value.PublicName,
Type = IndexChangeTypes.IndexPromotedFromIdle
});
});
}
var indexQueryOperation = new Index.IndexQueryOperation(value, query, shouldIncludeInResults, fieldsToFetch, indexQueryTriggers);
if (query.Query != null && query.Query.Contains(Constants.IntersectSeparator))
return indexQueryOperation.IntersectionQuery(token);
return indexQueryOperation.Query(token);
}
protected virtual IndexQueryResult RetrieveDocument(Document document, FieldsToFetch fieldsToFetch)
{
return new IndexQueryResult
{
Key = document.Get(Constants.DocumentIdFieldName),
Projection = fieldsToFetch.IsProjection ? CreateDocumentFromFields(document, fieldsToFetch) : null
};
}
public IEnumerable<IndexQueryResult> Query(
string index,
IndexQuery query,
Func<IndexQueryResult, bool> shouldIncludeInResults,
FieldsToFetch fieldsToFetch,
OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers)
{
Index value;
if (indexes.TryGetValue(index, out value) == false)
{
log.Debug("Query on non existing index '{0}'", index);
throw new InvalidOperationException("Index '" + index + "' does not exists");
}
var indexQueryOperation = new Index.IndexQueryOperation(value, query, shouldIncludeInResults, fieldsToFetch, indexQueryTriggers);
if (query.Query != null && query.Query.Contains(Constants.IntersectSeperator))
return indexQueryOperation.IntersectionQuery();
return indexQueryOperation.Query();
}
public QueryResultWithIncludes Query(string index, IndexQuery query, Action<QueryHeaderInformation> headerInfo, Action<RavenJObject> onResult)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var highlightings = new Dictionary<string, Dictionary<string, string[]>>();
Func<IndexQueryResult, object> tryRecordHighlighting = queryResult =>
{
if (queryResult.Highligtings != null && queryResult.Key != null)
highlightings.Add(queryResult.Key, queryResult.Highligtings);
return null;
};
var stale = false;
Tuple<DateTime, Etag> indexTimestamp = Tuple.Create(DateTime.MinValue, Etag.Empty);
Etag resultEtag = Etag.Empty;
var nonAuthoritativeInformation = false;
if (string.IsNullOrEmpty(query.ResultsTransformer) == false)
{
query.FieldsToFetch = new[] { Constants.AllFields };
}
var idsToLoad = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
TransactionalStorage.Batch(
actions =>
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
throw new IndexDoesNotExistsException("Could not find index named: " + index);
resultEtag = GetIndexEtag(index, null);
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers, query.QueryInputs, idsToLoad);
var indexDefinition = GetIndexDefinition(index);
var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
viewGenerator.ReduceDefinition == null
? Constants.DocumentIdFieldName
: Constants.ReduceKeyFieldName);
Func<IndexQueryResult, bool> shouldIncludeInResults =
result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch);
var indexQueryResults = IndexStorage.Query(index, query, shouldIncludeInResults, fieldsToFetch, IndexQueryTriggers);
indexQueryResults = new ActiveEnumerable<IndexQueryResult>(indexQueryResults);
var transformerErrors = new List<string>();
var results = GetQueryResults(query, viewGenerator, docRetriever,
from queryResult in indexQueryResults
let doc = docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch)
where doc != null
let _ = nonAuthoritativeInformation |= (doc.NonAuthoritativeInformation ?? false)
let __ = tryRecordHighlighting(queryResult)
select doc, transformerErrors);
if (headerInfo != null)
{
headerInfo(new QueryHeaderInformation
{
Index = index,
IsStable = stale,
ResultEtag = resultEtag,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2,
TotalResults = query.TotalSize.Value
});
}
using (new CurrentTransformationScope(docRetriever))
{
foreach (var result in results)
{
onResult(result);
}
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
}
});
return new QueryResultWithIncludes
{
IndexName = index,
IsStale = stale,
NonAuthoritativeInformation = nonAuthoritativeInformation,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2,
ResultEtag = resultEtag,
IdsToInclude = idsToLoad,
LastQueryTime = SystemTime.UtcNow,
Highlightings = highlightings
};
}
public QueryResult Query(string index, IndexQuery query)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var list = new List<RavenJObject>();
var stale = false;
Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty);
TransactionalStorage.Batch(
actions =>
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
throw new InvalidOperationException("Could not find index named: " + index);
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers);
var indexDefinition = GetIndexDefinition(index);
var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
viewGenerator.ReduceDefinition == null
? Constants.DocumentIdFieldName
: Constants.ReduceKeyFieldName);
var collection = from queryResult in IndexStorage.Query(index, query, result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch), fieldsToFetch, IndexQueryTriggers)
select docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch)
into doc
where doc != null
select doc;
var transformerErrors = new List<string>();
IEnumerable<RavenJObject> results;
if (query.SkipTransformResults == false &&
query.PageSize > 0 && // maybe they just want the stats?
viewGenerator.TransformResultsDefinition != null)
{
var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.ToJson())).ToArray();
var robustEnumerator = new RobustEnumerator(dynamicJsonObjects.Length)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
results =
robustEnumerator.RobustEnumeration(
dynamicJsonObjects,
source => viewGenerator.TransformResultsDefinition(docRetriever, source))
.Select(JsonExtensions.ToJObject);
}
else
{
results = collection.Select(x => x.ToJson());
}
if (query.PageSize > 0) // maybe they just want the query stats?
list.AddRange(results);
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
});
return new QueryResult
{
IndexName = index,
Results = list,
IsStale = stale,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2
};
}
public QueryResultWithIncludes Query(string index, IndexQuery query)
{
index = IndexDefinitionStorage.FixupIndexName(index);
var list = new List<RavenJObject>();
var highlightings = new Dictionary<string, Dictionary<string, string[]>>();
var stale = false;
Tuple<DateTime, Guid> indexTimestamp = Tuple.Create(DateTime.MinValue, Guid.Empty);
Guid resultEtag = Guid.Empty;
var nonAuthoritativeInformation = false;
var idsToLoad = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);
TransactionalStorage.Batch(
actions =>
{
var viewGenerator = IndexDefinitionStorage.GetViewGenerator(index);
if (viewGenerator == null)
throw new IndexDoesNotExistsException("Could not find index named: " + index);
resultEtag = GetIndexEtag(index, null);
stale = actions.Staleness.IsIndexStale(index, query.Cutoff, query.CutoffEtag);
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index);
var indexFailureInformation = actions.Indexing.GetFailureRate(index);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
var docRetriever = new DocumentRetriever(actions, ReadTriggers, idsToLoad);
var indexDefinition = GetIndexDefinition(index);
var fieldsToFetch = new FieldsToFetch(query.FieldsToFetch, query.AggregationOperation,
viewGenerator.ReduceDefinition == null
? Constants.DocumentIdFieldName
: Constants.ReduceKeyFieldName);
Func<IndexQueryResult, bool> shouldIncludeInResults =
result => docRetriever.ShouldIncludeResultInQuery(result, indexDefinition, fieldsToFetch);
var collection =
from queryResult in
IndexStorage.Query(index, query, shouldIncludeInResults, fieldsToFetch, IndexQueryTriggers)
select new
{
Document = docRetriever.RetrieveDocumentForQuery(queryResult, indexDefinition, fieldsToFetch),
Fragments = queryResult.Highligtings
}
into docWithFragments
where docWithFragments.Document != null
let _ = nonAuthoritativeInformation |= (docWithFragments.Document.NonAuthoritativeInformation ?? false)
select docWithFragments;
var transformerErrors = new List<string>();
IEnumerable<RavenJObject> results;
if (query.SkipTransformResults == false &&
query.PageSize > 0 && // maybe they just want the stats?
viewGenerator.TransformResultsDefinition != null)
{
var dynamicJsonObjects = collection.Select(x => new DynamicJsonObject(x.Document.ToJson())).ToArray();
var robustEnumerator = new RobustEnumerator(workContext, dynamicJsonObjects.Length)
{
OnError =
(exception, o) =>
transformerErrors.Add(string.Format("Doc '{0}', Error: {1}", Index.TryGetDocKey(o),
exception.Message))
};
results =
robustEnumerator.RobustEnumeration(
dynamicJsonObjects.Cast<object>().GetEnumerator(),
source => viewGenerator.TransformResultsDefinition(docRetriever, source))
.Select(JsonExtensions.ToJObject);
}
else
{
var resultList = new List<RavenJObject>();
foreach (var docWithFragments in collection)
{
resultList.Add(docWithFragments.Document.ToJson());
if (docWithFragments.Fragments != null && docWithFragments.Document.Key != null)
highlightings.Add(docWithFragments.Document.Key, docWithFragments.Fragments);
}
results = resultList;
}
list.AddRange(results);
if (transformerErrors.Count > 0)
{
throw new InvalidOperationException("The transform results function failed.\r\n" + string.Join("\r\n", transformerErrors));
}
});
return new QueryResultWithIncludes
{
IndexName = index,
Results = list,
IsStale = stale,
NonAuthoritativeInformation = nonAuthoritativeInformation,
SkippedResults = query.SkippedResults.Value,
TotalResults = query.TotalSize.Value,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2,
ResultEtag = resultEtag,
IdsToInclude = idsToLoad,
LastQueryTime = SystemTime.UtcNow,
Highlightings = highlightings
};
}
public IndexQueryOperation(Index parent, IndexQuery indexQuery, Func<IndexQueryResult, bool> shouldIncludeInResults,
FieldsToFetch fieldsToFetch, OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers)
{
this.parent = parent;
this.indexQuery = indexQuery;
this.shouldIncludeInResults = shouldIncludeInResults;
this.fieldsToFetch = fieldsToFetch;
this.indexQueryTriggers = indexQueryTriggers;
if (fieldsToFetch.IsDistinctQuery)
alreadyReturned = new HashSet<RavenJObject>(new RavenJTokenEqualityComparer());
}
public IEnumerable<IndexQueryResult> Query(
string index,
IndexQuery query,
Func<IndexQueryResult, bool> shouldIncludeInResults,
FieldsToFetch fieldsToFetch)
{
Index value;
if (indexes.TryGetValue(index, out value) == false)
{
log.DebugFormat("Query on non existing index {0}", index);
throw new InvalidOperationException("Index " + index + " does not exists");
}
return new Index.IndexQueryOperation(value, query, shouldIncludeInResults, fieldsToFetch).Query();
}
public Query GetDocumentQuery(string index, IndexQuery query, OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers)
{
var value = TryIndexByName(index);
if (value == null)
{
log.Debug("Query on non existing index {0}", index);
throw new InvalidOperationException("Index '" + index + "' does not exists");
}
var fieldsToFetch = new FieldsToFetch(new string[0], false, null);
return new Index.IndexQueryOperation(value, query, _ => false, fieldsToFetch, indexQueryTriggers).GetDocumentQuery();
}
public void Init()
{
highlightings = new Dictionary<string, Dictionary<string, string[]>>();
scoreExplanations = new Dictionary<string, string>();
Func<IndexQueryResult, object> tryRecordHighlightingAndScoreExplanation = queryResult =>
{
if (queryResult.Highligtings != null && (queryResult.Key != null || queryResult.HighlighterKey != null))
highlightings.Add(queryResult.Key ?? queryResult.HighlighterKey, queryResult.Highligtings);
if ((queryResult.Key != null || queryResult.ReduceVal != null) && queryResult.ScoreExplanation != null)
scoreExplanations.Add(queryResult.Key ?? queryResult.ReduceVal, queryResult.ScoreExplanation);
return null;
};
stale = false;
indexTimestamp = Tuple.Create(DateTime.MinValue, Etag.Empty);
resultEtag = Etag.Empty;
nonAuthoritativeInformation = false;
if (string.IsNullOrEmpty(query.ResultsTransformer) == false &&
(query.FieldsToFetch == null || query.FieldsToFetch.Length == 0))
{
query.FieldsToFetch = new[] { Constants.AllFields };
}
duration = Stopwatch.StartNew();
idsToLoad = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var viewGenerator = database.IndexDefinitionStorage.GetViewGenerator(indexName);
var index = database.IndexDefinitionStorage.GetIndexDefinition(indexName);
if (viewGenerator == null)
throw new IndexDoesNotExistsException("Could not find index named: " + indexName);
resultEtag = database.Indexes.GetIndexEtag(index.Name, null, query.ResultsTransformer);
stale = actions.Staleness.IsIndexStale(index.IndexId, query.Cutoff, query.CutoffEtag);
if (stale == false && query.Cutoff == null && query.CutoffEtag == null)
{
var indexInstance = database.IndexStorage.GetIndexInstance(indexName);
stale = stale || (indexInstance != null && indexInstance.IsMapIndexingInProgress);
}
if (stale &&
actions.Staleness.IsIndexStaleByTask(index.IndexId, query.Cutoff) == false &&
actions.Staleness.IsReduceStale(index.IndexId) == false)
{
var forEntityNames = viewGenerator.ForEntityNames.ToList();
var lastIndexedEtag = actions.Indexing.GetIndexStats(index.IndexId).LastIndexedEtag;
if (database.LastCollectionEtags.HasEtagGreaterThan(forEntityNames, lastIndexedEtag) == false)
stale = false;
}
indexTimestamp = actions.Staleness.IndexLastUpdatedAt(index.IndexId);
var indexFailureInformation = actions.Indexing.GetFailureRate(index.IndexId);
if (indexFailureInformation.IsInvalidIndex)
{
throw new IndexDisabledException(indexFailureInformation);
}
docRetriever = new DocumentRetriever(database.Configuration, actions, database.ReadTriggers, database.InFlightTransactionalState, query.TransformerParameters, idsToLoad);
var fieldsToFetch = new FieldsToFetch(query,
viewGenerator.ReduceDefinition == null
? Constants.DocumentIdFieldName
: Constants.ReduceKeyFieldName);
Func<IndexQueryResult, bool> shouldIncludeInResults =
result => docRetriever.ShouldIncludeResultInQuery(result, index, fieldsToFetch, ShouldSkipDuplicateChecking);
var indexQueryResults = database.IndexStorage.Query(indexName, query, shouldIncludeInResults, fieldsToFetch, database.IndexQueryTriggers, cancellationToken);
indexQueryResults = new ActiveEnumerable<IndexQueryResult>(indexQueryResults);
if (query.ShowTimings)
indexQueryResults = new TimedEnumerable<IndexQueryResult>(indexQueryResults, timeInMilliseconds => executionTimes[QueryTimings.Lucene] = timeInMilliseconds);
var docs = from queryResult in indexQueryResults
let doc = docRetriever.RetrieveDocumentForQuery(queryResult, index, fieldsToFetch, ShouldSkipDuplicateChecking)
where doc != null
let _ = nonAuthoritativeInformation |= (doc.NonAuthoritativeInformation ?? false)
let __ = tryRecordHighlightingAndScoreExplanation(queryResult)
select doc;
transformerErrors = new List<string>();
results = database
.Queries
.GetQueryResults(query, viewGenerator, docRetriever, docs, transformerErrors,
timeInMilliseconds => executionTimes[QueryTimings.LoadDocuments] = timeInMilliseconds,
timeInMilliseconds => executionTimes[QueryTimings.TransformResults] = timeInMilliseconds,
query.ShowTimings, cancellationToken);
Header = new QueryHeaderInformation
{
Index = indexName,
IsStale = stale,
ResultEtag = resultEtag,
IndexTimestamp = indexTimestamp.Item1,
IndexEtag = indexTimestamp.Item2,
TotalResults = query.TotalSize.Value
};
}
public Query GetLuceneQuery(string index, IndexQuery query, OrderedPartCollection<AbstractIndexQueryTrigger> indexQueryTriggers)
{
Index value;
if (indexes.TryGetValue(index, out value) == false)
{
log.Debug("Query on non existing index {0}", index);
throw new InvalidOperationException("Index '" + index + "' does not exists");
}
var fieldsToFetch = new FieldsToFetch(new string[0], AggregationOperation.None, null);
return new Index.IndexQueryOperation(value, query, _ => false, fieldsToFetch, indexQueryTriggers).GetLuceneQuery();
}
public IndexQueryOperation(
Index parent,
IndexQuery indexQuery,
Func<IndexQueryResult, bool> shouldIncludeInResults,
FieldsToFetch fieldsToFetch)
{
this.parent = parent;
this.indexQuery = indexQuery;
this.shouldIncludeInResults = shouldIncludeInResults;
this.fieldsToFetch = fieldsToFetch;
if (fieldsToFetch.IsDistinctQuery)
alreadyReturned = new HashSet<JObject>(new JTokenEqualityComparer());
}
