public JsonDocument RetrieveDocumentForQuery(IndexQueryResult queryResult, IndexDefinition indexDefinition, FieldsToFetch fieldsToFetch)
{
return(ExecuteReadTriggers(ProcessReadVetoes(
RetrieveDocumentInternal(queryResult, loadedIdsForRetrieval, fieldsToFetch, indexDefinition),
null, ReadOperation.Query), null, ReadOperation.Query));
}
private JsonDocument RetrieveDocumentInternal(
IndexQueryResult queryResult,
HashSet <string> loadedIds,
FieldsToFetch fieldsToFetch,
IndexDefinition indexDefinition)
{
var queryScore = queryResult.Score;
if (float.IsNaN(queryScore))
{
queryScore = 0f;
}
if (queryResult.Projection == null)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
{
return(null);
}
var document = GetDocumentWithCaching(queryResult.Key);
if (document != null)
{
document.Metadata[Constants.TemporaryScoreValue] = queryScore;
}
return(document);
}
JsonDocument doc = null;
if (fieldsToFetch.IsProjection)
{
if (indexDefinition.IsMapReduce == false)
{
bool hasStoredFields = false;
FieldStorage value;
if (indexDefinition.Stores.TryGetValue(Constants.AllFields, out value))
{
hasStoredFields = value != FieldStorage.No;
}
foreach (var fieldToFetch in fieldsToFetch.Fields)
{
if (indexDefinition.Stores.TryGetValue(fieldToFetch, out value) == false && value != FieldStorage.No)
{
continue;
}
hasStoredFields = true;
}
if (hasStoredFields == false)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
{
return(null);
}
}
}
// We have to load the document if user explicitly asked for the id, since
// we normalize the casing for the document id on the index, and we need to return
// the id to the user with the same casing they gave us.
var fetchingId = fieldsToFetch.HasField(Constants.DocumentIdFieldName);
var fieldsToFetchFromDocument = fieldsToFetch.Fields.Where(fieldToFetch => queryResult.Projection[fieldToFetch] == null).ToArray();
if (fieldsToFetchFromDocument.Length > 0 || fetchingId)
{
doc = GetDocumentWithCaching(queryResult.Key);
if (doc != null)
{
if (fetchingId)
{
queryResult.Projection[Constants.DocumentIdFieldName] = doc.Key;
}
var result = doc.DataAsJson.SelectTokenWithRavenSyntax(fieldsToFetchFromDocument.ToArray());
foreach (var property in result)
{
if (property.Value == null || property.Value.Type == JTokenType.Null)
{
continue;
}
queryResult.Projection[property.Key] = property.Value;
}
}
}
}
else if (fieldsToFetch.FetchAllStoredFields && string.IsNullOrEmpty(queryResult.Key) == false)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
{
return(null);
}
doc = GetDocumentWithCaching(queryResult.Key);
}
var metadata = GetMetadata(doc);
metadata.Remove("@id");
metadata[Constants.TemporaryScoreValue] = queryScore;
return(new JsonDocument
{
Key = queryResult.Key,
DataAsJson = queryResult.Projection,
Metadata = metadata
});
}
private JsonDocument RetrieveDocumentInternal(
IndexQueryResult queryResult,
HashSet <string> loadedIds,
FieldsToFetch fieldsToFetch,
IndexDefinition indexDefinition)
{
if (queryResult.Projection == null)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
{
return(null);
}
var document = GetDocumentWithCaching(queryResult.Key);
if (document != null)
{
document.Metadata[Constants.TemporaryScoreValue] = queryResult.Score;
}
return(document);
}
if (fieldsToFetch.IsProjection)
{
if (indexDefinition.IsMapReduce == false)
{
bool hasStoredFields = false;
foreach (var fieldToFetch in fieldsToFetch)
{
FieldStorage value;
if (indexDefinition.Stores.TryGetValue(fieldToFetch, out value) == false &&
value != FieldStorage.No)
{
continue;
}
hasStoredFields = true;
}
if (hasStoredFields == false)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
{
return(null);
}
}
}
var fieldsToFetchFromDocument = fieldsToFetch.Where(fieldToFetch => queryResult.Projection[fieldToFetch] == null);
var doc = GetDocumentWithCaching(queryResult.Key);
if (doc != null)
{
var result = doc.DataAsJson.SelectTokenWithRavenSyntax(fieldsToFetchFromDocument.ToArray());
foreach (var property in result)
{
if (property.Value == null || property.Value.Type == JTokenType.Null)
{
continue;
}
queryResult.Projection[property.Key] = property.Value;
}
}
}
return(new JsonDocument
{
Key = queryResult.Key,
DataAsJson = queryResult.Projection,
Metadata = new RavenJObject {
{ Constants.TemporaryScoreValue, queryResult.Score }
}
});
}
private JsonDocument RetrieveDocumentInternal(
IndexQueryResult queryResult,
HashSet <string> loadedIds,
FieldsToFetch fieldsToFetch,
IndexDefinition indexDefinition,
bool skipDuplicateCheck)
{
var queryScore = queryResult.Score;
if (float.IsNaN(queryScore))
{
queryScore = 0f;
}
if (queryResult.Projection == null)
{
// duplicate document, filter it out
if (skipDuplicateCheck == false && loadedIds.Add(queryResult.Key) == false)
{
return(null);
}
var document = GetDocumentWithCaching(queryResult);
if (document == null)
{
return(null);
}
document.Metadata = GetMetadata(document);
if (skipDuplicateCheck == false)
{
document.Metadata[Constants.TemporaryScoreValue] = queryScore;
}
return(document);
}
JsonDocument doc = null;
if (fieldsToFetch.IsProjection)
{
if (indexDefinition.IsMapReduce == false)
{
bool hasStoredFields = false;
FieldStorage value;
if (indexDefinition.Stores.TryGetValue(Constants.AllFields, out value))
{
hasStoredFields = value != FieldStorage.No;
}
foreach (var fieldToFetch in fieldsToFetch.Fields)
{
if (indexDefinition.Stores.TryGetValue(fieldToFetch, out value) == false && value != FieldStorage.No)
{
continue;
}
hasStoredFields = true;
}
if (hasStoredFields == false)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
{
return(null);
}
}
}
// We have to load the document if user explicitly asked for the id, since
// we normalize the casing for the document id on the index, and we need to return
// the id to the user with the same casing they gave us.
var fetchingId = fieldsToFetch.HasField(Constants.DocumentIdFieldName);
var fieldsToFetchFromDocument = fieldsToFetch.Fields.Where(fieldToFetch => queryResult.Projection[fieldToFetch] == null).ToArray();
if (fieldsToFetchFromDocument.Length > 0 || fetchingId)
{
switch (configuration.ImplicitFetchFieldsFromDocumentMode)
{
case ImplicitFetchFieldsMode.Enabled:
doc = GetDocumentWithCaching(queryResult);
if (doc != null)
{
if (fetchingId)
{
queryResult.Projection[Constants.DocumentIdFieldName] = doc.Key;
}
var result = doc.DataAsJson.SelectTokenWithRavenSyntax(fieldsToFetchFromDocument.ToArray());
foreach (var property in result)
{
if (property.Value == null || property.Value.Type == JTokenType.Null)
{
continue;
}
queryResult.Projection[property.Key] = property.Value;
}
}
break;
case ImplicitFetchFieldsMode.DoNothing:
break;
case ImplicitFetchFieldsMode.Exception:
string message = string.Format("Implicit fetching of fields from the document is disabled." + Environment.NewLine +
"Check your index ({0}) to make sure that all fields you want to project are stored in the index." + Environment.NewLine +
"You can control this behavior using the Raven/ImplicitFetchFieldsFromDocumentMode setting." + Environment.NewLine +
"Fields to fetch from document are: {1}" + Environment.NewLine +
"Fetching id: {2}", indexDefinition.Name, string.Join(", ", fieldsToFetchFromDocument), fetchingId);
throw new ImplicitFetchFieldsFromDocumentNotAllowedException(message);
default:
throw new ArgumentOutOfRangeException(configuration.ImplicitFetchFieldsFromDocumentMode.ToString());
}
}
}
else if (fieldsToFetch.FetchAllStoredFields && string.IsNullOrEmpty(queryResult.Key) == false &&
(fieldsToFetch.Query == null || fieldsToFetch.Query.AllowMultipleIndexEntriesForSameDocumentToResultTransformer == false)
)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
{
return(null);
}
doc = GetDocumentWithCaching(queryResult);
}
var metadata = GetMetadata(doc);
metadata.Remove("@id");
metadata[Constants.TemporaryScoreValue] = queryScore;
return(new JsonDocument
{
Key = queryResult.Key,
DataAsJson = queryResult.Projection,
Metadata = metadata
});
}
private JsonDocument RetrieveDocumentInternal(
IndexQueryResult queryResult,
HashSet <string> loadedIds,
IEnumerable <string> fieldsToFetch,
IndexDefinition indexDefinition,
AggregationOperation aggregationOperation)
{
if (queryResult.Projection == null)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
{
return(null);
}
return(GetDocumentWithCaching(queryResult.Key));
}
if (fieldsToFetch != null)
{
if (indexDefinition.IsMapReduce == false)
{
bool hasStoredFields = false;
foreach (var fieldToFetch in fieldsToFetch)
{
FieldStorage value;
if (indexDefinition.Stores.TryGetValue(fieldToFetch, out value) == false &&
value != FieldStorage.No)
{
continue;
}
hasStoredFields = true;
}
if (hasStoredFields == false)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
{
return(null);
}
}
}
if (aggregationOperation != AggregationOperation.None)
{
var aggOpr = aggregationOperation & ~AggregationOperation.Dynamic;
fieldsToFetch = fieldsToFetch.Concat(new[] { aggOpr.ToString() });
}
var fieldsToFetchFromDocument = fieldsToFetch.Where(fieldToFetch => queryResult.Projection.Property(fieldToFetch) == null);
var doc = GetDocumentWithCaching(queryResult.Key);
if (doc != null)
{
var result = doc.DataAsJson.SelectTokenWithRavenSyntax(fieldsToFetchFromDocument.ToArray());
foreach (var property in result.Properties())
{
if (property.Value == null || property.Value.Type == JTokenType.Null)
{
continue;
}
queryResult.Projection[property.Name] = property.Value;
}
}
}
return(new JsonDocument
{
Key = queryResult.Key,
Projection = queryResult.Projection,
});
}
public IEnumerable <IndexQueryResult> IntersectionQuery()
{
using (IndexStorage.EnsureInvariantCulture())
{
AssertQueryDoesNotContainFieldsThatAreNotIndexes();
IndexSearcher indexSearcher;
using (parent.GetSearcher(out indexSearcher))
{
int pageSizeBestGuess = (indexQuery.Start + indexQuery.PageSize) * 2;
int returnedResults = 0;
int skippedResultsInCurrentLoop = 0;
int previousIntersectMatches = 0;
var subQueries = indexQuery.Query.Split(new[] { Constants.IntersectSeperator }, StringSplitOptions.RemoveEmptyEntries);
if (subQueries.Length <= 1)
{
throw new InvalidOperationException("Invalid INTRESECT query, must have multiple intersect clauses.");
}
//Do the first sub-query in the normal way, so that sorting, filtering etc is accounted for
var firstSubLuceneQuery = ApplyIndexTriggers(GetLuceneQuery(subQueries[0]));
//Not sure how to select the page size here??? The problem is that only docs in this search can be part
//of the final result because we're doing an intersection query (but we might exclude some of them)
var search = ExecuteQuery(indexSearcher, firstSubLuceneQuery, 0, pageSizeBestGuess, indexQuery);
var intersectionCollector = new IntersectionCollector(indexSearcher, search.ScoreDocs);
var intersectMatches = 0;
//Keep going until we've pulled through enough intersecting docs to satisfy pageSize + start
//OR the current loop doesn't get us any more results, despite increasing the page size
do
{
if (skippedResultsInCurrentLoop > 0)
{
// We get here because out first attempt didn't get enough docs (after INTERSECTION was calculated)
pageSizeBestGuess = pageSizeBestGuess * 2;
search = ExecuteQuery(indexSearcher, firstSubLuceneQuery, 0, pageSizeBestGuess, indexQuery);
intersectionCollector = new IntersectionCollector(indexSearcher, search.ScoreDocs);
}
Filter filter = indexQuery.GetFilter();
for (int i = 1; i < subQueries.Length; i++)
{
var luceneSubQuery = ApplyIndexTriggers(GetLuceneQuery(subQueries[i]));
indexSearcher.Search(luceneSubQuery, filter, intersectionCollector);
}
var currentIntersectResults = intersectionCollector.DocumentsIdsForCount(subQueries.Length).ToList();
previousIntersectMatches = intersectMatches;
intersectMatches = currentIntersectResults.Count;
skippedResultsInCurrentLoop = pageSizeBestGuess - intersectMatches;
} while (previousIntersectMatches < intersectMatches && intersectMatches < indexQuery.PageSize);
var intersectResults = intersectionCollector.DocumentsIdsForCount(subQueries.Length).ToList();
//It's hard to know what to do here, the TotalHits from the base search isn't really the TotalSize,
//because it's before the INTERSECTION has been applied, so only some of those results make it out.
//Trying to give an accurate answer is going to be too costly, so we aren't going to try.
indexQuery.TotalSize.Value = search.TotalHits;
indexQuery.SkippedResults.Value = skippedResultsInCurrentLoop;
//Using the final set of results in the intersectionCollector
for (int i = indexQuery.Start; i < intersectResults.Count && (i - indexQuery.Start) < pageSizeBestGuess; i++)
{
Document document = indexSearcher.Doc(intersectResults[i].LuceneId);
IndexQueryResult indexQueryResult = parent.RetrieveDocument(document, fieldsToFetch, search.ScoreDocs[i].score);
if (ShouldIncludeInResults(indexQueryResult) == false)
{
indexQuery.SkippedResults.Value++;
skippedResultsInCurrentLoop++;
continue;
}
returnedResults++;
yield return(indexQueryResult);
if (returnedResults == indexQuery.PageSize)
{
yield break;
}
}
}
}
}
