public static Query BuildQuery(string query, IndexQuery indexQuery, PerFieldAnalyzerWrapper analyzer)
{
var originalQuery = query;
Analyzer keywordAnalyzer = new KeywordAnalyzer();
try
{
var queryParser = new RangeQueryParser(Version.LUCENE_29, indexQuery.DefaultField ?? string.Empty, analyzer)
{
DefaultOperator = indexQuery.DefaultOperator == QueryOperator.Or
? QueryParser.Operator.OR
: QueryParser.Operator.AND,
AllowLeadingWildcard = true
};
query = PreProcessUntokenizedTerms(query, queryParser);
query = PreProcessSearchTerms(query);
query = PreProcessDateTerms(query, queryParser);
var generatedQuery = queryParser.Parse(query);
generatedQuery = HandleMethods(generatedQuery);
return generatedQuery;
}
catch (ParseException pe)
{
if (originalQuery == query)
throw new ParseException("Could not parse: '" + query + "'", pe);
throw new ParseException("Could not parse modified query: '" + query + "' original was: '" + originalQuery + "'", pe);
}
finally
{
keywordAnalyzer.Close();
}
}
public static IList<string> GetRevisionsRequiringActivation(DocumentDatabase database)
{
var currentTime = SystemTime.UtcNow;
database.WaitForIndexToBecomeNonStale(TemporalConstants.PendingRevisionsIndex, currentTime, null);
const int pageSize = 1024;
var qs = string.Format("{0}:[* TO {1:o}]", Activation, currentTime);
var query = new IndexQuery {
Start = 0,
PageSize = pageSize,
Cutoff = currentTime,
Query = qs,
FieldsToFetch = new[] { Constants.DocumentIdFieldName },
SortedFields = new[] { new SortedField(Activation) }
};
var list = new List<string>();
while (true)
{
var results = database.Query(TemporalConstants.PendingRevisionsIndex, query).Results;
list.AddRange(results.Select(x => x.Value<string>(Constants.DocumentIdFieldName)));
if (results.Count < pageSize)
return list;
query.Start += pageSize;
}
}
/// <summary>
/// Initializes a new instance of the <see cref="SpatialIndexQuery"/> class.
/// </summary>
/// <param name="query">The query.</param>
public SpatialIndexQuery(IndexQuery query) : this()
{
Query = query.Query;
Start = query.Start;
Cutoff = query.Cutoff;
WaitForNonStaleResultsAsOfNow = query.WaitForNonStaleResultsAsOfNow;
CutoffEtag = query.CutoffEtag;
PageSize = query.PageSize;
FieldsToFetch = query.FieldsToFetch;
DefaultField = query.DefaultField; // keep?
DefaultOperator = query.DefaultOperator;
SortedFields = query.SortedFields;
HighlightedFields = query.HighlightedFields;
HighlighterPreTags = query.HighlighterPreTags;
HighlighterPostTags = query.HighlighterPostTags;
HighlighterKeyName = query.HighlighterKeyName;
ResultsTransformer = query.ResultsTransformer;
TransformerParameters = query.TransformerParameters;
ExplainScores = query.ExplainScores;
SortHints = query.SortHints;
IsDistinct = query.IsDistinct;
AllowMultipleIndexEntriesForSameDocumentToResultTransformer =
query.AllowMultipleIndexEntriesForSameDocumentToResultTransformer;
}
public void RestartNotCompletedOperations()
{
var lastEtag = db.GetLastEtag();
db.WaitForIndexNotStale(UpdateCascadeOperation.ByStatusIndexName, null, lastEtag, TimeSpan.FromHours(1), CancellationToken.None);
var indexQuery = new IndexQuery
{
CutoffEtag = lastEtag,
PageSize = 1024,
Query = "Status:Pending OR Status:Executing"
};
IList<string> docIds = null;
int restartedOperations = 0;
do
{
docIds = db.QueryDocumentIds(UpdateCascadeOperation.ByStatusIndexName, indexQuery, CancellationToken.None);
indexQuery.Start += docIds.Count;
db.TransactionalStorage.Batch(_ =>
{
foreach (var id in docIds)
{
var operation = repository.Get(id);
JsonDocument referenceDocument = null;
if (operation != null) referenceDocument = db.Get(operation.ReferencedDocId, null);
if (operation != null && referenceDocument != null && TryStartOperation(operation, referenceDocument))
{
restartedOperations++;
log.Debug("Operation {0} has been restarted after a reboot. {1} operations restarted so far", id, restartedOperations);
}
}
});
} while (docIds.Count == indexQuery.PageSize);
}
private void HandleTermsFacet(string index, Facet facet, IndexQuery indexQuery, IndexSearcher currentIndexSearcher, Dictionary<string, IEnumerable<FacetValue>> results)
{
var terms = database.ExecuteGetTermsQuery(index,
facet.Name, null,
database.Configuration.MaxPageSize);
var termResults = new List<FacetValue>();
var baseQuery = database.IndexStorage.GetLuceneQuery(index, indexQuery, database.IndexQueryTriggers);
foreach (var term in terms)
{
var termQuery = new TermQuery(new Term(facet.Name, term));
var joinedQuery = new BooleanQuery();
joinedQuery.Add(baseQuery, BooleanClause.Occur.MUST);
joinedQuery.Add(termQuery, BooleanClause.Occur.MUST);
var topDocs = currentIndexSearcher.Search(joinedQuery, null, 1);
if (topDocs.TotalHits > 0)
{
termResults.Add(new FacetValue
{
Count = topDocs.TotalHits,
Range = term
});
}
}
results[facet.Name] = termResults;
}
public void ShouldWork()
{
using (var store = NewDocumentStore(requestedStorage: "voron"))
{
using (var bulkInsert = store.BulkInsert())
{
for (int i = 0; i < 20000; i++)
{
bulkInsert.Store(new User { Id = i.ToString(), Name = "Name" + i });
}
}
WaitForIndexing(store);
var queryToDelete = new IndexQuery
{
Query = "Tag:Users"
};
var operation = store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", queryToDelete);
operation.WaitForCompletion();
using (var session = store.OpenSession())
{
var count = session
.Query<User>("Raven/DocumentsByEntityName")
.Customize(x => x.WaitForNonStaleResults())
.Count();
Assert.Equal(0, count);
}
}
}
public FacetResults GetFacets(string index, IndexQuery indexQuery, List<Facet> facets, int start = 0, int? pageSize = null)
{
var sp = Stopwatch.StartNew();
var results = new FacetResults();
var defaultFacets = new Dictionary<string, Facet>();
var rangeFacets = new Dictionary<string, List<ParsedRange>>();
var viewGenerator = database.IndexDefinitionStorage.GetViewGenerator(index);
Index.AssertQueryDoesNotContainFieldsThatAreNotIndexed(indexQuery, viewGenerator);
foreach (var facet in facets)
{
var key = string.IsNullOrWhiteSpace(facet.DisplayName) ? facet.Name : facet.DisplayName;
defaultFacets[key] = facet;
if (facet.Aggregation != FacetAggregation.Count && facet.Aggregation != FacetAggregation.None)
{
if (string.IsNullOrEmpty(facet.AggregationField))
throw new InvalidOperationException("Facet " + facet.Name + " cannot have aggregation set to " +
facet.Aggregation + " without having a value in AggregationField");
if (string.IsNullOrEmpty(facet.AggregationType))
throw new InvalidOperationException("Facet " + facet.Name + " cannot have aggregation set to " +
facet.Aggregation + " without having a value in AggregationType");
if (facet.AggregationField.EndsWith("_Range") == false)
{
if( QueryForFacets.IsAggregationTypeNumerical(facet.AggregationType))
facet.AggregationField = facet.AggregationField + "_Range";
}
}
switch (facet.Mode)
{
case FacetMode.Default:
results.Results[key] = new FacetResult();
break;
case FacetMode.Ranges:
rangeFacets[key] = facet.Ranges.Select(range => ParseRange(facet.Name, range)).ToList();
results.Results[key] = new FacetResult
{
Values = facet.Ranges.Select(range => new FacetValue
{
Range = range,
}).ToList()
};
break;
default:
throw new ArgumentException(string.Format("Could not understand '{0}'", facet.Mode));
}
}
var queryForFacets = new QueryForFacets(database, index, defaultFacets, rangeFacets, indexQuery, results, start, pageSize);
queryForFacets.Execute();
results.Duration = sp.Elapsed;
return results;
}
private static HashSet<string> GetFieldsInternal(IndexQuery query, Regex queryTerms)
{
var fields = new HashSet<string>();
if (string.IsNullOrEmpty(query.DefaultField) == false)
{
fields.Add(query.DefaultField);
}
if(query.Query == null)
return fields;
var dates = dateQuery.Matches(query.Query); // we need to exclude dates from this check
var queryTermMatches = queryTerms.Matches(query.Query);
for (int x = 0; x < queryTermMatches.Count; x++)
{
Match match = queryTermMatches[x];
String field = match.Groups[1].Value;
var isDate = false;
for (int i = 0; i < dates.Count; i++)
{
if(match.Index < dates[i].Index)
continue;
if (match.Index >= dates[i].Index + dates[i].Length)
continue;
isDate = true;
break;
}
if (isDate == false)
fields.Add(field);
}
return fields;
}
public LazyFacetsOperation( string index, string facetSetupDoc, IndexQuery query, int start = 0, int? pageSize = null ) {
this.index = index;
this.facetSetupDoc = facetSetupDoc;
this.query = query;
this.start = start;
this.pageSize = pageSize;
}
/// <summary>
/// Called when [delete].
/// </summary>
/// <param name="key">The key.</param>
/// <param name="transactionInformation">The transaction information.</param>
public override void OnDelete(string key, TransactionInformation transactionInformation)
{
if (key.StartsWith("Raven/")) // we don't deal with system documents
return;
var childrenQuery = new IndexQuery
{
Query = "Id:" + key
};
var queryResult = Database.Query("Documents/ByParent", childrenQuery);
if (queryResult.Results.Count > 0) {
foreach (var result in queryResult.Results) {
var metadataJObject = result.Value<RavenJObject>("@metadata");
if (metadataJObject != null)
{
var childId = metadataJObject.Value<string>("@id");
var childEtag = metadataJObject.Value<string>("@etag");
Database.Delete(childId, Guid.Parse(childEtag), transactionInformation);
}
}
}
base.OnDelete(key, transactionInformation);
}
protected void DoInResults(
Action<RavenJObject> action,
string query = "*:*",
int pageSize = 64,
string index = "Dynamic",
string[] includes = null)
{
int start = 0;
while (true)
{
var qry = new IndexQuery()
{
Query = query,
PageSize = pageSize,
Start = start //TODO: consider to use another thing in place of start
};
var results = DocumentStore.DatabaseCommands.Query(index, qry, includes);
if (results.Results.Count == 0)
break;
foreach (var result in results.Results)
action(result);
start += pageSize;
}
}
public static IndexQuery GetIndexQueryFromHttpContext(this IHttpContext context, int maxPageSize)
{
var query = new IndexQuery
{
Query = context.Request.QueryString["query"] ?? "",
Start = context.GetStart(),
Cutoff = context.GetCutOff(),
CutoffEtag = context.GetCutOffEtag(),
PageSize = context.GetPageSize(maxPageSize),
SkipTransformResults = context.GetSkipTransformResults(),
FieldsToFetch = context.Request.QueryString.GetValues("fetch"),
GroupBy = context.Request.QueryString.GetValues("groupBy"),
AggregationOperation = context.GetAggregationOperation(),
SortedFields = context.Request.QueryString.GetValues("sort")
.EmptyIfNull()
.Select(x => new SortedField(x))
.ToArray()
};
double lat = context.GetLat(), lng = context.GetLng(), radius = context.GetRadius();
if (lat != 0 || lng != 0 || radius != 0)
{
return new SpatialIndexQuery(query)
{
Latitude = lat,
Longitude = lng,
Radius = radius,
};
}
return query;
}
private void HandleRangeFacet(string index, Facet facet, IndexQuery indexQuery, IndexSearcher currentIndexSearcher, Dictionary<string, IEnumerable<FacetValue>> results)
{
var rangeResults = new List<FacetValue>();
foreach (var range in facet.Ranges)
{
var baseQuery = database.IndexStorage.GetLuceneQuery(index, indexQuery, database.IndexQueryTriggers);
//TODO the built-in parser can't handle [NULL TO 100.0}, i.e. a mix of [ and }
//so we need to handle this ourselves (greater and less-than-or-equal)
var rangeQuery = database.IndexStorage.GetLuceneQuery(index, new IndexQuery
{
Query = facet.Name + ":" + range
}, database.IndexQueryTriggers);
var joinedQuery = new BooleanQuery();
joinedQuery.Add(baseQuery, BooleanClause.Occur.MUST);
joinedQuery.Add(rangeQuery, BooleanClause.Occur.MUST);
var topDocs = currentIndexSearcher.Search(joinedQuery, null, 1);
if (topDocs.TotalHits > 0)
{
rangeResults.Add(new FacetValue
{
Count = topDocs.TotalHits,
Range = range
});
}
}
results[facet.Name] = rangeResults;
}
public IDictionary<string, IEnumerable<FacetValue>> GetFacets(string index, IndexQuery indexQuery, string facetSetupDoc)
{
var facetSetup = database.Get(facetSetupDoc, null);
if (facetSetup == null)
throw new InvalidOperationException("Could not find facets document: " + facetSetupDoc);
var facets = facetSetup.DataAsJson.JsonDeserialization<FacetSetup>().Facets;
var results = new Dictionary<string, IEnumerable<FacetValue>>();
IndexSearcher currentIndexSearcher;
using (database.IndexStorage.GetCurrentIndexSearcher(index, out currentIndexSearcher))
{
foreach (var facet in facets)
{
switch (facet.Mode)
{
case FacetMode.Default:
HandleTermsFacet(index, facet, indexQuery, currentIndexSearcher, results);
break;
case FacetMode.Ranges:
HandleRangeFacet(index, facet, indexQuery, currentIndexSearcher, results);
break;
default:
throw new ArgumentException(string.Format("Could not understand '{0}'", facet.Mode));
}
}
}
return results;
}
public static IndexQuery FromQueryString(string queryString)
{
var fields = queryString.Split(new[] {'&'}, StringSplitOptions.RemoveEmptyEntries)
.Select(segment =>
{
var parts = segment.Split(new[] {'='}, StringSplitOptions.RemoveEmptyEntries);
if (parts.Length == 1)
{
return new {Key = parts[0], Value = string.Empty};
}
else
{
return
new
{
Key = parts[0],
Value = Uri.UnescapeDataString(parts[1])
};
}
}).ToLookup(f => f.Key, f => f.Value);
var query = new IndexQuery
{
Query = Uri.UnescapeDataString(fields["query"].FirstOrDefault() ?? ""),
Start = fields.GetStart(),
Cutoff = fields.GetCutOff(),
CutoffEtag = fields.GetCutOffEtag(),
PageSize = fields.GetPageSize(250),
SkipTransformResults = fields.GetSkipTransformResults(),
FieldsToFetch = fields["fetch"].ToArray(),
GroupBy = fields["groupBy"].ToArray(),
AggregationOperation = fields.GetAggregationOperation(),
SortedFields = fields["sort"]
.EmptyIfNull()
.Select(x => new SortedField(x))
.ToArray(),
SortByAggregation = SortFieldAggregation.Default
};
SortFieldAggregation sortFieldAggregation;
var sortAggregationQuerystringValue = fields["sortAggregation"].FirstOrDefault();
if (!string.IsNullOrEmpty(sortAggregationQuerystringValue) &&
SortFieldAggregation.TryParse(sortAggregationQuerystringValue, out sortFieldAggregation))
{
query.SortByAggregation = sortFieldAggregation;
}
double lat = fields.GetLat(), lng = fields.GetLng(), radius = fields.GetRadius();
if (lat != 0 || lng != 0 || radius != 0)
{
return new SpatialIndexQuery(query)
{
QueryShape = SpatialIndexQuery.GetQueryShapeFromLatLon(lat, lng, radius),
SpatialRelation = SpatialRelation.Within, /* TODO */
SpatialFieldName = Constants.DefaultSpatialFieldName, /* TODO */
};
}
return query;
}
public LazyFacetsOperation(string index, IEnumerable<Facet> facets, IndexQuery query, int start = 0, int? pageSize = null)
{
this.index = index;
this.facets = facets;
this.query = query;
this.start = start;
this.pageSize = pageSize;
}
public RavenJArray UpdateByIndex(string indexName, IndexQuery queryToUpdate, ScriptedPatchRequest patch, BulkOperationOptions options = null)
{
return PerformBulkOperation(indexName, queryToUpdate, options, (docId, tx) =>
{
var patchResult = database.Patches.ApplyPatch(docId, null, patch, tx);
return new { Document = docId, Result = patchResult.Item1, Debug = patchResult.Item2 };
});
}
public static Filter MakeFilter(SpatialStrategy spatialStrategy, IndexQuery indexQuery)
{
var spatialQry = indexQuery as SpatialIndexQuery;
if (spatialQry == null) return null;
var args = new SpatialArgs(SpatialOperation.IsWithin, ShapeReadWriter.ReadShape(spatialQry.QueryShape));
return spatialStrategy.MakeFilter(args);
}
public RavenJArray DeleteByIndex(string indexName, IndexQuery queryToDelete, BulkOperationOptions options = null)
{
return PerformBulkOperation(indexName, queryToDelete, options, (docId, tx) =>
{
database.Documents.Delete(docId, null, tx);
return new { Document = docId, Deleted = true };
});
}
public static Filter MakeFilter(IndexQuery indexQuery)
{
var spatialQry = indexQuery as SpatialIndexQuery;
if (spatialQry == null) return null;
var args = new SpatialArgs(SpatialOperation.IsWithin, RavenSpatialContext.MakeCircle(spatialQry.Longitude, spatialQry.Latitude, spatialQry.Radius));
return strategy.MakeFilter(args, fieldInfo);
}
private Task<QueryResult> QueryIndexForDocument()
{
var query = new IndexQuery();
query.Start = itemIndex;
query.PageSize = 1;
query.SkipTransformResults = true;
return DatabaseCommands.QueryAsync(ConflictsIndexName, query, null);
}
/// <summary>
/// Initializes a new instance of the <see cref="SpatialIndexQuery"/> class.
/// </summary>
/// <param name="query">The query.</param>
public SpatialIndexQuery(IndexQuery query)
{
Query = query.Query;
Start = query.Start;
Cutoff = query.Cutoff;
PageSize = query.PageSize;
FieldsToFetch = query.FieldsToFetch;
SortedFields = query.SortedFields;
}
public static FacetResults ExecuteGetTermsQuery(this DocumentDatabase self, string index, IndexQuery query, string facetSetupDoc, int start, int? pageSize)
{
var facetSetup = self.Documents.Get(facetSetupDoc, null);
if (facetSetup == null)
throw new InvalidOperationException("Could not find facets document: " + facetSetupDoc);
var facets = facetSetup.DataAsJson.JsonDeserialization<FacetSetup>().Facets;
return self.ExecuteGetTermsQuery(index, query, facets, start, pageSize);
}
public void UpdateQuery(string indexName, IndexQuery templateQuery)
{
lock (_lockObject)
{
_templateQuery = templateQuery;
_indexName = indexName;
}
Refresh(RefreshMode.ClearStaleData);
}
/// <summary>
/// Initializes a new instance of the <see cref="SpatialIndexQuery"/> class.
/// </summary>
/// <param name="query">The query.</param>
public SpatialIndexQuery(IndexQuery query) : this()
{
Query = query.Query;
Start = query.Start;
Cutoff = query.Cutoff;
PageSize = query.PageSize;
FieldsToFetch = query.FieldsToFetch;
SortedFields = query.SortedFields;
SortByAggregation = query.SortByAggregation;
}
protected override System.Threading.Tasks.Task TimerTickedAsync()
{
var query = new IndexQuery { Start = GetSkipCount(), PageSize = 25, Query = "Tag:" + Name };
return databaseCommands
.QueryAsync("Raven/DocumentsByEntityName", query, new string[] {})
.ContinueOnSuccess(queryResult =>
{
var documents = SerializationHelper.RavenJObjectsToJsonDocuments(queryResult.Results);
Documents.Match(documents.Select(x=>new ViewableDocument(x)).ToArray());
});
}
public static HashSet<Tuple<string,string>> GetFieldsForDynamicQuery(IndexQuery query)
{
var results = new HashSet<Tuple<string,string>>();
foreach (var result in GetFieldsInternal(query, dynamicQueryTerms))
{
if(result == "*")
continue;
results.Add(Tuple.Create(TranslateField(result), result));
}
return results;
}
/// <summary>
/// Initializes a new instance of the <see cref="SpatialIndexQuery"/> class.
/// </summary>
/// <param name="query">The query.</param>
public SpatialIndexQuery(IndexQuery query) : this()
{
Query = query.Query;
Start = query.Start;
Cutoff = query.Cutoff;
PageSize = query.PageSize;
FieldsToFetch = query.FieldsToFetch;
SortedFields = query.SortedFields;
HighlighterPreTags = query.HighlighterPreTags;
HighlighterPostTags = query.HighlighterPostTags;
HighlightedFields = query.HighlightedFields;
}
public IndexDocumentsNavigator(string id, int itemIndex, string indexName, IndexQuery templateQuery)
{
this.id = id;
this.itemIndex = itemIndex;
this.indexName = indexName;
// canonicalize the template query to help keep the url more stable
templateQuery.Start = 0;
templateQuery.PageSize = 1;
this.templateQuery = templateQuery;
}
public static IndexQuery FromQueryString(string queryString)
{
var fields = queryString.Split(new[] {'&'}, StringSplitOptions.RemoveEmptyEntries)
.Select(segment =>
{
var parts = segment.Split(new[] {'='}, StringSplitOptions.RemoveEmptyEntries);
if (parts.Length == 1)
{
return new {Key = parts[0], Value = string.Empty};
}
else
{
return
new
{
Key = parts[0],
Value = Uri.UnescapeDataString(parts[1])
};
}
}).ToLookup(f => f.Key, f => f.Value);
var query = new IndexQuery
{
Query = Uri.UnescapeDataString(fields["query"].FirstOrDefault() ?? ""),
Start = fields.GetStart(),
Cutoff = fields.GetCutOff(),
CutoffEtag = fields.GetCutOffEtag(),
PageSize = fields.GetPageSize(250),
SkipTransformResults = fields.GetSkipTransformResults(),
FieldsToFetch = fields["fetch"].ToArray(),
GroupBy = fields["groupBy"].ToArray(),
AggregationOperation = fields.GetAggregationOperation(),
SortedFields = fields["sort"]
.EmptyIfNull()
.Select(x => new SortedField(x))
.ToArray()
};
double lat = fields.GetLat(), lng = fields.GetLng(), radius = fields.GetRadius();
if (lat != 0 || lng != 0 || radius != 0)
{
return new SpatialIndexQuery(query)
{
Latitude = lat,
Longitude = lng,
Radius = radius,
};
}
return query;
}
