public static Query BuildQuery(string query, IndexQuery indexQuery, PerFieldAnalyzerWrapper analyzer) { var originalQuery = query; Analyzer keywordAnalyzer = new KeywordAnalyzer(); try { var queryParser = new RangeQueryParser(Version.LUCENE_29, indexQuery.DefaultField ?? string.Empty, analyzer) { DefaultOperator = indexQuery.DefaultOperator == QueryOperator.Or ? QueryParser.Operator.OR : QueryParser.Operator.AND, AllowLeadingWildcard = true }; query = PreProcessUntokenizedTerms(query, queryParser); query = PreProcessSearchTerms(query); query = PreProcessDateTerms(query, queryParser); var generatedQuery = queryParser.Parse(query); generatedQuery = HandleMethods(generatedQuery); return generatedQuery; } catch (ParseException pe) { if (originalQuery == query) throw new ParseException("Could not parse: '" + query + "'", pe); throw new ParseException("Could not parse modified query: '" + query + "' original was: '" + originalQuery + "'", pe); } finally { keywordAnalyzer.Close(); } }
public static IList<string> GetRevisionsRequiringActivation(DocumentDatabase database) { var currentTime = SystemTime.UtcNow; database.WaitForIndexToBecomeNonStale(TemporalConstants.PendingRevisionsIndex, currentTime, null); const int pageSize = 1024; var qs = string.Format("{0}:[* TO {1:o}]", Activation, currentTime); var query = new IndexQuery { Start = 0, PageSize = pageSize, Cutoff = currentTime, Query = qs, FieldsToFetch = new[] { Constants.DocumentIdFieldName }, SortedFields = new[] { new SortedField(Activation) } }; var list = new List<string>(); while (true) { var results = database.Query(TemporalConstants.PendingRevisionsIndex, query).Results; list.AddRange(results.Select(x => x.Value<string>(Constants.DocumentIdFieldName))); if (results.Count < pageSize) return list; query.Start += pageSize; } }
/// <summary> /// Initializes a new instance of the <see cref="SpatialIndexQuery"/> class. /// </summary> /// <param name="query">The query.</param> public SpatialIndexQuery(IndexQuery query) : this() { Query = query.Query; Start = query.Start; Cutoff = query.Cutoff; WaitForNonStaleResultsAsOfNow = query.WaitForNonStaleResultsAsOfNow; CutoffEtag = query.CutoffEtag; PageSize = query.PageSize; FieldsToFetch = query.FieldsToFetch; DefaultField = query.DefaultField; // keep? DefaultOperator = query.DefaultOperator; SortedFields = query.SortedFields; HighlightedFields = query.HighlightedFields; HighlighterPreTags = query.HighlighterPreTags; HighlighterPostTags = query.HighlighterPostTags; HighlighterKeyName = query.HighlighterKeyName; ResultsTransformer = query.ResultsTransformer; TransformerParameters = query.TransformerParameters; ExplainScores = query.ExplainScores; SortHints = query.SortHints; IsDistinct = query.IsDistinct; AllowMultipleIndexEntriesForSameDocumentToResultTransformer = query.AllowMultipleIndexEntriesForSameDocumentToResultTransformer; }
public void RestartNotCompletedOperations() { var lastEtag = db.GetLastEtag(); db.WaitForIndexNotStale(UpdateCascadeOperation.ByStatusIndexName, null, lastEtag, TimeSpan.FromHours(1), CancellationToken.None); var indexQuery = new IndexQuery { CutoffEtag = lastEtag, PageSize = 1024, Query = "Status:Pending OR Status:Executing" }; IList<string> docIds = null; int restartedOperations = 0; do { docIds = db.QueryDocumentIds(UpdateCascadeOperation.ByStatusIndexName, indexQuery, CancellationToken.None); indexQuery.Start += docIds.Count; db.TransactionalStorage.Batch(_ => { foreach (var id in docIds) { var operation = repository.Get(id); JsonDocument referenceDocument = null; if (operation != null) referenceDocument = db.Get(operation.ReferencedDocId, null); if (operation != null && referenceDocument != null && TryStartOperation(operation, referenceDocument)) { restartedOperations++; log.Debug("Operation {0} has been restarted after a reboot. {1} operations restarted so far", id, restartedOperations); } } }); } while (docIds.Count == indexQuery.PageSize); }
private void HandleTermsFacet(string index, Facet facet, IndexQuery indexQuery, IndexSearcher currentIndexSearcher, Dictionary<string, IEnumerable<FacetValue>> results) { var terms = database.ExecuteGetTermsQuery(index, facet.Name, null, database.Configuration.MaxPageSize); var termResults = new List<FacetValue>(); var baseQuery = database.IndexStorage.GetLuceneQuery(index, indexQuery, database.IndexQueryTriggers); foreach (var term in terms) { var termQuery = new TermQuery(new Term(facet.Name, term)); var joinedQuery = new BooleanQuery(); joinedQuery.Add(baseQuery, BooleanClause.Occur.MUST); joinedQuery.Add(termQuery, BooleanClause.Occur.MUST); var topDocs = currentIndexSearcher.Search(joinedQuery, null, 1); if (topDocs.TotalHits > 0) { termResults.Add(new FacetValue { Count = topDocs.TotalHits, Range = term }); } } results[facet.Name] = termResults; }
public void ShouldWork() { using (var store = NewDocumentStore(requestedStorage: "voron")) { using (var bulkInsert = store.BulkInsert()) { for (int i = 0; i < 20000; i++) { bulkInsert.Store(new User { Id = i.ToString(), Name = "Name" + i }); } } WaitForIndexing(store); var queryToDelete = new IndexQuery { Query = "Tag:Users" }; var operation = store.DatabaseCommands.DeleteByIndex("Raven/DocumentsByEntityName", queryToDelete); operation.WaitForCompletion(); using (var session = store.OpenSession()) { var count = session .Query<User>("Raven/DocumentsByEntityName") .Customize(x => x.WaitForNonStaleResults()) .Count(); Assert.Equal(0, count); } } }
public FacetResults GetFacets(string index, IndexQuery indexQuery, List<Facet> facets, int start = 0, int? pageSize = null) { var sp = Stopwatch.StartNew(); var results = new FacetResults(); var defaultFacets = new Dictionary<string, Facet>(); var rangeFacets = new Dictionary<string, List<ParsedRange>>(); var viewGenerator = database.IndexDefinitionStorage.GetViewGenerator(index); Index.AssertQueryDoesNotContainFieldsThatAreNotIndexed(indexQuery, viewGenerator); foreach (var facet in facets) { var key = string.IsNullOrWhiteSpace(facet.DisplayName) ? facet.Name : facet.DisplayName; defaultFacets[key] = facet; if (facet.Aggregation != FacetAggregation.Count && facet.Aggregation != FacetAggregation.None) { if (string.IsNullOrEmpty(facet.AggregationField)) throw new InvalidOperationException("Facet " + facet.Name + " cannot have aggregation set to " + facet.Aggregation + " without having a value in AggregationField"); if (string.IsNullOrEmpty(facet.AggregationType)) throw new InvalidOperationException("Facet " + facet.Name + " cannot have aggregation set to " + facet.Aggregation + " without having a value in AggregationType"); if (facet.AggregationField.EndsWith("_Range") == false) { if( QueryForFacets.IsAggregationTypeNumerical(facet.AggregationType)) facet.AggregationField = facet.AggregationField + "_Range"; } } switch (facet.Mode) { case FacetMode.Default: results.Results[key] = new FacetResult(); break; case FacetMode.Ranges: rangeFacets[key] = facet.Ranges.Select(range => ParseRange(facet.Name, range)).ToList(); results.Results[key] = new FacetResult { Values = facet.Ranges.Select(range => new FacetValue { Range = range, }).ToList() }; break; default: throw new ArgumentException(string.Format("Could not understand '{0}'", facet.Mode)); } } var queryForFacets = new QueryForFacets(database, index, defaultFacets, rangeFacets, indexQuery, results, start, pageSize); queryForFacets.Execute(); results.Duration = sp.Elapsed; return results; }
private static HashSet<string> GetFieldsInternal(IndexQuery query, Regex queryTerms) { var fields = new HashSet<string>(); if (string.IsNullOrEmpty(query.DefaultField) == false) { fields.Add(query.DefaultField); } if(query.Query == null) return fields; var dates = dateQuery.Matches(query.Query); // we need to exclude dates from this check var queryTermMatches = queryTerms.Matches(query.Query); for (int x = 0; x < queryTermMatches.Count; x++) { Match match = queryTermMatches[x]; String field = match.Groups[1].Value; var isDate = false; for (int i = 0; i < dates.Count; i++) { if(match.Index < dates[i].Index) continue; if (match.Index >= dates[i].Index + dates[i].Length) continue; isDate = true; break; } if (isDate == false) fields.Add(field); } return fields; }
public LazyFacetsOperation( string index, string facetSetupDoc, IndexQuery query, int start = 0, int? pageSize = null ) { this.index = index; this.facetSetupDoc = facetSetupDoc; this.query = query; this.start = start; this.pageSize = pageSize; }
/// <summary> /// Called when [delete]. /// </summary> /// <param name="key">The key.</param> /// <param name="transactionInformation">The transaction information.</param> public override void OnDelete(string key, TransactionInformation transactionInformation) { if (key.StartsWith("Raven/")) // we don't deal with system documents return; var childrenQuery = new IndexQuery { Query = "Id:" + key }; var queryResult = Database.Query("Documents/ByParent", childrenQuery); if (queryResult.Results.Count > 0) { foreach (var result in queryResult.Results) { var metadataJObject = result.Value<RavenJObject>("@metadata"); if (metadataJObject != null) { var childId = metadataJObject.Value<string>("@id"); var childEtag = metadataJObject.Value<string>("@etag"); Database.Delete(childId, Guid.Parse(childEtag), transactionInformation); } } } base.OnDelete(key, transactionInformation); }
protected void DoInResults( Action<RavenJObject> action, string query = "*:*", int pageSize = 64, string index = "Dynamic", string[] includes = null) { int start = 0; while (true) { var qry = new IndexQuery() { Query = query, PageSize = pageSize, Start = start //TODO: consider to use another thing in place of start }; var results = DocumentStore.DatabaseCommands.Query(index, qry, includes); if (results.Results.Count == 0) break; foreach (var result in results.Results) action(result); start += pageSize; } }
public static IndexQuery GetIndexQueryFromHttpContext(this IHttpContext context, int maxPageSize) { var query = new IndexQuery { Query = context.Request.QueryString["query"] ?? "", Start = context.GetStart(), Cutoff = context.GetCutOff(), CutoffEtag = context.GetCutOffEtag(), PageSize = context.GetPageSize(maxPageSize), SkipTransformResults = context.GetSkipTransformResults(), FieldsToFetch = context.Request.QueryString.GetValues("fetch"), GroupBy = context.Request.QueryString.GetValues("groupBy"), AggregationOperation = context.GetAggregationOperation(), SortedFields = context.Request.QueryString.GetValues("sort") .EmptyIfNull() .Select(x => new SortedField(x)) .ToArray() }; double lat = context.GetLat(), lng = context.GetLng(), radius = context.GetRadius(); if (lat != 0 || lng != 0 || radius != 0) { return new SpatialIndexQuery(query) { Latitude = lat, Longitude = lng, Radius = radius, }; } return query; }
private void HandleRangeFacet(string index, Facet facet, IndexQuery indexQuery, IndexSearcher currentIndexSearcher, Dictionary<string, IEnumerable<FacetValue>> results) { var rangeResults = new List<FacetValue>(); foreach (var range in facet.Ranges) { var baseQuery = database.IndexStorage.GetLuceneQuery(index, indexQuery, database.IndexQueryTriggers); //TODO the built-in parser can't handle [NULL TO 100.0}, i.e. a mix of [ and } //so we need to handle this ourselves (greater and less-than-or-equal) var rangeQuery = database.IndexStorage.GetLuceneQuery(index, new IndexQuery { Query = facet.Name + ":" + range }, database.IndexQueryTriggers); var joinedQuery = new BooleanQuery(); joinedQuery.Add(baseQuery, BooleanClause.Occur.MUST); joinedQuery.Add(rangeQuery, BooleanClause.Occur.MUST); var topDocs = currentIndexSearcher.Search(joinedQuery, null, 1); if (topDocs.TotalHits > 0) { rangeResults.Add(new FacetValue { Count = topDocs.TotalHits, Range = range }); } } results[facet.Name] = rangeResults; }
public IDictionary<string, IEnumerable<FacetValue>> GetFacets(string index, IndexQuery indexQuery, string facetSetupDoc) { var facetSetup = database.Get(facetSetupDoc, null); if (facetSetup == null) throw new InvalidOperationException("Could not find facets document: " + facetSetupDoc); var facets = facetSetup.DataAsJson.JsonDeserialization<FacetSetup>().Facets; var results = new Dictionary<string, IEnumerable<FacetValue>>(); IndexSearcher currentIndexSearcher; using (database.IndexStorage.GetCurrentIndexSearcher(index, out currentIndexSearcher)) { foreach (var facet in facets) { switch (facet.Mode) { case FacetMode.Default: HandleTermsFacet(index, facet, indexQuery, currentIndexSearcher, results); break; case FacetMode.Ranges: HandleRangeFacet(index, facet, indexQuery, currentIndexSearcher, results); break; default: throw new ArgumentException(string.Format("Could not understand '{0}'", facet.Mode)); } } } return results; }
public static IndexQuery FromQueryString(string queryString) { var fields = queryString.Split(new[] {'&'}, StringSplitOptions.RemoveEmptyEntries) .Select(segment => { var parts = segment.Split(new[] {'='}, StringSplitOptions.RemoveEmptyEntries); if (parts.Length == 1) { return new {Key = parts[0], Value = string.Empty}; } else { return new { Key = parts[0], Value = Uri.UnescapeDataString(parts[1]) }; } }).ToLookup(f => f.Key, f => f.Value); var query = new IndexQuery { Query = Uri.UnescapeDataString(fields["query"].FirstOrDefault() ?? ""), Start = fields.GetStart(), Cutoff = fields.GetCutOff(), CutoffEtag = fields.GetCutOffEtag(), PageSize = fields.GetPageSize(250), SkipTransformResults = fields.GetSkipTransformResults(), FieldsToFetch = fields["fetch"].ToArray(), GroupBy = fields["groupBy"].ToArray(), AggregationOperation = fields.GetAggregationOperation(), SortedFields = fields["sort"] .EmptyIfNull() .Select(x => new SortedField(x)) .ToArray(), SortByAggregation = SortFieldAggregation.Default }; SortFieldAggregation sortFieldAggregation; var sortAggregationQuerystringValue = fields["sortAggregation"].FirstOrDefault(); if (!string.IsNullOrEmpty(sortAggregationQuerystringValue) && SortFieldAggregation.TryParse(sortAggregationQuerystringValue, out sortFieldAggregation)) { query.SortByAggregation = sortFieldAggregation; } double lat = fields.GetLat(), lng = fields.GetLng(), radius = fields.GetRadius(); if (lat != 0 || lng != 0 || radius != 0) { return new SpatialIndexQuery(query) { QueryShape = SpatialIndexQuery.GetQueryShapeFromLatLon(lat, lng, radius), SpatialRelation = SpatialRelation.Within, /* TODO */ SpatialFieldName = Constants.DefaultSpatialFieldName, /* TODO */ }; } return query; }
public LazyFacetsOperation(string index, IEnumerable<Facet> facets, IndexQuery query, int start = 0, int? pageSize = null) { this.index = index; this.facets = facets; this.query = query; this.start = start; this.pageSize = pageSize; }
public RavenJArray UpdateByIndex(string indexName, IndexQuery queryToUpdate, ScriptedPatchRequest patch, BulkOperationOptions options = null) { return PerformBulkOperation(indexName, queryToUpdate, options, (docId, tx) => { var patchResult = database.Patches.ApplyPatch(docId, null, patch, tx); return new { Document = docId, Result = patchResult.Item1, Debug = patchResult.Item2 }; }); }
public static Filter MakeFilter(SpatialStrategy spatialStrategy, IndexQuery indexQuery) { var spatialQry = indexQuery as SpatialIndexQuery; if (spatialQry == null) return null; var args = new SpatialArgs(SpatialOperation.IsWithin, ShapeReadWriter.ReadShape(spatialQry.QueryShape)); return spatialStrategy.MakeFilter(args); }
public RavenJArray DeleteByIndex(string indexName, IndexQuery queryToDelete, BulkOperationOptions options = null) { return PerformBulkOperation(indexName, queryToDelete, options, (docId, tx) => { database.Documents.Delete(docId, null, tx); return new { Document = docId, Deleted = true }; }); }
public static Filter MakeFilter(IndexQuery indexQuery) { var spatialQry = indexQuery as SpatialIndexQuery; if (spatialQry == null) return null; var args = new SpatialArgs(SpatialOperation.IsWithin, RavenSpatialContext.MakeCircle(spatialQry.Longitude, spatialQry.Latitude, spatialQry.Radius)); return strategy.MakeFilter(args, fieldInfo); }
private Task<QueryResult> QueryIndexForDocument() { var query = new IndexQuery(); query.Start = itemIndex; query.PageSize = 1; query.SkipTransformResults = true; return DatabaseCommands.QueryAsync(ConflictsIndexName, query, null); }
/// <summary> /// Initializes a new instance of the <see cref="SpatialIndexQuery"/> class. /// </summary> /// <param name="query">The query.</param> public SpatialIndexQuery(IndexQuery query) { Query = query.Query; Start = query.Start; Cutoff = query.Cutoff; PageSize = query.PageSize; FieldsToFetch = query.FieldsToFetch; SortedFields = query.SortedFields; }
public static FacetResults ExecuteGetTermsQuery(this DocumentDatabase self, string index, IndexQuery query, string facetSetupDoc, int start, int? pageSize) { var facetSetup = self.Documents.Get(facetSetupDoc, null); if (facetSetup == null) throw new InvalidOperationException("Could not find facets document: " + facetSetupDoc); var facets = facetSetup.DataAsJson.JsonDeserialization<FacetSetup>().Facets; return self.ExecuteGetTermsQuery(index, query, facets, start, pageSize); }
public void UpdateQuery(string indexName, IndexQuery templateQuery) { lock (_lockObject) { _templateQuery = templateQuery; _indexName = indexName; } Refresh(RefreshMode.ClearStaleData); }
/// <summary> /// Initializes a new instance of the <see cref="SpatialIndexQuery"/> class. /// </summary> /// <param name="query">The query.</param> public SpatialIndexQuery(IndexQuery query) : this() { Query = query.Query; Start = query.Start; Cutoff = query.Cutoff; PageSize = query.PageSize; FieldsToFetch = query.FieldsToFetch; SortedFields = query.SortedFields; SortByAggregation = query.SortByAggregation; }
protected override System.Threading.Tasks.Task TimerTickedAsync() { var query = new IndexQuery { Start = GetSkipCount(), PageSize = 25, Query = "Tag:" + Name }; return databaseCommands .QueryAsync("Raven/DocumentsByEntityName", query, new string[] {}) .ContinueOnSuccess(queryResult => { var documents = SerializationHelper.RavenJObjectsToJsonDocuments(queryResult.Results); Documents.Match(documents.Select(x=>new ViewableDocument(x)).ToArray()); }); }
public static HashSet<Tuple<string,string>> GetFieldsForDynamicQuery(IndexQuery query) { var results = new HashSet<Tuple<string,string>>(); foreach (var result in GetFieldsInternal(query, dynamicQueryTerms)) { if(result == "*") continue; results.Add(Tuple.Create(TranslateField(result), result)); } return results; }
/// <summary> /// Initializes a new instance of the <see cref="SpatialIndexQuery"/> class. /// </summary> /// <param name="query">The query.</param> public SpatialIndexQuery(IndexQuery query) : this() { Query = query.Query; Start = query.Start; Cutoff = query.Cutoff; PageSize = query.PageSize; FieldsToFetch = query.FieldsToFetch; SortedFields = query.SortedFields; HighlighterPreTags = query.HighlighterPreTags; HighlighterPostTags = query.HighlighterPostTags; HighlightedFields = query.HighlightedFields; }
public IndexDocumentsNavigator(string id, int itemIndex, string indexName, IndexQuery templateQuery) { this.id = id; this.itemIndex = itemIndex; this.indexName = indexName; // canonicalize the template query to help keep the url more stable templateQuery.Start = 0; templateQuery.PageSize = 1; this.templateQuery = templateQuery; }
public static IndexQuery FromQueryString(string queryString) { var fields = queryString.Split(new[] {'&'}, StringSplitOptions.RemoveEmptyEntries) .Select(segment => { var parts = segment.Split(new[] {'='}, StringSplitOptions.RemoveEmptyEntries); if (parts.Length == 1) { return new {Key = parts[0], Value = string.Empty}; } else { return new { Key = parts[0], Value = Uri.UnescapeDataString(parts[1]) }; } }).ToLookup(f => f.Key, f => f.Value); var query = new IndexQuery { Query = Uri.UnescapeDataString(fields["query"].FirstOrDefault() ?? ""), Start = fields.GetStart(), Cutoff = fields.GetCutOff(), CutoffEtag = fields.GetCutOffEtag(), PageSize = fields.GetPageSize(250), SkipTransformResults = fields.GetSkipTransformResults(), FieldsToFetch = fields["fetch"].ToArray(), GroupBy = fields["groupBy"].ToArray(), AggregationOperation = fields.GetAggregationOperation(), SortedFields = fields["sort"] .EmptyIfNull() .Select(x => new SortedField(x)) .ToArray() }; double lat = fields.GetLat(), lng = fields.GetLng(), radius = fields.GetRadius(); if (lat != 0 || lng != 0 || radius != 0) { return new SpatialIndexQuery(query) { Latitude = lat, Longitude = lng, Radius = radius, }; } return query; }