private JsonDocument RetrieveDocumentInternal( IndexQueryResult queryResult, HashSet<string> loadedIds, IEnumerable<string> fieldsToFetch, IndexDefinition indexDefinition, AggregationOperation aggregationOperation) { if (queryResult.Projection == null) { // duplicate document, filter it out if (loadedIds.Add(queryResult.Key) == false) return null; return GetDocumentWithCaching(queryResult.Key); } if (fieldsToFetch != null) { if (indexDefinition.IsMapReduce == false) { bool hasStoredFields = false; foreach (var fieldToFetch in fieldsToFetch) { FieldStorage value; if (indexDefinition.Stores.TryGetValue(fieldToFetch, out value) == false && value != FieldStorage.No) continue; hasStoredFields = true; } if (hasStoredFields == false) { // duplicate document, filter it out if (loadedIds.Add(queryResult.Key) == false) return null; } } if (aggregationOperation != AggregationOperation.None) { var aggOpr = aggregationOperation & ~AggregationOperation.Dynamic; fieldsToFetch = fieldsToFetch.Concat(new[] {aggOpr.ToString()}); } var fieldsToFetchFromDocument = fieldsToFetch.Where(fieldToFetch => queryResult.Projection.Property(fieldToFetch) == null); var doc = GetDocumentWithCaching(queryResult.Key); if (doc != null) { var result = doc.DataAsJson.SelectTokenWithRavenSyntax(fieldsToFetchFromDocument.ToArray()); foreach (var property in result.Properties()) { if(property.Value == null || property.Value.Type == JTokenType.Null) continue; queryResult.Projection[property.Name] = property.Value; } } } return new JsonDocument { Key = queryResult.Key, Projection = queryResult.Projection, }; }
protected Lucene.Net.Store.Directory OpenOrCreateLuceneDirectory(IndexDefinition indexDefinition, string indexName = null) { Lucene.Net.Store.Directory directory; if (indexDefinition.IsTemp || configuration.RunInMemory) { directory = new RAMDirectory(); new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Close(); // creating index structure } else { var indexDirectory = indexName ?? IndexDefinitionStorage.FixupIndexName(indexDefinition.Name, path); var indexFullPath = Path.Combine(path, MonoHttpUtility.UrlEncode(indexDirectory)); directory = FSDirectory.Open(new DirectoryInfo(indexFullPath)); if (!IndexReader.IndexExists(directory)) { //creating index structure if we need to new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Close(); } else { // forcefully unlock locked indexes if any if (IndexWriter.IsLocked(directory)) IndexWriter.Unlock(directory); } } return directory; }
public Lucene.Net.Search.SortField ToLuceneSortField(IndexDefinition definition) { var sortOptions = definition.GetSortOption(Field); if(sortOptions == null) return new Lucene.Net.Search.SortField(Field, CultureInfo.InvariantCulture, Descending); return new Lucene.Net.Search.SortField(Field, (int)sortOptions.Value); }
public void Convert_select_many_will_keep_doc_id() { IndexDefinition indexDefinition = new IndexDefinition<Order> { Map = orders => from order in orders from line in order.OrderLines select new { line.ProductId } }.ToIndexDefinition(new DocumentConvention()); var generator = new DynamicViewCompiler("test", indexDefinition, new AbstractDynamicCompilationExtension[0]) .GenerateInstance(); var results = generator.MapDefinition(new[] { GetDocumentFromString( @" { '@metadata': {'Raven-Entity-Name': 'Orders', '@id': 1}, 'OrderLines': [{'ProductId': 2}, {'ProductId': 3}] }"), GetDocumentFromString( @" { '@metadata': {'Raven-Entity-Name': 'Orders', '@id': 2}, 'OrderLines': [{'ProductId': 5}, {'ProductId': 4}] }") }).Cast<object>().ToArray(); foreach (var result in results) { Assert.NotNull(TypeDescriptor.GetProperties(result).Find("__document_id", true)); } }
public DynamicViewCompiler(string name, IndexDefinition indexDefinition, AbstractDynamicCompilationExtension[] extensions) { this.indexDefinition = indexDefinition; this.extensions = extensions; this.name = MonoHttpUtility.UrlEncode(name); RequiresSelectNewAnonymousType = true; }
private void ReadIndexesFromCatalog(IEnumerable<AbstractViewGenerator> compiledGenerators, ITransactionalStorage transactionalStorage) { foreach (var generator in compiledGenerators) { var copy = generator; var displayNameAtt = TypeDescriptor.GetAttributes(copy) .OfType<DisplayNameAttribute>() .FirstOrDefault(); var name = displayNameAtt != null ? displayNameAtt.DisplayName : copy.GetType().Name; transactionalStorage.Batch(actions => { if (actions.Indexing.GetIndexesStats().Any(x => x.Name == name)) return; actions.Indexing.AddIndex(name); }); var indexDefinition = new IndexDefinition { Map = "Compiled map function: " + generator.GetType().AssemblyQualifiedName, // need to supply this so the index storage will create map/reduce index Reduce = generator.ReduceDefinition == null ? null : "Compiled reduce function: " + generator.GetType().AssemblyQualifiedName, Indexes = generator.Indexes, Stores = generator.Stores, IsCompiled = true }; indexCache.AddOrUpdate(name, copy, (s, viewGenerator) => copy); indexDefinitions.AddOrUpdate(name, indexDefinition, (s1, definition) => indexDefinition); } }
public AnonymousObjectToLuceneDocumentConverter(DocumentDatabase database, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator, ILog log) { this.database = database; this.indexDefinition = indexDefinition; this.viewGenerator = viewGenerator; this.log = log; }
public MapReduceIndex(Directory directory, int id, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator, WorkContext context) : base(directory, id, indexDefinition, viewGenerator, context) { jsonSerializer = JsonExtensions.CreateDefaultJsonSerializer(); jsonSerializer.Converters = MapReduceConverters; }
public string AddIndex(string name, IndexDefinition indexDefinition) { DynamicViewCompiler transformer = AddAndCompileIndex(name, indexDefinition); if(configuration.RunInMemory == false) File.WriteAllText(Path.Combine(path, transformer.Name + ".index"), JsonConvert.SerializeObject(indexDefinition, Formatting.Indented, new JsonEnumConverter())); return transformer.Name; }
private Index CreateIndexImplementation(string name, IndexDefinition indexDefinition, Lucene.Net.Store.Directory directory) { var viewGenerator = indexDefinitionStorage.GetViewGenerator(name); return indexDefinition.IsMapReduce ? (Index)new MapReduceIndex(directory, name, indexDefinition, viewGenerator) : new SimpleIndex(directory, name, indexDefinition, viewGenerator); }
public static IEnumerable<AbstractField> Index(object val, PropertyDescriptorCollection properties, IndexDefinition indexDefinition, Field.Store defaultStorage) { return (from property in properties.Cast<PropertyDescriptor>() let name = property.Name where name != Constants.DocumentIdFieldName let value = property.GetValue(val) from field in CreateFields(name, value, indexDefinition, defaultStorage) select field); }
public IEnumerable<Field> Index(object val, PropertyDescriptorCollection properties, IndexDefinition indexDefinition) { return (from property in properties.Cast<PropertyDescriptor>() let name = property.Name where name != "__document_id" let value = property.GetValue(val) where value != null select new Field(name, ToIndexableString(value, indexDefinition.GetIndex(name)), indexDefinition.GetStorage(name), indexDefinition.GetIndex(name))); }
public static IEnumerable<AbstractField> Index(object val, PropertyDescriptorCollection properties, IndexDefinition indexDefinition, Field.Store defaultStorage) { return (from property in properties.Cast<PropertyDescriptor>() let name = property.Name where name != "__document_id" let value = property.GetValue(val) where value != null select Createfield(name, value, indexDefinition, defaultStorage)); }
public static IEnumerable<AbstractField> Index(JObject document, IndexDefinition indexDefinition, Field.Store defaultStorage) { return (from property in document.Cast<JProperty>() let name = property.Name where name != "__document_id" let value = GetPropertyValue(property) where value != null select Createfield(name, value, indexDefinition, defaultStorage)); }
internal override Lucene.Net.Search.Sort GetSort(Lucene.Net.Search.Filter filter, IndexDefinition indexDefinition) { if (SortByDistance == false) return base.GetSort(filter, indexDefinition); var dsort = new Lucene.Net.Spatial.Tier.DistanceFieldComparatorSource((Lucene.Net.Spatial.Tier.DistanceFilter)filter); return new Lucene.Net.Search.Sort(new Lucene.Net.Search.SortField("foo", dsort, false)); }
public static IEnumerable<AbstractField> Index(JObject document, IndexDefinition indexDefinition, Field.Store defaultStorage) { return (from property in document.Cast<JProperty>() let name = property.Name where name != Constants.DocumentIdFieldName let value = GetPropertyValue(property) from field in CreateFields(name, value, indexDefinition, defaultStorage) select field); }
public void CanPerformSpatialSearch() { var indexDefinition = new IndexDefinition { Map = "from e in docs.Events select new { Tag = \"Event\", _ = SpatialGenerate(e.Latitude, e.Longitude) }", Indexes = { { "Tag", FieldIndexing.NotAnalyzed } } }; store.DatabaseCommands.PutIndex("eventsByLatLng", indexDefinition); var events = SpatialIndexTestHelper.GetEvents(); for (int i = 0; i < events.Length; i++) { store.DatabaseCommands.Put("Events/" + (i + 1), null, RavenJObject.FromObject(events[i]), RavenJObject.Parse("{'Raven-Entity-Name': 'Events'}")); } const double lat = 38.96939, lng = -77.386398; const double radiusInKm = 6.0*1.609344; QueryResult queryResult; do { queryResult = store.DatabaseCommands.Query("eventsByLatLng", new SpatialIndexQuery() { Query = "Tag:[[Event]]", QueryShape = SpatialIndexQuery.GetQueryShapeFromLatLon(lat, lng, radiusInKm), SpatialRelation = SpatialRelation.Within, SpatialFieldName = Constants.DefaultSpatialFieldName, SortedFields = new[] { new SortedField("__distance"), } }, null); if (queryResult.IsStale) Thread.Sleep(100); } while (queryResult.IsStale); var expected = events.Count(e => GetGeographicalDistance(lat, lng, e.Latitude, e.Longitude) <= radiusInKm); Assert.Equal(expected, queryResult.Results.Count); Assert.Equal(7, queryResult.Results.Count); double previous = 0; foreach (var r in queryResult.Results) { Event e = r.JsonDeserialization<Event>(); double distance = GetGeographicalDistance(lat, lng, e.Latitude, e.Longitude); Console.WriteLine("Venue: " + e.Venue + ", Distance " + distance); Assert.True(distance < radiusInKm); Assert.True(distance >= previous); previous = distance; } }
public void CreateIndexImplementation(string name, IndexDefinition indexDefinition) { log.InfoFormat("Creating index {0}", name); indexes.AddOrUpdate(name, n => { var directory = OpenOrCreateLuceneDirectory(name); return CreateIndexImplementation(name, indexDefinition, directory); }, (s, index) => index); }
public IndexCreationOptions FindIndexCreationOptionsOptions(string name, IndexDefinition indexDef) { if (indexCache.ContainsKey(name)) { return GetIndexDefinition(name) == indexDef ? IndexCreationOptions.Noop : IndexCreationOptions.Update; } return IndexCreationOptions.Create; }
public MapReduceIndex(Directory directory, string name, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator, WorkContext context) : base(directory, name, indexDefinition, viewGenerator, context) { jsonSerializer = new JsonSerializer(); foreach (var jsonConverter in Default.Converters) { jsonSerializer.Converters.Add(jsonConverter); } }
public DynamicViewCompiler(string name, IndexDefinition indexDefinition, AbstractDynamicCompilationExtension[] extensions, string basePath) { this.indexDefinition = indexDefinition; this.extensions = extensions; this.basePath = Path.Combine(basePath, "TemporaryIndexDefinitionsAsSource"); if (Directory.Exists(this.basePath) == false) Directory.CreateDirectory(this.basePath); this.name = MonoHttpUtility.UrlEncode(name); RequiresSelectNewAnonymousType = true; }
public void CanPerformSpatialSearch() { var indexDefinition = new IndexDefinition { Map = "from e in docs.Events select new { Tag = \"Event\", _ = SpatialIndex.Generate(e.Latitude, e.Longitude) }", Indexes = { { "Tag", FieldIndexing.NotAnalyzed } } }; db.PutIndex("eventsByLatLng", indexDefinition); var events = SpatialIndexTestHelper.GetEvents(); for (int i = 0; i < events.Length; i++) { db.Put("Events/" + (i + 1), null, JObject.FromObject(events[i]), JObject.Parse("{'Raven-Entity-Name': 'Events'}"), null); } const double lat = 38.96939, lng = -77.386398; const double radius = 6.0; QueryResult queryResult; do { queryResult = db.Query("eventsByLatLng", new SpatialIndexQuery() { Query = "Tag:[[Event]]", Latitude = lat, Longitude = lng, Radius = radius, SortByDistance = true }); if (queryResult.IsStale) Thread.Sleep(100); } while (queryResult.IsStale); Assert.Equal(7, queryResult.Results.Count); double previous = 0; foreach (var r in queryResult.Results) { Event e = r.JsonDeserialization<Event>(); double distance = Raven.Database.Indexing.SpatialIndex.GetDistanceMi(lat, lng, e.Latitude, e.Longitude); Console.WriteLine("Venue: " + e.Venue + ", Distance " + distance); Assert.True(distance < radius); Assert.True(distance >= previous); previous = distance; } }
protected Index(Directory directory, string name,IndexDefinition indexDefinition) { this.name = name; this.indexDefinition = indexDefinition; log.DebugFormat("Creating index for {0}", name); this.directory = directory; searcher = new CurrentIndexSearcher { Searcher = new IndexSearcher(directory) }; }
private Index CreateIndexImplementation(string directoryPath, IndexDefinition indexDefinition, Lucene.Net.Store.Directory directory) { var viewGenerator = indexDefinitionStorage.GetViewGenerator(indexDefinition.Name); var indexImplementation = indexDefinition.IsMapReduce ? (Index)new MapReduceIndex(directory, directoryPath, indexDefinition, viewGenerator) : new SimpleIndex(directory, directoryPath, indexDefinition, viewGenerator); configuration.Container.SatisfyImportsOnce(indexImplementation); return indexImplementation; }
public MapReduceIndex(Directory directory, int id, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator, WorkContext context) : base(directory, id, indexDefinition, viewGenerator, context) { jsonSerializer = new JsonSerializer(); foreach (var jsonConverter in Default.Converters) { jsonSerializer.Converters.Add(jsonConverter); } jsonSerializer.Converters.Add(new IgnoreFieldable()); }
public IndexDefinitionStorage(TransactionalStorage transactionalStorage,string path, IEnumerable<AbstractViewGenerator> compiledGenerators) { this.path = Path.Combine(path, IndexDefDir); if (Directory.Exists(this.path) == false) Directory.CreateDirectory(this.path); foreach (var index in Directory.GetFiles(this.path, "*.index")) { try { AddAndCompileIndex( HttpUtility.UrlDecode(Path.GetFileNameWithoutExtension(index)), JsonConvert.DeserializeObject<IndexDefinition>(File.ReadAllText(index), new JsonEnumConverter()) ); } catch (Exception e) { logger.Warn("Could not compile index " + index + ", skipping bad index", e); } } //compiled view generators always overwrite dynamic views foreach (var generator in compiledGenerators) { var copy = generator; var displayNameAtt = TypeDescriptor.GetAttributes(copy) .OfType<DisplayNameAttribute>() .FirstOrDefault(); var name = displayNameAtt != null ? displayNameAtt.DisplayName : copy.GetType().Name; transactionalStorage.Batch(actions => { if (actions.GetIndexesStats().Any(x => x.Name == name)) return; actions.AddIndex(name); }); var indexDefinition = new IndexDefinition { Map = "Compiled map function: " + generator.GetType().AssemblyQualifiedName, // need to supply this so the index storage will create map/reduce index Reduce = generator.ReduceDefinition == null ? null : "Compiled reduce function: " + generator.GetType().AssemblyQualifiedName, Indexes = generator.Indexes, Stores = generator.Stores, IsCompiled = true }; indexCache.AddOrUpdate(name, copy, (s, viewGenerator) => copy); indexDefinitions.AddOrUpdate(name, indexDefinition, (s1, definition) => indexDefinition); } }
public void CreateIndexImplementation(string name, IndexDefinition indexDefinition) { log.InfoFormat("Creating index {0}", name); indexes.AddOrUpdate(name, n => { var nameOnDisk = HttpUtility.UrlEncode(name); var directory = FSDirectory.GetDirectory(Path.Combine(path, nameOnDisk), true); new IndexWriter(directory, new StandardAnalyzer()).Close(); //creating index structure return CreateIndexImplementation(name, indexDefinition, directory); }, (s, index) => index); }
public void CanCompileComplexQuery() { var indexDefinition = new IndexDefinition<Person>() { Map = people => from person in people from role in person.Roles where role == "Student" select new { role } }.ToIndexDefinition(new DocumentConvention()); new DynamicViewCompiler("test", indexDefinition, new AbstractDynamicCompilationExtension[0]) .GenerateInstance(); }
public DynamicViewCompiler(string name, IndexDefinition indexDefinition, OrderedPartCollection<AbstractDynamicCompilationExtension> extensions, string basePath, InMemoryRavenConfiguration configuration) { this.indexDefinition = indexDefinition; this.extensions = extensions; if (configuration.RunInMemory == false) { this.basePath = Path.Combine(basePath, "TemporaryIndexDefinitionsAsSource"); if (Directory.Exists(this.basePath) == false) Directory.CreateDirectory(this.basePath); } this.name = MonoHttpUtility.UrlEncode(name); RequiresSelectNewAnonymousType = true; }
internal override Lucene.Net.Search.Sort GetSort(IndexDefinition indexDefinition) { if (SortByDistance == false) return base.GetSort(indexDefinition); var dq = new Lucene.Net.Spatial.Tier.DistanceQueryBuilder( Latitude, Longitude, Radius, SpatialIndex.LatField, SpatialIndex.LngField, Lucene.Net.Spatial.Tier.Projectors.CartesianTierPlotter.DefaltFieldPrefix, true); var dsort = new Lucene.Net.Spatial.Tier.DistanceFieldComparatorSource(dq.DistanceFilter); return new Lucene.Net.Search.Sort(new Lucene.Net.Search.SortField("foo", dsort, false)); }
public SimpleIndex(Directory directory, string name, IndexDefinition indexDefinition) : base(directory, name, indexDefinition) { }
public AnonymousObjectToLuceneDocumentConverter(DocumentDatabase database, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator) { this.database = database; this.indexDefinition = indexDefinition; this.viewGenerator = viewGenerator; }
public IndexDefinitionCodeGenerator(IndexDefinition indexDefinition) { _indexDefinition = indexDefinition; }
private static Index CreateIndexImplementation(string name, IndexDefinition indexDefinition, Lucene.Net.Store.Directory directory) { return(indexDefinition.IsMapReduce ? (Index) new MapReduceIndex(directory, name, indexDefinition) : new SimpleIndex(directory, name, indexDefinition)); }
public SimpleIndex(Directory directory, string name, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator) : base(directory, name, indexDefinition, viewGenerator) { }
public SimpleIndex(Directory directory, int id, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator, WorkContext context) : base(directory, id, indexDefinition, viewGenerator, context) { }