private JsonDocument RetrieveDocumentInternal(
IndexQueryResult queryResult,
HashSet<string> loadedIds,
IEnumerable<string> fieldsToFetch,
IndexDefinition indexDefinition,
AggregationOperation aggregationOperation)
{
if (queryResult.Projection == null)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
return null;
return GetDocumentWithCaching(queryResult.Key);
}
if (fieldsToFetch != null)
{
if (indexDefinition.IsMapReduce == false)
{
bool hasStoredFields = false;
foreach (var fieldToFetch in fieldsToFetch)
{
FieldStorage value;
if (indexDefinition.Stores.TryGetValue(fieldToFetch, out value) == false &&
value != FieldStorage.No)
continue;
hasStoredFields = true;
}
if (hasStoredFields == false)
{
// duplicate document, filter it out
if (loadedIds.Add(queryResult.Key) == false)
return null;
}
}
if (aggregationOperation != AggregationOperation.None)
{
var aggOpr = aggregationOperation & ~AggregationOperation.Dynamic;
fieldsToFetch = fieldsToFetch.Concat(new[] {aggOpr.ToString()});
}
var fieldsToFetchFromDocument = fieldsToFetch.Where(fieldToFetch => queryResult.Projection.Property(fieldToFetch) == null);
var doc = GetDocumentWithCaching(queryResult.Key);
if (doc != null)
{
var result = doc.DataAsJson.SelectTokenWithRavenSyntax(fieldsToFetchFromDocument.ToArray());
foreach (var property in result.Properties())
{
if(property.Value == null || property.Value.Type == JTokenType.Null)
continue;
queryResult.Projection[property.Name] = property.Value;
}
}
}
return new JsonDocument
{
Key = queryResult.Key,
Projection = queryResult.Projection,
};
}
protected Lucene.Net.Store.Directory OpenOrCreateLuceneDirectory(IndexDefinition indexDefinition, string indexName = null)
{
Lucene.Net.Store.Directory directory;
if (indexDefinition.IsTemp || configuration.RunInMemory)
{
directory = new RAMDirectory();
new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Close(); // creating index structure
}
else
{
var indexDirectory = indexName ?? IndexDefinitionStorage.FixupIndexName(indexDefinition.Name, path);
var indexFullPath = Path.Combine(path, MonoHttpUtility.UrlEncode(indexDirectory));
directory = FSDirectory.Open(new DirectoryInfo(indexFullPath));
if (!IndexReader.IndexExists(directory))
{
//creating index structure if we need to
new IndexWriter(directory, dummyAnalyzer, IndexWriter.MaxFieldLength.UNLIMITED).Close();
}
else
{
// forcefully unlock locked indexes if any
if (IndexWriter.IsLocked(directory))
IndexWriter.Unlock(directory);
}
}
return directory;
}
public Lucene.Net.Search.SortField ToLuceneSortField(IndexDefinition definition)
{
var sortOptions = definition.GetSortOption(Field);
if(sortOptions == null)
return new Lucene.Net.Search.SortField(Field, CultureInfo.InvariantCulture, Descending);
return new Lucene.Net.Search.SortField(Field, (int)sortOptions.Value);
}
public void Convert_select_many_will_keep_doc_id()
{
IndexDefinition indexDefinition = new IndexDefinition<Order>
{
Map = orders => from order in orders
from line in order.OrderLines
select new { line.ProductId }
}.ToIndexDefinition(new DocumentConvention());
var generator = new DynamicViewCompiler("test", indexDefinition, new AbstractDynamicCompilationExtension[0])
.GenerateInstance();
var results = generator.MapDefinition(new[]
{
GetDocumentFromString(
@"
{
'@metadata': {'Raven-Entity-Name': 'Orders', '@id': 1},
'OrderLines': [{'ProductId': 2}, {'ProductId': 3}]
}"),
GetDocumentFromString(
@"
{
'@metadata': {'Raven-Entity-Name': 'Orders', '@id': 2},
'OrderLines': [{'ProductId': 5}, {'ProductId': 4}]
}")
}).Cast<object>().ToArray();
foreach (var result in results)
{
Assert.NotNull(TypeDescriptor.GetProperties(result).Find("__document_id", true));
}
}
public DynamicViewCompiler(string name, IndexDefinition indexDefinition, AbstractDynamicCompilationExtension[] extensions)
{
this.indexDefinition = indexDefinition;
this.extensions = extensions;
this.name = MonoHttpUtility.UrlEncode(name);
RequiresSelectNewAnonymousType = true;
}
private void ReadIndexesFromCatalog(IEnumerable<AbstractViewGenerator> compiledGenerators, ITransactionalStorage transactionalStorage)
{
foreach (var generator in compiledGenerators)
{
var copy = generator;
var displayNameAtt = TypeDescriptor.GetAttributes(copy)
.OfType<DisplayNameAttribute>()
.FirstOrDefault();
var name = displayNameAtt != null ? displayNameAtt.DisplayName : copy.GetType().Name;
transactionalStorage.Batch(actions =>
{
if (actions.Indexing.GetIndexesStats().Any(x => x.Name == name))
return;
actions.Indexing.AddIndex(name);
});
var indexDefinition = new IndexDefinition
{
Map = "Compiled map function: " + generator.GetType().AssemblyQualifiedName,
// need to supply this so the index storage will create map/reduce index
Reduce = generator.ReduceDefinition == null ? null : "Compiled reduce function: " + generator.GetType().AssemblyQualifiedName,
Indexes = generator.Indexes,
Stores = generator.Stores,
IsCompiled = true
};
indexCache.AddOrUpdate(name, copy, (s, viewGenerator) => copy);
indexDefinitions.AddOrUpdate(name, indexDefinition, (s1, definition) => indexDefinition);
}
}
public AnonymousObjectToLuceneDocumentConverter(DocumentDatabase database, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator, ILog log)
{
this.database = database;
this.indexDefinition = indexDefinition;
this.viewGenerator = viewGenerator;
this.log = log;
}
public MapReduceIndex(Directory directory, int id, IndexDefinition indexDefinition,
AbstractViewGenerator viewGenerator, WorkContext context)
: base(directory, id, indexDefinition, viewGenerator, context)
{
jsonSerializer = JsonExtensions.CreateDefaultJsonSerializer();
jsonSerializer.Converters = MapReduceConverters;
}
public string AddIndex(string name, IndexDefinition indexDefinition)
{
DynamicViewCompiler transformer = AddAndCompileIndex(name, indexDefinition);
if(configuration.RunInMemory == false)
File.WriteAllText(Path.Combine(path, transformer.Name + ".index"), JsonConvert.SerializeObject(indexDefinition, Formatting.Indented, new JsonEnumConverter()));
return transformer.Name;
}
private Index CreateIndexImplementation(string name, IndexDefinition indexDefinition, Lucene.Net.Store.Directory directory)
{
var viewGenerator = indexDefinitionStorage.GetViewGenerator(name);
return indexDefinition.IsMapReduce
? (Index)new MapReduceIndex(directory, name, indexDefinition, viewGenerator)
: new SimpleIndex(directory, name, indexDefinition, viewGenerator);
}
public static IEnumerable<AbstractField> Index(object val, PropertyDescriptorCollection properties, IndexDefinition indexDefinition, Field.Store defaultStorage)
{
return (from property in properties.Cast<PropertyDescriptor>()
let name = property.Name
where name != Constants.DocumentIdFieldName
let value = property.GetValue(val)
from field in CreateFields(name, value, indexDefinition, defaultStorage)
select field);
}
public IEnumerable<Field> Index(object val, PropertyDescriptorCollection properties, IndexDefinition indexDefinition)
{
return (from property in properties.Cast<PropertyDescriptor>()
let name = property.Name
where name != "__document_id"
let value = property.GetValue(val)
where value != null
select new Field(name, ToIndexableString(value, indexDefinition.GetIndex(name)), indexDefinition.GetStorage(name), indexDefinition.GetIndex(name)));
}
public static IEnumerable<AbstractField> Index(object val, PropertyDescriptorCollection properties, IndexDefinition indexDefinition, Field.Store defaultStorage)
{
return (from property in properties.Cast<PropertyDescriptor>()
let name = property.Name
where name != "__document_id"
let value = property.GetValue(val)
where value != null
select Createfield(name, value, indexDefinition, defaultStorage));
}
public static IEnumerable<AbstractField> Index(JObject document, IndexDefinition indexDefinition, Field.Store defaultStorage)
{
return (from property in document.Cast<JProperty>()
let name = property.Name
where name != "__document_id"
let value = GetPropertyValue(property)
where value != null
select Createfield(name, value, indexDefinition, defaultStorage));
}
internal override Lucene.Net.Search.Sort GetSort(Lucene.Net.Search.Filter filter, IndexDefinition indexDefinition)
{
if (SortByDistance == false)
return base.GetSort(filter, indexDefinition);
var dsort = new Lucene.Net.Spatial.Tier.DistanceFieldComparatorSource((Lucene.Net.Spatial.Tier.DistanceFilter)filter);
return new Lucene.Net.Search.Sort(new Lucene.Net.Search.SortField("foo", dsort, false));
}
public static IEnumerable<AbstractField> Index(JObject document, IndexDefinition indexDefinition, Field.Store defaultStorage)
{
return (from property in document.Cast<JProperty>()
let name = property.Name
where name != Constants.DocumentIdFieldName
let value = GetPropertyValue(property)
from field in CreateFields(name, value, indexDefinition, defaultStorage)
select field);
}
public void CanPerformSpatialSearch()
{
var indexDefinition = new IndexDefinition
{
Map = "from e in docs.Events select new { Tag = \"Event\", _ = SpatialGenerate(e.Latitude, e.Longitude) }",
Indexes = {
{ "Tag", FieldIndexing.NotAnalyzed }
}
};
store.DatabaseCommands.PutIndex("eventsByLatLng", indexDefinition);
var events = SpatialIndexTestHelper.GetEvents();
for (int i = 0; i < events.Length; i++)
{
store.DatabaseCommands.Put("Events/" + (i + 1), null,
RavenJObject.FromObject(events[i]),
RavenJObject.Parse("{'Raven-Entity-Name': 'Events'}"));
}
const double lat = 38.96939, lng = -77.386398;
const double radiusInKm = 6.0*1.609344;
QueryResult queryResult;
do
{
queryResult = store.DatabaseCommands.Query("eventsByLatLng", new SpatialIndexQuery()
{
Query = "Tag:[[Event]]",
QueryShape = SpatialIndexQuery.GetQueryShapeFromLatLon(lat, lng, radiusInKm),
SpatialRelation = SpatialRelation.Within,
SpatialFieldName = Constants.DefaultSpatialFieldName,
SortedFields = new[] { new SortedField("__distance"), }
}, null);
if (queryResult.IsStale)
Thread.Sleep(100);
} while (queryResult.IsStale);
var expected = events.Count(e => GetGeographicalDistance(lat, lng, e.Latitude, e.Longitude) <= radiusInKm);
Assert.Equal(expected, queryResult.Results.Count);
Assert.Equal(7, queryResult.Results.Count);
double previous = 0;
foreach (var r in queryResult.Results)
{
Event e = r.JsonDeserialization<Event>();
double distance = GetGeographicalDistance(lat, lng, e.Latitude, e.Longitude);
Console.WriteLine("Venue: " + e.Venue + ", Distance " + distance);
Assert.True(distance < radiusInKm);
Assert.True(distance >= previous);
previous = distance;
}
}
public void CreateIndexImplementation(string name, IndexDefinition indexDefinition)
{
log.InfoFormat("Creating index {0}", name);
indexes.AddOrUpdate(name, n =>
{
var directory = OpenOrCreateLuceneDirectory(name);
return CreateIndexImplementation(name, indexDefinition, directory);
}, (s, index) => index);
}
public IndexCreationOptions FindIndexCreationOptionsOptions(string name, IndexDefinition indexDef)
{
if (indexCache.ContainsKey(name))
{
return GetIndexDefinition(name) == indexDef
? IndexCreationOptions.Noop
: IndexCreationOptions.Update;
}
return IndexCreationOptions.Create;
}
public MapReduceIndex(Directory directory, string name, IndexDefinition indexDefinition,
AbstractViewGenerator viewGenerator, WorkContext context)
: base(directory, name, indexDefinition, viewGenerator, context)
{
jsonSerializer = new JsonSerializer();
foreach (var jsonConverter in Default.Converters)
{
jsonSerializer.Converters.Add(jsonConverter);
}
}
public DynamicViewCompiler(string name, IndexDefinition indexDefinition, AbstractDynamicCompilationExtension[] extensions, string basePath)
{
this.indexDefinition = indexDefinition;
this.extensions = extensions;
this.basePath = Path.Combine(basePath, "TemporaryIndexDefinitionsAsSource");
if (Directory.Exists(this.basePath) == false)
Directory.CreateDirectory(this.basePath);
this.name = MonoHttpUtility.UrlEncode(name);
RequiresSelectNewAnonymousType = true;
}
public void CanPerformSpatialSearch()
{
var indexDefinition = new IndexDefinition
{
Map = "from e in docs.Events select new { Tag = \"Event\", _ = SpatialIndex.Generate(e.Latitude, e.Longitude) }",
Indexes = {
{ "Tag", FieldIndexing.NotAnalyzed }
}
};
db.PutIndex("eventsByLatLng", indexDefinition);
var events = SpatialIndexTestHelper.GetEvents();
for (int i = 0; i < events.Length; i++)
{
db.Put("Events/" + (i + 1), null,
JObject.FromObject(events[i]),
JObject.Parse("{'Raven-Entity-Name': 'Events'}"), null);
}
const double lat = 38.96939, lng = -77.386398;
const double radius = 6.0;
QueryResult queryResult;
do
{
queryResult = db.Query("eventsByLatLng", new SpatialIndexQuery()
{
Query = "Tag:[[Event]]",
Latitude = lat,
Longitude = lng,
Radius = radius,
SortByDistance = true
});
if (queryResult.IsStale)
Thread.Sleep(100);
} while (queryResult.IsStale);
Assert.Equal(7, queryResult.Results.Count);
double previous = 0;
foreach (var r in queryResult.Results)
{
Event e = r.JsonDeserialization<Event>();
double distance = Raven.Database.Indexing.SpatialIndex.GetDistanceMi(lat, lng, e.Latitude, e.Longitude);
Console.WriteLine("Venue: " + e.Venue + ", Distance " + distance);
Assert.True(distance < radius);
Assert.True(distance >= previous);
previous = distance;
}
}
protected Index(Directory directory, string name,IndexDefinition indexDefinition)
{
this.name = name;
this.indexDefinition = indexDefinition;
log.DebugFormat("Creating index for {0}", name);
this.directory = directory;
searcher = new CurrentIndexSearcher
{
Searcher = new IndexSearcher(directory)
};
}
private Index CreateIndexImplementation(string directoryPath, IndexDefinition indexDefinition, Lucene.Net.Store.Directory directory)
{
var viewGenerator = indexDefinitionStorage.GetViewGenerator(indexDefinition.Name);
var indexImplementation = indexDefinition.IsMapReduce
? (Index)new MapReduceIndex(directory, directoryPath, indexDefinition, viewGenerator)
: new SimpleIndex(directory, directoryPath, indexDefinition, viewGenerator);
configuration.Container.SatisfyImportsOnce(indexImplementation);
return indexImplementation;
}
public MapReduceIndex(Directory directory, int id, IndexDefinition indexDefinition,
AbstractViewGenerator viewGenerator, WorkContext context)
: base(directory, id, indexDefinition, viewGenerator, context)
{
jsonSerializer = new JsonSerializer();
foreach (var jsonConverter in Default.Converters)
{
jsonSerializer.Converters.Add(jsonConverter);
}
jsonSerializer.Converters.Add(new IgnoreFieldable());
}
public IndexDefinitionStorage(TransactionalStorage transactionalStorage,string path, IEnumerable<AbstractViewGenerator> compiledGenerators)
{
this.path = Path.Combine(path, IndexDefDir);
if (Directory.Exists(this.path) == false)
Directory.CreateDirectory(this.path);
foreach (var index in Directory.GetFiles(this.path, "*.index"))
{
try
{
AddAndCompileIndex(
HttpUtility.UrlDecode(Path.GetFileNameWithoutExtension(index)),
JsonConvert.DeserializeObject<IndexDefinition>(File.ReadAllText(index), new JsonEnumConverter())
);
}
catch (Exception e)
{
logger.Warn("Could not compile index " + index + ", skipping bad index", e);
}
}
//compiled view generators always overwrite dynamic views
foreach (var generator in compiledGenerators)
{
var copy = generator;
var displayNameAtt = TypeDescriptor.GetAttributes(copy)
.OfType<DisplayNameAttribute>()
.FirstOrDefault();
var name = displayNameAtt != null ? displayNameAtt.DisplayName : copy.GetType().Name;
transactionalStorage.Batch(actions =>
{
if (actions.GetIndexesStats().Any(x => x.Name == name))
return;
actions.AddIndex(name);
});
var indexDefinition = new IndexDefinition
{
Map = "Compiled map function: " + generator.GetType().AssemblyQualifiedName,
// need to supply this so the index storage will create map/reduce index
Reduce = generator.ReduceDefinition == null ? null : "Compiled reduce function: " + generator.GetType().AssemblyQualifiedName,
Indexes = generator.Indexes,
Stores = generator.Stores,
IsCompiled = true
};
indexCache.AddOrUpdate(name, copy, (s, viewGenerator) => copy);
indexDefinitions.AddOrUpdate(name, indexDefinition, (s1, definition) => indexDefinition);
}
}
public void CreateIndexImplementation(string name, IndexDefinition indexDefinition)
{
log.InfoFormat("Creating index {0}", name);
indexes.AddOrUpdate(name, n =>
{
var nameOnDisk = HttpUtility.UrlEncode(name);
var directory = FSDirectory.GetDirectory(Path.Combine(path, nameOnDisk), true);
new IndexWriter(directory, new StandardAnalyzer()).Close(); //creating index structure
return CreateIndexImplementation(name, indexDefinition, directory);
}, (s, index) => index);
}
public void CanCompileComplexQuery()
{
var indexDefinition = new IndexDefinition<Person>()
{
Map = people => from person in people
from role in person.Roles
where role == "Student"
select new { role }
}.ToIndexDefinition(new DocumentConvention());
new DynamicViewCompiler("test", indexDefinition, new AbstractDynamicCompilationExtension[0])
.GenerateInstance();
}
public DynamicViewCompiler(string name, IndexDefinition indexDefinition, OrderedPartCollection<AbstractDynamicCompilationExtension> extensions, string basePath, InMemoryRavenConfiguration configuration)
{
this.indexDefinition = indexDefinition;
this.extensions = extensions;
if (configuration.RunInMemory == false)
{
this.basePath = Path.Combine(basePath, "TemporaryIndexDefinitionsAsSource");
if (Directory.Exists(this.basePath) == false)
Directory.CreateDirectory(this.basePath);
}
this.name = MonoHttpUtility.UrlEncode(name);
RequiresSelectNewAnonymousType = true;
}
internal override Lucene.Net.Search.Sort GetSort(IndexDefinition indexDefinition)
{
if (SortByDistance == false)
return base.GetSort(indexDefinition);
var dq = new Lucene.Net.Spatial.Tier.DistanceQueryBuilder(
Latitude, Longitude, Radius,
SpatialIndex.LatField,
SpatialIndex.LngField,
Lucene.Net.Spatial.Tier.Projectors.CartesianTierPlotter.DefaltFieldPrefix,
true);
var dsort = new Lucene.Net.Spatial.Tier.DistanceFieldComparatorSource(dq.DistanceFilter);
return new Lucene.Net.Search.Sort(new Lucene.Net.Search.SortField("foo", dsort, false));
}
public SimpleIndex(Directory directory, string name, IndexDefinition indexDefinition)
: base(directory, name, indexDefinition)
{
}
public AnonymousObjectToLuceneDocumentConverter(DocumentDatabase database, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator)
{
this.database = database;
this.indexDefinition = indexDefinition;
this.viewGenerator = viewGenerator;
}
public IndexDefinitionCodeGenerator(IndexDefinition indexDefinition)
{
_indexDefinition = indexDefinition;
}
private static Index CreateIndexImplementation(string name, IndexDefinition indexDefinition, Lucene.Net.Store.Directory directory)
{
return(indexDefinition.IsMapReduce
? (Index) new MapReduceIndex(directory, name, indexDefinition)
: new SimpleIndex(directory, name, indexDefinition));
}
public SimpleIndex(Directory directory, string name, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator)
: base(directory, name, indexDefinition, viewGenerator)
{
}
public SimpleIndex(Directory directory, int id, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator, WorkContext context)
: base(directory, id, indexDefinition, viewGenerator, context)
{
}