public void AdditionalAssemblies_NuGet() { using (var store = GetDocumentStore()) { store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition { Name = "XmlIndex", Maps = { "from c in docs.Companies select new { Name = typeof(System.Xml.XmlNode).Name }" }, AdditionalAssemblies = { AdditionalAssembly.FromRuntime("System.Private.Xml"), AdditionalAssembly.FromNuGet("System.Xml.ReaderWriter", "4.3.1") } })); } }
public void AdditionalAssemblies_NuGet_InvalidName() { using (var store = GetDocumentStore()) { var e = Assert.Throws <IndexCompilationException>(() => store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition { Name = "XmlIndex", Maps = { "from c in docs.Companies select new { Name = typeof(System.Xml.XmlNode).Name }" }, AdditionalAssemblies = { AdditionalAssembly.FromNuGet("Some.Assembly.That.Does.Not.Exist", "4.3.1") } }))); Assert.Contains("Cannot load NuGet package 'Some.Assembly.That.Does.Not.Exist'", e.Message); Assert.Contains("NuGet package 'Some.Assembly.That.Does.Not.Exist' version '4.3.1' from 'https://api.nuget.org/v3/index.json' does not exist", e.Message); } }
public void AdditionalAssemblies_Runtime_InvalidName() { using (var store = GetDocumentStore()) { var e = Assert.Throws <IndexCompilationException>(() => store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition { Name = "XmlIndex", Maps = { "from c in docs.Companies select new { Name = typeof(System.Xml.XmlNode).Name }" }, AdditionalAssemblies = { AdditionalAssembly.FromRuntime("Some.Assembly.That.Does.Not.Exist") } }))); Assert.Contains("Cannot load assembly 'Some.Assembly.That.Does.Not.Exist'", e.Message); Assert.Contains("Could not load file or assembly 'Some.Assembly.That.Does.Not.Exist", e.Message); } }
public void AdditionalAssemblies_NuGet_InvalidSource() { using (var store = GetDocumentStore()) { var e = Assert.Throws <IndexCompilationException>(() => store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition { Name = "XmlIndex", Maps = { "from c in docs.Companies select new { Name = typeof(System.Xml.XmlNode).Name }" }, AdditionalAssemblies = { AdditionalAssembly.FromNuGet("System.Xml.ReaderWriter", "4.3.1", "http://some.url.that.does.not.exist.com") } }))); Assert.Contains("Cannot load NuGet package 'System.Xml.ReaderWriter' version '4.3.1' from 'http://some.url.that.does.not.exist.com'", e.Message); Assert.Contains("Unable to load the service index for source", e.Message); } }
private static AdditionalAssembly GetAssembly(BlittableJsonReaderObject json) { json.TryGet(nameof(AdditionalAssembly.AssemblyName), out string assemblyName); json.TryGet(nameof(AdditionalAssembly.AssemblyPath), out string assemblyPath); json.TryGet(nameof(AdditionalAssembly.PackageName), out string packageName); json.TryGet(nameof(AdditionalAssembly.PackageVersion), out string packageVersion); json.TryGet(nameof(AdditionalAssembly.PackageSourceUrl), out string packageSourceUrl); var usings = new HashSet <string>(); json.TryGet(nameof(AdditionalAssembly.Usings), out BlittableJsonReaderArray usingsArray); if (usingsArray != null) { foreach (var item in usingsArray) { usings.Add(item.ToString()); } } if (string.IsNullOrWhiteSpace(assemblyName) == false) { return(AdditionalAssembly.FromRuntime(assemblyName, usings)); } if (string.IsNullOrWhiteSpace(assemblyPath) == false) { return(AdditionalAssembly.FromPath(assemblyPath, usings)); } if (string.IsNullOrWhiteSpace(packageName) == false && string.IsNullOrWhiteSpace(packageVersion) == false) { return(AdditionalAssembly.FromNuGet(packageName, packageVersion, packageSourceUrl, usings)); } return(null); }
public void CanUseMLNET_Omnx() { using (var store = GetDocumentStore()) { store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition { Name = "Employees/Tags", Maps = { @" from i in docs.Employees let pic = LoadAttachment(i, ""photo.png"") let classified = ObjectClassification.GetObjects(pic.GetContentAsStream()) select new { Tags = classified.Keys, Count = classified.Count } " }, AdditionalSources = new System.Collections.Generic.Dictionary <string, string> { { "ImageClassifier", @" using System; using System.IO; using System.Collections.Generic; using System.Drawing; using System.Linq; using Microsoft.ML; using Microsoft.ML.Data; using Microsoft.ML.Transforms.Image; using Microsoft.ML.Transforms.Onnx; using System.Net.Http; namespace ObjectDetection { public class ObjectClassification { public const int ROW_COUNT = 13; public const int COL_COUNT = 13; public const int BOXES_PER_CELL = 5; public const int CLASS_COUNT = 20; public const int BOX_INFO_FEATURE_COUNT = 5; private const int channelStride = ROW_COUNT * COL_COUNT; private static string[] labels = new string[] { ""aeroplane"", ""bicycle"", ""bird"", ""boat"", ""bottle"", ""bus"", ""car"", ""cat"", ""chair"", ""cow"", ""diningtable"", ""dog"", ""horse"", ""motorbike"", ""person"", ""pottedplant"", ""sheep"", ""sofa"", ""train"", ""tvmonitor"" }; [ThreadStatic] private static Context _context; public class Context { public TransformerChain<OnnxTransformer> Model; public MLContext MLContext; public MemoryStream Mem = new MemoryStream(); public InMemoryImageData[] Array = new InMemoryImageData[1] { new InMemoryImageData() }; private struct ImageNetSettings { public const int imageHeight = 416; public const int imageWidth = 416; } public class InMemoryImageData { [ImageType(ImageNetSettings.imageHeight, ImageNetSettings.imageWidth)] [LoadColumn(0)] public Bitmap Image; } private static string DownloadCache(string url) { var file = Path.GetFileName(new Uri(url).AbsolutePath); var cache = Path.Combine(Path.GetTempPath(), file); if (File.Exists(cache)) return cache; try { using var client = new HttpClient(); using var f = File.Create(cache); client.GetAsync(url).Result.Content.ReadAsStreamAsync().Result.CopyTo(f); return cache; } catch (Exception) { File.Delete(cache);//cleanup on failure throw; } } public Context() { var modelFilePath = DownloadCache(""https://media.githubusercontent.com/media/onnx/models/master/vision/object_detection_segmentation/tiny-yolov2/model/tinyyolov2-7.onnx""); MLContext = new MLContext(); var pipeline = MLContext.Transforms.ResizeImages(outputColumnName: ""image"", imageWidth: ImageNetSettings.imageWidth, imageHeight: ImageNetSettings.imageHeight, inputColumnName: ""Image"") .Append(MLContext.Transforms.ExtractPixels(outputColumnName: ""image"")) .Append(MLContext.Transforms.ApplyOnnxModel(""grid"", ""image"", modelFilePath)); var data = MLContext.Data.LoadFromEnumerable(new List<InMemoryImageData>()); Model = pipeline.Fit(data); } } public static dynamic GetObjects(Stream s) { _context ??= new Context(); _context.Mem.SetLength(0); s.CopyTo(_context.Mem); _context.Mem.Position = 0; using var bitmap = (Bitmap)Bitmap.FromStream(_context.Mem); _context.Array[0].Image = bitmap; var scoredData = _context.Model.Transform(_context.MLContext.Data.LoadFromEnumerable(_context.Array)); var matches = new Dictionary<string, float>(); foreach (var prob in scoredData.GetColumn<float[]>(TinyYoloModelSettings.ModelOutput)) { ParseOutputs(prob, matches); } _context.Array[0].Image = null; return matches; } private static void ParseOutputs(float[] yoloModelOutputs, Dictionary<string, float> matches, float threshold = .3F) { for (int row = 0; row < ROW_COUNT; row++) { for (int column = 0; column < COL_COUNT; column++) { for (int box = 0; box < BOXES_PER_CELL; box++) { var channel = (box * (CLASS_COUNT + BOX_INFO_FEATURE_COUNT)); float confidence = GetConfidence(yoloModelOutputs, row, column, channel); if (confidence < threshold) continue; float[] predictedClasses = ExtractClasses(yoloModelOutputs, row, column, channel); var (topResultIndex, topResultScore) = GetTopResult(predictedClasses); var topScore = topResultScore * confidence; if (topScore < threshold) continue; if(matches.TryGetValue(labels[topResultIndex], out var f) == false || f < topResultScore) { matches[labels[topResultIndex]] = topResultScore; } } } } } private static float GetConfidence(float[] modelOutput, int x, int y, int channel) { return Sigmoid(modelOutput[GetOffset(x, y, channel + 4)]); } private static float Sigmoid(float value) { var k = (float)Math.Exp(value); return k / (1.0f + k); } private static int GetOffset(int x, int y, int channel) { // YOLO outputs a tensor that has a shape of 125x13x13, which // WinML flattens into a 1D array. To access a specific channel // for a given (x,y) cell position, we need to calculate an offset // into the array return (channel * channelStride) + (y * COL_COUNT) + x; } private static float[] Softmax(float[] values) { var maxVal = values.Max(); var exp = values.Select(v => Math.Exp(v - maxVal)); var sumExp = exp.Sum(); return exp.Select(v => (float)(v / sumExp)).ToArray(); } private static float[] ExtractClasses(float[] modelOutput, int x, int y, int channel) { float[] predictedClasses = new float[CLASS_COUNT]; int predictedClassOffset = channel + BOX_INFO_FEATURE_COUNT; for (int predictedClass = 0; predictedClass < CLASS_COUNT; predictedClass++) { predictedClasses[predictedClass] = modelOutput[GetOffset(x, y, predictedClass + predictedClassOffset)]; } return Softmax(predictedClasses); } private static ValueTuple<int, float> GetTopResult(float[] predictedClasses) { return predictedClasses .Select((predictedClass, index) => (Index: index, Value: predictedClass)) .OrderByDescending(result => result.Value) .First(); } public struct TinyYoloModelSettings { // for checking Tiny yolo2 Model input and output parameter names, //you can use tools like Netron, // which is installed by Visual Studio AI Tools // input tensor name public const string ModelInput = ""image""; // output tensor name public const string ModelOutput = ""grid""; } } } " } }, AdditionalAssemblies = { AdditionalAssembly.FromRuntime("System.IO.FileSystem"), AdditionalAssembly.FromRuntime("System.Net.Http"), AdditionalAssembly.FromNuGet("Microsoft.ML", "1.5.5"), AdditionalAssembly.FromNuGet("Microsoft.ML.ImageAnalytics", "1.5.5"), AdditionalAssembly.FromNuGet("Microsoft.ML.OnnxTransformer", "1.5.5"), AdditionalAssembly.FromNuGet("Microsoft.ML.OnnxRuntime.Managed","1.7.1"), AdditionalAssembly.FromNuGet("Microsoft.ML.OnnxRuntime", "1.7.0") } })); using (var session = store.OpenSession()) { var employee = new Employee { FirstName = "John", LastName = "Doe" }; session.Store(employee); var image = GetImage("RavenDB_15753.cat.jpg"); // from: https://unsplash.com/photos/IuJc2qh2TcA session.Advanced.Attachments.Store(employee, "photo.png", image); session.SaveChanges(); } WaitForIndexing(store); RavenTestHelper.AssertNoIndexErrors(store); } }
public void CanUseMLNET() { using (var store = GetDocumentStore()) { store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition { Name = "Employees/Tags", Maps = { @" from e in docs.Employees let pic = LoadAttachment(e, ""photo.png"") where pic != null let classified = ImageClassifier.Classify(pic.GetContentAsStream()) select new { e.Name, Tag = classified.Where(x => x.Value > 0.75f).Select(x => x.Key), _ = classified.Select(x => CreateField(x.Key, x.Value)) } " }, AdditionalSources = new System.Collections.Generic.Dictionary <string, string> { { "ImageClassifier", @" using Microsoft.ML; using Microsoft.ML.Data; using System; using System.Collections.Generic; using System.Drawing; using System.IO; using System.Linq; public static class ImageClassifier { [ThreadStatic] private static PredictionEngine<ImageData, ImagePrediction> _predictor; [ThreadStatic] private static string[] _names; public static IDictionary<string, float> Classify(Stream s) { using var ms = new MemoryStream(); s.CopyTo(ms); ms.Position = 0; Dictionary<string, float> results = new Dictionary<string, float>(); _predictor ??= InitPredictor(); using Bitmap bitmap = (Bitmap)Image.FromStream(ms); ImageData imageData = new ImageData { Image = bitmap }; ImagePrediction prediction = _predictor.Predict(imageData); for (int i = 0; i < prediction.Score.Length; i++) { results[_names[i]] = prediction.Score[i]; } return results; } private static PredictionEngine<ImageData, ImagePrediction> InitPredictor() { MLContext mlContext = new MLContext(); ITransformer model = mlContext.Model.Load(""model.zip"", out _); PredictionEngine<ImageData, ImagePrediction> predictor = mlContext.Model.CreatePredictionEngine<ImageData, ImagePrediction>(model); VBuffer<ReadOnlyMemory<char>> slotNames = default; predictor.OutputSchema[nameof(ImagePrediction.Score)].GetSlotNames(ref slotNames); _names = slotNames.DenseValues().Select(x => x.ToString()).ToArray(); return predictor; } public class ImageData { public Bitmap Image; public string Label; } public class ImagePrediction : ImageData { public string PredictedLabelValue; public float[] Score; } } " } }, AdditionalAssemblies = { AdditionalAssembly.FromRuntime("System.Memory"), AdditionalAssembly.FromNuGet("System.Drawing.Common","4.7.0"), AdditionalAssembly.FromNuGet("Microsoft.ML", "1.5.2") } })); } }
public void AdditionalAssemblies_NuGet_Live() { using (var store = GetDocumentStore()) { store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition { Name = "HtmlIndex", Maps = { "from c in docs.Companies select new { Name = typeof(HtmlAgilityPack.HtmlNode).Assembly.FullName }" }, AdditionalAssemblies = { AdditionalAssembly.FromNuGet("HtmlAgilityPack", "1.11.28") } })); using (var session = store.OpenSession()) { session.Store(new Company { Name = "HR" }); session.SaveChanges(); } WaitForIndexing(store); RavenTestHelper.AssertNoIndexErrors(store); var terms = store.Maintenance.Send(new GetTermsOperation("HtmlIndex", "Name", null)); Assert.Equal(1, terms.Length); Assert.Contains("1.11.28.0", terms[0]); store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition { Name = "HtmlIndex", Maps = { "from c in docs.Companies select new { Name = typeof(HtmlAgilityPack.HtmlNode).Assembly.FullName }" }, AdditionalAssemblies = { AdditionalAssembly.FromNuGet("HtmlAgilityPack", "1.11.32") } })); WaitForIndexing(store); RavenTestHelper.AssertNoIndexErrors(store); terms = store.Maintenance.Send(new GetTermsOperation("HtmlIndex", "Name", null)); Assert.Equal(1, terms.Length); Assert.Contains("1.11.32.0", terms[0]); store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition { Name = "HtmlIndex_2", Maps = { "from c in docs.Companies select new { Name = typeof(HtmlAgilityPack.HtmlNode).Assembly.FullName }" }, AdditionalAssemblies = { AdditionalAssembly.FromNuGet("HtmlAgilityPack", "1.11.28") } })); WaitForIndexing(store); RavenTestHelper.AssertNoIndexErrors(store); terms = store.Maintenance.Send(new GetTermsOperation("HtmlIndex_2", "Name", null)); Assert.Equal(1, terms.Length); Assert.Contains("1.11.28.0", terms[0]); } }
public String ReadOutsideSql(String sqlFileEncoding, String dbmsSuffix) // Non Filtered { String standardPath = _outsideSqlPath.Replace("/", "."); // Replacing is required for CSharp! String dbmsPath = BuildDbmsPath(standardPath, dbmsSuffix); Assembly asm = GetType().Assembly; // In same assembly! AdditionalAssembly additionalAssemblyProvider = DBFluteConfig.GetInstance().AdditionalAssemblyProvider; // = = = = = = = = = = = = = = // From file system at first! // = = = = = = = = = =/ String fileSystemPath = BuildFileSystemPath(standardPath); String text = ReadTextFromFileSystem(fileSystemPath, sqlFileEncoding); if (text != null) { if (_log.IsDebugEnabled) { _log.Debug("...Searching from file system: (Found) " + fileSystemPath); } return(text); } else { if (_log.IsDebugEnabled) { _log.Debug("...Searching from file system: (Not Found) " + fileSystemPath); } } // = = = = = = = = = = = = = = // From embedded resource next! // = = = = = = = = = =/ // by standard assembly text = ReadOutsideSql(sqlFileEncoding, dbmsSuffix, standardPath, dbmsPath, asm); if (text != null) { if (_log.IsDebugEnabled) { _log.Debug("...Searching from embedded: (Found) " + asm.FullName); } return(text); } // by additional assembly if (additionalAssemblyProvider != null) { Assembly additionalAssembly = additionalAssemblyProvider.Invoke(); if (additionalAssembly != null && additionalAssembly != asm) { text = ReadOutsideSql(sqlFileEncoding, dbmsSuffix, standardPath, dbmsPath, additionalAssembly); if (text != null) { if (_log.IsDebugEnabled) { _log.Debug("...Searching from embedded: (Found) " + additionalAssembly.FullName); } return(text); } // Re-try after resolving path for the assembly of result type. String resolvedStandardPath = ResolvePathForResultTypeAssembly(standardPath, asm, additionalAssembly); String resolvedDbmsPath = ResolvePathForResultTypeAssembly(dbmsPath, asm, additionalAssembly); text = ReadOutsideSql(sqlFileEncoding, dbmsSuffix, resolvedStandardPath, resolvedDbmsPath, additionalAssembly); if (text != null) { if (_log.IsDebugEnabled) { _log.Debug("...Searching from embedded: (Found) " + resolvedStandardPath); } return(text); } } } // After all the outside SQL was not found! ThrowOutsideSqlNotFoundException(standardPath); return(null); // Unreachable! }
public ResourceOntologyIndex() { AddMap <ResourceMapping>(resources => from resource in resources where resource.Type == null || !resource.Type.Any() select new Resource { Context = resource.Context, ResourceId = resource.ResourceId, Tags = resource.Tags, Properties = resource.Properties, Source = resource.Source, Modified = MetadataFor(resource).Value <DateTime>("@last-modified") } ); AddMap <ResourceMapping>(resources => from resource in resources where resource.Type == null || !resource.Type.Any() from property in resource.Properties from propertyresource in property.Resources from propertyresourcetype in propertyresource.Type from inverseproperty in propertyresource.Properties select new Resource { Context = propertyresource.Context, ResourceId = propertyresourcetype, Tags = new string[] { }, Properties = new[] { new Property { Name = inverseproperty.Name, Properties = new[] { new Property { Name = property.Name, Resources = new[] { new Resource { Context = resource.Context, Type = new[] { resource.ResourceId } } } } } } }, Source = resource.Source, Modified = MetadataFor(resource).Value <DateTime>("@last-modified") } ); AddMap <ResourceMapping>(resources => from resource in resources from type in resource.Type from ontologyreference in LoadDocument <ResourceMappingReferences>("ResourceMappingReferences/" + resource.Context + "/" + type).ReduceOutputs let ontology = LoadDocument <ResourceMapping>(ontologyreference) where ontology != null from ontologyproperty in ontology.Properties.Where(p => !p.Name.StartsWith("@")) from ontologyresource in ontologyproperty.Resources let resourceIds = from resourceIdProperty in ontologyresource.Properties.Where(p => p.Name == "@resourceId") let derivedproperty = from ontologyderivedproperty in resourceIdProperty.Properties where resourceIdProperty.Tags.Contains("@derive") from derivedproperty in resource.Properties where ontologyderivedproperty.Name == derivedproperty.Name && ontologyderivedproperty.Tags.All(t => derivedproperty.Tags.Contains(t)) && (ontologyderivedproperty.From == null || ontologyderivedproperty.From <= (derivedproperty.Thru ?? DateTime.MaxValue)) && (ontologyderivedproperty.Thru == null || ontologyderivedproperty.Thru >= (derivedproperty.From ?? DateTime.MinValue)) select derivedproperty from resourceIdValue in resourceIdProperty.Value from resourceIdFormattedValue in ResourceFormat(resourceIdValue, resource, derivedproperty) select resourceIdFormattedValue from ontologypropertyresource in ( from tags in ontologyresource.Tags.Where(t => t == "@pull") from property in resource.Properties.Where(p => p.Name == ontologyproperty.Name) from propertyresource in property.Resources.Where(r => !String.IsNullOrEmpty(r.ResourceId)) select new Resource { Context = propertyresource.Context ?? ontology.Context, ResourceId = propertyresource.ResourceId, Tags = new[] { "@pull" }, Properties = new Property[] { }, Source = new string[] { } } ).Union( from tags in ontologyresource.Tags.Where(t => t == "@pull") from resourceId in resourceIds select new Resource { Context = ontologyresource.Context, ResourceId = resourceId, Tags = new[] { "@pull" }, Properties = new Property[] { }, Source = new string[] { } } ).Union( from property in ontologyresource.Properties.Take(1) where ontologyresource.Tags.Contains("@push") || ontologyresource.Properties.Any(p => p.Tags.Contains("@push")) from resourceId in resourceIds select new Resource { Context = ontologyresource.Context, ResourceId = resourceId, Tags = new string[] { "@push" }, Properties = new Property[] { }, Source = new[] { MetadataFor(resource).Value <String>("@id") } } ).Union( from aliasValue in ontology.Properties.Where(p => p.Name == "@alias").SelectMany(p => p.Value) from aliasFormattedValue in ResourceFormat(aliasValue, resource, null) select new Resource { Context = ontology.Context, ResourceId = aliasFormattedValue, Tags = new[] { "@alias" }, Properties = new[] { new Property { Name = "@resource", Resources = new[] { new Resource { Context = resource.Context, ResourceId = resource.ResourceId } }, Source = new[] { MetadataFor(resource).Value <String>("@id") } } }, Source = new string[] { } } ).Union( from aliasValue in ontology.Properties.Where(p => p.Name == "@alias").SelectMany(p => p.Value) from aliasFormattedValue in ResourceFormat(aliasValue, resource, null) select new Resource { Context = resource.Context, ResourceId = resource.ResourceId, Tags = new string[] { }, Properties = new[] { new Property { Name = "@alias", Source = new[] { "ResourceOntologyReferences/" + resource.Context + "/" + aliasFormattedValue } } }, Source = new string[] { } } ) select new Resource { Context = ontologypropertyresource.Context, ResourceId = ontologypropertyresource.ResourceId, Tags = ontologypropertyresource.Tags, Properties = ontologypropertyresource.Properties, Source = ontologypropertyresource.Source, Modified = MetadataFor(resource).Value <DateTime>("@last-modified") } ); Reduce = results => from result in results group result by new { result.Context, result.ResourceId } into g select new Resource { Context = g.Key.Context, ResourceId = g.Key.ResourceId, Tags = g.SelectMany(resource => resource.Tags).Distinct(), Properties = from property in g.SelectMany(r => r.Properties) group property by property.Name into propertyG select new Property { Name = propertyG.Key, Value = propertyG.SelectMany(p => p.Value).Distinct(), Tags = propertyG.SelectMany(p => p.Tags).Distinct(), Resources = propertyG.SelectMany(p => p.Resources).Distinct(), Properties = propertyG.SelectMany(p => p.Properties).Distinct(), Source = propertyG.SelectMany(p => p.Source).Distinct() }, Source = g.SelectMany(resource => resource.Source).Distinct(), Modified = g.Select(resource => resource.Modified).Max() }; Index(Raven.Client.Constants.Documents.Indexing.Fields.AllFields, FieldIndexing.No); OutputReduceToCollection = "ResourceOntology"; PatternReferencesCollectionName = "ResourceOntologyReferences"; PatternForOutputReduceToCollectionReferences = r => $"ResourceOntologyReferences/{r.Context}/{r.ResourceId}"; AdditionalAssemblies = new HashSet <AdditionalAssembly> { AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> { "Digitalisert.Dataplattform" }) }; }
public ResourceClusterIndex() { AddMap <ResourceProperty>(resources => from resource in resources from type in resource.Type from property in resource.Properties.Where(p => p.Tags.Contains("@wkt")) from wkt in property.Value from encodegeohash in WKTEncodeGeohash(wkt) let geohash = encodegeohash.Substring(0, encodegeohash.IndexOf('|')) select new ResourceProperty { Context = resource.Context, ResourceId = type + "/" + geohash, Name = property.Name, Properties = new[] { new Property { Name = property.Name, Value = new[] { encodegeohash }, Resources = new[] { new Resource { Context = resource.Context, ResourceId = resource.ResourceId, Source = new[] { MetadataFor(resource).Value <String>("@id") } } } } }, Source = ( from ontologyresource in property.Resources from ontologytype in ontologyresource.Type from ontologyresourceproperty in ontologyresource.Properties from i in Enumerable.Range(0, geohash.Length) let parentgeohash = geohash.Substring(0, geohash.Length - i) where !(ontologyresource.Context == resource.Context && ontologytype == type && geohash == parentgeohash && property.Name == ontologyresourceproperty.Name) select "ResourceClusterReferences/" + ontologyresource.Context + "/" + ontologytype + "/" + geohash.Substring(0, geohash.Length - i) + "/" + ontologyresourceproperty.Name ).Union( from ontologyresourceproperty in property.Properties from ontologyresource in ontologyresourceproperty.Resources from ontologytype in ontologyresource.Type from i in Enumerable.Range(0, geohash.Length) let parentgeohash = geohash.Substring(0, geohash.Length - i) where !(ontologyresource.Context == resource.Context && ontologytype == type && geohash == parentgeohash && property.Name == ontologyresourceproperty.Name) select "ResourceClusterReferences/" + ontologyresource.Context + "/" + ontologytype + "/" + geohash.Substring(0, geohash.Length - i) + "/" + ontologyresourceproperty.Name ) } ); Reduce = results => from result in results group result by new { result.Context, result.ResourceId, result.Name } into g select new ResourceProperty { Context = g.Key.Context, ResourceId = g.Key.ResourceId, Name = g.Key.Name, Properties = g.SelectMany(r => r.Properties), Source = g.SelectMany(r => r.Source).Distinct() }; Index(Raven.Client.Constants.Documents.Indexing.Fields.AllFields, FieldIndexing.No); OutputReduceToCollection = "ResourceCluster"; PatternReferencesCollectionName = "ResourceClusterReferences"; PatternForOutputReduceToCollectionReferences = r => $"ResourceClusterReferences/{r.Context}/{r.ResourceId}/{r.Name}"; AdditionalAssemblies = new HashSet <AdditionalAssembly> { AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> { "Digitalisert.Dataplattform" }) }; }
public ResourcePropertyIndex() { AddMap <ResourceMapping>(resources => from resource in resources from ontology in from type in resource.Type from ontologyreference in LoadDocument <ResourceOntologyReferences>("ResourceOntologyReferences/" + resource.Context + "/" + type).ReduceOutputs select LoadDocument <ResourceOntology>(ontologyreference) where !ontology.Tags.Contains("@fetch") || LoadDocument <ResourceOntologyReferences>("ResourceOntologyReferences/" + resource.Context + "/" + resource.ResourceId) != null select new Resource { Context = resource.Context, ResourceId = resource.ResourceId, Type = resource.Type, SubType = resource.SubType, Title = resource.Title, SubTitle = resource.SubTitle, Code = resource.Code, Status = resource.Status, Tags = resource.Tags, Properties = ( from ontologyproperty in ontology.Properties.Where(p => !p.Name.StartsWith("@")) let property = (!ontologyproperty.Tags.Contains("@derive")) ? resource.Properties.Where(p => p.Name == ontologyproperty.Name) : from ontologyderivedproperty in ontologyproperty.Properties from derivedproperty in resource.Properties where ontologyderivedproperty.Name == derivedproperty.Name && ontologyderivedproperty.Tags.All(t => derivedproperty.Tags.Contains(t)) && (ontologyderivedproperty.From == null || ontologyderivedproperty.From <= (derivedproperty.Thru ?? DateTime.MaxValue)) && (ontologyderivedproperty.Thru == null || ontologyderivedproperty.Thru >= (derivedproperty.From ?? DateTime.MinValue)) select derivedproperty select new Property { Name = ontologyproperty.Name, Value = property.SelectMany(p => p.Value).Concat(ontologyproperty.Value.SelectMany(v => ResourceFormat(v, resource, property))), Tags = property.SelectMany(p => p.Tags).Union(ontologyproperty.Tags).Select(v => v).Distinct(), Resources = ( from propertyresource in property.SelectMany(p => p.Resources) where String.IsNullOrEmpty(propertyresource.ResourceId) select propertyresource ).Union( from source in ( from propertyresource in property.SelectMany(p => p.Resources) where !String.IsNullOrEmpty(propertyresource.ResourceId) select LoadDocument <ResourceMappingReferences>("ResourceMappingReferences/" + (propertyresource.Context ?? ontology.Context) + "/" + propertyresource.ResourceId).ReduceOutputs ).Union( from ontologyresource in ontologyproperty.Resources from resourceId in from resourceIdValue in ontologyresource.Properties.Where(p => p.Name == "@resourceId").SelectMany(p => p.Value) from resourceIdFormattedValue in ResourceFormat(resourceIdValue, resource, property) select resourceIdFormattedValue select LoadDocument <ResourceMappingReferences>("ResourceMappingReferences/" + (ontologyresource.Context ?? ontology.Context) + "/" + resourceId).ReduceOutputs ).Union( from ontologyresource in ontologyproperty.Resources from resourceId in from resourceIdValue in ontologyresource.Properties.Where(p => p.Name == "@resourceId").SelectMany(p => p.Value) from resourceIdFormattedValue in ResourceFormat(resourceIdValue, resource, property) select resourceIdFormattedValue let aliasreference = LoadDocument <ResourceOntologyReferences>("ResourceOntologyReferences/" + (ontologyresource.Context ?? ontology.Context) + "/" + resourceId) from alias in LoadDocument <ResourceOntology>(aliasreference.ReduceOutputs).Where(r => r.Tags.Contains("@alias")) from resourceproperty in alias.Properties.Where(p => p.Name == "@resource") select resourceproperty.Source ).Union( from ontologyresource in ontologyproperty.Resources from resourceId in from resourceIdValue in ontologyresource.Properties.Where(p => p.Name == "@resourceId").SelectMany(p => p.Value) from resourceIdFormattedValue in ResourceFormat(resourceIdValue, resource, property) select resourceIdFormattedValue let aliasreference = LoadDocument <ResourceOntologyReferences>("ResourceOntologyReferences/" + (ontologyresource.Context ?? ontology.Context) + "/" + resourceId) from alias in LoadDocument <ResourceOntology>(aliasreference.ReduceOutputs) from aliasproperty in alias.Properties.Where(p => p.Name == "@alias") from aliaspropertyreference in LoadDocument <ResourceOntologyReferences>(aliasproperty.Source) from aliaspropertyresource in LoadDocument <ResourceOntology>(aliaspropertyreference.ReduceOutputs) from resourceproperty in aliaspropertyresource.Properties.Where(p => p.Name == "@resource") select resourceproperty.Source ) from propertyresource in from resourcemapping in LoadDocument <ResourceMapping>(source) let propertyresourceontologyreference = LoadDocument <ResourceOntologyReferences>(resourcemapping.Type.Select(type => "ResourceOntologyReferences/" + resourcemapping.Context + "/" + type)) let propertyresourceontology = LoadDocument <ResourceOntology>(propertyresourceontologyreference.SelectMany(r => r.ReduceOutputs)) select new Resource { Context = resourcemapping.Context, ResourceId = resourcemapping.ResourceId, Type = resourcemapping.Type, SubType = resourcemapping.SubType, Title = resourcemapping.Title, SubTitle = resourcemapping.SubTitle, Code = resourcemapping.Code, Status = resourcemapping.Status, Tags = resourcemapping.Tags, Properties = from resourcemappingontologyproperty in propertyresourceontology.SelectMany(r => r.Properties).Where(p => p.Name.StartsWith("@")) let derivedproperties = from derivedproperty in resourcemapping.Properties where resourcemappingontologyproperty.Tags.Contains("@derive") from ontologyderivedproperty in resourcemappingontologyproperty.Properties where ontologyderivedproperty.Name == derivedproperty.Name && ontologyderivedproperty.Tags.All(t => derivedproperty.Tags.Contains(t)) && (ontologyderivedproperty.From == null || ontologyderivedproperty.From <= (derivedproperty.Thru ?? DateTime.MaxValue)) && (ontologyderivedproperty.Thru == null || ontologyderivedproperty.Thru >= (derivedproperty.From ?? DateTime.MinValue)) select derivedproperty select new Property { Name = resourcemappingontologyproperty.Name, Value = ( from value in resourcemappingontologyproperty.Value from formattedvalue in ResourceFormat(value, resourcemapping, derivedproperties) select formattedvalue ).Where(v => !String.IsNullOrWhiteSpace(v)) }, Source = resourcemapping.Source } select new Resource { Context = propertyresource.Context, ResourceId = propertyresource.ResourceId, Type = propertyresource.Type.Union(propertyresource.Properties.Where(p => p.Name == "@type").SelectMany(p => p.Value)).Distinct(), SubType = propertyresource.SubType.Union(propertyresource.Properties.Where(p => p.Name == "@subtype").SelectMany(p => p.Value)).Distinct(), Title = propertyresource.Title.Union(propertyresource.Properties.Where(p => p.Name == "@title").SelectMany(p => p.Value)).Distinct(), SubTitle = propertyresource.SubTitle.Union(propertyresource.Properties.Where(p => p.Name == "@subtitle").SelectMany(p => p.Value)).Distinct(), Code = propertyresource.Code.Union(propertyresource.Properties.Where(p => p.Name == "@code").SelectMany(p => p.Value)).Distinct(), Status = propertyresource.Status.Union(propertyresource.Properties.Where(p => p.Name == "@status").SelectMany(p => p.Value)).Distinct(), Tags = propertyresource.Tags.Union(propertyresource.Properties.Where(p => p.Name == "@tags").SelectMany(p => p.Value)).Distinct(), Source = propertyresource.Source } ).Union( ontologyproperty.Resources.Where(r => !r.Properties.Any(p => p.Name == "@resourceId")) ), Properties = property.SelectMany(p => p.Properties).Union(ontologyproperty.Properties) }).Where(p => p.Value.Any() || p.Resources.Any()).Union(ontology.Properties.Where(p => p.Name.StartsWith("@"))), Source = new[] { MetadataFor(resource).Value <String>("@id") }, Modified = MetadataFor(resource).Value <DateTime>("@last-modified") } ); AddMap <ResourceOntology>(ontologies => from ontology in ontologies.Where(r => r.Tags.Contains("@push")) from pushresource in ( from tags in ontology.Tags.Where(t => t == "@push") select new Resource { Context = ontology.Context, ResourceId = ontology.ResourceId } ).Union( from resourceproperty in ontology.Properties.Where(p => p.Name == "@resource") where ontology.Tags.Any(t => t == "@alias") from aliasresource in resourceproperty.Resources select new Resource { Context = aliasresource.Context, ResourceId = aliasresource.ResourceId } ).Union( from aliasproperty in ontology.Properties.Where(p => p.Name == "@alias") from aliasreference in LoadDocument <ResourceOntologyReferences>(aliasproperty.Source).Where(r => r != null) from aliasresource in LoadDocument <ResourceOntology>(aliasreference.ReduceOutputs).Where(r => r != null) from resourceproperty in aliasresource.Properties.Where(p => p.Name == "@resource") from resourcealias in resourceproperty.Resources select new Resource { Context = resourcealias.Context, ResourceId = resourcealias.ResourceId } ) let pushmappingreference = LoadDocument <ResourceMappingReferences>("ResourceMappingReferences/" + pushresource.Context + "/" + pushresource.ResourceId) from pushpropertyresource in from resourcemapping in LoadDocument <ResourceMapping>(pushmappingreference.ReduceOutputs) let propertyresourceontologyreference = LoadDocument <ResourceMappingReferences>(resourcemapping.Type.Select(type => "ResourceMappingReferences/" + resourcemapping.Context + "/" + type)) let propertyresourceontology = LoadDocument <ResourceMapping>(propertyresourceontologyreference.SelectMany(r => r.ReduceOutputs)) select new Resource { Context = resourcemapping.Context, ResourceId = resourcemapping.ResourceId, Type = resourcemapping.Type, SubType = resourcemapping.SubType, Status = resourcemapping.Status, Tags = resourcemapping.Tags, Properties = from resourcemappingontologyproperty in propertyresourceontology.SelectMany(r => r.Properties).Where(p => new string[] { "@type", "@subtype", "@status", "@tags" }.Contains(p.Name)) let derivedproperties = from derivedproperty in resourcemapping.Properties where resourcemappingontologyproperty.Tags.Contains("@derive") from ontologyderivedproperty in resourcemappingontologyproperty.Properties where ontologyderivedproperty.Name == derivedproperty.Name && ontologyderivedproperty.Tags.All(t => derivedproperty.Tags.Contains(t)) && (ontologyderivedproperty.From == null || ontologyderivedproperty.From <= (derivedproperty.Thru ?? DateTime.MaxValue)) && (ontologyderivedproperty.Thru == null || ontologyderivedproperty.Thru >= (derivedproperty.From ?? DateTime.MinValue)) select derivedproperty select new Property { Name = resourcemappingontologyproperty.Name, Value = ( from value in resourcemappingontologyproperty.Value from formattedvalue in ResourceFormat(value, resourcemapping, derivedproperties) select formattedvalue ).Where(v => !String.IsNullOrWhiteSpace(v)) } } from resource in LoadDocument <ResourceMapping>(ontology.Source).Where(r => r != null) let resourceontologyreference = LoadDocument <ResourceMappingReferences>(resource.Type.Select(type => "ResourceMappingReferences/" + resource.Context + "/" + type)) from resourceontology in LoadDocument <ResourceMapping>(resourceontologyreference.SelectMany(r => r.ReduceOutputs)) from ontologyresource in resourceontology.Properties.SelectMany(p => p.Resources).Where(r => r.Tags.Contains("@push") || r.Properties.Any(p => p.Tags.Contains("@push"))) where ontologyresource.Context == pushpropertyresource.Context && ontologyresource.Type.All(t => pushpropertyresource.Type.Union(pushpropertyresource.Properties.Where(p => p.Name == "@type").SelectMany(p => p.Value).Select(v => v.ToString())).Contains(t.ToString())) && ontologyresource.SubType.All(t => pushpropertyresource.SubType.Union(pushpropertyresource.Properties.Where(p => p.Name == "@subtype").SelectMany(p => p.Value).Select(v => v.ToString())).Contains(t.ToString())) && ontologyresource.Status.All(s => pushpropertyresource.Status.Union(pushpropertyresource.Properties.Where(p => p.Name == "@status").SelectMany(p => p.Value).Select(v => v.ToString())).Contains(s.ToString())) && ontologyresource.Tags.All(t => pushpropertyresource.Tags.Union(pushpropertyresource.Properties.Where(p => p.Name == "@tags").SelectMany(p => p.Value).Select(v => v.ToString())).Contains(t.ToString())) select new Resource { Context = pushresource.Context, ResourceId = pushresource.ResourceId, Type = ontologyresource.Properties.Where(p => p.Name == "@type").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null)).Distinct(), SubType = ontologyresource.Properties.Where(p => p.Name == "@subtype").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null)).Distinct(), Title = ontologyresource.Properties.Where(p => p.Name == "@title").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null)).Distinct(), SubTitle = ontologyresource.Properties.Where(p => p.Name == "@subtitle").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null)).Distinct(), Code = ontologyresource.Properties.Where(p => p.Name == "@code").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null)).Distinct(), Status = ontologyresource.Properties.Where(p => p.Name == "@status").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null)).Distinct(), Tags = ontologyresource.Properties.Where(p => p.Name == "@tags").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null)).Distinct(), Properties = ( from ontologyproperty in ontologyresource.Properties.Where(p => !p.Name.StartsWith("@")) let property = (!ontologyproperty.Tags.Contains("@derive")) ? resource.Properties.Where(p => p.Name == ontologyproperty.Name) : from ontologyderivedproperty in ontologyproperty.Properties from derivedproperty in resource.Properties where ontologyderivedproperty.Name == derivedproperty.Name && ontologyderivedproperty.Tags.All(t => derivedproperty.Tags.Contains(t)) && (ontologyderivedproperty.From == null || ontologyderivedproperty.From <= (derivedproperty.Thru ?? DateTime.MaxValue)) && (ontologyderivedproperty.Thru == null || ontologyderivedproperty.Thru >= (derivedproperty.From ?? DateTime.MinValue)) select derivedproperty select new Property { Name = ontologyproperty.Name, Value = (ontologyproperty.Value.Any()) ? ontologyproperty.Value.SelectMany(v => ResourceFormat(v, resource, property)) : property.SelectMany(p => p.Value), Tags = property.SelectMany(p => p.Tags).Union(ontologyproperty.Tags).Select(v => v).Distinct(), Resources = ( from propertyresource in property.SelectMany(p => p.Resources) where String.IsNullOrEmpty(propertyresource.ResourceId) select propertyresource ).Union( from propertyresource in property.SelectMany(p => p.Resources) where !String.IsNullOrEmpty(propertyresource.ResourceId) select new Resource { Context = propertyresource.Context ?? ontology.Context, ResourceId = propertyresource.ResourceId } ).Union( from ontologyresource in ontologyproperty.Resources from resourceId in from resourceIdValue in ontologyresource.Properties.Where(p => p.Name == "@resourceId").SelectMany(p => p.Value) from resourceIdFormattedValue in ResourceFormat(resourceIdValue, resource, property) select resourceIdFormattedValue select new Resource { Context = ontologyresource.Context ?? ontology.Context, ResourceId = resourceId } ).Union( ontologyproperty.Resources.Where(r => !r.Properties.Any(p => p.Name == "@resourceId")) ), Properties = property.SelectMany(p => p.Properties).Union(ontologyproperty.Properties), Source = (ontologyproperty.Tags.Contains("@push")) ? new[] { MetadataFor(resource).Value <String>("@id") } : new string[] { }, }).Where(p => p.Value.Any() || p.Resources.Any()).Union(ontology.Properties.Where(p => p.Name.StartsWith("@") && p.Name != "@resource")), Source = (ontology.Tags.Contains("@push")) ? new[] { MetadataFor(resource).Value <String>("@id") } : new string[] { }, Modified = MetadataFor(resource).Value <DateTime>("@last-modified") } ); Reduce = results => from result in results group result by new { result.Context, result.ResourceId } into g let computedProperties = from property in g.SelectMany(r => r.Properties).Where(p => p.Name.StartsWith("@") && !p.Tags.Contains("@reasoning")) select new Property { Name = property.Name, Value = ( from value in property.Value from resource in g.ToList() from formattedvalue in ResourceFormat(value, resource, null) select formattedvalue ).Where(v => !String.IsNullOrWhiteSpace(v)) } select new Resource { Context = g.Key.Context, ResourceId = g.Key.ResourceId, Type = g.SelectMany(r => r.Type).Union(computedProperties.Where(p => p.Name == "@type").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(), SubType = g.SelectMany(r => r.SubType).Union(computedProperties.Where(p => p.Name == "@subtype").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(), Title = g.SelectMany(r => r.Title).Union(computedProperties.Where(p => p.Name == "@title").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(), SubTitle = g.SelectMany(r => r.SubTitle).Union(computedProperties.Where(p => p.Name == "@subtitle").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(), Code = g.SelectMany(r => r.Code).Union(computedProperties.Where(p => p.Name == "@code").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(), Status = g.SelectMany(r => r.Status).Union(computedProperties.Where(p => p.Name == "@status").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(), Tags = g.SelectMany(r => r.Tags).Union(computedProperties.Where(p => p.Name == "@tags").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(), Properties = (IEnumerable <Property>)Properties(g.SelectMany(r => r.Properties)), Source = g.SelectMany(r => r.Source).Distinct(), Modified = g.Select(r => r.Modified).Max() }; Index(Raven.Client.Constants.Documents.Indexing.Fields.AllFields, FieldIndexing.No); OutputReduceToCollection = "ResourceProperty"; PatternReferencesCollectionName = "ResourcePropertyReferences"; PatternForOutputReduceToCollectionReferences = r => $"ResourcePropertyReferences/{r.Context}/{r.ResourceId}"; AdditionalAssemblies = new HashSet <AdditionalAssembly> { AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> { "Digitalisert.Dataplattform" }) }; }
public N50KartdataResourceIndex() { AddMap <Kommune>(n50kartdata => from kommune in n50kartdata.WhereEntityIs <Kommune>("N50Kartdata") let metadata = MetadataFor(kommune) where metadata.Value <string>("@id").StartsWith("N50Kartdata/Kommune") select new Resource { ResourceId = "Kommune/" + kommune.kommunenummer, Type = new[] { "Kommune" }, SubType = new string[] { }, Title = new[] { kommune.navn }, Code = new[] { kommune.kommunenummer }, Status = new string[] { }, Properties = from wkt in new[] { kommune._omrade.wkt }.Where(v => !String.IsNullOrWhiteSpace(v)) select new Property { Name = "Område", Tags = new[] { "@wkt" }, Value = new[] { WKTProjectToWGS84(wkt, 0) } }, Source = new[] { metadata.Value <string>("@id") }, Modified = kommune.oppdateringsdato } ); AddMap <Kommune>(n50kartdata => from kommune in n50kartdata.WhereEntityIs <Kommune>("N50Kartdata") let metadata = MetadataFor(kommune) where metadata.Value <string>("@id").StartsWith("N50Kartdata/Kommune") from fylke in new[] { new { Code = "03", Title = "Oslo" }, new { Code = "11", Title = "Rogaland" }, new { Code = "15", Title = "Møre og Romsdal" }, new { Code = "18", Title = "Nordland" }, new { Code = "30", Title = "Viken" }, new { Code = "34", Title = "Innlandet" }, new { Code = "38", Title = "Vestfold og Telemark" }, new { Code = "42", Title = "Agder" }, new { Code = "46", Title = "Vestland" }, new { Code = "50", Title = "Trøndelag" }, new { Code = "54", Title = "Troms og Finnmark" }, } where fylke.Code == kommune.kommunenummer.Substring(0, 2) select new Resource { ResourceId = "Fylke/" + fylke.Code, Type = new[] { "Fylke" }, SubType = new string[] { }, Title = new[] { fylke.Title }, Code = new[] { fylke.Code }, Status = new string[] { }, Properties = ( new[] { new Property { Name = "Kommune", Resources = new[] { new Resource { ResourceId = "Kommune/" + kommune.kommunenummer } } } } ).Union( from wkt in new[] { kommune._omrade.wkt }.Where(v => !String.IsNullOrWhiteSpace(v)) select new Property { Name = "Område", Tags = new[] { "@wkt", "@union" }, Value = new[] { WKTProjectToWGS84(wkt, 0) } } ), Source = new[] { metadata.Value <string>("@id") }, Modified = kommune.oppdateringsdato } ); AddMap <NaturvernOmrade>(n50kartdata => from naturvernomrade in n50kartdata.WhereEntityIs <NaturvernOmrade>("N50Kartdata") let metadata = MetadataFor(naturvernomrade) where metadata.Value <string>("@id").StartsWith("N50Kartdata/NaturvernOmrade") select new Resource { ResourceId = "Naturvern/" + naturvernomrade.verneform + "/" + naturvernomrade.navn, Type = new[] { "Naturvernområde" }, SubType = new[] { LoadDocument <Verneform>("N50Kartdata/Verneform/" + naturvernomrade.verneform).description }.Where(s => !String.IsNullOrWhiteSpace(s)), Title = new[] { naturvernomrade.navn }, Code = new string[] { }, Status = new string[] { }, Properties = from wkt in new[] { naturvernomrade._omrade.wkt }.Where(v => !String.IsNullOrWhiteSpace(v)) select new Property { Name = "Område", Tags = new[] { "@wkt" }, Value = new[] { WKTProjectToWGS84(wkt, 0) } }, Source = new[] { metadata.Value <string>("@id") }, Modified = naturvernomrade.oppdateringsdato } ); AddMap <Sti>(n50kartdata => from sti in n50kartdata.WhereEntityIs <Sti>("N50Kartdata") let metadata = MetadataFor(sti) where metadata.Value <string>("@id").StartsWith("N50Kartdata/Sti") select new Resource { ResourceId = "Sti/" + sti.objid, Type = new[] { "Sti" }, SubType = new string[] { sti.vedlikeholdsansvarlig }.Where(t => !String.IsNullOrWhiteSpace(t)), Title = new string[] { }, Code = new string[] { }, Status = new string[] { (sti.merking == "JA") ? "Merket" : "" }.Where(t => !String.IsNullOrWhiteSpace(t)), Properties = from wkt in new[] { sti._senterlinje.wkt }.Where(v => !String.IsNullOrWhiteSpace(v)) select new Property { Name = "Senterlinje", Tags = new[] { "@wkt" }, Value = new[] { WKTProjectToWGS84(wkt, 0) } }, Source = new[] { metadata.Value <string>("@id") }, Modified = sti.oppdateringsdato } ); Reduce = results => from result in results group result by result.ResourceId into g select new Resource { ResourceId = g.Key, Type = g.SelectMany(r => r.Type).Distinct(), SubType = g.SelectMany(r => r.SubType).Distinct(), Title = g.SelectMany(r => r.Title).Distinct(), Code = g.SelectMany(r => r.Code).Distinct(), Status = g.SelectMany(r => r.Status).Distinct(), Properties = (IEnumerable <Property>)Properties(g.SelectMany(r => r.Properties)), Source = g.SelectMany(resource => resource.Source).Distinct(), Modified = g.Select(resource => resource.Modified).Max() }; Index(Raven.Client.Constants.Documents.Indexing.Fields.AllFields, FieldIndexing.No); OutputReduceToCollection = "N50KartdataResource"; AdditionalAssemblies = new HashSet <AdditionalAssembly> { AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> { "Digitalisert.Dataplattform" }) }; }
public ResourceIndex() { AddMap <Resource>(resources => from resource in resources let source = LoadDocument <ResourceMapping>(resource.Source).Where(r => r != null) let properties = from property in resource.Properties select new Property { Name = property.Name, Value = property.Value, Tags = property.Tags, Resources = ( from propertyresource in property.Resources where propertyresource.ResourceId == null select propertyresource ).Union( from propertyresource in property.Resources where propertyresource.ResourceId != null let propertyresourcereduceoutputs = LoadDocument <ResourceReferences>("ResourceReferences/" + propertyresource.Context + "/" + propertyresource.ResourceId).ReduceOutputs let propertyresourceoutputs = LoadDocument <Resource>(propertyresourcereduceoutputs) select new Resource { Context = propertyresource.Context, ResourceId = propertyresource.ResourceId, Type = propertyresourceoutputs.SelectMany(r => r.Type).Distinct(), SubType = propertyresourceoutputs.SelectMany(r => r.SubType).Distinct(), Title = propertyresourceoutputs.SelectMany(r => r.Title).Distinct(), SubTitle = propertyresourceoutputs.SelectMany(r => r.SubTitle).Distinct(), Code = propertyresourceoutputs.SelectMany(r => r.Code).Distinct(), Body = propertyresourceoutputs.SelectMany(r => r.Properties.Where(p => p.Name == "@body").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, r, null))).Distinct(), Status = propertyresourceoutputs.SelectMany(r => r.Status).Distinct(), Tags = propertyresourceoutputs.SelectMany(r => r.Tags).Union( propertyresourceoutputs.SelectMany(r => r.Properties).SelectMany(p => p.Tags).Where(t => t == "@wkt").Take(1) ).Distinct() } ) } let body = source.SelectMany(r => r.Body ?? new string[] { }).Union(properties.Where(p => p.Name == "@body").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null))).Distinct() select new Resource { Context = resource.Context, ResourceId = resource.ResourceId, Type = resource.Type, SubType = resource.SubType, Title = resource.Title, SubTitle = resource.SubTitle, Code = resource.Code, Body = body, Status = resource.Status, Tags = resource.Tags.Union(properties.Where(p => p.Name == "@tags").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, properties))).Select(v => v.ToString()).Distinct(), Properties = properties.Where(p => !p.Name.StartsWith("@")), _ = ( from property in properties.Where(p => !p.Name.StartsWith("@")) group property by property.Name into propertyG select CreateField( propertyG.Key, propertyG.Where(p => !p.Tags.Contains("@wkt")).SelectMany(p => p.Value).Select(v => v.ToString()).Union( from propertyresource in propertyG.SelectMany(p => p.Resources) from fieldvalue in new[] { propertyresource.ResourceId }.Union(propertyresource.Code).Union(propertyresource.Title).Union(propertyresource.SubTitle) select fieldvalue ).Where(v => !String.IsNullOrWhiteSpace(v)).Distinct() ) ).Union( from property in properties.Where(p => !p.Name.StartsWith("@")) group property by property.Name into propertyG from resourcetype in propertyG.SelectMany(p => p.Resources).SelectMany(r => r.Type).Distinct() select CreateField( propertyG.Key + "." + resourcetype, ( from propertyresource in propertyG.SelectMany(p => p.Resources).Where(r => r.Type.Contains(resourcetype)) from fieldvalue in new[] { propertyresource.ResourceId }.Union(propertyresource.Code).Union(propertyresource.Title) select fieldvalue ).Where(v => !String.IsNullOrWhiteSpace(v)).Distinct() ) ).Union( new object[] { CreateField( "@resources", properties.Where(p => !p.Name.StartsWith("@")).SelectMany(p => p.Resources).Select(r => r.Context + "/" + r.ResourceId).Distinct() ) } ).Union( new object[] { CreateField( "Properties", properties.Where(p => !p.Name.StartsWith("@")).Select(p => p.Name).Where(n => !n.StartsWith("@")).Distinct(), new CreateFieldOptions { Indexing = FieldIndexing.Exact } ) } ).Union( new object[] { CreateField( "Search", resource.Title.Union(resource.SubTitle).Union(resource.Code).Union(body).Distinct(), new CreateFieldOptions { Indexing = FieldIndexing.Search, Storage = FieldStorage.Yes, TermVector = FieldTermVector.WithPositionsAndOffsets } ) } ) } ); Index(r => r.Context, FieldIndexing.Exact); Index(r => r.Type, FieldIndexing.Exact); Index(r => r.SubType, FieldIndexing.Exact); Index(r => r.Code, FieldIndexing.Exact); Index(r => r.Status, FieldIndexing.Exact); Index(r => r.Tags, FieldIndexing.Exact); Index(r => r.Properties, FieldIndexing.No); Index(r => r.Title, FieldIndexing.Search); Index(r => r.SubTitle, FieldIndexing.Search); Index(r => r.Body, FieldIndexing.Search); Store(r => r.Context, FieldStorage.Yes); Store(r => r.Type, FieldStorage.Yes); Store(r => r.SubType, FieldStorage.Yes); Store(r => r.Title, FieldStorage.Yes); Store(r => r.SubTitle, FieldStorage.Yes); Store(r => r.Code, FieldStorage.Yes); Store(r => r.Body, FieldStorage.Yes); Store(r => r.Status, FieldStorage.Yes); Store(r => r.Tags, FieldStorage.Yes); Store(r => r.Properties, FieldStorage.Yes); Analyzers.Add(x => x.Title, "SimpleAnalyzer"); Analyzers.Add(x => x.SubTitle, "SimpleAnalyzer"); Analyzers.Add(x => x.Body, "SimpleAnalyzer"); AdditionalAssemblies = new HashSet <AdditionalAssembly> { AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> { "Digitalisert.Dataplattform" }) }; }
public ResourceReasonerIndex() { AddMap <ResourceProperty>(resources => from resource in resources select new Resource { Context = resource.Context, ResourceId = resource.ResourceId, Type = resource.Type, SubType = resource.SubType, Title = resource.Title, SubTitle = resource.SubTitle, Code = resource.Code, Status = resource.Status, Tags = resource.Tags, Properties = ( from property in resource.Properties.Where(r => !r.Name.StartsWith("@")) select new Property { Name = property.Name, Value = property.Value, Tags = property.Tags, Resources = ( property.Resources.Where(r => r.ResourceId == null) ).Union( from propertyresource in property.Resources.Where(r => r.ResourceId != null) let reduceoutputs = LoadDocument <ResourcePropertyReferences>("ResourcePropertyReferences/" + propertyresource.Context + "/" + propertyresource.ResourceId).ReduceOutputs let resourceoutputs = LoadDocument <ResourceProperty>(reduceoutputs) select new Resource { Context = propertyresource.Context, ResourceId = propertyresource.ResourceId, Type = (propertyresource.Type ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.Type)).Select(v => v.ToString()).Distinct(), SubType = (propertyresource.SubType ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.SubType)).Select(v => v.ToString()).Distinct(), Title = (propertyresource.Title ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.Title)).Select(v => v.ToString()).Distinct(), SubTitle = (propertyresource.SubTitle ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.SubTitle)).Select(v => v.ToString()).Distinct(), Code = (propertyresource.Code ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.Code)).Select(v => v.ToString()).Distinct(), Status = (propertyresource.Status ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.Status)).Select(v => v.ToString()).Distinct(), Tags = (propertyresource.Tags ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.Tags)).Select(v => v.ToString()).Distinct(), Source = (propertyresource.Source ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.Source)).Select(v => v.ToString()).Distinct() } ) } ).Union( resource.Properties.Where(r => r.Name.StartsWith("@")) ), Source = resource.Source, Modified = resource.Modified ?? DateTime.MinValue } ); AddMap <ResourceProperty>(resources => from resource in resources from property in resource.Properties from inverseproperty in property.Properties.Where(p => p.Tags.Contains("@inverse")) from propertyresource in property.Resources.Where(r => r.ResourceId != null) where LoadDocument <ResourcePropertyReferences>("ResourcePropertyReferences/" + propertyresource.Context + "/" + propertyresource.ResourceId) != null select new Resource { Context = propertyresource.Context, ResourceId = propertyresource.ResourceId, Type = new string[] {}, SubType = new string[] {}, Title = new string[] {}, SubTitle = new string[] {}, Code = new string[] {}, Status = new string[] {}, Tags = new string[] {}, Properties = new[] { new Property { Name = inverseproperty.Name, Resources = new[] { new Resource { Context = resource.Context, ResourceId = resource.ResourceId, Type = resource.Type, SubType = resource.SubType, Title = resource.Title, SubTitle = resource.SubTitle, Code = resource.Code, Status = resource.Status, Tags = resource.Tags, Source = resource.Source } } } }, Source = new string[] { }, Modified = DateTime.MinValue } ); AddMap <ResourceDerivedProperty>(resources => from resource in resources from resourceproperty in LoadDocument <ResourceProperty>(resource.Source).Where(r => r != null) from property in resourceproperty.Properties.Where(p => p.Name == resource.Name) from compareproperty in resource.Properties.Select(p => p.Name.Replace("+", "")).Distinct() from compareresourceproperty in ( from compare in resource.Properties.Where(p => p.Name == compareproperty + "+") from compareresource in LoadDocument <ResourceProperty>(compare.Source).Where(r => r != null) select compareresource ).Union( from compare in resource.Properties.Where(p => p.Name == compareproperty) let comparedsources = resource.Properties.Where(p => p.Name == compareproperty + "+").SelectMany(p => p.Source) let comparesources = compare.Source.Where(s => !comparedsources.Contains(s)) where comparesources.Any() from compareresource in LoadDocument <ResourceProperty>(comparesources).Where(r => r != null) where property.Value.Any(v1 => compareresource.Properties.Where(p => p.Name == compareproperty).SelectMany(p => p.Value).Any(v2 => WKTIntersects(v1, v2))) select compareresource ) from derivedproperty in ( from ontologyresource in property.Resources from ontologyproperty in ontologyresource.Properties where ontologyproperty.Name == compareproperty && ontologyresource.Context == compareresourceproperty.Context && ontologyresource.Type.All(t => compareresourceproperty.Type.Contains(t)) select new { fromresource = resourceproperty, name = property.Name, toresource = compareresourceproperty } ).Union( from ontologyproperty in property.Properties from ontologyresource in ontologyproperty.Resources where ontologyproperty.Name == compareproperty && ontologyresource.Context == compareresourceproperty.Context && ontologyresource.Type.All(t => compareresourceproperty.Type.Contains(t)) select new { fromresource = compareresourceproperty, name = compareproperty, toresource = resourceproperty } ) select new Resource { Context = derivedproperty.fromresource.Context, ResourceId = derivedproperty.fromresource.ResourceId, Type = new string[] {}, SubType = new string[] {}, Title = new string[] {}, SubTitle = new string[] {}, Code = new string[] {}, Status = new string[] {}, Tags = new string[] {}, Properties = new[] { new Property { Name = derivedproperty.name, Resources = new[] { new Resource { Context = derivedproperty.toresource.Context, ResourceId = derivedproperty.toresource.ResourceId, Type = derivedproperty.toresource.Type, SubType = derivedproperty.toresource.SubType, Title = derivedproperty.toresource.Title, SubTitle = derivedproperty.toresource.SubTitle, Code = derivedproperty.toresource.Code, Status = derivedproperty.toresource.Status, Tags = derivedproperty.toresource.Tags, Source = derivedproperty.toresource.Source } } } }, Source = new string[] { }, Modified = DateTime.MinValue } ); Reduce = results => from result in results group result by new { result.Context, result.ResourceId } into g let computedProperties = from property in g.SelectMany(r => r.Properties).Where(p => p.Name.StartsWith("@")) select new Property { Name = property.Name, Value = ( from value in property.Value from resource in g.ToList() from formattedvalue in ResourceFormat(value, resource, null) select formattedvalue ).Where(v => !String.IsNullOrWhiteSpace(v)) } select new Resource { Context = g.Key.Context, ResourceId = g.Key.ResourceId, Type = g.SelectMany(r => r.Type).Union(computedProperties.Where(p => p.Name == "@type").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(), SubType = g.SelectMany(r => r.SubType).Union(computedProperties.Where(p => p.Name == "@subtype").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(), Title = g.SelectMany(r => r.Title).Union(computedProperties.Where(p => p.Name == "@title").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(), SubTitle = g.SelectMany(r => r.SubTitle).Union(computedProperties.Where(p => p.Name == "@subtitle").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(), Code = g.SelectMany(r => r.Code).Union(computedProperties.Where(p => p.Name == "@code").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(), Status = g.SelectMany(r => r.Status).Union(computedProperties.Where(p => p.Name == "@status").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(), Tags = g.SelectMany(r => r.Tags).Union(computedProperties.Where(p => p.Name == "@tags").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(), Properties = ( from property in g.SelectMany(r => r.Properties).Where(r => !r.Name.StartsWith("@")) group property by property.Name into propertyG select new Property { Name = propertyG.Key, Value = propertyG.SelectMany(p => p.Value).Distinct(), Tags = propertyG.SelectMany(p => p.Tags).Distinct(), Resources = ( propertyG.SelectMany(p => p.Resources).Where(r => r.ResourceId == null).Distinct() ).Union( from resource in propertyG.SelectMany(p => p.Resources).Where(r => r.ResourceId != null) group resource by new { resource.Context, resource.ResourceId } into resourceG select new Resource { Context = resourceG.Key.Context, ResourceId = resourceG.Key.ResourceId, Type = resourceG.SelectMany(r => r.Type).Distinct(), SubType = resourceG.SelectMany(r => r.SubType).Distinct(), Title = resourceG.SelectMany(r => r.Title).Distinct(), SubTitle = resourceG.SelectMany(r => r.SubTitle).Distinct(), Code = resourceG.SelectMany(r => r.Code).Distinct(), Status = resourceG.SelectMany(r => r.Status).Distinct(), Tags = resourceG.SelectMany(r => r.Tags).Distinct(), Source = resourceG.SelectMany(r => r.Source).Distinct() } ) } ).Union( g.SelectMany(r => r.Properties).Where(r => r.Name.StartsWith("@")) ), Source = g.SelectMany(resource => resource.Source).Distinct(), Modified = g.Select(resource => resource.Modified ?? DateTime.MinValue).Max() }; Index(Raven.Client.Constants.Documents.Indexing.Fields.AllFields, FieldIndexing.No); OutputReduceToCollection = "Resource"; PatternReferencesCollectionName = "ResourceReferences"; PatternForOutputReduceToCollectionReferences = r => $"ResourceReferences/{r.Context}/{r.ResourceId}"; AdditionalAssemblies = new HashSet <AdditionalAssembly> { AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> { "Digitalisert.Dataplattform" }) }; }
public void Example() { using (var store = new DocumentStore()) { #region complex_index store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition { Name = "Photographs/Tags", Maps = { @" from p in docs.Photographs let photo = LoadAttachment(p, ""photo.png"") where photo != null let classified = ImageClassifier.Classify(photo.GetContentAsStream()) select new { e.Name, Tag = classified.Where(x => x.Value > 0.75f).Select(x => x.Key), _ = classified.Select(x => CreateField(x.Key, x.Value)) }" }, AdditionalSources = new System.Collections.Generic.Dictionary <string, string> { { "ImageClassifier", @" public static class ImageClassifier { public static IDictionary<string, float> Classify(Stream s) { // returns a list of descriptors with a // value between 0 and 1 of how well the // image matches that descriptor. } }" } }, AdditionalAssemblies = { AdditionalAssembly.FromRuntime("System.Memory"), AdditionalAssembly.FromNuGet("System.Drawing.Common","4.7.0"), AdditionalAssembly.FromNuGet("Microsoft.ML", "1.5.2") } })); #endregion } using (var store = new DocumentStore()) { using (var session = store.OpenSession()) { #region simple_index var runtimeindex = new IndexDefinition { Name = "Dog_Pictures", Maps = { @" from user in docs.Users let fileName = Path.GetFileName(user.ImagePath) where fileName = ""My_Dogs.jpeg"" select new { user.Name, fileName }" }, AdditionalAssemblies = { AdditionalAssembly.FromRuntime("System.IO") } }; #endregion } } }
public SSBResourceIndex() { AddMap <SSB>(statistikker => from statistikk in statistikker let metadata = MetadataFor(statistikk) where metadata.Value <string>("@id") == "SSB/1108" from data in statistikk.Data select new Resource { ResourceId = data["region"].Substring(0, 4), Type = new[] { "Kommune" }, SubType = new string [] { data["kvartal"] }, Title = new[] { data["region"].Substring(5) }, Code = new[] { data["region"].Substring(0, 4) }, Status = new string[] { }, Tags = new string[] { }, Properties = new[] { new Property { Name = data["statistikkvariabel"], Value = new[] { data["01222: Befolkning og kvartalsvise endringar, etter region, kvartal og statistikkvariabel"] } } }, Source = new[] { metadata.Value <string>("@id") } } ); AddMap <SSB>(statistikker => from statistikk in statistikker let metadata = MetadataFor(statistikk) where metadata.Value <string>("@id") == "SSB/26975" from data in statistikk.Data select new Resource { ResourceId = data["region"].Substring(2, 4), Type = new string[] { "Kommune" }, SubType = new string [] { }, Title = new[] { data["region"].Substring(7) }, Code = new[] { data["region"].Substring(2, 4) }, Status = new string[] { }, Tags = new string[] { }, Properties = new[] { new Property { Name = data["statistikkvariabel"], Value = new[] { data["07459: Befolkning, etter region, år og statistikkvariabel"] }, Tags = new[] { "@history" }, Thru = new DateTime(int.Parse(data["år"]), 12, 31) } }, Source = new[] { metadata.Value <string>("@id") } } ); Reduce = results => from result in results group result by result.ResourceId into g select new Resource { ResourceId = g.Key, Type = g.SelectMany(r => r.Type).Distinct(), SubType = g.SelectMany(r => r.SubType).Distinct(), Title = g.SelectMany(r => r.Title).Distinct(), Code = g.SelectMany(r => r.Code).Distinct(), Status = g.SelectMany(r => r.Status).Distinct(), Tags = g.SelectMany(r => r.Tags).Distinct(), Properties = (IEnumerable <Property>)Properties(g.SelectMany(r => r.Properties)), Source = g.SelectMany(resource => resource.Source).Distinct() }; Index(Raven.Client.Constants.Documents.Indexing.Fields.AllFields, FieldIndexing.No); OutputReduceToCollection = "SSBResource"; AdditionalAssemblies = new HashSet <AdditionalAssembly> { AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> { "Digitalisert.Dataplattform" }) }; }
public DataplattformResourceIndex() { AddMap <Drupal>(noder => from node in noder.WhereEntityIs <Drupal>("Dataplattform") let metadata = MetadataFor(node) where metadata.Value <string>("@id").StartsWith("Dataplattform/Drupal/Node") from resource in ( from resourceid in node["resourceid"] select new Resource { Context = (node["context"].Length > 0) ? node["context"][0]["value"].ToString() : "Dataplattform", ResourceId = resourceid["value"].ToString() } ).Union( from resource in node["resource"] let uuid = resource["target_uuid"].ToString() select new Resource { Context = uuid.Substring(0, uuid.IndexOf('/')), ResourceId = uuid.Substring(uuid.IndexOf('/') + 1) } ) select new Resource { Context = resource.Context, ResourceId = resource.ResourceId, Type = node["resourcetype"].Select(t => t["value"].ToString()), Title = node["title"].Select(t => t["value"].ToString()), Body = node["body"].Select(b => b["value"].ToString()), Properties = from property in node["properties"] let paragraph = LoadDocument <Drupal>("Dataplattform/Drupal/Paragraph/" + property["target_id"], "Dataplattform") let paragraphmetadata = MetadataFor(paragraph) from name in paragraph["name"] select new Property { Name = name["value"].ToString(), Value = ( paragraph["value"].Select(v => v["value"].ToString()) ).Union( paragraph["value_geofield"].Select(v => v["value"].ToString()) ), Tags = ( paragraph["tags"].Select(v => v["value"].ToString()) ).Union( paragraph["value_geofield"].Take(1).Select(t => "@wkt") ).Union( paragraph["type"].Where(t => t["target_id"].ToString() == "property_query").Take(1).Select(t => "@query") ), Resources = from propertyresource in paragraph["resources"] let resourceparagraph = LoadDocument <Drupal>("Dataplattform/Drupal/Paragraph/" + propertyresource["target_id"], "Dataplattform") let resourceparagraphmetadata = MetadataFor(resourceparagraph) from context in resourceparagraph["context"] select new Resource { Context = context["value"].ToString(), Type = resourceparagraph["resourcetype"].Select(t => t["value"].ToString()), Properties = from resourceproperty in resourceparagraph["properties"] let resourcepropertyparagraph = LoadDocument <Drupal>("Dataplattform/Drupal/Paragraph/" + resourceproperty["target_id"], "Dataplattform") let resourcepropertyparagraphmetadata = MetadataFor(resourcepropertyparagraph) from resourcepropertyname in resourcepropertyparagraph["name"] select new Property { Name = resourcepropertyname["value"].ToString(), Value = resourcepropertyparagraph["value"].Select(v => v["value"].ToString()), Source = new[] { resourcepropertyparagraphmetadata.Value <string>("@id") } }, Source = new[] { resourceparagraphmetadata.Value <string>("@id") } }, Properties = from propertyproperty in paragraph["properties"] let propertypropertyparagraph = LoadDocument <Drupal>("Dataplattform/Drupal/Paragraph/" + propertyproperty["target_id"], "Dataplattform") let propertypropertyparagraphmetadata = MetadataFor(propertypropertyparagraph) from propertypropertyname in propertypropertyparagraph["name"] select new Property { Name = propertypropertyname["value"].ToString(), Tags = propertypropertyparagraph["tags"].Select(v => v["value"].ToString()), Source = new[] { propertypropertyparagraphmetadata.Value <string>("@id") } }, From = (paragraph["from"] != null && paragraph["from"].Any()) ? paragraph["from"].Select(v => DateTime.Parse(v["value"].ToString())).First() : null, Thru = (paragraph["thru"] != null && paragraph["thru"].Any()) ? paragraph["thru"].Select(v => DateTime.Parse(v["value"].ToString())).First() : null, Source = new[] { paragraphmetadata.Value <string>("@id") } }, Source = new[] { metadata.Value <string>("@id") }, Modified = DateTime.MinValue } ); Reduce = results => from result in results group result by new { result.Context, result.ResourceId } into g select new Resource { Context = g.Key.Context, ResourceId = g.Key.ResourceId, Type = g.SelectMany(r => r.Type).Distinct(), Title = g.SelectMany(r => r.Title).Distinct(), Body = g.SelectMany(r => r.Body).Distinct(), Properties = (IEnumerable <Property>)Properties(g.SelectMany(r => r.Properties)), Source = g.SelectMany(resource => resource.Source).Distinct(), Modified = g.Select(resource => resource.Modified).Max() }; Index(Raven.Client.Constants.Documents.Indexing.Fields.AllFields, FieldIndexing.No); OutputReduceToCollection = "DataplattformResource"; AdditionalAssemblies = new HashSet <AdditionalAssembly> { AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> { "Digitalisert.Dataplattform" }) }; }