public void AdditionalAssemblies_NuGet()
{
using (var store = GetDocumentStore())
{
store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition
{
Name = "XmlIndex",
Maps =
{
"from c in docs.Companies select new { Name = typeof(System.Xml.XmlNode).Name }"
},
AdditionalAssemblies =
{
AdditionalAssembly.FromRuntime("System.Private.Xml"),
AdditionalAssembly.FromNuGet("System.Xml.ReaderWriter", "4.3.1")
}
}));
}
}
public void AdditionalAssemblies_NuGet_InvalidName()
{
using (var store = GetDocumentStore())
{
var e = Assert.Throws <IndexCompilationException>(() => store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition
{
Name = "XmlIndex",
Maps =
{
"from c in docs.Companies select new { Name = typeof(System.Xml.XmlNode).Name }"
},
AdditionalAssemblies =
{
AdditionalAssembly.FromNuGet("Some.Assembly.That.Does.Not.Exist", "4.3.1")
}
})));
Assert.Contains("Cannot load NuGet package 'Some.Assembly.That.Does.Not.Exist'", e.Message);
Assert.Contains("NuGet package 'Some.Assembly.That.Does.Not.Exist' version '4.3.1' from 'https://api.nuget.org/v3/index.json' does not exist", e.Message);
}
}
public void AdditionalAssemblies_Runtime_InvalidName()
{
using (var store = GetDocumentStore())
{
var e = Assert.Throws <IndexCompilationException>(() => store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition
{
Name = "XmlIndex",
Maps =
{
"from c in docs.Companies select new { Name = typeof(System.Xml.XmlNode).Name }"
},
AdditionalAssemblies =
{
AdditionalAssembly.FromRuntime("Some.Assembly.That.Does.Not.Exist")
}
})));
Assert.Contains("Cannot load assembly 'Some.Assembly.That.Does.Not.Exist'", e.Message);
Assert.Contains("Could not load file or assembly 'Some.Assembly.That.Does.Not.Exist", e.Message);
}
}
public void AdditionalAssemblies_NuGet_InvalidSource()
{
using (var store = GetDocumentStore())
{
var e = Assert.Throws <IndexCompilationException>(() => store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition
{
Name = "XmlIndex",
Maps =
{
"from c in docs.Companies select new { Name = typeof(System.Xml.XmlNode).Name }"
},
AdditionalAssemblies =
{
AdditionalAssembly.FromNuGet("System.Xml.ReaderWriter", "4.3.1", "http://some.url.that.does.not.exist.com")
}
})));
Assert.Contains("Cannot load NuGet package 'System.Xml.ReaderWriter' version '4.3.1' from 'http://some.url.that.does.not.exist.com'", e.Message);
Assert.Contains("Unable to load the service index for source", e.Message);
}
}
private static AdditionalAssembly GetAssembly(BlittableJsonReaderObject json)
{
json.TryGet(nameof(AdditionalAssembly.AssemblyName), out string assemblyName);
json.TryGet(nameof(AdditionalAssembly.AssemblyPath), out string assemblyPath);
json.TryGet(nameof(AdditionalAssembly.PackageName), out string packageName);
json.TryGet(nameof(AdditionalAssembly.PackageVersion), out string packageVersion);
json.TryGet(nameof(AdditionalAssembly.PackageSourceUrl), out string packageSourceUrl);
var usings = new HashSet <string>();
json.TryGet(nameof(AdditionalAssembly.Usings), out BlittableJsonReaderArray usingsArray);
if (usingsArray != null)
{
foreach (var item in usingsArray)
{
usings.Add(item.ToString());
}
}
if (string.IsNullOrWhiteSpace(assemblyName) == false)
{
return(AdditionalAssembly.FromRuntime(assemblyName, usings));
}
if (string.IsNullOrWhiteSpace(assemblyPath) == false)
{
return(AdditionalAssembly.FromPath(assemblyPath, usings));
}
if (string.IsNullOrWhiteSpace(packageName) == false && string.IsNullOrWhiteSpace(packageVersion) == false)
{
return(AdditionalAssembly.FromNuGet(packageName, packageVersion, packageSourceUrl, usings));
}
return(null);
}
public void CanUseMLNET_Omnx()
{
using (var store = GetDocumentStore())
{
store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition
{
Name = "Employees/Tags",
Maps =
{
@"
from i in docs.Employees
let pic = LoadAttachment(i, ""photo.png"")
let classified = ObjectClassification.GetObjects(pic.GetContentAsStream())
select new
{
Tags = classified.Keys,
Count = classified.Count
}
"
},
AdditionalSources = new System.Collections.Generic.Dictionary <string, string>
{
{ "ImageClassifier", @"
using System;
using System.IO;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using Microsoft.ML;
using Microsoft.ML.Data;
using Microsoft.ML.Transforms.Image;
using Microsoft.ML.Transforms.Onnx;
using System.Net.Http;
namespace ObjectDetection
{
public class ObjectClassification
{
public const int ROW_COUNT = 13;
public const int COL_COUNT = 13;
public const int BOXES_PER_CELL = 5;
public const int CLASS_COUNT = 20;
public const int BOX_INFO_FEATURE_COUNT = 5;
private const int channelStride = ROW_COUNT * COL_COUNT;
private static string[] labels = new string[]
{
""aeroplane"", ""bicycle"", ""bird"", ""boat"", ""bottle"",
""bus"", ""car"", ""cat"", ""chair"", ""cow"",
""diningtable"", ""dog"", ""horse"", ""motorbike"", ""person"",
""pottedplant"", ""sheep"", ""sofa"", ""train"", ""tvmonitor""
};
[ThreadStatic]
private static Context _context;
public class Context
{
public TransformerChain<OnnxTransformer> Model;
public MLContext MLContext;
public MemoryStream Mem = new MemoryStream();
public InMemoryImageData[] Array = new InMemoryImageData[1]
{
new InMemoryImageData()
};
private struct ImageNetSettings
{
public const int imageHeight = 416;
public const int imageWidth = 416;
}
public class InMemoryImageData
{
[ImageType(ImageNetSettings.imageHeight, ImageNetSettings.imageWidth)]
[LoadColumn(0)]
public Bitmap Image;
}
private static string DownloadCache(string url)
{
var file = Path.GetFileName(new Uri(url).AbsolutePath);
var cache = Path.Combine(Path.GetTempPath(), file);
if (File.Exists(cache))
return cache;
try
{
using var client = new HttpClient();
using var f = File.Create(cache);
client.GetAsync(url).Result.Content.ReadAsStreamAsync().Result.CopyTo(f);
return cache;
}
catch (Exception)
{
File.Delete(cache);//cleanup on failure
throw;
}
}
public Context()
{
var modelFilePath = DownloadCache(""https://media.githubusercontent.com/media/onnx/models/master/vision/object_detection_segmentation/tiny-yolov2/model/tinyyolov2-7.onnx"");
MLContext = new MLContext();
var pipeline = MLContext.Transforms.ResizeImages(outputColumnName: ""image"", imageWidth: ImageNetSettings.imageWidth, imageHeight: ImageNetSettings.imageHeight, inputColumnName: ""Image"")
.Append(MLContext.Transforms.ExtractPixels(outputColumnName: ""image""))
.Append(MLContext.Transforms.ApplyOnnxModel(""grid"", ""image"", modelFilePath));
var data = MLContext.Data.LoadFromEnumerable(new List<InMemoryImageData>());
Model = pipeline.Fit(data);
}
}
public static dynamic GetObjects(Stream s)
{
_context ??= new Context();
_context.Mem.SetLength(0);
s.CopyTo(_context.Mem);
_context.Mem.Position = 0;
using var bitmap = (Bitmap)Bitmap.FromStream(_context.Mem);
_context.Array[0].Image = bitmap;
var scoredData = _context.Model.Transform(_context.MLContext.Data.LoadFromEnumerable(_context.Array));
var matches = new Dictionary<string, float>();
foreach (var prob in scoredData.GetColumn<float[]>(TinyYoloModelSettings.ModelOutput))
{
ParseOutputs(prob, matches);
}
_context.Array[0].Image = null;
return matches;
}
private static void ParseOutputs(float[] yoloModelOutputs, Dictionary<string, float> matches, float threshold = .3F)
{
for (int row = 0; row < ROW_COUNT; row++)
{
for (int column = 0; column < COL_COUNT; column++)
{
for (int box = 0; box < BOXES_PER_CELL; box++)
{
var channel = (box * (CLASS_COUNT + BOX_INFO_FEATURE_COUNT));
float confidence = GetConfidence(yoloModelOutputs, row, column, channel);
if (confidence < threshold)
continue;
float[] predictedClasses = ExtractClasses(yoloModelOutputs, row, column, channel);
var (topResultIndex, topResultScore) = GetTopResult(predictedClasses);
var topScore = topResultScore * confidence;
if (topScore < threshold)
continue;
if(matches.TryGetValue(labels[topResultIndex], out var f) == false || f < topResultScore)
{
matches[labels[topResultIndex]] = topResultScore;
}
}
}
}
}
private static float GetConfidence(float[] modelOutput, int x, int y, int channel)
{
return Sigmoid(modelOutput[GetOffset(x, y, channel + 4)]);
}
private static float Sigmoid(float value)
{
var k = (float)Math.Exp(value);
return k / (1.0f + k);
}
private static int GetOffset(int x, int y, int channel)
{
// YOLO outputs a tensor that has a shape of 125x13x13, which
// WinML flattens into a 1D array. To access a specific channel
// for a given (x,y) cell position, we need to calculate an offset
// into the array
return (channel * channelStride) + (y * COL_COUNT) + x;
}
private static float[] Softmax(float[] values)
{
var maxVal = values.Max();
var exp = values.Select(v => Math.Exp(v - maxVal));
var sumExp = exp.Sum();
return exp.Select(v => (float)(v / sumExp)).ToArray();
}
private static float[] ExtractClasses(float[] modelOutput, int x, int y, int channel)
{
float[] predictedClasses = new float[CLASS_COUNT];
int predictedClassOffset = channel + BOX_INFO_FEATURE_COUNT;
for (int predictedClass = 0; predictedClass < CLASS_COUNT; predictedClass++)
{
predictedClasses[predictedClass] = modelOutput[GetOffset(x, y, predictedClass + predictedClassOffset)];
}
return Softmax(predictedClasses);
}
private static ValueTuple<int, float> GetTopResult(float[] predictedClasses)
{
return predictedClasses
.Select((predictedClass, index) => (Index: index, Value: predictedClass))
.OrderByDescending(result => result.Value)
.First();
}
public struct TinyYoloModelSettings
{
// for checking Tiny yolo2 Model input and output parameter names,
//you can use tools like Netron,
// which is installed by Visual Studio AI Tools
// input tensor name
public const string ModelInput = ""image"";
// output tensor name
public const string ModelOutput = ""grid"";
}
}
}
" }
},
AdditionalAssemblies =
{
AdditionalAssembly.FromRuntime("System.IO.FileSystem"),
AdditionalAssembly.FromRuntime("System.Net.Http"),
AdditionalAssembly.FromNuGet("Microsoft.ML", "1.5.5"),
AdditionalAssembly.FromNuGet("Microsoft.ML.ImageAnalytics", "1.5.5"),
AdditionalAssembly.FromNuGet("Microsoft.ML.OnnxTransformer", "1.5.5"),
AdditionalAssembly.FromNuGet("Microsoft.ML.OnnxRuntime.Managed","1.7.1"),
AdditionalAssembly.FromNuGet("Microsoft.ML.OnnxRuntime", "1.7.0")
}
}));
using (var session = store.OpenSession())
{
var employee = new Employee {
FirstName = "John", LastName = "Doe"
};
session.Store(employee);
var image = GetImage("RavenDB_15753.cat.jpg"); // from: https://unsplash.com/photos/IuJc2qh2TcA
session.Advanced.Attachments.Store(employee, "photo.png", image);
session.SaveChanges();
}
WaitForIndexing(store);
RavenTestHelper.AssertNoIndexErrors(store);
}
}
public void CanUseMLNET()
{
using (var store = GetDocumentStore())
{
store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition
{
Name = "Employees/Tags",
Maps =
{
@"
from e in docs.Employees
let pic = LoadAttachment(e, ""photo.png"")
where pic != null
let classified = ImageClassifier.Classify(pic.GetContentAsStream())
select new {
e.Name,
Tag = classified.Where(x => x.Value > 0.75f).Select(x => x.Key),
_ = classified.Select(x => CreateField(x.Key, x.Value))
}
"
},
AdditionalSources = new System.Collections.Generic.Dictionary <string, string>
{
{ "ImageClassifier", @"
using Microsoft.ML;
using Microsoft.ML.Data;
using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
public static class ImageClassifier
{
[ThreadStatic] private static PredictionEngine<ImageData, ImagePrediction> _predictor;
[ThreadStatic] private static string[] _names;
public static IDictionary<string, float> Classify(Stream s)
{
using var ms = new MemoryStream();
s.CopyTo(ms);
ms.Position = 0;
Dictionary<string, float> results = new Dictionary<string, float>();
_predictor ??= InitPredictor();
using Bitmap bitmap = (Bitmap)Image.FromStream(ms);
ImageData imageData = new ImageData { Image = bitmap };
ImagePrediction prediction = _predictor.Predict(imageData);
for (int i = 0; i < prediction.Score.Length; i++)
{
results[_names[i]] = prediction.Score[i];
}
return results;
}
private static PredictionEngine<ImageData, ImagePrediction> InitPredictor()
{
MLContext mlContext = new MLContext();
ITransformer model = mlContext.Model.Load(""model.zip"", out _);
PredictionEngine<ImageData, ImagePrediction> predictor =
mlContext.Model.CreatePredictionEngine<ImageData, ImagePrediction>(model);
VBuffer<ReadOnlyMemory<char>> slotNames = default;
predictor.OutputSchema[nameof(ImagePrediction.Score)].GetSlotNames(ref slotNames);
_names = slotNames.DenseValues().Select(x => x.ToString()).ToArray();
return predictor;
}
public class ImageData
{
public Bitmap Image;
public string Label;
}
public class ImagePrediction : ImageData
{
public string PredictedLabelValue;
public float[] Score;
}
}
" }
},
AdditionalAssemblies =
{
AdditionalAssembly.FromRuntime("System.Memory"),
AdditionalAssembly.FromNuGet("System.Drawing.Common","4.7.0"),
AdditionalAssembly.FromNuGet("Microsoft.ML", "1.5.2")
}
}));
}
}
public void AdditionalAssemblies_NuGet_Live()
{
using (var store = GetDocumentStore())
{
store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition
{
Name = "HtmlIndex",
Maps =
{
"from c in docs.Companies select new { Name = typeof(HtmlAgilityPack.HtmlNode).Assembly.FullName }"
},
AdditionalAssemblies =
{
AdditionalAssembly.FromNuGet("HtmlAgilityPack", "1.11.28")
}
}));
using (var session = store.OpenSession())
{
session.Store(new Company
{
Name = "HR"
});
session.SaveChanges();
}
WaitForIndexing(store);
RavenTestHelper.AssertNoIndexErrors(store);
var terms = store.Maintenance.Send(new GetTermsOperation("HtmlIndex", "Name", null));
Assert.Equal(1, terms.Length);
Assert.Contains("1.11.28.0", terms[0]);
store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition
{
Name = "HtmlIndex",
Maps =
{
"from c in docs.Companies select new { Name = typeof(HtmlAgilityPack.HtmlNode).Assembly.FullName }"
},
AdditionalAssemblies =
{
AdditionalAssembly.FromNuGet("HtmlAgilityPack", "1.11.32")
}
}));
WaitForIndexing(store);
RavenTestHelper.AssertNoIndexErrors(store);
terms = store.Maintenance.Send(new GetTermsOperation("HtmlIndex", "Name", null));
Assert.Equal(1, terms.Length);
Assert.Contains("1.11.32.0", terms[0]);
store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition
{
Name = "HtmlIndex_2",
Maps =
{
"from c in docs.Companies select new { Name = typeof(HtmlAgilityPack.HtmlNode).Assembly.FullName }"
},
AdditionalAssemblies =
{
AdditionalAssembly.FromNuGet("HtmlAgilityPack", "1.11.28")
}
}));
WaitForIndexing(store);
RavenTestHelper.AssertNoIndexErrors(store);
terms = store.Maintenance.Send(new GetTermsOperation("HtmlIndex_2", "Name", null));
Assert.Equal(1, terms.Length);
Assert.Contains("1.11.28.0", terms[0]);
}
}
public String ReadOutsideSql(String sqlFileEncoding, String dbmsSuffix) // Non Filtered
{
String standardPath = _outsideSqlPath.Replace("/", "."); // Replacing is required for CSharp!
String dbmsPath = BuildDbmsPath(standardPath, dbmsSuffix);
Assembly asm = GetType().Assembly; // In same assembly!
AdditionalAssembly additionalAssemblyProvider = DBFluteConfig.GetInstance().AdditionalAssemblyProvider;
// = = = = = = = = = = = = = =
// From file system at first!
// = = = = = = = = = =/
String fileSystemPath = BuildFileSystemPath(standardPath);
String text = ReadTextFromFileSystem(fileSystemPath, sqlFileEncoding);
if (text != null)
{
if (_log.IsDebugEnabled)
{
_log.Debug("...Searching from file system: (Found) " + fileSystemPath);
}
return(text);
}
else
{
if (_log.IsDebugEnabled)
{
_log.Debug("...Searching from file system: (Not Found) " + fileSystemPath);
}
}
// = = = = = = = = = = = = = =
// From embedded resource next!
// = = = = = = = = = =/
// by standard assembly
text = ReadOutsideSql(sqlFileEncoding, dbmsSuffix, standardPath, dbmsPath, asm);
if (text != null)
{
if (_log.IsDebugEnabled)
{
_log.Debug("...Searching from embedded: (Found) " + asm.FullName);
}
return(text);
}
// by additional assembly
if (additionalAssemblyProvider != null)
{
Assembly additionalAssembly = additionalAssemblyProvider.Invoke();
if (additionalAssembly != null && additionalAssembly != asm)
{
text = ReadOutsideSql(sqlFileEncoding, dbmsSuffix, standardPath, dbmsPath, additionalAssembly);
if (text != null)
{
if (_log.IsDebugEnabled)
{
_log.Debug("...Searching from embedded: (Found) " + additionalAssembly.FullName);
}
return(text);
}
// Re-try after resolving path for the assembly of result type.
String resolvedStandardPath = ResolvePathForResultTypeAssembly(standardPath, asm, additionalAssembly);
String resolvedDbmsPath = ResolvePathForResultTypeAssembly(dbmsPath, asm, additionalAssembly);
text = ReadOutsideSql(sqlFileEncoding, dbmsSuffix, resolvedStandardPath, resolvedDbmsPath, additionalAssembly);
if (text != null)
{
if (_log.IsDebugEnabled)
{
_log.Debug("...Searching from embedded: (Found) " + resolvedStandardPath);
}
return(text);
}
}
}
// After all the outside SQL was not found!
ThrowOutsideSqlNotFoundException(standardPath);
return(null); // Unreachable!
}
public ResourceOntologyIndex()
{
AddMap <ResourceMapping>(resources =>
from resource in resources
where resource.Type == null || !resource.Type.Any()
select new Resource
{
Context = resource.Context,
ResourceId = resource.ResourceId,
Tags = resource.Tags,
Properties = resource.Properties,
Source = resource.Source,
Modified = MetadataFor(resource).Value <DateTime>("@last-modified")
}
);
AddMap <ResourceMapping>(resources =>
from resource in resources
where resource.Type == null || !resource.Type.Any()
from property in resource.Properties
from propertyresource in property.Resources
from propertyresourcetype in propertyresource.Type
from inverseproperty in propertyresource.Properties
select new Resource
{
Context = propertyresource.Context,
ResourceId = propertyresourcetype,
Tags = new string[] { },
Properties = new[] {
new Property {
Name = inverseproperty.Name,
Properties = new[] {
new Property {
Name = property.Name,
Resources = new[] {
new Resource {
Context = resource.Context,
Type = new[] { resource.ResourceId }
}
}
}
}
}
},
Source = resource.Source,
Modified = MetadataFor(resource).Value <DateTime>("@last-modified")
}
);
AddMap <ResourceMapping>(resources =>
from resource in resources
from type in resource.Type
from ontologyreference in LoadDocument <ResourceMappingReferences>("ResourceMappingReferences/" + resource.Context + "/" + type).ReduceOutputs
let ontology = LoadDocument <ResourceMapping>(ontologyreference)
where ontology != null
from ontologyproperty in ontology.Properties.Where(p => !p.Name.StartsWith("@"))
from ontologyresource in ontologyproperty.Resources
let resourceIds =
from resourceIdProperty in ontologyresource.Properties.Where(p => p.Name == "@resourceId")
let derivedproperty =
from ontologyderivedproperty in resourceIdProperty.Properties
where resourceIdProperty.Tags.Contains("@derive")
from derivedproperty in resource.Properties
where ontologyderivedproperty.Name == derivedproperty.Name &&
ontologyderivedproperty.Tags.All(t => derivedproperty.Tags.Contains(t)) &&
(ontologyderivedproperty.From == null || ontologyderivedproperty.From <= (derivedproperty.Thru ?? DateTime.MaxValue)) &&
(ontologyderivedproperty.Thru == null || ontologyderivedproperty.Thru >= (derivedproperty.From ?? DateTime.MinValue))
select derivedproperty
from resourceIdValue in resourceIdProperty.Value
from resourceIdFormattedValue in ResourceFormat(resourceIdValue, resource, derivedproperty)
select resourceIdFormattedValue
from ontologypropertyresource in (
from tags in ontologyresource.Tags.Where(t => t == "@pull")
from property in resource.Properties.Where(p => p.Name == ontologyproperty.Name)
from propertyresource in property.Resources.Where(r => !String.IsNullOrEmpty(r.ResourceId))
select new Resource
{
Context = propertyresource.Context ?? ontology.Context,
ResourceId = propertyresource.ResourceId,
Tags = new[] { "@pull" },
Properties = new Property[] { },
Source = new string[] { }
}
).Union(
from tags in ontologyresource.Tags.Where(t => t == "@pull")
from resourceId in resourceIds
select new Resource
{
Context = ontologyresource.Context,
ResourceId = resourceId,
Tags = new[] { "@pull" },
Properties = new Property[] { },
Source = new string[] { }
}
).Union(
from property in ontologyresource.Properties.Take(1)
where ontologyresource.Tags.Contains("@push") || ontologyresource.Properties.Any(p => p.Tags.Contains("@push"))
from resourceId in resourceIds
select new Resource
{
Context = ontologyresource.Context,
ResourceId = resourceId,
Tags = new string[] { "@push" },
Properties = new Property[] { },
Source = new[] { MetadataFor(resource).Value <String>("@id") }
}
).Union(
from aliasValue in ontology.Properties.Where(p => p.Name == "@alias").SelectMany(p => p.Value)
from aliasFormattedValue in ResourceFormat(aliasValue, resource, null)
select new Resource
{
Context = ontology.Context,
ResourceId = aliasFormattedValue,
Tags = new[] { "@alias" },
Properties = new[] {
new Property {
Name = "@resource",
Resources = new[] {
new Resource {
Context = resource.Context,
ResourceId = resource.ResourceId
}
},
Source = new[] { MetadataFor(resource).Value <String>("@id") }
}
},
Source = new string[] { }
}
).Union(
from aliasValue in ontology.Properties.Where(p => p.Name == "@alias").SelectMany(p => p.Value)
from aliasFormattedValue in ResourceFormat(aliasValue, resource, null)
select new Resource
{
Context = resource.Context,
ResourceId = resource.ResourceId,
Tags = new string[] { },
Properties = new[] {
new Property {
Name = "@alias",
Source = new[] { "ResourceOntologyReferences/" + resource.Context + "/" + aliasFormattedValue }
}
},
Source = new string[] { }
}
)
select new Resource
{
Context = ontologypropertyresource.Context,
ResourceId = ontologypropertyresource.ResourceId,
Tags = ontologypropertyresource.Tags,
Properties = ontologypropertyresource.Properties,
Source = ontologypropertyresource.Source,
Modified = MetadataFor(resource).Value <DateTime>("@last-modified")
}
);
Reduce = results =>
from result in results
group result by new { result.Context, result.ResourceId } into g
select new Resource
{
Context = g.Key.Context,
ResourceId = g.Key.ResourceId,
Tags = g.SelectMany(resource => resource.Tags).Distinct(),
Properties =
from property in g.SelectMany(r => r.Properties)
group property by property.Name into propertyG
select new Property {
Name = propertyG.Key,
Value = propertyG.SelectMany(p => p.Value).Distinct(),
Tags = propertyG.SelectMany(p => p.Tags).Distinct(),
Resources = propertyG.SelectMany(p => p.Resources).Distinct(),
Properties = propertyG.SelectMany(p => p.Properties).Distinct(),
Source = propertyG.SelectMany(p => p.Source).Distinct()
},
Source = g.SelectMany(resource => resource.Source).Distinct(),
Modified = g.Select(resource => resource.Modified).Max()
};
Index(Raven.Client.Constants.Documents.Indexing.Fields.AllFields, FieldIndexing.No);
OutputReduceToCollection = "ResourceOntology";
PatternReferencesCollectionName = "ResourceOntologyReferences";
PatternForOutputReduceToCollectionReferences = r => $"ResourceOntologyReferences/{r.Context}/{r.ResourceId}";
AdditionalAssemblies = new HashSet <AdditionalAssembly> {
AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> {
"Digitalisert.Dataplattform"
})
};
}
public ResourceClusterIndex()
{
AddMap <ResourceProperty>(resources =>
from resource in resources
from type in resource.Type
from property in resource.Properties.Where(p => p.Tags.Contains("@wkt"))
from wkt in property.Value
from encodegeohash in WKTEncodeGeohash(wkt)
let geohash = encodegeohash.Substring(0, encodegeohash.IndexOf('|'))
select new ResourceProperty
{
Context = resource.Context,
ResourceId = type + "/" + geohash,
Name = property.Name,
Properties = new[] {
new Property {
Name = property.Name,
Value = new[] { encodegeohash },
Resources = new[] {
new Resource {
Context = resource.Context,
ResourceId = resource.ResourceId,
Source = new[] { MetadataFor(resource).Value <String>("@id") }
}
}
}
},
Source = (
from ontologyresource in property.Resources
from ontologytype in ontologyresource.Type
from ontologyresourceproperty in ontologyresource.Properties
from i in Enumerable.Range(0, geohash.Length)
let parentgeohash = geohash.Substring(0, geohash.Length - i)
where !(ontologyresource.Context == resource.Context && ontologytype == type && geohash == parentgeohash && property.Name == ontologyresourceproperty.Name)
select "ResourceClusterReferences/" + ontologyresource.Context + "/" + ontologytype + "/" + geohash.Substring(0, geohash.Length - i) + "/" + ontologyresourceproperty.Name
).Union(
from ontologyresourceproperty in property.Properties
from ontologyresource in ontologyresourceproperty.Resources
from ontologytype in ontologyresource.Type
from i in Enumerable.Range(0, geohash.Length)
let parentgeohash = geohash.Substring(0, geohash.Length - i)
where !(ontologyresource.Context == resource.Context && ontologytype == type && geohash == parentgeohash && property.Name == ontologyresourceproperty.Name)
select "ResourceClusterReferences/" + ontologyresource.Context + "/" + ontologytype + "/" + geohash.Substring(0, geohash.Length - i) + "/" + ontologyresourceproperty.Name
)
}
);
Reduce = results =>
from result in results
group result by new { result.Context, result.ResourceId, result.Name } into g
select new ResourceProperty
{
Context = g.Key.Context,
ResourceId = g.Key.ResourceId,
Name = g.Key.Name,
Properties = g.SelectMany(r => r.Properties),
Source = g.SelectMany(r => r.Source).Distinct()
};
Index(Raven.Client.Constants.Documents.Indexing.Fields.AllFields, FieldIndexing.No);
OutputReduceToCollection = "ResourceCluster";
PatternReferencesCollectionName = "ResourceClusterReferences";
PatternForOutputReduceToCollectionReferences = r => $"ResourceClusterReferences/{r.Context}/{r.ResourceId}/{r.Name}";
AdditionalAssemblies = new HashSet <AdditionalAssembly> {
AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> {
"Digitalisert.Dataplattform"
})
};
}
public ResourcePropertyIndex()
{
AddMap <ResourceMapping>(resources =>
from resource in resources
from ontology in
from type in resource.Type
from ontologyreference in LoadDocument <ResourceOntologyReferences>("ResourceOntologyReferences/" + resource.Context + "/" + type).ReduceOutputs
select LoadDocument <ResourceOntology>(ontologyreference)
where !ontology.Tags.Contains("@fetch") || LoadDocument <ResourceOntologyReferences>("ResourceOntologyReferences/" + resource.Context + "/" + resource.ResourceId) != null
select new Resource {
Context = resource.Context,
ResourceId = resource.ResourceId,
Type = resource.Type,
SubType = resource.SubType,
Title = resource.Title,
SubTitle = resource.SubTitle,
Code = resource.Code,
Status = resource.Status,
Tags = resource.Tags,
Properties = (
from ontologyproperty in ontology.Properties.Where(p => !p.Name.StartsWith("@"))
let property = (!ontologyproperty.Tags.Contains("@derive")) ? resource.Properties.Where(p => p.Name == ontologyproperty.Name) :
from ontologyderivedproperty in ontologyproperty.Properties
from derivedproperty in resource.Properties
where ontologyderivedproperty.Name == derivedproperty.Name &&
ontologyderivedproperty.Tags.All(t => derivedproperty.Tags.Contains(t)) &&
(ontologyderivedproperty.From == null || ontologyderivedproperty.From <= (derivedproperty.Thru ?? DateTime.MaxValue)) &&
(ontologyderivedproperty.Thru == null || ontologyderivedproperty.Thru >= (derivedproperty.From ?? DateTime.MinValue))
select derivedproperty
select new Property {
Name = ontologyproperty.Name,
Value = property.SelectMany(p => p.Value).Concat(ontologyproperty.Value.SelectMany(v => ResourceFormat(v, resource, property))),
Tags = property.SelectMany(p => p.Tags).Union(ontologyproperty.Tags).Select(v => v).Distinct(),
Resources = (
from propertyresource in property.SelectMany(p => p.Resources)
where String.IsNullOrEmpty(propertyresource.ResourceId)
select propertyresource
).Union(
from source in (
from propertyresource in property.SelectMany(p => p.Resources)
where !String.IsNullOrEmpty(propertyresource.ResourceId)
select LoadDocument <ResourceMappingReferences>("ResourceMappingReferences/" + (propertyresource.Context ?? ontology.Context) + "/" + propertyresource.ResourceId).ReduceOutputs
).Union(
from ontologyresource in ontologyproperty.Resources
from resourceId in
from resourceIdValue in ontologyresource.Properties.Where(p => p.Name == "@resourceId").SelectMany(p => p.Value)
from resourceIdFormattedValue in ResourceFormat(resourceIdValue, resource, property)
select resourceIdFormattedValue
select LoadDocument <ResourceMappingReferences>("ResourceMappingReferences/" + (ontologyresource.Context ?? ontology.Context) + "/" + resourceId).ReduceOutputs
).Union(
from ontologyresource in ontologyproperty.Resources
from resourceId in
from resourceIdValue in ontologyresource.Properties.Where(p => p.Name == "@resourceId").SelectMany(p => p.Value)
from resourceIdFormattedValue in ResourceFormat(resourceIdValue, resource, property)
select resourceIdFormattedValue
let aliasreference = LoadDocument <ResourceOntologyReferences>("ResourceOntologyReferences/" + (ontologyresource.Context ?? ontology.Context) + "/" + resourceId)
from alias in LoadDocument <ResourceOntology>(aliasreference.ReduceOutputs).Where(r => r.Tags.Contains("@alias"))
from resourceproperty in alias.Properties.Where(p => p.Name == "@resource")
select resourceproperty.Source
).Union(
from ontologyresource in ontologyproperty.Resources
from resourceId in
from resourceIdValue in ontologyresource.Properties.Where(p => p.Name == "@resourceId").SelectMany(p => p.Value)
from resourceIdFormattedValue in ResourceFormat(resourceIdValue, resource, property)
select resourceIdFormattedValue
let aliasreference = LoadDocument <ResourceOntologyReferences>("ResourceOntologyReferences/" + (ontologyresource.Context ?? ontology.Context) + "/" + resourceId)
from alias in LoadDocument <ResourceOntology>(aliasreference.ReduceOutputs)
from aliasproperty in alias.Properties.Where(p => p.Name == "@alias")
from aliaspropertyreference in LoadDocument <ResourceOntologyReferences>(aliasproperty.Source)
from aliaspropertyresource in LoadDocument <ResourceOntology>(aliaspropertyreference.ReduceOutputs)
from resourceproperty in aliaspropertyresource.Properties.Where(p => p.Name == "@resource")
select resourceproperty.Source
)
from propertyresource in
from resourcemapping in LoadDocument <ResourceMapping>(source)
let propertyresourceontologyreference = LoadDocument <ResourceOntologyReferences>(resourcemapping.Type.Select(type => "ResourceOntologyReferences/" + resourcemapping.Context + "/" + type))
let propertyresourceontology = LoadDocument <ResourceOntology>(propertyresourceontologyreference.SelectMany(r => r.ReduceOutputs))
select new Resource {
Context = resourcemapping.Context,
ResourceId = resourcemapping.ResourceId,
Type = resourcemapping.Type,
SubType = resourcemapping.SubType,
Title = resourcemapping.Title,
SubTitle = resourcemapping.SubTitle,
Code = resourcemapping.Code,
Status = resourcemapping.Status,
Tags = resourcemapping.Tags,
Properties =
from resourcemappingontologyproperty in propertyresourceontology.SelectMany(r => r.Properties).Where(p => p.Name.StartsWith("@"))
let derivedproperties =
from derivedproperty in resourcemapping.Properties
where resourcemappingontologyproperty.Tags.Contains("@derive")
from ontologyderivedproperty in resourcemappingontologyproperty.Properties
where ontologyderivedproperty.Name == derivedproperty.Name &&
ontologyderivedproperty.Tags.All(t => derivedproperty.Tags.Contains(t)) &&
(ontologyderivedproperty.From == null || ontologyderivedproperty.From <= (derivedproperty.Thru ?? DateTime.MaxValue)) &&
(ontologyderivedproperty.Thru == null || ontologyderivedproperty.Thru >= (derivedproperty.From ?? DateTime.MinValue))
select derivedproperty
select new Property {
Name = resourcemappingontologyproperty.Name,
Value = (
from value in resourcemappingontologyproperty.Value
from formattedvalue in ResourceFormat(value, resourcemapping, derivedproperties)
select formattedvalue
).Where(v => !String.IsNullOrWhiteSpace(v))
},
Source = resourcemapping.Source
}
select new Resource {
Context = propertyresource.Context,
ResourceId = propertyresource.ResourceId,
Type = propertyresource.Type.Union(propertyresource.Properties.Where(p => p.Name == "@type").SelectMany(p => p.Value)).Distinct(),
SubType = propertyresource.SubType.Union(propertyresource.Properties.Where(p => p.Name == "@subtype").SelectMany(p => p.Value)).Distinct(),
Title = propertyresource.Title.Union(propertyresource.Properties.Where(p => p.Name == "@title").SelectMany(p => p.Value)).Distinct(),
SubTitle = propertyresource.SubTitle.Union(propertyresource.Properties.Where(p => p.Name == "@subtitle").SelectMany(p => p.Value)).Distinct(),
Code = propertyresource.Code.Union(propertyresource.Properties.Where(p => p.Name == "@code").SelectMany(p => p.Value)).Distinct(),
Status = propertyresource.Status.Union(propertyresource.Properties.Where(p => p.Name == "@status").SelectMany(p => p.Value)).Distinct(),
Tags = propertyresource.Tags.Union(propertyresource.Properties.Where(p => p.Name == "@tags").SelectMany(p => p.Value)).Distinct(),
Source = propertyresource.Source
}
).Union(
ontologyproperty.Resources.Where(r => !r.Properties.Any(p => p.Name == "@resourceId"))
),
Properties = property.SelectMany(p => p.Properties).Union(ontologyproperty.Properties)
}).Where(p => p.Value.Any() || p.Resources.Any()).Union(ontology.Properties.Where(p => p.Name.StartsWith("@"))),
Source = new[] { MetadataFor(resource).Value <String>("@id") },
Modified = MetadataFor(resource).Value <DateTime>("@last-modified")
}
);
AddMap <ResourceOntology>(ontologies =>
from ontology in ontologies.Where(r => r.Tags.Contains("@push"))
from pushresource in (
from tags in ontology.Tags.Where(t => t == "@push")
select new Resource {
Context = ontology.Context,
ResourceId = ontology.ResourceId
}
).Union(
from resourceproperty in ontology.Properties.Where(p => p.Name == "@resource")
where ontology.Tags.Any(t => t == "@alias")
from aliasresource in resourceproperty.Resources
select new Resource {
Context = aliasresource.Context,
ResourceId = aliasresource.ResourceId
}
).Union(
from aliasproperty in ontology.Properties.Where(p => p.Name == "@alias")
from aliasreference in LoadDocument <ResourceOntologyReferences>(aliasproperty.Source).Where(r => r != null)
from aliasresource in LoadDocument <ResourceOntology>(aliasreference.ReduceOutputs).Where(r => r != null)
from resourceproperty in aliasresource.Properties.Where(p => p.Name == "@resource")
from resourcealias in resourceproperty.Resources
select new Resource {
Context = resourcealias.Context,
ResourceId = resourcealias.ResourceId
}
)
let pushmappingreference = LoadDocument <ResourceMappingReferences>("ResourceMappingReferences/" + pushresource.Context + "/" + pushresource.ResourceId)
from pushpropertyresource in
from resourcemapping in LoadDocument <ResourceMapping>(pushmappingreference.ReduceOutputs)
let propertyresourceontologyreference = LoadDocument <ResourceMappingReferences>(resourcemapping.Type.Select(type => "ResourceMappingReferences/" + resourcemapping.Context + "/" + type))
let propertyresourceontology = LoadDocument <ResourceMapping>(propertyresourceontologyreference.SelectMany(r => r.ReduceOutputs))
select new Resource {
Context = resourcemapping.Context,
ResourceId = resourcemapping.ResourceId,
Type = resourcemapping.Type,
SubType = resourcemapping.SubType,
Status = resourcemapping.Status,
Tags = resourcemapping.Tags,
Properties =
from resourcemappingontologyproperty in propertyresourceontology.SelectMany(r => r.Properties).Where(p => new string[] { "@type", "@subtype", "@status", "@tags" }.Contains(p.Name))
let derivedproperties =
from derivedproperty in resourcemapping.Properties
where resourcemappingontologyproperty.Tags.Contains("@derive")
from ontologyderivedproperty in resourcemappingontologyproperty.Properties
where ontologyderivedproperty.Name == derivedproperty.Name &&
ontologyderivedproperty.Tags.All(t => derivedproperty.Tags.Contains(t)) &&
(ontologyderivedproperty.From == null || ontologyderivedproperty.From <= (derivedproperty.Thru ?? DateTime.MaxValue)) &&
(ontologyderivedproperty.Thru == null || ontologyderivedproperty.Thru >= (derivedproperty.From ?? DateTime.MinValue))
select derivedproperty
select new Property {
Name = resourcemappingontologyproperty.Name,
Value = (
from value in resourcemappingontologyproperty.Value
from formattedvalue in ResourceFormat(value, resourcemapping, derivedproperties)
select formattedvalue
).Where(v => !String.IsNullOrWhiteSpace(v))
}
}
from resource in LoadDocument <ResourceMapping>(ontology.Source).Where(r => r != null)
let resourceontologyreference = LoadDocument <ResourceMappingReferences>(resource.Type.Select(type => "ResourceMappingReferences/" + resource.Context + "/" + type))
from resourceontology in LoadDocument <ResourceMapping>(resourceontologyreference.SelectMany(r => r.ReduceOutputs))
from ontologyresource in resourceontology.Properties.SelectMany(p => p.Resources).Where(r => r.Tags.Contains("@push") || r.Properties.Any(p => p.Tags.Contains("@push")))
where ontologyresource.Context == pushpropertyresource.Context &&
ontologyresource.Type.All(t => pushpropertyresource.Type.Union(pushpropertyresource.Properties.Where(p => p.Name == "@type").SelectMany(p => p.Value).Select(v => v.ToString())).Contains(t.ToString())) &&
ontologyresource.SubType.All(t => pushpropertyresource.SubType.Union(pushpropertyresource.Properties.Where(p => p.Name == "@subtype").SelectMany(p => p.Value).Select(v => v.ToString())).Contains(t.ToString())) &&
ontologyresource.Status.All(s => pushpropertyresource.Status.Union(pushpropertyresource.Properties.Where(p => p.Name == "@status").SelectMany(p => p.Value).Select(v => v.ToString())).Contains(s.ToString())) &&
ontologyresource.Tags.All(t => pushpropertyresource.Tags.Union(pushpropertyresource.Properties.Where(p => p.Name == "@tags").SelectMany(p => p.Value).Select(v => v.ToString())).Contains(t.ToString()))
select new Resource {
Context = pushresource.Context,
ResourceId = pushresource.ResourceId,
Type = ontologyresource.Properties.Where(p => p.Name == "@type").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null)).Distinct(),
SubType = ontologyresource.Properties.Where(p => p.Name == "@subtype").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null)).Distinct(),
Title = ontologyresource.Properties.Where(p => p.Name == "@title").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null)).Distinct(),
SubTitle = ontologyresource.Properties.Where(p => p.Name == "@subtitle").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null)).Distinct(),
Code = ontologyresource.Properties.Where(p => p.Name == "@code").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null)).Distinct(),
Status = ontologyresource.Properties.Where(p => p.Name == "@status").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null)).Distinct(),
Tags = ontologyresource.Properties.Where(p => p.Name == "@tags").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null)).Distinct(),
Properties = (
from ontologyproperty in ontologyresource.Properties.Where(p => !p.Name.StartsWith("@"))
let property = (!ontologyproperty.Tags.Contains("@derive")) ? resource.Properties.Where(p => p.Name == ontologyproperty.Name) :
from ontologyderivedproperty in ontologyproperty.Properties
from derivedproperty in resource.Properties
where ontologyderivedproperty.Name == derivedproperty.Name &&
ontologyderivedproperty.Tags.All(t => derivedproperty.Tags.Contains(t)) &&
(ontologyderivedproperty.From == null || ontologyderivedproperty.From <= (derivedproperty.Thru ?? DateTime.MaxValue)) &&
(ontologyderivedproperty.Thru == null || ontologyderivedproperty.Thru >= (derivedproperty.From ?? DateTime.MinValue))
select derivedproperty
select new Property {
Name = ontologyproperty.Name,
Value = (ontologyproperty.Value.Any()) ? ontologyproperty.Value.SelectMany(v => ResourceFormat(v, resource, property)) : property.SelectMany(p => p.Value),
Tags = property.SelectMany(p => p.Tags).Union(ontologyproperty.Tags).Select(v => v).Distinct(),
Resources = (
from propertyresource in property.SelectMany(p => p.Resources)
where String.IsNullOrEmpty(propertyresource.ResourceId)
select propertyresource
).Union(
from propertyresource in property.SelectMany(p => p.Resources)
where !String.IsNullOrEmpty(propertyresource.ResourceId)
select new Resource {
Context = propertyresource.Context ?? ontology.Context,
ResourceId = propertyresource.ResourceId
}
).Union(
from ontologyresource in ontologyproperty.Resources
from resourceId in
from resourceIdValue in ontologyresource.Properties.Where(p => p.Name == "@resourceId").SelectMany(p => p.Value)
from resourceIdFormattedValue in ResourceFormat(resourceIdValue, resource, property)
select resourceIdFormattedValue
select new Resource {
Context = ontologyresource.Context ?? ontology.Context,
ResourceId = resourceId
}
).Union(
ontologyproperty.Resources.Where(r => !r.Properties.Any(p => p.Name == "@resourceId"))
),
Properties = property.SelectMany(p => p.Properties).Union(ontologyproperty.Properties),
Source = (ontologyproperty.Tags.Contains("@push")) ? new[] { MetadataFor(resource).Value <String>("@id") } : new string[] { },
}).Where(p => p.Value.Any() || p.Resources.Any()).Union(ontology.Properties.Where(p => p.Name.StartsWith("@") && p.Name != "@resource")),
Source = (ontology.Tags.Contains("@push")) ? new[] { MetadataFor(resource).Value <String>("@id") } : new string[] { },
Modified = MetadataFor(resource).Value <DateTime>("@last-modified")
}
);
Reduce = results =>
from result in results
group result by new { result.Context, result.ResourceId } into g
let computedProperties =
from property in g.SelectMany(r => r.Properties).Where(p => p.Name.StartsWith("@") && !p.Tags.Contains("@reasoning"))
select new Property {
Name = property.Name,
Value = (
from value in property.Value
from resource in g.ToList()
from formattedvalue in ResourceFormat(value, resource, null)
select formattedvalue
).Where(v => !String.IsNullOrWhiteSpace(v))
}
select new Resource {
Context = g.Key.Context,
ResourceId = g.Key.ResourceId,
Type = g.SelectMany(r => r.Type).Union(computedProperties.Where(p => p.Name == "@type").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(),
SubType = g.SelectMany(r => r.SubType).Union(computedProperties.Where(p => p.Name == "@subtype").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(),
Title = g.SelectMany(r => r.Title).Union(computedProperties.Where(p => p.Name == "@title").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(),
SubTitle = g.SelectMany(r => r.SubTitle).Union(computedProperties.Where(p => p.Name == "@subtitle").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(),
Code = g.SelectMany(r => r.Code).Union(computedProperties.Where(p => p.Name == "@code").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(),
Status = g.SelectMany(r => r.Status).Union(computedProperties.Where(p => p.Name == "@status").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(),
Tags = g.SelectMany(r => r.Tags).Union(computedProperties.Where(p => p.Name == "@tags").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(),
Properties = (IEnumerable <Property>)Properties(g.SelectMany(r => r.Properties)),
Source = g.SelectMany(r => r.Source).Distinct(),
Modified = g.Select(r => r.Modified).Max()
};
Index(Raven.Client.Constants.Documents.Indexing.Fields.AllFields, FieldIndexing.No);
OutputReduceToCollection = "ResourceProperty";
PatternReferencesCollectionName = "ResourcePropertyReferences";
PatternForOutputReduceToCollectionReferences = r => $"ResourcePropertyReferences/{r.Context}/{r.ResourceId}";
AdditionalAssemblies = new HashSet <AdditionalAssembly> {
AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> {
"Digitalisert.Dataplattform"
})
};
}
public N50KartdataResourceIndex()
{
AddMap <Kommune>(n50kartdata =>
from kommune in n50kartdata.WhereEntityIs <Kommune>("N50Kartdata")
let metadata = MetadataFor(kommune)
where metadata.Value <string>("@id").StartsWith("N50Kartdata/Kommune")
select new Resource
{
ResourceId = "Kommune/" + kommune.kommunenummer,
Type = new[] { "Kommune" },
SubType = new string[] { },
Title = new[] { kommune.navn },
Code = new[] { kommune.kommunenummer },
Status = new string[] { },
Properties =
from wkt in new[] { kommune._omrade.wkt }.Where(v => !String.IsNullOrWhiteSpace(v))
select new Property {
Name = "Område", Tags = new[] { "@wkt" }, Value = new[] { WKTProjectToWGS84(wkt, 0) }
},
Source = new[] { metadata.Value <string>("@id") },
Modified = kommune.oppdateringsdato
}
);
AddMap <Kommune>(n50kartdata =>
from kommune in n50kartdata.WhereEntityIs <Kommune>("N50Kartdata")
let metadata = MetadataFor(kommune)
where metadata.Value <string>("@id").StartsWith("N50Kartdata/Kommune")
from fylke in new[] {
new { Code = "03", Title = "Oslo" },
new { Code = "11", Title = "Rogaland" },
new { Code = "15", Title = "Møre og Romsdal" },
new { Code = "18", Title = "Nordland" },
new { Code = "30", Title = "Viken" },
new { Code = "34", Title = "Innlandet" },
new { Code = "38", Title = "Vestfold og Telemark" },
new { Code = "42", Title = "Agder" },
new { Code = "46", Title = "Vestland" },
new { Code = "50", Title = "Trøndelag" },
new { Code = "54", Title = "Troms og Finnmark" },
}
where fylke.Code == kommune.kommunenummer.Substring(0, 2)
select new Resource
{
ResourceId = "Fylke/" + fylke.Code,
Type = new[] { "Fylke" },
SubType = new string[] { },
Title = new[] { fylke.Title },
Code = new[] { fylke.Code },
Status = new string[] { },
Properties = (
new[] {
new Property {
Name = "Kommune",
Resources = new[] { new Resource {
ResourceId = "Kommune/" + kommune.kommunenummer
} }
}
}
).Union(
from wkt in new[] { kommune._omrade.wkt }.Where(v => !String.IsNullOrWhiteSpace(v))
select new Property {
Name = "Område", Tags = new[] { "@wkt", "@union" }, Value = new[] { WKTProjectToWGS84(wkt, 0) }
}
),
Source = new[] { metadata.Value <string>("@id") },
Modified = kommune.oppdateringsdato
}
);
AddMap <NaturvernOmrade>(n50kartdata =>
from naturvernomrade in n50kartdata.WhereEntityIs <NaturvernOmrade>("N50Kartdata")
let metadata = MetadataFor(naturvernomrade)
where metadata.Value <string>("@id").StartsWith("N50Kartdata/NaturvernOmrade")
select new Resource
{
ResourceId = "Naturvern/" + naturvernomrade.verneform + "/" + naturvernomrade.navn,
Type = new[] { "Naturvernområde" },
SubType = new[] { LoadDocument <Verneform>("N50Kartdata/Verneform/" + naturvernomrade.verneform).description }.Where(s => !String.IsNullOrWhiteSpace(s)),
Title = new[] { naturvernomrade.navn },
Code = new string[] { },
Status = new string[] { },
Properties =
from wkt in new[] { naturvernomrade._omrade.wkt }.Where(v => !String.IsNullOrWhiteSpace(v))
select new Property {
Name = "Område", Tags = new[] { "@wkt" }, Value = new[] { WKTProjectToWGS84(wkt, 0) }
},
Source = new[] { metadata.Value <string>("@id") },
Modified = naturvernomrade.oppdateringsdato
}
);
AddMap <Sti>(n50kartdata =>
from sti in n50kartdata.WhereEntityIs <Sti>("N50Kartdata")
let metadata = MetadataFor(sti)
where metadata.Value <string>("@id").StartsWith("N50Kartdata/Sti")
select new Resource
{
ResourceId = "Sti/" + sti.objid,
Type = new[] { "Sti" },
SubType = new string[] { sti.vedlikeholdsansvarlig }.Where(t => !String.IsNullOrWhiteSpace(t)),
Title = new string[] { },
Code = new string[] { },
Status = new string[] { (sti.merking == "JA") ? "Merket" : "" }.Where(t => !String.IsNullOrWhiteSpace(t)),
Properties =
from wkt in new[] { sti._senterlinje.wkt }.Where(v => !String.IsNullOrWhiteSpace(v))
select new Property {
Name = "Senterlinje", Tags = new[] { "@wkt" }, Value = new[] { WKTProjectToWGS84(wkt, 0) }
},
Source = new[] { metadata.Value <string>("@id") },
Modified = sti.oppdateringsdato
}
);
Reduce = results =>
from result in results
group result by result.ResourceId into g
select new Resource
{
ResourceId = g.Key,
Type = g.SelectMany(r => r.Type).Distinct(),
SubType = g.SelectMany(r => r.SubType).Distinct(),
Title = g.SelectMany(r => r.Title).Distinct(),
Code = g.SelectMany(r => r.Code).Distinct(),
Status = g.SelectMany(r => r.Status).Distinct(),
Properties = (IEnumerable <Property>)Properties(g.SelectMany(r => r.Properties)),
Source = g.SelectMany(resource => resource.Source).Distinct(),
Modified = g.Select(resource => resource.Modified).Max()
};
Index(Raven.Client.Constants.Documents.Indexing.Fields.AllFields, FieldIndexing.No);
OutputReduceToCollection = "N50KartdataResource";
AdditionalAssemblies = new HashSet <AdditionalAssembly> {
AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> {
"Digitalisert.Dataplattform"
})
};
}
public ResourceIndex()
{
AddMap <Resource>(resources =>
from resource in resources
let source = LoadDocument <ResourceMapping>(resource.Source).Where(r => r != null)
let properties =
from property in resource.Properties
select new Property {
Name = property.Name,
Value = property.Value,
Tags = property.Tags,
Resources = (
from propertyresource in property.Resources
where propertyresource.ResourceId == null
select propertyresource
).Union(
from propertyresource in property.Resources
where propertyresource.ResourceId != null
let propertyresourcereduceoutputs = LoadDocument <ResourceReferences>("ResourceReferences/" + propertyresource.Context + "/" + propertyresource.ResourceId).ReduceOutputs
let propertyresourceoutputs = LoadDocument <Resource>(propertyresourcereduceoutputs)
select new Resource {
Context = propertyresource.Context,
ResourceId = propertyresource.ResourceId,
Type = propertyresourceoutputs.SelectMany(r => r.Type).Distinct(),
SubType = propertyresourceoutputs.SelectMany(r => r.SubType).Distinct(),
Title = propertyresourceoutputs.SelectMany(r => r.Title).Distinct(),
SubTitle = propertyresourceoutputs.SelectMany(r => r.SubTitle).Distinct(),
Code = propertyresourceoutputs.SelectMany(r => r.Code).Distinct(),
Body = propertyresourceoutputs.SelectMany(r => r.Properties.Where(p => p.Name == "@body").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, r, null))).Distinct(),
Status = propertyresourceoutputs.SelectMany(r => r.Status).Distinct(),
Tags = propertyresourceoutputs.SelectMany(r => r.Tags).Union(
propertyresourceoutputs.SelectMany(r => r.Properties).SelectMany(p => p.Tags).Where(t => t == "@wkt").Take(1)
).Distinct()
}
)
}
let body = source.SelectMany(r => r.Body ?? new string[] { }).Union(properties.Where(p => p.Name == "@body").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, null))).Distinct()
select new Resource
{
Context = resource.Context,
ResourceId = resource.ResourceId,
Type = resource.Type,
SubType = resource.SubType,
Title = resource.Title,
SubTitle = resource.SubTitle,
Code = resource.Code,
Body = body,
Status = resource.Status,
Tags = resource.Tags.Union(properties.Where(p => p.Name == "@tags").SelectMany(p => p.Value).SelectMany(v => ResourceFormat(v, resource, properties))).Select(v => v.ToString()).Distinct(),
Properties = properties.Where(p => !p.Name.StartsWith("@")),
_ = (
from property in properties.Where(p => !p.Name.StartsWith("@"))
group property by property.Name into propertyG
select CreateField(
propertyG.Key,
propertyG.Where(p => !p.Tags.Contains("@wkt")).SelectMany(p => p.Value).Select(v => v.ToString()).Union(
from propertyresource in propertyG.SelectMany(p => p.Resources)
from fieldvalue in new[] { propertyresource.ResourceId }.Union(propertyresource.Code).Union(propertyresource.Title).Union(propertyresource.SubTitle)
select fieldvalue
).Where(v => !String.IsNullOrWhiteSpace(v)).Distinct()
)
).Union(
from property in properties.Where(p => !p.Name.StartsWith("@"))
group property by property.Name into propertyG
from resourcetype in propertyG.SelectMany(p => p.Resources).SelectMany(r => r.Type).Distinct()
select CreateField(
propertyG.Key + "." + resourcetype,
(
from propertyresource in propertyG.SelectMany(p => p.Resources).Where(r => r.Type.Contains(resourcetype))
from fieldvalue in new[] { propertyresource.ResourceId }.Union(propertyresource.Code).Union(propertyresource.Title)
select fieldvalue
).Where(v => !String.IsNullOrWhiteSpace(v)).Distinct()
)
).Union(
new object[] {
CreateField(
"@resources",
properties.Where(p => !p.Name.StartsWith("@")).SelectMany(p => p.Resources).Select(r => r.Context + "/" + r.ResourceId).Distinct()
)
}
).Union(
new object[] {
CreateField(
"Properties",
properties.Where(p => !p.Name.StartsWith("@")).Select(p => p.Name).Where(n => !n.StartsWith("@")).Distinct(),
new CreateFieldOptions {
Indexing = FieldIndexing.Exact
}
)
}
).Union(
new object[] {
CreateField(
"Search",
resource.Title.Union(resource.SubTitle).Union(resource.Code).Union(body).Distinct(),
new CreateFieldOptions {
Indexing = FieldIndexing.Search, Storage = FieldStorage.Yes, TermVector = FieldTermVector.WithPositionsAndOffsets
}
)
}
)
}
);
Index(r => r.Context, FieldIndexing.Exact);
Index(r => r.Type, FieldIndexing.Exact);
Index(r => r.SubType, FieldIndexing.Exact);
Index(r => r.Code, FieldIndexing.Exact);
Index(r => r.Status, FieldIndexing.Exact);
Index(r => r.Tags, FieldIndexing.Exact);
Index(r => r.Properties, FieldIndexing.No);
Index(r => r.Title, FieldIndexing.Search);
Index(r => r.SubTitle, FieldIndexing.Search);
Index(r => r.Body, FieldIndexing.Search);
Store(r => r.Context, FieldStorage.Yes);
Store(r => r.Type, FieldStorage.Yes);
Store(r => r.SubType, FieldStorage.Yes);
Store(r => r.Title, FieldStorage.Yes);
Store(r => r.SubTitle, FieldStorage.Yes);
Store(r => r.Code, FieldStorage.Yes);
Store(r => r.Body, FieldStorage.Yes);
Store(r => r.Status, FieldStorage.Yes);
Store(r => r.Tags, FieldStorage.Yes);
Store(r => r.Properties, FieldStorage.Yes);
Analyzers.Add(x => x.Title, "SimpleAnalyzer");
Analyzers.Add(x => x.SubTitle, "SimpleAnalyzer");
Analyzers.Add(x => x.Body, "SimpleAnalyzer");
AdditionalAssemblies = new HashSet <AdditionalAssembly> {
AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> {
"Digitalisert.Dataplattform"
})
};
}
public ResourceReasonerIndex()
{
AddMap <ResourceProperty>(resources =>
from resource in resources
select new Resource
{
Context = resource.Context,
ResourceId = resource.ResourceId,
Type = resource.Type,
SubType = resource.SubType,
Title = resource.Title,
SubTitle = resource.SubTitle,
Code = resource.Code,
Status = resource.Status,
Tags = resource.Tags,
Properties = (
from property in resource.Properties.Where(r => !r.Name.StartsWith("@"))
select new Property
{
Name = property.Name,
Value = property.Value,
Tags = property.Tags,
Resources = (
property.Resources.Where(r => r.ResourceId == null)
).Union(
from propertyresource in property.Resources.Where(r => r.ResourceId != null)
let reduceoutputs = LoadDocument <ResourcePropertyReferences>("ResourcePropertyReferences/" + propertyresource.Context + "/" + propertyresource.ResourceId).ReduceOutputs
let resourceoutputs = LoadDocument <ResourceProperty>(reduceoutputs)
select new Resource
{
Context = propertyresource.Context,
ResourceId = propertyresource.ResourceId,
Type = (propertyresource.Type ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.Type)).Select(v => v.ToString()).Distinct(),
SubType = (propertyresource.SubType ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.SubType)).Select(v => v.ToString()).Distinct(),
Title = (propertyresource.Title ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.Title)).Select(v => v.ToString()).Distinct(),
SubTitle = (propertyresource.SubTitle ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.SubTitle)).Select(v => v.ToString()).Distinct(),
Code = (propertyresource.Code ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.Code)).Select(v => v.ToString()).Distinct(),
Status = (propertyresource.Status ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.Status)).Select(v => v.ToString()).Distinct(),
Tags = (propertyresource.Tags ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.Tags)).Select(v => v.ToString()).Distinct(),
Source = (propertyresource.Source ?? new string[] { }).Union(resourceoutputs.SelectMany(r => r.Source)).Select(v => v.ToString()).Distinct()
}
)
}
).Union(
resource.Properties.Where(r => r.Name.StartsWith("@"))
),
Source = resource.Source,
Modified = resource.Modified ?? DateTime.MinValue
}
);
AddMap <ResourceProperty>(resources =>
from resource in resources
from property in resource.Properties
from inverseproperty in property.Properties.Where(p => p.Tags.Contains("@inverse"))
from propertyresource in property.Resources.Where(r => r.ResourceId != null)
where LoadDocument <ResourcePropertyReferences>("ResourcePropertyReferences/" + propertyresource.Context + "/" + propertyresource.ResourceId) != null
select new Resource {
Context = propertyresource.Context,
ResourceId = propertyresource.ResourceId,
Type = new string[] {},
SubType = new string[] {},
Title = new string[] {},
SubTitle = new string[] {},
Code = new string[] {},
Status = new string[] {},
Tags = new string[] {},
Properties = new[] {
new Property {
Name = inverseproperty.Name,
Resources = new[] {
new Resource {
Context = resource.Context,
ResourceId = resource.ResourceId,
Type = resource.Type,
SubType = resource.SubType,
Title = resource.Title,
SubTitle = resource.SubTitle,
Code = resource.Code,
Status = resource.Status,
Tags = resource.Tags,
Source = resource.Source
}
}
}
},
Source = new string[] { },
Modified = DateTime.MinValue
}
);
AddMap <ResourceDerivedProperty>(resources =>
from resource in resources
from resourceproperty in LoadDocument <ResourceProperty>(resource.Source).Where(r => r != null)
from property in resourceproperty.Properties.Where(p => p.Name == resource.Name)
from compareproperty in resource.Properties.Select(p => p.Name.Replace("+", "")).Distinct()
from compareresourceproperty in (
from compare in resource.Properties.Where(p => p.Name == compareproperty + "+")
from compareresource in LoadDocument <ResourceProperty>(compare.Source).Where(r => r != null)
select compareresource
).Union(
from compare in resource.Properties.Where(p => p.Name == compareproperty)
let comparedsources = resource.Properties.Where(p => p.Name == compareproperty + "+").SelectMany(p => p.Source)
let comparesources = compare.Source.Where(s => !comparedsources.Contains(s))
where comparesources.Any()
from compareresource in LoadDocument <ResourceProperty>(comparesources).Where(r => r != null)
where property.Value.Any(v1 => compareresource.Properties.Where(p => p.Name == compareproperty).SelectMany(p => p.Value).Any(v2 => WKTIntersects(v1, v2)))
select compareresource
)
from derivedproperty in (
from ontologyresource in property.Resources
from ontologyproperty in ontologyresource.Properties
where ontologyproperty.Name == compareproperty &&
ontologyresource.Context == compareresourceproperty.Context &&
ontologyresource.Type.All(t => compareresourceproperty.Type.Contains(t))
select new {
fromresource = resourceproperty,
name = property.Name,
toresource = compareresourceproperty
}
).Union(
from ontologyproperty in property.Properties
from ontologyresource in ontologyproperty.Resources
where ontologyproperty.Name == compareproperty &&
ontologyresource.Context == compareresourceproperty.Context &&
ontologyresource.Type.All(t => compareresourceproperty.Type.Contains(t))
select new {
fromresource = compareresourceproperty,
name = compareproperty,
toresource = resourceproperty
}
)
select new Resource
{
Context = derivedproperty.fromresource.Context,
ResourceId = derivedproperty.fromresource.ResourceId,
Type = new string[] {},
SubType = new string[] {},
Title = new string[] {},
SubTitle = new string[] {},
Code = new string[] {},
Status = new string[] {},
Tags = new string[] {},
Properties = new[] {
new Property
{
Name = derivedproperty.name,
Resources = new[] {
new Resource
{
Context = derivedproperty.toresource.Context,
ResourceId = derivedproperty.toresource.ResourceId,
Type = derivedproperty.toresource.Type,
SubType = derivedproperty.toresource.SubType,
Title = derivedproperty.toresource.Title,
SubTitle = derivedproperty.toresource.SubTitle,
Code = derivedproperty.toresource.Code,
Status = derivedproperty.toresource.Status,
Tags = derivedproperty.toresource.Tags,
Source = derivedproperty.toresource.Source
}
}
}
},
Source = new string[] { },
Modified = DateTime.MinValue
}
);
Reduce = results =>
from result in results
group result by new { result.Context, result.ResourceId } into g
let computedProperties =
from property in g.SelectMany(r => r.Properties).Where(p => p.Name.StartsWith("@"))
select new Property {
Name = property.Name,
Value = (
from value in property.Value
from resource in g.ToList()
from formattedvalue in ResourceFormat(value, resource, null)
select formattedvalue
).Where(v => !String.IsNullOrWhiteSpace(v))
}
select new Resource
{
Context = g.Key.Context,
ResourceId = g.Key.ResourceId,
Type = g.SelectMany(r => r.Type).Union(computedProperties.Where(p => p.Name == "@type").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(),
SubType = g.SelectMany(r => r.SubType).Union(computedProperties.Where(p => p.Name == "@subtype").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(),
Title = g.SelectMany(r => r.Title).Union(computedProperties.Where(p => p.Name == "@title").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(),
SubTitle = g.SelectMany(r => r.SubTitle).Union(computedProperties.Where(p => p.Name == "@subtitle").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(),
Code = g.SelectMany(r => r.Code).Union(computedProperties.Where(p => p.Name == "@code").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(),
Status = g.SelectMany(r => r.Status).Union(computedProperties.Where(p => p.Name == "@status").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(),
Tags = g.SelectMany(r => r.Tags).Union(computedProperties.Where(p => p.Name == "@tags").SelectMany(p => p.Value)).Select(v => v.ToString()).Distinct(),
Properties = (
from property in g.SelectMany(r => r.Properties).Where(r => !r.Name.StartsWith("@"))
group property by property.Name into propertyG
select new Property {
Name = propertyG.Key,
Value = propertyG.SelectMany(p => p.Value).Distinct(),
Tags = propertyG.SelectMany(p => p.Tags).Distinct(),
Resources = (
propertyG.SelectMany(p => p.Resources).Where(r => r.ResourceId == null).Distinct()
).Union(
from resource in propertyG.SelectMany(p => p.Resources).Where(r => r.ResourceId != null)
group resource by new { resource.Context, resource.ResourceId } into resourceG
select new Resource {
Context = resourceG.Key.Context,
ResourceId = resourceG.Key.ResourceId,
Type = resourceG.SelectMany(r => r.Type).Distinct(),
SubType = resourceG.SelectMany(r => r.SubType).Distinct(),
Title = resourceG.SelectMany(r => r.Title).Distinct(),
SubTitle = resourceG.SelectMany(r => r.SubTitle).Distinct(),
Code = resourceG.SelectMany(r => r.Code).Distinct(),
Status = resourceG.SelectMany(r => r.Status).Distinct(),
Tags = resourceG.SelectMany(r => r.Tags).Distinct(),
Source = resourceG.SelectMany(r => r.Source).Distinct()
}
)
}
).Union(
g.SelectMany(r => r.Properties).Where(r => r.Name.StartsWith("@"))
),
Source = g.SelectMany(resource => resource.Source).Distinct(),
Modified = g.Select(resource => resource.Modified ?? DateTime.MinValue).Max()
};
Index(Raven.Client.Constants.Documents.Indexing.Fields.AllFields, FieldIndexing.No);
OutputReduceToCollection = "Resource";
PatternReferencesCollectionName = "ResourceReferences";
PatternForOutputReduceToCollectionReferences = r => $"ResourceReferences/{r.Context}/{r.ResourceId}";
AdditionalAssemblies = new HashSet <AdditionalAssembly> {
AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> {
"Digitalisert.Dataplattform"
})
};
}
public void Example()
{
using (var store = new DocumentStore())
{
#region complex_index
store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition
{
Name = "Photographs/Tags",
Maps =
{
@"
from p in docs.Photographs
let photo = LoadAttachment(p, ""photo.png"")
where photo != null
let classified = ImageClassifier.Classify(photo.GetContentAsStream())
select new {
e.Name,
Tag = classified.Where(x => x.Value > 0.75f).Select(x => x.Key),
_ = classified.Select(x => CreateField(x.Key, x.Value))
}"
},
AdditionalSources = new System.Collections.Generic.Dictionary <string, string>
{
{
"ImageClassifier",
@"
public static class ImageClassifier
{
public static IDictionary<string, float> Classify(Stream s)
{
// returns a list of descriptors with a
// value between 0 and 1 of how well the
// image matches that descriptor.
}
}"
}
},
AdditionalAssemblies =
{
AdditionalAssembly.FromRuntime("System.Memory"),
AdditionalAssembly.FromNuGet("System.Drawing.Common","4.7.0"),
AdditionalAssembly.FromNuGet("Microsoft.ML", "1.5.2")
}
}));
#endregion
}
using (var store = new DocumentStore())
{
using (var session = store.OpenSession())
{
#region simple_index
var runtimeindex = new IndexDefinition
{
Name = "Dog_Pictures",
Maps = { @"
from user in docs.Users
let fileName = Path.GetFileName(user.ImagePath)
where fileName = ""My_Dogs.jpeg""
select new {
user.Name,
fileName
}" },
AdditionalAssemblies =
{
AdditionalAssembly.FromRuntime("System.IO")
}
};
#endregion
}
}
}
public SSBResourceIndex()
{
AddMap <SSB>(statistikker =>
from statistikk in statistikker
let metadata = MetadataFor(statistikk)
where metadata.Value <string>("@id") == "SSB/1108"
from data in statistikk.Data
select new Resource
{
ResourceId = data["region"].Substring(0, 4),
Type = new[] { "Kommune" },
SubType = new string [] { data["kvartal"] },
Title = new[] { data["region"].Substring(5) },
Code = new[] { data["region"].Substring(0, 4) },
Status = new string[] { },
Tags = new string[] { },
Properties = new[] {
new Property {
Name = data["statistikkvariabel"], Value = new[] { data["01222: Befolkning og kvartalsvise endringar, etter region, kvartal og statistikkvariabel"] }
}
},
Source = new[] { metadata.Value <string>("@id") }
}
);
AddMap <SSB>(statistikker =>
from statistikk in statistikker
let metadata = MetadataFor(statistikk)
where metadata.Value <string>("@id") == "SSB/26975"
from data in statistikk.Data
select new Resource
{
ResourceId = data["region"].Substring(2, 4),
Type = new string[] { "Kommune" },
SubType = new string [] { },
Title = new[] { data["region"].Substring(7) },
Code = new[] { data["region"].Substring(2, 4) },
Status = new string[] { },
Tags = new string[] { },
Properties = new[] {
new Property {
Name = data["statistikkvariabel"],
Value = new[] { data["07459: Befolkning, etter region, år og statistikkvariabel"] },
Tags = new[] { "@history" },
Thru = new DateTime(int.Parse(data["år"]), 12, 31)
}
},
Source = new[] { metadata.Value <string>("@id") }
}
);
Reduce = results =>
from result in results
group result by result.ResourceId into g
select new Resource
{
ResourceId = g.Key,
Type = g.SelectMany(r => r.Type).Distinct(),
SubType = g.SelectMany(r => r.SubType).Distinct(),
Title = g.SelectMany(r => r.Title).Distinct(),
Code = g.SelectMany(r => r.Code).Distinct(),
Status = g.SelectMany(r => r.Status).Distinct(),
Tags = g.SelectMany(r => r.Tags).Distinct(),
Properties = (IEnumerable <Property>)Properties(g.SelectMany(r => r.Properties)),
Source = g.SelectMany(resource => resource.Source).Distinct()
};
Index(Raven.Client.Constants.Documents.Indexing.Fields.AllFields, FieldIndexing.No);
OutputReduceToCollection = "SSBResource";
AdditionalAssemblies = new HashSet <AdditionalAssembly> {
AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> {
"Digitalisert.Dataplattform"
})
};
}
public DataplattformResourceIndex()
{
AddMap <Drupal>(noder =>
from node in noder.WhereEntityIs <Drupal>("Dataplattform")
let metadata = MetadataFor(node)
where metadata.Value <string>("@id").StartsWith("Dataplattform/Drupal/Node")
from resource in (
from resourceid in node["resourceid"]
select new Resource {
Context = (node["context"].Length > 0) ? node["context"][0]["value"].ToString() : "Dataplattform",
ResourceId = resourceid["value"].ToString()
}
).Union(
from resource in node["resource"]
let uuid = resource["target_uuid"].ToString()
select new Resource {
Context = uuid.Substring(0, uuid.IndexOf('/')),
ResourceId = uuid.Substring(uuid.IndexOf('/') + 1)
}
)
select new Resource
{
Context = resource.Context,
ResourceId = resource.ResourceId,
Type = node["resourcetype"].Select(t => t["value"].ToString()),
Title = node["title"].Select(t => t["value"].ToString()),
Body = node["body"].Select(b => b["value"].ToString()),
Properties =
from property in node["properties"]
let paragraph = LoadDocument <Drupal>("Dataplattform/Drupal/Paragraph/" + property["target_id"], "Dataplattform")
let paragraphmetadata = MetadataFor(paragraph)
from name in paragraph["name"]
select new Property {
Name = name["value"].ToString(),
Value = (
paragraph["value"].Select(v => v["value"].ToString())
).Union(
paragraph["value_geofield"].Select(v => v["value"].ToString())
),
Tags = (
paragraph["tags"].Select(v => v["value"].ToString())
).Union(
paragraph["value_geofield"].Take(1).Select(t => "@wkt")
).Union(
paragraph["type"].Where(t => t["target_id"].ToString() == "property_query").Take(1).Select(t => "@query")
),
Resources =
from propertyresource in paragraph["resources"]
let resourceparagraph = LoadDocument <Drupal>("Dataplattform/Drupal/Paragraph/" + propertyresource["target_id"], "Dataplattform")
let resourceparagraphmetadata = MetadataFor(resourceparagraph)
from context in resourceparagraph["context"]
select new Resource
{
Context = context["value"].ToString(),
Type = resourceparagraph["resourcetype"].Select(t => t["value"].ToString()),
Properties =
from resourceproperty in resourceparagraph["properties"]
let resourcepropertyparagraph = LoadDocument <Drupal>("Dataplattform/Drupal/Paragraph/" + resourceproperty["target_id"], "Dataplattform")
let resourcepropertyparagraphmetadata = MetadataFor(resourcepropertyparagraph)
from resourcepropertyname in resourcepropertyparagraph["name"]
select new Property {
Name = resourcepropertyname["value"].ToString(),
Value = resourcepropertyparagraph["value"].Select(v => v["value"].ToString()),
Source = new[] { resourcepropertyparagraphmetadata.Value <string>("@id") }
},
Source = new[] { resourceparagraphmetadata.Value <string>("@id") }
},
Properties =
from propertyproperty in paragraph["properties"]
let propertypropertyparagraph = LoadDocument <Drupal>("Dataplattform/Drupal/Paragraph/" + propertyproperty["target_id"], "Dataplattform")
let propertypropertyparagraphmetadata = MetadataFor(propertypropertyparagraph)
from propertypropertyname in propertypropertyparagraph["name"]
select new Property {
Name = propertypropertyname["value"].ToString(),
Tags = propertypropertyparagraph["tags"].Select(v => v["value"].ToString()),
Source = new[] { propertypropertyparagraphmetadata.Value <string>("@id") }
},
From = (paragraph["from"] != null && paragraph["from"].Any()) ? paragraph["from"].Select(v => DateTime.Parse(v["value"].ToString())).First() : null,
Thru = (paragraph["thru"] != null && paragraph["thru"].Any()) ? paragraph["thru"].Select(v => DateTime.Parse(v["value"].ToString())).First() : null,
Source = new[] { paragraphmetadata.Value <string>("@id") }
},
Source = new[] { metadata.Value <string>("@id") },
Modified = DateTime.MinValue
}
);
Reduce = results =>
from result in results
group result by new { result.Context, result.ResourceId } into g
select new Resource
{
Context = g.Key.Context,
ResourceId = g.Key.ResourceId,
Type = g.SelectMany(r => r.Type).Distinct(),
Title = g.SelectMany(r => r.Title).Distinct(),
Body = g.SelectMany(r => r.Body).Distinct(),
Properties = (IEnumerable <Property>)Properties(g.SelectMany(r => r.Properties)),
Source = g.SelectMany(resource => resource.Source).Distinct(),
Modified = g.Select(resource => resource.Modified).Max()
};
Index(Raven.Client.Constants.Documents.Indexing.Fields.AllFields, FieldIndexing.No);
OutputReduceToCollection = "DataplattformResource";
AdditionalAssemblies = new HashSet <AdditionalAssembly> {
AdditionalAssembly.FromPath("Digitalisert.Dataplattform.ResourceModel.dll", new HashSet <string> {
"Digitalisert.Dataplattform"
})
};
}
