public void AdditionalAssemblies_NuGet() { using (var store = GetDocumentStore()) { store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition { Name = "XmlIndex", Maps = { "from c in docs.Companies select new { Name = typeof(System.Xml.XmlNode).Name }" }, AdditionalAssemblies = { AdditionalAssembly.FromRuntime("System.Private.Xml"), AdditionalAssembly.FromNuGet("System.Xml.ReaderWriter", "4.3.1") } })); } }
public void AdditionalAssemblies_Runtime_InvalidName() { using (var store = GetDocumentStore()) { var e = Assert.Throws <IndexCompilationException>(() => store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition { Name = "XmlIndex", Maps = { "from c in docs.Companies select new { Name = typeof(System.Xml.XmlNode).Name }" }, AdditionalAssemblies = { AdditionalAssembly.FromRuntime("Some.Assembly.That.Does.Not.Exist") } }))); Assert.Contains("Cannot load assembly 'Some.Assembly.That.Does.Not.Exist'", e.Message); Assert.Contains("Could not load file or assembly 'Some.Assembly.That.Does.Not.Exist", e.Message); } }
private static AdditionalAssembly GetAssembly(BlittableJsonReaderObject json) { json.TryGet(nameof(AdditionalAssembly.AssemblyName), out string assemblyName); json.TryGet(nameof(AdditionalAssembly.AssemblyPath), out string assemblyPath); json.TryGet(nameof(AdditionalAssembly.PackageName), out string packageName); json.TryGet(nameof(AdditionalAssembly.PackageVersion), out string packageVersion); json.TryGet(nameof(AdditionalAssembly.PackageSourceUrl), out string packageSourceUrl); var usings = new HashSet <string>(); json.TryGet(nameof(AdditionalAssembly.Usings), out BlittableJsonReaderArray usingsArray); if (usingsArray != null) { foreach (var item in usingsArray) { usings.Add(item.ToString()); } } if (string.IsNullOrWhiteSpace(assemblyName) == false) { return(AdditionalAssembly.FromRuntime(assemblyName, usings)); } if (string.IsNullOrWhiteSpace(assemblyPath) == false) { return(AdditionalAssembly.FromPath(assemblyPath, usings)); } if (string.IsNullOrWhiteSpace(packageName) == false && string.IsNullOrWhiteSpace(packageVersion) == false) { return(AdditionalAssembly.FromNuGet(packageName, packageVersion, packageSourceUrl, usings)); } return(null); }
public void CanUseMLNET_Omnx() { using (var store = GetDocumentStore()) { store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition { Name = "Employees/Tags", Maps = { @" from i in docs.Employees let pic = LoadAttachment(i, ""photo.png"") let classified = ObjectClassification.GetObjects(pic.GetContentAsStream()) select new { Tags = classified.Keys, Count = classified.Count } " }, AdditionalSources = new System.Collections.Generic.Dictionary <string, string> { { "ImageClassifier", @" using System; using System.IO; using System.Collections.Generic; using System.Drawing; using System.Linq; using Microsoft.ML; using Microsoft.ML.Data; using Microsoft.ML.Transforms.Image; using Microsoft.ML.Transforms.Onnx; using System.Net.Http; namespace ObjectDetection { public class ObjectClassification { public const int ROW_COUNT = 13; public const int COL_COUNT = 13; public const int BOXES_PER_CELL = 5; public const int CLASS_COUNT = 20; public const int BOX_INFO_FEATURE_COUNT = 5; private const int channelStride = ROW_COUNT * COL_COUNT; private static string[] labels = new string[] { ""aeroplane"", ""bicycle"", ""bird"", ""boat"", ""bottle"", ""bus"", ""car"", ""cat"", ""chair"", ""cow"", ""diningtable"", ""dog"", ""horse"", ""motorbike"", ""person"", ""pottedplant"", ""sheep"", ""sofa"", ""train"", ""tvmonitor"" }; [ThreadStatic] private static Context _context; public class Context { public TransformerChain<OnnxTransformer> Model; public MLContext MLContext; public MemoryStream Mem = new MemoryStream(); public InMemoryImageData[] Array = new InMemoryImageData[1] { new InMemoryImageData() }; private struct ImageNetSettings { public const int imageHeight = 416; public const int imageWidth = 416; } public class InMemoryImageData { [ImageType(ImageNetSettings.imageHeight, ImageNetSettings.imageWidth)] [LoadColumn(0)] public Bitmap Image; } private static string DownloadCache(string url) { var file = Path.GetFileName(new Uri(url).AbsolutePath); var cache = Path.Combine(Path.GetTempPath(), file); if (File.Exists(cache)) return cache; try { using var client = new HttpClient(); using var f = File.Create(cache); client.GetAsync(url).Result.Content.ReadAsStreamAsync().Result.CopyTo(f); return cache; } catch (Exception) { File.Delete(cache);//cleanup on failure throw; } } public Context() { var modelFilePath = DownloadCache(""https://media.githubusercontent.com/media/onnx/models/master/vision/object_detection_segmentation/tiny-yolov2/model/tinyyolov2-7.onnx""); MLContext = new MLContext(); var pipeline = MLContext.Transforms.ResizeImages(outputColumnName: ""image"", imageWidth: ImageNetSettings.imageWidth, imageHeight: ImageNetSettings.imageHeight, inputColumnName: ""Image"") .Append(MLContext.Transforms.ExtractPixels(outputColumnName: ""image"")) .Append(MLContext.Transforms.ApplyOnnxModel(""grid"", ""image"", modelFilePath)); var data = MLContext.Data.LoadFromEnumerable(new List<InMemoryImageData>()); Model = pipeline.Fit(data); } } public static dynamic GetObjects(Stream s) { _context ??= new Context(); _context.Mem.SetLength(0); s.CopyTo(_context.Mem); _context.Mem.Position = 0; using var bitmap = (Bitmap)Bitmap.FromStream(_context.Mem); _context.Array[0].Image = bitmap; var scoredData = _context.Model.Transform(_context.MLContext.Data.LoadFromEnumerable(_context.Array)); var matches = new Dictionary<string, float>(); foreach (var prob in scoredData.GetColumn<float[]>(TinyYoloModelSettings.ModelOutput)) { ParseOutputs(prob, matches); } _context.Array[0].Image = null; return matches; } private static void ParseOutputs(float[] yoloModelOutputs, Dictionary<string, float> matches, float threshold = .3F) { for (int row = 0; row < ROW_COUNT; row++) { for (int column = 0; column < COL_COUNT; column++) { for (int box = 0; box < BOXES_PER_CELL; box++) { var channel = (box * (CLASS_COUNT + BOX_INFO_FEATURE_COUNT)); float confidence = GetConfidence(yoloModelOutputs, row, column, channel); if (confidence < threshold) continue; float[] predictedClasses = ExtractClasses(yoloModelOutputs, row, column, channel); var (topResultIndex, topResultScore) = GetTopResult(predictedClasses); var topScore = topResultScore * confidence; if (topScore < threshold) continue; if(matches.TryGetValue(labels[topResultIndex], out var f) == false || f < topResultScore) { matches[labels[topResultIndex]] = topResultScore; } } } } } private static float GetConfidence(float[] modelOutput, int x, int y, int channel) { return Sigmoid(modelOutput[GetOffset(x, y, channel + 4)]); } private static float Sigmoid(float value) { var k = (float)Math.Exp(value); return k / (1.0f + k); } private static int GetOffset(int x, int y, int channel) { // YOLO outputs a tensor that has a shape of 125x13x13, which // WinML flattens into a 1D array. To access a specific channel // for a given (x,y) cell position, we need to calculate an offset // into the array return (channel * channelStride) + (y * COL_COUNT) + x; } private static float[] Softmax(float[] values) { var maxVal = values.Max(); var exp = values.Select(v => Math.Exp(v - maxVal)); var sumExp = exp.Sum(); return exp.Select(v => (float)(v / sumExp)).ToArray(); } private static float[] ExtractClasses(float[] modelOutput, int x, int y, int channel) { float[] predictedClasses = new float[CLASS_COUNT]; int predictedClassOffset = channel + BOX_INFO_FEATURE_COUNT; for (int predictedClass = 0; predictedClass < CLASS_COUNT; predictedClass++) { predictedClasses[predictedClass] = modelOutput[GetOffset(x, y, predictedClass + predictedClassOffset)]; } return Softmax(predictedClasses); } private static ValueTuple<int, float> GetTopResult(float[] predictedClasses) { return predictedClasses .Select((predictedClass, index) => (Index: index, Value: predictedClass)) .OrderByDescending(result => result.Value) .First(); } public struct TinyYoloModelSettings { // for checking Tiny yolo2 Model input and output parameter names, //you can use tools like Netron, // which is installed by Visual Studio AI Tools // input tensor name public const string ModelInput = ""image""; // output tensor name public const string ModelOutput = ""grid""; } } } " } }, AdditionalAssemblies = { AdditionalAssembly.FromRuntime("System.IO.FileSystem"), AdditionalAssembly.FromRuntime("System.Net.Http"), AdditionalAssembly.FromNuGet("Microsoft.ML", "1.5.5"), AdditionalAssembly.FromNuGet("Microsoft.ML.ImageAnalytics", "1.5.5"), AdditionalAssembly.FromNuGet("Microsoft.ML.OnnxTransformer", "1.5.5"), AdditionalAssembly.FromNuGet("Microsoft.ML.OnnxRuntime.Managed","1.7.1"), AdditionalAssembly.FromNuGet("Microsoft.ML.OnnxRuntime", "1.7.0") } })); using (var session = store.OpenSession()) { var employee = new Employee { FirstName = "John", LastName = "Doe" }; session.Store(employee); var image = GetImage("RavenDB_15753.cat.jpg"); // from: https://unsplash.com/photos/IuJc2qh2TcA session.Advanced.Attachments.Store(employee, "photo.png", image); session.SaveChanges(); } WaitForIndexing(store); RavenTestHelper.AssertNoIndexErrors(store); } }
public void CanUseMLNET() { using (var store = GetDocumentStore()) { store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition { Name = "Employees/Tags", Maps = { @" from e in docs.Employees let pic = LoadAttachment(e, ""photo.png"") where pic != null let classified = ImageClassifier.Classify(pic.GetContentAsStream()) select new { e.Name, Tag = classified.Where(x => x.Value > 0.75f).Select(x => x.Key), _ = classified.Select(x => CreateField(x.Key, x.Value)) } " }, AdditionalSources = new System.Collections.Generic.Dictionary <string, string> { { "ImageClassifier", @" using Microsoft.ML; using Microsoft.ML.Data; using System; using System.Collections.Generic; using System.Drawing; using System.IO; using System.Linq; public static class ImageClassifier { [ThreadStatic] private static PredictionEngine<ImageData, ImagePrediction> _predictor; [ThreadStatic] private static string[] _names; public static IDictionary<string, float> Classify(Stream s) { using var ms = new MemoryStream(); s.CopyTo(ms); ms.Position = 0; Dictionary<string, float> results = new Dictionary<string, float>(); _predictor ??= InitPredictor(); using Bitmap bitmap = (Bitmap)Image.FromStream(ms); ImageData imageData = new ImageData { Image = bitmap }; ImagePrediction prediction = _predictor.Predict(imageData); for (int i = 0; i < prediction.Score.Length; i++) { results[_names[i]] = prediction.Score[i]; } return results; } private static PredictionEngine<ImageData, ImagePrediction> InitPredictor() { MLContext mlContext = new MLContext(); ITransformer model = mlContext.Model.Load(""model.zip"", out _); PredictionEngine<ImageData, ImagePrediction> predictor = mlContext.Model.CreatePredictionEngine<ImageData, ImagePrediction>(model); VBuffer<ReadOnlyMemory<char>> slotNames = default; predictor.OutputSchema[nameof(ImagePrediction.Score)].GetSlotNames(ref slotNames); _names = slotNames.DenseValues().Select(x => x.ToString()).ToArray(); return predictor; } public class ImageData { public Bitmap Image; public string Label; } public class ImagePrediction : ImageData { public string PredictedLabelValue; public float[] Score; } } " } }, AdditionalAssemblies = { AdditionalAssembly.FromRuntime("System.Memory"), AdditionalAssembly.FromNuGet("System.Drawing.Common","4.7.0"), AdditionalAssembly.FromNuGet("Microsoft.ML", "1.5.2") } })); } }
public void Example() { using (var store = new DocumentStore()) { #region complex_index store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition { Name = "Photographs/Tags", Maps = { @" from p in docs.Photographs let photo = LoadAttachment(p, ""photo.png"") where photo != null let classified = ImageClassifier.Classify(photo.GetContentAsStream()) select new { e.Name, Tag = classified.Where(x => x.Value > 0.75f).Select(x => x.Key), _ = classified.Select(x => CreateField(x.Key, x.Value)) }" }, AdditionalSources = new System.Collections.Generic.Dictionary <string, string> { { "ImageClassifier", @" public static class ImageClassifier { public static IDictionary<string, float> Classify(Stream s) { // returns a list of descriptors with a // value between 0 and 1 of how well the // image matches that descriptor. } }" } }, AdditionalAssemblies = { AdditionalAssembly.FromRuntime("System.Memory"), AdditionalAssembly.FromNuGet("System.Drawing.Common","4.7.0"), AdditionalAssembly.FromNuGet("Microsoft.ML", "1.5.2") } })); #endregion } using (var store = new DocumentStore()) { using (var session = store.OpenSession()) { #region simple_index var runtimeindex = new IndexDefinition { Name = "Dog_Pictures", Maps = { @" from user in docs.Users let fileName = Path.GetFileName(user.ImagePath) where fileName = ""My_Dogs.jpeg"" select new { user.Name, fileName }" }, AdditionalAssemblies = { AdditionalAssembly.FromRuntime("System.IO") } }; #endregion } } }