public void Run(IDictionary <string, string> args, ILogger logger) { var time = Stopwatch.StartNew(); var collectionId = args["collection"].ToHash(); var images = new MnistReader(args["imageFileName"], args["labelFileName"]).Read(); var count = 0; VectorNode tree; using (var sessionFactory = new SessionFactory(new KeyValueConfiguration("sir.ini"), logger)) using (var writeSession = sessionFactory.CreateWriteSession(collectionId)) { var debugger = new IndexDebugger(); var keyId = writeSession.EnsureKeyExists("image"); using (var indexSession = sessionFactory.CreateIndexSession(collectionId, new ImageModel())) { foreach (var image in images) { var document = new Dictionary <string, object>() { { "label", image.Label } }; var storeFields = new HashSet <string> { "label" }; var documentId = writeSession.Put(document, storeFields); indexSession.Put(documentId, keyId, image); count++; var debugInfo = debugger.GetDebugInfo(indexSession); if (debugInfo != null) { logger.LogInformation(debugInfo); } } tree = indexSession.GetInMemoryIndex(keyId); } } Print(tree); logger.LogInformation($"indexed {count} mnist images in {time.Elapsed}"); }
public void Run(IDictionary <string, string> args, ILogger logger) { var time = Stopwatch.StartNew(); var dataDirectory = args["dataDirectory"]; var collectionId = args["collection"].ToHash(); var images = new MnistReader(args["imageFileName"], args["labelFileName"]).Read(); VectorNode tree; var debugger = new IndexDebugger(logger); var model = new LinearClassifierImageModel(); using (var sessionFactory = new SessionFactory(dataDirectory, logger)) { sessionFactory.Truncate(collectionId); using (var writeSession = new WriteSession(new DocumentWriter(collectionId, sessionFactory))) using (var indexSession = new IndexSession <IImage>(model, model)) { var imageIndexId = writeSession.EnsureKeyExists("image"); foreach (var image in images) { var imageField = new Field("image", image.Pixels, index: true, store: true); var labelField = new Field("label", image.Label, index: false, store: true); var document = new Document(new Field[] { imageField, labelField }); writeSession.Put(document); indexSession.Put(document.Id, imageField.KeyId, image); debugger.Step(indexSession); } var indices = indexSession.GetInMemoryIndex(); tree = indices[imageIndexId]; using (var stream = new WritableIndexStream(collectionId, sessionFactory, logger: logger)) { stream.Write(indices); } } } logger.LogInformation($"indexed {debugger.Steps} mnist images in {time.Elapsed}"); Print(tree); }
public void Run(IDictionary <string, string> args, ILogger logger) { var time = Stopwatch.StartNew(); var dataDirectory = args["dataDirectory"]; var images = new MnistReader(args["imageFileName"], args["labelFileName"]).Read(); var collection = args["collection"]; var count = 0; var errors = 0; var model = new LinearClassifierImageModel(); using (var sessionFactory = new SessionFactory(directory: dataDirectory, logger: logger)) using (var querySession = sessionFactory.CreateSearchSession(model)) { var queryParser = new QueryParser <IImage>(sessionFactory, model, logger); foreach (var image in images) { var query = queryParser.Parse(collection, image, field: "image", select: "label", and: true, or: false); var result = querySession.Search(query, 0, 1); count++; if (result.Total == 0) { errors++; } else { var documentLabel = (string)result.Documents.First().Get("label").Value; if (!documentLabel.Equals(image.Label)) { errors++; logger.LogDebug($"error. label: {image.Label} document label: {documentLabel}\n{((MnistImage)image).Print()}\n{((MnistImage)image).Print()}"); } } logger.LogInformation($"errors: {errors}. total tests {count}. error rate: {(float)errors / count * 100}%"); } } logger.LogInformation($"tested {count} mnist images in {time.Elapsed}"); }