public void Run(IDictionary <string, string> args, ILogger logger) { var dataDirectory = args["dataDirectory"]; var fileName = args["fileName"]; var collection = args["collection"]; var skip = args.ContainsKey("skip") ? int.Parse(args["skip"]) : 0; var take = args.ContainsKey("take") ? int.Parse(args["take"]) : int.MaxValue; var sampleSize = args.ContainsKey("sampleSize") ? int.Parse(args["sampleSize"]) : 1000; var pageSize = args.ContainsKey("pageSize") ? int.Parse(args["pageSize"]) : 100000; var collectionId = collection.ToHash(); var fieldsToStore = new HashSet <string> { "language", "wikibase_item", "title", "text", "url" }; var fieldsToIndex = new HashSet <string> { "title", "text" }; if (take == 0) { take = int.MaxValue; } var model = new BagOfCharsModel(); var payload = WikipediaHelper.ReadWP(fileName, skip, take, fieldsToStore, fieldsToIndex); using (var sessionFactory = new SessionFactory(dataDirectory, logger)) { var debugger = new IndexDebugger(logger, sampleSize); using (var writeSession = new WriteSession(new DocumentWriter(collectionId, sessionFactory))) { foreach (var page in payload.Batch(pageSize)) { using (var indexStream = new WritableIndexStream(collectionId, sessionFactory, logger: logger)) using (var indexSession = new IndexSession <string>(model, model)) { foreach (var document in page) { writeSession.Put(document); foreach (var field in document.IndexableFields) { indexSession.Put(document.Id, field.KeyId, (string)field.Value); } debugger.Step(indexSession); } indexStream.Write(indexSession.GetInMemoryIndex()); //foreach (var column in indexSession.InMemoryIndex) //{ // Print($"wikipedia.{column.Key}", column.Value); //} } } } } }
private static void WriteEmbeddings( SessionFactory sessionFactory, WriteSession writeSession, WordEmbeddingsSession indexSession, string fileName, string collection, IStringModel model, ILogger logger, string refFileName) { var documents = ReadWatFile(fileName, refFileName); var collectionId = collection.ToHash(); var time = Stopwatch.StartNew(); var storedFieldNames = new HashSet <string>(); var indexedFieldNames = new HashSet <string> { "title", "description", "url" }; sessionFactory.CreateWordEmbeddings( new WriteJob( collectionId, documents, model, storedFieldNames, indexedFieldNames), writeSession, indexSession, reportSize: 1000); logger.LogInformation($"indexed {fileName} in {time.Elapsed}"); }
public void FinalizeWriteSession() { if (m_WriteSession != null) { m_WriteSession.Dispose(); m_WriteSession = null; } }
private async void GetLoginData(LoginRequest _credentialValue, object sender) { SetDialogHostStatus("True"); SetStatusUpdate("Logging in..."); for (int i = 0; i < 50; i++) { SetProgressBarValue(i); } //BeginLoading("Making Dummy Data", "Checking Credentials"); SetStatusUpdate("Checking Permisson..."); readLoginResponse = new ReadLoginResponse(); LoginResponse loginResponse = await readLoginResponse.GetLoginAsync(_credentialValue); if (loginResponse.error == null) { for (int i = 50; i < 100; i++) { SetProgressBarValue(i); } writeSession = new WriteSession(); Session sessionData = new Session { user_id = loginResponse.result.res_user.id.ToString(), user_name = loginResponse.result.res_user.name, user_login = _credentialValue.username, token = loginResponse.result.token, refresh_token = loginResponse.result.refresh_token, token_live = loginResponse.result.token_live.ToString() }; writeSession.UpdateSession(sessionData); SetStatusUpdate("Logged in."); ConfigList = readConfig.GetAllConfigs(); IpAddressValue = ConfigList[0].current_ip; SetDialogHostStatus("False"); SetProgressBarValue(0); UsernameValue = ""; if (await readPosSession.GetPosSessionAsync(IpAddressValue) == null) { ShowDepositWindow(sender); } else { ShowMainWindow(sender); } } else { for (int i = 50; i < 100; i++) { SetProgressBarValue(i); } MaterialMessageBox.ShowDialog(loginResponse.error.message, loginResponse.error.code.ToString(), MessageBoxButton.OK, PackIconKind.Error, PrimaryColor.LightBlue); UsernameValue = ""; SetDialogHostStatus("False"); SetProgressBarValue(0); } //EndLoading(); }
public void InitializeWriteSession() { if (m_WriteSession != null) { this.FinalizeWriteSession(); } this.OpenOrCreateShapeFile(); m_WriteSession = new WriteSession(FeatureClass); }
public void InitializeWriteSession(bool a_Overwrite) { if (m_WriteSession != null) { this.FinalizeWriteSession(); } this.OpenOrCreateResultsShapeFile(); m_WriteSession = new WriteSession(ResultsFeatureClass, a_Overwrite); }
public void Can_search_filestreamed_with_multiple_pages() { var model = new BagOfCharsModel(); const string collection = "Can_search_streamed_with_one_page_per_document"; var collectionId = collection.ToHash(); const string fieldName = "description"; _sessionFactory.Truncate(collectionId); using (var stream = new WritableIndexStream(collectionId, _sessionFactory)) using (var writeSession = new WriteSession(new DocumentWriter(collectionId, _sessionFactory))) { var keyId = writeSession.EnsureKeyExists(fieldName); for (long i = 0; i < _data.Length; i++) { var data = _data[i]; using (var indexSession = new IndexSession <string>(model, model)) { var doc = new Document(new Field[] { new Field(fieldName, data, index: true, store: true) }); writeSession.Put(doc); indexSession.Put(doc.Id, keyId, data); stream.Write(indexSession.GetInMemoryIndex()); } } } var queryParser = new QueryParser <string>(_sessionFactory, model); using (var searchSession = new SearchSession(_sessionFactory, model, new PostingsReader(_sessionFactory))) { Assert.DoesNotThrow(() => { foreach (var word in _data) { var query = queryParser.Parse(collection, word, fieldName, fieldName, and: true, or: false); var result = searchSession.Search(query, 0, 1); var document = result.Documents.FirstOrDefault(); if (document == null) { throw new Exception($"unable to find {word}."); } if (document.Score < model.IdenticalAngle) { throw new Exception($"unable to score {word}."); } Debug.WriteLine($"{word} matched with {document.Score * 100}% certainty."); } }); } }
private static void RunMonoSyncTest(int entityCount) { Console.WriteLine(); Console.WriteLine($"Benchmark with {entityCount} entities"); // Initialization Console.Write("Initializing: "); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); var world = new MonoSyncWorld(); var syncSourceRoot = new SourceSynchronizerRoot(world); using (world.Entities.BeginMassUpdate()) { for (int i = 0; i < entityCount; i++) { world.Entities.Add(i, new Entity()); } } stopwatch.Stop(); Console.WriteLine(stopwatch.ElapsedMilliseconds + "MS"); stopwatch.Reset(); // Full write Console.Write("Full write: "); stopwatch.Start(); WriteSession fullWriteSession = syncSourceRoot.BeginWrite(); var syncTargetRoot = new TargetSynchronizerRoot <MonoSyncWorld>(fullWriteSession.WriteFull()); fullWriteSession.Dispose(); stopwatch.Stop(); Console.WriteLine(stopwatch.ElapsedMilliseconds + "MS"); stopwatch.Reset(); int changes = entityCount / 10; Console.Write($"{changes} changes write: "); stopwatch.Start(); for (int i = 0; i < changes; i++) { world.Entities[i].XPos = 2; } using (WriteSession writeSession = syncSourceRoot.BeginWrite()) { var data = writeSession.WriteChanges().SetTick(TimeSpan.Zero); syncTargetRoot.Read(data); } stopwatch.Stop(); Console.WriteLine(stopwatch.ElapsedMilliseconds + "MS"); }
public void Run(IDictionary <string, string> args, ILogger logger) { var dataDirectory = args["dataDirectory"]; var fileName = args["fileName"]; var collection = args["collection"]; var skip = args.ContainsKey("skip") ? int.Parse(args["skip"]) : 0; var take = args.ContainsKey("take") ? int.Parse(args["take"]) : int.MaxValue; var sampleSize = args.ContainsKey("sampleSize") ? int.Parse(args["sampleSize"]) : 1000; var pageSize = args.ContainsKey("pageSize") ? int.Parse(args["pageSize"]) : 100000; var collectionId = collection.ToHash(); var fieldsToStore = new HashSet <string> { "language", "wikibase_item", "title", "text" }; var fieldsToIndex = new HashSet <string> { "language", "title", "text" }; if (take == 0) { take = int.MaxValue; } var model = new BagOfCharsModel(); var payload = WikipediaHelper.ReadWP(fileName, skip, take, fieldsToStore, fieldsToIndex); var debugger = new BatchDebugger(logger, sampleSize); using (var sessionFactory = new SessionFactory(dataDirectory, logger)) { using (var writeSession = new WriteSession(new DocumentWriter(collectionId, sessionFactory))) { foreach (var page in payload.Batch(pageSize)) { using (var indexSession = new IndexSession <string>(model, model)) { foreach (var document in page) { writeSession.Put(document); foreach (var field in document.IndexableFields) { foreach (var token in model.Tokenize((string)field.Value)) { debugger.Step(); } } } } } logger.LogInformation($"tokenized {debugger.StepCount} in {debugger.Time}."); } } }
public static void WriteWatSegment( string dataDirectory, string fileName, string collection, IModel <string> model, ILogger logger, string refFileName) { var time = Stopwatch.StartNew(); var collectionId = collection.ToHash(); var storeFields = new HashSet <string> { "title", "description", "url", "filename" }; var indexFields = new HashSet <string> { "title", "description", "url" }; using (var sessionFactory = new SessionFactory(dataDirectory, logger)) using (var writeSession = new WriteSession(new DocumentWriter(collectionId, sessionFactory))) using (var indexSession = new IndexSession <string>(model, model)) { using (var queue = new ProducerConsumerQueue <Document>(document => { sessionFactory.Write(document, writeSession, indexSession); })) { foreach (var document in ReadWatFile(fileName, refFileName).Select(dic => new Document( dic.Select(kvp => new Field( kvp.Key, kvp.Value, index: indexFields.Contains(kvp.Key), store: storeFields.Contains(kvp.Key))).ToList()))) { queue.Enqueue(document); } } using (var stream = new WritableIndexStream(collectionId, sessionFactory, logger: logger)) { stream.Write(indexSession.GetInMemoryIndex()); } } logger.LogInformation($"indexed {fileName} in {time.Elapsed}"); }
public void Run(IDictionary <string, string> args, ILogger logger) { var time = Stopwatch.StartNew(); var dataDirectory = args["dataDirectory"]; var collectionId = args["collection"].ToHash(); var images = new MnistReader(args["imageFileName"], args["labelFileName"]).Read(); VectorNode tree; var debugger = new IndexDebugger(logger); var model = new LinearClassifierImageModel(); using (var sessionFactory = new SessionFactory(dataDirectory, logger)) { sessionFactory.Truncate(collectionId); using (var writeSession = new WriteSession(new DocumentWriter(collectionId, sessionFactory))) using (var indexSession = new IndexSession <IImage>(model, model)) { var imageIndexId = writeSession.EnsureKeyExists("image"); foreach (var image in images) { var imageField = new Field("image", image.Pixels, index: true, store: true); var labelField = new Field("label", image.Label, index: false, store: true); var document = new Document(new Field[] { imageField, labelField }); writeSession.Put(document); indexSession.Put(document.Id, imageField.KeyId, image); debugger.Step(indexSession); } var indices = indexSession.GetInMemoryIndex(); tree = indices[imageIndexId]; using (var stream = new WritableIndexStream(collectionId, sessionFactory, logger: logger)) { stream.Write(indices); } } } logger.LogInformation($"indexed {debugger.Steps} mnist images in {time.Elapsed}"); Print(tree); }
public void Run(IDictionary <string, string> args, ILogger logger) { var dataDirectory = args["dataDirectory"]; var fileName = args["fileName"]; var collection = args["collection"]; var skip = args.ContainsKey("skip") ? int.Parse(args["skip"]) : 0; var take = args.ContainsKey("take") ? int.Parse(args["take"]) : int.MaxValue; var sampleSize = args.ContainsKey("sampleSize") ? int.Parse(args["sampleSize"]) : 1000; var collectionId = collection.ToHash(); var fieldsToStore = new HashSet <string> { "language", "wikibase_item", "title", "text", "url" }; var fieldsToIndex = new HashSet <string>(); if (take == 0) { take = int.MaxValue; } var payload = WikipediaHelper.ReadWP(fileName, skip, take, fieldsToStore, fieldsToIndex); using (var sessionFactory = new SessionFactory(dataDirectory, logger)) { sessionFactory.Truncate(collectionId); var debugger = new BatchDebugger(logger, sampleSize); using (var writeSession = new WriteSession(new DocumentWriter(collectionId, sessionFactory))) { foreach (var document in payload) { writeSession.Put(document); debugger.Step(); } } } }
public static byte[] WriteFullAndDispose(this SourceSynchronizerRoot SourceSynchronizerRoot) { using WriteSession session = SourceSynchronizerRoot.BeginWrite(); return(session.WriteFull()); }
public static SynchronizationPacket WriteChangesAndDispose(this SourceSynchronizerRoot SourceSynchronizerRoot) { using WriteSession session = SourceSynchronizerRoot.BeginWrite(); return(session.WriteChanges()); }
public static byte[] WriteFullAndDispose(this SyncSourceRoot syncSourceRoot) { using WriteSession session = syncSourceRoot.BeginWrite(); return(session.WriteFull()); }