public ColumnSerializer(ulong collectionId, long keyId, SessionFactory sessionFactory, RemotePostingsWriter postingsWriter = null, string ixFileExtension = "ix", string pageFileExtension = "ixp") { _keyId = keyId; _collectionId = collectionId; _postingsWriter = postingsWriter; _sessionFactory = sessionFactory; var pixFileName = Path.Combine(_sessionFactory.Dir, string.Format("{0}.{1}.{2}", _collectionId, keyId, pageFileExtension)); var ixFileName = Path.Combine(_sessionFactory.Dir, string.Format("{0}.{1}.{2}", _collectionId, keyId, ixFileExtension)); try { _pageIndexWriter = new PageIndexWriter(_sessionFactory.CreateAppendStream(pixFileName)); _ixStream = _sessionFactory.CreateAppendStream(ixFileName); } catch (IOException) { Thread.Sleep(100); if (_pageIndexWriter != null) { _pageIndexWriter.Dispose(); } if (_ixStream != null) { _ixStream.Dispose(); } _pageIndexWriter = new PageIndexWriter(_sessionFactory.CreateAppendStream(pixFileName)); _ixStream = _sessionFactory.CreateAppendStream(ixFileName); } }
public ColumnSerializer(ulong collectionId, long keyId, SessionFactory sessionFactory, RemotePostingsWriter postingsWriter, string ixFileExtension = "ix", string pageFileExtension = "ixp") { _keyId = keyId; _collectionId = collectionId; _postingsWriter = postingsWriter; _sessionFactory = sessionFactory; var pixFileName = Path.Combine(_sessionFactory.Dir, string.Format("{0}.{1}.{2}", _collectionId, keyId, pageFileExtension)); var ixFileName = Path.Combine(_sessionFactory.Dir, string.Format("{0}.{1}.{2}", _collectionId, keyId, ixFileExtension)); _ixPageIndexWriter = new PageIndexWriter(_sessionFactory.CreateAppendStream(pixFileName)); _ixStream = _sessionFactory.CreateAppendStream(ixFileName); }
public ColumnSerializer( ulong collectionId, long keyId, SessionFactory sessionFactory) { _keyId = keyId; _collectionId = collectionId; _sessionFactory = sessionFactory; var pixFileName = Path.Combine(_sessionFactory.Dir, string.Format("{0}.{1}.ixp", _collectionId, keyId)); var ixFileName = Path.Combine(_sessionFactory.Dir, string.Format("{0}.{1}.ix", _collectionId, keyId)); _ixPageIndexWriter = new PageIndexWriter(_sessionFactory.CreateAppendStream(pixFileName)); _ixStream = _sessionFactory.CreateAppendStream(ixFileName); }
public async Task Commit() { if (_committing || _committed) { return; } _committing = true; this.Log($"merges: {_merges}"); foreach (var column in _dirty) { using (var vectorStream = SessionFactory.CreateAppendStream( Path.Combine(SessionFactory.Dir, $"{CollectionId}.{column.Key}.vec"))) { using (var writer = new ColumnSerializer( CollectionId, column.Key, SessionFactory, new RemotePostingsWriter(_config, CollectionName))) { await writer.CreateColumnSegment(column.Value, vectorStream); } } } _committed = true; _committing = false; this.Log(string.Format("***FLUSHED***")); }
private void BuildInMemoryIndex(ulong docId, long keyId, VectorNode index, IEnumerable <string> tokens) { var count = 0; using (var vectorStream = SessionFactory.CreateAppendStream( Path.Combine(SessionFactory.Dir, string.Format("{0}.{1}.vec", CollectionId.ToHash(), keyId)))) { foreach (var token in tokens) { index.Add(new VectorNode(token, docId), vectorStream); count++; } } }
public WriteSession( string collectionName, ulong collectionId, SessionFactory sessionFactory) : base(collectionName, collectionId, sessionFactory) { ValueStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.val", CollectionId))); KeyStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.key", CollectionId))); DocStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.docs", CollectionId))); ValueIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.vix", CollectionId))); KeyIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.kix", CollectionId))); DocIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.dix", CollectionId))); _vals = new ValueWriter(ValueStream); _keys = new ValueWriter(KeyStream); _docs = new DocMapWriter(DocStream); _valIx = new ValueIndexWriter(ValueIndexStream); _keyIx = new ValueIndexWriter(KeyIndexStream); _docIx = new DocIndexWriter(DocIndexStream); }
public WriteSession( string collectionId, SessionFactory sessionFactory) : base(collectionId, sessionFactory) { _log = Logging.CreateWriter("writesession"); var collection = collectionId.ToHash(); ValueStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.val", collection))); KeyStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.key", collection))); DocStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.docs", collection))); ValueIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.vix", collection))); KeyIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.kix", collection))); DocIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.dix", collection))); _vals = new ValueWriter(ValueStream); _keys = new ValueWriter(KeyStream); _docs = new DocWriter(DocStream); _valIx = new ValueIndexWriter(ValueIndexStream); _keyIx = new ValueIndexWriter(KeyIndexStream); _docIx = new DocIndexWriter(DocIndexStream); }
public WriteSession( string collectionName, ulong collectionId, SessionFactory sessionFactory, TermIndexSession indexSession, IConfigurationProvider config) : base(collectionName, collectionId, sessionFactory) { ValueStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.val", CollectionId)), int.Parse(config.Get("value_stream_buffer_size"))); KeyStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.key", CollectionId))); DocStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.docs", CollectionId)), int.Parse(config.Get("doc_map_stream_buffer_size"))); ValueIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.vix", CollectionId))); KeyIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.kix", CollectionId))); DocIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.dix", CollectionId))); _config = config; _vals = new ValueWriter(ValueStream); _keys = new ValueWriter(KeyStream); _docs = new DocMapWriter(DocStream); _valIx = new ValueIndexWriter(ValueIndexStream); _keyIx = new ValueIndexWriter(KeyIndexStream); _docIx = new DocIndexWriter(DocIndexStream); _indexSession = indexSession; }