public ColumnSerializer(ulong collectionId, long keyId, SessionFactory sessionFactory, RemotePostingsWriter postingsWriter = null, string ixFileExtension = "ix", string pageFileExtension = "ixp")
{
_keyId = keyId;
_collectionId = collectionId;
_postingsWriter = postingsWriter;
_sessionFactory = sessionFactory;
var pixFileName = Path.Combine(_sessionFactory.Dir, string.Format("{0}.{1}.{2}", _collectionId, keyId, pageFileExtension));
var ixFileName = Path.Combine(_sessionFactory.Dir, string.Format("{0}.{1}.{2}", _collectionId, keyId, ixFileExtension));
try
{
_pageIndexWriter = new PageIndexWriter(_sessionFactory.CreateAppendStream(pixFileName));
_ixStream = _sessionFactory.CreateAppendStream(ixFileName);
}
catch (IOException)
{
Thread.Sleep(100);
if (_pageIndexWriter != null)
{
_pageIndexWriter.Dispose();
}
if (_ixStream != null)
{
_ixStream.Dispose();
}
_pageIndexWriter = new PageIndexWriter(_sessionFactory.CreateAppendStream(pixFileName));
_ixStream = _sessionFactory.CreateAppendStream(ixFileName);
}
}
public ColumnSerializer(ulong collectionId, long keyId, SessionFactory sessionFactory, RemotePostingsWriter postingsWriter, string ixFileExtension = "ix", string pageFileExtension = "ixp")
{
_keyId = keyId;
_collectionId = collectionId;
_postingsWriter = postingsWriter;
_sessionFactory = sessionFactory;
var pixFileName = Path.Combine(_sessionFactory.Dir, string.Format("{0}.{1}.{2}", _collectionId, keyId, pageFileExtension));
var ixFileName = Path.Combine(_sessionFactory.Dir, string.Format("{0}.{1}.{2}", _collectionId, keyId, ixFileExtension));
_ixPageIndexWriter = new PageIndexWriter(_sessionFactory.CreateAppendStream(pixFileName));
_ixStream = _sessionFactory.CreateAppendStream(ixFileName);
}
public ColumnSerializer(
ulong collectionId,
long keyId,
SessionFactory sessionFactory)
{
_keyId = keyId;
_collectionId = collectionId;
_sessionFactory = sessionFactory;
var pixFileName = Path.Combine(_sessionFactory.Dir, string.Format("{0}.{1}.ixp", _collectionId, keyId));
var ixFileName = Path.Combine(_sessionFactory.Dir, string.Format("{0}.{1}.ix", _collectionId, keyId));
_ixPageIndexWriter = new PageIndexWriter(_sessionFactory.CreateAppendStream(pixFileName));
_ixStream = _sessionFactory.CreateAppendStream(ixFileName);
}
public async Task Commit()
{
if (_committing || _committed)
{
return;
}
_committing = true;
this.Log($"merges: {_merges}");
foreach (var column in _dirty)
{
using (var vectorStream = SessionFactory.CreateAppendStream(
Path.Combine(SessionFactory.Dir, $"{CollectionId}.{column.Key}.vec")))
{
using (var writer = new ColumnSerializer(
CollectionId, column.Key, SessionFactory, new RemotePostingsWriter(_config, CollectionName)))
{
await writer.CreateColumnSegment(column.Value, vectorStream);
}
}
}
_committed = true;
_committing = false;
this.Log(string.Format("***FLUSHED***"));
}
private void BuildInMemoryIndex(ulong docId, long keyId, VectorNode index, IEnumerable <string> tokens)
{
var count = 0;
using (var vectorStream = SessionFactory.CreateAppendStream(
Path.Combine(SessionFactory.Dir, string.Format("{0}.{1}.vec", CollectionId.ToHash(), keyId))))
{
foreach (var token in tokens)
{
index.Add(new VectorNode(token, docId), vectorStream);
count++;
}
}
}
public WriteSession(
string collectionName,
ulong collectionId,
SessionFactory sessionFactory) : base(collectionName, collectionId, sessionFactory)
{
ValueStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.val", CollectionId)));
KeyStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.key", CollectionId)));
DocStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.docs", CollectionId)));
ValueIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.vix", CollectionId)));
KeyIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.kix", CollectionId)));
DocIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.dix", CollectionId)));
_vals = new ValueWriter(ValueStream);
_keys = new ValueWriter(KeyStream);
_docs = new DocMapWriter(DocStream);
_valIx = new ValueIndexWriter(ValueIndexStream);
_keyIx = new ValueIndexWriter(KeyIndexStream);
_docIx = new DocIndexWriter(DocIndexStream);
}
public WriteSession(
string collectionId,
SessionFactory sessionFactory) : base(collectionId, sessionFactory)
{
_log = Logging.CreateWriter("writesession");
var collection = collectionId.ToHash();
ValueStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.val", collection)));
KeyStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.key", collection)));
DocStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.docs", collection)));
ValueIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.vix", collection)));
KeyIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.kix", collection)));
DocIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.dix", collection)));
_vals = new ValueWriter(ValueStream);
_keys = new ValueWriter(KeyStream);
_docs = new DocWriter(DocStream);
_valIx = new ValueIndexWriter(ValueIndexStream);
_keyIx = new ValueIndexWriter(KeyIndexStream);
_docIx = new DocIndexWriter(DocIndexStream);
}
public WriteSession(
string collectionName,
ulong collectionId,
SessionFactory sessionFactory,
TermIndexSession indexSession,
IConfigurationProvider config) : base(collectionName, collectionId, sessionFactory)
{
ValueStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.val", CollectionId)), int.Parse(config.Get("value_stream_buffer_size")));
KeyStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.key", CollectionId)));
DocStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.docs", CollectionId)), int.Parse(config.Get("doc_map_stream_buffer_size")));
ValueIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.vix", CollectionId)));
KeyIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.kix", CollectionId)));
DocIndexStream = sessionFactory.CreateAppendStream(Path.Combine(sessionFactory.Dir, string.Format("{0}.dix", CollectionId)));
_config = config;
_vals = new ValueWriter(ValueStream);
_keys = new ValueWriter(KeyStream);
_docs = new DocMapWriter(DocStream);
_valIx = new ValueIndexWriter(ValueIndexStream);
_keyIx = new ValueIndexWriter(KeyIndexStream);
_docIx = new DocIndexWriter(DocIndexStream);
_indexSession = indexSession;
}
