/// <summary>
/// Fields prefixed with "__" will not be written.
/// The "__docid" field, if it exists, will be persisted as "_original".
/// The reason a model may already have a "__docid" field even before it has been persisted is that it originates from another collection.
/// </summary>
/// <returns>Document ID</returns>
public async Task <long> Write(IDictionary model)
{
var timer = new Stopwatch();
timer.Start();
var docId = _docIx.GetNextDocId();
var docMap = new List <(long keyId, long valId)>();
if (model.Contains("__docid") && !model.Contains("_original"))
{
model.Add("_original", model["__docid"]);
}
foreach (var key in model.Keys)
{
var keyStr = key.ToString();
if (keyStr.StartsWith("__"))
{
continue;
}
var keyHash = keyStr.ToHash();
var val = (IComparable)model[key];
var str = val as string;
long keyId, valId;
if (!SessionFactory.TryGetKeyId(CollectionId, keyHash, out keyId))
{
// We have a new key!
// store key
var keyInfo = await _keys.Append(keyStr);
keyId = await _keyIx.Append(keyInfo.offset, keyInfo.len, keyInfo.dataType);
SessionFactory.PersistKeyMapping(CollectionId, keyHash, keyId);
}
// store value
var valInfo = await _vals.Append(val);
valId = await _valIx.Append(valInfo.offset, valInfo.len, valInfo.dataType);
// store refs to keys and values
docMap.Add((keyId, valId));
}
var docMeta = await _docs.Append(docMap);
await _docIx.Append(docMeta.offset, docMeta.length);
model["__docid"] = docId;
this.Log(string.Format("processed document {0} in {1}", docId, timer.Elapsed));
return(docId);
}
public async Task <IList <ulong> > Write(WriteJob job)
{
var docIds = new List <ulong>();
var docCount = 0;
var timer = new Stopwatch();
timer.Start();
foreach (var model in job.Documents)
{
var docId = _docIx.GetNextDocId();
var docMap = new List <(long keyId, long valId)>();
foreach (var key in model.Keys)
{
var keyStr = key.ToString();
var keyHash = keyStr.ToHash();
var val = (IComparable)model[key];
var str = val as string;
long keyId, valId;
if (!SessionFactory.TryGetKeyId(keyHash, out keyId))
{
// We have a new key!
// store key
var keyInfo = await _keys.Append(keyStr);
keyId = await _keyIx.Append(keyInfo.offset, keyInfo.len, keyInfo.dataType);
await SessionFactory.PersistKeyMapping(keyHash, keyId);
}
// store value
var valInfo = await _vals.Append(val);
valId = await _valIx.Append(valInfo.offset, valInfo.len, valInfo.dataType);
// store refs to keys and values
docMap.Add((keyId, valId));
}
var docMeta = await _docs.Append(docMap);
await _docIx.Append(docMeta.offset, docMeta.length);
model.Add("__docid", docId);
docIds.Add(docId);
docCount++;
}
_log.Log(string.Format("processed {0} documents in {1}", docCount, timer.Elapsed));
return(docIds);
}
public bool IndexExists(ulong key)
{
long keyId;
if (!SessionFactory.TryGetKeyId(key, out keyId))
{
return(false);
}
return(true);
}
/// <summary>
/// Fields prefixed with "___" will not be stored.
/// </summary>
/// <returns>Document ID</returns>
public void Write(IDictionary <string, object> document)
{
document["__created"] = DateTime.Now.ToBinary();
var docMap = new List <(long keyId, long valId)>();
var docId = _docIx.GetNextDocId();
foreach (var key in document.Keys)
{
var val = document[key];
if (val == null)
{
continue;
}
var keyStr = key.ToString();
if (keyStr.StartsWith("___"))
{
continue;
}
var keyHash = keyStr.ToHash();
long keyId;
if (!SessionFactory.TryGetKeyId(CollectionId, keyHash, out keyId))
{
// We have a new key!
// store key
var keyInfo = _keys.Append(keyStr);
keyId = _keyIx.Append(keyInfo.offset, keyInfo.len, keyInfo.dataType);
SessionFactory.PersistKeyMapping(CollectionId, keyHash, keyId);
}
// store value
var valInfo = _vals.Append(val);
var valId = _valIx.Append(valInfo.offset, valInfo.len, valInfo.dataType);
// store refs to keys and values
docMap.Add((keyId, valId));
// index
if (!keyStr.StartsWith("_") && valInfo.dataType == DataType.STRING)
{
_indexSession.Put(docId, keyId, (string)val);
}
}
var docMeta = _docs.Append(docMap);
_docIx.Append(docMeta.offset, docMeta.length);
}
public VectorNode GetIndex(ulong keyHash)
{
long keyId;
if (!SessionFactory.TryGetKeyId(keyHash, out keyId))
{
return(null);
}
return(GetIndex(keyId));
}
public NodeReader CreateIndexReader(ulong keyHash)
{
long keyId;
if (!SessionFactory.TryGetKeyId(CollectionId, keyHash, out keyId))
{
return(null);
}
return(CreateIndexReader(keyId));
}
public async Task <Result> Read(string collectionId, HttpRequest request)
{
Query query = null;
try
{
query = _httpQueryParser.Parse(collectionId, request, _tokenizer);
ulong keyHash = query.Term.Key.ToString().ToHash();
long keyId;
var timer = new Stopwatch();
timer.Start();
if (_sessionFactory.TryGetKeyId(keyHash, out keyId))
{
using (var session = _sessionFactory.CreateReadSession(collectionId))
{
var result = await session.Read(query, query.Take);
var docs = result.Docs;
_log.Log(string.Format("fetched {0} docs from disk in {1}", docs.Count, timer.Elapsed));
timer.Restart();
var stream = new MemoryStream();
Serialize(docs, stream);
_log.Log(string.Format("serialized {0} docs in {1}", docs.Count, timer.Elapsed));
return(new Result {
MediaType = "application/json", Data = stream, Documents = docs, Total = result.Total
});
}
}
return(new Result {
Total = 0
});
}
catch (Exception ex)
{
_log.Log(string.Format("read failed for query: {0} {1}", query.ToString() ?? "unknown", ex));
throw;
}
}
public IEnumerable <IDictionary> Read(Query query)
{
ulong keyHash = query.Term.Key.ToString().ToHash();
uint keyId;
if (_sessionFactory.TryGetKeyId(keyHash, out keyId))
{
using (var session = _sessionFactory.CreateReadSession(query.CollectionId))
{
foreach (var model in session.Read(query))
{
yield return(model);
}
}
}
}
public VectorNode GetIndex(ulong key)
{
uint keyId;
if (!SessionFactory.TryGetKeyId(key, out keyId))
{
return(null);
}
VectorNode root;
if (!Index.TryGetValue(keyId, out root))
{
return(null);
}
return(root);
}
public void Write(IEnumerable <IDictionary> models, bool writeToIndex = false)
{
foreach (var model in models)
{
var docId = _docIx.GetNextDocId();
var docMap = new List <(long keyId, long valId)>();
foreach (var key in model.Keys)
{
var keyStr = key.ToString();
var keyHash = keyStr.ToHash();
var val = (IComparable)model[key];
var str = val as string;
long keyId, valId;
if (!SessionFactory.TryGetKeyId(keyHash, out keyId))
{
// We have a new key!
// store key
var keyInfo = _keys.Append(keyStr);
keyId = _keyIx.Append(keyInfo.offset, keyInfo.len, keyInfo.dataType);
SessionFactory.PersistKeyMapping(keyHash, keyId);
}
// store value
var valInfo = _vals.Append(val);
valId = _valIx.Append(valInfo.offset, valInfo.len, valInfo.dataType);
// store refs to keys and values
docMap.Add((keyId, valId));
}
var docMeta = _docs.Append(docMap);
_docIx.Append(docMeta.offset, docMeta.length);
model.Add("__docid", docId);
}
if (writeToIndex)
{
WriteToIndex(new IndexJob(CollectionId, models));
}
}
public VectorNode CloneIndex(ulong keyHash)
{
long keyId;
if (!SessionFactory.TryGetKeyId(keyHash, out keyId))
{
return(null);
}
VectorNode dirty;
if (_dirty.TryGetValue(keyId, out dirty))
{
return(dirty);
}
var ixFileName = Path.Combine(SessionFactory.Dir, string.Format("{0}.{1}.ix", CollectionId, keyId));
var vecFileName = Path.Combine(SessionFactory.Dir, string.Format("{0}.vec", CollectionId));
return(SessionFactory.DeserializeIndex(ixFileName, vecFileName));
}