/// <summary>
/// Fields prefixed with "__" will not be written.
/// The "__docid" field, if it exists, will be persisted as "_original".
/// The reason a model may already have a "__docid" field even before it has been persisted is that it originates from another collection.
/// </summary>
/// <returns>Document ID</returns>
public async Task <long> Write(IDictionary model)
{
var timer = new Stopwatch();
timer.Start();
var docId = _docIx.GetNextDocId();
var docMap = new List <(long keyId, long valId)>();
if (model.Contains("__docid") && !model.Contains("_original"))
{
model.Add("_original", model["__docid"]);
}
foreach (var key in model.Keys)
{
var keyStr = key.ToString();
if (keyStr.StartsWith("__"))
{
continue;
}
var keyHash = keyStr.ToHash();
var val = (IComparable)model[key];
var str = val as string;
long keyId, valId;
if (!SessionFactory.TryGetKeyId(CollectionId, keyHash, out keyId))
{
// We have a new key!
// store key
var keyInfo = await _keys.Append(keyStr);
keyId = await _keyIx.Append(keyInfo.offset, keyInfo.len, keyInfo.dataType);
SessionFactory.PersistKeyMapping(CollectionId, keyHash, keyId);
}
// store value
var valInfo = await _vals.Append(val);
valId = await _valIx.Append(valInfo.offset, valInfo.len, valInfo.dataType);
// store refs to keys and values
docMap.Add((keyId, valId));
}
var docMeta = await _docs.Append(docMap);
await _docIx.Append(docMeta.offset, docMeta.length);
model["__docid"] = docId;
this.Log(string.Format("processed document {0} in {1}", docId, timer.Elapsed));
return(docId);
}
public async Task <IList <ulong> > Write(WriteJob job)
{
var docIds = new List <ulong>();
var docCount = 0;
var timer = new Stopwatch();
timer.Start();
foreach (var model in job.Documents)
{
var docId = _docIx.GetNextDocId();
var docMap = new List <(long keyId, long valId)>();
foreach (var key in model.Keys)
{
var keyStr = key.ToString();
var keyHash = keyStr.ToHash();
var val = (IComparable)model[key];
var str = val as string;
long keyId, valId;
if (!SessionFactory.TryGetKeyId(keyHash, out keyId))
{
// We have a new key!
// store key
var keyInfo = await _keys.Append(keyStr);
keyId = await _keyIx.Append(keyInfo.offset, keyInfo.len, keyInfo.dataType);
await SessionFactory.PersistKeyMapping(keyHash, keyId);
}
// store value
var valInfo = await _vals.Append(val);
valId = await _valIx.Append(valInfo.offset, valInfo.len, valInfo.dataType);
// store refs to keys and values
docMap.Add((keyId, valId));
}
var docMeta = await _docs.Append(docMap);
await _docIx.Append(docMeta.offset, docMeta.length);
model.Add("__docid", docId);
docIds.Add(docId);
docCount++;
}
_log.Log(string.Format("processed {0} documents in {1}", docCount, timer.Elapsed));
return(docIds);
}
/// <summary>
/// Fields prefixed with "___" will not be stored.
/// </summary>
/// <returns>Document ID</returns>
public void Write(IDictionary <string, object> document)
{
document["__created"] = DateTime.Now.ToBinary();
var docMap = new List <(long keyId, long valId)>();
var docId = _docIx.GetNextDocId();
foreach (var key in document.Keys)
{
var val = document[key];
if (val == null)
{
continue;
}
var keyStr = key.ToString();
if (keyStr.StartsWith("___"))
{
continue;
}
var keyHash = keyStr.ToHash();
long keyId;
if (!SessionFactory.TryGetKeyId(CollectionId, keyHash, out keyId))
{
// We have a new key!
// store key
var keyInfo = _keys.Append(keyStr);
keyId = _keyIx.Append(keyInfo.offset, keyInfo.len, keyInfo.dataType);
SessionFactory.PersistKeyMapping(CollectionId, keyHash, keyId);
}
// store value
var valInfo = _vals.Append(val);
var valId = _valIx.Append(valInfo.offset, valInfo.len, valInfo.dataType);
// store refs to keys and values
docMap.Add((keyId, valId));
// index
if (!keyStr.StartsWith("_") && valInfo.dataType == DataType.STRING)
{
_indexSession.Put(docId, keyId, (string)val);
}
}
var docMeta = _docs.Append(docMap);
_docIx.Append(docMeta.offset, docMeta.length);
}
public void Write(IEnumerable <IDictionary> models, bool writeToIndex = false)
{
foreach (var model in models)
{
var docId = _docIx.GetNextDocId();
var docMap = new List <(long keyId, long valId)>();
foreach (var key in model.Keys)
{
var keyStr = key.ToString();
var keyHash = keyStr.ToHash();
var val = (IComparable)model[key];
var str = val as string;
long keyId, valId;
if (!SessionFactory.TryGetKeyId(keyHash, out keyId))
{
// We have a new key!
// store key
var keyInfo = _keys.Append(keyStr);
keyId = _keyIx.Append(keyInfo.offset, keyInfo.len, keyInfo.dataType);
SessionFactory.PersistKeyMapping(keyHash, keyId);
}
// store value
var valInfo = _vals.Append(val);
valId = _valIx.Append(valInfo.offset, valInfo.len, valInfo.dataType);
// store refs to keys and values
docMap.Add((keyId, valId));
}
var docMeta = _docs.Append(docMap);
_docIx.Append(docMeta.offset, docMeta.length);
model.Add("__docid", docId);
}
if (writeToIndex)
{
WriteToIndex(new IndexJob(CollectionId, models));
}
}