/// <summary> /// Fields prefixed with "__" will not be written. /// The "__docid" field, if it exists, will be persisted as "_original". /// The reason a model may already have a "__docid" field even before it has been persisted is that it originates from another collection. /// </summary> /// <returns>Document ID</returns> public async Task <long> Write(IDictionary model) { var timer = new Stopwatch(); timer.Start(); var docId = _docIx.GetNextDocId(); var docMap = new List <(long keyId, long valId)>(); if (model.Contains("__docid") && !model.Contains("_original")) { model.Add("_original", model["__docid"]); } foreach (var key in model.Keys) { var keyStr = key.ToString(); if (keyStr.StartsWith("__")) { continue; } var keyHash = keyStr.ToHash(); var val = (IComparable)model[key]; var str = val as string; long keyId, valId; if (!SessionFactory.TryGetKeyId(CollectionId, keyHash, out keyId)) { // We have a new key! // store key var keyInfo = await _keys.Append(keyStr); keyId = await _keyIx.Append(keyInfo.offset, keyInfo.len, keyInfo.dataType); SessionFactory.PersistKeyMapping(CollectionId, keyHash, keyId); } // store value var valInfo = await _vals.Append(val); valId = await _valIx.Append(valInfo.offset, valInfo.len, valInfo.dataType); // store refs to keys and values docMap.Add((keyId, valId)); } var docMeta = await _docs.Append(docMap); await _docIx.Append(docMeta.offset, docMeta.length); model["__docid"] = docId; this.Log(string.Format("processed document {0} in {1}", docId, timer.Elapsed)); return(docId); }
public async Task <IList <ulong> > Write(WriteJob job) { var docIds = new List <ulong>(); var docCount = 0; var timer = new Stopwatch(); timer.Start(); foreach (var model in job.Documents) { var docId = _docIx.GetNextDocId(); var docMap = new List <(long keyId, long valId)>(); foreach (var key in model.Keys) { var keyStr = key.ToString(); var keyHash = keyStr.ToHash(); var val = (IComparable)model[key]; var str = val as string; long keyId, valId; if (!SessionFactory.TryGetKeyId(keyHash, out keyId)) { // We have a new key! // store key var keyInfo = await _keys.Append(keyStr); keyId = await _keyIx.Append(keyInfo.offset, keyInfo.len, keyInfo.dataType); await SessionFactory.PersistKeyMapping(keyHash, keyId); } // store value var valInfo = await _vals.Append(val); valId = await _valIx.Append(valInfo.offset, valInfo.len, valInfo.dataType); // store refs to keys and values docMap.Add((keyId, valId)); } var docMeta = await _docs.Append(docMap); await _docIx.Append(docMeta.offset, docMeta.length); model.Add("__docid", docId); docIds.Add(docId); docCount++; } _log.Log(string.Format("processed {0} documents in {1}", docCount, timer.Elapsed)); return(docIds); }
public bool IndexExists(ulong key) { long keyId; if (!SessionFactory.TryGetKeyId(key, out keyId)) { return(false); } return(true); }
/// <summary> /// Fields prefixed with "___" will not be stored. /// </summary> /// <returns>Document ID</returns> public void Write(IDictionary <string, object> document) { document["__created"] = DateTime.Now.ToBinary(); var docMap = new List <(long keyId, long valId)>(); var docId = _docIx.GetNextDocId(); foreach (var key in document.Keys) { var val = document[key]; if (val == null) { continue; } var keyStr = key.ToString(); if (keyStr.StartsWith("___")) { continue; } var keyHash = keyStr.ToHash(); long keyId; if (!SessionFactory.TryGetKeyId(CollectionId, keyHash, out keyId)) { // We have a new key! // store key var keyInfo = _keys.Append(keyStr); keyId = _keyIx.Append(keyInfo.offset, keyInfo.len, keyInfo.dataType); SessionFactory.PersistKeyMapping(CollectionId, keyHash, keyId); } // store value var valInfo = _vals.Append(val); var valId = _valIx.Append(valInfo.offset, valInfo.len, valInfo.dataType); // store refs to keys and values docMap.Add((keyId, valId)); // index if (!keyStr.StartsWith("_") && valInfo.dataType == DataType.STRING) { _indexSession.Put(docId, keyId, (string)val); } } var docMeta = _docs.Append(docMap); _docIx.Append(docMeta.offset, docMeta.length); }
public VectorNode GetIndex(ulong keyHash) { long keyId; if (!SessionFactory.TryGetKeyId(keyHash, out keyId)) { return(null); } return(GetIndex(keyId)); }
public NodeReader CreateIndexReader(ulong keyHash) { long keyId; if (!SessionFactory.TryGetKeyId(CollectionId, keyHash, out keyId)) { return(null); } return(CreateIndexReader(keyId)); }
public async Task <Result> Read(string collectionId, HttpRequest request) { Query query = null; try { query = _httpQueryParser.Parse(collectionId, request, _tokenizer); ulong keyHash = query.Term.Key.ToString().ToHash(); long keyId; var timer = new Stopwatch(); timer.Start(); if (_sessionFactory.TryGetKeyId(keyHash, out keyId)) { using (var session = _sessionFactory.CreateReadSession(collectionId)) { var result = await session.Read(query, query.Take); var docs = result.Docs; _log.Log(string.Format("fetched {0} docs from disk in {1}", docs.Count, timer.Elapsed)); timer.Restart(); var stream = new MemoryStream(); Serialize(docs, stream); _log.Log(string.Format("serialized {0} docs in {1}", docs.Count, timer.Elapsed)); return(new Result { MediaType = "application/json", Data = stream, Documents = docs, Total = result.Total }); } } return(new Result { Total = 0 }); } catch (Exception ex) { _log.Log(string.Format("read failed for query: {0} {1}", query.ToString() ?? "unknown", ex)); throw; } }
public IEnumerable <IDictionary> Read(Query query) { ulong keyHash = query.Term.Key.ToString().ToHash(); uint keyId; if (_sessionFactory.TryGetKeyId(keyHash, out keyId)) { using (var session = _sessionFactory.CreateReadSession(query.CollectionId)) { foreach (var model in session.Read(query)) { yield return(model); } } } }
public VectorNode GetIndex(ulong key) { uint keyId; if (!SessionFactory.TryGetKeyId(key, out keyId)) { return(null); } VectorNode root; if (!Index.TryGetValue(keyId, out root)) { return(null); } return(root); }
public void Write(IEnumerable <IDictionary> models, bool writeToIndex = false) { foreach (var model in models) { var docId = _docIx.GetNextDocId(); var docMap = new List <(long keyId, long valId)>(); foreach (var key in model.Keys) { var keyStr = key.ToString(); var keyHash = keyStr.ToHash(); var val = (IComparable)model[key]; var str = val as string; long keyId, valId; if (!SessionFactory.TryGetKeyId(keyHash, out keyId)) { // We have a new key! // store key var keyInfo = _keys.Append(keyStr); keyId = _keyIx.Append(keyInfo.offset, keyInfo.len, keyInfo.dataType); SessionFactory.PersistKeyMapping(keyHash, keyId); } // store value var valInfo = _vals.Append(val); valId = _valIx.Append(valInfo.offset, valInfo.len, valInfo.dataType); // store refs to keys and values docMap.Add((keyId, valId)); } var docMeta = _docs.Append(docMap); _docIx.Append(docMeta.offset, docMeta.length); model.Add("__docid", docId); } if (writeToIndex) { WriteToIndex(new IndexJob(CollectionId, models)); } }
public VectorNode CloneIndex(ulong keyHash) { long keyId; if (!SessionFactory.TryGetKeyId(keyHash, out keyId)) { return(null); } VectorNode dirty; if (_dirty.TryGetValue(keyId, out dirty)) { return(dirty); } var ixFileName = Path.Combine(SessionFactory.Dir, string.Format("{0}.{1}.ix", CollectionId, keyId)); var vecFileName = Path.Combine(SessionFactory.Dir, string.Format("{0}.vec", CollectionId)); return(SessionFactory.DeserializeIndex(ixFileName, vecFileName)); }