public void RemoveMany(IList <PackedObject> items) { Dbg.Trace($"remove many called for {items.Count} items"); foreach (var item in items) { foreach (var metadata in CollectionSchema.ServerSide) { if (metadata.IndexType == IndexType.Unique) { _dataByUniqueKey[metadata.Name].Remove(item[metadata.Order]); } } // if present remove it from the full-text index _fullTextIndex?.DeleteDocument(item.PrimaryKey); EvictionPolicy.TryRemove(item); } foreach (var index in _dataByIndexKey) { index.Value.RemoveMany(items); } foreach (var o in items) { DataByPrimaryKey.Remove(o.PrimaryKey); } }
public void Truncate() { EvictionPolicy.Clear(); DataByPrimaryKey.Clear(); foreach (var indexByKey in _dataByUniqueKey) { indexByKey.Value.Clear(); } foreach (var index in _dataByIndexKey) { index.Value.Clear(); } _fullTextIndex?.Clear(); Interlocked.Exchange(ref _readCount, 0); Interlocked.Exchange(ref _hitCount, 0); // also reset the domain description DomainDescription = null; }
/// <summary> /// Store a new object in all the indexes /// REQUIRE: an object with the same primary key is not present in the datastore /// </summary> /// <param name="packedObject"></param> /// <param name="excludeFromEviction">if true the item will never be evicted</param> internal void InternalAddNew(PackedObject packedObject, bool excludeFromEviction) { InternalAddNew(packedObject); if (!excludeFromEviction) { EvictionPolicy.AddItem(packedObject); } }
/// <summary> /// </summary> /// <param name="items"></param> /// <param name="excludeFromEviction">used only for non persistent case</param> internal void InternalPutMany(IList <PackedObject> items, bool excludeFromEviction) { var isBulkOperation = items.Count > Constants.BulkThreshold; var toUpdate = new List <PackedObject>(); try { Dbg.Trace($"begin InternalPutMany with {items.Count} object"); InternalBeginBulkInsert(isBulkOperation); foreach (var item in items) { if (DataByPrimaryKey.ContainsKey(item.PrimaryKey)) { toUpdate.Add(item); } else { InternalAddNew(item, excludeFromEviction); } } } finally { InternalEndBulkInsert(isBulkOperation); // update items outside the bulk insert if (toUpdate.Count > Constants.BulkThreshold) // bulk optimization for updates { RemoveMany(toUpdate); InternalPutMany(toUpdate, true); } else { foreach (var cachedObject in toUpdate) { InternalUpdate(cachedObject); } } foreach (var cachedObject in toUpdate) { EvictionPolicy.Touch(cachedObject); } Dbg.Trace($"end InternalPutMany with {items.Count} object"); } }
public PackedObject RemoveByPrimaryKey(KeyValue primary) { var removed = InternalRemoveByPrimaryKey(primary); if (removed != null) { EvictionPolicy.TryRemove(removed); } return(removed); }
/// <summary> /// Initialize an empty datastore from a type description /// </summary> /// <param name="typeDescription"></param> /// <param name="evictionPolicy"></param> /// <param name="config"></param> public DataStore(TypeDescription typeDescription, EvictionPolicy evictionPolicy, NodeConfig config) { TypeDescription = typeDescription ?? throw new ArgumentNullException(nameof(typeDescription)); EvictionPolicy = evictionPolicy ?? throw new ArgumentNullException(nameof(evictionPolicy)); //initialize the primary key dictionary _dataByPrimaryKey = new Dictionary <KeyValue, CachedObject>(); //initialize the unique keys dictionaries (une by unique key) _dataByUniqueKey = new Dictionary <string, Dictionary <KeyValue, CachedObject> >(); foreach (var keyInfo in typeDescription.UniqueKeyFields) { _dataByUniqueKey.Add(keyInfo.Name, new Dictionary <KeyValue, CachedObject>()); } //initialize the indexes (one by index key) _dataByIndexKey = new Dictionary <string, IndexBase>(); // scalar indexed fields foreach (var indexField in typeDescription.IndexFields) { var index = IndexFactory.CreateIndex(indexField); _dataByIndexKey.Add(indexField.Name, index); } // list indexed fields foreach (var indexField in typeDescription.ListFields) { var index = IndexFactory.CreateIndex(indexField); _dataByIndexKey.Add(indexField.Name, index); } // create the full-text index if required if (typeDescription.FullText.Count > 0) { _fullTextIndex = new FullTextIndex(config.FullTextConfig) { // a function that allows the full text engine to find the original line of text LineProvider = pointer => _dataByPrimaryKey[pointer.PrimaryKey].FullText[pointer.Line] }; } }
/// <summary> /// Update an object previously stored /// The primary key must be the same, all others can change /// </summary> /// <param name="item"></param> public void InternalUpdate(PackedObject item) { if (item == null) { throw new ArgumentNullException(nameof(item)); } if (!DataByPrimaryKey.ContainsKey(item.PrimaryKey)) { var msg = $"Update called for the object {item} which is not stored in the cache"; throw new NotSupportedException(msg); } InternalRemoveByPrimaryKey(item.PrimaryKey); InternalAddNew(item); EvictionPolicy.Touch(item); }
/// <summary> /// Initialize an empty datastore from a type description /// </summary> /// <param name="collectionSchema"></param> /// <param name="evictionPolicy"></param> /// <param name="fullTextConfig"></param> public DataStore(CollectionSchema collectionSchema, EvictionPolicy evictionPolicy, FullTextConfig fullTextConfig) { _fullTextConfig = fullTextConfig; CollectionSchema = collectionSchema ?? throw new ArgumentNullException(nameof(collectionSchema)); EvictionPolicy = evictionPolicy ?? throw new ArgumentNullException(nameof(evictionPolicy)); //initialize the primary key dictionary DataByPrimaryKey = new Dictionary <KeyValue, PackedObject>(); //initialize the unique keys dictionaries (one by unique key) _dataByUniqueKey = new Dictionary <string, Dictionary <KeyValue, PackedObject> >(); foreach (var keyInfo in collectionSchema.UniqueKeyFields) { _dataByUniqueKey.Add(keyInfo.Name, new Dictionary <KeyValue, PackedObject>()); } //initialize the indexes (one by index key) _dataByIndexKey = new Dictionary <string, IndexBase>(); // scalar indexed fields foreach (var indexField in collectionSchema.IndexFields) { var index = IndexFactory.CreateIndex(indexField); _dataByIndexKey.Add(indexField.Name, index); } // create the full-text index if required if (collectionSchema.FullText.Count > 0) { _fullTextIndex = new FullTextIndex(fullTextConfig) { // a function that allows the full text engine to find the original line of text LineProvider = pointer => DataByPrimaryKey[pointer.PrimaryKey].TokenizedFullText[pointer.Line] } } ; }
public void ProcessEviction() { if (EvictionPolicy.IsEvictionRequired) { var itemsToEvict = EvictionPolicy.DoEviction(); foreach (var item in itemsToEvict) { InternalRemoveByPrimaryKey(item.PrimaryKey); } var requestDescription = string.Empty; var processedItems = itemsToEvict.Count; var requestType = "EVICTION"; ServerLog.AddEntry(new ServerLogEntry(0, requestType, requestDescription, processedItems)); } }
/// <summary> /// Like a bulk insert on an empty datastore. Used internally to reindex data when type description changed /// </summary> /// <param name="items"></param> private void InternalReindex(IEnumerable <PackedObject> items) { var toUpdate = new List <PackedObject>(); try { InternalBeginBulkInsert(true); foreach (var item in items) { InternalAddNew(item, true); } } finally { InternalEndBulkInsert(true); // update items outside the bulk insert if (toUpdate.Count > Constants.BulkThreshold) // bulk optimization for updates { RemoveMany(toUpdate); InternalPutMany(toUpdate, true); } else { foreach (var cachedObject in toUpdate) { InternalUpdate(cachedObject); } } //TODO check if necessary foreach (var cachedObject in toUpdate) { EvictionPolicy.Touch(cachedObject); } } }
/// <summary> /// Initialize an empty datastore from a type description /// </summary> /// <param name="typeDescription"></param> /// <param name="evictionPolicy"></param> public DataStore(TypeDescription typeDescription, EvictionPolicy evictionPolicy) { TypeDescription = typeDescription ?? throw new ArgumentNullException(nameof(typeDescription)); EvictionPolicy = evictionPolicy ?? throw new ArgumentNullException(nameof(evictionPolicy)); //initialize the primary key dictionary _dataByPrimaryKey = new Dictionary <KeyValue, CachedObject>(); //initialize the unique keys dictionaries (une by unique key) _dataByUniqueKey = new Dictionary <string, Dictionary <KeyValue, CachedObject> >(); foreach (var keyInfo in typeDescription.UniqueKeyFields) { _dataByUniqueKey.Add(keyInfo.Name, new Dictionary <KeyValue, CachedObject>()); } //initialize the indexes (one by index key) _dataByIndexKey = new Dictionary <string, IndexBase>(); // scalar indexed fields foreach (var indexField in typeDescription.IndexFields) { var index = IndexFactory.CreateIndex(indexField); _dataByIndexKey.Add(indexField.Name, index); } // list indexed fields foreach (var indexField in typeDescription.ListFields) { var index = IndexFactory.CreateIndex(indexField); _dataByIndexKey.Add(indexField.Name, index); } }
public void Touch(PackedObject packedObject) { EvictionPolicy?.Touch(packedObject); }