public void RemoveMany(IList <PackedObject> items)
{
Dbg.Trace($"remove many called for {items.Count} items");
foreach (var item in items)
{
foreach (var metadata in CollectionSchema.ServerSide)
{
if (metadata.IndexType == IndexType.Unique)
{
_dataByUniqueKey[metadata.Name].Remove(item[metadata.Order]);
}
}
// if present remove it from the full-text index
_fullTextIndex?.DeleteDocument(item.PrimaryKey);
EvictionPolicy.TryRemove(item);
}
foreach (var index in _dataByIndexKey)
{
index.Value.RemoveMany(items);
}
foreach (var o in items)
{
DataByPrimaryKey.Remove(o.PrimaryKey);
}
}
public void Truncate()
{
EvictionPolicy.Clear();
DataByPrimaryKey.Clear();
foreach (var indexByKey in _dataByUniqueKey)
{
indexByKey.Value.Clear();
}
foreach (var index in _dataByIndexKey)
{
index.Value.Clear();
}
_fullTextIndex?.Clear();
Interlocked.Exchange(ref _readCount, 0);
Interlocked.Exchange(ref _hitCount, 0);
// also reset the domain description
DomainDescription = null;
}
/// <summary>
/// Store a new object in all the indexes
/// REQUIRE: an object with the same primary key is not present in the datastore
/// </summary>
/// <param name="packedObject"></param>
/// <param name="excludeFromEviction">if true the item will never be evicted</param>
internal void InternalAddNew(PackedObject packedObject, bool excludeFromEviction)
{
InternalAddNew(packedObject);
if (!excludeFromEviction)
{
EvictionPolicy.AddItem(packedObject);
}
}
/// <summary>
/// </summary>
/// <param name="items"></param>
/// <param name="excludeFromEviction">used only for non persistent case</param>
internal void InternalPutMany(IList <PackedObject> items, bool excludeFromEviction)
{
var isBulkOperation = items.Count > Constants.BulkThreshold;
var toUpdate = new List <PackedObject>();
try
{
Dbg.Trace($"begin InternalPutMany with {items.Count} object");
InternalBeginBulkInsert(isBulkOperation);
foreach (var item in items)
{
if (DataByPrimaryKey.ContainsKey(item.PrimaryKey))
{
toUpdate.Add(item);
}
else
{
InternalAddNew(item, excludeFromEviction);
}
}
}
finally
{
InternalEndBulkInsert(isBulkOperation);
// update items outside the bulk insert
if (toUpdate.Count > Constants.BulkThreshold) // bulk optimization for updates
{
RemoveMany(toUpdate);
InternalPutMany(toUpdate, true);
}
else
{
foreach (var cachedObject in toUpdate)
{
InternalUpdate(cachedObject);
}
}
foreach (var cachedObject in toUpdate)
{
EvictionPolicy.Touch(cachedObject);
}
Dbg.Trace($"end InternalPutMany with {items.Count} object");
}
}
public PackedObject RemoveByPrimaryKey(KeyValue primary)
{
var removed = InternalRemoveByPrimaryKey(primary);
if (removed != null)
{
EvictionPolicy.TryRemove(removed);
}
return(removed);
}
/// <summary>
/// Initialize an empty datastore from a type description
/// </summary>
/// <param name="typeDescription"></param>
/// <param name="evictionPolicy"></param>
/// <param name="config"></param>
public DataStore(TypeDescription typeDescription, EvictionPolicy evictionPolicy, NodeConfig config)
{
TypeDescription = typeDescription ?? throw new ArgumentNullException(nameof(typeDescription));
EvictionPolicy = evictionPolicy ?? throw new ArgumentNullException(nameof(evictionPolicy));
//initialize the primary key dictionary
_dataByPrimaryKey = new Dictionary <KeyValue, CachedObject>();
//initialize the unique keys dictionaries (une by unique key)
_dataByUniqueKey = new Dictionary <string, Dictionary <KeyValue, CachedObject> >();
foreach (var keyInfo in typeDescription.UniqueKeyFields)
{
_dataByUniqueKey.Add(keyInfo.Name, new Dictionary <KeyValue, CachedObject>());
}
//initialize the indexes (one by index key)
_dataByIndexKey = new Dictionary <string, IndexBase>();
// scalar indexed fields
foreach (var indexField in typeDescription.IndexFields)
{
var index = IndexFactory.CreateIndex(indexField);
_dataByIndexKey.Add(indexField.Name, index);
}
// list indexed fields
foreach (var indexField in typeDescription.ListFields)
{
var index = IndexFactory.CreateIndex(indexField);
_dataByIndexKey.Add(indexField.Name, index);
}
// create the full-text index if required
if (typeDescription.FullText.Count > 0)
{
_fullTextIndex = new FullTextIndex(config.FullTextConfig)
{
// a function that allows the full text engine to find the original line of text
LineProvider = pointer => _dataByPrimaryKey[pointer.PrimaryKey].FullText[pointer.Line]
};
}
}
/// <summary>
/// Update an object previously stored
/// The primary key must be the same, all others can change
/// </summary>
/// <param name="item"></param>
public void InternalUpdate(PackedObject item)
{
if (item == null)
{
throw new ArgumentNullException(nameof(item));
}
if (!DataByPrimaryKey.ContainsKey(item.PrimaryKey))
{
var msg = $"Update called for the object {item} which is not stored in the cache";
throw new NotSupportedException(msg);
}
InternalRemoveByPrimaryKey(item.PrimaryKey);
InternalAddNew(item);
EvictionPolicy.Touch(item);
}
/// <summary>
/// Initialize an empty datastore from a type description
/// </summary>
/// <param name="collectionSchema"></param>
/// <param name="evictionPolicy"></param>
/// <param name="fullTextConfig"></param>
public DataStore(CollectionSchema collectionSchema, EvictionPolicy evictionPolicy,
FullTextConfig fullTextConfig)
{
_fullTextConfig = fullTextConfig;
CollectionSchema = collectionSchema ?? throw new ArgumentNullException(nameof(collectionSchema));
EvictionPolicy = evictionPolicy ?? throw new ArgumentNullException(nameof(evictionPolicy));
//initialize the primary key dictionary
DataByPrimaryKey = new Dictionary <KeyValue, PackedObject>();
//initialize the unique keys dictionaries (one by unique key)
_dataByUniqueKey = new Dictionary <string, Dictionary <KeyValue, PackedObject> >();
foreach (var keyInfo in collectionSchema.UniqueKeyFields)
{
_dataByUniqueKey.Add(keyInfo.Name, new Dictionary <KeyValue, PackedObject>());
}
//initialize the indexes (one by index key)
_dataByIndexKey = new Dictionary <string, IndexBase>();
// scalar indexed fields
foreach (var indexField in collectionSchema.IndexFields)
{
var index = IndexFactory.CreateIndex(indexField);
_dataByIndexKey.Add(indexField.Name, index);
}
// create the full-text index if required
if (collectionSchema.FullText.Count > 0)
{
_fullTextIndex = new FullTextIndex(fullTextConfig)
{
// a function that allows the full text engine to find the original line of text
LineProvider = pointer => DataByPrimaryKey[pointer.PrimaryKey].TokenizedFullText[pointer.Line]
}
}
;
}
public void ProcessEviction()
{
if (EvictionPolicy.IsEvictionRequired)
{
var itemsToEvict = EvictionPolicy.DoEviction();
foreach (var item in itemsToEvict)
{
InternalRemoveByPrimaryKey(item.PrimaryKey);
}
var requestDescription = string.Empty;
var processedItems = itemsToEvict.Count;
var requestType = "EVICTION";
ServerLog.AddEntry(new ServerLogEntry(0, requestType, requestDescription,
processedItems));
}
}
/// <summary>
/// Like a bulk insert on an empty datastore. Used internally to reindex data when type description changed
/// </summary>
/// <param name="items"></param>
private void InternalReindex(IEnumerable <PackedObject> items)
{
var toUpdate = new List <PackedObject>();
try
{
InternalBeginBulkInsert(true);
foreach (var item in items)
{
InternalAddNew(item, true);
}
}
finally
{
InternalEndBulkInsert(true);
// update items outside the bulk insert
if (toUpdate.Count > Constants.BulkThreshold) // bulk optimization for updates
{
RemoveMany(toUpdate);
InternalPutMany(toUpdate, true);
}
else
{
foreach (var cachedObject in toUpdate)
{
InternalUpdate(cachedObject);
}
}
//TODO check if necessary
foreach (var cachedObject in toUpdate)
{
EvictionPolicy.Touch(cachedObject);
}
}
}
/// <summary>
/// Initialize an empty datastore from a type description
/// </summary>
/// <param name="typeDescription"></param>
/// <param name="evictionPolicy"></param>
public DataStore(TypeDescription typeDescription, EvictionPolicy evictionPolicy)
{
TypeDescription = typeDescription ?? throw new ArgumentNullException(nameof(typeDescription));
EvictionPolicy = evictionPolicy ?? throw new ArgumentNullException(nameof(evictionPolicy));
//initialize the primary key dictionary
_dataByPrimaryKey = new Dictionary <KeyValue, CachedObject>();
//initialize the unique keys dictionaries (une by unique key)
_dataByUniqueKey = new Dictionary <string, Dictionary <KeyValue, CachedObject> >();
foreach (var keyInfo in typeDescription.UniqueKeyFields)
{
_dataByUniqueKey.Add(keyInfo.Name, new Dictionary <KeyValue, CachedObject>());
}
//initialize the indexes (one by index key)
_dataByIndexKey = new Dictionary <string, IndexBase>();
// scalar indexed fields
foreach (var indexField in typeDescription.IndexFields)
{
var index = IndexFactory.CreateIndex(indexField);
_dataByIndexKey.Add(indexField.Name, index);
}
// list indexed fields
foreach (var indexField in typeDescription.ListFields)
{
var index = IndexFactory.CreateIndex(indexField);
_dataByIndexKey.Add(indexField.Name, index);
}
}
public void Touch(PackedObject packedObject)
{
EvictionPolicy?.Touch(packedObject);
}