public async Task <bool> UpsertAsync(DataKind kind, string key, SerializedItemDescriptor newItem)
{
var encodedItem = MarshalItem(kind, key, newItem);
if (!CheckSizeLimit(encodedItem))
{
return(false);
}
try
{
var request = new PutItemRequest(_tableName, encodedItem);
request.ConditionExpression = "attribute_not_exists(#namespace) or attribute_not_exists(#key) or :version > #version";
request.ExpressionAttributeNames = new Dictionary <string, string>()
{
{ "#namespace", DynamoDB.DataStorePartitionKey },
{ "#key", DynamoDB.DataStoreSortKey },
{ "#version", VersionAttribute }
};
request.ExpressionAttributeValues = new Dictionary <string, AttributeValue>()
{
{ ":version", new AttributeValue()
{
N = Convert.ToString(newItem.Version)
} }
};
await _client.PutItemAsync(request);
}
catch (ConditionalCheckFailedException)
{
return(false);
}
return(true);
}
private SerializedItemDescriptor StorableItem(DataKind kind, SerializedItemDescriptor item)
{
if (item.Deleted && !PersistOnlyAsString)
{
// This simulates the kind of store implementation that *can* track metadata separately, so we don't
// have to persist the placeholder string for deleted items
return(new SerializedItemDescriptor(item.Version, true, null));
}
return(item);
}
public bool Upsert(DataKind kind, string key, SerializedItemDescriptor item)
{
var dict = _db.DataForPrefixAndKind(_prefix, kind);
if (dict.TryGetValue(key, out var oldItem) && oldItem.Version >= item.Version)
{
return(false);
}
dict[key] = item;
return(true);
}
private ItemDescriptor Deserialize(DataKind kind, SerializedItemDescriptor serializedItemDesc)
{
if (serializedItemDesc.Deleted || serializedItemDesc.SerializedItem is null)
{
return(ItemDescriptor.Deleted(serializedItemDesc.Version));
}
var deserializedItem = kind.Deserialize(serializedItemDesc.SerializedItem);
if (serializedItemDesc.Version == 0 || serializedItemDesc.Version == deserializedItem.Version ||
deserializedItem.Item is null)
{
return(deserializedItem);
}
// If the store gave us a version number that isn't what was encoded in the object, trust it
return(new ItemDescriptor(serializedItemDesc.Version, deserializedItem.Item));
}
private static void GetTooLargeItemParams(
string flagOrSegment,
out DataKind dataKind,
out int collIndex,
out SerializedItemDescriptor item
)
{
string tooBigKeysListJson = "[";
for (var i = 0; i < 40000; i++)
{
if (i != 0)
{
tooBigKeysListJson += ",";
}
tooBigKeysListJson += @"""key" + i + @"""";
}
tooBigKeysListJson += "]";
Assert.NotInRange(tooBigKeysListJson.Length, 0, 400 * 1024);
string badItemJson;
switch (flagOrSegment)
{
case "flag":
dataKind = DataModel.Features;
collIndex = 0;
badItemJson = @"{""key"":""" + BadItemKey + @""", ""version"": 1, ""targets"":[{""variation"":0,""values"":" +
tooBigKeysListJson + "}]}";
break;
case "segment":
dataKind = DataModel.Segments;
collIndex = 1;
badItemJson = @"{""key"":""" + BadItemKey + @""", ""version"": 1, ""included"":" + tooBigKeysListJson + "]}";
break;
default:
throw new ArgumentException("invalid type parameter");
}
item = new SerializedItemDescriptor(1, false, badItemJson);
}
public bool Upsert(DataKind kind, string key, SerializedItemDescriptor item)
{
MaybeThrowError();
if (!Data.ContainsKey(kind))
{
Data[kind] = new Dictionary <string, SerializedItemDescriptor>();
}
if (Data[kind].TryGetValue(key, out var oldItem))
{
// If PersistOnlyAsString is true, simulate the kind of implementation where we can't see the
// version as a separate attribute in the database and must deserialize the item to get it.
var oldVersion = PersistOnlyAsString ?
kind.Deserialize(oldItem.SerializedItem).Version :
oldItem.Version;
if (oldVersion >= item.Version)
{
return(false);
}
}
Data[kind][key] = StorableItem(kind, item);
return(true);
}
public async Task <bool> UpsertAsync(DataKind kind, string key, SerializedItemDescriptor newItem)
{
var fullKey = ItemKey(kind, key);
// We will potentially keep retrying indefinitely until someone's write succeeds
while (true)
{
var oldValue = (await _client.KV.Get(fullKey)).Response;
var oldVersion = oldValue is null ? 0 :
kind.Deserialize(Encoding.UTF8.GetString(oldValue.Value)).Version;
// Check whether the item is stale. If so, don't do the update (and return the existing item to
// FeatureStoreWrapper so it can be cached)
if (oldVersion >= newItem.Version)
{
return(false);
}
// Otherwise, try to write. We will do a compare-and-set operation, so the write will only succeed if
// the key's ModifyIndex is still equal to the previous value returned by getEvenIfDeleted. If the
// previous ModifyIndex was zero, it means the key did not previously exist and the write will only
// succeed if it still doesn't exist.
var modifyIndex = oldValue == null ? 0 : oldValue.ModifyIndex;
var pair = new KVPair(fullKey)
{
Value = Encoding.UTF8.GetBytes(newItem.SerializedItem),
ModifyIndex = modifyIndex
};
var result = await _client.KV.CAS(pair);
if (result.Response)
{
return(true);
}
// If we failed, retry the whole shebang
_log.Debug("Concurrent modification detected, retrying");
}
}
public bool Upsert(DataKind kind, string key, SerializedItemDescriptor newItem)
{
IDatabase db = _redis.GetDatabase();
string baseKey = ItemsKey(kind);
while (true)
{
string oldData;
try
{
oldData = db.HashGet(baseKey, key);
}
catch (RedisTimeoutException e)
{
_log.Error("Timeout in update when reading {0} from {1}: {2}", key, baseKey, e.ToString());
throw;
}
// Here, unfortunately, we have to deserialize the old item (if any) just to find
// out its version number (see implementation notes).
var oldVersion = (oldData is null) ? 0 : kind.Deserialize(oldData).Version;
if (oldVersion >= newItem.Version)
{
_log.Debug("Attempted to {0} key: {1} version: {2} with a version that is" +
" the same or older: {3} in \"{4}\"",
newItem.Deleted ? "delete" : "update",
key, oldVersion, newItem.Version, kind.Name);
return(false);
}
// This hook is used only in unit tests
_updateHook?.Invoke();
// Note that transactions work a bit differently in StackExchange.Redis than in other
// Redis clients. The same Redis connection is shared across all threads, so it can't
// set a WATCH at the moment we start the transaction. Instead, it saves up all of
// the actions we send during the transaction, and replays them all within a MULTI
// when the transaction. AddCondition() is this client's way of doing a WATCH, and it
// can only use an equality match on the whole value (which is unfortunate since a
// serialized flag value could be fairly large).
ITransaction txn = db.CreateTransaction();
txn.AddCondition(oldData is null ? Condition.HashNotExists(baseKey, key) :
Condition.HashEqual(baseKey, key, oldData));
txn.HashSetAsync(baseKey, key, newItem.SerializedItem);
try
{
bool success = txn.Execute();
if (!success)
{
// The watch was triggered, we should retry
_log.Debug("Concurrent modification detected, retrying");
continue;
}
}
catch (RedisTimeoutException e)
{
_log.Error("Timeout on update of {0} in {1}: {2}", key, baseKey, e.ToString());
throw;
}
return(true);
}
}
public async Task <bool> UpsertAsync(DataKind kind, string key, SerializedItemDescriptor item)
{
await ArbitraryTask();
return(Upsert(kind, key, item));
}
private Dictionary <string, AttributeValue> MarshalItem(DataKind kind, string key, SerializedItemDescriptor item)
{
var ret = MakeKeysMap(NamespaceForKind(kind), key);
ret[VersionAttribute] = new AttributeValue()
{
N = item.Version.ToString()
};
ret[SerializedItemAttribute] = new AttributeValue(item.Deleted ? DeletedItemPlaceholder : item.SerializedItem);
return(ret);
}
private static async Task <bool> Upsert(IDisposable store, DataKind kind, string key, SerializedItemDescriptor item)
{
if (store is IPersistentDataStore syncStore)
{
return(syncStore.Upsert(kind, key, item));
}
return(await(store as IPersistentDataStoreAsync).UpsertAsync(kind, key, item));
}
public bool Upsert(DataKind kind, string key, SerializedItemDescriptor item)
{
return(WaitSafely(() => _coreAsync.UpsertAsync(kind, key, item)));
}
public async Task <bool> UpsertAsync(DataKind kind, string key, SerializedItemDescriptor item) => _syncStore.Upsert(kind, key, item);
