private async Task <IVersionedKeyValue> ReadPropertiesFromRemoteStoreAsync(Guid activityId, string keyName, string versionKeyName)
        {
            traceType.WriteInfo("Starting ReadPropertiesFromRemoteStoreAsync for store name '{0}'. ActivityId: {1}", storeName, activityId);

            var propertyOperations = new List <PropertyBatchOperation>
            {
                // operation index
                new GetPropertyOperation(versionKeyName, true), // 0
                new GetPropertyOperation(keyName, true),        // 1
            };

            try
            {
                IPropertyBatchResultWrapper result = await retryPolicyFactory.Create().ExecuteAsync(
                    async token =>
                    await propertyManager.SubmitPropertyBatchAsync(
                        storeName,
                        propertyOperations).ConfigureAwait(false),
                    "SubmitPropertyBatchAsync",
                    CancellationToken.None).ConfigureAwait(false);

                traceType.WriteInfo(
                    "SubmitPropertyBatchAsync returned result.FailedOperationIndex: {0}, result.FailedOperationException: {1}, ActivityId: {2}",
                    result.FailedOperationIndex,
                    result.FailedOperationException != null ? result.FailedOperationException.ToString() : "<null>",
                    activityId);

                if (result.FailedOperationIndex == -1)
                {
                    traceType.WriteInfo(
                        "ReadPropertiesFromRemoteStoreAsync succeeded for store name '{0}'. Key: {1}, Version Key: {2}, ActivityId: {3}",
                        storeName,
                        keyName,
                        versionKeyName,
                        activityId);

                    // operation succeeded
                    var version = result.GetProperty(0).GetValue <Int64>();
                    var value   = result.GetProperty(1).GetValue <string>();

                    return(new VersionedKeyValue(keyName, versionKeyName, value, version));
                }

                string message = string.Format(
                    CultureInfo.InvariantCulture,
                    "ReadPropertiesFromRemoteStoreAsync failed for store name '{0}'. Key: {1}, VersionKey: {2}, ActivityId: {3}, Error: {4}",
                    storeName,
                    keyName,
                    versionKeyName,
                    activityId,
                    result.FailedOperationException);

                throw result.FailedOperationException ?? new FabricException(message, FabricErrorCode.Unknown);
            }
            catch (Exception ex)
            {
                string message =
                    "Error while reading key for store name '{0}'. Key: {1}, VersionKey: {2}, ActivityId: {3}, Error: {4}"
                    .ToString(storeName, keyName, versionKeyName, activityId, ex);
                traceType.WriteInfo(message);
                throw;
            }
        }
        public async Task UpdateValueAsync(Guid activityId, string keyName, string versionKeyName, string value)
        {
            keyName.Validate("keyName");
            versionKeyName.Validate("versionKeyName");
            value.Validate("value");

            traceType.WriteInfo(
                "Starting VersionedPropertyStore.UpdateValueAsync for store name '{0}'. Key: {1}, VersionKey: {2}, Value: {3}, ActivityId: {4}",
                storeName, keyName, versionKeyName, value, activityId);

            Int64 oldVersion, newVersion;

            lock (locker)
            {
                oldVersion = cachedProperties.ContainsKey(keyName) ? cachedProperties[keyName].Version : -1;
                newVersion = oldVersion + 1;
            }

            // Publish the new values to the naming service, including retrieving the new values
            // to update the local cache
            var propertyOperations = new List <PropertyBatchOperation>
            {
                // operation index
                new CheckValuePropertyOperation(versionKeyName, oldVersion), // 0
                new PutPropertyOperation(versionKeyName, newVersion),        // 1
                new GetPropertyOperation(versionKeyName, true),              // 2
                new PutPropertyOperation(keyName, value),                    // 3
                new GetPropertyOperation(keyName, true)                      // 4
            };

            try
            {
                IPropertyBatchResultWrapper result = await retryPolicyFactory.Create().ExecuteAsync(
                    async token => await propertyManager.SubmitPropertyBatchAsync(
                        storeName,
                        propertyOperations).ConfigureAwait(false),
                    "SubmitPropertyBatchAsync",
                    CancellationToken.None).ConfigureAwait(false);

                traceType.WriteInfo(
                    "SubmitPropertyBatchAsync returned result.FailedOperationIndex: {0}, result.FailedOperationException: {1}, ActivityId: {2}",
                    result.FailedOperationIndex,
                    result.FailedOperationException != null ? result.FailedOperationException.ToString() : "<null>",
                    activityId);

                switch (result.FailedOperationIndex)
                {
                case -1:
                    lock (locker)
                    {
                        // Note; these indexes are tied to the batch operation array above. If you adjust
                        // the set of operations, need to update these offsets accordingly
                        Int64  version  = result.GetProperty(2).GetValue <Int64>();
                        string newValue = result.GetProperty(4).GetValue <string>();

                        var updatedVkv = UpdateCache(activityId, new VersionedKeyValue(keyName, versionKeyName, newValue, version));

                        string message =
                            "UpdateValueAsync successful for store name '{0}'. New cached value: {1}, ActivityId: {2}"
                            .ToString(storeName, updatedVkv, activityId);

                        traceType.WriteInfo(message);
                    }
                    break;

                case 0:
                {
                    // optimistic concurrency check failed (someone else has updated the version in the remote store)
                    string message =
                        ("UpdateValueAsync policy failed for store name '{0}' due to version mismatch. This is because of a possible write conflict. Please retry. " +
                         "Key: {1}, VersionKey: {2}, ActivityId: {3}").ToString(
                            storeName,
                            keyName,
                            versionKeyName,
                            activityId);
                    traceType.WriteInfo(message);

                    // get the latest version so that the cache is updated
                    await GetValueAsync(activityId, keyName, versionKeyName, false).ConfigureAwait(false);

                    throw new FabricException(message, FabricErrorCode.WriteConflict);
                }

                default:
                {
                    string message =
                        "UpdateValueAsync policy failed for store name '{0}'. " +
                        "Key: {1}, VersionKey: {2}, ActivityId: {3}".ToString(
                            storeName,
                            keyName,
                            versionKeyName,
                            activityId);
                    var ex = result.FailedOperationException ?? new FabricException(message, FabricErrorCode.Unknown);
                    throw ex;
                }
                }
            }
            catch (Exception ex)
            {
                string message =
                    "Error while updating key for store name '{0}'. Key: {1}, VersionKey: {2}, Value: {3}, ActivityId: {4}, Error: {5}"
                    .ToString(storeName, keyName, versionKeyName, value, activityId, ex);
                traceType.WriteInfo(message);
                throw;
            }
        }