private FatEntity BuildEntity(string partitionKey, string rowKey)
{
var e = new FatEntity { PartitionKey = partitionKey, RowKey = rowKey, ETag = "*" };
var data = new { testdata = "blah" };
var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(data));
e.SetData(bytes, bytes.Length);
return e;
}
public void Delete(E entity)
{
CheckNotSaved();
// Delete is special in that we do not need the data to delete
var fat = new FatEntity
{
PartitionKey = entity.PartitionKey,
RowKey = entity.RowKey,
ETag = "*"
};
_context.Delete(fat);
_deleteBackupContext.InsertOrReplace(fat);
_isDirty = true;
}
/// <summary>Converts a <c>CloudEntity</c> toward a <c>FatEntity</c>.</summary>
public static FatEntity Convert <T>(CloudEntity <T> cloudEntity, IDataSerializer serializer)
{
var fatEntity = new FatEntity
{
PartitionKey = cloudEntity.PartitionKey,
RowKey = cloudEntity.RowKey,
Timestamp = cloudEntity.Timestamp
};
using (var stream = new MemoryStream())
{
serializer.Serialize(cloudEntity.Value, stream, typeof(T));
fatEntity.SetData(stream.ToArray());
return(fatEntity);
}
}
/// <summary>Converts a <c>FatEntity</c> toward a <c>CloudEntity</c>.</summary>
public static CloudEntity <T> Convert <T>(FatEntity fatEntity, IDataSerializer serializer, string etag)
{
using (var stream = new MemoryStream(fatEntity.GetData())
{
Position = 0
})
{
var val = (T)serializer.Deserialize(stream, typeof(T));
return(new CloudEntity <T>
{
PartitionKey = fatEntity.PartitionKey,
RowKey = fatEntity.RowKey,
Timestamp = fatEntity.Timestamp,
ETag = etag,
Value = val
});
}
}
private FatEntity ConvertToFatEntity(E entity)
{
byte[] data;
if (entity.DataObject == null)
{
data = new byte[0];
}
else
{
data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(entity.DataObject));
if (_encryptor != null)
{
using(var ms = new MemoryStream())
{
using(var enc = _encryptor.Encrypt(ms, false))
{
enc.Write(data, 0, data.Length);
}
data = ms.ToArray();
}
}
}
var fat = new FatEntity()
{
PartitionKey = entity.PartitionKey,
RowKey = entity.RowKey
};
fat.SetData(data, data.Length);
return fat;
}
/// <remarks></remarks>
void DeleteInternal <T>(string tableName, string partitionKey, IEnumerable <Tuple <string, string> > rowKeysAndETags, bool force)
{
var context = _tableStorage.GetTableServiceContext();
var stopwatch = new Stopwatch();
// CAUTION: make sure to get rid of potential duplicate in rowkeys.
// (otherwise insertion in 'context' is likely to fail)
foreach (var s in Slice(rowKeysAndETags
// Similar effect than 'Distinct' based on 'RowKey'
.ToLookup(p => p.Item1, p => p).Select(g => g.First()),
MaxEntityTransactionCount))
{
stopwatch.Restart();
var slice = s;
DeletionStart: // 'slice' might have been refreshed if some entities were already deleted
foreach (var rowKeyAndETag in slice)
{
// Deleting entities in 1 roundtrip
// http://blog.smarx.com/posts/deleting-entities-from-windows-azure-without-querying-first
var mock = new FatEntity
{
PartitionKey = partitionKey,
RowKey = rowKeyAndETag.Item1
};
context.AttachTo(tableName, mock, rowKeyAndETag.Item2);
context.DeleteObject(mock);
}
try // HACK: [vermorel] if a single entity is missing, then the whole batch operation is aborded
{
try // HACK: nested try/catch to handle the special case where the table is missing
{
Retry.Do(_policies.TransientTableErrorBackOff(), CancellationToken.None, () => context.SaveChanges(SaveChangesOptions.Batch));
}
catch (DataServiceRequestException ex)
{
// if the table is missing, no need to go on with the deletion
var errorCode = RetryPolicies.GetErrorCode(ex);
if (TableErrorCodeStrings.TableNotFound == errorCode)
{
NotifySucceeded(StorageOperationType.TableDelete, stopwatch);
return;
}
throw;
}
}
// if some entities exist
catch (DataServiceRequestException ex)
{
var errorCode = RetryPolicies.GetErrorCode(ex);
// HACK: Table Storage both implement a bizarre non-idempotent semantic
// but in addition, it throws a non-documented exception as well.
if (errorCode != "ResourceNotFound")
{
throw;
}
slice = Get <T>(tableName, partitionKey, slice.Select(p => p.Item1))
.Select(e => Tuple.Create(e.RowKey, MapETag(e.ETag, force))).ToArray();
// entities with same name will be added again
context = _tableStorage.GetTableServiceContext();
// HACK: [vermorel] yes, gotos are horrid, but other solutions are worst here.
goto DeletionStart;
}
NotifySucceeded(StorageOperationType.TableDelete, stopwatch);
}
}
/// <remarks>Upsert is making several storage calls to emulate the
/// missing semantic from the Table Storage.</remarks>
void UpsertInternal <T>(string tableName, IEnumerable <CloudEntity <T> > entities)
{
var context = _tableStorage.GetTableServiceContext();
context.MergeOption = MergeOption.AppendOnly;
context.ResolveType = ResolveFatEntityType;
var stopwatch = new Stopwatch();
var fatEntities = entities.Select(e => Tuple.Create(FatEntity.Convert(e, _serializer), e));
var noBatchMode = false;
foreach (var slice in SliceEntities(fatEntities, e => e.Item1.GetPayload()))
{
stopwatch.Restart();
var cloudEntityOfFatEntity = new Dictionary <object, CloudEntity <T> >();
foreach (var fatEntity in slice)
{
// entities should be updated in a single round-trip
context.AttachTo(tableName, fatEntity.Item1);
context.UpdateObject(fatEntity.Item1);
cloudEntityOfFatEntity.Add(fatEntity.Item1, fatEntity.Item2);
}
Retry.Do(_policies.TransientTableErrorBackOff(), CancellationToken.None, () =>
{
try
{
context.SaveChanges(noBatchMode ? SaveChangesOptions.ReplaceOnUpdate : SaveChangesOptions.ReplaceOnUpdate | SaveChangesOptions.Batch);
ReadETagsAndDetach(context, (entity, etag) => cloudEntityOfFatEntity[entity].ETag = etag);
}
catch (DataServiceRequestException ex)
{
var errorCode = RetryPolicies.GetErrorCode(ex);
if (errorCode == StorageErrorCodeStrings.OperationTimedOut)
{
// if batch does not work, then split into elementary requests
// PERF: it would be better to split the request in two and retry
context.SaveChanges(SaveChangesOptions.ReplaceOnUpdate);
ReadETagsAndDetach(context, (entity, etag) => cloudEntityOfFatEntity[entity].ETag = etag);
noBatchMode = true;
}
else if (errorCode == TableErrorCodeStrings.TableNotFound)
{
Retry.Do(_policies.SlowInstantiation(), CancellationToken.None, () =>
{
try
{
var table = _tableStorage.GetTableReference(tableName);
table.CreateIfNotExists();
}
// HACK: incorrect behavior of the StorageClient (2010-09)
// Fails to behave properly in multi-threaded situations
catch (StorageException cex)
{
var extended = cex.RequestInformation != null ? cex.RequestInformation.ExtendedErrorInformation : null;
if (extended == null || extended.ErrorCode != TableErrorCodeStrings.TableAlreadyExists)
{
throw;
}
}
context.SaveChanges(noBatchMode ? SaveChangesOptions.ReplaceOnUpdate : SaveChangesOptions.ReplaceOnUpdate | SaveChangesOptions.Batch);
ReadETagsAndDetach(context, (entity, etag) => cloudEntityOfFatEntity[entity].ETag = etag);
});
}
else if (errorCode == StorageErrorCodeStrings.ResourceNotFound)
{
throw new InvalidOperationException("Cannot call update on a resource that does not exist", ex);
}
else
{
throw;
}
}
catch (DataServiceQueryException ex)
{
// HACK: code duplicated
var errorCode = RetryPolicies.GetErrorCode(ex);
if (errorCode == StorageErrorCodeStrings.OperationTimedOut)
{
// if batch does not work, then split into elementary requests
// PERF: it would be better to split the request in two and retry
context.SaveChanges(SaveChangesOptions.ReplaceOnUpdate);
ReadETagsAndDetach(context, (entity, etag) => cloudEntityOfFatEntity[entity].ETag = etag);
noBatchMode = true;
}
else
{
throw;
}
}
});
NotifySucceeded(StorageOperationType.TableUpsert, stopwatch);
}
}
/// <remarks></remarks>
private IEnumerable <CloudEntity <T> > GetInternal <T>(TableServiceContext context, string tableName, Maybe <string> filter)
{
string continuationRowKey = null;
string continuationPartitionKey = null;
var stopwatch = Stopwatch.StartNew();
context.MergeOption = MergeOption.AppendOnly;
context.ResolveType = ResolveFatEntityType;
do
{
var query = context.CreateQuery <FatEntity>(tableName);
if (filter.HasValue)
{
query = query.AddQueryOption("$filter", filter.Value);
}
if (null != continuationRowKey)
{
query = query.AddQueryOption(NextRowKeyToken, continuationRowKey)
.AddQueryOption(NextPartitionKeyToken, continuationPartitionKey);
}
QueryOperationResponse response = null;
FatEntity[] fatEntities = null;
Retry.Do(_policies.TransientTableErrorBackOff(), CancellationToken.None, () =>
{
try
{
response = query.Execute() as QueryOperationResponse;
fatEntities = ((IEnumerable <FatEntity>)response).ToArray();
}
catch (DataServiceQueryException ex)
{
// if the table does not exist, there is nothing to return
var errorCode = RetryPolicies.GetErrorCode(ex);
if (TableErrorCodeStrings.TableNotFound == errorCode ||
StorageErrorCodeStrings.ResourceNotFound == errorCode)
{
fatEntities = new FatEntity[0];
return;
}
throw;
}
});
NotifySucceeded(StorageOperationType.TableQuery, stopwatch);
foreach (var fatEntity in fatEntities)
{
var etag = context.Entities.First(e => e.Entity == fatEntity).ETag;
context.Detach(fatEntity);
yield return(FatEntity.Convert <T>(fatEntity, _serializer, etag));
}
Debug.Assert(context.Entities.Count == 0);
if (null != response && response.Headers.ContainsKey(ContinuationNextRowKeyToken))
{
continuationRowKey = response.Headers[ContinuationNextRowKeyToken];
continuationPartitionKey = response.Headers[ContinuationNextPartitionKeyToken];
stopwatch.Restart();
}
else
{
continuationRowKey = null;
continuationPartitionKey = null;
}
} while (null != continuationRowKey);
}