/// <summary>
/// Test upsert (insert or replace) on entities inside and outside the given range.
/// </summary>
/// <param name="testClient">The table client to test.</param>
/// <param name="tableName">The name of the table to test.</param>
/// <param name="accessPermissions">The access permissions of the table client.</param>
/// <param name="startPk">The start partition key range.</param>
/// <param name="startRk">The start row key range.</param>
/// <param name="endPk">The end partition key range.</param>
/// <param name="endRk">The end row key range.</param>
private void TestUpsertReplace(
CloudTableClient testClient,
string tableName,
SharedAccessTablePermissions accessPermissions,
string startPk,
string startRk,
string endPk,
string endRk)
{
Action <BaseEntity> upsertDelegate = (tableEntity) =>
{
TableServiceContext context = testClient.GetTableServiceContext();
// Replace entity
tableEntity.A = "10";
context.AttachTo(tableName, tableEntity);
context.UpdateObject(tableEntity);
context.SaveChangesWithRetries(SaveChangesOptions.ReplaceOnUpdate);
};
SharedAccessTablePermissions upsertPermissions = (SharedAccessTablePermissions.Update | SharedAccessTablePermissions.Add);
bool expectSuccess = (accessPermissions & upsertPermissions) == upsertPermissions;
// Perform test
TestOperationWithRange(
tableName,
startPk,
startRk,
endPk,
endRk,
upsertDelegate,
"upsert replace",
expectSuccess,
expectSuccess ? HttpStatusCode.NoContent : HttpStatusCode.NotFound);
}
/// <summary>
/// Test deleting entities inside and outside the given range.
/// </summary>
/// <param name="testClient">The table client to test.</param>
/// <param name="tableName">The name of the table to test.</param>
/// <param name="accessPermissions">The access permissions of the table client.</param>
/// <param name="startPk">The start partition key range.</param>
/// <param name="startRk">The start row key range.</param>
/// <param name="endPk">The end partition key range.</param>
/// <param name="endRk">The end row key range.</param>
private void TestDelete(
CloudTableClient testClient,
string tableName,
SharedAccessTablePermissions accessPermissions,
string startPk,
string startRk,
string endPk,
string endRk)
{
TableServiceContext referenceContext = testClient.GetTableServiceContext();
Action <BaseEntity> deleteDelegate = (tableEntity) =>
{
TableServiceContext context = testClient.GetTableServiceContext();
context.AttachTo(tableName, tableEntity, "*");
context.DeleteObject(tableEntity);
context.SaveChangesWithRetries();
context.Detach(tableEntity);
};
bool expectSuccess = (accessPermissions & SharedAccessTablePermissions.Delete) != 0;
// Perform test
TestOperationWithRange(
tableName,
startPk,
startRk,
endPk,
endRk,
deleteDelegate,
"delete",
expectSuccess,
expectSuccess ? HttpStatusCode.NoContent : HttpStatusCode.NotFound);
}
public void Delete(T item)
{
int trycount = 0;
bool success = false;
while (trycount++ < 1)
{
try
{
TableServiceContext tableServiceContext = TableContext();
tableServiceContext.AttachTo(tableName, item, "*");
tableServiceContext.DeleteObject(item);
tableServiceContext.SaveChangesWithRetries();
success = true;
break;
}
catch
{
Connect();
}
}
if (!success)
{
throw new Exception("Could not delete " + typeof(T).Name);
}
}
public void AddOrUpdateEntity(IEnumerable <T> objs)
{
foreach (var obj in objs)
{
var pk = obj.PartitionKey;
var rk = obj.RowKey;
T existingObj = null;
try
{
existingObj = (from o in this.Query
where o.PartitionKey == pk && o.RowKey == rk
select o).SingleOrDefault();
}
catch
{
}
if (existingObj == null)
{
this.AddEntity(obj);
}
else
{
TableServiceContext context = this.CreateContext();
context.AttachTo(this.tableName, obj, "*");
context.UpdateObject(obj);
context.SaveChanges(SaveChangesOptions.ReplaceOnUpdate);
}
}
}
public void AddOrUpdateEntity <T>(T obj, string tableName) where T : ITableServiceEntity
{
var pk = obj.PartitionKey;
var rk = obj.RowKey;
T existingObj = default(T);
try
{
existingObj = (from o in this.Query <T>(tableName)
where o.PartitionKey == pk && o.RowKey == rk
select o).SingleOrDefault();
}
catch
{
}
if (existingObj == null)
{
this.AddEntity(obj, tableName);
}
else
{
TableServiceContext context = this.CreateContext <T>();
context.AttachTo(tableName, obj, "*");
context.UpdateObject(obj);
context.SaveChanges(SaveChangesOptions.ReplaceOnUpdate);
}
}
public bool SaveBtsAssemblyFilesMetadata(BtsAssemblyFilesMetadata btsAssemblyFilesMetadata)
{
this.tableClient = account.CreateCloudTableClient();
this.tableClient.CreateTableIfNotExist(MapFilesTableName);
this.tableContext = tableClient.GetDataServiceContext();
btsAssemblyFilesMetadata.PartitionKey = btsAssemblyFilesMetadata.FileName;
btsAssemblyFilesMetadata.RowKey = btsAssemblyFilesMetadata.FileName;
btsAssemblyFilesMetadata.Timestamp = DateTime.UtcNow;
if (MapFiles.FirstOrDefault(t => t.PartitionKey == btsAssemblyFilesMetadata.PartitionKey && t.RowKey == btsAssemblyFilesMetadata.RowKey) == null)
{
MapFiles.Add(btsAssemblyFilesMetadata);
}
// We need upsert functionality here, hence removing AddObject call and adding UpdateObject
// this.tableContext.AddObject(MapFilesTableName, tradingPartnerSpecCert);
// http://social.msdn.microsoft.com/Forums/windowsazure/en-US/892340f1-bfe1-4433-9246-b617abe6078c/upsert-operation-in-the-table
// http://msdn.microsoft.com/en-us/library/windowsazure/hh452242.aspx
// http://www.windowsazure.com/en-us/develop/net/how-to-guides/table-services/#replace-entity
tableContext.AttachTo(MapFilesTableName, btsAssemblyFilesMetadata);
tableContext.UpdateObject(btsAssemblyFilesMetadata);
DataServiceResponse response = this.tableContext.SaveChangesWithRetries(SaveChangesOptions.Batch | SaveChangesOptions.ReplaceOnUpdate);
return(response.BatchStatusCode == Http200 || response.BatchStatusCode == Http201 || response.BatchStatusCode == Http202);
}
/// <summary>
/// Update a new item in a table
/// </summary>
/// <param name="tableName">Precisely table's name</param>
/// <param name="entity">The item which need to be updated</param>
public void Update(string tableName, TElement entity)
{
TableServiceContext.Detach(entity);
TableServiceContext.AttachTo(tableName, entity, "*");
TableServiceContext.UpdateObject(entity);
TableServiceContext.SaveChanges(SaveChangesOptions.ReplaceOnUpdate);
}
private void MovePageRecursively(Models.Site site, string pageFullName, string newParent, TableServiceContext serviceContext)
{
var oldPage = Get(new Page(site, pageFullName));
var entity = PageEntityHelper.ToPageEntity(oldPage);
if (!string.IsNullOrEmpty(newParent))
{
var newPage = new Page(new Page(site, newParent), oldPage.Name);
entity.FullName = newPage.FullName;
entity.ParentPage = newPage.Parent.FullName;
}
else
{
entity.FullName = oldPage.Name;
entity.ParentPage = "";
}
foreach (var item in ChildPages(oldPage))
{
MovePageRecursively(site, item.FullName, entity.FullName, serviceContext);
}
serviceContext.AddObject(PageTable, entity);
var oldEntity = PageEntityHelper.ToPageEntity(oldPage);
serviceContext.AttachTo(PageTable, oldEntity, "*");
serviceContext.DeleteObject(oldEntity);
}
public void DeleteEntity(IEnumerable <T> objs)
{
TableServiceContext context = this.CreateContext();
foreach (var obj in objs)
{
context.AttachTo(this.tableName, obj, "*");
context.DeleteObject(obj);
}
try
{
context.SaveChanges();
}
catch (DataServiceRequestException ex)
{
var dataServiceClientException = ex.InnerException as DataServiceClientException;
if (dataServiceClientException != null)
{
if (dataServiceClientException.StatusCode == 404)
{
return;
}
}
throw;
}
}
public void Delete(IEnumerable <T> objs)
{
TableServiceContext context = this.CreateContext();
foreach (var obj in objs)
{
context.AttachTo(this.tableName, obj, "*");
context.DeleteObject(obj);
}
this.StorageRetryPolicy.ExecuteAction(() =>
{
try
{
context.SaveChanges();
}
catch (DataServiceRequestException ex)
{
var dataServiceClientException = ex.InnerException as DataServiceClientException;
if (dataServiceClientException != null)
{
if (dataServiceClientException.StatusCode == 404)
{
TraceHelper.TraceWarning(ex.TraceInformation());
return;
}
}
TraceHelper.TraceError(ex.TraceInformation());
throw;
}
});
}
private async Task Update(T entity)
{
TableServiceContext context = this.CreateContext();
context.AttachTo(this._tableName, entity, "*");
context.UpdateObject(entity);
await this.SaveChangesAsync(context, SaveChangesOptions.ReplaceOnUpdate);
}
internal async Task <string> UpdateTableEntryConditionallyAsync(T data, string dataEtag, T tableVersion, string tableVersionEtag)
{
const string operation = "UpdateTableEntryConditionally";
string tableVersionData = (tableVersion == null ? "null" : tableVersion.ToString());
var startTime = DateTime.UtcNow;
if (Logger.IsVerbose2)
{
Logger.Verbose2("{0} table {1} version {2} entry {3}", operation, TableName, tableVersionData, data);
}
try
{
TableServiceContext svc = tableOperationsClient.GetDataServiceContext();
svc.AttachTo(TableName, data, dataEtag);
svc.UpdateObject(data);
if (tableVersion != null && tableVersionEtag != null)
{
svc.AttachTo(TableName, tableVersion, tableVersionEtag);
svc.UpdateObject(tableVersion);
}
try
{
await Task <DataServiceResponse> .Factory.FromAsync(
svc.BeginSaveChangesWithRetries,
svc.EndSaveChangesWithRetries,
SaveChangesOptions.ReplaceOnUpdate | SaveChangesOptions.Batch,
null);
EntityDescriptor dataResult = svc.GetEntityDescriptor(data);
return(dataResult.ETag);
}
catch (Exception exc)
{
CheckAlertWriteError(operation, data, tableVersionData, exc);
throw;
}
}
finally
{
CheckAlertSlowAccess(startTime, operation);
}
}
public static void UnregisterRoute(string routeName)
{
TableServiceContext tableContext = GetTableContext();
RouteEntity entity = new RouteEntity(routeName, null);
tableContext.AttachTo(_tableName, entity, "*");
tableContext.DeleteObject(entity);
tableContext.SaveChanges();
_configCache.Remove(_partitionKey);
}
public static void RegisterRoute(string routeName, RoutingInfo routingInfo)
{
TableServiceContext tableContext = GetTableContext();
RouteEntity entity = new RouteEntity(routeName, routingInfo);
//this performs an Upsert (InsertOrRepalce)
tableContext.AttachTo(_tableName, entity, null);
tableContext.UpdateObject(entity);
tableContext.SaveChanges();
_configCache.Remove(_partitionKey);
}
public async Task Delete(IEnumerable <T> entities)
{
TableServiceContext context = this.CreateContext();
foreach (var entity in entities)
{
context.AttachTo(this._tableName, entity, "*");
context.DeleteObject(entity);
}
await this.SaveChangesAsync(context, context.SaveChangesDefaultOptions);
}
public static void DeleteEntity(string account, string key, string table, string partitionKey, string rowKey)
{
CloudTableClient tableClient = Client.GetTableClient(account, key);
TableServiceContext context = tableClient.GetDataServiceContext();
TableEntity entity = new TableEntity()
{
PartitionKey = partitionKey, RowKey = rowKey
};
context.AttachTo(table, entity, "*");
context.DeleteObject(entity);
context.SaveChangesWithRetries(SaveChangesOptions.None);
}
public bool DeleteBtsAssemblyFilesMetadata(BtsAssemblyFilesMetadata btsAssemblyFilesMetadata)
{
this.tableClient = account.CreateCloudTableClient();
this.tableClient.CreateTableIfNotExist(MapFilesTableName);
this.tableContext = tableClient.GetDataServiceContext();
btsAssemblyFilesMetadata.PartitionKey = btsAssemblyFilesMetadata.FileName;
btsAssemblyFilesMetadata.RowKey = btsAssemblyFilesMetadata.FileName;
btsAssemblyFilesMetadata.Timestamp = DateTime.UtcNow;
MapFiles.Remove(btsAssemblyFilesMetadata);
tableContext.AttachTo(MapFilesTableName, btsAssemblyFilesMetadata, "*");
tableContext.DeleteObject(btsAssemblyFilesMetadata);
DataServiceResponse response = this.tableContext.SaveChangesWithRetries(SaveChangesOptions.Batch | SaveChangesOptions.ReplaceOnUpdate);
return(response.BatchStatusCode == Http200 || response.BatchStatusCode == Http201 || response.BatchStatusCode == Http202);
}
internal async Task <string> InsertTableEntryConditionallyAsync(T data, T tableVersion, string tableVersionEtag, bool updateTableVersion = true)
{
const string operation = "InsertTableEntryConditionally";
string tableVersionData = (tableVersion == null ? "null" : tableVersion.ToString());
var startTime = DateTime.UtcNow;
if (Logger.IsVerbose2)
{
Logger.Verbose2("{0} into table {1} version {2} entry {3}", operation, TableName, tableVersionData, data);
}
try
{
TableServiceContext svc = tableOperationsClient.GetDataServiceContext();
// Only AddObject, do NOT AttachTo. If we did both UpdateObject and AttachTo, it would have been equivalent to InsertOrReplace.
svc.AddObject(TableName, data);
if (updateTableVersion)
{
svc.AttachTo(TableName, tableVersion, tableVersionEtag);
svc.UpdateObject(tableVersion);
}
try
{
await Task <DataServiceResponse> .Factory.FromAsync(
svc.BeginSaveChangesWithRetries,
svc.EndSaveChangesWithRetries,
SaveChangesOptions.ReplaceOnUpdate | SaveChangesOptions.Batch,
null);
EntityDescriptor dataResult = svc.GetEntityDescriptor(data);
return(dataResult.ETag);
}
catch (Exception exc)
{
CheckAlertWriteError(operation, data, tableVersionData, exc);
throw;
}
}
finally
{
CheckAlertSlowAccess(startTime, operation);
}
}
public bool DeleteTradingPartnerSpecCertMetadata(TradingPartnerSpecCertMetadata tradingPartnerSpecCert)
{
this.tableClient = account.CreateCloudTableClient();
this.tableClient.CreateTableIfNotExist(TradingPartnerSpecCertTableName);
this.tableContext = tableClient.GetDataServiceContext();
tradingPartnerSpecCert.PartitionKey = tradingPartnerSpecCert.TradingPartnerName;
tradingPartnerSpecCert.RowKey = string.Format("{0}_{1}", tradingPartnerSpecCert.DocumentType, tradingPartnerSpecCert.Direction);
tradingPartnerSpecCert.Timestamp = DateTime.UtcNow;
TradingPartnerSpecCerts.Remove(tradingPartnerSpecCert);
tableContext.AttachTo(TradingPartnerSpecCertTableName, tradingPartnerSpecCert, "*");
tableContext.DeleteObject(tradingPartnerSpecCert);
DataServiceResponse response = this.tableContext.SaveChangesWithRetries(SaveChangesOptions.Batch | SaveChangesOptions.ReplaceOnUpdate);
return(response.BatchStatusCode == Http200 || response.BatchStatusCode == Http201 || response.BatchStatusCode == Http202);
}
public bool DeleteBizRuleCertMetadata(BizRuleCertMetadata bizRuleCertMetadata)
{
this.tableClient = account.CreateCloudTableClient();
this.tableClient.CreateTableIfNotExist(BizRuleCertTableName);
this.tableContext = tableClient.GetDataServiceContext();
bizRuleCertMetadata.PartitionKey = bizRuleCertMetadata.TradingPartnerName;
bizRuleCertMetadata.RowKey = bizRuleCertMetadata.RuleCertFileName;
bizRuleCertMetadata.Timestamp = DateTime.UtcNow;
BizRuleCerts.Remove(bizRuleCertMetadata);
tableContext.AttachTo(BizRuleCertTableName, bizRuleCertMetadata, "*");
tableContext.DeleteObject(bizRuleCertMetadata);
DataServiceResponse response = this.tableContext.SaveChangesWithRetries(SaveChangesOptions.Batch | SaveChangesOptions.ReplaceOnUpdate);
return(response.BatchStatusCode == Http200 || response.BatchStatusCode == Http201 || response.BatchStatusCode == Http202);
}
public void AddOrUpdate(IEnumerable <T> objs)
{
foreach (var obj in objs)
{
T objCopy = obj;
var existingObj = (from o in this.Query
where o.PartitionKey == objCopy.PartitionKey &&
o.RowKey == objCopy.RowKey
select o).SingleOrDefault();
if (existingObj == null)
{
this.Add(obj);
}
else
{
TableServiceContext context = this.CreateContext();
context.AttachTo(this.tableName, obj, "*");
context.UpdateObject(obj);
context.SaveChanges(SaveChangesOptions.ReplaceOnUpdate);
}
}
}
/// <summary>
/// Deletes a set of already existing data entries in the table, by using eTag.
/// Fails if the data does not already exist or if eTag does not match.
/// </summary>
/// <param name="list">List of data entries and their corresponding etags to be deleted from the table.</param>
/// <returns>Completion promise for this storage operation.</returns>
internal async Task DeleteTableEntriesAsync(IReadOnlyCollection <Tuple <T, string> > list)
{
const string operation = "DeleteTableEntries";
var startTime = DateTime.UtcNow;
if (Logger.IsVerbose2)
{
Logger.Verbose2("Deleting {0} table entries: {1}", TableName, Utils.EnumerableToString(list));
}
try
{
TableServiceContext svc = tableOperationsClient.GetDataServiceContext();
foreach (var tuple in list)
{
svc.AttachTo(TableName, tuple.Item1, tuple.Item2);
svc.DeleteObject(tuple.Item1);
}
try
{
await Task <DataServiceResponse> .Factory.FromAsync(
svc.BeginSaveChangesWithRetries,
svc.EndSaveChangesWithRetries,
SaveChangesOptions.ReplaceOnUpdate | SaveChangesOptions.Batch,
null);
}
catch (Exception exc)
{
Logger.Warn(ErrorCode.AzureTable_08,
String.Format("Intermediate error deleting entries {0} from the table {1}.",
Utils.EnumerableToString(list), TableName), exc);
throw;
}
}
finally
{
CheckAlertSlowAccess(startTime, operation);
}
}
internal async Task <string> MergeTableEntryAsync(T data)
{
const string operation = "MergeTableEntry";
var startTime = DateTime.UtcNow;
if (Logger.IsVerbose2)
{
Logger.Verbose2("{0} entry {1} into table {2}", operation, data, TableName);
}
try
{
TableServiceContext svc = tableOperationsClient.GetDataServiceContext();
svc.AttachTo(TableName, data, ANY_ETAG);
svc.UpdateObject(data);
try
{
await Task <DataServiceResponse> .Factory.FromAsync(
svc.BeginSaveChangesWithRetries,
svc.EndSaveChangesWithRetries,
SaveChangesOptions.None,
null);
return(svc.GetEntityDescriptor(data).ETag);
}
catch (Exception exc)
{
Logger.Warn(ErrorCode.AzureTable_07, String.Format("Intermediate error merging entry {0} to the table {1}",
(data == null ? "null" : data.ToString()), TableName), exc);
throw;
}
}
finally
{
CheckAlertSlowAccess(startTime, operation);
}
}
private void SaveBook()
{
if (String.IsNullOrEmpty(bookTitle.Text))
{
MessageBox.Show("Please provide a book title before continuing.");
bookTitle.Focus();
return;
}
if (String.IsNullOrEmpty(bookAuthor.Text))
{
MessageBox.Show("Please provide a book author before continuing.");
bookAuthor.Focus();
return;
}
if (String.IsNullOrEmpty(bookCategory.Text))
{
MessageBox.Show("Please provide a book category before continuing.");
bookCategory.Focus();
return;
}
// Save the book...
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
TableServiceContext tableContext = tableClient.GetDataServiceContext();
var book = new Book(bookTitle.Text, bookAuthor.Text, bookCategory.Text);
tableClient.CreateTableIfNotExist("Book");
tableContext.AttachTo("Book", book);
tableContext.UpdateObject(book);
tableContext.SaveChangesWithRetries(SaveChangesOptions.ReplaceOnUpdate);
CreateNewBook();
}
/// <summary>
/// Test a table operation on entities inside and outside the given range.
/// </summary>
/// <param name="tableName">The name of the table to test.</param>
/// <param name="startPk">The start partition key range.</param>
/// <param name="startRk">The start row key range.</param>
/// <param name="endPk">The end partition key range.</param>
/// <param name="endRk">The end row key range.</param>
/// <param name="runOperationDelegate">A delegate with the table operation to test.</param>
/// <param name="opName">The name of the operation being tested.</param>
/// <param name="expectSuccess">Whether the operation should succeed on entities within the range.</param>
private void TestOperationWithRange(
string tableName,
string startPk,
string startRk,
string endPk,
string endRk,
Action <BaseEntity> runOperationDelegate,
string opName,
bool expectSuccess,
HttpStatusCode expectedStatusCode,
bool isRangeQuery)
{
CloudTableClient referenceClient = GenerateCloudTableClient();
TableServiceContext referenceContext = referenceClient.GetTableServiceContext();
string partitionKey = startPk ?? endPk ?? "M";
string rowKey = startRk ?? endRk ?? "S";
// if we expect a success for creation - avoid inserting duplicate entities
BaseEntity tableEntity = new BaseEntity(partitionKey, rowKey);
if (expectedStatusCode == HttpStatusCode.Created)
{
referenceContext.AttachTo(tableName, tableEntity, "*");
referenceContext.DeleteObject(tableEntity);
try
{
referenceContext.SaveChangesWithRetries();
}
catch (Exception)
{
}
}
else
{
// only for add we should not be adding the entity
referenceContext.AttachTo(tableName, tableEntity);
referenceContext.UpdateObject(tableEntity);
referenceContext.SaveChangesWithRetries(SaveChangesOptions.ReplaceOnUpdate);
}
if (expectSuccess)
{
runOperationDelegate(tableEntity);
}
else
{
TestHelper.ExpectedException(
() => runOperationDelegate(tableEntity),
string.Format("{0} without appropriate permission.", opName),
HttpStatusCode.NotFound);
}
if (startPk != null)
{
tableEntity.PartitionKey = "A";
if (startPk.CompareTo(tableEntity.PartitionKey) <= 0)
{
Assert.Inconclusive("Test error: partition key for this test must not be less than or equal to \"A\"");
}
TestHelper.ExpectedException(
() => runOperationDelegate(tableEntity),
string.Format("{0} before allowed partition key range", opName),
HttpStatusCode.NotFound);
tableEntity.PartitionKey = partitionKey;
}
if (endPk != null)
{
tableEntity.PartitionKey = "Z";
if (endPk.CompareTo(tableEntity.PartitionKey) >= 0)
{
Assert.Inconclusive("Test error: partition key for this test must not be greater than or equal to \"Z\"");
}
TestHelper.ExpectedException(
() => runOperationDelegate(tableEntity),
string.Format("{0} after allowed partition key range", opName),
HttpStatusCode.NotFound);
tableEntity.PartitionKey = partitionKey;
}
if (startRk != null)
{
if (isRangeQuery || startPk != null)
{
tableEntity.PartitionKey = startPk;
tableEntity.RowKey = "A";
if (startRk.CompareTo(tableEntity.RowKey) <= 0)
{
Assert.Inconclusive("Test error: row key for this test must not be less than or equal to \"A\"");
}
TestHelper.ExpectedException(
() => runOperationDelegate(tableEntity),
string.Format("{0} before allowed row key range", opName),
HttpStatusCode.NotFound);
tableEntity.RowKey = rowKey;
}
}
if (endRk != null)
{
if (isRangeQuery || endPk != null)
{
tableEntity.PartitionKey = endPk;
tableEntity.RowKey = "Z";
if (endRk.CompareTo(tableEntity.RowKey) >= 0)
{
Assert.Inconclusive("Test error: row key for this test must not be greater than or equal to \"Z\"");
}
TestHelper.ExpectedException(
() => runOperationDelegate(tableEntity),
string.Format("{0} after allowed row key range", opName),
HttpStatusCode.NotFound);
tableEntity.RowKey = rowKey;
}
}
}
public void AddOrUpdate(IEnumerable <T> objs)
{
foreach (var obj in objs)
{
var existingObj = default(T);
var added = this.StorageRetryPolicy.ExecuteAction <bool>(() =>
{
var result = false;
try
{
existingObj = (from o in this.Query
where o.PartitionKey == obj.PartitionKey &&
o.RowKey == obj.RowKey
select o).SingleOrDefault();
}
catch (DataServiceQueryException ex)
{
var dataServiceClientException = ex.InnerException as DataServiceClientException;
if (dataServiceClientException != null)
{
if (dataServiceClientException.StatusCode == 404)
{
TraceHelper.TraceWarning(ex.TraceInformation());
this.Add(obj);
result = true;
}
else
{
TraceHelper.TraceError(ex.TraceInformation());
throw;
}
}
else
{
TraceHelper.TraceError(ex.TraceInformation());
throw;
}
}
return(result);
});
if (added)
{
return;
}
if (existingObj == null)
{
this.Add(obj);
}
else
{
TableServiceContext context = this.CreateContext();
context.AttachTo(this.tableName, obj, "*");
context.UpdateObject(obj);
this.StorageRetryPolicy.ExecuteAction(() => context.SaveChanges(SaveChangesOptions.ReplaceOnUpdate));
}
}
}
internal async Task BulkInsertTableEntries(IReadOnlyCollection <T> data)
{
const string operation = "BulkInsertTableEntries";
if (data == null)
{
throw new ArgumentNullException("data");
}
if (data.Count > AzureTableDefaultPolicies.MAX_BULK_UPDATE_ROWS)
{
throw new ArgumentOutOfRangeException("data", data.Count,
"Too many rows for bulk update - max " + AzureTableDefaultPolicies.MAX_BULK_UPDATE_ROWS);
}
var startTime = DateTime.UtcNow;
if (Logger.IsVerbose2)
{
Logger.Verbose2("Bulk inserting {0} entries to {1} table", data.Count, TableName);
}
try
{
TableServiceContext svc = tableOperationsClient.GetDataServiceContext();
foreach (T entry in data)
{
svc.AttachTo(TableName, entry);
svc.UpdateObject(entry);
}
bool fallbackToInsertOneByOne = false;
try
{
// SaveChangesOptions.None == Insert-or-merge operation, SaveChangesOptions.Batch == Batch transaction
// http://msdn.microsoft.com/en-us/library/hh452241.aspx
await Task <DataServiceResponse> .Factory.FromAsync(
svc.BeginSaveChangesWithRetries,
svc.EndSaveChangesWithRetries,
SaveChangesOptions.None | SaveChangesOptions.Batch,
null);
return;
}
catch (Exception exc)
{
Logger.Warn(ErrorCode.AzureTable_37, String.Format("Intermediate error bulk inserting {0} entries in the table {1}",
data.Count, TableName), exc);
var dsre = exc.GetBaseException() as DataServiceRequestException;
if (dsre != null)
{
var dsce = dsre.GetBaseException() as DataServiceClientException;
if (dsce != null)
{
// Fallback to insert rows one by one
fallbackToInsertOneByOne = true;
}
}
if (!fallbackToInsertOneByOne)
{
throw;
}
}
// Bulk insert failed, so try to insert rows one by one instead
var promises = new List <Task>();
foreach (T entry in data)
{
promises.Add(UpsertTableEntryAsync(entry));
}
await Task.WhenAll(promises);
}
finally
{
CheckAlertSlowAccess(startTime, operation);
}
}
/// <summary>
/// Inserts a data entry in the Azure table: creates a new one if does not exists or overwrites (without eTag) an already existing version (the "update in place" semantincs).
/// </summary>
/// <param name="data">Data to be inserted or replaced in the table.</param>
/// <returns>Value promise with new Etag for this data entry after completing this storage operation.</returns>
public async Task <string> UpsertTableEntryAsync(T data)
{
const string operation = "UpsertTableEntry";
var startTime = DateTime.UtcNow;
if (Logger.IsVerbose2)
{
Logger.Verbose2("{0} entry {1} into table {2}", operation, data, TableName);
}
try
{
TableServiceContext svc = tableOperationsClient.GetDataServiceContext();
try
{
Task <DataServiceResponse> savePromise;
Func <int, Task <DataServiceResponse> > doSaveChanges = retryNum =>
{
if (retryNum > 0)
{
svc.Detach(data);
}
// Try to do update first
svc.AttachTo(TableName, data, ANY_ETAG);
svc.UpdateObject(data);
return(Task <DataServiceResponse> .Factory.FromAsync(
svc.BeginSaveChangesWithRetries,
svc.EndSaveChangesWithRetries,
SaveChangesOptions.ReplaceOnUpdate,
null));
};
if (AzureTableDefaultPolicies.MaxBusyRetries > 0)
{
IBackoffProvider backoff = new FixedBackoff(AzureTableDefaultPolicies.PauseBetweenBusyRetries);
savePromise = AsyncExecutorWithRetries.ExecuteWithRetries(
doSaveChanges,
AzureTableDefaultPolicies.MaxBusyRetries,
// Retry automatically iff we get ServerBusy reply from Azure storage
(exc, retryNum) => IsServerBusy(exc),
AzureTableDefaultPolicies.BusyRetriesTimeout,
backoff);
}
else
{
// Try single Write only once
savePromise = doSaveChanges(0);
}
await savePromise;
EntityDescriptor result = svc.GetEntityDescriptor(data);
return(result.ETag);
}
catch (Exception exc)
{
Logger.Warn(ErrorCode.AzureTable_06, String.Format("Intermediate error upserting entry {0} to the table {1}",
(data == null ? "null" : data.ToString()), TableName), exc);
throw;
}
}
finally
{
CheckAlertSlowAccess(startTime, operation);
}
}
// Write a DataTable to an AzureTable.
// DataTable's Rows are an unstructured property bag.
// columnTypes - type of the column, or null if column should be skipped. Length of columnTypes should be the same as number of columns.
public static void SaveToAzureTable(DataTable table, CloudStorageAccount account, string tableName, Type[] columnTypes, Func <int, Row, PartitionRowKey> funcComputeKeys)
{
if (table == null)
{
throw new ArgumentNullException("table");
}
if (account == null)
{
throw new ArgumentNullException("account");
}
if (columnTypes == null)
{
throw new ArgumentNullException("columnTypes");
}
if (tableName == null)
{
throw new ArgumentNullException("tableName");
}
ValidateAzureTableName(tableName);
// Azure tables have "special" columns.
// We can skip these by settings columnType[i] to null, which means don't write that column
string[] columnNames = table.ColumnNames.ToArray();
if (columnNames.Length != columnTypes.Length)
{
throw new ArgumentException(string.Format("columnTypes should have {0} elements", columnNames.Length), "columnTypes");
}
columnTypes = columnTypes.ToArray(); // create a copy for mutation.
for (int i = 0; i < columnNames.Length; i++)
{
if (IsSpecialColumnName(columnNames[i]))
{
columnTypes[i] = null;
}
}
if (funcComputeKeys == null)
{
funcComputeKeys = GetPartitionRowKeyFunc(columnNames);
}
// Validate columnTypes
string [] edmTypeNames = Array.ConvertAll(columnTypes,
columnType => {
if (columnType == null)
{
return(null);
}
string edmTypeName;
_edmNameMapping.TryGetValue(columnType, out edmTypeName);
if (edmTypeName == null)
{
// Unsupported type!
throw new InvalidOperationException(string.Format("Type '{0}' is not a supported type on azure tables", columnType.FullName));
}
return(edmTypeName);
});
CloudTableClient tableClient = account.CreateCloudTableClient();
tableClient.DeleteTableIfExist(tableName);
tableClient.CreateTableIfNotExist(tableName);
GenericTableWriter w = new GenericTableWriter
{
_edmTypeNames = edmTypeNames,
_columnNames = table.ColumnNames.ToArray()
};
// Batch rows for performance,
// but all rows in the batch must have the same partition key
TableServiceContext ctx = null;
string lastPartitionKey = null;
HashSet <PartitionRowKey> dups = new HashSet <PartitionRowKey>();
int rowCounter = 0;
int batchSize = 0;
foreach (Row row in table.Rows)
{
GenericWriterEntity entity = new GenericWriterEntity {
_source = row
};
// Compute row and partition keys too.
var partRow = funcComputeKeys(rowCounter, row);
entity.PartitionKey = partRow.PartitionKey;
entity.RowKey = partRow.RowKey;
rowCounter++;
// but all rows in the batch must have the same partition key
if ((ctx != null) && (lastPartitionKey != null) && (lastPartitionKey != entity.PartitionKey))
{
ctx.SaveChangesWithRetries(SaveChangesOptions.Batch | SaveChangesOptions.ReplaceOnUpdate);
ctx = null;
}
if (ctx == null)
{
dups.Clear();
lastPartitionKey = null;
ctx = tableClient.GetDataServiceContext();
ctx.WritingEntity += new EventHandler <ReadingWritingEntityEventArgs>(w.ctx_WritingEntity);
batchSize = 0;
}
// Add enty to the current batch
// Upsert means insert+Replace. But still need uniqueness within a batch.
bool allowUpsert = true;
// Check for dups within a batch.
var key = new PartitionRowKey {
PartitionKey = entity.PartitionKey, RowKey = entity.RowKey
};
bool dupWithinBatch = dups.Contains(key);
dups.Add(key);
if (allowUpsert)
{
// Upsert allows overwriting existing keys. But still must be unique within a batch.
if (!dupWithinBatch)
{
ctx.AttachTo(tableName, entity);
ctx.UpdateObject(entity);
}
}
else
{
// AddObject requires uniquess.
if (dupWithinBatch)
{
// Azure REST APIs will give us a horrible cryptic error (400 with no message).
// Provide users a useful error instead.
throw new InvalidOperationException(string.Format("Table has duplicate keys: {0}", key));
}
ctx.AddObject(tableName, entity);
}
lastPartitionKey = entity.PartitionKey;
batchSize++;
if (batchSize % UploadBatchSize == 0)
{
// Beware, if keys collide within a batch, we get a very cryptic error and 400.
// If they collide across batches, we get a more useful 409 (conflict).
try
{
ctx.SaveChangesWithRetries(SaveChangesOptions.Batch | SaveChangesOptions.ReplaceOnUpdate);
}
catch (DataServiceRequestException de)
{
var e = de.InnerException as DataServiceClientException;
if (e != null)
{
if (e.StatusCode == 409)
{
// Conflict. Duplicate keys. We don't get the specific duplicate key.
// Server shouldn't do this if we support upsert.
// (although an old emulator that doesn't yet support upsert may throw it).
throw new InvalidOperationException(string.Format("Table has duplicate keys. {0}", e.Message));
}
}
}
ctx = null;
}
}
if (ctx != null)
{
ctx.SaveChangesWithRetries(SaveChangesOptions.Batch | SaveChangesOptions.ReplaceOnUpdate);
}
}