public async Task BlobOpenWriteTestAsync()
{
byte[] buffer = GetRandomBuffer(2 * 1024);
CloudBlobContainer container = GetRandomContainerReference();
try
{
await container.CreateAsync();
CloudPageBlob blob = container.GetPageBlobReference("blob1");
using (CloudBlobStream blobStream = await blob.OpenWriteAsync(2048))
{
Stream blobStreamForWrite = blobStream;
await blobStreamForWrite.WriteAsync(buffer, 0, 2048);
await blobStreamForWrite.FlushAsync();
byte[] testBuffer = new byte[2048];
MemoryStream dstStream = new MemoryStream(testBuffer);
await blob.DownloadRangeToStreamAsync(dstStream, null, null);
MemoryStream memStream = new MemoryStream(buffer);
TestHelper.AssertStreamsAreEqual(memStream, dstStream);
}
}
finally
{
container.DeleteIfExistsAsync().Wait();
}
}
public async Task BlobReadWhenOpenWriteAsync()
{
byte[] buffer = GetRandomBuffer(2 * 1024);
bool thrown = false;
CloudBlobContainer container = GetRandomContainerReference();
try
{
await container.CreateAsync();
CloudPageBlob blob = container.GetPageBlobReference("blob1");
MemoryStream memoryStream = new MemoryStream(buffer);
using (CloudBlobStream blobStream = await blob.OpenWriteAsync(2048))
{
Stream blobStreamForWrite = blobStream;
await blobStreamForWrite.WriteAsync(buffer, 0, 2048);
byte[] testBuffer = new byte[2048];
try
{
await blobStreamForWrite.ReadAsync(testBuffer, 0, 2048);
}
catch (NotSupportedException)
{
thrown = true;
}
Assert.IsTrue(thrown);
}
}
finally
{
container.DeleteIfExistsAsync().Wait();
}
}
public void BlobOpenWriteTest()
{
byte[] buffer = GetRandomBuffer(2 * 1024);
CloudBlobContainer container = GetRandomContainerReference();
try
{
container.Create();
CloudPageBlob blob = container.GetPageBlobReference("blob1");
using (CloudBlobStream blobStream = blob.OpenWrite(2048))
{
blobStream.Write(buffer, 0, 2048);
blobStream.Flush();
byte[] testBuffer = new byte[2048];
MemoryStream dstStream = new MemoryStream(testBuffer);
blob.DownloadRangeToStream(dstStream, null, null);
MemoryStream memStream = new MemoryStream(buffer);
TestHelper.AssertStreamsAreEqual(memStream, dstStream);
}
}
finally
{
container.DeleteIfExists();
}
}
public async Task BlobOpenReadWriteTestAsync()
{
byte[] buffer = GetRandomBuffer(2 * 1024);
CloudBlobContainer container = GetRandomContainerReference();
try
{
await container.CreateAsync();
CloudPageBlob blob = container.GetPageBlobReference("blob1");
using (CloudBlobStream blobStream = await blob.OpenWriteAsync(2048))
{
Stream blobStreamForWrite = blobStream;
await blobStreamForWrite.WriteAsync(buffer, 0, 2048);
await blobStreamForWrite.FlushAsync();
}
using (Stream dstStream = await blob.OpenReadAsync())
{
Stream dstStreamForRead = dstStream;
MemoryStream memoryStream = new MemoryStream(buffer);
TestHelper.AssertStreamsAreEqual(memoryStream, dstStreamForRead);
}
}
finally
{
await container.DeleteAsync();
}
}
public async Task BlobOpenWriteSeekReadTestAsync()
{
byte[] buffer = GetRandomBuffer(2 * 1024);
CloudBlobContainer container = GetRandomContainerReference();
try
{
await container.CreateAsync();
CloudPageBlob blob = container.GetPageBlobReference("blob1");
MemoryStream memoryStream = new MemoryStream(buffer);
using (CloudBlobStream blobStream = await blob.OpenWriteAsync(2048))
{
Stream blobStreamForWrite = blobStream;
await blobStreamForWrite.WriteAsync(buffer, 0, 2048);
Assert.AreEqual(blobStreamForWrite.Position, 2048);
blobStreamForWrite.Seek(1024, 0);
memoryStream.Seek(1024, 0);
Assert.AreEqual(blobStreamForWrite.Position, 1024);
byte[] testBuffer = GetRandomBuffer(1024);
await memoryStream.WriteAsync(testBuffer, 0, 1024);
await blobStreamForWrite.WriteAsync(testBuffer, 0, 1024);
Assert.AreEqual(blobStreamForWrite.Position, memoryStream.Position);
await blobStreamForWrite.FlushAsync();
}
using (Stream dstStream = await blob.OpenReadAsync())
{
Stream dstStreamForRead = dstStream;
TestHelper.AssertStreamsAreEqual(memoryStream, dstStreamForRead);
}
}
finally
{
container.DeleteIfExistsAsync().Wait();
}
}
/// <summary>
/// Restore file from blob storage by BackupAzureTables to the destination table name specified.
/// File will be read directly from blob storage. If the file is compressed, it will be decompressed to blob storage and then read.
/// </summary>
/// <param name="DestinationTableName">Name of the Azure Table to restore to - may be different than name backed up originally.</param>
/// <param name="OriginalTableName">Name of the Azure Table originally backed (required for determining blob directory to use)</param>
/// <param name="BlobRoot">Name to use as blob root folder.</param>
/// <param name="BlobFileName">Name of the blob file to restore.</param>
/// <param name="TimeoutSeconds">Set timeout for table client.</param>
/// <returns>A string indicating the table restored and record count.</returns>
public string RestoreTableFromBlobDirect(string DestinationTableName, string OriginalTableName, string BlobRoot, string BlobFileName, int TimeoutSeconds = 30)
{
if (String.IsNullOrWhiteSpace(DestinationTableName))
{
throw new ParameterSpecException("DestinationTableName is missing.");
}
if (String.IsNullOrWhiteSpace(OriginalTableName))
{
throw new ParameterSpecException("OriginalTableName is missing.");
}
if (String.IsNullOrWhiteSpace(BlobFileName))
{
throw new ParameterSpecException(String.Format("Invalid BlobFileName '{0}' specified.", BlobFileName));
}
bool Decompress = BlobFileName.EndsWith(".7z");
string TempFileName = String.Format("{0}.temp", BlobFileName);
if (String.IsNullOrWhiteSpace(BlobRoot))
{
throw new ParameterSpecException(String.Format("Invalid BlobRoot '{0}' specified.", BlobRoot));
}
try
{
if (!CosmosTable.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureTableConnectionSpec).Password, out CosmosTable.CloudStorageAccount StorageAccount))
{
throw new ConnectionException("Can not connect to CloudStorage Account. Verify connection string.");
}
if (!AZStorage.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureBlobConnectionSpec).Password, out AZStorage.CloudStorageAccount StorageAccountAZ))
{
throw new ConnectionException("Can not connect to CloudStorage Account. Verify connection string.");
}
AZBlob.CloudBlobClient ClientBlob = AZBlob.BlobAccountExtensions.CreateCloudBlobClient(StorageAccountAZ);
var container = ClientBlob.GetContainerReference(BlobRoot);
container.CreateIfNotExists();
AZBlob.CloudBlobDirectory directory = container.GetDirectoryReference(BlobRoot.ToLower() + "-table-" + OriginalTableName.ToLower());
// If file is compressed, Decompress to a temp file in the blob
if (Decompress)
{
AZBlob.CloudBlockBlob BlobBlockTemp = directory.GetBlockBlobReference(TempFileName);
AZBlob.CloudBlockBlob BlobBlockRead = directory.GetBlockBlobReference(BlobFileName);
using (AZBlob.CloudBlobStream decompressedStream = BlobBlockTemp.OpenWrite())
{
using (Stream readstream = BlobBlockRead.OpenRead())
{
using (var zip = new GZipStream(readstream, CompressionMode.Decompress, true))
{
zip.CopyTo(decompressedStream);
}
}
}
BlobFileName = TempFileName;
}
AZBlob.CloudBlockBlob BlobBlock = directory.GetBlockBlobReference(BlobFileName);
CosmosTable.CloudTableClient client = CosmosTable.CloudStorageAccountExtensions.CreateCloudTableClient(StorageAccount, new CosmosTable.TableClientConfiguration());
CosmosTable.CloudTable TableDest = client.GetTableReference(DestinationTableName);
TableDest.ServiceClient.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, 0, TimeoutSeconds);
TableDest.CreateIfNotExists();
using (Stream BlobStream = BlobBlock.OpenRead())
{
using (StreamReader InputFileStream = new StreamReader(BlobStream))
{
string result = RestoreFromStream(InputFileStream, TableDest, DestinationTableName);
if (Decompress)
{
AZBlob.CloudBlockBlob BlobBlockTemp = directory.GetBlockBlobReference(TempFileName);
BlobBlockTemp.DeleteIfExists();
}
return(result);
}
}
}
catch (ConnectionException cex)
{
throw cex;
}
catch (RestoreFailedException rex)
{
throw rex;
}
catch (Exception ex)
{
throw new RestoreFailedException(String.Format("Table '{0}' restore failed.", DestinationTableName), ex);
}
finally
{
}
} // RestoreTableFromBlobDirect
public async Task CloudBlobClientMaximumExecutionTimeoutShouldNotBeHonoredForStreamsAsync()
{
CloudBlobClient blobClient = GenerateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(Guid.NewGuid().ToString("N"));
byte[] buffer = BlobTestBase.GetRandomBuffer(1024 * 1024);
try
{
await container.CreateAsync();
blobClient.DefaultRequestOptions.MaximumExecutionTime = TimeSpan.FromSeconds(30);
CloudBlockBlob blockBlob = container.GetBlockBlobReference("blob1");
CloudPageBlob pageBlob = container.GetPageBlobReference("blob2");
blockBlob.StreamWriteSizeInBytes = 1024 * 1024;
blockBlob.StreamMinimumReadSizeInBytes = 1024 * 1024;
pageBlob.StreamWriteSizeInBytes = 1024 * 1024;
pageBlob.StreamMinimumReadSizeInBytes = 1024 * 1024;
using (CloudBlobStream bos = await blockBlob.OpenWriteAsync())
{
DateTime start = DateTime.Now;
for (int i = 0; i < 7; i++)
{
await bos.WriteAsync(buffer, 0, buffer.Length);
}
// Sleep to ensure we are over the Max execution time when we do the last write
int msRemaining = (int)(blobClient.DefaultRequestOptions.MaximumExecutionTime.Value - (DateTime.Now - start)).TotalMilliseconds;
if (msRemaining > 0)
{
await Task.Delay(msRemaining);
}
await bos.WriteAsync(buffer, 0, buffer.Length);
await bos.CommitAsync();
}
using (Stream bis = (await blockBlob.OpenReadAsync()))
{
DateTime start = DateTime.Now;
int total = 0;
while (total < 7 * 1024 * 1024)
{
total += await bis.ReadAsync(buffer, 0, buffer.Length);
}
// Sleep to ensure we are over the Max execution time when we do the last read
int msRemaining = (int)(blobClient.DefaultRequestOptions.MaximumExecutionTime.Value - (DateTime.Now - start)).TotalMilliseconds;
if (msRemaining > 0)
{
await Task.Delay(msRemaining);
}
while (true)
{
int count = await bis.ReadAsync(buffer, 0, buffer.Length);
total += count;
if (count == 0)
{
break;
}
}
}
using (CloudBlobStream bos = await pageBlob.OpenWriteAsync(8 * 1024 * 1024))
{
DateTime start = DateTime.Now;
for (int i = 0; i < 7; i++)
{
await bos.WriteAsync(buffer, 0, buffer.Length);
}
// Sleep to ensure we are over the Max execution time when we do the last write
int msRemaining = (int)(blobClient.DefaultRequestOptions.MaximumExecutionTime.Value - (DateTime.Now - start)).TotalMilliseconds;
if (msRemaining > 0)
{
await Task.Delay(msRemaining);
}
await bos.WriteAsync(buffer, 0, buffer.Length);
await bos.CommitAsync();
}
using (Stream bis = (await pageBlob.OpenReadAsync()))
{
DateTime start = DateTime.Now;
int total = 0;
while (total < 7 * 1024 * 1024)
{
total += await bis.ReadAsync(buffer, 0, buffer.Length);
}
// Sleep to ensure we are over the Max execution time when we do the last read
int msRemaining = (int)(blobClient.DefaultRequestOptions.MaximumExecutionTime.Value - (DateTime.Now - start)).TotalMilliseconds;
if (msRemaining > 0)
{
await Task.Delay(msRemaining);
}
while (true)
{
int count = await bis.ReadAsync(buffer, 0, buffer.Length);
total += count;
if (count == 0)
{
break;
}
}
}
}
finally
{
blobClient.DefaultRequestOptions.MaximumExecutionTime = null;
container.DeleteIfExistsAsync().Wait();
}
}
/// <summary>
/// Backup table directly to Blob.
///
/// </summary>
/// <param name="TableName">Name of Azure Table to backup.</param>
/// <param name="BlobRoot">Name to use as blob root folder.</param>
/// <param name="Compress">True to compress the file.</param>
/// <param name="RetentionDays">Process will age files in blob created more than x days ago.</param>
/// <param name="TimeoutSeconds">Set timeout for table client.</param>
/// <param name="filters">A list of Filter objects to be applied to table values extracted.</param>
/// <returns>A string containing the name of the file created.</returns>
public string BackupTableToBlobDirect(string TableName, string BlobRoot, bool Compress = false, int RetentionDays = 30, int TimeoutSeconds = 30, List <Filter> filters = default(List <Filter>))
{
string OutFileName = "";
int RecordCount = 0;
int BackupsAged = 0;
if (String.IsNullOrWhiteSpace(TableName))
{
throw new ParameterSpecException("TableName is missing.");
}
if (String.IsNullOrWhiteSpace(BlobRoot))
{
throw new ParameterSpecException("BlobRoot is missing.");
}
try
{
if (Compress)
{
OutFileName = String.Format(TableName + "_Backup_" + System.DateTime.Now.ToString("yyyyMMddHHmmss") + ".txt.7z");
}
else
{
OutFileName = String.Format(TableName + "_Backup_" + System.DateTime.Now.ToString("yyyyMMddHHmmss") + ".txt");
}
if (!CosmosTable.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureTableConnectionSpec).Password, out CosmosTable.CloudStorageAccount StorageAccount))
{
throw new ConnectionException("Can not connect to CloudStorage Account. Verify connection string.");
}
if (!AZStorage.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureBlobConnectionSpec).Password, out AZStorage.CloudStorageAccount StorageAccountAZ))
{
throw new ConnectionException("Can not connect to CloudStorage Account. Verify connection string.");
}
AZBlob.CloudBlobClient ClientBlob = AZBlob.BlobAccountExtensions.CreateCloudBlobClient(StorageAccountAZ);
var container = ClientBlob.GetContainerReference(BlobRoot);
container.CreateIfNotExists();
AZBlob.CloudBlobDirectory directory = container.GetDirectoryReference(BlobRoot.ToLower() + "-table-" + TableName.ToLower());
AZBlob.CloudBlockBlob BlobBlock = directory.GetBlockBlobReference(OutFileName);
// start upload from stream, iterate through table, possible inline compress
try
{
CosmosTable.CloudTableClient client = CosmosTable.CloudStorageAccountExtensions.CreateCloudTableClient(StorageAccount, new CosmosTable.TableClientConfiguration());
CosmosTable.CloudTable table = client.GetTableReference(TableName);
table.ServiceClient.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, 0, TimeoutSeconds);
CosmosTable.TableContinuationToken token = null;
var entities = new List <CosmosTable.DynamicTableEntity>();
var entitiesSerialized = new List <string>();
DynamicTableEntityJsonSerializer serializer = new DynamicTableEntityJsonSerializer();
TableSpec TableSpecStart = new TableSpec(TableName);
var NewLineAsBytes = Encoding.UTF8.GetBytes("\n");
var tempTableSpecStart = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(TableSpecStart));
AZBlob.CloudBlobStream bs2 = BlobBlock.OpenWrite();
Stream bs = BlobBlock.OpenWrite();
if (Compress)
{
bs = new GZipStream(bs2, CompressionMode.Compress);
}
else
{
bs = bs2;
}
bs.Write(tempTableSpecStart, 0, tempTableSpecStart.Length);
bs.Flush();
bs.Write(NewLineAsBytes, 0, NewLineAsBytes.Length);
bs.Flush();
CosmosTable.TableQuery <CosmosTable.DynamicTableEntity> tq;
if (default(List <Filter>) == filters)
{
tq = new CosmosTable.TableQuery <CosmosTable.DynamicTableEntity>();
}
else
{
tq = new CosmosTable.TableQuery <CosmosTable.DynamicTableEntity>().Where(Filter.BuildFilterSpec(filters));
}
do
{
var queryResult = table.ExecuteQuerySegmented(tq, token);
foreach (CosmosTable.DynamicTableEntity dte in queryResult.Results)
{
var tempDTE = Encoding.UTF8.GetBytes(serializer.Serialize(dte));
bs.Write(tempDTE, 0, tempDTE.Length);
bs.Flush();
bs.Write(NewLineAsBytes, 0, NewLineAsBytes.Length);
bs.Flush();
RecordCount++;
}
token = queryResult.ContinuationToken;
} while (token != null);
TableSpec TableSpecEnd = new TableSpec(TableName, RecordCount);
var tempTableSpecEnd = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(TableSpecEnd));
bs.Write(tempTableSpecEnd, 0, tempTableSpecEnd.Length);
bs.Flush();
bs.Write(NewLineAsBytes, 0, NewLineAsBytes.Length);
bs.Flush();
bs.Close();
}
catch (Exception ex)
{
throw new BackupFailedException(String.Format("Table '{0}' backup failed.", TableName), ex);
}
DateTimeOffset OffsetTimeNow = System.DateTimeOffset.Now;
DateTimeOffset OffsetTimeRetain = System.DateTimeOffset.Now.AddDays(-1 * RetentionDays);
//Cleanup old versions
var BlobList = directory.ListBlobs().OfType <AZBlob.CloudBlockBlob>().ToList();;
foreach (var blob in BlobList)
{
if (blob.Properties.Created < OffsetTimeRetain)
{
try
{
blob.Delete();
BackupsAged++;
}
catch (Exception ex)
{
throw new AgingException(String.Format("Error aging file '{0}'.", blob.Name), ex);
}
}
}
return(String.Format("Table '{0}' backed up as '{2}' under blob '{3}\\{4}'; {1} files aged.", TableName, BackupsAged, OutFileName, BlobRoot, directory.ToString()));
}
catch (ConnectionException cex)
{
throw cex;
}
catch (Exception ex)
{
throw new BackupFailedException(String.Format("Table '{0}' backup failed.", TableName), ex);
}
finally
{
}
} // BackupTableToBlobDirect
