/// <summary>
/// The main entry point.
/// </summary>
/// <param name="args">
/// The command line arguments. Not used.
/// </param>
private static void Main(string[] args)
{
var credentials = new StorageCredentials("rwwa", "vyMfaSPxURHXZaIhhFJQRg5ZLEN6qDj4yU78r3oeOH+pZzdcf4S86QvGAsB6L8JaPti9qJbB929hy1Y9hipFmw==");
// Retrieve storage account from connection string.
var storageAccount = new CloudStorageAccount(credentials, true);
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
// Retrieve a reference to a queue
CloudQueue queue = queueClient.GetQueueReference("dataqueue");
queue.CreateIfNotExists();
Console.WriteLine("Up and listening to the queue.");
while (true)
{
CloudQueueMessage message;
do
{
message = queue.GetMessage();
if (message != null)
{
Console.WriteLine("Received Message for BLOB " + message.AsString);
var blobUrl = message.AsString;
var blockBlob = new CloudBlockBlob(new Uri(blobUrl), credentials);
if (blockBlob.Exists())
{
// TODO: download and process BLOB
/*using (var fileStream = System.IO.File.OpenWrite(@"path\myfile"))
{
blockBlob.DownloadToStream(fileStream);
}*/
blockBlob.Delete();
}
queue.DeleteMessage(message);
Console.WriteLine("BLOB " + message.AsString + " and queue message deleted.");
}
}
while (message != null);
Thread.Sleep(TimeSpan.FromSeconds(5));
}
}
/// <summary>
/// The main entry point.
/// </summary>
/// <param name="args">
/// The command line arguments. Not used.
/// </param>
private static void Main(string[] args)
{
var credentials = new StorageCredentials("[Enter your account name]", "[Enter your account key]");
// Retrieve storage account from connection string.
var storageAccount = new CloudStorageAccount(credentials, true);
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
// Retrieve a reference to a queue
CloudQueue queue = queueClient.GetQueueReference("dataqueue");
Console.WriteLine("Up and listening to the queue.");
while (true)
{
CloudQueueMessage message;
do
{
message = queue.GetMessage();
if (message != null)
{
Console.WriteLine("Received Message for BLOB " + message.AsString);
var blobUrl = message.AsString;
var blockBlob = new CloudBlockBlob(new Uri(blobUrl), credentials);
if (blockBlob.Exists())
{
// TODO: download and process BLOB
/*using (var fileStream = System.IO.File.OpenWrite(@"path\myfile"))
{
blockBlob.DownloadToStream(fileStream);
}*/
blockBlob.Delete();
}
queue.DeleteMessage(message);
Console.WriteLine("BLOB " + message.AsString + " and queue message deleted.");
}
}
while (message != null);
Thread.Sleep(TimeSpan.FromSeconds(5));
}
}
public static async Task WaitForBlobAsync(CloudBlockBlob blob)
{
await TestHelpers.Await(() =>
{
return blob.Exists();
});
}
/// <summary>
/// Creates an archived copy of the original package blob if it doesn't already exist.
/// </summary>
private void ArchiveOriginalPackageBlob(CloudBlockBlob originalPackageBlob, CloudBlockBlob latestPackageBlob)
{
// Copy the blob to backup only if it isn't already successfully copied
if ((!originalPackageBlob.Exists()) || (originalPackageBlob.CopyState != null && originalPackageBlob.CopyState.Status != CopyStatus.Success))
{
if (!WhatIf)
{
Log.Info("Backing up blob: {0} to {1}", latestPackageBlob.Name, originalPackageBlob.Name);
originalPackageBlob.StartCopyFromBlob(latestPackageBlob);
CopyState state = originalPackageBlob.CopyState;
for (int i = 0; (state == null || state.Status == CopyStatus.Pending) && i < SleepTimes.Length; i++)
{
Log.Info("(sleeping for a copy completion)");
Thread.Sleep(SleepTimes[i]);
originalPackageBlob.FetchAttributes(); // To get a refreshed CopyState
//refresh state
state = originalPackageBlob.CopyState;
}
if (state.Status != CopyStatus.Success)
{
string msg = string.Format("Blob copy failed: CopyState={0}", state.StatusDescription);
Log.Error("(error) " + msg);
throw new BlobBackupFailedException(msg);
}
}
}
}
//Delete a blob file
public void DeleteBlob(string UserId, string BlobName)
{
Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer container = GetCloudBlobContainer(UserId);
Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob blockBlob = container.GetBlockBlobReference(BlobName);
if (blockBlob.Exists())
{
blockBlob.Delete();
}
}
public static async Task<string> WaitForBlobAsync(CloudBlockBlob blob)
{
await TestHelpers.Await(() =>
{
return blob.Exists();
});
string result = await blob.DownloadTextAsync(Encoding.UTF8,
null, new BlobRequestOptions(), new Microsoft.WindowsAzure.Storage.OperationContext());
return result;
}
private void DownloadIfExists(CloudBlockBlob blob)
{
string target = Path.Combine(Root, blob.Name);
if (blob.Exists())
{
if (!Directory.Exists(Root))
{
Directory.CreateDirectory(Root);
}
using (var file = File.OpenWrite(target))
{
blob.DownloadToStream(file);
}
}
}
private static string SetUpInputString(CloudBlockBlob blockBlobReference)
{
string inputStr;
if (blockBlobReference.Exists() == false)
{
inputStr = "";
} else
{
using (StreamReader blobReader = new StreamReader(blockBlobReference.OpenRead()))
{
inputStr = blobReader.ReadToEnd();
}
}
return inputStr;
}
public void Run()
{
var storageAccount = CloudStorageAccount.Parse(_storageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
var container = blobClient.GetContainerReference("mycontainer");
container.CreateIfNotExists();
_blob = container.GetBlockBlobReference("myblob.txt");
if (!_blob.Exists())
{
using (var s = new MemoryStream())
{
_blob.UploadFromStream(s);
}
}
Task.Factory.StartNew(() => GenerateContentAsync());
Task.Factory.StartNew(() => AppendStreamToBlobAsync());
Task.WaitAll();
}
private DateTime GetLastModified()
{
Microsoft.WindowsAzure.Storage.CloudStorageAccount storageAccount = Microsoft.WindowsAzure.Storage.CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("CineStorageConStr"));
Microsoft.WindowsAzure.Storage.Blob.CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve a reference to a container.
Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer container = blobClient.GetContainerReference("data");
// Retrieve reference to a blob.
Microsoft.WindowsAzure.Storage.Blob.CloudBlockBlob cinemasUKBlob = container.GetBlockBlobReference(CinemasUKFileName);
if (cinemasUKBlob.Exists())
{
DateTimeOffset?dto = cinemasUKBlob.Properties.LastModified;
if (dto.HasValue)
{
return(dto.Value.DateTime);
}
}
return(DateTime.MinValue);
}
private void ArchiveOldReport(string reportPath, string reportName, string fileExt, CloudBlockBlob report)
{
if (!report.Exists()) return;
report.FetchAttributes();
string archiveDirString = (archiveBlobPath + (
(archiveBlobPath.EndsWith("/") && reportPath.StartsWith("/")) ?
reportPath.Substring(1) : reportPath
)).ToLower();
if (!archiveDirString.EndsWith("/"))
archiveDirString = archiveDirString + "/";
CloudBlobDirectory archiveDir = container.GetDirectoryReference(archiveDirString);
//if the archive path exists, see if we need to delete the oldest blob in the path
IEnumerable<IListBlobItem> blobs = archiveDir.ListBlobs(true);
Dictionary<DateTimeOffset, string> oldReports = new Dictionary<DateTimeOffset, string>();
DateTimeOffset oldest = DateTimeOffset.MaxValue;
ICloudBlob oldestBlob = null;
int count = 0;
foreach (ICloudBlob blob in blobs)
{
++count;
if (blob.Properties.LastModified.Value < oldest)
{
oldest = blob.Properties.LastModified.Value;
oldestBlob = blob;
}
}
if (count >= maxNumOfReportsArchived)
oldestBlob.Delete();
//copy report to archive
string newUri = (archiveDirString + reportName + "_" + report.Properties.LastModified.Value.ToString("u") + fileExt).ToLower();
CloudBlockBlob newReport = container.GetBlockBlobReference(newUri);
newReport.StartCopyFromBlob(report);
int retries = 3;
while (retries > 0)
{
if (newReport.CopyState.Status == CopyStatus.Success)
{
Trace.TraceInformation("Blob archiving succeeded: " + newUri);
break;
}
else if (newReport.CopyState.Status == CopyStatus.Aborted ||
newReport.CopyState.Status == CopyStatus.Failed ||
newReport.CopyState.Status == CopyStatus.Invalid)
{
Trace.TraceError("Blob archiving failed: " + newUri);
break;
}
else
{
Thread.Sleep(1000);
--retries;
if (retries == 0) Trace.TraceError("Blob archiving timed out: " + newUri);
}
}
}
private void CreateBlob()
{
var account = CloudStorageAccount.Parse(Config.Get("DeepStorage.OutputConnectionString"));
var blobClient = account.CreateCloudBlobClient();
var container = blobClient.GetContainerReference(Config.Get("DeepStorage.OutputContainerName"));
container.CreateIfNotExists();
_blob = container.GetBlockBlobReference(_blobPath);
if (!_blob.Exists())
{
_blob.UploadFromByteArray(new byte[] {},0, 0);
}
}
/// <summary>
/// Uploads Memory Stream to Blob
/// </summary>
/// <param name="outputBlob"></param>
/// <param name="line"></param>
private static void UploadBlobStream(CloudBlockBlob outputBlob, string line)
{
using (MemoryStream ms = new MemoryStream())
{
if (outputBlob.Exists())
{
outputBlob.DownloadToStream(ms);
}
byte[] dataToWrite = Encoding.UTF8.GetBytes(line + "\r\n");
ms.Write(dataToWrite, 0, dataToWrite.Length);
ms.Position = 0;
outputBlob.UploadFromStream(ms);
}
}
/// <summary>
/// Process sample Files for generating test data
/// </summary>
/// <param name="path"></param>
/// <param name="outputStorageAccount"></param>
/// <param name="folderPath"></param>
/// <param name="outputTableName"></param>
/// <param name="logger"></param>
public static void ProcessFiles(string path, CloudStorageAccount outputStorageAccount, string folderPath, string outputTableName, IActivityLogger logger)
{
string[] files = Directory.GetFiles(path);
foreach (var file in files)
{
if (file.Contains(@"artist_data.txt") && outputTableName.ToLower().Contains(@"catalog"))
{
Uri outputBlobUri = new Uri(outputStorageAccount.BlobEndpoint, folderPath + "artist_data.txt");
CloudBlockBlob outputBlob = new CloudBlockBlob(outputBlobUri, outputStorageAccount.Credentials);
if (outputBlob.Exists())
{
outputBlob.Delete();
}
outputBlob.UploadFromFile(path + "/artist_data.txt", FileMode.Open);
}
if (file.Contains(@"customers.txt") && outputTableName.ToLower().Contains(@"customers"))
{
Uri outputBlobUri = new Uri(outputStorageAccount.BlobEndpoint, folderPath + "customers.txt");
CloudBlockBlob outputBlob = new CloudBlockBlob(outputBlobUri, outputStorageAccount.Credentials);
if (outputBlob.Exists())
{
outputBlob.Delete();
}
outputBlob.UploadFromFile(path + "/customers.txt", FileMode.Open);
}
else if (file.Contains(@"artist_customer_data.txt") && outputTableName.ToLower().Contains(@"rawproducts"))
{
Uri outputBlobUri = new Uri(outputStorageAccount.BlobEndpoint, folderPath + "artist_customer_data.txt");
CloudBlockBlob outputBlob = new CloudBlockBlob(outputBlobUri, outputStorageAccount.Credentials);
List<string> lines = File.ReadAllLines(path + "/artist_customer_data.txt").ToList();
int index = 0;
if (outputBlob.Exists() && index == 0)
{
outputBlob.Delete();
}
//add new column value for each row.
lines.Skip(0).ToList().ForEach(line =>
{
if (index >= 0 & index <= 1000)
{
lines[index] += "," + DateTime.UtcNow.AddMonths(-1).ToString("yyyy-MM-dd");
UploadBlobStream(outputBlob, lines[index]);
index++;
}
else if (index >= 1001 & index <= 2000)
{
lines[index] += "," + DateTime.UtcNow.AddMonths(-2).ToString("yyyy-MM-dd");
UploadBlobStream(outputBlob, lines[index]);
index++;
}
else if (index >= 2001 & index <= 3000)
{
lines[index] += "," + DateTime.UtcNow.AddMonths(-3).ToString("yyyy-MM-dd");
UploadBlobStream(outputBlob, lines[index]);
index++;
}
else if (index >= 3001 & index <= 4000)
{
lines[index] += "," + DateTime.UtcNow.AddMonths(-4).ToString("yyyy-MM-dd");
UploadBlobStream(outputBlob, lines[index]);
index++;
}
else if (index >= 4001 & index <= 5000)
{
lines[index] += "," + DateTime.UtcNow.AddMonths(-5).ToString("yyyy-MM-dd");
UploadBlobStream(outputBlob, lines[index]);
index++;
}
else if (index >= 5001 & index <= 6000)
{
lines[index] += "," + DateTime.UtcNow.AddMonths(-6).ToString("yyyy-MM-dd");
UploadBlobStream(outputBlob, lines[index]);
index++;
}
else if (index >= 6001 & index <= 7000)
{
lines[index] += "," + DateTime.UtcNow.AddMonths(-7).ToString("yyyy-MM-dd");
UploadBlobStream(outputBlob, lines[index]);
index++;
}
else if (index >= 7001 & index <= 8000)
{
lines[index] += "," + DateTime.UtcNow.AddMonths(-8).ToString("yyyy-MM-dd");
UploadBlobStream(outputBlob, lines[index]);
index++;
}
else if (index >= 8001 & index <= 9000)
{
lines[index] += "," + DateTime.UtcNow.AddMonths(-9).ToString("yyyy-MM-dd");
UploadBlobStream(outputBlob, lines[index]);
index++;
}
else if (index >= 9001 & index <= 10000)
{
lines[index] += "," + DateTime.UtcNow.AddMonths(-10).ToString("yyyy-MM-dd");
UploadBlobStream(outputBlob, lines[index]);
index++;
}
else if (index >= 10001 & index <= 11000)
{
lines[index] += "," + DateTime.UtcNow.AddMonths(-11).ToString("yyyy-MM-dd");
UploadBlobStream(outputBlob, lines[index]);
index++;
}
else if (index >= 11001 & index <= 12000)
{
lines[index] += "," + DateTime.UtcNow.AddMonths(-12).ToString("yyyy-MM-dd");
UploadBlobStream(outputBlob, lines[index]);
index++;
}
Console.WriteLine("Writing blob number: {0}", index);
logger.Write(TraceEventType.Information, "Writing blob number: {0}", index);
});
}
}
}
public void DownloadBlob_using_SharedAccessPolicies()
{
// In this unit test demo there is a timing aspect below where a policy expires.
// If run in debug the test might fail
var cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(BlobName);
Assert.IsTrue(cloudBlockBlob.Exists(), "If the blob does not exist then execute the test above!");
const string storedPolicyKey = "demostoredpolicy";
// 1) Set the policy in Azure Storage on the container.
// Make sure our policy does not already exist.
var blobContainerPermissions = cloudBlobContainer.GetPermissions();
if (blobContainerPermissions.SharedAccessPolicies.ContainsKey(storedPolicyKey))
{
blobContainerPermissions.SharedAccessPolicies.Remove(storedPolicyKey);
}
// Or use blobContainerPermissions.SharedAccessPolicies.Clear();
// Set our access policy in Azure to grant read access for a short time given the key.
blobContainerPermissions.SharedAccessPolicies.Add(
storedPolicyKey,
new SharedAccessBlobPolicy
{
Permissions = SharedAccessBlobPermissions.Read,
SharedAccessExpiryTime = DateTime.UtcNow.AddSeconds(5)
});
cloudBlobContainer.SetPermissions(blobContainerPermissions);
// 2) Use the policy only to
// This is the information a server would send to a client: Blob Uri including the SAS key!
var blobAbsoluteUri = new Uri(cloudBlockBlob.Uri.AbsoluteUri);
var sharedAccessSignature = cloudBlockBlob.GetSharedAccessSignature(null, storedPolicyKey);
var blobAccess = blobAbsoluteUri + sharedAccessSignature;
// Use the named key to get access
// This is typically done on a client that wants access using the policy they have been given
var blobAccessUri = new Uri(blobAccess);
var blobAccessBySAS = new CloudBlockBlob(blobAccessUri);
// Here we use the access by being able to read that the blob does indeed exist
Assert.IsTrue(blobAccessBySAS.Exists());
// Allow the policy to expire on the server
Task.Delay(TimeSpan.FromSeconds(5)).Wait();
try
{
blobAccessBySAS.Exists();
Assert.Fail("The policy should have been expired!");
}
catch (StorageException sex)
{
// We have no access to the blob on an expired policy. This is forbidden!
const HttpStatusCode expectedStatusCode = HttpStatusCode.Forbidden;
var returnedStatusCode = (HttpStatusCode)sex.RequestInformation.HttpStatusCode;
Assert.AreEqual(expectedStatusCode, returnedStatusCode);
}
// Prolong the policy on the server
blobContainerPermissions.SharedAccessPolicies[storedPolicyKey].SharedAccessExpiryTime = DateTime.UtcNow.AddSeconds(5);
cloudBlobContainer.SetPermissions(blobContainerPermissions);
// Access is again granted by the same stored policy as before! We did not need to send a new url to the client!
Assert.IsTrue(blobAccessBySAS.Exists());
}
private void InitFormBlob(CloudBlockBlob blob, bool withFileData)
{
if (blob == null || !blob.Exists())
return;
BlobKey = blob.Name;
FileName = Uri.UnescapeDataString(blob.Metadata["FileName"]);
ContentType = Uri.UnescapeDataString(blob.Metadata["ContentType"]);
int contentLength;
if (int.TryParse(Uri.UnescapeDataString(blob.Metadata["ContentLength"]), out contentLength))
ContentLength = contentLength;
BlobFileType type;
if (Enum.TryParse<BlobFileType>(Uri.UnescapeDataString(blob.Metadata["BlobFileType"]), true, out type))
BlobFileType = type;
DateTime dtUploaded;
if (DateTime.TryParse(Uri.UnescapeDataString(blob.Metadata["DtUploaded"]), out dtUploaded))
UploadedDate = dtUploaded;
if (withFileData)
{
Data = new MemoryStream();
blob.DownloadToStream(Data);
Data.Position = 0;
}
}
/// <summary>
/// Adds the diagnostic message to block blob.
/// </summary>
/// <param name="blob">The cloud blob.</param>
/// <param name="message">The message.</param>
protected virtual void AddMessageToBlock(CloudBlockBlob blob, string message)
{
Sitecore.Diagnostics.Assert.ArgumentNotNull(blob, "blob");
Sitecore.Diagnostics.Assert.ArgumentNotNull(message, "message");
var blockIds = new List<string>();
if (blob.Exists())
{
blockIds.AddRange(blob.DownloadBlockList().Select(b => b.Name));
}
string blockId = Guid.NewGuid().ToString().Replace("-", string.Empty);
blockIds.Add(blockId);
using (var blockData = new MemoryStream(LogStorageManager.DefaultTextEncoding.GetBytes(message), false))
{
blob.PutBlock(blockId, blockData, null);
blob.PutBlockList(blockIds);
}
}
