public static void ProcessMessageLoop(ref int msgProcessed, ref int msgErrors, int?invisibilityTimeout = null)
{
IMessageQueue queue = new AzureMessageQueue();
while (true)
{
try
{
QueueMessage payload = queue.Dequeue(invisibilityTimeout);
if (payload == null)
{
break;
}
// Right now, success/failure is indicated through a bool
// Do we want to surround this with a try/catch and use exceptions instead?
if (ProcessMessage(payload, invisibilityTimeout))
{
queue.DeleteCurrentMessage();
msgProcessed++;
}
else
{
// Leave it in the queue for retry after invisibility period expires
msgErrors++;
}
}
catch (Exception ex)
{
DashTrace.TraceWarning("Unhandled exception processing async message. Message will be left in queue. Details: {0}", ex);
msgErrors++;
}
}
}
public async Task <HttpResponseMessage> OptionsCallAsync()
{
return(await DoHandlerAsync("AccountController.OptionsCallAsync", async() =>
{
var request = HttpContextFactory.Current.Request;
HttpResponseMessage response;
if (request.Headers["Origin"] != null)
{
DashTrace.TraceInformation("Forwarding real CORS OPTIONS request");
Uri forwardUri = ControllerOperations.ForwardUriToNamespace(request);
var forwardRequest = new HttpRequestMessage(HttpMethod.Options, forwardUri);
foreach (string key in _corsOptionsHeaders)
{
forwardRequest.Headers.TryAddWithoutValidation(key, request.Headers[key]);
}
HttpClient client = new HttpClient();
response = await client.SendAsync(forwardRequest);
DashTrace.TraceInformation("CORS OPTIONS response: {0}, {1}", response.StatusCode, response.ReasonPhrase);
}
else
{
response = this.Request.CreateResponse(HttpStatusCode.OK);
}
response.Headers.Add("x-ms-dash-client", "true");
return response;
}));
}
static bool ProcessMessage(QueueMessage message, int?invisibilityTimeout = null)
{
var messageProcessed = false;
Guid contextId = message.CorrelationId.HasValue ? message.CorrelationId.Value : Guid.Empty;
using (var ctx = new CorrelationContext(contextId))
{
switch (message.MessageType)
{
case MessageTypes.BeginReplicate:
messageProcessed = DoReplicateJob(message, invisibilityTimeout);
break;
case MessageTypes.ReplicateProgress:
messageProcessed = DoReplicateProgressJob(message, invisibilityTimeout);
break;
case MessageTypes.DeleteReplica:
messageProcessed = DoDeleteReplicaJob(message, invisibilityTimeout);
break;
case MessageTypes.Unknown:
default:
DashTrace.TraceWarning("Unable to process unknown message type from async queue [{0}]. Payload: {1}",
message.MessageType,
message.ToString());
// Let this message bounce around for a bit - there may be a different version running
// on another instance that knows about this message. It will be automatically discarded
// after exceeding the deque limit.
messageProcessed = false;
break;
}
}
return(messageProcessed);
}
public static bool ProgressBlobReplication(string sourceAccount, string destAccount, string container, string blobName, string copyId, int?waitDelay = null)
{
bool retval = false;
ICloudBlob destBlob = null;
try
{
var destContainer = DashConfiguration.GetDataAccountByAccountName(destAccount).CreateCloudBlobClient().GetContainerReference(container);
destBlob = destContainer.GetBlobReferenceFromServer(blobName);
retval = ProcessBlobCopyStatus(destBlob, sourceAccount, copyId, waitDelay);
}
catch (StorageException ex)
{
// Classify the errors into retryable & non-retryable
retval = !RecoverReplicationError(ex, destBlob,
String.Format("Storage error checking replication progress for blob [{0}][{1}] to account [{2}]. Details: {3}",
sourceAccount,
PathUtils.CombineContainerAndBlob(container, blobName),
destAccount,
ex));
}
catch (Exception ex)
{
DashTrace.TraceWarning("Error checking replication progress for blob [{0}][{1}] to account [{2}]. Details: {3}",
sourceAccount,
PathUtils.CombineContainerAndBlob(container, blobName),
destAccount,
ex);
}
return(retval);
}
public async Task UpdateStatus(string message, States newState, TraceLevel traceLevel)
{
this.Message = message;
if (newState != States.Unknown)
{
this.State = newState;
}
await AccountStatus.UpdateAccountStatus(this);
switch (traceLevel)
{
case TraceLevel.Error:
DashTrace.TraceError(message);
break;
case TraceLevel.Warning:
DashTrace.TraceWarning(message);
break;
case TraceLevel.Info:
case TraceLevel.Verbose:
DashTrace.TraceInformation(message);
break;
}
}
public QueueMessage Dequeue(int?invisibilityTimeout = null)
{
while (true)
{
invisibilityTimeout = invisibilityTimeout ?? _timeout;
if (invisibilityTimeout.Value < 1)
{
invisibilityTimeout = 1;
}
this.CurrentMessage = Queue.GetMessage(new TimeSpan(0, 0, invisibilityTimeout.Value));
if (this.CurrentMessage != null)
{
if (this.CurrentMessage.DequeueCount >= _dequeLimit)
{
DashTrace.TraceWarning("Discarding message after exceeding deque limit of {0}. Message details: {1}",
this.CurrentMessage.DequeueCount,
this.CurrentMessage.AsString);
DeleteCurrentMessage();
}
else
{
return(JsonConvert.DeserializeObject <QueueMessage>(this.CurrentMessage.AsString));
}
}
else
{
break;
}
}
return(null);
}
public IEnumerable <T> Values <T>(string itemName) where T : IConvertible
{
IList <string> values = null;
if (_items.TryGetValue(itemName, out values))
{
return(values
.Select(value =>
{
try
{
if (typeof(T).IsEnum)
{
return (T)Enum.Parse(typeof(T), value);
}
return (T)Convert.ChangeType(value, typeof(T));
}
catch (Exception ex)
{
DashTrace.TraceWarning(new TraceMessage
{
Operation = "General Exception",
Message = String.Format("Failure converting item type. Item: {0}, new type: {1}", value, typeof(T).Name),
ErrorDetails = new DashErrorInformation {
ErrorMessage = ex.ToString()
},
});
}
return default(T);
}));
}
return(Enumerable.Empty <T>());
}
// TODO: Consider storage abstraction if we want to switch between XTable & Redis for this info
static AccountStatus()
{
try
{
GetAccountStatusTable().CreateIfNotExists();
}
catch (Exception ex)
{
DashTrace.TraceWarning("Error creating/referencing account status table: {0}. Details: {1}", StatusTableName, ex);
}
}
private static async Task <TableResult> ExecuteAsync(TableOperation operation)
{
try
{
return(await GetAccountStatusTable().ExecuteAsync(operation));
}
catch (Exception ex)
{
DashTrace.TraceWarning("Error interacting with account status. Details: {0}", ex);
}
return(null);
}
protected void Application_Start()
{
AzureUtils.AddAzureDiagnosticsListener();
DashTrace.TraceInformation("Starting application instance");
GlobalConfiguration.Configure(WebApiConfig.Register);
// Import any statically configured accounts
var importAccounts = DashConfiguration.ImportAccounts;
if (importAccounts.Any())
{
Task.Factory.StartNew(() => AccountManager.ImportAccounts(importAccounts));
}
}
public static string GenerateSignature(Func <string> stringToSignFactory, byte[] accountKey)
{
string stringToSign = stringToSignFactory();
DashTrace.TraceInformation("Authentication signing string: {0}", stringToSign);
var bytesToSign = Encoding.UTF8.GetBytes(stringToSign);
using (var hmac = new HMACSHA256(accountKey))
{
return(Convert.ToBase64String(hmac.ComputeHash(bytesToSign)));
}
}
static void Main(string[] args)
{
if (!Trace.Listeners.OfType <ConsoleTraceListener>().Any())
{
Trace.Listeners.Add(new ConsoleTraceListener());
}
AzureUtils.AddAzureDiagnosticsListener();
DashTrace.TraceInformation("DashAsync (version: {0}): Asynchronous worker starting up.", Assembly.GetEntryAssembly().GetName().Version);
int msgProcessed = 0, msgErrors = 0;
MessageProcessor.ProcessMessageLoop(ref msgProcessed, ref msgErrors);
DashTrace.TraceInformation("DashAsync completed. Messages processed [{0}], messages unprocessed [{1}]", msgProcessed, msgErrors);
}
static bool CleanupAbortedBlobReplication(NamespaceBlob namespaceBlob, ICloudBlob destBlob)
{
try
{
destBlob.DeleteIfExists();
}
catch (Exception ex1)
{
DashTrace.TraceWarning("Error deleting aborted replication target [{0}][{1}]. Details: {2}",
destBlob.ServiceClient.Credentials.AccountName,
destBlob.Name,
ex1);
}
return(namespaceBlob.RemoveDataAccount(destBlob.ServiceClient.Credentials.AccountName));
}
public static async Task EnqueueBlobReplicationAsync(NamespaceBlob namespaceBlob, bool deleteReplica, bool saveNamespaceEntry = true)
{
if (!await namespaceBlob.ExistsAsync())
{
return;
}
// Trim down the namespace replication list to the first 'master' item. This is sufficient to ensure that the
// orphaned blobs are not effectively in the account. The master blob will be replicated over the top of the
// orphaned blobs.
string primaryAccount = namespaceBlob.PrimaryAccountName;
if (namespaceBlob.IsReplicated)
{
namespaceBlob.PrimaryAccountName = primaryAccount;
if (saveNamespaceEntry)
{
await namespaceBlob.SaveAsync();
}
}
// This rest of this method does not block. Enqueueing the replication is a completely async process
var task = Task.Factory.StartNew(() =>
{
var queue = new AzureMessageQueue();
var tasks = DashConfiguration.DataAccounts
.Where(dataAccount => !dataAccount.Credentials.AccountName.Equals(primaryAccount, StringComparison.OrdinalIgnoreCase))
.Select(async dataAccount => await queue.EnqueueAsync(ConstructReplicationMessage(deleteReplica,
primaryAccount,
dataAccount.Credentials.AccountName,
namespaceBlob.Container,
namespaceBlob.BlobName,
deleteReplica ? await GetBlobETagAsync(dataAccount, namespaceBlob.Container, namespaceBlob.BlobName) : null)));
Task.WhenAll(tasks)
.ContinueWith(antecedent =>
{
if (antecedent.Exception != null)
{
DashTrace.TraceWarning("Error queueing replication message for blob: {0}. Details: {1}",
PathUtils.CombineContainerAndBlob(namespaceBlob.Container, namespaceBlob.BlobName),
antecedent.Exception.Flatten());
}
else
{
DashTrace.TraceInformation("Blob: {0} has been enqueued for replication.",
PathUtils.CombineContainerAndBlob(namespaceBlob.Container, namespaceBlob.BlobName));
}
});
});
}
static async Task <string> GetBlobETagAsync(CloudStorageAccount dataAccount, string containerName, string blobName)
{
try
{
var container = dataAccount.CreateCloudBlobClient().GetContainerReference(containerName);
var blob = await container.GetBlobReferenceFromServerAsync(blobName);
return(blob.Properties.ETag);
}
catch (StorageException ex)
{
DashTrace.TraceWarning("Exception attempting to retrieve ETag value for data blob [{0}/{1}/{2}]. Details: {3}",
dataAccount.Credentials.AccountName, containerName, blobName, ex);
}
return(String.Empty);
}
static bool RecoverReplicationError(StorageException ex, ICloudBlob destBlob, string exceptionMessage)
{
bool retval = true;
switch ((HttpStatusCode)ex.RequestInformation.HttpStatusCode)
{
case HttpStatusCode.Conflict:
case HttpStatusCode.PreconditionFailed:
// Destination has been modified - no retry
if (destBlob != null)
{
DashTrace.TraceInformation("A pre-condition was not met attempting to replicate or cleanup blob [{0}] in account [{1}]. This operation will be aborted",
destBlob.Name,
destBlob.ServiceClient.Credentials.AccountName);
}
else
{
DashTrace.TraceWarning("A pre-condition was not met attempting to replicate or cleanup an unknown blob. This operation will be aborted.");
}
retval = false;
break;
case HttpStatusCode.NotFound:
// The source blob could not be found - delete the target to prevent orphaning
if (destBlob != null)
{
DashTrace.TraceInformation("Replication of blob [{0}] to account [{1}] cannot be completed because the source blob does not exist.",
destBlob.Name,
destBlob.ServiceClient.Credentials.AccountName);
CleanupAbortedBlobReplication(destBlob);
}
else
{
DashTrace.TraceWarning("Replication of unknown blob cannot be completed because the source blob does not exist.");
}
retval = false;
break;
default:
// Unexpected exceptions are warnings
DashTrace.TraceWarning(exceptionMessage);
break;
}
return(retval);
}
static bool FinalizeBlobReplication(string dataAccount, string container, string blobName, bool deleteReplica, ICloudBlob destBlob = null)
{
return(NamespaceHandler.PerformNamespaceOperation(container, blobName, async(namespaceBlob) =>
{
bool exists = await namespaceBlob.ExistsAsync();
if (!exists || namespaceBlob.IsMarkedForDeletion)
{
// It's ok for a deleted replica not to have a corresponding namespace blob
if (!deleteReplica)
{
DashTrace.TraceWarning("Replication of blob [{0}] to account [{1}] cannot be completed because the namespace blob either does not exist or is marked for deletion.",
PathUtils.CombineContainerAndBlob(container, blobName),
dataAccount);
// Attempt to not leave the replica orphaned
if (CleanupAbortedBlobReplication(namespaceBlob, destBlob))
{
await namespaceBlob.SaveAsync();
}
}
// Do not attempt retry in this state
return true;
}
string message = "replicated to";
bool nsDirty = false;
if (deleteReplica)
{
nsDirty = namespaceBlob.RemoveDataAccount(dataAccount);
message = "dereplicated from";
}
else
{
nsDirty = namespaceBlob.AddDataAccount(dataAccount);
}
if (nsDirty)
{
await namespaceBlob.SaveAsync();
DashTrace.TraceInformation("Blob [{0}] has been successfully {1} account [{2}].",
PathUtils.CombineContainerAndBlob(container, blobName),
message,
dataAccount);
}
return true;
}).Result);
}
static DashConfigurationSource()
{
_dataAccounts = new Lazy <IList <CloudStorageAccount> >(() => GetDataStorageAccountsFromConfig().ToList(), LazyThreadSafetyMode.PublicationOnly);
_dataAccountsByName = new Lazy <IDictionary <string, CloudStorageAccount> >(() => DashConfiguration.DataAccounts
.Where(account => account != null)
.ToDictionary(account => account.Credentials.AccountName, StringComparer.OrdinalIgnoreCase), LazyThreadSafetyMode.PublicationOnly);
_namespaceAccount = new Lazy <CloudStorageAccount>(() =>
{
CloudStorageAccount account;
string connectString = AzureUtils.GetConfigSetting("StorageConnectionStringMaster", "");
if (!CloudStorageAccount.TryParse(connectString, out account))
{
DashTrace.TraceError("Error reading namespace account connection string from configuration. Details: {0}", connectString);
return(null);
}
ServicePointManager.FindServicePoint(account.BlobEndpoint).ConnectionLimit = int.MaxValue;
return(account);
}, LazyThreadSafetyMode.PublicationOnly);
}
static IEnumerable <CloudStorageAccount> GetDataStorageAccountsFromConfig()
{
for (int accountIndex = 0; true; accountIndex++)
{
var connectString = AzureUtils.GetConfigSetting("ScaleoutStorage" + accountIndex.ToString(), "");
if (String.IsNullOrWhiteSpace(connectString))
{
yield break;
}
CloudStorageAccount account;
if (CloudStorageAccount.TryParse(connectString, out account))
{
ServicePointManager.FindServicePoint(account.BlobEndpoint).ConnectionLimit = int.MaxValue;
yield return(account);
}
else
{
DashTrace.TraceWarning("Error reading data account connection string from configuration. Configuration details: {0}:{1}", accountIndex, connectString);
}
}
}
public static bool DeleteReplica(string accountName, string container, string blobName, string eTag)
{
bool retval = false;
ICloudBlob replicaBlob = null;
try
{
DashTrace.TraceInformation("Deleting replica blob [{0}] from account [{1}]", PathUtils.CombineContainerAndBlob(container, blobName), accountName);
var blobContainer = DashConfiguration.GetDataAccountByAccountName(accountName).CreateCloudBlobClient().GetContainerReference(container);
replicaBlob = blobContainer.GetBlobReferenceFromServer(blobName);
AccessCondition accessCondition = null;
if (!String.IsNullOrWhiteSpace(eTag))
{
accessCondition = AccessCondition.GenerateIfMatchCondition(eTag);
}
replicaBlob.Delete(DeleteSnapshotsOption.IncludeSnapshots, accessCondition);
FinalizeBlobReplication(accountName, container, blobName, true);
retval = true;
}
catch (StorageException ex)
{
// Classify the errors into retryable & non-retryable
retval = !RecoverReplicationError(ex, replicaBlob,
String.Format("Storage error deleting replica blob [{0}] from account [{1}]. Details: {2}",
PathUtils.CombineContainerAndBlob(container, blobName),
accountName,
ex));
}
catch (Exception ex)
{
DashTrace.TraceWarning("Error deleting replica blob [{0}] from account [{1}]. Details: {2}",
PathUtils.CombineContainerAndBlob(container, blobName),
accountName,
ex);
}
return(retval);
}
public static async Task <HandlerResult> HandlePrePipelineOperationAsync(IHttpRequestWrapper requestWrapper)
{
string containerName = requestWrapper.UriParts.Container;
string blobName = requestWrapper.UriParts.BlobName;
StorageOperationTypes requestOperation = StorageOperations.GetBlobOperation(requestWrapper);
DashClientCapabilities client = DashClientDetector.DetectClient(requestWrapper);
HandlerResult result = null;
bool servePrimaryOnly = false;
switch (requestOperation)
{
case StorageOperationTypes.GetBlobProperties:
case StorageOperationTypes.SetBlobProperties:
case StorageOperationTypes.GetBlobMetadata:
case StorageOperationTypes.SetBlobMetadata:
case StorageOperationTypes.LeaseBlob:
case StorageOperationTypes.GetBlockList:
servePrimaryOnly = true;
// Fall through
goto case StorageOperationTypes.GetBlob;
case StorageOperationTypes.GetBlob:
case StorageOperationTypes.SnapshotBlob:
case StorageOperationTypes.GetPageRanges:
if (client.HasFlag(DashClientCapabilities.FollowRedirects))
{
// If the client has specified a concurrent operation, then we always serve the primary account
// as that is where the concurrency controls exist
if (!servePrimaryOnly && requestWrapper.Headers.IsConcurrentRequest())
{
servePrimaryOnly = true;
}
result = await BlobHandler.BasicBlobAsync(requestWrapper,
containerName,
blobName,
servePrimaryOnly,
BlobReplicationOperations.DoesOperationTriggerReplication(requestOperation));
}
break;
case StorageOperationTypes.PutPage:
if (client.HasFlag(DashClientCapabilities.NoPayloadToDash))
{
result = await BlobHandler.BasicBlobAsync(requestWrapper,
containerName,
blobName,
true,
BlobReplicationOperations.DoesOperationTriggerReplication(requestOperation));
}
break;
case StorageOperationTypes.PutBlob:
case StorageOperationTypes.PutBlock:
case StorageOperationTypes.PutBlockList:
if (client.HasFlag(DashClientCapabilities.NoPayloadToDash))
{
result = await BlobHandler.PutBlobAsync(requestWrapper,
containerName,
blobName,
BlobReplicationOperations.DoesOperationTriggerReplication(requestOperation));
}
break;
default:
// All other operations flow through to the controller action
break;
}
DashTrace.TraceInformation("Operation: {0}, client capability: {1}, action: {2}",
requestOperation,
client,
result == null ? "Forward" : "Redirect");
return(result);
}
public static bool IsAuthorized(IHttpRequestWrapper request, RequestHeaders headers, RequestQueryParameters queryParams, bool ignoreRequestAge)
{
// Quick request age check
var authHeader = headers.Value <string>("Authorization");
string requestDateHeader = headers.Value <string>("x-ms-date");
string dateHeader = String.Empty;
if (String.IsNullOrWhiteSpace(requestDateHeader))
{
requestDateHeader = headers.Value <string>("Date");
dateHeader = requestDateHeader;
}
if (String.IsNullOrWhiteSpace(requestDateHeader))
{
// One of the date headers is mandatory
return(false);
}
if (!ignoreRequestAge)
{
DateTime requestDate;
if (!DateTime.TryParse(requestDateHeader, out requestDate))
{
return(false);
}
else if (requestDate < DateTime.Now.AddMinutes(-15))
{
return(false);
}
}
var parts = authHeader.Split(' ', ':');
if (parts.Length != 3)
{
return(false);
}
else if (parts[0] != SharedKeySignature.AlgorithmSharedKey && parts[0] != SharedKeySignature.AlgorithmSharedKeyLite)
{
return(false);
}
var account = parts[1];
var signature = parts[2];
if (!String.Equals(account, SharedKeySignature.AccountName, StringComparison.OrdinalIgnoreCase))
{
return(false);
}
// We have to deal with multiple encodings (the spec is a bit ambiguous on what an 'encoded' path actually is).
// Only run the validation if the encodings result in different strings
var requestUriParts = request.UriParts;
var pathsToCheck = new List <string>()
{
requestUriParts.OriginalUriPath
};
var unencodedPath = requestUriParts.PublicUriPath;
if (unencodedPath != pathsToCheck[0])
{
pathsToCheck.Add(unencodedPath);
}
var alternateEncodingPaths = AlternateEncodeString(pathsToCheck[0]);
if (alternateEncodingPaths != null)
{
pathsToCheck.AddRange(alternateEncodingPaths);
}
// For some verbs we can't tell if the Content-Length header was specified as 0 or that IIS/UrlRewrite/ASP.NET has constructed
// the header value for us. The difference is significant to the signature as content length is included for SharedKey
bool fullKeyAlgorithm = parts[0] == SharedKeySignature.AlgorithmSharedKey;
bool runBlankContentLengthComparison = false;
string method = request.HttpMethod.ToUpper();
var contentLength = headers.Value("Content-Length", String.Empty);
if (fullKeyAlgorithm)
{
int length;
if (!int.TryParse(contentLength, out length) || length <= 0)
{
// Preserve a Content-Length: 0 header for PUT methods
runBlankContentLengthComparison = !method.Equals(WebRequestMethods.Http.Put, StringComparison.OrdinalIgnoreCase);
}
}
var validationChecks = pathsToCheck.SelectMany(uriPath => new[] {
runBlankContentLengthComparison?Tuple.Create(true, uriPath, String.Empty) : null,
Tuple.Create(true, uriPath, contentLength),
runBlankContentLengthComparison ? Tuple.Create(false, uriPath, String.Empty) : null,
Tuple.Create(false, uriPath, contentLength),
})
.Where(check => check != null)
.ToArray();
var evaluationResult = validationChecks
.FirstOrDefault(validatationCheck =>
VerifyRequestAuthorization(signature, validatationCheck.Item1, !fullKeyAlgorithm, method, validatationCheck.Item2, headers, queryParams, dateHeader, validatationCheck.Item3));
if (evaluationResult != null)
{
// Remember the Auth Scheme & Key for when we have to sign the response
request.AuthenticationScheme = parts[0];
request.AuthenticationKey = evaluationResult.Item1 ? SharedKeySignature.PrimaryAccountKey : SharedKeySignature.SecondaryAccountKey;
return(true);
}
DashTrace.TraceWarning("Failed to authenticate SharedKey request: {0}:{1}:{2}:{3}:{4}", parts[0], account, method, request.Url, signature);
return(false);
}
public static async Task ImportAccountAsync(string accountName)
{
await OperationRunner.DoActionAsync(String.Format("Importing data account: {0}", accountName), async() =>
{
// This method will only import the blobs into the namespace. A future task may be
// to redistribute the blobs to balance the entire virtual account.
var account = DashConfiguration.GetDataAccountByAccountName(accountName);
if (account == null)
{
DashTrace.TraceWarning("Failure importing storage account: {0}. The storage account has not been configured as part of this virtual account",
accountName);
return;
}
// Check if we've already imported this account
var accountClient = account.CreateCloudBlobClient();
var namespaceClient = DashConfiguration.NamespaceAccount.CreateCloudBlobClient();
var accountContainers = await ListContainersAsync(accountClient);
var status = await AccountStatus.GetAccountStatus(accountName);
await status.UpdateStatusInformation(AccountStatus.States.Healthy, "Importing storage account: {0} into virtual account", accountName);
bool alreadyImported = false;
await GetAccountBlobs(accountClient, async(blobItem) =>
{
var blob = (ICloudBlob)blobItem;
var namespaceBlob = await NamespaceBlob.FetchForBlobAsync(
(CloudBlockBlob)NamespaceHandler.GetBlobByName(DashConfiguration.NamespaceAccount, blob.Container.Name, blob.Name, blob.IsSnapshot ? blob.SnapshotTime.ToString() : String.Empty));
alreadyImported = await namespaceBlob.ExistsAsync(true) &&
namespaceBlob.DataAccounts.Contains(accountName, StringComparer.OrdinalIgnoreCase);
return(false);
});
if (alreadyImported)
{
await status.UpdateStatusWarning("Importing storage account: {0} has already been imported. This account cannot be imported again.", accountName);
return;
}
// Sync the container structure first - add containers in the imported account to the virtual account
await status.UpdateStatusInformation("Importing storage account: {0}. Synchronizing container structure", accountName);
int containersAddedCount = 0, containersWarningCount = 0;
var namespaceContainers = await ListContainersAsync(namespaceClient);
await ProcessContainerDifferencesAsync(accountContainers, namespaceContainers, async(newContainerName, accountContainer) =>
{
var createContainerResult = await ContainerHandler.DoForAllContainersAsync(newContainerName,
HttpStatusCode.Created,
async newContainer => await CopyContainer(accountContainer, newContainer),
true,
new[] { account });
if (createContainerResult.StatusCode < HttpStatusCode.OK || createContainerResult.StatusCode >= HttpStatusCode.Ambiguous)
{
await status.UpdateStatusWarning("Importing storage account: {0}. Failed to create container: {1} in virtual account. Details: {2}, {3}",
accountName,
newContainerName,
createContainerResult.StatusCode.ToString(),
createContainerResult.ReasonPhrase);
containersWarningCount++;
}
else
{
containersAddedCount++;
}
},
(newContainerName, ex) =>
{
status.UpdateStatusWarning("Importing storage account: {0}. Error processing container {1}. Details: {2}",
accountName,
newContainerName,
ex.ToString()).Wait();
containersWarningCount++;
});
// Sync the other way
await ProcessContainerDifferencesAsync(namespaceContainers, accountContainers, async(newContainerName, namespaceContainer) =>
{
await CopyContainer(namespaceContainer, accountClient.GetContainerReference(newContainerName));
},
(newContainerName, ex) =>
{
status.UpdateStatusWarning("Importing storage account: {0}. Error replicating container {1} to imported account. Details: {2}",
accountName,
newContainerName,
ex.ToString()).Wait();
containersWarningCount++;
});
DashTrace.TraceInformation("Importing storage account: {0}. Synchronized containers structure. {1} containers added to virtual account. {2} failures/warnings.",
accountName,
containersAddedCount,
containersWarningCount);
// Start importing namespace entries
await status.UpdateStatusInformation("Importing storage account: {0}. Adding blob entries to namespace", accountName);
int blobsAddedCount = 0, warningCount = 0, duplicateCount = 0;
await GetAccountBlobs(accountClient, async(blobItem) =>
{
var blob = (ICloudBlob)blobItem;
try
{
var namespaceBlob = await NamespaceBlob.FetchForBlobAsync(
(CloudBlockBlob)NamespaceHandler.GetBlobByName(DashConfiguration.NamespaceAccount, blob.Container.Name, blob.Name, blob.IsSnapshot ? blob.SnapshotTime.ToString() : String.Empty));
if (await namespaceBlob.ExistsAsync())
{
if (!String.Equals(namespaceBlob.PrimaryAccountName, accountName, StringComparison.OrdinalIgnoreCase))
{
await status.UpdateStatusWarning("Importing storage account: {0}. Adding blob: {1}/{2} would result in a duplicate blob entry. This blob will NOT be imported into the virtual account. Manually add the contents of this blob to the virtual account.",
accountName,
blob.Container.Name,
blob.Name);
duplicateCount++;
}
}
else
{
namespaceBlob.PrimaryAccountName = accountName;
namespaceBlob.Container = blob.Container.Name;
namespaceBlob.BlobName = blob.Name;
namespaceBlob.IsMarkedForDeletion = false;
await namespaceBlob.SaveAsync();
blobsAddedCount++;
}
}
catch (StorageException ex)
{
status.UpdateStatusWarning("Importing storage account: {0}. Error importing blob: {0}/{1} into virtual namespace. Details: {3}",
accountName,
blob.Container.Name,
blob.Name,
ex.ToString()).Wait();
warningCount++;
}
return(true);
});
if (status.State < AccountStatus.States.Warning)
{
await status.UpdateStatus(String.Empty, AccountStatus.States.Unknown, TraceLevel.Off);
}
DashTrace.TraceInformation("Successfully imported the contents of storage account: '{0}' into the virtual namespace. Blobs added: {1}, duplicates detected: {2}, errors encountered: {3}",
accountName, blobsAddedCount, duplicateCount, warningCount);
},
ex =>
{
var status = AccountStatus.GetAccountStatus(accountName).Result;
status.UpdateStatusWarning("Error importing storage account: {0} into virtual account. Details: {1}", accountName, ex.ToString()).Wait();
}, false, true);
}
public static bool BeginBlobReplication(string sourceAccount, string destAccount, string container, string blobName, int?waitDelay = null)
{
bool retval = false;
ICloudBlob destBlob = null;
try
{
// Process is:
// - start the copy
// - wait around for a little while to see if it finishes - if so, we're done - update the namespace
// - if the copy is still in progress, enqueue a ReplicateProgress message to revisit the progress & update the namespace
// Attempt to acquire a reference to the specified destination first, because if the source no longer exists we must cleanup this orphaned replica
var destContainer = DashConfiguration.GetDataAccountByAccountName(destAccount).CreateCloudBlobClient().GetContainerReference(container);
try
{
destBlob = destContainer.GetBlobReferenceFromServer(blobName);
}
catch
{
destBlob = null;
}
var sourceClient = DashConfiguration.GetDataAccountByAccountName(sourceAccount).CreateCloudBlobClient();
var sourceBlob = sourceClient.GetContainerReference(container).GetBlobReferenceFromServer(blobName);
if (destBlob == null)
{
if (sourceBlob.BlobType == BlobType.PageBlob)
{
destBlob = destContainer.GetPageBlobReference(blobName);
}
else
{
destBlob = destContainer.GetBlockBlobReference(blobName);
}
}
DashTrace.TraceInformation("Replicating blob [{0}] to account [{1}]", sourceBlob.Uri, destAccount);
// If the source is still being copied to (we kicked off the replication as the result of a copy operation), then just recycle
// this message (don't return false as that may ultimately cause us to give up replicating).
if (sourceBlob.CopyState != null && sourceBlob.CopyState.Status == CopyStatus.Pending)
{
DashTrace.TraceInformation("Waiting for replication source [{0}] to complete copy.", sourceBlob.Uri);
new AzureMessageQueue().Enqueue(
BlobReplicationHandler.ConstructReplicationMessage(
false,
sourceAccount,
destAccount,
container,
blobName,
String.Empty));
return(true);
}
var sasUri = new UriBuilder(sourceBlob.Uri);
sasUri.Query = sourceBlob.GetSharedAccessSignature(new SharedAccessBlobPolicy
{
SharedAccessStartTime = DateTime.UtcNow.AddMinutes(-5),
SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(60),
Permissions = SharedAccessBlobPermissions.Read,
}).TrimStart('?');
string copyId = destBlob.StartCopyFromBlob(sasUri.Uri);
DateTime waitForCompleteGiveUp = DateTime.UtcNow.AddSeconds(waitDelay ?? (DashConfiguration.AsyncWorkerTimeout / 2));
while (DateTime.UtcNow < waitForCompleteGiveUp)
{
destBlob.FetchAttributes();
if (destBlob.CopyState.CopyId != copyId || destBlob.CopyState.Status != CopyStatus.Pending)
{
break;
}
Thread.Sleep(1000);
}
retval = ProcessBlobCopyStatus(destBlob, sourceAccount, copyId, waitDelay);
}
catch (StorageException ex)
{
// Classify the errors into retryable & non-retryable
retval = !RecoverReplicationError(ex, destBlob,
String.Format("Storage error initiating replication for blob [{0}][{1}] to account [{2}]. Details: {3}",
sourceAccount,
PathUtils.CombineContainerAndBlob(container, blobName),
destAccount,
ex));
}
catch (Exception ex)
{
DashTrace.TraceWarning("Error initiating replication for blob [{0}][{1}] to account [{2}]. Details: {3}",
sourceAccount,
PathUtils.CombineContainerAndBlob(container, blobName),
destAccount,
ex);
}
return(retval);
}
static bool ProcessBlobCopyStatus(ICloudBlob destBlob, string sourceAccount, string copyId, int?waitDelay = null)
{
bool retval = false;
string destAccount = destBlob.ServiceClient.Credentials.AccountName;
Uri sourceUri = destBlob.CopyState.Source;
// If the blob has moved on to another copy, just assume that it overrode our copy
if (destBlob.CopyState.CopyId == copyId)
{
switch (destBlob.CopyState.Status)
{
case CopyStatus.Aborted:
// Copy has been abandoned - we don't automatically come back from here
DashTrace.TraceWarning("Replicating blob [{0}] to account [{1}]. Copy Id [{3}] has been aborted.",
sourceUri, destAccount, copyId);
// Make sure we don't orphan the replica
CleanupAbortedBlobReplication(destBlob);
retval = true;
break;
case CopyStatus.Failed:
case CopyStatus.Invalid:
// Possibly temporaral issues - allow the message to retry after a period
DashTrace.TraceWarning("Replicating blob [{0}] to account [{1}]. Copy Id [{3}] has been failed or is invalid.",
sourceUri, destAccount, copyId);
// Make sure we don't orphan the replica
CleanupAbortedBlobReplication(destBlob);
retval = false;
break;
case CopyStatus.Pending:
// Enqueue a new message to check on the copy status
DashTrace.TraceInformation("Replicating blob [{0}] to account [{1}]. Copy Id [{2}] is pending. Copied [{3}]/[{4}] bytes. Enqueing progress message.",
sourceUri, destBlob.ServiceClient.Credentials.AccountName, copyId, destBlob.CopyState.BytesCopied, destBlob.CopyState.TotalBytes);
new AzureMessageQueue().Enqueue(new QueueMessage(MessageTypes.ReplicateProgress,
new Dictionary <string, string>
{
{ ReplicateProgressPayload.Source, sourceAccount },
{ ReplicateProgressPayload.Destination, destAccount },
{ ReplicateProgressPayload.Container, destBlob.Container.Name },
{ ReplicateProgressPayload.BlobName, destBlob.Name },
{ ReplicateProgressPayload.CopyID, copyId },
},
DashTrace.CorrelationId),
waitDelay ?? (DashConfiguration.WorkerQueueInitialDelay + 10));
retval = true;
break;
case CopyStatus.Success:
retval = FinalizeBlobReplication(destAccount, destBlob.Container.Name, destBlob.Name, false, destBlob);
break;
}
}
else
{
DashTrace.TraceInformation("Replication of blob [{0}] to account [{1}] has been aborted as the destination blob has a different copy id. Expected [{2}], actual [{3}]",
sourceUri, destAccount, copyId, destBlob.CopyState.CopyId);
// Return true to indicate that this operation shouldn't be retried
retval = true;
}
return(retval);
}
