Example #1
0
        void AssertReplicationWorker(string container, string blobName, string primaryAccount, bool isDeleteReplica)
        {
            // Wait for the messages to be fully enqueued
            Task.Delay(1000).Wait();
            int processed = 0, errors = 0;

            MessageProcessor.ProcessMessageLoop(ref processed, ref errors, 0);
            Assert.AreEqual(0, errors);
            Assert.IsTrue(DashConfiguration.DataAccounts.Count - 1 <= processed);
            foreach (var account in DashConfiguration.DataAccounts
                     .Where(dataAccount => !String.Equals(dataAccount.Credentials.AccountName, primaryAccount, StringComparison.OrdinalIgnoreCase)))
            {
                var dataBlob = NamespaceHandler.GetBlobByName(account, container, blobName);
                Assert.AreEqual(!isDeleteReplica, dataBlob.Exists());
            }
        }
Example #2
0
        public static async Task <HandlerResult> AbortCopyBlobAsync(IHttpRequestWrapper requestWrapper, string destContainer, string destBlob, string copyId)
        {
            return(await WebOperationRunner.DoHandlerAsync("BlobHandler.AbortCopyBlobAsync", async() =>
            {
                var destNamespaceBlob = await NamespaceHandler.FetchNamespaceBlobAsync(destContainer, destBlob);

                var destCloudBlob = NamespaceHandler.GetBlobByName(
                    DashConfiguration.GetDataAccountByAccountName(destNamespaceBlob.PrimaryAccountName),
                    destContainer,
                    destBlob);
                await destCloudBlob.AbortCopyAsync(copyId);
                return new HandlerResult
                {
                    StatusCode = HttpStatusCode.NoContent,
                };
            }));
        }
Example #3
0
        public static async Task ImportAccountAsync(string accountName)
        {
            await OperationRunner.DoActionAsync(String.Format("Importing data account: {0}", accountName), async() =>
            {
                // This method will only import the blobs into the namespace. A future task may be
                // to redistribute the blobs to balance the entire virtual account.
                var account = DashConfiguration.GetDataAccountByAccountName(accountName);
                if (account == null)
                {
                    DashTrace.TraceWarning("Failure importing storage account: {0}. The storage account has not been configured as part of this virtual account",
                                           accountName);
                    return;
                }
                // Check if we've already imported this account
                var accountClient     = account.CreateCloudBlobClient();
                var namespaceClient   = DashConfiguration.NamespaceAccount.CreateCloudBlobClient();
                var accountContainers = await ListContainersAsync(accountClient);
                var status            = await AccountStatus.GetAccountStatus(accountName);
                await status.UpdateStatusInformation(AccountStatus.States.Healthy, "Importing storage account: {0} into virtual account", accountName);
                bool alreadyImported = false;
                await GetAccountBlobs(accountClient, async(blobItem) =>
                {
                    var blob          = (ICloudBlob)blobItem;
                    var namespaceBlob = await NamespaceBlob.FetchForBlobAsync(
                        (CloudBlockBlob)NamespaceHandler.GetBlobByName(DashConfiguration.NamespaceAccount, blob.Container.Name, blob.Name, blob.IsSnapshot ? blob.SnapshotTime.ToString() : String.Empty));
                    alreadyImported = await namespaceBlob.ExistsAsync(true) &&
                                      namespaceBlob.DataAccounts.Contains(accountName, StringComparer.OrdinalIgnoreCase);
                    return(false);
                });
                if (alreadyImported)
                {
                    await status.UpdateStatusWarning("Importing storage account: {0} has already been imported. This account cannot be imported again.", accountName);
                    return;
                }
                // Sync the container structure first - add containers in the imported account to the virtual account
                await status.UpdateStatusInformation("Importing storage account: {0}. Synchronizing container structure", accountName);
                int containersAddedCount = 0, containersWarningCount = 0;
                var namespaceContainers  = await ListContainersAsync(namespaceClient);
                await ProcessContainerDifferencesAsync(accountContainers, namespaceContainers, async(newContainerName, accountContainer) =>
                {
                    var createContainerResult = await ContainerHandler.DoForAllContainersAsync(newContainerName,
                                                                                               HttpStatusCode.Created,
                                                                                               async newContainer => await CopyContainer(accountContainer, newContainer),
                                                                                               true,
                                                                                               new[] { account });
                    if (createContainerResult.StatusCode < HttpStatusCode.OK || createContainerResult.StatusCode >= HttpStatusCode.Ambiguous)
                    {
                        await status.UpdateStatusWarning("Importing storage account: {0}. Failed to create container: {1} in virtual account. Details: {2}, {3}",
                                                         accountName,
                                                         newContainerName,
                                                         createContainerResult.StatusCode.ToString(),
                                                         createContainerResult.ReasonPhrase);
                        containersWarningCount++;
                    }
                    else
                    {
                        containersAddedCount++;
                    }
                },
                                                       (newContainerName, ex) =>
                {
                    status.UpdateStatusWarning("Importing storage account: {0}. Error processing container {1}. Details: {2}",
                                               accountName,
                                               newContainerName,
                                               ex.ToString()).Wait();
                    containersWarningCount++;
                });
                // Sync the other way
                await ProcessContainerDifferencesAsync(namespaceContainers, accountContainers, async(newContainerName, namespaceContainer) =>
                {
                    await CopyContainer(namespaceContainer, accountClient.GetContainerReference(newContainerName));
                },
                                                       (newContainerName, ex) =>
                {
                    status.UpdateStatusWarning("Importing storage account: {0}. Error replicating container {1} to imported account. Details: {2}",
                                               accountName,
                                               newContainerName,
                                               ex.ToString()).Wait();
                    containersWarningCount++;
                });
                DashTrace.TraceInformation("Importing storage account: {0}. Synchronized containers structure. {1} containers added to virtual account. {2} failures/warnings.",
                                           accountName,
                                           containersAddedCount,
                                           containersWarningCount);

                // Start importing namespace entries
                await status.UpdateStatusInformation("Importing storage account: {0}. Adding blob entries to namespace", accountName);
                int blobsAddedCount = 0, warningCount = 0, duplicateCount = 0;
                await GetAccountBlobs(accountClient, async(blobItem) =>
                {
                    var blob = (ICloudBlob)blobItem;
                    try
                    {
                        var namespaceBlob = await NamespaceBlob.FetchForBlobAsync(
                            (CloudBlockBlob)NamespaceHandler.GetBlobByName(DashConfiguration.NamespaceAccount, blob.Container.Name, blob.Name, blob.IsSnapshot ? blob.SnapshotTime.ToString() : String.Empty));
                        if (await namespaceBlob.ExistsAsync())
                        {
                            if (!String.Equals(namespaceBlob.PrimaryAccountName, accountName, StringComparison.OrdinalIgnoreCase))
                            {
                                await status.UpdateStatusWarning("Importing storage account: {0}. Adding blob: {1}/{2} would result in a duplicate blob entry. This blob will NOT be imported into the virtual account. Manually add the contents of this blob to the virtual account.",
                                                                 accountName,
                                                                 blob.Container.Name,
                                                                 blob.Name);
                                duplicateCount++;
                            }
                        }
                        else
                        {
                            namespaceBlob.PrimaryAccountName  = accountName;
                            namespaceBlob.Container           = blob.Container.Name;
                            namespaceBlob.BlobName            = blob.Name;
                            namespaceBlob.IsMarkedForDeletion = false;
                            await namespaceBlob.SaveAsync();
                            blobsAddedCount++;
                        }
                    }
                    catch (StorageException ex)
                    {
                        status.UpdateStatusWarning("Importing storage account: {0}. Error importing blob: {0}/{1} into virtual namespace. Details: {3}",
                                                   accountName,
                                                   blob.Container.Name,
                                                   blob.Name,
                                                   ex.ToString()).Wait();
                        warningCount++;
                    }
                    return(true);
                });

                if (status.State < AccountStatus.States.Warning)
                {
                    await status.UpdateStatus(String.Empty, AccountStatus.States.Unknown, TraceLevel.Off);
                }
                DashTrace.TraceInformation("Successfully imported the contents of storage account: '{0}' into the virtual namespace. Blobs added: {1}, duplicates detected: {2}, errors encountered: {3}",
                                           accountName, blobsAddedCount, duplicateCount, warningCount);
            },
                                                ex =>
            {
                var status = AccountStatus.GetAccountStatus(accountName).Result;
                status.UpdateStatusWarning("Error importing storage account: {0} into virtual account. Details: {1}", accountName, ex.ToString()).Wait();
            }, false, true);
        }
Example #4
0
 public static async Task <HandlerResult> CopyBlobAsync(IHttpRequestWrapper requestWrapper, string destContainer, string destBlob, string source)
 {
     return(await WebOperationRunner.DoHandlerAsync(String.Format("BlobHandler.CopyBlobAsync: {0}/{1} from {2}", destContainer, destBlob, source), async() =>
     {
         // source is a naked URI supplied by client
         Uri sourceUri;
         if (Uri.TryCreate(source, UriKind.RelativeOrAbsolute, out sourceUri))
         {
             string sourceContainer = String.Empty;
             string sourceBlobName = String.Empty;
             string sourceQuery = String.Empty;
             BlobType sourceBlobType = BlobType.BlockBlob;
             var requestVersion = requestWrapper.Headers.Value("x-ms-version", StorageServiceVersions.Version_2009_09_19);
             bool processRelativeSource = false;
             if (!sourceUri.IsAbsoluteUri)
             {
                 if (requestVersion >= StorageServiceVersions.Version_2012_02_12)
                 {
                     // 2012-02-12 onwards doesn't accept relative URIs
                     return new HandlerResult
                     {
                         StatusCode = HttpStatusCode.BadRequest,
                     };
                 }
                 // Make sourceUri absolute here because a bunch of Uri functionality fails for relative URIs
                 sourceUri = new Uri(new Uri("http://dummyhost"), sourceUri);
                 processRelativeSource = true;
             }
             if (processRelativeSource ||
                 (String.Equals(sourceUri.Host, requestWrapper.Url.Host, StringComparison.OrdinalIgnoreCase) &&
                  ((sourceUri.IsDefaultPort && requestWrapper.Url.IsDefaultPort) || (sourceUri.Port == requestWrapper.Url.Port))))
             {
                 var segments = PathUtils.GetPathSegments(sourceUri.AbsolutePath);
                 if (processRelativeSource)
                 {
                     // Blob in named container: /accountName/containerName/blobName
                     // Snapshot in named container: /accountName/containerName/blobName?snapshot=<DateTime>
                     // Blob in root container: /accountName/blobName
                     // Snapshot in root container: /accountName/blobName?snapshot=<DateTime>
                     if (!String.Equals(segments.FirstOrDefault(), DashConfiguration.AccountName))
                     {
                         return new HandlerResult
                         {
                             StatusCode = HttpStatusCode.BadRequest,
                             ErrorInformation = new DashErrorInformation
                             {
                                 ErrorCode = "CopyAcrossAccountsNotSupported",
                                 ErrorMessage = "The copy source account and destination account must be the same.",
                             },
                         };
                     }
                     if (segments.Count() == 2)
                     {
                         sourceContainer = "root";
                         sourceBlobName = segments[1];
                     }
                     else if (segments.Count() > 2)
                     {
                         sourceContainer = segments[1];
                         sourceBlobName = PathUtils.CombinePathSegments(segments.Skip(2));
                     }
                 }
                 else
                 {
                     sourceContainer = segments.FirstOrDefault();
                     sourceBlobName = PathUtils.CombinePathSegments(segments.Skip(1));
                 }
             }
             var destNamespaceBlob = await NamespaceHandler.FetchNamespaceBlobAsync(destContainer, destBlob);
             string destAccount = String.Empty;
             if (!String.IsNullOrEmpty(sourceContainer) && !String.IsNullOrEmpty(sourceBlobName))
             {
                 var sourceQueryParams = HttpUtility.ParseQueryString(sourceUri.Query);
                 var sourceNamespaceBlob = await NamespaceHandler.FetchNamespaceBlobAsync(sourceContainer, sourceBlobName, sourceQueryParams["snapshot"]);
                 if (!await sourceNamespaceBlob.ExistsAsync())
                 {
                     // This isn't actually documented (what happens when the source doesn't exist), but by obervation the service emits 404
                     return new HandlerResult
                     {
                         StatusCode = HttpStatusCode.NotFound,
                     };
                 }
                 var sourceCloudContainer = NamespaceHandler.GetContainerByName(
                     DashConfiguration.GetDataAccountByAccountName(sourceNamespaceBlob.PrimaryAccountName), sourceContainer);
                 sourceBlobType = sourceCloudContainer.GetBlobReferenceFromServer(sourceBlobName).BlobType;
                 // This is effectively an intra-account copy which is expected to be atomic. Therefore, even if the destination already
                 // exists, we need to place the destination in the same data account as the source.
                 // If the destination blob already exists, we delete it below to prevent an orphaned data blob
                 destAccount = sourceNamespaceBlob.PrimaryAccountName;
                 var sourceUriBuilder = ControllerOperations.GetRedirectUriBuilder("GET",
                                                                                   requestWrapper.Url.Scheme,
                                                                                   DashConfiguration.GetDataAccountByAccountName(destAccount),
                                                                                   sourceContainer,
                                                                                   sourceBlobName,
                                                                                   false,
                                                                                   String.Empty);
                 sourceUri = sourceUriBuilder.Uri;
             }
             else if (await destNamespaceBlob.ExistsAsync())
             {
                 destAccount = destNamespaceBlob.PrimaryAccountName;
             }
             else
             {
                 destAccount = NamespaceHandler.GetDataStorageAccountForBlob(destBlob).Credentials.AccountName;
             }
             bool replicateDestination = false;
             if (await destNamespaceBlob.ExistsAsync() && destNamespaceBlob.PrimaryAccountName != destAccount)
             {
                 // Delete the existing blob to prevent orphaning it
                 replicateDestination = destNamespaceBlob.IsReplicated;
                 var dataBlob = NamespaceHandler.GetBlobByName(
                     DashConfiguration.GetDataAccountByAccountName(destNamespaceBlob.PrimaryAccountName), destContainer, destBlob);
                 await dataBlob.DeleteIfExistsAsync();
             }
             destNamespaceBlob.PrimaryAccountName = destAccount;
             destNamespaceBlob.Container = destContainer;
             destNamespaceBlob.BlobName = destBlob;
             destNamespaceBlob.IsMarkedForDeletion = false;
             await destNamespaceBlob.SaveAsync();
             // Now that we've got the metadata tucked away - do the actual copy
             var destCloudContainer = NamespaceHandler.GetContainerByName(DashConfiguration.GetDataAccountByAccountName(destAccount), destContainer);
             ICloudBlob destCloudBlob = null;
             if (sourceBlobType == BlobType.PageBlob)
             {
                 destCloudBlob = destCloudContainer.GetPageBlobReference(destBlob);
             }
             else
             {
                 destCloudBlob = destCloudContainer.GetBlockBlobReference(destBlob);
             }
             // Storage client will retry failed copy. Let our clients decide that.
             var copyId = await destCloudBlob.StartCopyFromBlobAsync(sourceUri,
                                                                     AccessCondition.GenerateEmptyCondition(),
                                                                     AccessCondition.GenerateEmptyCondition(),
                                                                     new BlobRequestOptions
             {
                 RetryPolicy = new NoRetry(),
             },
                                                                     new OperationContext());
             // Check if we should replicate the copied destination blob
             if (replicateDestination || BlobReplicationHandler.ShouldReplicateBlob(requestWrapper.Headers, destContainer, destBlob))
             {
                 await BlobReplicationHandler.EnqueueBlobReplicationAsync(destNamespaceBlob, false);
             }
             return new HandlerResult
             {
                 StatusCode = requestVersion >= StorageServiceVersions.Version_2012_02_12 ? HttpStatusCode.Accepted : HttpStatusCode.Created,
                 Headers = new ResponseHeaders(new[]
                 {
                     new KeyValuePair <string, string>("x-ms-copy-id", copyId),
                     new KeyValuePair <string, string>("x-ms-copy-status", destCloudBlob.CopyState.Status == CopyStatus.Success ? "success" : "pending"),
                 })
             };
         }
         return new HandlerResult
         {
             StatusCode = HttpStatusCode.BadRequest,
         };
     }));
 }