public void ReplicateDeleteBlobControllerTest() { // We need exclusive access to the queue to validate queue behavior lock (this) { string blobName = Guid.NewGuid().ToString(); string blobUri = "http://localhost/blob/" + ContainerName + "/" + blobName; var content = new StringContent("hello world", System.Text.Encoding.UTF8, "text/plain"); content.Headers.Add("x-ms-version", "2013-08-15"); content.Headers.Add("x-ms-date", "Wed, 23 Oct 2013 22:33:355 GMT"); content.Headers.Add("x-ms-blob-type", "BlockBlob"); var response = _runner.ExecuteRequest(blobUri, "PUT", content, HttpStatusCode.Created); AssertQueueIsDrained(); // Directly manipulate the namespace blob so it appears that the blob is replicated var namespaceClient = DashConfiguration.NamespaceAccount.CreateCloudBlobClient(); var containerReference = namespaceClient.GetContainerReference(ContainerName); var nsBlob = NamespaceBlob.FetchForBlobAsync(containerReference.GetBlockBlobReference(blobName)).Result; foreach (var dataAccount in DashConfiguration.DataAccounts .Where(account => !String.Equals(account.Credentials.AccountName, nsBlob.PrimaryAccountName, StringComparison.OrdinalIgnoreCase))) { nsBlob.DataAccounts.Add(dataAccount.Credentials.AccountName); } nsBlob.SaveAsync().Wait(); // Now we delete - we should get a delete replica message for every data account except the primary _runner.ExecuteRequest(blobUri, "DELETE", expectedStatusCode: HttpStatusCode.Accepted); AssertReplicationMessageIsEnqueued(MessageTypes.DeleteReplica, ContainerName, blobName, nsBlob.PrimaryAccountName); } }
static bool CleanupAbortedBlobReplication(NamespaceBlob namespaceBlob, ICloudBlob destBlob) { try { destBlob.DeleteIfExists(); } catch (Exception ex1) { DashTrace.TraceWarning("Error deleting aborted replication target [{0}][{1}]. Details: {2}", destBlob.ServiceClient.Credentials.AccountName, destBlob.Name, ex1); } return(namespaceBlob.RemoveDataAccount(destBlob.ServiceClient.Credentials.AccountName)); }
public static async Task ImportAccountAsync(string accountName) { await OperationRunner.DoActionAsync(String.Format("Importing data account: {0}", accountName), async() => { // This method will only import the blobs into the namespace. A future task may be // to redistribute the blobs to balance the entire virtual account. var account = DashConfiguration.GetDataAccountByAccountName(accountName); if (account == null) { DashTrace.TraceWarning("Failure importing storage account: {0}. The storage account has not been configured as part of this virtual account", accountName); return; } // Check if we've already imported this account var accountClient = account.CreateCloudBlobClient(); var namespaceClient = DashConfiguration.NamespaceAccount.CreateCloudBlobClient(); var accountContainers = await ListContainersAsync(accountClient); var status = await AccountStatus.GetAccountStatus(accountName); await status.UpdateStatusInformation(AccountStatus.States.Healthy, "Importing storage account: {0} into virtual account", accountName); bool alreadyImported = false; await GetAccountBlobs(accountClient, async(blobItem) => { var blob = (ICloudBlob)blobItem; var namespaceBlob = await NamespaceBlob.FetchForBlobAsync( (CloudBlockBlob)NamespaceHandler.GetBlobByName(DashConfiguration.NamespaceAccount, blob.Container.Name, blob.Name, blob.IsSnapshot ? blob.SnapshotTime.ToString() : String.Empty)); alreadyImported = await namespaceBlob.ExistsAsync(true) && namespaceBlob.DataAccounts.Contains(accountName, StringComparer.OrdinalIgnoreCase); return(false); }); if (alreadyImported) { await status.UpdateStatusWarning("Importing storage account: {0} has already been imported. This account cannot be imported again.", accountName); return; } // Sync the container structure first - add containers in the imported account to the virtual account await status.UpdateStatusInformation("Importing storage account: {0}. Synchronizing container structure", accountName); int containersAddedCount = 0, containersWarningCount = 0; var namespaceContainers = await ListContainersAsync(namespaceClient); await ProcessContainerDifferencesAsync(accountContainers, namespaceContainers, async(newContainerName, accountContainer) => { var createContainerResult = await ContainerHandler.DoForAllContainersAsync(newContainerName, HttpStatusCode.Created, async newContainer => await CopyContainer(accountContainer, newContainer), true, new[] { account }); if (createContainerResult.StatusCode < HttpStatusCode.OK || createContainerResult.StatusCode >= HttpStatusCode.Ambiguous) { await status.UpdateStatusWarning("Importing storage account: {0}. Failed to create container: {1} in virtual account. Details: {2}, {3}", accountName, newContainerName, createContainerResult.StatusCode.ToString(), createContainerResult.ReasonPhrase); containersWarningCount++; } else { containersAddedCount++; } }, (newContainerName, ex) => { status.UpdateStatusWarning("Importing storage account: {0}. Error processing container {1}. Details: {2}", accountName, newContainerName, ex.ToString()).Wait(); containersWarningCount++; }); // Sync the other way await ProcessContainerDifferencesAsync(namespaceContainers, accountContainers, async(newContainerName, namespaceContainer) => { await CopyContainer(namespaceContainer, accountClient.GetContainerReference(newContainerName)); }, (newContainerName, ex) => { status.UpdateStatusWarning("Importing storage account: {0}. Error replicating container {1} to imported account. Details: {2}", accountName, newContainerName, ex.ToString()).Wait(); containersWarningCount++; }); DashTrace.TraceInformation("Importing storage account: {0}. Synchronized containers structure. {1} containers added to virtual account. {2} failures/warnings.", accountName, containersAddedCount, containersWarningCount); // Start importing namespace entries await status.UpdateStatusInformation("Importing storage account: {0}. Adding blob entries to namespace", accountName); int blobsAddedCount = 0, warningCount = 0, duplicateCount = 0; await GetAccountBlobs(accountClient, async(blobItem) => { var blob = (ICloudBlob)blobItem; try { var namespaceBlob = await NamespaceBlob.FetchForBlobAsync( (CloudBlockBlob)NamespaceHandler.GetBlobByName(DashConfiguration.NamespaceAccount, blob.Container.Name, blob.Name, blob.IsSnapshot ? blob.SnapshotTime.ToString() : String.Empty)); if (await namespaceBlob.ExistsAsync()) { if (!String.Equals(namespaceBlob.PrimaryAccountName, accountName, StringComparison.OrdinalIgnoreCase)) { await status.UpdateStatusWarning("Importing storage account: {0}. Adding blob: {1}/{2} would result in a duplicate blob entry. This blob will NOT be imported into the virtual account. Manually add the contents of this blob to the virtual account.", accountName, blob.Container.Name, blob.Name); duplicateCount++; } } else { namespaceBlob.PrimaryAccountName = accountName; namespaceBlob.Container = blob.Container.Name; namespaceBlob.BlobName = blob.Name; namespaceBlob.IsMarkedForDeletion = false; await namespaceBlob.SaveAsync(); blobsAddedCount++; } } catch (StorageException ex) { status.UpdateStatusWarning("Importing storage account: {0}. Error importing blob: {0}/{1} into virtual namespace. Details: {3}", accountName, blob.Container.Name, blob.Name, ex.ToString()).Wait(); warningCount++; } return(true); }); if (status.State < AccountStatus.States.Warning) { await status.UpdateStatus(String.Empty, AccountStatus.States.Unknown, TraceLevel.Off); } DashTrace.TraceInformation("Successfully imported the contents of storage account: '{0}' into the virtual namespace. Blobs added: {1}, duplicates detected: {2}, errors encountered: {3}", accountName, blobsAddedCount, duplicateCount, warningCount); }, ex => { var status = AccountStatus.GetAccountStatus(accountName).Result; status.UpdateStatusWarning("Error importing storage account: {0} into virtual account. Details: {1}", accountName, ex.ToString()).Wait(); }, false, true); }
private async Task <HttpResponseMessage> ForwardRequestHandler(NamespaceBlob namespaceBlob, StorageOperationTypes operation) { // Clone the inbound request var sourceRequest = this.Request; // We always target the primary data account for forwarded messages. If the operation invalidates the replicas, then // separate logic will enqueue the new blob to be replicated. var clonedRequest = new HttpRequestMessage(sourceRequest.Method, ControllerOperations.GetRedirectUri(sourceRequest.RequestUri, sourceRequest.Method.Method, DashConfiguration.GetDataAccountByAccountName(namespaceBlob.PrimaryAccountName), namespaceBlob.Container, namespaceBlob.BlobName, false)); clonedRequest.Version = sourceRequest.Version; foreach (var property in sourceRequest.Properties) { clonedRequest.Properties.Add(property); } foreach (var header in sourceRequest.Headers) { if (!_noCopyHeaders.Contains(header.Key)) { clonedRequest.Headers.TryAddWithoutValidation(header.Key, header.Value); } } // Depending on the operation, we have to do some fixup to unwind HttpRequestMessage a bit - we also have to fixup some responses switch (operation) { case StorageOperationTypes.GetBlob: case StorageOperationTypes.GetBlobMetadata: case StorageOperationTypes.GetBlobProperties: case StorageOperationTypes.GetBlockList: case StorageOperationTypes.GetPageRanges: case StorageOperationTypes.LeaseBlob: case StorageOperationTypes.SetBlobMetadata: case StorageOperationTypes.SetBlobProperties: case StorageOperationTypes.SnapshotBlob: // Push any headers that are assigned to Content onto the request itself as these operations do not have any body if (sourceRequest.Content != null) { foreach (var header in sourceRequest.Content.Headers) { clonedRequest.Headers.TryAddWithoutValidation(header.Key, header.Value); } } break; default: clonedRequest.Content = sourceRequest.Content; break; } var client = new HttpClient(); var response = await client.SendAsync(clonedRequest, HttpCompletionOption.ResponseHeadersRead); // Fixup response for HEAD requests switch (operation) { case StorageOperationTypes.GetBlobProperties: var content = response.Content; if (response.IsSuccessStatusCode && content != null) { string mediaType = null; string dummyContent = String.Empty; if (content.Headers.ContentType != null) { mediaType = content.Headers.ContentType.MediaType; } // For some reason, a HEAD request requires some content otherwise the Content-Length is set to 0 dummyContent = "A"; response.Content = new StringContent(dummyContent, null, mediaType); foreach (var header in content.Headers) { response.Content.Headers.TryAddWithoutValidation(header.Key, header.Value); } response.Content.Headers.ContentLength = content.Headers.ContentLength; content.Dispose(); } break; } return(response); }