void AssertReplicationWorker(string container, string blobName, string primaryAccount, bool isDeleteReplica) { // Wait for the messages to be fully enqueued Task.Delay(1000).Wait(); int processed = 0, errors = 0; MessageProcessor.ProcessMessageLoop(ref processed, ref errors, 0); Assert.AreEqual(0, errors); Assert.IsTrue(DashConfiguration.DataAccounts.Count - 1 <= processed); foreach (var account in DashConfiguration.DataAccounts .Where(dataAccount => !String.Equals(dataAccount.Credentials.AccountName, primaryAccount, StringComparison.OrdinalIgnoreCase))) { var dataBlob = NamespaceHandler.GetBlobByName(account, container, blobName); Assert.AreEqual(!isDeleteReplica, dataBlob.Exists()); } }
public static async Task <HandlerResult> AbortCopyBlobAsync(IHttpRequestWrapper requestWrapper, string destContainer, string destBlob, string copyId) { return(await WebOperationRunner.DoHandlerAsync("BlobHandler.AbortCopyBlobAsync", async() => { var destNamespaceBlob = await NamespaceHandler.FetchNamespaceBlobAsync(destContainer, destBlob); var destCloudBlob = NamespaceHandler.GetBlobByName( DashConfiguration.GetDataAccountByAccountName(destNamespaceBlob.PrimaryAccountName), destContainer, destBlob); await destCloudBlob.AbortCopyAsync(copyId); return new HandlerResult { StatusCode = HttpStatusCode.NoContent, }; })); }
public static async Task ImportAccountAsync(string accountName) { await OperationRunner.DoActionAsync(String.Format("Importing data account: {0}", accountName), async() => { // This method will only import the blobs into the namespace. A future task may be // to redistribute the blobs to balance the entire virtual account. var account = DashConfiguration.GetDataAccountByAccountName(accountName); if (account == null) { DashTrace.TraceWarning("Failure importing storage account: {0}. The storage account has not been configured as part of this virtual account", accountName); return; } // Check if we've already imported this account var accountClient = account.CreateCloudBlobClient(); var namespaceClient = DashConfiguration.NamespaceAccount.CreateCloudBlobClient(); var accountContainers = await ListContainersAsync(accountClient); var status = await AccountStatus.GetAccountStatus(accountName); await status.UpdateStatusInformation(AccountStatus.States.Healthy, "Importing storage account: {0} into virtual account", accountName); bool alreadyImported = false; await GetAccountBlobs(accountClient, async(blobItem) => { var blob = (ICloudBlob)blobItem; var namespaceBlob = await NamespaceBlob.FetchForBlobAsync( (CloudBlockBlob)NamespaceHandler.GetBlobByName(DashConfiguration.NamespaceAccount, blob.Container.Name, blob.Name, blob.IsSnapshot ? blob.SnapshotTime.ToString() : String.Empty)); alreadyImported = await namespaceBlob.ExistsAsync(true) && namespaceBlob.DataAccounts.Contains(accountName, StringComparer.OrdinalIgnoreCase); return(false); }); if (alreadyImported) { await status.UpdateStatusWarning("Importing storage account: {0} has already been imported. This account cannot be imported again.", accountName); return; } // Sync the container structure first - add containers in the imported account to the virtual account await status.UpdateStatusInformation("Importing storage account: {0}. Synchronizing container structure", accountName); int containersAddedCount = 0, containersWarningCount = 0; var namespaceContainers = await ListContainersAsync(namespaceClient); await ProcessContainerDifferencesAsync(accountContainers, namespaceContainers, async(newContainerName, accountContainer) => { var createContainerResult = await ContainerHandler.DoForAllContainersAsync(newContainerName, HttpStatusCode.Created, async newContainer => await CopyContainer(accountContainer, newContainer), true, new[] { account }); if (createContainerResult.StatusCode < HttpStatusCode.OK || createContainerResult.StatusCode >= HttpStatusCode.Ambiguous) { await status.UpdateStatusWarning("Importing storage account: {0}. Failed to create container: {1} in virtual account. Details: {2}, {3}", accountName, newContainerName, createContainerResult.StatusCode.ToString(), createContainerResult.ReasonPhrase); containersWarningCount++; } else { containersAddedCount++; } }, (newContainerName, ex) => { status.UpdateStatusWarning("Importing storage account: {0}. Error processing container {1}. Details: {2}", accountName, newContainerName, ex.ToString()).Wait(); containersWarningCount++; }); // Sync the other way await ProcessContainerDifferencesAsync(namespaceContainers, accountContainers, async(newContainerName, namespaceContainer) => { await CopyContainer(namespaceContainer, accountClient.GetContainerReference(newContainerName)); }, (newContainerName, ex) => { status.UpdateStatusWarning("Importing storage account: {0}. Error replicating container {1} to imported account. Details: {2}", accountName, newContainerName, ex.ToString()).Wait(); containersWarningCount++; }); DashTrace.TraceInformation("Importing storage account: {0}. Synchronized containers structure. {1} containers added to virtual account. {2} failures/warnings.", accountName, containersAddedCount, containersWarningCount); // Start importing namespace entries await status.UpdateStatusInformation("Importing storage account: {0}. Adding blob entries to namespace", accountName); int blobsAddedCount = 0, warningCount = 0, duplicateCount = 0; await GetAccountBlobs(accountClient, async(blobItem) => { var blob = (ICloudBlob)blobItem; try { var namespaceBlob = await NamespaceBlob.FetchForBlobAsync( (CloudBlockBlob)NamespaceHandler.GetBlobByName(DashConfiguration.NamespaceAccount, blob.Container.Name, blob.Name, blob.IsSnapshot ? blob.SnapshotTime.ToString() : String.Empty)); if (await namespaceBlob.ExistsAsync()) { if (!String.Equals(namespaceBlob.PrimaryAccountName, accountName, StringComparison.OrdinalIgnoreCase)) { await status.UpdateStatusWarning("Importing storage account: {0}. Adding blob: {1}/{2} would result in a duplicate blob entry. This blob will NOT be imported into the virtual account. Manually add the contents of this blob to the virtual account.", accountName, blob.Container.Name, blob.Name); duplicateCount++; } } else { namespaceBlob.PrimaryAccountName = accountName; namespaceBlob.Container = blob.Container.Name; namespaceBlob.BlobName = blob.Name; namespaceBlob.IsMarkedForDeletion = false; await namespaceBlob.SaveAsync(); blobsAddedCount++; } } catch (StorageException ex) { status.UpdateStatusWarning("Importing storage account: {0}. Error importing blob: {0}/{1} into virtual namespace. Details: {3}", accountName, blob.Container.Name, blob.Name, ex.ToString()).Wait(); warningCount++; } return(true); }); if (status.State < AccountStatus.States.Warning) { await status.UpdateStatus(String.Empty, AccountStatus.States.Unknown, TraceLevel.Off); } DashTrace.TraceInformation("Successfully imported the contents of storage account: '{0}' into the virtual namespace. Blobs added: {1}, duplicates detected: {2}, errors encountered: {3}", accountName, blobsAddedCount, duplicateCount, warningCount); }, ex => { var status = AccountStatus.GetAccountStatus(accountName).Result; status.UpdateStatusWarning("Error importing storage account: {0} into virtual account. Details: {1}", accountName, ex.ToString()).Wait(); }, false, true); }
public static async Task <HandlerResult> CopyBlobAsync(IHttpRequestWrapper requestWrapper, string destContainer, string destBlob, string source) { return(await WebOperationRunner.DoHandlerAsync(String.Format("BlobHandler.CopyBlobAsync: {0}/{1} from {2}", destContainer, destBlob, source), async() => { // source is a naked URI supplied by client Uri sourceUri; if (Uri.TryCreate(source, UriKind.RelativeOrAbsolute, out sourceUri)) { string sourceContainer = String.Empty; string sourceBlobName = String.Empty; string sourceQuery = String.Empty; BlobType sourceBlobType = BlobType.BlockBlob; var requestVersion = requestWrapper.Headers.Value("x-ms-version", StorageServiceVersions.Version_2009_09_19); bool processRelativeSource = false; if (!sourceUri.IsAbsoluteUri) { if (requestVersion >= StorageServiceVersions.Version_2012_02_12) { // 2012-02-12 onwards doesn't accept relative URIs return new HandlerResult { StatusCode = HttpStatusCode.BadRequest, }; } // Make sourceUri absolute here because a bunch of Uri functionality fails for relative URIs sourceUri = new Uri(new Uri("http://dummyhost"), sourceUri); processRelativeSource = true; } if (processRelativeSource || (String.Equals(sourceUri.Host, requestWrapper.Url.Host, StringComparison.OrdinalIgnoreCase) && ((sourceUri.IsDefaultPort && requestWrapper.Url.IsDefaultPort) || (sourceUri.Port == requestWrapper.Url.Port)))) { var segments = PathUtils.GetPathSegments(sourceUri.AbsolutePath); if (processRelativeSource) { // Blob in named container: /accountName/containerName/blobName // Snapshot in named container: /accountName/containerName/blobName?snapshot=<DateTime> // Blob in root container: /accountName/blobName // Snapshot in root container: /accountName/blobName?snapshot=<DateTime> if (!String.Equals(segments.FirstOrDefault(), DashConfiguration.AccountName)) { return new HandlerResult { StatusCode = HttpStatusCode.BadRequest, ErrorInformation = new DashErrorInformation { ErrorCode = "CopyAcrossAccountsNotSupported", ErrorMessage = "The copy source account and destination account must be the same.", }, }; } if (segments.Count() == 2) { sourceContainer = "root"; sourceBlobName = segments[1]; } else if (segments.Count() > 2) { sourceContainer = segments[1]; sourceBlobName = PathUtils.CombinePathSegments(segments.Skip(2)); } } else { sourceContainer = segments.FirstOrDefault(); sourceBlobName = PathUtils.CombinePathSegments(segments.Skip(1)); } } var destNamespaceBlob = await NamespaceHandler.FetchNamespaceBlobAsync(destContainer, destBlob); string destAccount = String.Empty; if (!String.IsNullOrEmpty(sourceContainer) && !String.IsNullOrEmpty(sourceBlobName)) { var sourceQueryParams = HttpUtility.ParseQueryString(sourceUri.Query); var sourceNamespaceBlob = await NamespaceHandler.FetchNamespaceBlobAsync(sourceContainer, sourceBlobName, sourceQueryParams["snapshot"]); if (!await sourceNamespaceBlob.ExistsAsync()) { // This isn't actually documented (what happens when the source doesn't exist), but by obervation the service emits 404 return new HandlerResult { StatusCode = HttpStatusCode.NotFound, }; } var sourceCloudContainer = NamespaceHandler.GetContainerByName( DashConfiguration.GetDataAccountByAccountName(sourceNamespaceBlob.PrimaryAccountName), sourceContainer); sourceBlobType = sourceCloudContainer.GetBlobReferenceFromServer(sourceBlobName).BlobType; // This is effectively an intra-account copy which is expected to be atomic. Therefore, even if the destination already // exists, we need to place the destination in the same data account as the source. // If the destination blob already exists, we delete it below to prevent an orphaned data blob destAccount = sourceNamespaceBlob.PrimaryAccountName; var sourceUriBuilder = ControllerOperations.GetRedirectUriBuilder("GET", requestWrapper.Url.Scheme, DashConfiguration.GetDataAccountByAccountName(destAccount), sourceContainer, sourceBlobName, false, String.Empty); sourceUri = sourceUriBuilder.Uri; } else if (await destNamespaceBlob.ExistsAsync()) { destAccount = destNamespaceBlob.PrimaryAccountName; } else { destAccount = NamespaceHandler.GetDataStorageAccountForBlob(destBlob).Credentials.AccountName; } bool replicateDestination = false; if (await destNamespaceBlob.ExistsAsync() && destNamespaceBlob.PrimaryAccountName != destAccount) { // Delete the existing blob to prevent orphaning it replicateDestination = destNamespaceBlob.IsReplicated; var dataBlob = NamespaceHandler.GetBlobByName( DashConfiguration.GetDataAccountByAccountName(destNamespaceBlob.PrimaryAccountName), destContainer, destBlob); await dataBlob.DeleteIfExistsAsync(); } destNamespaceBlob.PrimaryAccountName = destAccount; destNamespaceBlob.Container = destContainer; destNamespaceBlob.BlobName = destBlob; destNamespaceBlob.IsMarkedForDeletion = false; await destNamespaceBlob.SaveAsync(); // Now that we've got the metadata tucked away - do the actual copy var destCloudContainer = NamespaceHandler.GetContainerByName(DashConfiguration.GetDataAccountByAccountName(destAccount), destContainer); ICloudBlob destCloudBlob = null; if (sourceBlobType == BlobType.PageBlob) { destCloudBlob = destCloudContainer.GetPageBlobReference(destBlob); } else { destCloudBlob = destCloudContainer.GetBlockBlobReference(destBlob); } // Storage client will retry failed copy. Let our clients decide that. var copyId = await destCloudBlob.StartCopyFromBlobAsync(sourceUri, AccessCondition.GenerateEmptyCondition(), AccessCondition.GenerateEmptyCondition(), new BlobRequestOptions { RetryPolicy = new NoRetry(), }, new OperationContext()); // Check if we should replicate the copied destination blob if (replicateDestination || BlobReplicationHandler.ShouldReplicateBlob(requestWrapper.Headers, destContainer, destBlob)) { await BlobReplicationHandler.EnqueueBlobReplicationAsync(destNamespaceBlob, false); } return new HandlerResult { StatusCode = requestVersion >= StorageServiceVersions.Version_2012_02_12 ? HttpStatusCode.Accepted : HttpStatusCode.Created, Headers = new ResponseHeaders(new[] { new KeyValuePair <string, string>("x-ms-copy-id", copyId), new KeyValuePair <string, string>("x-ms-copy-status", destCloudBlob.CopyState.Status == CopyStatus.Success ? "success" : "pending"), }) }; } return new HandlerResult { StatusCode = HttpStatusCode.BadRequest, }; })); }