private CloudBlockBlob GetBlockBlobReference(CloudBlobContainer container, string blobName) { try { return(BackupRestoreUtility.PerformWithRetries <string, CloudBlockBlob>( (string name) => { return container.GetBlockBlobReference(name); }, blobName, new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler), this.retryCount)); } catch (Exception e) { this.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to get cloud block blob {0} from Azure storage blob container {1}.", blobName, container.Name); throw; } }
private CloudBlobContainer GetOrCreateContainer(string containerName) { try { return(BackupRestoreUtility.PerformWithRetries <string, CloudBlobContainer>( (string name) => { CloudBlobContainer blobContainer = this.GetContainerRef(containerName); // Create the container blobContainer.CreateIfNotExists(null, null); return blobContainer; }, containerName, new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler), this.retryCount)); } catch (Exception e) { this.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to GetOrCreate container {0} with Azure storage.", containerName); throw; } }
private CloudBlobContainer GetContainer(string containerName) { try { return(BackupRestoreUtility.PerformWithRetries <string, CloudBlobContainer>( (string name) => { CloudBlobContainer container = this.GetContainerRef(name); // Create the container if (!container.Exists()) { throw new ArgumentException(string.Format("Given container {0} doesn't exist.", name), "containerName"); } return container; }, containerName, new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler), this.retryCount)); } catch (Exception e) { this.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to get container {0} from Azure storage.", containerName); throw; } }
private bool CopyFileToDestination(CloudBlockBlob blob, string destinationFullFileName, int retryCount) { CloudFileCopyInfo cloudFileCopyInfo = new CloudFileCopyInfo() { SourceBlob = blob, FullFileName = destinationFullFileName, RetryCount = retryCount }; try { BackupRestoreUtility.PerformWithRetries( this.CopyBlobToDestinationFile, cloudFileCopyInfo, new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler), this.retryCount); } catch (Exception e) { this.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to copy blob: {0} to file: {1}.", blob.Name, destinationFullFileName); throw; } return(true); }
public void Upload(string sourceFileOrFolderPath, string destinationFolderName, bool sourceIsFile = false) { var containerName = this.GetContainerName(); var destinationFolder = Path.Combine(this.GetFolderPath(), destinationFolderName == null ? string.Empty : destinationFolderName); this.TraceSource.WriteInfo(TraceType, "Moving backup from {0} to {1}/{2}", sourceFileOrFolderPath, containerName, destinationFolder); CloudBlobContainer container = null; // Create container try { container = BackupRestoreUtility.PerformWithRetries <string, CloudBlobContainer>( this.GetOrCreateContainer, containerName, new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler), this.retryCount); } catch (Exception e) { this.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to create container {0} on Azure storage.", containerName); throw; } string sourceFolder; if (!sourceIsFile) { sourceFolder = sourceFileOrFolderPath; } else { sourceFolder = Path.GetDirectoryName(sourceFileOrFolderPath); } if (sourceIsFile) { CopyFileToDestination(container, sourceFileOrFolderPath, destinationFolder, this.retryCount); } else { var sourceDirInfo = new DirectoryInfo(sourceFolder); // TODO: Add following files in concurrent bag and run loop with scheduled number of threads to upload these in parallel. foreach (var file in sourceDirInfo.GetFiles("*", SearchOption.AllDirectories)) { string sourceRelative = file.DirectoryName.Substring(sourceFolder.Trim('\\').Length).Trim('\\'); string effectiveDestinationPath = Path.Combine(destinationFolder, sourceRelative); this.TraceSource.WriteInfo(TraceType, "Full file name: {0} \t destinationFolder: {1} ", file.FullName, effectiveDestinationPath); CopyFileToDestination(container, file.FullName, effectiveDestinationPath, this.retryCount); } } }
/// <summary> /// Create a new <see cref="RecoveryPointMetadataFile"/> and write it to the given file. /// </summary> /// <returns>Task that represents the asynchronous operation.</returns> public static async Task <RecoveryPointMetadataFile> CreateAsync( string fileName, DateTime backupTime, Guid backupId, Guid parentBackupId, Guid backupChainId, Epoch epochOfLastBackupRecord, long lsnOfLastBackupRecord, string backupLocation, string parentBackupLocation, ServicePartitionInformation partitionInformation, string serviceManifestVersion, CancellationToken cancellationToken) { // Create the file with asynchronous flag and 4096 cache size (C# default). using (var filestream = FabricFile.Open( fileName, FileMode.CreateNew, FileAccess.Write, FileShare.Read, 4096, FileOptions.Asynchronous)) { BackupRestoreUtility.SetIoPriorityHint(filestream.SafeFileHandle, Kernel32Types.PRIORITY_HINT.IoPriorityHintLow); var recoveryPointMetadataFile = new RecoveryPointMetadataFile(fileName); recoveryPointMetadataFile.properties = new RecoveryPointMetadataFileProperties { BackupTime = backupTime, BackupLocation = backupLocation, ParentBackupLocation = parentBackupLocation, BackupId = backupId, ParentBackupId = parentBackupId, BackupChainId = backupChainId, EpochOfLastBackupRecord = epochOfLastBackupRecord, LsnOfLastBackupRecord = lsnOfLastBackupRecord, PartitionInformation = partitionInformation, ServiceManifestVersion = serviceManifestVersion, }; // Write the properties. var propertiesHandle = await FileBlock.WriteBlockAsync(filestream, writer => recoveryPointMetadataFile.properties.Write(writer)).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); // Write the footer. recoveryPointMetadataFile.footer = new FileFooter(propertiesHandle, Version); await FileBlock.WriteBlockAsync(filestream, writer => recoveryPointMetadataFile.footer.Write(writer)).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); // Flush to underlying stream. await filestream.FlushAsync(cancellationToken).ConfigureAwait(false); return(recoveryPointMetadataFile); } }
public void Upload(string sourceFileOrFolderPath, string destinationFolderName, bool sourceIsFile = false) { var destinationFolder = this._storeInformation.FileSharePath; if (!String.IsNullOrEmpty(destinationFolderName)) { destinationFolder += PathSeparator + destinationFolderName; } AppTrace.TraceSource.WriteInfo(TraceType, "Moving backup from {0} to {1}", sourceFileOrFolderPath, destinationFolder); var workFolder = Path.GetTempPath(); string sourceFolder; if (!sourceIsFile) { sourceFolder = sourceFileOrFolderPath; } else { sourceFolder = Path.GetDirectoryName(sourceFileOrFolderPath); } // Check if the source folder can be accessed under impersonation, if not ACL it accordingly try { BackupRestoreUtility.PerformIOWithRetries( () => { if (CheckFolderAccessUnderImpersonation(sourceFolder, FileAccess.Read)) { UpdateAclOnFolder(sourceFolder, FileSystemRights.Read); } }); } catch (Exception e) { throw new IOException(String.Format("Unable to create staging directory for work directory {0}.", workFolder), e); } if (sourceIsFile) { CopyFileToDestination(sourceFileOrFolderPath, destinationFolder, "", 3); } else { var sourceDirInfo = new DirectoryInfo(sourceFolder); foreach (var file in sourceDirInfo.GetFiles("*", SearchOption.AllDirectories)) { var sourceRelative = file.DirectoryName.Substring(sourceFolder.Trim('\\').Length).Trim('\\'); CopyFileToDestination(file.FullName, destinationFolder, sourceRelative, 3); } } }
private IEnumerable <IListBlobItem> GetBlobList(CloudBlobDirectory directory, bool useFlatBlobListing) { try { return(BackupRestoreUtility.PerformWithRetries <bool, IEnumerable <IListBlobItem> >( (bool flatBlobListing) => { return directory.ListBlobs(flatBlobListing); }, useFlatBlobListing, new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler), this.retryCount)); } catch (Exception e) { this.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to list blobs under directory {0} from Azure storage.", directory.Prefix); throw; } }
private CloudBlobDirectory GetCloudBlobDirectoryRef(CloudBlobContainer container, string directoryName) { try { return(BackupRestoreUtility.PerformWithRetries <string, CloudBlobDirectory>( (string relDirName) => { return container.GetDirectoryReference(relDirName); }, directoryName, new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler), this.retryCount)); } catch (Exception e) { this.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to get directory reference {0} from Azure storage.", directoryName); throw; } }
private void CopyFileToDestinationBlobWorker(FileCopyInfo fileCopyInfo, CloudBlockBlob destinationBlob) { // First let's try to copy directly from the source location. This should work fine if the // source file is not in use. bool exceptionWithDirectCopy = false; try { CreateStreamAndUploadToBlob(fileCopyInfo.SourceFullPath, destinationBlob); this.TraceSource.WriteInfo( TraceType, "Uploaded file {0} to destination blob {1}", fileCopyInfo.SourceFullPath, destinationBlob.Name); } catch (Exception e) { if ((false == (e is IOException)) && (false == (e is FabricException))) { throw; } exceptionWithDirectCopy = true; } if (exceptionWithDirectCopy) { // We couldn't copy the file directly to blob. This can happen when the file is in use, // because we would be unable to open a handle to the file in that case. Therefore, // we make a copy of the file to a temp location and then open a handle to that temp // copy for uploading to blob. string tempDir = Path.GetTempPath(); if (!String.IsNullOrWhiteSpace(fileCopyInfo.RelativeDirectoryName)) { tempDir = Path.Combine(tempDir, fileCopyInfo.RelativeDirectoryName); } string tempDest = Path.Combine(tempDir, fileCopyInfo.SourceFileName); FabricDirectory.CreateDirectory(tempDir); try { FabricFile.Copy(fileCopyInfo.SourceFullPath, tempDest, true); CreateStreamAndUploadToBlob(tempDest, destinationBlob); this.TraceSource.WriteInfo( TraceType, "Uploaded file {0} to destination blob {1}", fileCopyInfo.SourceFullPath, destinationBlob.Name); } finally { try { BackupRestoreUtility.PerformIOWithRetries( () => { FabricDirectory.Delete(tempDir, true); }); } catch (Exception e) { this.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to delete temporary directory {0}.", tempDir); } } } }
private bool CopyFileToDestination(CloudBlobContainer container, string source, string destinationPath, int retryCount) { // Get the name of the source file string sourceFileName = Path.GetFileName(source); // Create a list to hold the directory hierarchy of the source List <string> sourceRelativeParts = new List <string>(); string relativeDirectoryName = string.Empty; if (!string.IsNullOrWhiteSpace(destinationPath)) { // Add each directory in the directory hierarchy to the list string sourceRelativePartialPath = destinationPath; while (false == string.IsNullOrEmpty(sourceRelativePartialPath)) { string sourceRelativePart = Path.GetFileName(sourceRelativePartialPath); sourceRelativeParts.Add(sourceRelativePart); sourceRelativePartialPath = Path.GetDirectoryName(sourceRelativePartialPath); } // Reverse the list, so that top-level directories appear first sourceRelativeParts.Reverse(); // Compute the directory name under the container relativeDirectoryName = string.Join( "/", sourceRelativeParts.ToArray()); } // Copy the file to the Azure blob FileCopyInfo fileCopyInfo = new FileCopyInfo() { SourceFullPath = source, SourceFileName = sourceFileName, RelativeDirectoryName = relativeDirectoryName, RetryCount = retryCount, Container = container }; try { BackupRestoreUtility.PerformWithRetries( this.CopyFileToDestinationBlob, fileCopyInfo, new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler), retryCount); } catch (Exception e) { this.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to copy file {0} to Azure blob storage container {1}.", source, container.Name); throw; } return(true); }
public void Download(string sourceFileOrFolderName, string destinationFolder, bool sourceIsFile = false) { var sourceFileOrFolderPath = Path.Combine(this._storeInformation.FileSharePath, sourceFileOrFolderName); AppTrace.TraceSource.WriteInfo(TraceType, "Moving backup from {0} to {1}", sourceFileOrFolderPath, destinationFolder); // First check if the destination folder exists or not, if not create it BackupRestoreUtility.PerformIOWithRetries( () => { if (!Directory.Exists(destinationFolder)) { Directory.CreateDirectory(destinationFolder); } }); // Check if post impersonation we would be able to copy the content to the destination dir, // if not, add ACL appropriately. try { BackupRestoreUtility.PerformIOWithRetries( () => { if (CheckFolderAccessUnderImpersonation(destinationFolder, FileAccess.ReadWrite)) { UpdateAclOnFolder(destinationFolder, FileSystemRights.FullControl); } }); } catch (Exception e) { throw new IOException(String.Format("Unable to check for folder access or update ACL for destination folder {0}.", destinationFolder), e); } if (!sourceIsFile) { FileInfo[] files = null; var sourceDirInfo = new DirectoryInfo(sourceFileOrFolderPath); BackupRestoreUtility.PerformIOWithRetries( () => PerformOperationUnderImpersonation( () => { files = sourceDirInfo.GetFiles("*", SearchOption.AllDirectories); return(true); }) ); if (null == files) { throw new IOException(String.Format("Unable to enumerate files under directory {0}", sourceFileOrFolderPath)); } foreach (var file in files) { var sourceRelative = file.DirectoryName.Substring(sourceFileOrFolderPath.Trim(PathSeparator).Length).Trim(PathSeparator); CopyFileToDestination(file.FullName, destinationFolder, sourceRelative, 3); } } else { CopyFileToDestination(sourceFileOrFolderPath, destinationFolder, String.Empty, 3); } }
private void CopyFileToDestination(string source, string destinationPath, string sourceRelative, int retryCount) { // Compute the destination path var sourceFileName = Path.GetFileName(source); string destination = Path.Combine(destinationPath, sourceRelative); destination = Path.Combine(destination, sourceFileName); // Figure out the name of the directory at the destination string destinationDir; try { destinationDir = Path.GetDirectoryName(destination); } catch (PathTooLongException e) { // The path to the destination directory is too long. Skip it. AppTrace.TraceSource.WriteExceptionAsError( TraceType, e, "The destination path is too long {0}", destination); throw; } // If the directory at the destination doesn't exist, then create it if (!String.IsNullOrEmpty(destinationDir)) { try { BackupRestoreUtility.PerformIOWithRetries( () => this.PerformOperationUnderImpersonation( () => { if (!FabricDirectory.Exists(destinationDir)) { FabricDirectory.CreateDirectory(destinationDir); } return(true); }), retryCount); } catch (Exception e) { AppTrace.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to create directory {0} for copying file {1}.", destinationDir, sourceRelative); throw; } } // Copy the file over to its destination try { BackupRestoreUtility.PerformIOWithRetries( () => { FileCopyInfo copyInfo = new FileCopyInfo { Source = source, Destination = destination }; PerformOperationUnderImpersonation( () => CopyFileToDestination(copyInfo)); }, retryCount); } catch (Exception e) { AppTrace.TraceSource.WriteExceptionAsError( TraceType, e, "File copy failed. Source: {0}, destination: {1}", source, destination); throw; } }