Example #1
0
        private void TryFileCompression(ref FileCopyInfo fileCopyInfo, out bool fileCompressed)
        {
            fileCompressed = false;

            if (fileCopyInfo.SourceFileName.EndsWith(".zip"))
            {
                // already compressed skip it.
                return;
            }

            string tempArchivePath  = string.Empty;
            string actualFilePath   = fileCopyInfo.SourceFullPath;
            string archiveEntryName = Path.GetFileNameWithoutExtension(fileCopyInfo.SourceFileName);

            try
            {
                FileCompressor.Compress(fileCopyInfo.SourceFullPath, archiveEntryName, out tempArchivePath);
                Utility.PerformIOWithRetries(
                    () => { FabricFile.Delete(actualFilePath); });
            }
            catch (Exception e)
            {
                // Consume the exception and upload the uncompressed file.
                this.TraceSource.WriteExceptionAsWarning(
                    this.LogSourceId,
                    e,
                    "Failed to compress crash dump for upload");
                return;
            }

            fileCopyInfo.SourceFileName = archiveEntryName + ".zip";
            fileCopyInfo.SourceFullPath = tempArchivePath;
            fileCompressed = true;
        }
Example #2
0
        private void CopyWorker_DoWork(object sender, DoWorkEventArgs e)
        {
            FileCopyInfo finfo = (FileCopyInfo)e.Argument;

            CopyDirectory(finfo.SourcePath, finfo.DestinationPath);

            CopyWorker.ReportProgress(100);
        }
Example #3
0
        public FileIOProgress(string SourcePath, string DestinationPath, ComparisonResults.RefreshAndCloseDelegate rCD)
        {
            InitializeComponent();

            fileCopyInfo = new FileCopyInfo();

            fileCopyInfo.SourcePath      = SourcePath;
            fileCopyInfo.DestinationPath = DestinationPath;
            refreshAndCloseDelegate      = rCD;
        }
Example #4
0
        private void UploadFileWorker(object context)
        {
            FileCopyInfo fileCopyInfo = (FileCopyInfo)context;

            var destinationDirectory = this.GetDirectoryForUpload(this.containerName, this.directoryName, fileCopyInfo.RelativeDirectoryName);

            // Get the blob object
            CloudBlockBlob destinationBlob = destinationDirectory.GetBlockBlobReference(fileCopyInfo.SourceFileName);

            // Copy the file to the blob
            this.UploadFileWorker(fileCopyInfo.SourceFullPath, destinationBlob);
        }
Example #5
0
        private void CopyFileToDestinationBlob(object context)
        {
            FileCopyInfo fileCopyInfo = (FileCopyInfo)context;

            // Get the directory object under the container
            CloudBlobDirectory destinationDirectory = fileCopyInfo.Container.GetDirectoryReference(fileCopyInfo.RelativeDirectoryName);

            // Get the blob object
            CloudBlockBlob destinationBlob = destinationDirectory.GetBlockBlobReference(fileCopyInfo.SourceFileName);

            // Copy the file to the blob
            this.CopyFileToDestinationBlobWorker(fileCopyInfo, destinationBlob);
        }
Example #6
0
        private void CopyFileToDestinationBlob(object context)
        {
            this.perfHelper.ExternalOperationBegin(
                ExternalOperationTime.ExternalOperationType.BlobCopy,
                0);

            FileCopyInfo fileCopyInfo         = (FileCopyInfo)context;
            var          destinationDirectory = this.GetDirectoryForUpload(this.containerName, this.directoryName, fileCopyInfo.RelativeDirectoryName);

            // Get the blob object
            CloudBlockBlob destinationBlob = destinationDirectory.GetBlockBlobReference(fileCopyInfo.SourceFileName);

            // Copy the file to the blob
            this.CopyFileToDestinationBlobWorker(fileCopyInfo, destinationBlob);

            this.perfHelper.ExternalOperationEnd(
                ExternalOperationTime.ExternalOperationType.BlobCopy,
                0);
            this.perfHelper.AzureBlobUploaded();
        }
Example #7
0
        private bool UploadFile(string traceFileSubFolder, string filePath, string fileName)
        {
            FileCopyInfo fileCopyInfo = new FileCopyInfo()
            {
                SourceFullPath        = filePath,
                SourceFileName        = fileName,
                RelativeDirectoryName = traceFileSubFolder
            };

            // Figure out the name of the directory in the local map.
            string localMapDestination = Path.Combine(this.localMap, fileCopyInfo.RelativeDirectoryName, fileCopyInfo.SourceFileName);
            string localMapDestinationDir;

            try
            {
                localMapDestinationDir = FabricPath.GetDirectoryName(localMapDestination);
            }
            catch (PathTooLongException e)
            {
                // The path to the local map directory is too long. Skip it.
                this.traceSource.WriteError(
                    this.logSourceId,
                    "Failed to get local map directory for uploading file {0}. Exception information: {1}",
                    fileCopyInfo.SourceFileName,
                    e);
                return(false);
            }

            // File has already been uploaded before
            if (true == FabricFile.Exists(localMapDestination))
            {
                return(true);
            }

            try
            {
                Utility.PerformWithRetries(
                    this.UploadFileWorker,
                    fileCopyInfo,
                    new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler),
                    AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs,
                    AzureBlobEtwConstants.MethodExecutionMaxRetryCount,
                    AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs);
            }
            catch (Exception e)
            {
                this.traceSource.WriteExceptionAsError(
                    this.logSourceId,
                    e,
                    "Failed to copy file {0} to Azure blob account {1}, container {2}.",
                    fileName,
                    this.storageAccountFactory.Connection.AccountName,
                    this.containerName);
                return(false);
            }

            if (false == this.UpdateLocalMap(localMapDestinationDir, localMapDestination))
            {
                return(false);
            }

            this.traceSource.WriteInfo(
                this.logSourceId,
                "Successfully uploaded file {0} to blob storage.",
                fileCopyInfo.SourceFileName);

            return(true);
        }
Example #8
0
 public void AddFileCopyInfo(FileCopyInfo fileCopyInfo)
 {
     FileCopyInfos.Add(fileCopyInfo);
 }
Example #9
0
        private void CopyFileToDestinationBlobWorker(FileCopyInfo fileCopyInfo, CloudBlockBlob destinationBlob)
        {
            // First let's try to copy directly from the source location. This should work fine if the
            // source file is not in use.
            bool exceptionWithDirectCopy = false;

            try
            {
                CreateStreamAndUploadToBlob(fileCopyInfo.SourceFullPath, destinationBlob);
                this.TraceSource.WriteInfo(
                    this.LogSourceId,
                    "Uploaded file {0} to destination blob {1}",
                    fileCopyInfo.SourceFullPath,
                    destinationBlob.Name);
            }
            catch (Exception e)
            {
                if ((false == (e is IOException)) && (false == (e is FabricException)))
                {
                    throw;
                }

                exceptionWithDirectCopy = true;
            }

            if (exceptionWithDirectCopy)
            {
                // We couldn't copy the file directly to blob. This can happen when the file is in use,
                // because we would be unable to open a handle to the file in that case. Therefore,
                // we make a copy of the file to a temp location and then open a handle to that temp
                // copy for uploading to blob.
                string tempDir  = Utility.GetTempDirName();
                string tempDest = Path.Combine(tempDir, fileCopyInfo.SourceFileName);
                FabricDirectory.CreateDirectory(tempDir);
                try
                {
                    FabricFile.Copy(fileCopyInfo.SourceFullPath, tempDest, true);
                    CreateStreamAndUploadToBlob(tempDest, destinationBlob);
                    this.TraceSource.WriteInfo(
                        this.LogSourceId,
                        "Uploaded file {0} to destination blob {1}",
                        fileCopyInfo.SourceFullPath,
                        destinationBlob.Name);
                }
                finally
                {
                    try
                    {
                        Utility.PerformIOWithRetries(
                            () =>
                        {
                            FabricDirectory.Delete(tempDir, true);
                        });
                    }
                    catch (Exception e)
                    {
                        this.TraceSource.WriteExceptionAsError(
                            this.LogSourceId,
                            e,
                            "Failed to delete temporary directory {0}.",
                            tempDir);
                    }
                }
            }
        }
Example #10
0
        protected override bool CopyFileToDestination(string source, string sourceRelative, int retryCount, out bool fileSkipped, out bool fileCompressed)
        {
            fileSkipped    = true;
            fileCompressed = false;

            // Get the name of the source file
            string sourceFileName = Path.GetFileName(sourceRelative);

            // Create a list to hold the directory hierarchy of the source
            List <string> sourceRelativeParts = new List <string>();

            // Add each directory in the directory hierarchy to the list
            string sourceRelativePath = FabricPath.GetDirectoryName(sourceRelative);

            while (false == string.IsNullOrEmpty(sourceRelativePath))
            {
                string sourceRelativePart = Path.GetFileName(sourceRelativePath);
                sourceRelativeParts.Add(sourceRelativePart);

                sourceRelativePath = FabricPath.GetDirectoryName(sourceRelativePath);
            }

            // Reverse the list, so that top-level directories appear first
            sourceRelativeParts.Reverse();

            // Compute the directory name under the container
            string relativeDirectoryName = string.Join(
                "/",
                sourceRelativeParts.ToArray());

            // Copy the file to the Azure blob
            FileCopyInfo fileCopyInfo = new FileCopyInfo()
            {
                SourceFullPath        = source,
                SourceFileName        = sourceFileName,
                RelativeDirectoryName = relativeDirectoryName,
                RetryCount            = retryCount
            };

            try
            {
                Utility.PerformWithRetries(
                    this.CopyFileToDestinationBlob,
                    fileCopyInfo,
                    new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler),
                    retryCount);
            }
            catch (Exception e)
            {
                this.TraceSource.WriteExceptionAsError(
                    this.LogSourceId,
                    e,
                    "Failed to copy file {0} to Azure blob account {1}, container {2}.",
                    source,
                    this.storageAccountFactory.Connection.AccountName,
                    this.containerName);
                return(false);
            }

            fileSkipped = false;
            return(true);
        }
Example #11
0
        private bool CopyFileToDestination(CloudBlobContainer container, string source, string destinationPath, int retryCount)
        {
            // Get the name of the source file
            string sourceFileName = Path.GetFileName(source);

            // Create a list to hold the directory hierarchy of the source
            List <string> sourceRelativeParts = new List <string>();

            string relativeDirectoryName = string.Empty;

            if (!string.IsNullOrWhiteSpace(destinationPath))
            {
                // Add each directory in the directory hierarchy to the list
                string sourceRelativePartialPath = destinationPath;
                while (false == string.IsNullOrEmpty(sourceRelativePartialPath))
                {
                    string sourceRelativePart = Path.GetFileName(sourceRelativePartialPath);
                    sourceRelativeParts.Add(sourceRelativePart);

                    sourceRelativePartialPath = Path.GetDirectoryName(sourceRelativePartialPath);
                }

                // Reverse the list, so that top-level directories appear first
                sourceRelativeParts.Reverse();

                // Compute the directory name under the container
                relativeDirectoryName = string.Join(
                    "/",
                    sourceRelativeParts.ToArray());
            }

            // Copy the file to the Azure blob
            FileCopyInfo fileCopyInfo = new FileCopyInfo()
            {
                SourceFullPath        = source,
                SourceFileName        = sourceFileName,
                RelativeDirectoryName = relativeDirectoryName,
                RetryCount            = retryCount,
                Container             = container
            };

            try
            {
                BackupRestoreUtility.PerformWithRetries(
                    this.CopyFileToDestinationBlob,
                    fileCopyInfo,
                    new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler),
                    retryCount);
            }
            catch (Exception e)
            {
                this.TraceSource.WriteExceptionAsError(
                    TraceType,
                    e,
                    "Failed to copy file {0} to Azure blob storage container {1}.",
                    source,
                    container.Name);

                throw;
            }

            return(true);
        }
Example #12
0
        private async void MoveFiles(ObjectListView sourceListView, ObjectListView targetListView, IEnumerable <MyFileSystemInfo> sources, MyFileSystemInfo target, FileMoveMode mode = FileMoveMode.None)
        {
            if (mode == FileMoveMode.None)
            {
                return;
            }

            long totalSize        = 0;
            int  fileCount        = 0;
            long totalTransferred = 0;

            foreach (MyFileSystemInfo source in sources)
            {
                if (source.IsDirectory)
                {
                    fileCount += FileHelper.GetFolderFileCount(source.AsDirectory);
                    totalSize += FileHelper.GetFolderSize(source.AsDirectory);
                }
                else
                {
                    totalSize += source.AsFile.Length;
                    fileCount++;
                }
            }

            this.cancellationTokenSource = new CancellationTokenSource();

            FileCopyInfo info = new FileCopyInfo()
            {
                TotalSize = totalSize, FileCount = fileCount
            };

            Dictionary <string, TransferProgress> fileTransferred = new Dictionary <string, TransferProgress>();

            Action <TransferProgress> fileCopyProgress = (progress) =>
            {
                if (this.isCanceled)
                {
                    return;
                }

                if (!fileTransferred.ContainsKey(progress.SourcePath))
                {
                    fileTransferred.Add(progress.SourcePath, progress);
                }
                else
                {
                    fileTransferred[progress.SourcePath] = progress;
                }

                totalTransferred   = fileTransferred.Sum(item => item.Value.Transferred);
                info.FinishedCount = fileTransferred.Where(item => item.Value.Transferred == item.Value.Total).Count();

                info.TotalTransferred   = totalTransferred;
                info.CurrentSize        = progress.StreamSize;
                info.CurrentTransferred = progress.Transferred;
                info.CurrentPercent     = progress.Fraction;
                info.TotalPercent       = totalTransferred / (totalSize * 1.0);

                this.Feedback($"Progress:({info.FinishedCount}/{info.FileCount}) {FileHelper.FormatFileSize(info.TotalTransferred)}/{FileHelper.FormatFileSize(info.TotalSize)}, {info.TotalPercent.ToString("P")}");
            };

            try
            {
                this.isCanceled = false;
                this.isMoving   = true;

                foreach (MyFileSystemInfo source in sources)
                {
                    if (this.isCanceled)
                    {
                        break;
                    }

                    if (source.IsDirectory)
                    {
                        DirectoryInfo sourceFolder = source.AsDirectory;

                        string targetFolder = Path.Combine(target.FullName, sourceFolder.Name);

                        if (FileHelper.IsSameDrive(source.AsDirectory.FullName, targetFolder) && (mode == FileMoveMode.Cut || mode == FileMoveMode.Drag))
                        {
                            await FileTransferManager.MoveWithProgressAsync(source.AsDirectory.FullName, targetFolder, fileCopyProgress, this.cancellationTokenSource.Token);
                        }
                        else
                        {
                            await FileHelper.CopyFolder(source.AsDirectory.FullName, targetFolder, true, fileCopyProgress, this.cancellationTokenSource.Token);

                            if (mode == FileMoveMode.Cut)
                            {
                                source.AsDirectory.Delete(true);
                            }
                        }
                    }
                    else
                    {
                        FileInfo sourceFile = source.AsFile;

                        string targetFilePath = Path.Combine(target.FullName, sourceFile.Name);

                        if (FileHelper.IsSameDrive(sourceFile.DirectoryName, target.FullName) && (mode == FileMoveMode.Cut || mode == FileMoveMode.Drag))
                        {
                            await FileHelper.MoveFile(sourceFile.FullName, targetFilePath, fileCopyProgress, this.cancellationTokenSource.Token);
                        }
                        else
                        {
                            await FileHelper.CopyFile(sourceFile.FullName, targetFilePath, fileCopyProgress, this.cancellationTokenSource.Token);

                            if (mode == FileMoveMode.Cut)
                            {
                                sourceFile.Delete();
                            }
                        }
                    }
                }

                this.MoveObjects(sourceListView, targetListView, sources, target, mode);
            }
            catch (Exception ex)
            {
                this.Feedback(ex.Message, true);
            }
            finally
            {
                this.isMoving = false;

                if (mode != FileMoveMode.Copy)
                {
                    this.RefreshListView(this.navigator);
                }

                this.RefreshListView(this.lvFiles);
            }
        }
Example #13
0
 private static bool CopyFileToDestination(object context)
 {
     FileCopyInfo fileCopyInfo = (FileCopyInfo)context;
     FabricFile.Copy(fileCopyInfo.Source, fileCopyInfo.Destination, true);
     return true;
 }
Example #14
0
        protected override bool CopyFileToDestination(string source, string sourceRelative, int retryCount, out bool fileSkipped, out bool fileCompressed)
        {
            // No compression
            fileCompressed = false;

            // Compute the destination path
            string destination = Path.Combine(this.destinationPath, sourceRelative);

            // Figure out the name of the directory at the destination and in the
            // local map.
            var destinationDir = FabricPath.GetDirectoryName(destination);

            // If the directory at the destination doesn't exist, then create it
            if ((false == string.IsNullOrEmpty(destinationDir)) &&
                (false == this.PerformDestinationOperation(DirectoryExistsAtDestination, destinationDir)))
            {
                try
                {
                    Utility.PerformIOWithRetries(
                        () =>
                        {
                            PerformDestinationOperation(
                                CreateDirectoryAtDestination,
                                destinationDir);
                        },
                        retryCount);
                }
                catch (Exception e)
                {
                    this.TraceSource.WriteExceptionAsError(
                        this.LogSourceId,
                        e,
                        "Failed to create directory {0} for copying file {1}.",
                        destinationDir,
                        sourceRelative);
                    fileSkipped = true;
                    return false;
                }
            }

            // Determine whether we need to first copy the file to the staging folder
            string stagingFilePath = null;
            if (false == string.IsNullOrEmpty(this.stagingFolderPath))
            {
                stagingFilePath = Path.Combine(this.stagingFolderPath, Path.GetFileName(sourceRelative));
            }

            // Copy the file over to its destination
            DateTime sourceLastWriteTime = DateTime.MinValue;
            bool sourceFileNotFound = false;
            try
            {
                try
                {
                    Utility.PerformIOWithRetries(
                        () =>
                        {
                            if (string.IsNullOrEmpty(stagingFilePath))
                            {
                                this.perfHelper.ExternalOperationBegin(
                                    ExternalOperationTime.ExternalOperationType.FileShareCopy,
                                    0);
                            }

                            try
                            {
                                // Copy the file
                                if (false == string.IsNullOrEmpty(stagingFilePath))
                                {
                                    FabricFile.Copy(source, stagingFilePath, true);
                                }
                                else
                                {
                                    FileCopyInfo copyInfo = new FileCopyInfo
                                    {
                                        Source = source,
                                        Destination = destination
                                    };
                                    PerformDestinationOperation(CopyFileToDestination, copyInfo);
                                }
                            }
                            catch (FileNotFoundException)
                            {
                                sourceFileNotFound = true;
                            }

                            if (string.IsNullOrEmpty(stagingFilePath))
                            {
                                this.perfHelper.ExternalOperationEnd(
                                    ExternalOperationTime.ExternalOperationType.FileShareCopy,
                                    0);
                                this.perfHelper.FileUploaded();
                            }
                        },
                        retryCount);
                }
                catch (Exception e)
                {
                    this.TraceSource.WriteExceptionAsError(
                        this.LogSourceId,
                        e,
                        "File copy failed. Source: {0}, destination: {1}",
                        source,
                        destination);
                    fileSkipped = true;
                    return false;
                }

                if (sourceFileNotFound)
                {
                    // The source file was not found. Maybe it
                    // got deleted before we had a chance to copy
                    // it. Handle the error and move on.
                    this.TraceSource.WriteWarning(
                        this.LogSourceId,
                        "File {0} could not be uploaded to {1} because it was not found.",
                        source,
                        destination);
                    fileSkipped = true;
                    return true;
                }

                if (false == string.IsNullOrEmpty(stagingFilePath))
                {
                    try
                    {
                        Utility.PerformIOWithRetries(
                            () =>
                            {
                                this.perfHelper.ExternalOperationBegin(
                                    ExternalOperationTime.ExternalOperationType.FileShareCopy,
                                    0);

                                FileCopyInfo copyInfo = new FileCopyInfo
                                {
                                    Source = stagingFilePath,
                                    Destination = destination
                                };
                                PerformDestinationOperation(CopyFileToDestination, copyInfo);

                                this.perfHelper.ExternalOperationEnd(
                                    ExternalOperationTime.ExternalOperationType.FileShareCopy,
                                    0);
                                this.perfHelper.FileUploaded();
                            },
                            retryCount);
                    }
                    catch (Exception e)
                    {
                        this.TraceSource.WriteExceptionAsError(
                            this.LogSourceId,
                            e,
                            "File copy failed. Source: {0}, destination: {1}",
                            source,
                            destination);
                        fileSkipped = true;
                        return false;
                    }
                }
            }
            finally
            {
                if (false == string.IsNullOrEmpty(stagingFilePath))
                {
                    try
                    {
                        Utility.PerformIOWithRetries(
                            () =>
                            {
                                FabricFile.Delete(stagingFilePath);
                            },
                            retryCount);
                    }
                    catch (Exception e)
                    {
                        this.TraceSource.WriteExceptionAsError(
                            this.LogSourceId,
                            e,
                            "Failed to delete file {0}",
                            stagingFilePath);
                    }
                }
            }

            fileSkipped = false;
            return true;
        }
        private void CopyFileToDestination(string source, string destinationPath, string sourceRelative, int retryCount)
        {
            // Compute the destination path
            var    sourceFileName = Path.GetFileName(source);
            string destination    = Path.Combine(destinationPath, sourceRelative);

            destination = Path.Combine(destination, sourceFileName);

            // Figure out the name of the directory at the destination
            string destinationDir;

            try
            {
                destinationDir = Path.GetDirectoryName(destination);
            }
            catch (PathTooLongException e)
            {
                // The path to the destination directory is too long. Skip it.
                AppTrace.TraceSource.WriteExceptionAsError(
                    TraceType,
                    e,
                    "The destination path is too long {0}",
                    destination);
                throw;
            }

            // If the directory at the destination doesn't exist, then create it
            if (!String.IsNullOrEmpty(destinationDir))
            {
                try
                {
                    BackupRestoreUtility.PerformIOWithRetries(
                        () =>
                        this.PerformOperationUnderImpersonation(
                            () =>
                    {
                        if (!FabricDirectory.Exists(destinationDir))
                        {
                            FabricDirectory.CreateDirectory(destinationDir);
                        }

                        return(true);
                    }),
                        retryCount);
                }
                catch (Exception e)
                {
                    AppTrace.TraceSource.WriteExceptionAsError(
                        TraceType,
                        e,
                        "Failed to create directory {0} for copying file {1}.",
                        destinationDir,
                        sourceRelative);
                    throw;
                }
            }

            // Copy the file over to its destination
            try
            {
                BackupRestoreUtility.PerformIOWithRetries(
                    () =>
                {
                    FileCopyInfo copyInfo = new FileCopyInfo
                    {
                        Source      = source,
                        Destination = destination
                    };
                    PerformOperationUnderImpersonation(
                        () => CopyFileToDestination(copyInfo));
                },
                    retryCount);
            }
            catch (Exception e)
            {
                AppTrace.TraceSource.WriteExceptionAsError(
                    TraceType,
                    e,
                    "File copy failed. Source: {0}, destination: {1}",
                    source,
                    destination);
                throw;
            }
        }