private void TryFileCompression(ref FileCopyInfo fileCopyInfo, out bool fileCompressed)
{
fileCompressed = false;
if (fileCopyInfo.SourceFileName.EndsWith(".zip"))
{
// already compressed skip it.
return;
}
string tempArchivePath = string.Empty;
string actualFilePath = fileCopyInfo.SourceFullPath;
string archiveEntryName = Path.GetFileNameWithoutExtension(fileCopyInfo.SourceFileName);
try
{
FileCompressor.Compress(fileCopyInfo.SourceFullPath, archiveEntryName, out tempArchivePath);
Utility.PerformIOWithRetries(
() => { FabricFile.Delete(actualFilePath); });
}
catch (Exception e)
{
// Consume the exception and upload the uncompressed file.
this.TraceSource.WriteExceptionAsWarning(
this.LogSourceId,
e,
"Failed to compress crash dump for upload");
return;
}
fileCopyInfo.SourceFileName = archiveEntryName + ".zip";
fileCopyInfo.SourceFullPath = tempArchivePath;
fileCompressed = true;
}
private void CopyWorker_DoWork(object sender, DoWorkEventArgs e)
{
FileCopyInfo finfo = (FileCopyInfo)e.Argument;
CopyDirectory(finfo.SourcePath, finfo.DestinationPath);
CopyWorker.ReportProgress(100);
}
public FileIOProgress(string SourcePath, string DestinationPath, ComparisonResults.RefreshAndCloseDelegate rCD)
{
InitializeComponent();
fileCopyInfo = new FileCopyInfo();
fileCopyInfo.SourcePath = SourcePath;
fileCopyInfo.DestinationPath = DestinationPath;
refreshAndCloseDelegate = rCD;
}
private void UploadFileWorker(object context)
{
FileCopyInfo fileCopyInfo = (FileCopyInfo)context;
var destinationDirectory = this.GetDirectoryForUpload(this.containerName, this.directoryName, fileCopyInfo.RelativeDirectoryName);
// Get the blob object
CloudBlockBlob destinationBlob = destinationDirectory.GetBlockBlobReference(fileCopyInfo.SourceFileName);
// Copy the file to the blob
this.UploadFileWorker(fileCopyInfo.SourceFullPath, destinationBlob);
}
private void CopyFileToDestinationBlob(object context)
{
FileCopyInfo fileCopyInfo = (FileCopyInfo)context;
// Get the directory object under the container
CloudBlobDirectory destinationDirectory = fileCopyInfo.Container.GetDirectoryReference(fileCopyInfo.RelativeDirectoryName);
// Get the blob object
CloudBlockBlob destinationBlob = destinationDirectory.GetBlockBlobReference(fileCopyInfo.SourceFileName);
// Copy the file to the blob
this.CopyFileToDestinationBlobWorker(fileCopyInfo, destinationBlob);
}
private void CopyFileToDestinationBlob(object context)
{
this.perfHelper.ExternalOperationBegin(
ExternalOperationTime.ExternalOperationType.BlobCopy,
0);
FileCopyInfo fileCopyInfo = (FileCopyInfo)context;
var destinationDirectory = this.GetDirectoryForUpload(this.containerName, this.directoryName, fileCopyInfo.RelativeDirectoryName);
// Get the blob object
CloudBlockBlob destinationBlob = destinationDirectory.GetBlockBlobReference(fileCopyInfo.SourceFileName);
// Copy the file to the blob
this.CopyFileToDestinationBlobWorker(fileCopyInfo, destinationBlob);
this.perfHelper.ExternalOperationEnd(
ExternalOperationTime.ExternalOperationType.BlobCopy,
0);
this.perfHelper.AzureBlobUploaded();
}
private bool UploadFile(string traceFileSubFolder, string filePath, string fileName)
{
FileCopyInfo fileCopyInfo = new FileCopyInfo()
{
SourceFullPath = filePath,
SourceFileName = fileName,
RelativeDirectoryName = traceFileSubFolder
};
// Figure out the name of the directory in the local map.
string localMapDestination = Path.Combine(this.localMap, fileCopyInfo.RelativeDirectoryName, fileCopyInfo.SourceFileName);
string localMapDestinationDir;
try
{
localMapDestinationDir = FabricPath.GetDirectoryName(localMapDestination);
}
catch (PathTooLongException e)
{
// The path to the local map directory is too long. Skip it.
this.traceSource.WriteError(
this.logSourceId,
"Failed to get local map directory for uploading file {0}. Exception information: {1}",
fileCopyInfo.SourceFileName,
e);
return(false);
}
// File has already been uploaded before
if (true == FabricFile.Exists(localMapDestination))
{
return(true);
}
try
{
Utility.PerformWithRetries(
this.UploadFileWorker,
fileCopyInfo,
new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler),
AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs,
AzureBlobEtwConstants.MethodExecutionMaxRetryCount,
AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Failed to copy file {0} to Azure blob account {1}, container {2}.",
fileName,
this.storageAccountFactory.Connection.AccountName,
this.containerName);
return(false);
}
if (false == this.UpdateLocalMap(localMapDestinationDir, localMapDestination))
{
return(false);
}
this.traceSource.WriteInfo(
this.logSourceId,
"Successfully uploaded file {0} to blob storage.",
fileCopyInfo.SourceFileName);
return(true);
}
public void AddFileCopyInfo(FileCopyInfo fileCopyInfo)
{
FileCopyInfos.Add(fileCopyInfo);
}
private void CopyFileToDestinationBlobWorker(FileCopyInfo fileCopyInfo, CloudBlockBlob destinationBlob)
{
// First let's try to copy directly from the source location. This should work fine if the
// source file is not in use.
bool exceptionWithDirectCopy = false;
try
{
CreateStreamAndUploadToBlob(fileCopyInfo.SourceFullPath, destinationBlob);
this.TraceSource.WriteInfo(
this.LogSourceId,
"Uploaded file {0} to destination blob {1}",
fileCopyInfo.SourceFullPath,
destinationBlob.Name);
}
catch (Exception e)
{
if ((false == (e is IOException)) && (false == (e is FabricException)))
{
throw;
}
exceptionWithDirectCopy = true;
}
if (exceptionWithDirectCopy)
{
// We couldn't copy the file directly to blob. This can happen when the file is in use,
// because we would be unable to open a handle to the file in that case. Therefore,
// we make a copy of the file to a temp location and then open a handle to that temp
// copy for uploading to blob.
string tempDir = Utility.GetTempDirName();
string tempDest = Path.Combine(tempDir, fileCopyInfo.SourceFileName);
FabricDirectory.CreateDirectory(tempDir);
try
{
FabricFile.Copy(fileCopyInfo.SourceFullPath, tempDest, true);
CreateStreamAndUploadToBlob(tempDest, destinationBlob);
this.TraceSource.WriteInfo(
this.LogSourceId,
"Uploaded file {0} to destination blob {1}",
fileCopyInfo.SourceFullPath,
destinationBlob.Name);
}
finally
{
try
{
Utility.PerformIOWithRetries(
() =>
{
FabricDirectory.Delete(tempDir, true);
});
}
catch (Exception e)
{
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"Failed to delete temporary directory {0}.",
tempDir);
}
}
}
}
protected override bool CopyFileToDestination(string source, string sourceRelative, int retryCount, out bool fileSkipped, out bool fileCompressed)
{
fileSkipped = true;
fileCompressed = false;
// Get the name of the source file
string sourceFileName = Path.GetFileName(sourceRelative);
// Create a list to hold the directory hierarchy of the source
List <string> sourceRelativeParts = new List <string>();
// Add each directory in the directory hierarchy to the list
string sourceRelativePath = FabricPath.GetDirectoryName(sourceRelative);
while (false == string.IsNullOrEmpty(sourceRelativePath))
{
string sourceRelativePart = Path.GetFileName(sourceRelativePath);
sourceRelativeParts.Add(sourceRelativePart);
sourceRelativePath = FabricPath.GetDirectoryName(sourceRelativePath);
}
// Reverse the list, so that top-level directories appear first
sourceRelativeParts.Reverse();
// Compute the directory name under the container
string relativeDirectoryName = string.Join(
"/",
sourceRelativeParts.ToArray());
// Copy the file to the Azure blob
FileCopyInfo fileCopyInfo = new FileCopyInfo()
{
SourceFullPath = source,
SourceFileName = sourceFileName,
RelativeDirectoryName = relativeDirectoryName,
RetryCount = retryCount
};
try
{
Utility.PerformWithRetries(
this.CopyFileToDestinationBlob,
fileCopyInfo,
new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler),
retryCount);
}
catch (Exception e)
{
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"Failed to copy file {0} to Azure blob account {1}, container {2}.",
source,
this.storageAccountFactory.Connection.AccountName,
this.containerName);
return(false);
}
fileSkipped = false;
return(true);
}
private bool CopyFileToDestination(CloudBlobContainer container, string source, string destinationPath, int retryCount)
{
// Get the name of the source file
string sourceFileName = Path.GetFileName(source);
// Create a list to hold the directory hierarchy of the source
List <string> sourceRelativeParts = new List <string>();
string relativeDirectoryName = string.Empty;
if (!string.IsNullOrWhiteSpace(destinationPath))
{
// Add each directory in the directory hierarchy to the list
string sourceRelativePartialPath = destinationPath;
while (false == string.IsNullOrEmpty(sourceRelativePartialPath))
{
string sourceRelativePart = Path.GetFileName(sourceRelativePartialPath);
sourceRelativeParts.Add(sourceRelativePart);
sourceRelativePartialPath = Path.GetDirectoryName(sourceRelativePartialPath);
}
// Reverse the list, so that top-level directories appear first
sourceRelativeParts.Reverse();
// Compute the directory name under the container
relativeDirectoryName = string.Join(
"/",
sourceRelativeParts.ToArray());
}
// Copy the file to the Azure blob
FileCopyInfo fileCopyInfo = new FileCopyInfo()
{
SourceFullPath = source,
SourceFileName = sourceFileName,
RelativeDirectoryName = relativeDirectoryName,
RetryCount = retryCount,
Container = container
};
try
{
BackupRestoreUtility.PerformWithRetries(
this.CopyFileToDestinationBlob,
fileCopyInfo,
new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler),
retryCount);
}
catch (Exception e)
{
this.TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to copy file {0} to Azure blob storage container {1}.",
source,
container.Name);
throw;
}
return(true);
}
private async void MoveFiles(ObjectListView sourceListView, ObjectListView targetListView, IEnumerable <MyFileSystemInfo> sources, MyFileSystemInfo target, FileMoveMode mode = FileMoveMode.None)
{
if (mode == FileMoveMode.None)
{
return;
}
long totalSize = 0;
int fileCount = 0;
long totalTransferred = 0;
foreach (MyFileSystemInfo source in sources)
{
if (source.IsDirectory)
{
fileCount += FileHelper.GetFolderFileCount(source.AsDirectory);
totalSize += FileHelper.GetFolderSize(source.AsDirectory);
}
else
{
totalSize += source.AsFile.Length;
fileCount++;
}
}
this.cancellationTokenSource = new CancellationTokenSource();
FileCopyInfo info = new FileCopyInfo()
{
TotalSize = totalSize, FileCount = fileCount
};
Dictionary <string, TransferProgress> fileTransferred = new Dictionary <string, TransferProgress>();
Action <TransferProgress> fileCopyProgress = (progress) =>
{
if (this.isCanceled)
{
return;
}
if (!fileTransferred.ContainsKey(progress.SourcePath))
{
fileTransferred.Add(progress.SourcePath, progress);
}
else
{
fileTransferred[progress.SourcePath] = progress;
}
totalTransferred = fileTransferred.Sum(item => item.Value.Transferred);
info.FinishedCount = fileTransferred.Where(item => item.Value.Transferred == item.Value.Total).Count();
info.TotalTransferred = totalTransferred;
info.CurrentSize = progress.StreamSize;
info.CurrentTransferred = progress.Transferred;
info.CurrentPercent = progress.Fraction;
info.TotalPercent = totalTransferred / (totalSize * 1.0);
this.Feedback($"Progress:({info.FinishedCount}/{info.FileCount}) {FileHelper.FormatFileSize(info.TotalTransferred)}/{FileHelper.FormatFileSize(info.TotalSize)}, {info.TotalPercent.ToString("P")}");
};
try
{
this.isCanceled = false;
this.isMoving = true;
foreach (MyFileSystemInfo source in sources)
{
if (this.isCanceled)
{
break;
}
if (source.IsDirectory)
{
DirectoryInfo sourceFolder = source.AsDirectory;
string targetFolder = Path.Combine(target.FullName, sourceFolder.Name);
if (FileHelper.IsSameDrive(source.AsDirectory.FullName, targetFolder) && (mode == FileMoveMode.Cut || mode == FileMoveMode.Drag))
{
await FileTransferManager.MoveWithProgressAsync(source.AsDirectory.FullName, targetFolder, fileCopyProgress, this.cancellationTokenSource.Token);
}
else
{
await FileHelper.CopyFolder(source.AsDirectory.FullName, targetFolder, true, fileCopyProgress, this.cancellationTokenSource.Token);
if (mode == FileMoveMode.Cut)
{
source.AsDirectory.Delete(true);
}
}
}
else
{
FileInfo sourceFile = source.AsFile;
string targetFilePath = Path.Combine(target.FullName, sourceFile.Name);
if (FileHelper.IsSameDrive(sourceFile.DirectoryName, target.FullName) && (mode == FileMoveMode.Cut || mode == FileMoveMode.Drag))
{
await FileHelper.MoveFile(sourceFile.FullName, targetFilePath, fileCopyProgress, this.cancellationTokenSource.Token);
}
else
{
await FileHelper.CopyFile(sourceFile.FullName, targetFilePath, fileCopyProgress, this.cancellationTokenSource.Token);
if (mode == FileMoveMode.Cut)
{
sourceFile.Delete();
}
}
}
}
this.MoveObjects(sourceListView, targetListView, sources, target, mode);
}
catch (Exception ex)
{
this.Feedback(ex.Message, true);
}
finally
{
this.isMoving = false;
if (mode != FileMoveMode.Copy)
{
this.RefreshListView(this.navigator);
}
this.RefreshListView(this.lvFiles);
}
}
private static bool CopyFileToDestination(object context)
{
FileCopyInfo fileCopyInfo = (FileCopyInfo)context;
FabricFile.Copy(fileCopyInfo.Source, fileCopyInfo.Destination, true);
return true;
}
protected override bool CopyFileToDestination(string source, string sourceRelative, int retryCount, out bool fileSkipped, out bool fileCompressed)
{
// No compression
fileCompressed = false;
// Compute the destination path
string destination = Path.Combine(this.destinationPath, sourceRelative);
// Figure out the name of the directory at the destination and in the
// local map.
var destinationDir = FabricPath.GetDirectoryName(destination);
// If the directory at the destination doesn't exist, then create it
if ((false == string.IsNullOrEmpty(destinationDir)) &&
(false == this.PerformDestinationOperation(DirectoryExistsAtDestination, destinationDir)))
{
try
{
Utility.PerformIOWithRetries(
() =>
{
PerformDestinationOperation(
CreateDirectoryAtDestination,
destinationDir);
},
retryCount);
}
catch (Exception e)
{
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"Failed to create directory {0} for copying file {1}.",
destinationDir,
sourceRelative);
fileSkipped = true;
return false;
}
}
// Determine whether we need to first copy the file to the staging folder
string stagingFilePath = null;
if (false == string.IsNullOrEmpty(this.stagingFolderPath))
{
stagingFilePath = Path.Combine(this.stagingFolderPath, Path.GetFileName(sourceRelative));
}
// Copy the file over to its destination
DateTime sourceLastWriteTime = DateTime.MinValue;
bool sourceFileNotFound = false;
try
{
try
{
Utility.PerformIOWithRetries(
() =>
{
if (string.IsNullOrEmpty(stagingFilePath))
{
this.perfHelper.ExternalOperationBegin(
ExternalOperationTime.ExternalOperationType.FileShareCopy,
0);
}
try
{
// Copy the file
if (false == string.IsNullOrEmpty(stagingFilePath))
{
FabricFile.Copy(source, stagingFilePath, true);
}
else
{
FileCopyInfo copyInfo = new FileCopyInfo
{
Source = source,
Destination = destination
};
PerformDestinationOperation(CopyFileToDestination, copyInfo);
}
}
catch (FileNotFoundException)
{
sourceFileNotFound = true;
}
if (string.IsNullOrEmpty(stagingFilePath))
{
this.perfHelper.ExternalOperationEnd(
ExternalOperationTime.ExternalOperationType.FileShareCopy,
0);
this.perfHelper.FileUploaded();
}
},
retryCount);
}
catch (Exception e)
{
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"File copy failed. Source: {0}, destination: {1}",
source,
destination);
fileSkipped = true;
return false;
}
if (sourceFileNotFound)
{
// The source file was not found. Maybe it
// got deleted before we had a chance to copy
// it. Handle the error and move on.
this.TraceSource.WriteWarning(
this.LogSourceId,
"File {0} could not be uploaded to {1} because it was not found.",
source,
destination);
fileSkipped = true;
return true;
}
if (false == string.IsNullOrEmpty(stagingFilePath))
{
try
{
Utility.PerformIOWithRetries(
() =>
{
this.perfHelper.ExternalOperationBegin(
ExternalOperationTime.ExternalOperationType.FileShareCopy,
0);
FileCopyInfo copyInfo = new FileCopyInfo
{
Source = stagingFilePath,
Destination = destination
};
PerformDestinationOperation(CopyFileToDestination, copyInfo);
this.perfHelper.ExternalOperationEnd(
ExternalOperationTime.ExternalOperationType.FileShareCopy,
0);
this.perfHelper.FileUploaded();
},
retryCount);
}
catch (Exception e)
{
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"File copy failed. Source: {0}, destination: {1}",
source,
destination);
fileSkipped = true;
return false;
}
}
}
finally
{
if (false == string.IsNullOrEmpty(stagingFilePath))
{
try
{
Utility.PerformIOWithRetries(
() =>
{
FabricFile.Delete(stagingFilePath);
},
retryCount);
}
catch (Exception e)
{
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"Failed to delete file {0}",
stagingFilePath);
}
}
}
fileSkipped = false;
return true;
}
private void CopyFileToDestination(string source, string destinationPath, string sourceRelative, int retryCount)
{
// Compute the destination path
var sourceFileName = Path.GetFileName(source);
string destination = Path.Combine(destinationPath, sourceRelative);
destination = Path.Combine(destination, sourceFileName);
// Figure out the name of the directory at the destination
string destinationDir;
try
{
destinationDir = Path.GetDirectoryName(destination);
}
catch (PathTooLongException e)
{
// The path to the destination directory is too long. Skip it.
AppTrace.TraceSource.WriteExceptionAsError(
TraceType,
e,
"The destination path is too long {0}",
destination);
throw;
}
// If the directory at the destination doesn't exist, then create it
if (!String.IsNullOrEmpty(destinationDir))
{
try
{
BackupRestoreUtility.PerformIOWithRetries(
() =>
this.PerformOperationUnderImpersonation(
() =>
{
if (!FabricDirectory.Exists(destinationDir))
{
FabricDirectory.CreateDirectory(destinationDir);
}
return(true);
}),
retryCount);
}
catch (Exception e)
{
AppTrace.TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to create directory {0} for copying file {1}.",
destinationDir,
sourceRelative);
throw;
}
}
// Copy the file over to its destination
try
{
BackupRestoreUtility.PerformIOWithRetries(
() =>
{
FileCopyInfo copyInfo = new FileCopyInfo
{
Source = source,
Destination = destination
};
PerformOperationUnderImpersonation(
() => CopyFileToDestination(copyInfo));
},
retryCount);
}
catch (Exception e)
{
AppTrace.TraceSource.WriteExceptionAsError(
TraceType,
e,
"File copy failed. Source: {0}, destination: {1}",
source,
destination);
throw;
}
}
