private static void WriteStringToFile(string fileName, string value, bool writeLine = true, Encoding encoding = null) { var directory = FabricPath.GetDirectoryName(fileName); if (!FabricDirectory.Exists(directory)) { FabricDirectory.CreateDirectory(directory); } if (encoding == null) { #if DotNetCoreClrLinux encoding = new UTF8Encoding(false); #else encoding = Encoding.GetEncoding(0); #endif } using (StreamWriter writer = new StreamWriter(FabricFile.Open(fileName, FileMode.OpenOrCreate), encoding)) { if (writeLine) { writer.WriteLine(value); } else { writer.Write(value); } #if DotNetCoreClrLinux Helpers.UpdateFilePermission(fileName); #endif } }
private void CreateDirectoryPath(string path) { string directoryName = FabricPath.GetDirectoryName(path); if ((directoryName != null) && (directoryName.Length > 0) && (!FabricDirectory.Exists(directoryName))) { FabricDirectory.CreateDirectory(directoryName); } }
private void ValidateRelativePath(string value, string source, string destination, string elementName) { if (value == null) { return; } if (!this.ShouldValidate(value)) { return; } if (Path.IsPathRooted(value)) { ImageBuilderUtility.TraceAndThrowValidationError( TraceType, StringResources.ImageBuilderError_DiagnosticValidator_AbsolutePathNotAllowed, value, source, (destination != null) ? destination : String.Empty, elementName); } string remainingPath = value; while (!String.IsNullOrEmpty(remainingPath)) { string pathPart = Path.GetFileName(remainingPath); if (pathPart.Equals(StringConstants.DoubleDot, StringComparison.Ordinal)) { ImageBuilderUtility.TraceAndThrowValidationError( TraceType, StringResources.ImageBuilderError_DiagnosticValidator_InvalidRelativePath, value, source, (destination != null) ? destination : String.Empty, elementName); } remainingPath = FabricPath.GetDirectoryName(remainingPath); } }
internal static void ValidateRelativePath(string path, string[] reservedDirectoryNames = null) { if (Path.IsPathRooted(path.TrimStart('\\'))) { ImageBuilderUtility.TraceAndThrowValidationError( TraceType, StringResources.ImageBuilderError_AbsolutePathNotAllowed, path); } string remainingPath = path; while (!string.IsNullOrEmpty(remainingPath)) { string pathPart = Path.GetFileName(remainingPath); if (pathPart.Equals(StringConstants.DoubleDot, StringComparison.Ordinal)) { ImageBuilderUtility.TraceAndThrowValidationError( TraceType, StringResources.ImageBuilderError_InvalidRelativePath, path); } remainingPath = FabricPath.GetDirectoryName(remainingPath); if (string.IsNullOrEmpty(remainingPath) && reservedDirectoryNames != null && reservedDirectoryNames.Count() > 0) { foreach (string directoryName in reservedDirectoryNames) { if (string.Compare(directoryName, pathPart, true) == 0) { ImageBuilderUtility.TraceAndThrowReservedDirectoryError( TraceType, StringResources.ImageBuilderError_ReservedDirectoryName, path, pathPart); } } } } }
public static void WriteXml <T>(string fileName, T value) { var directoryName = FabricPath.GetDirectoryName(fileName); if (!FabricDirectory.Exists(directoryName)) { FabricDirectory.CreateDirectory(directoryName); } using (var stream = FabricFile.Open(fileName, FileMode.Create, FileAccess.Write, FileShare.Read)) using (var writer = XmlWriter.Create(stream)) { var serializer = new XmlSerializer(typeof(T)); serializer.Serialize(writer, value); } #if DotNetCoreClrLinux Helpers.UpdateFilePermission(fileName); #endif }
private void CopyCallerHoldsReaderLock(string source, string destination, CopyFlag copyFlag, TimeoutHelper helper) { string destinationDirectory = FabricPath.GetDirectoryName(destination); if (!string.IsNullOrEmpty(destinationDirectory) && !FabricDirectory.Exists(destinationDirectory)) { FabricDirectory.CreateDirectory(destinationDirectory); } using (FileWriterLock writerLock = new FileWriterLock(destination)) { if (!writerLock.Acquire()) { throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, FabricErrorCode.ImageStoreAcquireFileLockFailed); } if (helper != null) { helper.ThrowIfExpired(); } if (FabricFile.Exists(source)) { // This is a file copy if (FabricFile.Exists(destination)) { FabricFile.Delete(destination, deleteReadonly: true); } int retryCount = 0; while (helper == null || !TimeoutHelper.HasExpired(helper)) { try { bool shouldOverwrite = (copyFlag != CopyFlag.AtomicCopySkipIfExists); FabricFile.Copy(source, destination, shouldOverwrite); break; } catch (UnauthorizedAccessException) { TraceSource.WriteInfo( TraceType, "Uploading {0} to {1} caused UnauthorizedAccessException. RetryCount: {2}.", source, destination, retryCount); if (retryCount++ > 3) { throw; } // This could happen when a file is marked for delete and we try to delete // it again or try to open the file. Retrying after sometime should fix the issue. Thread.Sleep(TimeSpan.FromSeconds(retryCount)); } if (helper != null) { helper.ThrowIfExpired(); } } } else { // This is a folder copy using (FolderCopy fc = new FolderCopy(copyFlag, null)) { fc.Copy(source, destination); } } } }
private bool UploadFile(string traceFileSubFolder, string filePath, string fileName) { FileCopyInfo fileCopyInfo = new FileCopyInfo() { SourceFullPath = filePath, SourceFileName = fileName, RelativeDirectoryName = traceFileSubFolder }; // Figure out the name of the directory in the local map. string localMapDestination = Path.Combine(this.localMap, fileCopyInfo.RelativeDirectoryName, fileCopyInfo.SourceFileName); string localMapDestinationDir; try { localMapDestinationDir = FabricPath.GetDirectoryName(localMapDestination); } catch (PathTooLongException e) { // The path to the local map directory is too long. Skip it. this.traceSource.WriteError( this.logSourceId, "Failed to get local map directory for uploading file {0}. Exception information: {1}", fileCopyInfo.SourceFileName, e); return(false); } // File has already been uploaded before if (true == FabricFile.Exists(localMapDestination)) { return(true); } try { Utility.PerformWithRetries( this.UploadFileWorker, fileCopyInfo, new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler), AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs, AzureBlobEtwConstants.MethodExecutionMaxRetryCount, AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs); } catch (Exception e) { this.traceSource.WriteExceptionAsError( this.logSourceId, e, "Failed to copy file {0} to Azure blob account {1}, container {2}.", fileName, this.storageAccountFactory.Connection.AccountName, this.containerName); return(false); } if (false == this.UpdateLocalMap(localMapDestinationDir, localMapDestination)) { return(false); } this.traceSource.WriteInfo( this.logSourceId, "Successfully uploaded file {0} to blob storage.", fileCopyInfo.SourceFileName); return(true); }
protected override bool CopyFileToDestination(string source, string sourceRelative, int retryCount, out bool fileSkipped, out bool fileCompressed) { fileSkipped = true; fileCompressed = false; // Get the name of the source file string sourceFileName = Path.GetFileName(sourceRelative); // Create a list to hold the directory hierarchy of the source List <string> sourceRelativeParts = new List <string>(); // Add each directory in the directory hierarchy to the list string sourceRelativePath = FabricPath.GetDirectoryName(sourceRelative); while (false == string.IsNullOrEmpty(sourceRelativePath)) { string sourceRelativePart = Path.GetFileName(sourceRelativePath); sourceRelativeParts.Add(sourceRelativePart); sourceRelativePath = FabricPath.GetDirectoryName(sourceRelativePath); } // Reverse the list, so that top-level directories appear first sourceRelativeParts.Reverse(); // Compute the directory name under the container string relativeDirectoryName = string.Join( "/", sourceRelativeParts.ToArray()); // Copy the file to the Azure blob FileCopyInfo fileCopyInfo = new FileCopyInfo() { SourceFullPath = source, SourceFileName = sourceFileName, RelativeDirectoryName = relativeDirectoryName, RetryCount = retryCount }; try { Utility.PerformWithRetries( this.CopyFileToDestinationBlob, fileCopyInfo, new RetriableOperationExceptionHandler(this.AzureStorageExceptionHandler), retryCount); } catch (Exception e) { this.TraceSource.WriteExceptionAsError( this.LogSourceId, e, "Failed to copy file {0} to Azure blob account {1}, container {2}.", source, this.storageAccountFactory.Connection.AccountName, this.containerName); return(false); } fileSkipped = false; return(true); }
/// <summary> /// Download file from XStore to SMB /// </summary> /// <param name="blobContainer">The blob container having the file.</param> /// <param name="task">The task to be performed.</param> private void TransferFileFromXStoreToSMB(CloudBlobContainer blobContainer, XStoreFileOperationTask task) { if (!this.ShouldCopy(task.SrcUri)) { return; } // download the file if (task.PartialID >= 0) { // This is already a divided work; let's go ahead and download string dstSMBPath = XStoreCommon.GetPartialFileName(task.DstUri, task.PartialID); // Delete the file if it exists string directoryName = FabricPath.GetDirectoryName(dstSMBPath); if (!FabricDirectory.Exists(directoryName)) { FabricDirectory.CreateDirectory(directoryName); } if (FabricFile.Exists(dstSMBPath)) { FabricFile.Delete(dstSMBPath, deleteReadonly: true); } var blob = blobContainer.GetBlockBlobReference(task.SrcUri); var blobWrapper = new XStoreBlobWrapper(blob); if (task.TimeoutHelper != null) { //task.TimeoutHelper.ThrowIfExpired(); blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, task.TimeoutHelper.GetRemainingTime()); } else { blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, TimeSpan.MaxValue); } } else { // we are going to download a file, which we don't know the size yet var blob = blobContainer.GetBlockBlobReference(task.SrcUri); #if !DotNetCoreClr blob.FetchAttributes(null, this.defaultRequestOption); #else blob.FetchAttributesAsync(null, this.defaultRequestOption, null).Wait(); #endif int blobLength = (int)blob.Properties.Length; // Delete the file if it exists string directoryName = FabricPath.GetDirectoryName(task.DstUri); if (!FabricDirectory.Exists(directoryName)) { FabricDirectory.CreateDirectory(directoryName); } if (FabricFile.Exists(task.DstUri)) { FabricFile.Delete(task.DstUri, deleteReadonly: true); } if (blobLength < DownloadSizeThreshold) { var blobWrapper = new XStoreBlobWrapper(blob); if (task.TimeoutHelper != null) { blobWrapper.DownloadToFile(task.DstUri, blobLength, task.TimeoutHelper.GetRemainingTime(), task.OperationContext); } else { blobWrapper.DownloadToFile(task.DstUri, blobLength, TimeSpan.MaxValue, task.OperationContext); } return; } else { // For large files we divided the work to couple threads. int numThreads = Math.Min( blobLength / ParrallelDownloadSize, ParrallelDownloadThreadCount); // Create new tasks to parallel download Queue <XStoreFileOperationTask> tasks = new Queue <XStoreFileOperationTask>(); int offset = 0; for (int i = 0; i < numThreads; i++) { int length; if (i < numThreads - 1) { length = blobLength / numThreads; } else { length = blobLength - offset; } XStoreFileOperationTask newTask = new XStoreFileOperationTask( XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB, task.SrcUri, task.DstUri, false, 0, task.TimeoutHelper); newTask.FileCopyFlag = task.FileCopyFlag; newTask.Offset = offset; newTask.Length = length; newTask.PartialID = i; tasks.Enqueue(newTask); offset += length; } // enqueue all divided tasks this.xstoreTaskPool.AddTaskToTaskQueue(tasks); // Add an EndTask to the endTaskQueue as well; to combine all downloads XStoreFileOperationTask endTask = new XStoreFileOperationTask( XStoreFileOperationTask.XStoreTaskType.CombinePartialFiles, task.SrcUri, task.DstUri, false, 0, task.TimeoutHelper); endTask.IsSucceeded = true; endTask.FileCopyFlag = task.FileCopyFlag; endTask.PartialID = numThreads; endTask.Content = task.DstUri; this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask); } } }
private bool TryUploadFileToFileShare(DateTime dueTime, string fileToCopy, int retryCount, List <FolderInfo> foldersToCopy) { bool interruptedDueToStop = false; bool interruptedDueToTimeout = false; if (this.CheckForInterruptions(dueTime, ref interruptedDueToStop, ref interruptedDueToTimeout)) { this.TraceSource.WriteInfo( this.LogSourceId, "No more files from {0} will be uploaded to {1} because {2}.", this.folderName, this.destinationId, interruptedDueToStop ? "the consumer is being stopped" : "of a timeout"); return(false); } string source = Path.Combine(this.folderName, fileToCopy); FileAttributes attr; if (!this.TryGetFileAttributes(retryCount, source, out attr)) { return(false); } if ((attr & FileAttributes.Directory) == FileAttributes.Directory) { // The path is a directory. We'll need to copy all the files // and folders inside it. FolderInfo folderToCopy = new FolderInfo { FullPath = source, RelativePath = fileToCopy }; foldersToCopy.Add(folderToCopy); return(false); } // Figure out the name of the directory in the local map. string localMapDestination = Path.Combine(this.localMap, fileToCopy); string localMapDestinationDir; try { localMapDestinationDir = FabricPath.GetDirectoryName(localMapDestination); } catch (PathTooLongException e) { // The path to the local map directory is too long. Skip it. this.TraceSource.WriteError( this.LogSourceId, "Failed to create local map directory for copying file {0}. Exception information: {1}", source, e); return(false); } bool sourceFileNotFound; DateTime sourceLastWriteTime; if (!this.TryGetLastWriteTime(retryCount, source, out sourceFileNotFound, out sourceLastWriteTime)) { return(false); } bool fileSkipped, fileCompressed = false; if (sourceFileNotFound) { // The source file was not found. Maybe it // got deleted before we had a chance to copy // it. Skip it and move on. fileSkipped = true; } else { // Copy the file over to its destination if (false == this.CopyFileToDestination(source, fileToCopy, retryCount, out fileSkipped, out fileCompressed)) { this.fileSyncNeeded = true; // Add the file back for retry next interval. this.AddFileToCopy(fileToCopy); return(false); } } if (fileSkipped) { this.TraceSource.WriteWarning( this.LogSourceId, "Errors encountered while uploading file {0} to {1}. Skipping this file.", source, this.destinationId); return(false); } if (fileCompressed) { // Note that we only change this for local Map // but not fileToCopy, as FileToCopy is used to // conditionally write an event for crash dump // based on extension below. localMapDestination = Path.ChangeExtension(localMapDestination, "zip"); } if (!this.TryUpdateLocalMap(retryCount, localMapDestinationDir, localMapDestination, sourceLastWriteTime)) { return(false); } if (this.uploadingWinFabCrashDumps) { // Before logging an event that a crash dump was found, double-check // to make sure that the file name extension is ".dmp". string extension = Path.GetExtension(fileToCopy); if (extension.Equals(".dmp", StringComparison.OrdinalIgnoreCase)) { string fileNameWithoutPath = Path.GetFileName(fileToCopy); FabricEvents.Events.WindowsFabricCrashDumpFound(fileNameWithoutPath); } } return(true); }
private void OnRemoveLogMarkerFileCreated(string containerRemoveLogMarkerFullPath) { Utility.TraceSource.WriteInfo(TraceType, "Removing container traces folder : {0}", FabricPath.GetDirectoryName(containerRemoveLogMarkerFullPath)); this.ScheduleRemovalOfContainerAppInstanceAndFolder(FabricPath.GetDirectoryName(containerRemoveLogMarkerFullPath)); }
private void OnProcessLogMarkerFileCreated(string containerProcessLogMarkerFullPath) { Utility.TraceSource.WriteInfo(TraceType, "Processing container traces folder : {0}", FabricPath.GetDirectoryName(containerProcessLogMarkerFullPath)); this.InitializeProcessingOfContainerLogsFolder(FabricPath.GetDirectoryName(containerProcessLogMarkerFullPath)); }
protected override bool CopyFileToDestination(string source, string sourceRelative, int retryCount, out bool fileSkipped, out bool fileCompressed) { // No compression fileCompressed = false; // Compute the destination path string destination = Path.Combine(this.destinationPath, sourceRelative); // Figure out the name of the directory at the destination and in the // local map. var destinationDir = FabricPath.GetDirectoryName(destination); // If the directory at the destination doesn't exist, then create it if ((false == string.IsNullOrEmpty(destinationDir)) && (false == this.PerformDestinationOperation(DirectoryExistsAtDestination, destinationDir))) { try { Utility.PerformIOWithRetries( () => { PerformDestinationOperation( CreateDirectoryAtDestination, destinationDir); }, retryCount); } catch (Exception e) { this.TraceSource.WriteExceptionAsError( this.LogSourceId, e, "Failed to create directory {0} for copying file {1}.", destinationDir, sourceRelative); fileSkipped = true; return false; } } // Determine whether we need to first copy the file to the staging folder string stagingFilePath = null; if (false == string.IsNullOrEmpty(this.stagingFolderPath)) { stagingFilePath = Path.Combine(this.stagingFolderPath, Path.GetFileName(sourceRelative)); } // Copy the file over to its destination DateTime sourceLastWriteTime = DateTime.MinValue; bool sourceFileNotFound = false; try { try { Utility.PerformIOWithRetries( () => { if (string.IsNullOrEmpty(stagingFilePath)) { this.perfHelper.ExternalOperationBegin( ExternalOperationTime.ExternalOperationType.FileShareCopy, 0); } try { // Copy the file if (false == string.IsNullOrEmpty(stagingFilePath)) { FabricFile.Copy(source, stagingFilePath, true); } else { FileCopyInfo copyInfo = new FileCopyInfo { Source = source, Destination = destination }; PerformDestinationOperation(CopyFileToDestination, copyInfo); } } catch (FileNotFoundException) { sourceFileNotFound = true; } if (string.IsNullOrEmpty(stagingFilePath)) { this.perfHelper.ExternalOperationEnd( ExternalOperationTime.ExternalOperationType.FileShareCopy, 0); this.perfHelper.FileUploaded(); } }, retryCount); } catch (Exception e) { this.TraceSource.WriteExceptionAsError( this.LogSourceId, e, "File copy failed. Source: {0}, destination: {1}", source, destination); fileSkipped = true; return false; } if (sourceFileNotFound) { // The source file was not found. Maybe it // got deleted before we had a chance to copy // it. Handle the error and move on. this.TraceSource.WriteWarning( this.LogSourceId, "File {0} could not be uploaded to {1} because it was not found.", source, destination); fileSkipped = true; return true; } if (false == string.IsNullOrEmpty(stagingFilePath)) { try { Utility.PerformIOWithRetries( () => { this.perfHelper.ExternalOperationBegin( ExternalOperationTime.ExternalOperationType.FileShareCopy, 0); FileCopyInfo copyInfo = new FileCopyInfo { Source = stagingFilePath, Destination = destination }; PerformDestinationOperation(CopyFileToDestination, copyInfo); this.perfHelper.ExternalOperationEnd( ExternalOperationTime.ExternalOperationType.FileShareCopy, 0); this.perfHelper.FileUploaded(); }, retryCount); } catch (Exception e) { this.TraceSource.WriteExceptionAsError( this.LogSourceId, e, "File copy failed. Source: {0}, destination: {1}", source, destination); fileSkipped = true; return false; } } } finally { if (false == string.IsNullOrEmpty(stagingFilePath)) { try { Utility.PerformIOWithRetries( () => { FabricFile.Delete(stagingFilePath); }, retryCount); } catch (Exception e) { this.TraceSource.WriteExceptionAsError( this.LogSourceId, e, "Failed to delete file {0}", stagingFilePath); } } } fileSkipped = false; return true; }
public void FabricFile_EndToEndPositive() { var folderPath = this.testPath; folderPath = this.ExtendPath(folderPath); var filePath = Path.Combine(folderPath, this.testFileName); Assert.IsTrue(filePath.Length > 260); LogHelper.Log("FabricDirectory.Create {0}", folderPath); FabricDirectory.CreateDirectory(folderPath); LogHelper.Log("FabricFile.Create {0}", filePath); using (StreamWriter streamWriter = new StreamWriter(FabricFile.Create(filePath))) { LogHelper.Log("Write {0}", this.testString); streamWriter.WriteLine(this.testString); } LogHelper.Log("FabricDirectory.GetDirectories {0}", this.testPath); var result = FabricDirectory.GetDirectories(this.testPath); Assert.AreEqual(1, result.Length); LogHelper.Log("FabricDirectory.GetFiles {0}", this.testPath); result = FabricDirectory.GetFiles(this.testPath); Assert.AreEqual(0, result.Length); LogHelper.Log("FabricDirectory.GetFiles {0}, AllDirectories", this.testPath); result = FabricDirectory.GetFiles(this.testPath, "*", SearchOption.AllDirectories); Assert.AreEqual(1, result.Length); LogHelper.Log("FabricDirectory.GetDirectories {0}", folderPath); result = FabricDirectory.GetDirectories(folderPath); Assert.AreEqual(0, result.Length); LogHelper.Log("FabricDirectory.GetFiles {0}", folderPath); result = FabricDirectory.GetFiles(folderPath); Assert.AreEqual(1, result.Length); LogHelper.Log("FabricFile.Open {0}", filePath); using (StreamReader streamReader = new StreamReader(FabricFile.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.None))) { string actual = streamReader.ReadLine(); LogHelper.Log("Read {0}", actual); Assert.AreEqual(this.testString, actual); } LogHelper.Log("FabricFile.GetSize {0}", filePath); long size = FabricFile.GetSize(filePath); Assert.IsTrue(size > 0); LogHelper.Log("FabricPath.GetDirectoryName {0}", filePath); string directoryName = FabricPath.GetDirectoryName(filePath); Assert.AreEqual(folderPath, directoryName); LogHelper.Log("FabricFile.GetLastWriteTime {0}", filePath); DateTime oldTime = FabricFile.GetLastWriteTime(filePath); Thread.Sleep(TimeSpan.FromSeconds(1)); using (StreamWriter streamWriter = new StreamWriter(FabricFile.Open(filePath, FileMode.Open, FileAccess.Write))) { LogHelper.Log("Write {0}", this.testString); streamWriter.WriteLine(this.testString); } DateTime newTime = FabricFile.GetLastWriteTime(filePath); Assert.IsTrue(newTime > oldTime); }