private Task DeleteLogFileAsync(string fileName) { var path = FileLogicalLog.GetFullPathToLog(this.LogFileDirectoryPath, fileName); if (FabricFile.Exists(path)) { try { FabricFile.Delete(path); } catch (Exception ex) { // TODO: Exception/HRESULT work needs to be finished: catch (FabricElementNotFoundException) FabricEvents.Events.LogManagerExceptionInfo( this.Tracer.Type, "DeleteLogFileAsync: Delete logical log: " + fileName, ex.GetType().ToString(), ex.Message, ex.HResult, ex.StackTrace); } } return(Task.FromResult(0)); }
public bool Release() { if (this.readerStream != null) { try { this.readerStream.Dispose(); this.readerStream = null; FabricFile.Delete(this.readerLockPath, deleteReadonly: true); } catch (Exception e) { this.traceSource.WriteInfo(TraceTag, "Unable to delete {0} because of {1}", this.readerLockPath, e); } } if (this.writerStream != null) { this.writerStream.Dispose(); this.writerStream = null; FabricFile.Delete(this.writerLockPath, deleteReadonly: true); } this.traceSource.WriteInfo(TraceTag, "Released {0} lock on {1}", this.operation, this.path); return(true); }
private bool DeleteFile(object context) { var filePath = (string)context; FabricFile.Delete(filePath); return(true); }
private void TryFileCompression(ref FileCopyInfo fileCopyInfo, out bool fileCompressed) { fileCompressed = false; if (fileCopyInfo.SourceFileName.EndsWith(".zip")) { // already compressed skip it. return; } string tempArchivePath = string.Empty; string actualFilePath = fileCopyInfo.SourceFullPath; string archiveEntryName = Path.GetFileNameWithoutExtension(fileCopyInfo.SourceFileName); try { FileCompressor.Compress(fileCopyInfo.SourceFullPath, archiveEntryName, out tempArchivePath); Utility.PerformIOWithRetries( () => { FabricFile.Delete(actualFilePath); }); } catch (Exception e) { // Consume the exception and upload the uncompressed file. this.TraceSource.WriteExceptionAsWarning( this.LogSourceId, e, "Failed to compress crash dump for upload"); return; } fileCopyInfo.SourceFileName = archiveEntryName + ".zip"; fileCopyInfo.SourceFullPath = tempArchivePath; fileCompressed = true; }
public void Dispose() { lock (this.disposeLock) { if (this.disposed) { return; } if (this.CheckpointFile != null) { var keyFileName = this.CheckpointFile.KeyCheckpointFileName; var valueFileName = this.CheckpointFile.ValueCheckpointFileName; this.CheckpointFile.Dispose(); if (this.CanBeDeleted) { FabricEvents.Events.FileMetadataDeleteKeyValue(this.tracer, keyFileName, valueFileName); FabricFile.Delete(keyFileName); FabricFile.Delete(valueFileName); } this.disposed = true; } } }
public static void SafeFileReplace(string currentFilePath, string newFilePath, string backupFilePath, string traceType) { if (FabricFile.Exists(backupFilePath)) { FabricFile.Delete(backupFilePath); } if (FabricFile.Exists(backupFilePath)) { TestableAssertHelper.FailInvalidData(traceType, "CheckpointFileHelper.SafeFileReplace", "!FabricFile.Exists(backupFilePath) : {0}", backupFilePath); } // Previous replace could have failed in the middle before the next metadata table file got renamed to current. if (!FabricFile.Exists(currentFilePath)) { FabricFile.Move(newFilePath, currentFilePath); } else { FabricFile.Replace(newFilePath, currentFilePath, backupFilePath, ignoreMetadataErrors: false); } if (FabricFile.Exists(backupFilePath)) { FabricFile.Delete(backupFilePath); } if (FabricFile.Exists(backupFilePath)) { TestableAssertHelper.FailInvalidData(traceType, "CheckpointFileHelper.SafeFileReplace", "!FabricFile.Exists(backupFilePath) : {0}", backupFilePath); } }
internal static void CopyFile(string source, string destination, bool overwrite, bool compressDestination) { string tempArchivePath = string.Empty; try { if (compressDestination) { string archiveEntryName = Path.GetFileNameWithoutExtension(destination); FileCompressor.Compress(source, archiveEntryName, out tempArchivePath); source = tempArchivePath; } CopyFileParameters copyParam = new CopyFileParameters() { Source = source, Destination = destination, Overwrite = true }; Utility.PerformIOWithRetries( CopyFileWorker, (object)copyParam); } finally { if (false == string.IsNullOrEmpty(tempArchivePath)) { Utility.PerformIOWithRetries( () => { FabricFile.Delete(tempArchivePath); }); } } }
public static void DeleteTempLocation(string location) { if (!string.IsNullOrEmpty(location) && FabricDirectory.Exists(location)) { FabricDirectory.Delete(location, recursive: true, deleteReadOnlyFiles: true); } else if (!string.IsNullOrEmpty(location) && FabricFile.Exists(location)) { FabricFile.Delete(location, deleteReadonly: true); } }
/// <summary> /// Moves the temporary checkpoint file to the checkpoint file. /// </summary> /// <returns>Asynchronous operation the represents the file replace.</returns> /// <remarks> /// Code depends on only being called in cases where next (tmp) checkpoint exists. /// </remarks> public static async Task SafeFileReplaceAsync( string checkpointFileName, string temporaryCheckpointFileName, string backupCheckpointFileName, string traceType) { if (string.IsNullOrEmpty(checkpointFileName)) { throw new ArgumentException("Checkpoint file name is null or empty.", "checkpointFileName"); } if (string.IsNullOrEmpty(temporaryCheckpointFileName)) { throw new ArgumentException("Temporary file name is null or empty.", "temporaryCheckpointFileName"); } if (string.IsNullOrEmpty(backupCheckpointFileName)) { throw new ArgumentException("Backup file name is null or empty.", "backupCheckpointFileName"); } // Delete previous backup, if it exists. if (FabricFile.Exists(backupCheckpointFileName)) { FabricFile.Delete(backupCheckpointFileName); } // Previous replace could have failed in the middle before the next checkpoint file got renamed to current. if (!FabricFile.Exists(checkpointFileName)) { // Validate the next file is complete (this will throw InvalidDataException if it's not valid). var nextCheckpointFile = await OpenAsync(temporaryCheckpointFileName, traceType, CancellationToken.None).ConfigureAwait(false); // Next checkpoint file is valid. Move it to be current. // MCoskun: Note that FabricFile.Move is MOVEFILE_WRITE_THROUGH by default. // Note using this flag can cause undetected dataloss. FabricFile.Move(temporaryCheckpointFileName, checkpointFileName); } else { // Current exists, so we must have gotten here only after writing a valid next checkpoint file. // MCoskun: Note that FabricFile.Move is MOVEFILE_WRITE_THROUGH by default. // NTFS guarantees that this operation will not be lost as long as MOVEFILE_WRITE_THROUGH is on. // If NTFS metadata log's tail is not flushed yet, it will be flushed before this operation is logged with FUA. // Note that ReplaceFile is not guaranteed to be persisted even after it returns. // #9291020: Using ReplaceFile here instead of MoveFile with MOVEFILE_WRITE_THROUGH can cause data-loss FabricFile.Move(checkpointFileName, backupCheckpointFileName); FabricFile.Move(temporaryCheckpointFileName, checkpointFileName); // Delete the backup file. FabricFile.Delete(backupCheckpointFileName); } }
public void ReleaseRef() { var newRefCount = Interlocked.Decrement(ref this.refCount); if (newRefCount < 0) { TestableAssertHelper.FailInvalidOperation(this.traceType, "Checkpoint.ReleaseRef", "newRefCount >= 0. newRefCount: {0}", newRefCount); } if (newRefCount == 0) { FabricFile.Delete(this.FilePath); } }
public void FabricFile_DeleteNegative() { try { LogHelper.Log("FabricFile.Delete {0}", this.badPath); FabricFile.Delete(this.badPath); Assert.Fail("should never reach here"); } catch (Exception e) { LogHelper.Log("caught exception {0}", e); Assert.IsTrue(e is IOException); } }
public static void RemoveFabricAssemblies(string localApplicationPackagePath) { var filePaths = FabricDirectory.GetFiles(localApplicationPackagePath, "*.dll", true, SearchOption.AllDirectories); foreach (var filePath in filePaths) { var fileName = Path.GetFileName(filePath); if (fabricAssemblies.Contains(fileName, StringComparer.OrdinalIgnoreCase)) { FabricFile.Delete(filePath, deleteReadonly: true); ImageBuilder.TraceSource.WriteWarning(TraceType, "File {0} has been removed from the ApplicationPackage since fabric assemblies are not allowed.", filePath); } } }
private static void GenerateChecksumFile( string fileOrDirectoryPath, BuildLayoutSpecification layoutSpecification, bool isImageStoreServiceEnabled) { var checksumFileName = layoutSpecification.GetChecksumFile(fileOrDirectoryPath); if (FabricFile.Exists(checksumFileName)) { FabricFile.Delete(checksumFileName); } var checksumValue = ChecksumUtility.ComputeHash(fileOrDirectoryPath, isImageStoreServiceEnabled); WriteStringToFile(checksumFileName, checksumValue); }
public static bool TryDeleteCopyFile(string directory, string traceType) { var filePath = GetCopyFilePath(directory); if (!FabricFile.Exists(filePath)) { return(false); } FabricFile.Delete(filePath); if (FabricFile.Exists(filePath)) { TestableAssertHelper.FailInvalidData(traceType, "CheckpointFileHelper.TryDeleteCopyFile", "!FabricFile.Exists(filePath). filePath: {0}", filePath); } return(true); }
internal override async Task <IndexingLogRecord> CreateCopyLogAsync( Epoch startingEpoch, LogicalSequenceNumber startingLsn) { var flushCallback = this.PhysicalLogWriter.CallbackManager.Callback; await this.CloseCurrentLogAsync().ConfigureAwait(false); this.CurrentLogFileAlias = this.BaseLogFileAlias + CopySuffix; try { FabricFile.Delete(FileLogicalLog.GetFullPathToLog(this.LogFileDirectoryPath, this.CurrentLogFileAlias)); } catch (Exception ex) { FabricEvents.Events.LogManager( this.Tracer.Type, "CreateCopyLog: Delete logical log: " + this.CurrentLogFileAlias + " failed: " + ex); } this.LogicalLog = await this.CreateLogFileAsync(true, CancellationToken.None).ConfigureAwait(false); var callbackManager = new PhysicalLogWriterCallbackManager( flushCallback, this.Tracer); this.PhysicalLogWriter = new PhysicalLogWriter( this.LogicalLog, callbackManager, this.Tracer, this.MaxWriteCacheSizeInMB, this.IncomingBytesRateCounterWriter, this.LogFlushBytesRateCounterWriter, this.BytesPerFlushCounterWriter, this.AvgFlushLatencyCounterWriter, this.AvgSerializationLatencyCounterWriter, false); var firstIndexingRecord = new IndexingLogRecord(startingEpoch, startingLsn, null); this.PhysicalLogWriter.InsertBufferedRecord(firstIndexingRecord); await this.PhysicalLogWriter.FlushAsync("CreateCopyLogAsync").ConfigureAwait(false); this.LogHeadRecordPosition = firstIndexingRecord.RecordPosition; return(firstIndexingRecord); }
/// <summary> /// Try to delete the file. /// </summary> /// <param name="filePath">The complete path of the file to be deleted.</param> private static void TryDeleteFile(string filePath) { try { if (FabricFile.Exists(filePath)) { FabricFile.Delete(filePath, deleteReadonly: true); } } catch (Exception e) { if (!ExceptionHandler.IsIOException(e)) { throw; } } }
internal void Delete() { try { Utility.PerformIOWithRetries( () => { FabricFile.Delete(this.TempFileName); }); } catch (Exception e) { this.traceSource.WriteExceptionAsError( this.logSourceId, e, "Failed to delete temporary file {0}.", this.TempFileName); } }
private static void DeleteTargetInformationFile(string targetInformationFile) { if (FabricFile.Exists(targetInformationFile)) { try { DeployerTrace.WriteInfo("Attempting to delete TargetInformationFile."); FabricFile.Delete(targetInformationFile, true); } catch (Exception ex) { DeployerTrace.WriteError("Failed to delete: {0}. Exception: {1}.", targetInformationFile, ex); throw; } } else { DeployerTrace.WriteWarning("TargetInformationFile does not exist."); } }
private bool DeleteFilesFromFolderWorker(object context) { DeleteFilesData data = (DeleteFilesData)context; string folder = data.FolderInfo.FolderName; DateTime cutoffTime = data.CutoffTime; IsOkToDeleteFile isOkToDelete = data.IsOkToDelete; if (false == FabricDirectory.Exists(folder)) { // Directory does not exist. Nothing more to be done here. return(true); } // Iterate over the files in the current folder and figure out which // ones need to be deleted DirectoryInfo dirInfo = new DirectoryInfo(folder); IEnumerable <FileInfo> files = dirInfo.EnumerateFiles().Where(file => file.LastWriteTimeUtc.CompareTo(cutoffTime) < 0); foreach (FileInfo file in files) { if (this.stopping) { this.traceSource.WriteInfo( this.logSourceId, "The consumer is being stopped. Therefore, no more files from folder {0} will be deleted.", folder); break; } if (null != this.OnFileEnumerated) { this.OnFileEnumerated(this, EventArgs.Empty); } if ((null != isOkToDelete) && (false == isOkToDelete(data.FolderInfo, file.Name))) { continue; } try { Utility.PerformIOWithRetries( () => { if (null != OnPreFileDeletion) { OnPreFileDeletion(this, EventArgs.Empty); } FabricFile.Delete(file.FullName); if (null != OnPostFileDeletion) { OnPostFileDeletion(this, EventArgs.Empty); } }); } catch (Exception e) { this.traceSource.WriteExceptionAsError( this.logSourceId, e, "Failed to delete file {0}.", file.FullName); } } return(true); }
/// <summary> /// Download file from XStore to SMB /// </summary> /// <param name="blobContainer">The blob container having the file.</param> /// <param name="task">The task to be performed.</param> private void TransferFileFromXStoreToSMB(CloudBlobContainer blobContainer, XStoreFileOperationTask task) { if (!this.ShouldCopy(task.SrcUri)) { return; } // download the file if (task.PartialID >= 0) { // This is already a divided work; let's go ahead and download string dstSMBPath = XStoreCommon.GetPartialFileName(task.DstUri, task.PartialID); // Delete the file if it exists string directoryName = FabricPath.GetDirectoryName(dstSMBPath); if (!FabricDirectory.Exists(directoryName)) { FabricDirectory.CreateDirectory(directoryName); } if (FabricFile.Exists(dstSMBPath)) { FabricFile.Delete(dstSMBPath, deleteReadonly: true); } var blob = blobContainer.GetBlockBlobReference(task.SrcUri); var blobWrapper = new XStoreBlobWrapper(blob); if (task.TimeoutHelper != null) { //task.TimeoutHelper.ThrowIfExpired(); blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, task.TimeoutHelper.GetRemainingTime()); } else { blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, TimeSpan.MaxValue); } } else { // we are going to download a file, which we don't know the size yet var blob = blobContainer.GetBlockBlobReference(task.SrcUri); #if !DotNetCoreClr blob.FetchAttributes(null, this.defaultRequestOption); #else blob.FetchAttributesAsync(null, this.defaultRequestOption, null).Wait(); #endif int blobLength = (int)blob.Properties.Length; // Delete the file if it exists string directoryName = FabricPath.GetDirectoryName(task.DstUri); if (!FabricDirectory.Exists(directoryName)) { FabricDirectory.CreateDirectory(directoryName); } if (FabricFile.Exists(task.DstUri)) { FabricFile.Delete(task.DstUri, deleteReadonly: true); } if (blobLength < DownloadSizeThreshold) { var blobWrapper = new XStoreBlobWrapper(blob); if (task.TimeoutHelper != null) { blobWrapper.DownloadToFile(task.DstUri, blobLength, task.TimeoutHelper.GetRemainingTime(), task.OperationContext); } else { blobWrapper.DownloadToFile(task.DstUri, blobLength, TimeSpan.MaxValue, task.OperationContext); } return; } else { // For large files we divided the work to couple threads. int numThreads = Math.Min( blobLength / ParrallelDownloadSize, ParrallelDownloadThreadCount); // Create new tasks to parallel download Queue <XStoreFileOperationTask> tasks = new Queue <XStoreFileOperationTask>(); int offset = 0; for (int i = 0; i < numThreads; i++) { int length; if (i < numThreads - 1) { length = blobLength / numThreads; } else { length = blobLength - offset; } XStoreFileOperationTask newTask = new XStoreFileOperationTask( XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB, task.SrcUri, task.DstUri, false, 0, task.TimeoutHelper); newTask.FileCopyFlag = task.FileCopyFlag; newTask.Offset = offset; newTask.Length = length; newTask.PartialID = i; tasks.Enqueue(newTask); offset += length; } // enqueue all divided tasks this.xstoreTaskPool.AddTaskToTaskQueue(tasks); // Add an EndTask to the endTaskQueue as well; to combine all downloads XStoreFileOperationTask endTask = new XStoreFileOperationTask( XStoreFileOperationTask.XStoreTaskType.CombinePartialFiles, task.SrcUri, task.DstUri, false, 0, task.TimeoutHelper); endTask.IsSucceeded = true; endTask.FileCopyFlag = task.FileCopyFlag; endTask.PartialID = numThreads; endTask.Content = task.DstUri; this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask); } } }
/// <summary> /// Renames/Moves tempSourceFilePath to destinationFilePath. /// Function retries if there is another process/thread currently /// reading from destinationFilePath. /// <para></para> /// Throws if after several retries the move wasn't successful. /// For effectivity purposes, the tempSourceFilePath and destinationFilePath should /// be located in the same directory / at least in the same volume. /// <para></para> /// The use case for using this function is to quickly move the file with retry /// so other readers don't have to wait/retry for a long time. /// e.g. File.Replace() or File.Copy() would take a long time if the file is large. /// <para></para> /// If the process/thread is stopped externally during execution of this function, /// one of the results can be that a temporary file (original destinationFilePath renamed) can stay /// orphaned on the disk and the file in destinationFilePath does not exist. /// Next time the process is started programmer should expect /// the destinationFilePath may exist but needn't. /// </summary> /// <param name="sourceFilePath"> /// File has to exist. /// It is supposed that the file is temporary, so nobody is using it. /// (so no IO conflicts/race condition is applied here) /// Path cannot be null or empty. /// </param> /// <param name="destinationFilePath"> /// File can exist but doesn't have to. /// File can be access by other threads/processes while moving - retry policy is applied. /// Path cannot be null or empty /// </param> public static void MoveFileWithRetry(string sourceFilePath, string destinationFilePath) { // Steps of atomic file switch // 1. Remove old temp file (if exists) // 2. Rename destination file to temp file (if destination file exist) // 3. Rename/Move sourceFile to destinationFile // 4. Delete old temp file string tempFileName = destinationFilePath + ".oldTemp" + DateTime.UtcNow.Ticks.ToString(CultureInfo.InvariantCulture); int maxRetryCount = 4; for (int i = 0; i <= maxRetryCount; i++) { try { // Step 1. Remove old temp file (if exists) // There shouldn't be any other process/thread using this file // Remark: If 1st attempt failed after step 2 the tempFileName // now contains the original destinationFilePath (if existed). // We are deleting the tempFileName now. if (FabricFile.Exists(tempFileName)) { FabricFile.Delete(tempFileName, deleteReadonly: true); } // Step 2. Rename destination file to temp file // Retry because there can be other threads/processes // reading/writing/moving to that file - race condition. if (FabricFile.Exists(destinationFilePath)) { FabricFile.Move(destinationFilePath, tempFileName); } // REMARK: If the process crashes/stops here // before completing the step 3 // the tempFileName will stay orphaned // and destinationFilePath will not exist. // Step 3. Rename sourceFile to destinationFile // This can fail if another process/thread meanwhile created // the destinationFilePath after it was deleted in this thread in step 2. FabricFile.Move(sourceFilePath, destinationFilePath); break; } catch (Exception e) { if ((i >= maxRetryCount) || !ExceptionHandler.IsIOException(e)) { // Step 4. Delete old temp file TryDeleteFile(tempFileName); throw; } int timeToSleep = 40; lock (FolderCopy.Randomizer) { timeToSleep = FolderCopy.Randomizer.Next(40, 70); } Thread.Sleep(timeToSleep); } } // REMARK: If the process crashes/stops here // before completing the step 4 // the tempFileName will stay orphaned. // Step 4. Delete old temp file TryDeleteFile(tempFileName); }
private bool CheckAccess(string localRoot, FileAccess access) { if (string.IsNullOrEmpty(localRoot)) { return(false); } string tempFileName = Path.Combine(localRoot, Path.GetRandomFileName()); using (FileStream fileStream = FabricFile.Open(tempFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite)) {} try { // Issue : https://github.com/dotnet/corefx/issues/17164 #if !DotNetCoreClr using (WindowsImpersonationContext impersonationContext = this.WindowsIdentity.Impersonate()) #endif { FileStream accessCheckStream = null; try { FileShare fileShareOption = FileShare.None; switch (access) { case FileAccess.Read: fileShareOption = FileShare.Read; break; case FileAccess.Write: fileShareOption = FileShare.Write; break; case FileAccess.ReadWrite: fileShareOption = FileShare.ReadWrite; break; } accessCheckStream = FabricFile.Open(tempFileName, FileMode.Open, access, fileShareOption); } catch (UnauthorizedAccessException) { return(false); } finally { if (accessCheckStream != null) { accessCheckStream.Dispose(); } } } } finally { if (FabricFile.Exists(tempFileName)) { FabricFile.Delete(tempFileName, deleteReadonly: true); } ; } return(true); }
private bool SaveToFile() { // Create a new temp file string tempFilePath = Utility.GetTempFileName(); try { // Open the temp file StreamWriter writer = null; try { Utility.PerformIOWithRetries( () => { FileStream fileStream = FabricFile.Open(tempFilePath, FileMode.Create, FileAccess.Write); #if !DotNetCoreClrLinux Helpers.SetIoPriorityHint(fileStream.SafeFileHandle, Kernel32Types.PRIORITY_HINT.IoPriorityHintVeryLow); #endif writer = new StreamWriter(fileStream); }); } catch (Exception e) { this.traceSource.WriteExceptionAsError( this.logSourceId, e, "Failed to open temp file {0} for persisting index map.", tempFilePath); return(false); } try { // Write the version information to the map file string versionString = string.Concat(VersionPrefix, this.fileFormatVersion.ToString(CultureInfo.InvariantCulture)); try { Utility.PerformIOWithRetries( () => { writer.WriteLine(versionString); }); } catch (Exception e) { this.traceSource.WriteExceptionAsError( this.logSourceId, e, "Failed to write version information to temp file {0} for persisting index map.", tempFilePath); return(false); } // Write the map records to the map file foreach (TItem item in this.dictionary.Keys) { string mapRecord = string.Concat( item.ToString(), ", ", this.dictionary[item].ToString(CultureInfo.InvariantCulture)); try { Utility.PerformIOWithRetries( () => { writer.WriteLine(mapRecord); }); } catch (Exception e) { this.traceSource.WriteExceptionAsError( this.logSourceId, e, "Failed to write record {0} to temp file {1} for persisting index map.", mapRecord, tempFilePath); return(false); } } } finally { writer.Dispose(); } // Copy the temp file as the new map file try { Utility.PerformIOWithRetries( () => { FabricFile.Copy(tempFilePath, this.fileFullPath, true); }); } catch (Exception e) { this.traceSource.WriteExceptionAsError( this.logSourceId, e, "Failed to copy file {0} to {1} for persisting index map", tempFilePath, this.fileFullPath); return(false); } this.traceSource.WriteInfo( this.logSourceId, "Index map file {0} created.", this.fileFullPath); } finally { try { Utility.PerformIOWithRetries( () => { FabricFile.Delete(tempFilePath); }); } catch (Exception e) { this.traceSource.WriteExceptionAsError( this.logSourceId, e, "Failed to delete temp file {0} which was created for persisting index map.", tempFilePath); } } return(true); }
private void CopyCallerHoldsReaderLock(string source, string destination, CopyFlag copyFlag, TimeoutHelper helper) { string destinationDirectory = FabricPath.GetDirectoryName(destination); if (!string.IsNullOrEmpty(destinationDirectory) && !FabricDirectory.Exists(destinationDirectory)) { FabricDirectory.CreateDirectory(destinationDirectory); } using (FileWriterLock writerLock = new FileWriterLock(destination)) { if (!writerLock.Acquire()) { throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, FabricErrorCode.ImageStoreAcquireFileLockFailed); } if (helper != null) { helper.ThrowIfExpired(); } if (FabricFile.Exists(source)) { // This is a file copy if (FabricFile.Exists(destination)) { FabricFile.Delete(destination, deleteReadonly: true); } int retryCount = 0; while (helper == null || !TimeoutHelper.HasExpired(helper)) { try { bool shouldOverwrite = (copyFlag != CopyFlag.AtomicCopySkipIfExists); FabricFile.Copy(source, destination, shouldOverwrite); break; } catch (UnauthorizedAccessException) { TraceSource.WriteInfo( TraceType, "Uploading {0} to {1} caused UnauthorizedAccessException. RetryCount: {2}.", source, destination, retryCount); if (retryCount++ > 3) { throw; } // This could happen when a file is marked for delete and we try to delete // it again or try to open the file. Retrying after sometime should fix the issue. Thread.Sleep(TimeSpan.FromSeconds(retryCount)); } if (helper != null) { helper.ThrowIfExpired(); } } } else { // This is a folder copy using (FolderCopy fc = new FolderCopy(copyFlag, null)) { fc.Copy(source, destination); } } } }
/// <summary> /// Remove tag from store and clear the data associated with tag. /// </summary> /// <param name="tag">Location (relative to RootUri) from where to delete the content.</param> /// <param name="timeout">The timeout for performing the delete operation.</param> public void DeleteContent(string tag, TimeSpan timeout) { TimeoutHelper helper = (timeout == TimeSpan.MaxValue) ? null : new TimeoutHelper(timeout); #if !DotNetCoreClr using (WindowsImpersonationContext impersonationContext = this.GetImpersonationContext()) #endif { try { string smbTag = this.ConvertTagToSMBPath(tag); if ((!FabricFile.Exists(smbTag)) && (!FabricDirectory.Exists(smbTag))) { return; } using (FileWriterLock fileWriterLock = new FileWriterLock(smbTag)) { if (!fileWriterLock.Acquire()) { throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, FabricErrorCode.ImageStoreAcquireFileLockFailed); } if (helper != null) { helper.ThrowIfExpired(); } if (FabricFile.Exists(smbTag)) { FabricFile.Delete(smbTag, deleteReadonly: true); } else if (FabricDirectory.Exists(smbTag)) { FabricDirectory.Delete(smbTag, recursive: true, deleteReadOnlyFiles: true); } } } catch (IOException exception) { // HResult 0x80070020: THE PROCESS CANNOT ACCESS THE FILE BECAUSE IT IS BEING USED BY ANOTHER PROCESS // This work-around is done here since when we read a file from a folder we only take a reader lock on the file and not the // entire folder. When we delete a folder, we only take writer lock on the folder and hence we might run into scenarios // where a folder is attempted to be deleted even when a file in it is being read. // Returning FabricImageStoreException, causes CM to retry the delete operation. int hrError = Marshal.GetHRForException(exception); TraceSource.WriteWarning( TraceType, "Delete of {0} failed because of {1}, hrError {2:X}.", tag, exception.Message, hrError); if (hrError == unchecked ((int)0x80070020)) { throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, exception, FabricErrorCode.ImageStoreAcquireFileLockFailed); } throw; } } }
/// <summary> /// Replaces current master table file with the next file and handles intermittent failures of ReplaceW. /// SafeFileReplaceAsync assumes that the tempFilePath already exists. /// </summary> /// <returns></returns> internal static async Task SafeFileReplaceAsync(string currentFilePath, string bkpFilePath, string tempFilePath, string traceType) { // Temp should always exist // Current or Backup may or may not exist // 1. Non exists : First ever checkpoint // 2. Current exists but no backup : Last safe file replace went through without any failure. // 3. Backup exists but no current : Last safe file replace in the replace step. Moved the current to backup but could not move temp to current. // 4. Both current and backup exist : A previous safe file replace failed after moving current to backup and temp to current but before backup could be deleted. // The CompleteCheckpointAsync was retried but as next does not exist, SafeFileReplace was not executed. Hence backup could not be completed. // A new checkpoint was taken at a later stage. Current, Backup already existed and Temp was added. So all 3 exist now. var currentExists = FabricFile.Exists(currentFilePath); var tempExists = FabricFile.Exists(tempFilePath); var bkpExists = FabricFile.Exists(bkpFilePath); FabricEvents.Events.CompleteCheckpointAsync( traceType, string.Format( "Safe file replace starting. CurrentFilePath : {0}, TempFilePath : {1}, BackupFilePath : {2}, CurrentExists : {3}, TempExists : {4}, BackupExists : {5}", currentFilePath, tempFilePath, bkpFilePath, currentExists, tempExists, bkpExists)); // SafeFileReplace assumes that the temp checkpoint file always exists Diagnostics.Assert( tempExists, traceType, string.Format("Temp checkpoint file does not exist. TempFilePath : {0}", tempFilePath)); // If the backup file exists, or the checkpoint file does not exist, then we need to validate the temp file before we delete the backup. // If the temp is corrupted, this would the best effort to save the last good checkpoint file. // In case the temp is corrupted, we throw and the replica would get stuck in an open loop. // Mitigation for this would be to copy all the checkpoint files for RCA. Then force drop the replica and do manual cleanup for the same. if (bkpExists || !currentExists) { try { // Check if the next file is valid by opening and reading it. await ValidateAsync(tempFilePath, traceType).ConfigureAwait(false); } catch (Exception ex) { FabricEvents.Events.SafeFileReplace( traceType, "MetadataManager.SafeFileReplaceAsync failed due to temp file validation check failure. Exception : ", ex.ToString()); throw; } } // If a previous iteration failed, the backup file might exist. // FabricFile.Move does not overwrite the target. If the target exists, FabricFile.Move will throw with an error code 0x800700B7. // Hence we need to remove the bkpFilePath before we start the file replace. // Fix for ICM 43706105. if (bkpExists) { FabricFile.Delete(bkpFilePath); } // Previous replace could have failed in the middle before the next metadata table file got renamed to current. if (!currentExists) { FabricFile.Move(tempFilePath, currentFilePath); } else { // Note: Using FabricFile.Replace causes security issues during subsequent FileReplace for the current checkpoint file (second FileReplace will throw AccessDenied Exception) in container scenarios. FabricFile.Move(currentFilePath, bkpFilePath); FabricFile.Move(tempFilePath, currentFilePath); // Since the move completed successfully, we need to delete the bkpFilePath. This is best-effort cleanup. FabricFile.Delete(bkpFilePath); } FabricEvents.Events.CompleteCheckpointAsync(traceType, "safe file replace: completed"); }
private void CopyTraceFileForUpload(string fileName) { // Build the destination name for the filtered trace file // // !!! WARNING !!! // The trace viewer tool parses the file names of the filtered trace files. // Changing the file name format might require a change to trace viewer as well. // !!! WARNING !!! // // If the ETL file is an active ETL file, the trace file name is of the form: // <FabricNodeID>_<etlFileName>_<TimestampOfLastEventProcessed>_<TimestampDifferentiatorOfLastEventProcessed>.dtr // // If the ETL file is an inactive ETL file, the trace file name is of the form: // <FabricNodeID>_<etlFileName>_<TimestampOfLastEventProcessed>_<Int32.MaxValue>.dtr // // Using Int32.MaxValue as a component of the trace file name makes // it easy to identify gaps in the filtered traces if the DCA is // falling behind on trace processing. Recall that an inactive ETL // file is always processed fully. Only active ETL files are processed // in chunks. Therefore, the presence of Int32.MaxValue indicates that // the corresponding ETL file is inactive and has been fully processed // by the DCA. Thus, gaps **within** an ETL file (i.e. unprocessed // chunks within the file) can be identified by the absence of a file // containing Int32.MaxValue in its name. // // It is also worth noting that ETL file names are sequentially // numbered, which helps in identifying gaps **between** ETL files // (i.e. ETL files that were not processed at all). And the use of // Int32.MaxValue is an enhancement that enables us to identify gaps // within an ETL file. Using these two concepts, we can look at a set // of filtered trace files and determine whether they are complete. // And if not complete, we can also identify where all the gaps are. string differentiator = string.Format( CultureInfo.InvariantCulture, "{0:D10}", 0); string newTraceFileName = ""; long lastEventTicks; try { if (logStartTime == 0) { // We just need the start of log tracing file generation time stamp, // for alligning with etw logs //this.logStartTime = DateTime.Parse(Path.GetFileNameWithoutExtension(fileName).Replace("_", " ")).Ticks; this.logStartTime = GetFirstEventTicks(fileName); } lastEventTicks = GetLastEventTicks(fileName); } catch (Exception e) { var fileNameDiscard = fileName + ".discard"; this.traceSource.WriteExceptionAsWarning( this.logSourceId, e, "Could not create filename for trace files for upload. Renaming file to {0}", fileNameDiscard); // Rename the file and do not process it. try { // Delete the file so that storage is not blocked FabricFile.Delete(fileName); } catch (Exception ex) { this.traceSource.WriteExceptionAsWarning( this.logSourceId, ex, "Failed to rename file to {0}", fileNameDiscard); } return; } // Create the filename which TraceViewer understands newTraceFileName = string.Format( CultureInfo.InvariantCulture, "fabric_traces_{0}_{1}_{2:D6}", fabricVersion, this.logStartTime, 1); string traceFileNamePrefix = string.Format( CultureInfo.InvariantCulture, "{0}_{1}_{2:D20}_{3}.", this.fabricNodeId, newTraceFileName, lastEventTicks, differentiator); var applicationInstanceId = ContainerEnvironment.GetContainerApplicationInstanceId(this.logSourceId); if (ContainerEnvironment.IsContainerApplication(applicationInstanceId)) { // Note that the a hash of the applicationInstanceId is being used to reduce file name length in around 70 characters // This is done to workaround PathTooLong exception in FileUploaderBase.cs since we don't have an interop for FileSystemWatcher // and .NET 4.5 used does not support long paths yet. traceFileNamePrefix = string.Format( CultureInfo.InvariantCulture, "{0}_{1:X8}_{2}_{3:D20}_{4}.", this.fabricNodeId, Path.GetFileName(applicationInstanceId).GetHashCode(), newTraceFileName, lastEventTicks, differentiator); } string traceFileNameWithoutPath = string.Concat( traceFileNamePrefix, "dtr"); string compressedTraceFileNameWithoutPath = string.Concat( traceFileNamePrefix, "dtr.zip"); string subFolder = GetTraceFileSubFolder(fileName); string traceFileDestinationPath = Path.Combine( this.csvFolder, subFolder); string traceFileDestinationName = Path.Combine( traceFileDestinationPath, this.compressCsvFiles ? compressedTraceFileNameWithoutPath : traceFileNameWithoutPath); string alternateTraceFileDestinationName = Path.Combine( traceFileDestinationPath, this.compressCsvFiles ? traceFileNameWithoutPath : compressedTraceFileNameWithoutPath); try { InternalFileSink.CopyFile(fileName, traceFileDestinationName, false, this.compressCsvFiles); FabricFile.Delete(fileName); this.traceSource.WriteInfo( this.logSourceId, "Traces are ready. They have been moved from {0} to {1}.", fileName, traceFileDestinationName); } catch (Exception e) { this.traceSource.WriteExceptionAsError( this.logSourceId, e, "Failed to move file from {0} to {1}.", fileName, traceFileDestinationName); } }
internal EtlToCsvFileWriter( ITraceEventSourceFactory traceEventSourceFactory, string logSourceId, string fabricNodeId, string etwCsvFolder, bool dtrCompressionDisabled, DiskSpaceManager diskSpaceManager, IEtlToCsvFileWriterConfigReader configReader) : base(traceEventSourceFactory, logSourceId) { this.organizeWindowsFabricTracesByType = true; this.fabricNodeId = fabricNodeId; this.dtrCompressionDisabledByConsumer = dtrCompressionDisabled; this.diskSpaceManager = diskSpaceManager; this.configReader = configReader; #if !DotNetCoreClr this.perfHelper = new EtlToCsvPerformance(this.TraceSource, this.LogSourceId); #endif // Create the directory that containing filtered traces, in case it // doesn't already exist this.filteredTraceDirName = etwCsvFolder; FabricDirectory.CreateDirectory(this.filteredTraceDirName); this.TraceSource.WriteInfo( this.LogSourceId, "Directory containing filtered ETW traces: {0}", this.filteredTraceDirName); // Create a timer to delete old logs // Figure out the retention time for the CSV files // Time after which the CSV file on disk becomes a candidate for deletion // Read this value from config every time. Do not cache it. That's how // we pick up the latest value when an update happens. // // From the above files, get the ones whose corresponding ETL files // have already been fully processed. We should delete only the files // whose corresponding ETL files have been fully processed. All other // files should be kept around because their file name gives us the // bookmark up to which we have processed events. var deletionAge = configReader.GetDtrDeletionAge(); diskSpaceManager.RegisterFolder( logSourceId, () => { // Get the filtered ETW trace files that are old enough to be deleted var dirInfo = new DirectoryInfo(this.filteredTraceDirName); return(dirInfo.EnumerateFiles(EtlConsumerConstants.FilteredEtwTraceSearchPattern, SearchOption.AllDirectories)); }, f => f.LastWriteTimeUtc < DateTime.UtcNow - deletionAge, // we don't have any indication whether work is done currently, use timer to estimate f => f.LastWriteTimeUtc >= DateTime.UtcNow - deletionAge, f => { DateTime lastEventTimeStamp; GetLastEventTimestamp(f.Name, out lastEventTimeStamp); this.lastDeletedDtrName = f.Name; this.lastDeletedDtrEventTime = lastEventTimeStamp; try { FabricFile.Delete(f.FullName); return(true); } catch (Exception) { return(false); } }); diskSpaceManager.RetentionPassCompleted += () => { this.TraceSource.WriteInfo( this.LogSourceId, "The last dtr file deleted during disk space manager pass {0} with events up til {1}.", this.lastDeletedDtrName, this.lastDeletedDtrEventTime); }; diskSpaceManager.RegisterFolder( logSourceId, () => new DirectoryInfo(this.filteredTraceDirName) .EnumerateFiles(EtlConsumerConstants.BootstrapTraceSearchPattern, SearchOption.AllDirectories), f => true, f => f.LastWriteTimeUtc >= DateTime.UtcNow - deletionAge); }
private static void SafeDeleteDirectory(string directoryName, string directoryFilter, string fileFilter) { #if DotNetCoreClr // Disable compiling on windows for now. Need to correct when porting FabricDeployer for windows. if (!Directory.Exists(directoryName) || IsDirectorySymbolicLink(directoryName) || string.Equals(directoryName, directoryFilter, StringComparison.OrdinalIgnoreCase)) { return; } // Avoid deleting any file locks var files = Directory.GetFiles(directoryName).Where( (f => !f.EndsWith(FileLock.ReaderLockExtension, StringComparison.OrdinalIgnoreCase) && !f.EndsWith(FileLock.WriterLockExtension, StringComparison.CurrentCultureIgnoreCase))); #else if (!Directory.Exists(directoryName) || IsDirectorySymbolicLink(directoryName) || string.Equals(directoryName, directoryFilter, StringComparison.InvariantCultureIgnoreCase)) { return; } // Avoid deleting any file locks var files = Directory.GetFiles(directoryName).Where( (f => !f.EndsWith(FileLock.ReaderLockExtension, StringComparison.InvariantCultureIgnoreCase) && !f.EndsWith(FileLock.WriterLockExtension, StringComparison.InvariantCultureIgnoreCase))); #endif foreach (var file in files) { if (FabricFile.Exists(file) && !string.Equals(file, fileFilter, StringComparison.OrdinalIgnoreCase)) { try { FabricFile.Delete(file, true); } catch (Exception ex) { DeployerTrace.WriteError("SafeDeleteDirectory failed deleting file: {0}. Exception: {1}.", file, ex); throw; } } } foreach (var subDirectoryName in Directory.GetDirectories(directoryName)) { SafeDeleteDirectory(subDirectoryName, directoryFilter, fileFilter); } if (!Directory.GetDirectories(directoryName).Any() && !Directory.GetFiles(directoryName).Any()) { try { TimeSpan retryInterval = TimeSpan.FromSeconds(5); int retryCount = 12; Type[] exceptionTypes = { typeof(System.IO.FileLoadException) }; Helpers.PerformWithRetry(() => { try { FabricDirectory.Delete(directoryName); } catch (Exception ex) { DeployerTrace.WriteWarning("Directory Delete failed for {0} due to: {1}.", directoryName, ex.Message); throw; } }, exceptionTypes, retryInterval, retryCount); } catch (Exception ex) { DeployerTrace.WriteError("SafeDeleteDirectory failed deleting directory: {0}. Exception: {1}.", directoryName, ex); throw; } } DeployerTrace.WriteInfo("Removed folder: {0}", directoryName); }
internal bool Update(Dictionary <string, ServicePackageTableRecord> servicePackageTable, EtwEventTimestamp latestDataTimestamp) { // Create a new temp file string tempFilePath = Utility.GetTempFileName(); string backupFilePath = string.Empty; try { // Open the temp file StreamWriter writer = null; try { Utility.PerformIOWithRetries( () => { FileStream file = FabricFile.Open(tempFilePath, FileMode.Create, FileAccess.Write); #if !DotNetCoreClrLinux Helpers.SetIoPriorityHint(file.SafeFileHandle, Kernel32Types.PRIORITY_HINT.IoPriorityHintVeryLow); #endif writer = new StreamWriter(file); }); } catch (Exception e) { Utility.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to open temp file {0}.", tempFilePath); return(false); } try { // Write the version information to the backup file try { Utility.PerformIOWithRetries( () => { writer.WriteLine(BackupFileVersionString); }); } catch (Exception e) { Utility.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to write version information to temp file {0}.", tempFilePath); return(false); } // Write the table records to the backup file foreach (string tableKey in servicePackageTable.Keys) { string tableRecord = string.Concat( servicePackageTable[tableKey].NodeName, ", ", servicePackageTable[tableKey].ApplicationInstanceId, ", ", servicePackageTable[tableKey].ApplicationRolloutVersion, ", ", servicePackageTable[tableKey].ServicePackageName, ", ", servicePackageTable[tableKey].ServiceRolloutVersion, ", ", servicePackageTable[tableKey].RunLayoutRoot); try { Utility.PerformIOWithRetries( () => { writer.WriteLine(tableRecord); }); } catch (Exception e) { Utility.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to write record {0} to temp file {1}", tableRecord, tempFilePath); return(false); } } } finally { writer.Dispose(); } // Compute the name of the backup file long timstampBinary = latestDataTimestamp.Timestamp.ToBinary(); string fileName = string.Concat( BackupFilePrefix, timstampBinary.ToString("D20", CultureInfo.InvariantCulture), "_", latestDataTimestamp.Differentiator.ToString("D10", CultureInfo.InvariantCulture), ".", BackupFileExt); backupFilePath = Path.Combine(this.backupDirectory, fileName); // Copy the temp file as the new backup file try { Utility.PerformIOWithRetries( () => { FabricFile.Copy(tempFilePath, backupFilePath, true); }); } catch (Exception e) { Utility.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to copy file {0} to {1}", tempFilePath, backupFilePath); return(false); } Utility.TraceSource.WriteInfo( TraceType, "Backup file {0} created. The backup file is valid up to timestamp {1} ({2}, {3}).", backupFilePath, latestDataTimestamp.Timestamp, latestDataTimestamp.Timestamp.Ticks, latestDataTimestamp.Differentiator); } finally { try { Utility.PerformIOWithRetries( () => { FabricFile.Delete(tempFilePath); }); } catch (Exception e) { Utility.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to delete temp file {0}", tempFilePath); } } // Update the latest backup time this.LatestBackupTime = latestDataTimestamp; // Delete older backup files string backupFilePattern = string.Concat( BackupFilePrefix, "*.", BackupFileExt); string[] backupFiles = FabricDirectory.GetFiles( this.backupDirectory, backupFilePattern); foreach (string fileToDelete in backupFiles) { if (fileToDelete.Equals( backupFilePath, StringComparison.OrdinalIgnoreCase)) { // Don't delete the current backup file continue; } try { Utility.PerformIOWithRetries( () => { FabricFile.Delete(fileToDelete); }); } catch (Exception e) { // Deletion is on a best-effort basis. Log an error and // continue. Utility.TraceSource.WriteExceptionAsError( TraceType, e, "Failed to delete old backup file {0}", fileToDelete); } } return(true); }