private Task DeleteLogFileAsync(string fileName)
{
var path = FileLogicalLog.GetFullPathToLog(this.LogFileDirectoryPath, fileName);
if (FabricFile.Exists(path))
{
try
{
FabricFile.Delete(path);
}
catch (Exception ex)
{
// TODO: Exception/HRESULT work needs to be finished: catch (FabricElementNotFoundException)
FabricEvents.Events.LogManagerExceptionInfo(
this.Tracer.Type,
"DeleteLogFileAsync: Delete logical log: " + fileName,
ex.GetType().ToString(),
ex.Message,
ex.HResult,
ex.StackTrace);
}
}
return(Task.FromResult(0));
}
public bool Release()
{
if (this.readerStream != null)
{
try
{
this.readerStream.Dispose();
this.readerStream = null;
FabricFile.Delete(this.readerLockPath, deleteReadonly: true);
}
catch (Exception e)
{
this.traceSource.WriteInfo(TraceTag, "Unable to delete {0} because of {1}", this.readerLockPath, e);
}
}
if (this.writerStream != null)
{
this.writerStream.Dispose();
this.writerStream = null;
FabricFile.Delete(this.writerLockPath, deleteReadonly: true);
}
this.traceSource.WriteInfo(TraceTag, "Released {0} lock on {1}", this.operation, this.path);
return(true);
}
private bool DeleteFile(object context)
{
var filePath = (string)context;
FabricFile.Delete(filePath);
return(true);
}
private void TryFileCompression(ref FileCopyInfo fileCopyInfo, out bool fileCompressed)
{
fileCompressed = false;
if (fileCopyInfo.SourceFileName.EndsWith(".zip"))
{
// already compressed skip it.
return;
}
string tempArchivePath = string.Empty;
string actualFilePath = fileCopyInfo.SourceFullPath;
string archiveEntryName = Path.GetFileNameWithoutExtension(fileCopyInfo.SourceFileName);
try
{
FileCompressor.Compress(fileCopyInfo.SourceFullPath, archiveEntryName, out tempArchivePath);
Utility.PerformIOWithRetries(
() => { FabricFile.Delete(actualFilePath); });
}
catch (Exception e)
{
// Consume the exception and upload the uncompressed file.
this.TraceSource.WriteExceptionAsWarning(
this.LogSourceId,
e,
"Failed to compress crash dump for upload");
return;
}
fileCopyInfo.SourceFileName = archiveEntryName + ".zip";
fileCopyInfo.SourceFullPath = tempArchivePath;
fileCompressed = true;
}
public void Dispose()
{
lock (this.disposeLock)
{
if (this.disposed)
{
return;
}
if (this.CheckpointFile != null)
{
var keyFileName = this.CheckpointFile.KeyCheckpointFileName;
var valueFileName = this.CheckpointFile.ValueCheckpointFileName;
this.CheckpointFile.Dispose();
if (this.CanBeDeleted)
{
FabricEvents.Events.FileMetadataDeleteKeyValue(this.tracer, keyFileName, valueFileName);
FabricFile.Delete(keyFileName);
FabricFile.Delete(valueFileName);
}
this.disposed = true;
}
}
}
public static void SafeFileReplace(string currentFilePath, string newFilePath, string backupFilePath, string traceType)
{
if (FabricFile.Exists(backupFilePath))
{
FabricFile.Delete(backupFilePath);
}
if (FabricFile.Exists(backupFilePath))
{
TestableAssertHelper.FailInvalidData(traceType, "CheckpointFileHelper.SafeFileReplace", "!FabricFile.Exists(backupFilePath) : {0}", backupFilePath);
}
// Previous replace could have failed in the middle before the next metadata table file got renamed to current.
if (!FabricFile.Exists(currentFilePath))
{
FabricFile.Move(newFilePath, currentFilePath);
}
else
{
FabricFile.Replace(newFilePath, currentFilePath, backupFilePath, ignoreMetadataErrors: false);
}
if (FabricFile.Exists(backupFilePath))
{
FabricFile.Delete(backupFilePath);
}
if (FabricFile.Exists(backupFilePath))
{
TestableAssertHelper.FailInvalidData(traceType, "CheckpointFileHelper.SafeFileReplace", "!FabricFile.Exists(backupFilePath) : {0}", backupFilePath);
}
}
internal static void CopyFile(string source, string destination, bool overwrite, bool compressDestination)
{
string tempArchivePath = string.Empty;
try
{
if (compressDestination)
{
string archiveEntryName = Path.GetFileNameWithoutExtension(destination);
FileCompressor.Compress(source, archiveEntryName, out tempArchivePath);
source = tempArchivePath;
}
CopyFileParameters copyParam = new CopyFileParameters()
{
Source = source,
Destination = destination,
Overwrite = true
};
Utility.PerformIOWithRetries(
CopyFileWorker,
(object)copyParam);
}
finally
{
if (false == string.IsNullOrEmpty(tempArchivePath))
{
Utility.PerformIOWithRetries(
() => { FabricFile.Delete(tempArchivePath); });
}
}
}
public static void DeleteTempLocation(string location)
{
if (!string.IsNullOrEmpty(location) && FabricDirectory.Exists(location))
{
FabricDirectory.Delete(location, recursive: true, deleteReadOnlyFiles: true);
}
else if (!string.IsNullOrEmpty(location) && FabricFile.Exists(location))
{
FabricFile.Delete(location, deleteReadonly: true);
}
}
/// <summary>
/// Moves the temporary checkpoint file to the checkpoint file.
/// </summary>
/// <returns>Asynchronous operation the represents the file replace.</returns>
/// <remarks>
/// Code depends on only being called in cases where next (tmp) checkpoint exists.
/// </remarks>
public static async Task SafeFileReplaceAsync(
string checkpointFileName,
string temporaryCheckpointFileName,
string backupCheckpointFileName,
string traceType)
{
if (string.IsNullOrEmpty(checkpointFileName))
{
throw new ArgumentException("Checkpoint file name is null or empty.", "checkpointFileName");
}
if (string.IsNullOrEmpty(temporaryCheckpointFileName))
{
throw new ArgumentException("Temporary file name is null or empty.", "temporaryCheckpointFileName");
}
if (string.IsNullOrEmpty(backupCheckpointFileName))
{
throw new ArgumentException("Backup file name is null or empty.", "backupCheckpointFileName");
}
// Delete previous backup, if it exists.
if (FabricFile.Exists(backupCheckpointFileName))
{
FabricFile.Delete(backupCheckpointFileName);
}
// Previous replace could have failed in the middle before the next checkpoint file got renamed to current.
if (!FabricFile.Exists(checkpointFileName))
{
// Validate the next file is complete (this will throw InvalidDataException if it's not valid).
var nextCheckpointFile = await OpenAsync(temporaryCheckpointFileName, traceType, CancellationToken.None).ConfigureAwait(false);
// Next checkpoint file is valid. Move it to be current.
// MCoskun: Note that FabricFile.Move is MOVEFILE_WRITE_THROUGH by default.
// Note using this flag can cause undetected dataloss.
FabricFile.Move(temporaryCheckpointFileName, checkpointFileName);
}
else
{
// Current exists, so we must have gotten here only after writing a valid next checkpoint file.
// MCoskun: Note that FabricFile.Move is MOVEFILE_WRITE_THROUGH by default.
// NTFS guarantees that this operation will not be lost as long as MOVEFILE_WRITE_THROUGH is on.
// If NTFS metadata log's tail is not flushed yet, it will be flushed before this operation is logged with FUA.
// Note that ReplaceFile is not guaranteed to be persisted even after it returns.
// #9291020: Using ReplaceFile here instead of MoveFile with MOVEFILE_WRITE_THROUGH can cause data-loss
FabricFile.Move(checkpointFileName, backupCheckpointFileName);
FabricFile.Move(temporaryCheckpointFileName, checkpointFileName);
// Delete the backup file.
FabricFile.Delete(backupCheckpointFileName);
}
}
public void ReleaseRef()
{
var newRefCount = Interlocked.Decrement(ref this.refCount);
if (newRefCount < 0)
{
TestableAssertHelper.FailInvalidOperation(this.traceType, "Checkpoint.ReleaseRef", "newRefCount >= 0. newRefCount: {0}", newRefCount);
}
if (newRefCount == 0)
{
FabricFile.Delete(this.FilePath);
}
}
public void FabricFile_DeleteNegative()
{
try
{
LogHelper.Log("FabricFile.Delete {0}", this.badPath);
FabricFile.Delete(this.badPath);
Assert.Fail("should never reach here");
}
catch (Exception e)
{
LogHelper.Log("caught exception {0}", e);
Assert.IsTrue(e is IOException);
}
}
public static void RemoveFabricAssemblies(string localApplicationPackagePath)
{
var filePaths = FabricDirectory.GetFiles(localApplicationPackagePath, "*.dll", true, SearchOption.AllDirectories);
foreach (var filePath in filePaths)
{
var fileName = Path.GetFileName(filePath);
if (fabricAssemblies.Contains(fileName, StringComparer.OrdinalIgnoreCase))
{
FabricFile.Delete(filePath, deleteReadonly: true);
ImageBuilder.TraceSource.WriteWarning(TraceType, "File {0} has been removed from the ApplicationPackage since fabric assemblies are not allowed.", filePath);
}
}
}
private static void GenerateChecksumFile(
string fileOrDirectoryPath,
BuildLayoutSpecification layoutSpecification,
bool isImageStoreServiceEnabled)
{
var checksumFileName = layoutSpecification.GetChecksumFile(fileOrDirectoryPath);
if (FabricFile.Exists(checksumFileName))
{
FabricFile.Delete(checksumFileName);
}
var checksumValue = ChecksumUtility.ComputeHash(fileOrDirectoryPath, isImageStoreServiceEnabled);
WriteStringToFile(checksumFileName, checksumValue);
}
public static bool TryDeleteCopyFile(string directory, string traceType)
{
var filePath = GetCopyFilePath(directory);
if (!FabricFile.Exists(filePath))
{
return(false);
}
FabricFile.Delete(filePath);
if (FabricFile.Exists(filePath))
{
TestableAssertHelper.FailInvalidData(traceType, "CheckpointFileHelper.TryDeleteCopyFile", "!FabricFile.Exists(filePath). filePath: {0}", filePath);
}
return(true);
}
internal override async Task <IndexingLogRecord> CreateCopyLogAsync(
Epoch startingEpoch,
LogicalSequenceNumber startingLsn)
{
var flushCallback = this.PhysicalLogWriter.CallbackManager.Callback;
await this.CloseCurrentLogAsync().ConfigureAwait(false);
this.CurrentLogFileAlias = this.BaseLogFileAlias + CopySuffix;
try
{
FabricFile.Delete(FileLogicalLog.GetFullPathToLog(this.LogFileDirectoryPath, this.CurrentLogFileAlias));
}
catch (Exception ex)
{
FabricEvents.Events.LogManager(
this.Tracer.Type,
"CreateCopyLog: Delete logical log: " + this.CurrentLogFileAlias + " failed: " + ex);
}
this.LogicalLog = await this.CreateLogFileAsync(true, CancellationToken.None).ConfigureAwait(false);
var callbackManager = new PhysicalLogWriterCallbackManager(
flushCallback,
this.Tracer);
this.PhysicalLogWriter = new PhysicalLogWriter(
this.LogicalLog,
callbackManager,
this.Tracer,
this.MaxWriteCacheSizeInMB,
this.IncomingBytesRateCounterWriter,
this.LogFlushBytesRateCounterWriter,
this.BytesPerFlushCounterWriter,
this.AvgFlushLatencyCounterWriter,
this.AvgSerializationLatencyCounterWriter,
false);
var firstIndexingRecord = new IndexingLogRecord(startingEpoch, startingLsn, null);
this.PhysicalLogWriter.InsertBufferedRecord(firstIndexingRecord);
await this.PhysicalLogWriter.FlushAsync("CreateCopyLogAsync").ConfigureAwait(false);
this.LogHeadRecordPosition = firstIndexingRecord.RecordPosition;
return(firstIndexingRecord);
}
/// <summary>
/// Try to delete the file.
/// </summary>
/// <param name="filePath">The complete path of the file to be deleted.</param>
private static void TryDeleteFile(string filePath)
{
try
{
if (FabricFile.Exists(filePath))
{
FabricFile.Delete(filePath, deleteReadonly: true);
}
}
catch (Exception e)
{
if (!ExceptionHandler.IsIOException(e))
{
throw;
}
}
}
internal void Delete()
{
try
{
Utility.PerformIOWithRetries(
() =>
{
FabricFile.Delete(this.TempFileName);
});
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Failed to delete temporary file {0}.",
this.TempFileName);
}
}
private static void DeleteTargetInformationFile(string targetInformationFile)
{
if (FabricFile.Exists(targetInformationFile))
{
try
{
DeployerTrace.WriteInfo("Attempting to delete TargetInformationFile.");
FabricFile.Delete(targetInformationFile, true);
}
catch (Exception ex)
{
DeployerTrace.WriteError("Failed to delete: {0}. Exception: {1}.", targetInformationFile, ex);
throw;
}
}
else
{
DeployerTrace.WriteWarning("TargetInformationFile does not exist.");
}
}
private bool DeleteFilesFromFolderWorker(object context)
{
DeleteFilesData data = (DeleteFilesData)context;
string folder = data.FolderInfo.FolderName;
DateTime cutoffTime = data.CutoffTime;
IsOkToDeleteFile isOkToDelete = data.IsOkToDelete;
if (false == FabricDirectory.Exists(folder))
{
// Directory does not exist. Nothing more to be done here.
return(true);
}
// Iterate over the files in the current folder and figure out which
// ones need to be deleted
DirectoryInfo dirInfo = new DirectoryInfo(folder);
IEnumerable <FileInfo> files = dirInfo.EnumerateFiles().Where(file => file.LastWriteTimeUtc.CompareTo(cutoffTime) < 0);
foreach (FileInfo file in files)
{
if (this.stopping)
{
this.traceSource.WriteInfo(
this.logSourceId,
"The consumer is being stopped. Therefore, no more files from folder {0} will be deleted.",
folder);
break;
}
if (null != this.OnFileEnumerated)
{
this.OnFileEnumerated(this, EventArgs.Empty);
}
if ((null != isOkToDelete) &&
(false == isOkToDelete(data.FolderInfo, file.Name)))
{
continue;
}
try
{
Utility.PerformIOWithRetries(
() =>
{
if (null != OnPreFileDeletion)
{
OnPreFileDeletion(this, EventArgs.Empty);
}
FabricFile.Delete(file.FullName);
if (null != OnPostFileDeletion)
{
OnPostFileDeletion(this, EventArgs.Empty);
}
});
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Failed to delete file {0}.",
file.FullName);
}
}
return(true);
}
/// <summary>
/// Download file from XStore to SMB
/// </summary>
/// <param name="blobContainer">The blob container having the file.</param>
/// <param name="task">The task to be performed.</param>
private void TransferFileFromXStoreToSMB(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
{
if (!this.ShouldCopy(task.SrcUri))
{
return;
}
// download the file
if (task.PartialID >= 0)
{
// This is already a divided work; let's go ahead and download
string dstSMBPath = XStoreCommon.GetPartialFileName(task.DstUri, task.PartialID);
// Delete the file if it exists
string directoryName = FabricPath.GetDirectoryName(dstSMBPath);
if (!FabricDirectory.Exists(directoryName))
{
FabricDirectory.CreateDirectory(directoryName);
}
if (FabricFile.Exists(dstSMBPath))
{
FabricFile.Delete(dstSMBPath, deleteReadonly: true);
}
var blob = blobContainer.GetBlockBlobReference(task.SrcUri);
var blobWrapper = new XStoreBlobWrapper(blob);
if (task.TimeoutHelper != null)
{
//task.TimeoutHelper.ThrowIfExpired();
blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, task.TimeoutHelper.GetRemainingTime());
}
else
{
blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, TimeSpan.MaxValue);
}
}
else
{
// we are going to download a file, which we don't know the size yet
var blob = blobContainer.GetBlockBlobReference(task.SrcUri);
#if !DotNetCoreClr
blob.FetchAttributes(null, this.defaultRequestOption);
#else
blob.FetchAttributesAsync(null, this.defaultRequestOption, null).Wait();
#endif
int blobLength = (int)blob.Properties.Length;
// Delete the file if it exists
string directoryName = FabricPath.GetDirectoryName(task.DstUri);
if (!FabricDirectory.Exists(directoryName))
{
FabricDirectory.CreateDirectory(directoryName);
}
if (FabricFile.Exists(task.DstUri))
{
FabricFile.Delete(task.DstUri, deleteReadonly: true);
}
if (blobLength < DownloadSizeThreshold)
{
var blobWrapper = new XStoreBlobWrapper(blob);
if (task.TimeoutHelper != null)
{
blobWrapper.DownloadToFile(task.DstUri, blobLength, task.TimeoutHelper.GetRemainingTime(), task.OperationContext);
}
else
{
blobWrapper.DownloadToFile(task.DstUri, blobLength, TimeSpan.MaxValue, task.OperationContext);
}
return;
}
else
{
// For large files we divided the work to couple threads.
int numThreads = Math.Min(
blobLength / ParrallelDownloadSize,
ParrallelDownloadThreadCount);
// Create new tasks to parallel download
Queue <XStoreFileOperationTask> tasks = new Queue <XStoreFileOperationTask>();
int offset = 0;
for (int i = 0; i < numThreads; i++)
{
int length;
if (i < numThreads - 1)
{
length = blobLength / numThreads;
}
else
{
length = blobLength - offset;
}
XStoreFileOperationTask newTask = new XStoreFileOperationTask(
XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB,
task.SrcUri,
task.DstUri,
false,
0,
task.TimeoutHelper);
newTask.FileCopyFlag = task.FileCopyFlag;
newTask.Offset = offset;
newTask.Length = length;
newTask.PartialID = i;
tasks.Enqueue(newTask);
offset += length;
}
// enqueue all divided tasks
this.xstoreTaskPool.AddTaskToTaskQueue(tasks);
// Add an EndTask to the endTaskQueue as well; to combine all downloads
XStoreFileOperationTask endTask = new XStoreFileOperationTask(
XStoreFileOperationTask.XStoreTaskType.CombinePartialFiles,
task.SrcUri,
task.DstUri,
false,
0,
task.TimeoutHelper);
endTask.IsSucceeded = true;
endTask.FileCopyFlag = task.FileCopyFlag;
endTask.PartialID = numThreads;
endTask.Content = task.DstUri;
this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask);
}
}
}
/// <summary>
/// Renames/Moves tempSourceFilePath to destinationFilePath.
/// Function retries if there is another process/thread currently
/// reading from destinationFilePath.
/// <para></para>
/// Throws if after several retries the move wasn't successful.
/// For effectivity purposes, the tempSourceFilePath and destinationFilePath should
/// be located in the same directory / at least in the same volume.
/// <para></para>
/// The use case for using this function is to quickly move the file with retry
/// so other readers don't have to wait/retry for a long time.
/// e.g. File.Replace() or File.Copy() would take a long time if the file is large.
/// <para></para>
/// If the process/thread is stopped externally during execution of this function,
/// one of the results can be that a temporary file (original destinationFilePath renamed) can stay
/// orphaned on the disk and the file in destinationFilePath does not exist.
/// Next time the process is started programmer should expect
/// the destinationFilePath may exist but needn't.
/// </summary>
/// <param name="sourceFilePath">
/// File has to exist.
/// It is supposed that the file is temporary, so nobody is using it.
/// (so no IO conflicts/race condition is applied here)
/// Path cannot be null or empty.
/// </param>
/// <param name="destinationFilePath">
/// File can exist but doesn't have to.
/// File can be access by other threads/processes while moving - retry policy is applied.
/// Path cannot be null or empty
/// </param>
public static void MoveFileWithRetry(string sourceFilePath, string destinationFilePath)
{
// Steps of atomic file switch
// 1. Remove old temp file (if exists)
// 2. Rename destination file to temp file (if destination file exist)
// 3. Rename/Move sourceFile to destinationFile
// 4. Delete old temp file
string tempFileName = destinationFilePath + ".oldTemp" + DateTime.UtcNow.Ticks.ToString(CultureInfo.InvariantCulture);
int maxRetryCount = 4;
for (int i = 0; i <= maxRetryCount; i++)
{
try
{
// Step 1. Remove old temp file (if exists)
// There shouldn't be any other process/thread using this file
// Remark: If 1st attempt failed after step 2 the tempFileName
// now contains the original destinationFilePath (if existed).
// We are deleting the tempFileName now.
if (FabricFile.Exists(tempFileName))
{
FabricFile.Delete(tempFileName, deleteReadonly: true);
}
// Step 2. Rename destination file to temp file
// Retry because there can be other threads/processes
// reading/writing/moving to that file - race condition.
if (FabricFile.Exists(destinationFilePath))
{
FabricFile.Move(destinationFilePath, tempFileName);
}
// REMARK: If the process crashes/stops here
// before completing the step 3
// the tempFileName will stay orphaned
// and destinationFilePath will not exist.
// Step 3. Rename sourceFile to destinationFile
// This can fail if another process/thread meanwhile created
// the destinationFilePath after it was deleted in this thread in step 2.
FabricFile.Move(sourceFilePath, destinationFilePath);
break;
}
catch (Exception e)
{
if ((i >= maxRetryCount) || !ExceptionHandler.IsIOException(e))
{
// Step 4. Delete old temp file
TryDeleteFile(tempFileName);
throw;
}
int timeToSleep = 40;
lock (FolderCopy.Randomizer)
{
timeToSleep = FolderCopy.Randomizer.Next(40, 70);
}
Thread.Sleep(timeToSleep);
}
}
// REMARK: If the process crashes/stops here
// before completing the step 4
// the tempFileName will stay orphaned.
// Step 4. Delete old temp file
TryDeleteFile(tempFileName);
}
private bool CheckAccess(string localRoot, FileAccess access)
{
if (string.IsNullOrEmpty(localRoot))
{
return(false);
}
string tempFileName = Path.Combine(localRoot, Path.GetRandomFileName());
using (FileStream fileStream = FabricFile.Open(tempFileName, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite)) {}
try
{
// Issue : https://github.com/dotnet/corefx/issues/17164
#if !DotNetCoreClr
using (WindowsImpersonationContext impersonationContext = this.WindowsIdentity.Impersonate())
#endif
{
FileStream accessCheckStream = null;
try
{
FileShare fileShareOption = FileShare.None;
switch (access)
{
case FileAccess.Read:
fileShareOption = FileShare.Read;
break;
case FileAccess.Write:
fileShareOption = FileShare.Write;
break;
case FileAccess.ReadWrite:
fileShareOption = FileShare.ReadWrite;
break;
}
accessCheckStream = FabricFile.Open(tempFileName, FileMode.Open, access, fileShareOption);
}
catch (UnauthorizedAccessException)
{
return(false);
}
finally
{
if (accessCheckStream != null)
{
accessCheckStream.Dispose();
}
}
}
}
finally
{
if (FabricFile.Exists(tempFileName))
{
FabricFile.Delete(tempFileName, deleteReadonly: true);
}
;
}
return(true);
}
private bool SaveToFile()
{
// Create a new temp file
string tempFilePath = Utility.GetTempFileName();
try
{
// Open the temp file
StreamWriter writer = null;
try
{
Utility.PerformIOWithRetries(
() =>
{
FileStream fileStream = FabricFile.Open(tempFilePath, FileMode.Create, FileAccess.Write);
#if !DotNetCoreClrLinux
Helpers.SetIoPriorityHint(fileStream.SafeFileHandle, Kernel32Types.PRIORITY_HINT.IoPriorityHintVeryLow);
#endif
writer = new StreamWriter(fileStream);
});
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Failed to open temp file {0} for persisting index map.",
tempFilePath);
return(false);
}
try
{
// Write the version information to the map file
string versionString = string.Concat(VersionPrefix, this.fileFormatVersion.ToString(CultureInfo.InvariantCulture));
try
{
Utility.PerformIOWithRetries(
() =>
{
writer.WriteLine(versionString);
});
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Failed to write version information to temp file {0} for persisting index map.",
tempFilePath);
return(false);
}
// Write the map records to the map file
foreach (TItem item in this.dictionary.Keys)
{
string mapRecord = string.Concat(
item.ToString(),
", ",
this.dictionary[item].ToString(CultureInfo.InvariantCulture));
try
{
Utility.PerformIOWithRetries(
() =>
{
writer.WriteLine(mapRecord);
});
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Failed to write record {0} to temp file {1} for persisting index map.",
mapRecord,
tempFilePath);
return(false);
}
}
}
finally
{
writer.Dispose();
}
// Copy the temp file as the new map file
try
{
Utility.PerformIOWithRetries(
() =>
{
FabricFile.Copy(tempFilePath, this.fileFullPath, true);
});
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Failed to copy file {0} to {1} for persisting index map",
tempFilePath,
this.fileFullPath);
return(false);
}
this.traceSource.WriteInfo(
this.logSourceId,
"Index map file {0} created.",
this.fileFullPath);
}
finally
{
try
{
Utility.PerformIOWithRetries(
() =>
{
FabricFile.Delete(tempFilePath);
});
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Failed to delete temp file {0} which was created for persisting index map.",
tempFilePath);
}
}
return(true);
}
private void CopyCallerHoldsReaderLock(string source, string destination, CopyFlag copyFlag, TimeoutHelper helper)
{
string destinationDirectory = FabricPath.GetDirectoryName(destination);
if (!string.IsNullOrEmpty(destinationDirectory) && !FabricDirectory.Exists(destinationDirectory))
{
FabricDirectory.CreateDirectory(destinationDirectory);
}
using (FileWriterLock writerLock = new FileWriterLock(destination))
{
if (!writerLock.Acquire())
{
throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, FabricErrorCode.ImageStoreAcquireFileLockFailed);
}
if (helper != null)
{
helper.ThrowIfExpired();
}
if (FabricFile.Exists(source))
{
// This is a file copy
if (FabricFile.Exists(destination))
{
FabricFile.Delete(destination, deleteReadonly: true);
}
int retryCount = 0;
while (helper == null || !TimeoutHelper.HasExpired(helper))
{
try
{
bool shouldOverwrite = (copyFlag != CopyFlag.AtomicCopySkipIfExists);
FabricFile.Copy(source, destination, shouldOverwrite);
break;
}
catch (UnauthorizedAccessException)
{
TraceSource.WriteInfo(
TraceType,
"Uploading {0} to {1} caused UnauthorizedAccessException. RetryCount: {2}.",
source,
destination,
retryCount);
if (retryCount++ > 3)
{
throw;
}
// This could happen when a file is marked for delete and we try to delete
// it again or try to open the file. Retrying after sometime should fix the issue.
Thread.Sleep(TimeSpan.FromSeconds(retryCount));
}
if (helper != null)
{
helper.ThrowIfExpired();
}
}
}
else
{
// This is a folder copy
using (FolderCopy fc = new FolderCopy(copyFlag, null))
{
fc.Copy(source, destination);
}
}
}
}
/// <summary>
/// Remove tag from store and clear the data associated with tag.
/// </summary>
/// <param name="tag">Location (relative to RootUri) from where to delete the content.</param>
/// <param name="timeout">The timeout for performing the delete operation.</param>
public void DeleteContent(string tag, TimeSpan timeout)
{
TimeoutHelper helper = (timeout == TimeSpan.MaxValue) ? null : new TimeoutHelper(timeout);
#if !DotNetCoreClr
using (WindowsImpersonationContext impersonationContext = this.GetImpersonationContext())
#endif
{
try
{
string smbTag = this.ConvertTagToSMBPath(tag);
if ((!FabricFile.Exists(smbTag)) &&
(!FabricDirectory.Exists(smbTag)))
{
return;
}
using (FileWriterLock fileWriterLock = new FileWriterLock(smbTag))
{
if (!fileWriterLock.Acquire())
{
throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, FabricErrorCode.ImageStoreAcquireFileLockFailed);
}
if (helper != null)
{
helper.ThrowIfExpired();
}
if (FabricFile.Exists(smbTag))
{
FabricFile.Delete(smbTag, deleteReadonly: true);
}
else if (FabricDirectory.Exists(smbTag))
{
FabricDirectory.Delete(smbTag, recursive: true, deleteReadOnlyFiles: true);
}
}
}
catch (IOException exception)
{
// HResult 0x80070020: THE PROCESS CANNOT ACCESS THE FILE BECAUSE IT IS BEING USED BY ANOTHER PROCESS
// This work-around is done here since when we read a file from a folder we only take a reader lock on the file and not the
// entire folder. When we delete a folder, we only take writer lock on the folder and hence we might run into scenarios
// where a folder is attempted to be deleted even when a file in it is being read.
// Returning FabricImageStoreException, causes CM to retry the delete operation.
int hrError = Marshal.GetHRForException(exception);
TraceSource.WriteWarning(
TraceType,
"Delete of {0} failed because of {1}, hrError {2:X}.",
tag,
exception.Message,
hrError);
if (hrError == unchecked ((int)0x80070020))
{
throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, exception, FabricErrorCode.ImageStoreAcquireFileLockFailed);
}
throw;
}
}
}
/// <summary>
/// Replaces current master table file with the next file and handles intermittent failures of ReplaceW.
/// SafeFileReplaceAsync assumes that the tempFilePath already exists.
/// </summary>
/// <returns></returns>
internal static async Task SafeFileReplaceAsync(string currentFilePath, string bkpFilePath, string tempFilePath, string traceType)
{
// Temp should always exist
// Current or Backup may or may not exist
// 1. Non exists : First ever checkpoint
// 2. Current exists but no backup : Last safe file replace went through without any failure.
// 3. Backup exists but no current : Last safe file replace in the replace step. Moved the current to backup but could not move temp to current.
// 4. Both current and backup exist : A previous safe file replace failed after moving current to backup and temp to current but before backup could be deleted.
// The CompleteCheckpointAsync was retried but as next does not exist, SafeFileReplace was not executed. Hence backup could not be completed.
// A new checkpoint was taken at a later stage. Current, Backup already existed and Temp was added. So all 3 exist now.
var currentExists = FabricFile.Exists(currentFilePath);
var tempExists = FabricFile.Exists(tempFilePath);
var bkpExists = FabricFile.Exists(bkpFilePath);
FabricEvents.Events.CompleteCheckpointAsync(
traceType,
string.Format(
"Safe file replace starting. CurrentFilePath : {0}, TempFilePath : {1}, BackupFilePath : {2}, CurrentExists : {3}, TempExists : {4}, BackupExists : {5}",
currentFilePath,
tempFilePath,
bkpFilePath,
currentExists,
tempExists,
bkpExists));
// SafeFileReplace assumes that the temp checkpoint file always exists
Diagnostics.Assert(
tempExists,
traceType,
string.Format("Temp checkpoint file does not exist. TempFilePath : {0}", tempFilePath));
// If the backup file exists, or the checkpoint file does not exist, then we need to validate the temp file before we delete the backup.
// If the temp is corrupted, this would the best effort to save the last good checkpoint file.
// In case the temp is corrupted, we throw and the replica would get stuck in an open loop.
// Mitigation for this would be to copy all the checkpoint files for RCA. Then force drop the replica and do manual cleanup for the same.
if (bkpExists || !currentExists)
{
try
{
// Check if the next file is valid by opening and reading it.
await ValidateAsync(tempFilePath, traceType).ConfigureAwait(false);
}
catch (Exception ex)
{
FabricEvents.Events.SafeFileReplace(
traceType,
"MetadataManager.SafeFileReplaceAsync failed due to temp file validation check failure. Exception : ",
ex.ToString());
throw;
}
}
// If a previous iteration failed, the backup file might exist.
// FabricFile.Move does not overwrite the target. If the target exists, FabricFile.Move will throw with an error code 0x800700B7.
// Hence we need to remove the bkpFilePath before we start the file replace.
// Fix for ICM 43706105.
if (bkpExists)
{
FabricFile.Delete(bkpFilePath);
}
// Previous replace could have failed in the middle before the next metadata table file got renamed to current.
if (!currentExists)
{
FabricFile.Move(tempFilePath, currentFilePath);
}
else
{
// Note: Using FabricFile.Replace causes security issues during subsequent FileReplace for the current checkpoint file (second FileReplace will throw AccessDenied Exception) in container scenarios.
FabricFile.Move(currentFilePath, bkpFilePath);
FabricFile.Move(tempFilePath, currentFilePath);
// Since the move completed successfully, we need to delete the bkpFilePath. This is best-effort cleanup.
FabricFile.Delete(bkpFilePath);
}
FabricEvents.Events.CompleteCheckpointAsync(traceType, "safe file replace: completed");
}
private void CopyTraceFileForUpload(string fileName)
{
// Build the destination name for the filtered trace file
//
// !!! WARNING !!!
// The trace viewer tool parses the file names of the filtered trace files.
// Changing the file name format might require a change to trace viewer as well.
// !!! WARNING !!!
//
// If the ETL file is an active ETL file, the trace file name is of the form:
// <FabricNodeID>_<etlFileName>_<TimestampOfLastEventProcessed>_<TimestampDifferentiatorOfLastEventProcessed>.dtr
//
// If the ETL file is an inactive ETL file, the trace file name is of the form:
// <FabricNodeID>_<etlFileName>_<TimestampOfLastEventProcessed>_<Int32.MaxValue>.dtr
//
// Using Int32.MaxValue as a component of the trace file name makes
// it easy to identify gaps in the filtered traces if the DCA is
// falling behind on trace processing. Recall that an inactive ETL
// file is always processed fully. Only active ETL files are processed
// in chunks. Therefore, the presence of Int32.MaxValue indicates that
// the corresponding ETL file is inactive and has been fully processed
// by the DCA. Thus, gaps **within** an ETL file (i.e. unprocessed
// chunks within the file) can be identified by the absence of a file
// containing Int32.MaxValue in its name.
//
// It is also worth noting that ETL file names are sequentially
// numbered, which helps in identifying gaps **between** ETL files
// (i.e. ETL files that were not processed at all). And the use of
// Int32.MaxValue is an enhancement that enables us to identify gaps
// within an ETL file. Using these two concepts, we can look at a set
// of filtered trace files and determine whether they are complete.
// And if not complete, we can also identify where all the gaps are.
string differentiator = string.Format(
CultureInfo.InvariantCulture,
"{0:D10}",
0);
string newTraceFileName = "";
long lastEventTicks;
try
{
if (logStartTime == 0)
{
// We just need the start of log tracing file generation time stamp,
// for alligning with etw logs
//this.logStartTime = DateTime.Parse(Path.GetFileNameWithoutExtension(fileName).Replace("_", " ")).Ticks;
this.logStartTime = GetFirstEventTicks(fileName);
}
lastEventTicks = GetLastEventTicks(fileName);
}
catch (Exception e)
{
var fileNameDiscard = fileName + ".discard";
this.traceSource.WriteExceptionAsWarning(
this.logSourceId,
e,
"Could not create filename for trace files for upload. Renaming file to {0}",
fileNameDiscard);
// Rename the file and do not process it.
try
{
// Delete the file so that storage is not blocked
FabricFile.Delete(fileName);
}
catch (Exception ex)
{
this.traceSource.WriteExceptionAsWarning(
this.logSourceId,
ex,
"Failed to rename file to {0}",
fileNameDiscard);
}
return;
}
// Create the filename which TraceViewer understands
newTraceFileName = string.Format(
CultureInfo.InvariantCulture,
"fabric_traces_{0}_{1}_{2:D6}",
fabricVersion,
this.logStartTime,
1);
string traceFileNamePrefix = string.Format(
CultureInfo.InvariantCulture,
"{0}_{1}_{2:D20}_{3}.",
this.fabricNodeId,
newTraceFileName,
lastEventTicks,
differentiator);
var applicationInstanceId = ContainerEnvironment.GetContainerApplicationInstanceId(this.logSourceId);
if (ContainerEnvironment.IsContainerApplication(applicationInstanceId))
{
// Note that the a hash of the applicationInstanceId is being used to reduce file name length in around 70 characters
// This is done to workaround PathTooLong exception in FileUploaderBase.cs since we don't have an interop for FileSystemWatcher
// and .NET 4.5 used does not support long paths yet.
traceFileNamePrefix = string.Format(
CultureInfo.InvariantCulture,
"{0}_{1:X8}_{2}_{3:D20}_{4}.",
this.fabricNodeId,
Path.GetFileName(applicationInstanceId).GetHashCode(),
newTraceFileName,
lastEventTicks,
differentiator);
}
string traceFileNameWithoutPath = string.Concat(
traceFileNamePrefix,
"dtr");
string compressedTraceFileNameWithoutPath = string.Concat(
traceFileNamePrefix,
"dtr.zip");
string subFolder = GetTraceFileSubFolder(fileName);
string traceFileDestinationPath = Path.Combine(
this.csvFolder,
subFolder);
string traceFileDestinationName = Path.Combine(
traceFileDestinationPath,
this.compressCsvFiles ?
compressedTraceFileNameWithoutPath :
traceFileNameWithoutPath);
string alternateTraceFileDestinationName = Path.Combine(
traceFileDestinationPath,
this.compressCsvFiles ?
traceFileNameWithoutPath :
compressedTraceFileNameWithoutPath);
try
{
InternalFileSink.CopyFile(fileName, traceFileDestinationName, false, this.compressCsvFiles);
FabricFile.Delete(fileName);
this.traceSource.WriteInfo(
this.logSourceId,
"Traces are ready. They have been moved from {0} to {1}.",
fileName,
traceFileDestinationName);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Failed to move file from {0} to {1}.",
fileName,
traceFileDestinationName);
}
}
internal EtlToCsvFileWriter(
ITraceEventSourceFactory traceEventSourceFactory,
string logSourceId,
string fabricNodeId,
string etwCsvFolder,
bool dtrCompressionDisabled,
DiskSpaceManager diskSpaceManager,
IEtlToCsvFileWriterConfigReader configReader)
: base(traceEventSourceFactory, logSourceId)
{
this.organizeWindowsFabricTracesByType = true;
this.fabricNodeId = fabricNodeId;
this.dtrCompressionDisabledByConsumer = dtrCompressionDisabled;
this.diskSpaceManager = diskSpaceManager;
this.configReader = configReader;
#if !DotNetCoreClr
this.perfHelper = new EtlToCsvPerformance(this.TraceSource, this.LogSourceId);
#endif
// Create the directory that containing filtered traces, in case it
// doesn't already exist
this.filteredTraceDirName = etwCsvFolder;
FabricDirectory.CreateDirectory(this.filteredTraceDirName);
this.TraceSource.WriteInfo(
this.LogSourceId,
"Directory containing filtered ETW traces: {0}",
this.filteredTraceDirName);
// Create a timer to delete old logs
// Figure out the retention time for the CSV files
// Time after which the CSV file on disk becomes a candidate for deletion
// Read this value from config every time. Do not cache it. That's how
// we pick up the latest value when an update happens.
//
// From the above files, get the ones whose corresponding ETL files
// have already been fully processed. We should delete only the files
// whose corresponding ETL files have been fully processed. All other
// files should be kept around because their file name gives us the
// bookmark up to which we have processed events.
var deletionAge = configReader.GetDtrDeletionAge();
diskSpaceManager.RegisterFolder(
logSourceId,
() =>
{
// Get the filtered ETW trace files that are old enough to be deleted
var dirInfo = new DirectoryInfo(this.filteredTraceDirName);
return(dirInfo.EnumerateFiles(EtlConsumerConstants.FilteredEtwTraceSearchPattern, SearchOption.AllDirectories));
},
f => f.LastWriteTimeUtc < DateTime.UtcNow - deletionAge, // we don't have any indication whether work is done currently, use timer to estimate
f => f.LastWriteTimeUtc >= DateTime.UtcNow - deletionAge,
f =>
{
DateTime lastEventTimeStamp;
GetLastEventTimestamp(f.Name, out lastEventTimeStamp);
this.lastDeletedDtrName = f.Name;
this.lastDeletedDtrEventTime = lastEventTimeStamp;
try
{
FabricFile.Delete(f.FullName);
return(true);
}
catch (Exception)
{
return(false);
}
});
diskSpaceManager.RetentionPassCompleted += () =>
{
this.TraceSource.WriteInfo(
this.LogSourceId,
"The last dtr file deleted during disk space manager pass {0} with events up til {1}.",
this.lastDeletedDtrName,
this.lastDeletedDtrEventTime);
};
diskSpaceManager.RegisterFolder(
logSourceId,
() => new DirectoryInfo(this.filteredTraceDirName)
.EnumerateFiles(EtlConsumerConstants.BootstrapTraceSearchPattern, SearchOption.AllDirectories),
f => true,
f => f.LastWriteTimeUtc >= DateTime.UtcNow - deletionAge);
}
private static void SafeDeleteDirectory(string directoryName, string directoryFilter, string fileFilter)
{
#if DotNetCoreClr // Disable compiling on windows for now. Need to correct when porting FabricDeployer for windows.
if (!Directory.Exists(directoryName) ||
IsDirectorySymbolicLink(directoryName) ||
string.Equals(directoryName, directoryFilter, StringComparison.OrdinalIgnoreCase))
{
return;
}
// Avoid deleting any file locks
var files = Directory.GetFiles(directoryName).Where(
(f => !f.EndsWith(FileLock.ReaderLockExtension, StringComparison.OrdinalIgnoreCase) && !f.EndsWith(FileLock.WriterLockExtension, StringComparison.CurrentCultureIgnoreCase)));
#else
if (!Directory.Exists(directoryName) ||
IsDirectorySymbolicLink(directoryName) ||
string.Equals(directoryName, directoryFilter, StringComparison.InvariantCultureIgnoreCase))
{
return;
}
// Avoid deleting any file locks
var files = Directory.GetFiles(directoryName).Where(
(f => !f.EndsWith(FileLock.ReaderLockExtension, StringComparison.InvariantCultureIgnoreCase) && !f.EndsWith(FileLock.WriterLockExtension, StringComparison.InvariantCultureIgnoreCase)));
#endif
foreach (var file in files)
{
if (FabricFile.Exists(file) && !string.Equals(file, fileFilter, StringComparison.OrdinalIgnoreCase))
{
try
{
FabricFile.Delete(file, true);
}
catch (Exception ex)
{
DeployerTrace.WriteError("SafeDeleteDirectory failed deleting file: {0}. Exception: {1}.", file, ex);
throw;
}
}
}
foreach (var subDirectoryName in Directory.GetDirectories(directoryName))
{
SafeDeleteDirectory(subDirectoryName, directoryFilter, fileFilter);
}
if (!Directory.GetDirectories(directoryName).Any() && !Directory.GetFiles(directoryName).Any())
{
try
{
TimeSpan retryInterval = TimeSpan.FromSeconds(5);
int retryCount = 12;
Type[] exceptionTypes = { typeof(System.IO.FileLoadException) };
Helpers.PerformWithRetry(() =>
{
try
{
FabricDirectory.Delete(directoryName);
}
catch (Exception ex)
{
DeployerTrace.WriteWarning("Directory Delete failed for {0} due to: {1}.", directoryName, ex.Message);
throw;
}
},
exceptionTypes,
retryInterval,
retryCount);
}
catch (Exception ex)
{
DeployerTrace.WriteError("SafeDeleteDirectory failed deleting directory: {0}. Exception: {1}.", directoryName, ex);
throw;
}
}
DeployerTrace.WriteInfo("Removed folder: {0}", directoryName);
}
internal bool Update(Dictionary <string, ServicePackageTableRecord> servicePackageTable, EtwEventTimestamp latestDataTimestamp)
{
// Create a new temp file
string tempFilePath = Utility.GetTempFileName();
string backupFilePath = string.Empty;
try
{
// Open the temp file
StreamWriter writer = null;
try
{
Utility.PerformIOWithRetries(
() =>
{
FileStream file = FabricFile.Open(tempFilePath, FileMode.Create, FileAccess.Write);
#if !DotNetCoreClrLinux
Helpers.SetIoPriorityHint(file.SafeFileHandle, Kernel32Types.PRIORITY_HINT.IoPriorityHintVeryLow);
#endif
writer = new StreamWriter(file);
});
}
catch (Exception e)
{
Utility.TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to open temp file {0}.",
tempFilePath);
return(false);
}
try
{
// Write the version information to the backup file
try
{
Utility.PerformIOWithRetries(
() =>
{
writer.WriteLine(BackupFileVersionString);
});
}
catch (Exception e)
{
Utility.TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to write version information to temp file {0}.",
tempFilePath);
return(false);
}
// Write the table records to the backup file
foreach (string tableKey in servicePackageTable.Keys)
{
string tableRecord = string.Concat(
servicePackageTable[tableKey].NodeName,
", ",
servicePackageTable[tableKey].ApplicationInstanceId,
", ",
servicePackageTable[tableKey].ApplicationRolloutVersion,
", ",
servicePackageTable[tableKey].ServicePackageName,
", ",
servicePackageTable[tableKey].ServiceRolloutVersion,
", ",
servicePackageTable[tableKey].RunLayoutRoot);
try
{
Utility.PerformIOWithRetries(
() =>
{
writer.WriteLine(tableRecord);
});
}
catch (Exception e)
{
Utility.TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to write record {0} to temp file {1}",
tableRecord,
tempFilePath);
return(false);
}
}
}
finally
{
writer.Dispose();
}
// Compute the name of the backup file
long timstampBinary = latestDataTimestamp.Timestamp.ToBinary();
string fileName = string.Concat(
BackupFilePrefix,
timstampBinary.ToString("D20", CultureInfo.InvariantCulture),
"_",
latestDataTimestamp.Differentiator.ToString("D10", CultureInfo.InvariantCulture),
".",
BackupFileExt);
backupFilePath = Path.Combine(this.backupDirectory, fileName);
// Copy the temp file as the new backup file
try
{
Utility.PerformIOWithRetries(
() =>
{
FabricFile.Copy(tempFilePath, backupFilePath, true);
});
}
catch (Exception e)
{
Utility.TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to copy file {0} to {1}",
tempFilePath,
backupFilePath);
return(false);
}
Utility.TraceSource.WriteInfo(
TraceType,
"Backup file {0} created. The backup file is valid up to timestamp {1} ({2}, {3}).",
backupFilePath,
latestDataTimestamp.Timestamp,
latestDataTimestamp.Timestamp.Ticks,
latestDataTimestamp.Differentiator);
}
finally
{
try
{
Utility.PerformIOWithRetries(
() =>
{
FabricFile.Delete(tempFilePath);
});
}
catch (Exception e)
{
Utility.TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to delete temp file {0}",
tempFilePath);
}
}
// Update the latest backup time
this.LatestBackupTime = latestDataTimestamp;
// Delete older backup files
string backupFilePattern = string.Concat(
BackupFilePrefix,
"*.",
BackupFileExt);
string[] backupFiles = FabricDirectory.GetFiles(
this.backupDirectory,
backupFilePattern);
foreach (string fileToDelete in backupFiles)
{
if (fileToDelete.Equals(
backupFilePath,
StringComparison.OrdinalIgnoreCase))
{
// Don't delete the current backup file
continue;
}
try
{
Utility.PerformIOWithRetries(
() =>
{
FabricFile.Delete(fileToDelete);
});
}
catch (Exception e)
{
// Deletion is on a best-effort basis. Log an error and
// continue.
Utility.TraceSource.WriteExceptionAsError(
TraceType,
e,
"Failed to delete old backup file {0}",
fileToDelete);
}
}
return(true);
}
