private bool TryCheckForFilesToBeSynced(DateTime dueTime)
{
var syncCompletedSuccessfully = true;
this.perfHelper.FileSyncPassBegin();
this.TraceSource.WriteInfo(
this.LogSourceId,
"Starting to examine folder {0} for files to synchronize with the file share ...",
this.folderName);
// Do a breadth-first traversal of the folder from which files need
// to be uploaded
FolderInfo folderInfo = new FolderInfo
{
FullPath = this.folderName,
RelativePath = string.Empty,
LocalMapPath = this.localMap
};
Queue <FolderInfo> folderQueue = new Queue <FolderInfo>();
folderQueue.Enqueue(folderInfo);
bool syncInterruptedDueToStop = false;
bool syncInterruptedDueToTimeout = false;
while (folderQueue.Count != 0)
{
if (this.CheckForInterruptions(dueTime, ref syncInterruptedDueToStop, ref syncInterruptedDueToTimeout))
{
break;
}
folderInfo = folderQueue.Dequeue();
try
{
// Create a DirectoryInfo object representing the current folder
DirectoryInfo dirInfo = new DirectoryInfo(folderInfo.FullPath);
// Add the current folder's sub-folders to the queue
IEnumerable <DirectoryInfo> subFolders = dirInfo.EnumerateDirectories();
foreach (DirectoryInfo subFolder in subFolders)
{
if (this.CheckForInterruptions(dueTime, ref syncInterruptedDueToStop, ref syncInterruptedDueToTimeout))
{
break;
}
FolderInfo subFolderInfo = new FolderInfo();
subFolderInfo.FullPath = subFolder.FullName;
subFolderInfo.RelativePath = string.IsNullOrEmpty(folderInfo.RelativePath) ?
subFolder.Name :
Path.Combine(
folderInfo.RelativePath,
subFolder.Name);
subFolderInfo.LocalMapPath = Path.Combine(
folderInfo.LocalMapPath,
subFolder.Name);
folderQueue.Enqueue(subFolderInfo);
}
// Iterate over the files in the current folder and figure out which
// ones need to be synchronized
IEnumerable <FileInfo> files = dirInfo.EnumerateFiles();
foreach (FileInfo file in files)
{
try
{
if (this.CheckForInterruptions(dueTime, ref syncInterruptedDueToStop, ref syncInterruptedDueToTimeout))
{
break;
}
bool needsSync = false;
string localMapFile = Path.Combine(
folderInfo.LocalMapPath,
file.Name);
if (false == FabricFile.Exists(localMapFile))
{
// We don't have any record of this file in our local map.
// So we definitely need to synchronize this file.
needsSync = true;
}
else
{
// We have a record of this file in our local map. Compare
// the timestamp of the file with the timestamp of its
// counterpart in the local map to determine if the file
// has changed since we last copied it to the destination.
FileInfo localMapFileInfo = new FileInfo(localMapFile);
if (localMapFileInfo.LastWriteTime.CompareTo(file.LastWriteTime) < 0)
{
// The file has changed since we last copied it to the
// destination. We need to synchronize this file.
needsSync = true;
}
}
if (needsSync)
{
// This file needs to be synchronized. Add it to the list
// of files to be copied to the file share.
this.AddFileToCopy(Path.Combine(
folderInfo.RelativePath,
file.Name));
}
this.perfHelper.FileCheckedForSync();
}
catch (IOException)
{
// Unable to read information about the file. Maybe
// it got deleted. Handle the exception and move on.
syncCompletedSuccessfully = false;
}
}
}
catch (IOException e)
{
// Exception occurred during enumeration of directories or files.
// Handle the exception and move on.
this.TraceSource.WriteError(
this.LogSourceId,
"Failed to enumerate folder {0} while looking for files to sync. Exception information: {1}",
folderInfo.FullPath,
e);
syncCompletedSuccessfully = false;
}
}
if (syncInterruptedDueToStop)
{
this.TraceSource.WriteInfo(
this.LogSourceId,
"The consumer is being stopped. Therefore, examination of folder {0} for files to synchronize with the file share was interrupted.",
this.folderName);
syncCompletedSuccessfully = false;
}
else if (syncInterruptedDueToTimeout)
{
this.TraceSource.WriteInfo(
this.LogSourceId,
"Examination of folder {0} for files to synchronize with the file share was interrupted due to timeout.",
this.folderName);
syncCompletedSuccessfully = false;
}
else
{
this.TraceSource.WriteInfo(
this.LogSourceId,
"Finished examining folder {0} for files to synchronize with the file share.",
this.folderName);
}
this.perfHelper.FileSyncPassEnd();
return(syncCompletedSuccessfully);
}
/// <summary>
/// Download file from XStore to SMB
/// </summary>
/// <param name="blobContainer">The blob container having the file.</param>
/// <param name="task">The task to be performed.</param>
private void TransferFileFromXStoreToSMB(CloudBlobContainer blobContainer, XStoreFileOperationTask task)
{
if (!this.ShouldCopy(task.SrcUri))
{
return;
}
// download the file
if (task.PartialID >= 0)
{
// This is already a divided work; let's go ahead and download
string dstSMBPath = XStoreCommon.GetPartialFileName(task.DstUri, task.PartialID);
// Delete the file if it exists
string directoryName = FabricPath.GetDirectoryName(dstSMBPath);
if (!FabricDirectory.Exists(directoryName))
{
FabricDirectory.CreateDirectory(directoryName);
}
if (FabricFile.Exists(dstSMBPath))
{
FabricFile.Delete(dstSMBPath, deleteReadonly: true);
}
var blob = blobContainer.GetBlockBlobReference(task.SrcUri);
var blobWrapper = new XStoreBlobWrapper(blob);
if (task.TimeoutHelper != null)
{
//task.TimeoutHelper.ThrowIfExpired();
blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, task.TimeoutHelper.GetRemainingTime());
}
else
{
blobWrapper.DownloadPartToFile(dstSMBPath, task.Offset, task.Length, TimeSpan.MaxValue);
}
}
else
{
// we are going to download a file, which we don't know the size yet
var blob = blobContainer.GetBlockBlobReference(task.SrcUri);
#if !DotNetCoreClr
blob.FetchAttributes(null, this.defaultRequestOption);
#else
blob.FetchAttributesAsync(null, this.defaultRequestOption, null).Wait();
#endif
int blobLength = (int)blob.Properties.Length;
// Delete the file if it exists
string directoryName = FabricPath.GetDirectoryName(task.DstUri);
if (!FabricDirectory.Exists(directoryName))
{
FabricDirectory.CreateDirectory(directoryName);
}
if (FabricFile.Exists(task.DstUri))
{
FabricFile.Delete(task.DstUri, deleteReadonly: true);
}
if (blobLength < DownloadSizeThreshold)
{
var blobWrapper = new XStoreBlobWrapper(blob);
if (task.TimeoutHelper != null)
{
blobWrapper.DownloadToFile(task.DstUri, blobLength, task.TimeoutHelper.GetRemainingTime(), task.OperationContext);
}
else
{
blobWrapper.DownloadToFile(task.DstUri, blobLength, TimeSpan.MaxValue, task.OperationContext);
}
return;
}
else
{
// For large files we divided the work to couple threads.
int numThreads = Math.Min(
blobLength / ParrallelDownloadSize,
ParrallelDownloadThreadCount);
// Create new tasks to parallel download
Queue <XStoreFileOperationTask> tasks = new Queue <XStoreFileOperationTask>();
int offset = 0;
for (int i = 0; i < numThreads; i++)
{
int length;
if (i < numThreads - 1)
{
length = blobLength / numThreads;
}
else
{
length = blobLength - offset;
}
XStoreFileOperationTask newTask = new XStoreFileOperationTask(
XStoreFileOperationTask.XStoreTaskType.CopyFromXStoreToSMB,
task.SrcUri,
task.DstUri,
false,
0,
task.TimeoutHelper);
newTask.FileCopyFlag = task.FileCopyFlag;
newTask.Offset = offset;
newTask.Length = length;
newTask.PartialID = i;
tasks.Enqueue(newTask);
offset += length;
}
// enqueue all divided tasks
this.xstoreTaskPool.AddTaskToTaskQueue(tasks);
// Add an EndTask to the endTaskQueue as well; to combine all downloads
XStoreFileOperationTask endTask = new XStoreFileOperationTask(
XStoreFileOperationTask.XStoreTaskType.CombinePartialFiles,
task.SrcUri,
task.DstUri,
false,
0,
task.TimeoutHelper);
endTask.IsSucceeded = true;
endTask.FileCopyFlag = task.FileCopyFlag;
endTask.PartialID = numThreads;
endTask.Content = task.DstUri;
this.xstoreTaskPool.AddTaskToEndTaskQueue(endTask);
}
}
}
private static bool CopyFileToDestination(object context)
{
FileCopyInfo fileCopyInfo = (FileCopyInfo)context;
FabricFile.Copy(fileCopyInfo.Source, fileCopyInfo.Destination, true);
return true;
}
public void ProvisionFabric(
string localCodePath,
string localConfigPath,
#if !DotNetCoreClrLinux && !DotNetCoreClrIOT
string configurationCsvFilePath,
#endif
string infrastructureManifestFilePath,
bool validateOnly,
TimeSpan timeout)
{
TimeoutHelper timeoutHelper = new TimeoutHelper(timeout);
#if DotNetCoreClrLinux || DotNetCoreClrIOT
ImageBuilder.TraceSource.WriteInfo(
TraceType,
"Starting ProvisionFabric. CodePath:{0}, ConfigPath:{1}, InfrastructureManifestFilePath:{2}, Timeout:{3}",
localCodePath,
localConfigPath,
infrastructureManifestFilePath,
timeoutHelper.GetRemainingTime());
#else
ImageBuilder.TraceSource.WriteInfo(
TraceType,
"Starting ProvisionFabric. CodePath:{0}, ConfigPath:{1}, ConfigurationCsvFilePath:{2}, InfrastructureManifestFilePath:{3}, Timeout:{4}",
localCodePath,
localConfigPath,
configurationCsvFilePath,
infrastructureManifestFilePath,
timeoutHelper.GetRemainingTime());
#endif
// TODO: Once FabricTest has been modified to generate InfrastructureManifest.xml file, we should not
// allow InfrastructureManifest file to be optional
if (string.IsNullOrEmpty(infrastructureManifestFilePath) || !FabricFile.Exists(infrastructureManifestFilePath))
{
infrastructureManifestFilePath = null;
ImageBuilder.TraceSource.WriteWarning(
TraceType,
"The InfrastrucutreManifestFile:{0} is not found",
infrastructureManifestFilePath);
}
string codeVersion = string.Empty, clusterManifestVersion = string.Empty;
if (!string.IsNullOrEmpty(localCodePath))
{
codeVersion = GetCodeVersion(localCodePath);
}
if (!string.IsNullOrEmpty(localConfigPath))
{
try
{
var parameters = new Dictionary <string, dynamic>
{
{ DeploymentParameters.ClusterManifestString, localConfigPath },
{ DeploymentParameters.InfrastructureManifestString, infrastructureManifestFilePath }
};
var deploymentParameters = new DeploymentParameters();
deploymentParameters.SetParameters(parameters, DeploymentOperations.ValidateClusterManifest);
DeploymentOperation.ExecuteOperation(deploymentParameters);
}
catch (Exception e)
{
ImageBuilderUtility.TraceAndThrowValidationError(
FabricProvisionOperation.TraceType,
StringResources.ImageBuilderError_ClusterManifestValidationFailed,
e.ToString(),
localConfigPath);
}
}
if (!validateOnly)
{
timeoutHelper.ThrowIfExpired();
WinFabStoreLayoutSpecification winFabLayout = WinFabStoreLayoutSpecification.Create();
if (!string.IsNullOrEmpty(localCodePath))
{
// Upload MSP file
string destinationCodePath;
if (ImageBuilderUtility.IsInstaller(localCodePath))
{
destinationCodePath = winFabLayout.GetPatchFile(codeVersion);
}
else if (ImageBuilderUtility.IsCabFile(localCodePath))
{
// Upload CAB file
destinationCodePath = winFabLayout.GetCabPatchFile(codeVersion);
}
else
{
destinationCodePath = winFabLayout.GetCodePackageFolder(codeVersion);
}
this.imageStoreWrapper.UploadContent(destinationCodePath, localCodePath, timeoutHelper.GetRemainingTime());
}
ClusterManifestType clusterManifest = null;
if (!string.IsNullOrEmpty(localConfigPath))
{
clusterManifest = ImageBuilderUtility.ReadXml <ClusterManifestType>(localConfigPath, this.validatingXmlReaderSettings);
clusterManifestVersion = clusterManifest.Version;
}
if (clusterManifest != null)
{
// Upload ClusterManifest file
ReplaceDefaultImageStoreConnectionString(clusterManifest);
string destinationConfigPath = winFabLayout.GetClusterManifestFile(clusterManifestVersion);
imageStoreWrapper.SetToStore <ClusterManifestType>(destinationConfigPath, clusterManifest, timeoutHelper.GetRemainingTime());
}
}
#if DotNetCoreClrLinux || DotNetCoreClrIOT
ImageBuilder.TraceSource.WriteInfo(
TraceType,
"Completed ProvisionFabric. CodePath:{0}, ConfigPath:{1}",
localCodePath,
localConfigPath);
#else
ImageBuilder.TraceSource.WriteInfo(
TraceType,
"Completed ProvisionFabric. CodePath:{0}, ConfigPath:{1}, ConfigurationCsvFilePath:{2}",
localCodePath,
localConfigPath,
configurationCsvFilePath);
#endif
}
public void OnEtwEventProcessingPeriodStart()
{
this.streamWriter = null;
this.perfHelper.EtlReadPassBegin();
// Build the full path to our buffered event file
string tempCacheFileName = string.Format(
CultureInfo.InvariantCulture,
"{0}{1}.{2}",
TempCacheFileNamePrefix,
DateTime.Now.Ticks,
TempCacheFileExtension);
string tempCacheFileFullPath = Path.Combine(this.etwEventCache, tempCacheFileName);
// Open the file
StreamWriter writer = null;
try
{
Utility.PerformIOWithRetries(
ctx =>
{
string fileName = ctx;
FileStream fileStream = FabricFile.Open(fileName, FileMode.Create, FileAccess.Write);
#if !DotNetCoreClr
Helpers.SetIoPriorityHint(fileStream.SafeFileHandle, Kernel32Types.PRIORITY_HINT.IoPriorityHintVeryLow);
#endif
writer = new StreamWriter(fileStream);
},
tempCacheFileFullPath);
}
catch (Exception e)
{
// Log an error and move on. No events from this pass will be
// written to the destination.
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"Failed to create a new file in the buffered event directory. None of the events from this pass will be written to the event buffer.");
return;
}
// Write the version number. This will help us read data from the file.
try
{
Utility.PerformIOWithRetries(
() =>
{
writer.WriteLine(EtlConsumerConstants.EtwEventCacheFormatVersionString);
});
}
catch (Exception e)
{
// Log an error and move on. No events from this pass will be
// written to the destination.
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"Failed to write buffered event file format version number to file {0} in the buffered event file. None of the events from this pass will be written to the event buffer.",
tempCacheFileFullPath);
return;
}
this.streamWriter = writer;
this.TraceSource.WriteInfo(
this.LogSourceId,
"Filtered ETW events for consumer will be buffered in file {0}.",
tempCacheFileFullPath);
}
/// <summary>
/// Remove tag from store and clear the data associated with tag.
/// </summary>
/// <param name="tag">Location (relative to RootUri) from where to delete the content.</param>
/// <param name="timeout">The timeout for performing the delete operation.</param>
public void DeleteContent(string tag, TimeSpan timeout)
{
TimeoutHelper helper = (timeout == TimeSpan.MaxValue) ? null : new TimeoutHelper(timeout);
#if !DotNetCoreClr
using (WindowsImpersonationContext impersonationContext = this.GetImpersonationContext())
#endif
{
try
{
string smbTag = this.ConvertTagToSMBPath(tag);
if ((!FabricFile.Exists(smbTag)) &&
(!FabricDirectory.Exists(smbTag)))
{
return;
}
using (FileWriterLock fileWriterLock = new FileWriterLock(smbTag))
{
if (!fileWriterLock.Acquire())
{
throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, FabricErrorCode.ImageStoreAcquireFileLockFailed);
}
if (helper != null)
{
helper.ThrowIfExpired();
}
if (FabricFile.Exists(smbTag))
{
FabricFile.Delete(smbTag, deleteReadonly: true);
}
else if (FabricDirectory.Exists(smbTag))
{
FabricDirectory.Delete(smbTag, recursive: true, deleteReadOnlyFiles: true);
}
}
}
catch (IOException exception)
{
// HResult 0x80070020: THE PROCESS CANNOT ACCESS THE FILE BECAUSE IT IS BEING USED BY ANOTHER PROCESS
// This work-around is done here since when we read a file from a folder we only take a reader lock on the file and not the
// entire folder. When we delete a folder, we only take writer lock on the folder and hence we might run into scenarios
// where a folder is attempted to be deleted even when a file in it is being read.
// Returning FabricImageStoreException, causes CM to retry the delete operation.
int hrError = Marshal.GetHRForException(exception);
TraceSource.WriteWarning(
TraceType,
"Delete of {0} failed because of {1}, hrError {2:X}.",
tag,
exception.Message,
hrError);
if (hrError == unchecked ((int)0x80070020))
{
throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, exception, FabricErrorCode.ImageStoreAcquireFileLockFailed);
}
throw;
}
}
}
internal static bool FileExists(string fileName)
{
return(FabricFile.Exists(fileName));
}
/// <summary>
/// Creates the bookmark file if it does not exist.
/// </summary>
/// <param name="bookmarkFolder"></param>
/// <param name="bytesWritten"></param>
/// <returns></returns>
internal bool CreateBookmarkFile(
string bookmarkFolder,
out long bytesWritten)
{
FileStream fs = null;
StreamWriter writer = null;
bytesWritten = 0;
string bookmarkFile = Path.Combine(bookmarkFolder, BookmarkFileName);
if (true == FabricFile.Exists(bookmarkFile))
{
return(true);
}
try
{
// Create the bookmark file
try
{
Utility.PerformIOWithRetries(
() =>
{
fs = new FileStream(bookmarkFile, FileMode.Create);
writer = new StreamWriter(fs);
},
this.methodExecutionInitialRetryIntervalMs,
this.methodExecutionMaxRetryCount,
this.methodExecutionMaxRetryIntervalMs);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Unable to create bookmark file {0} while processing events.",
bookmarkFile);
return(false);
}
long localBytesWritten = 0;
// Write the version information
try
{
Utility.PerformIOWithRetries(
() =>
{
writer.WriteLine(BookmarkFileFormatVersionString);
localBytesWritten += BookmarkFileFormatVersionString.Length + 2;
writer.Flush();
},
this.methodExecutionInitialRetryIntervalMs,
this.methodExecutionMaxRetryCount,
this.methodExecutionMaxRetryIntervalMs);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Unable to write version information in bookmark file {0} while processing events.",
bookmarkFile);
return(false);
}
// Write the timestamp and index of the last event read and close the file
int lastEventIndex = -1;
string lastEventReadString = string.Format("{0},{1},{2}", DateTime.MinValue.ToBinary(), DateTime.MinValue.ToString(), lastEventIndex.ToString());
try
{
Utility.PerformIOWithRetries(
() =>
{
writer.WriteLine(lastEventReadString);
localBytesWritten += lastEventReadString.Length + 2;
writer.Dispose();
fs.Dispose();
},
this.methodExecutionInitialRetryIntervalMs,
this.methodExecutionMaxRetryCount,
this.methodExecutionMaxRetryIntervalMs);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Unable to write last event read {0} in bookmark file {1} while processing events.",
lastEventReadString,
bookmarkFile);
return(false);
}
// record bytes written
bytesWritten = localBytesWritten;
}
finally
{
if (null != writer)
{
writer.Dispose();
}
if (null != fs)
{
fs.Dispose();
}
}
return(true);
}
/// <summary>
/// Gets the file position where last event index is recorded.
/// </summary>
/// <param name="bookmarkFolder"></param>
/// <param name="lastEventReadPosition"></param>
/// <param name="bytesRead"></param>
/// <returns></returns>
internal bool GetLastEventReadPosition(
string bookmarkFolder,
out long lastEventReadPosition,
out long bytesRead)
{
lastEventReadPosition = 0;
bytesRead = 0;
string bookmarkFile = Path.Combine(bookmarkFolder, BookmarkFileName);
if (false == FabricFile.Exists(bookmarkFile))
{
// Bookmark file doesn't exist
return(false);
}
StreamReader reader = null;
try
{
// Open the file
try
{
Utility.PerformIOWithRetries(
() =>
{
FileStream fileStream = FabricFile.Open(bookmarkFile, FileMode.Open, FileAccess.Read);
Helpers.SetIoPriorityHint(fileStream.SafeFileHandle, Kernel32Types.PRIORITY_HINT.IoPriorityHintVeryLow);
reader = new StreamReader(fileStream);
},
this.methodExecutionInitialRetryIntervalMs,
this.methodExecutionMaxRetryCount,
this.methodExecutionMaxRetryIntervalMs);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Unable to open bookmark file {0}.",
bookmarkFile);
return(false);
}
long localBytesRead = 0;
long streamPosition = 0;
// Get the version
string versionString = string.Empty;
try
{
Utility.PerformIOWithRetries(
() =>
{
versionString = reader.ReadLine();
localBytesRead += versionString.Length + 2;
streamPosition = localBytesRead;
},
this.methodExecutionInitialRetryIntervalMs,
this.methodExecutionMaxRetryCount,
this.methodExecutionMaxRetryIntervalMs);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Unable to read version information from bookmark file {0}.",
bookmarkFile);
return(false);
}
// Check the version
if (false == versionString.Equals(BookmarkFileFormatVersionString, StringComparison.Ordinal))
{
this.traceSource.WriteError(
this.logSourceId,
"Unexpected version string {0} encountered in bookmark file {1}.",
versionString,
bookmarkFile);
return(false);
}
// Get information about the last event that we read
string infoLine = string.Empty;
try
{
Utility.PerformIOWithRetries(
() =>
{
infoLine = reader.ReadLine();
localBytesRead += infoLine.Length + 2;
},
this.methodExecutionInitialRetryIntervalMs,
this.methodExecutionMaxRetryCount,
this.methodExecutionMaxRetryIntervalMs);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Unable to read information about last event read from bookmark file {0}.",
bookmarkFile);
return(false);
}
// record bytes read
bytesRead = localBytesRead;
lastEventReadPosition = streamPosition;
}
finally
{
if (null != reader)
{
reader.Dispose();
}
}
return(true);
}
private void TestIntersectingCopyAndReplace(bool isLongPath)
{
var folderPath = this.testPath;
if (true == isLongPath)
{
folderPath = this.ExtendPath(folderPath);
}
var filePath = Path.Combine(folderPath, this.testFileName);
Assert.IsTrue(!isLongPath || filePath.Length > 260, "file path must be greater than max path size.");
var copyFilePath = Path.Combine(folderPath, this.testHardLinkedFileName);
Assert.IsTrue(!isLongPath || copyFilePath.Length > 260, "hard linked file path must be greater than max path size.");
var replaceFilePath = Path.Combine(folderPath, this.testReplaceFileName);
Assert.IsTrue(!isLongPath || replaceFilePath.Length > 260, "replace file path must be greater than max path size.");
var backupFilePath = Path.Combine(folderPath, this.testBackupFileName);
Assert.IsTrue(!isLongPath || backupFilePath.Length > 260, "backup file path must be greater than max path size.");
LogHelper.Log("FabricDirectory.Create {0}", folderPath);
FabricDirectory.CreateDirectory(folderPath);
LogHelper.Log("FabricFile.Create {0}", filePath);
using (StreamWriter streamWriter = new StreamWriter(FabricFile.Create(filePath)))
{
LogHelper.Log("Write {0}", this.testString);
streamWriter.WriteLine(this.testString);
streamWriter.Flush();
}
using (StreamWriter streamWriter = new StreamWriter(FabricFile.Create(replaceFilePath)))
{
LogHelper.Log("Write {0}", this.testNewString);
streamWriter.WriteLine(this.testNewString);
streamWriter.Flush();
}
FabricFile.Copy(filePath, copyFilePath, false);
using (StreamReader reader = new StreamReader(FabricFile.Open(filePath, FileMode.Open, FileAccess.Read)))
{
var content = reader.ReadLine();
Assert.AreEqual <string>(this.testString, content, "before replace current file must have the old content.");
}
using (StreamReader copyReader0 = new StreamReader(FabricFile.Open(copyFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
var hardLinkContent0 = copyReader0.ReadLine();
Assert.AreEqual <string>(this.testString, hardLinkContent0, "after replace hard link file must have the old content.");
FabricFile.Replace(replaceFilePath, filePath, backupFilePath, false);
using (StreamReader fileReader = new StreamReader(FabricFile.Open(filePath, FileMode.Open, FileAccess.Read)))
{
var content = fileReader.ReadLine();
Assert.AreEqual <string>(this.testNewString, content, "after replace current file must have the new content.");
}
using (StreamReader copyReader1 = new StreamReader(FabricFile.Open(copyFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
var content = copyReader1.ReadLine();
Assert.AreEqual <string>(this.testString, content, "after replace hard link file must have the old content.");
}
}
}
internal override async Task RenameCopyLogAtomicallyAsync()
{
try
{
// Retrieve head and tail records of the copy log
var tailRecord = this.PhysicalLogWriter.CurrentLogTailRecord;
var callbackManager = this.PhysicalLogWriter.CallbackManager;
this.PhysicalLogWriter.ResetCallbackManager = true;
var closedException = this.PhysicalLogWriter.ClosedException;
var exitTcs = this.PhysicalLogWriter.ExitTcs;
// Now, close the copy log
await this.CloseCurrentLogAsync().ConfigureAwait(false);
var fullcurrentLogfileName = FileLogicalLog.GetFullPathToLog(
this.LogFileDirectoryPath,
this.CurrentLogFileAlias);
var fullBaseLogFileName = FileLogicalLog.GetFullPathToLog(
this.LogFileDirectoryPath,
this.BaseLogFileAlias);
var fullbackupLogFileName = FileLogicalLog.GetFullPathToLog(
this.LogFileDirectoryPath,
this.BaseLogFileAlias + BackupSuffix);
FabricFile.Replace(fullcurrentLogfileName, fullBaseLogFileName, fullbackupLogFileName, false);
// opens using currentLogFileAlias
this.CurrentLogFileAlias = this.BaseLogFileAlias;
this.LogicalLog = await this.CreateLogFileAsync(false, CancellationToken.None).ConfigureAwait(false);
// Write cursor is auto-position to eos
this.PhysicalLogWriter = new PhysicalLogWriter(
this.LogicalLog,
tailRecord,
callbackManager,
closedException,
this.Tracer,
this.MaxWriteCacheSizeInMB,
this.IncomingBytesRateCounterWriter,
this.LogFlushBytesRateCounterWriter,
this.BytesPerFlushCounterWriter,
this.AvgFlushLatencyCounterWriter,
this.AvgSerializationLatencyCounterWriter)
{
ExitTcs = exitTcs
};
var message =
string.Format(
CultureInfo.InvariantCulture,
"RenameCopyLogAtomically: Renamed log. Head record Position: {0}" + Environment.NewLine
+ " Tail record. Type: {1} LSN: {2} PSN: {3} Position: {4}",
this.LogHeadRecordPosition,
tailRecord.RecordType,
tailRecord.Lsn.LSN,
tailRecord.Psn.PSN,
tailRecord.RecordPosition);
FabricEvents.Events.LogManager(this.Tracer.Type, message);
}
catch (Exception e)
{
Utility.ProcessUnexpectedException(
"LogManager::RenameCopyLogAtomically",
this.Tracer,
"renaming log",
e);
}
}
private void CopyTest(bool copyFileExists, bool isLongPath, bool overWrite)
{
var folderPath = this.testPath;
if (true == isLongPath)
{
folderPath = this.ExtendPath(folderPath);
}
var filePath = Path.Combine(folderPath, this.testFileName);
Assert.IsTrue(!isLongPath || filePath.Length > 260, "file path must be greater than max path size.");
var copyFilePath = Path.Combine(folderPath, this.testReplaceFileName);
Assert.IsTrue(!isLongPath || copyFilePath.Length > 260, "replace file path must be greater than max path size.");
LogHelper.Log("FabricDirectory.Create {0}", folderPath);
FabricDirectory.CreateDirectory(folderPath);
LogHelper.Log("FabricFile.Create {0}", filePath);
using (StreamWriter streamWriter = new StreamWriter(FabricFile.Create(filePath)))
{
LogHelper.Log("Write {0}", this.testNewString);
streamWriter.WriteLine(this.testNewString);
streamWriter.Flush();
}
if (copyFileExists)
{
using (StreamWriter streamWriter = new StreamWriter(FabricFile.Create(copyFilePath)))
{
LogHelper.Log("Write {0}", this.testString);
streamWriter.WriteLine(this.testString);
streamWriter.Flush();
}
using (StreamReader copyReader = new StreamReader(FabricFile.Open(copyFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
var content = copyReader.ReadLine();
Assert.AreEqual <string>(this.testString, content, "before copy state.");
}
}
else
{
Assert.IsFalse(FabricFile.Exists(copyFilePath));
}
try
{
FabricFile.Copy(filePath, copyFilePath, overWrite);
Assert.IsTrue(overWrite);
}
catch (FabricException e)
{
Assert.IsFalse(overWrite);
LogHelper.Log("Exception thrown as expected {0}", e.GetType().ToString());
}
if (true == overWrite || false == copyFileExists)
{
using (StreamReader copyReader = new StreamReader(FabricFile.Open(copyFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
var content = copyReader.ReadLine();
Assert.AreEqual <string>(this.testNewString, content, "after copy file must have the new content.");
LogHelper.Log("Read {0} as expected", this.testNewString);
}
}
}
public void FabricFile_ExistsNegative()
{
LogHelper.Log("FabricFile.Exists {0}", this.badPath);
Assert.IsFalse(FabricFile.Exists(this.badPath));
}
public void FabricFile_EndToEndPositive()
{
var folderPath = this.testPath;
folderPath = this.ExtendPath(folderPath);
var filePath = Path.Combine(folderPath, this.testFileName);
Assert.IsTrue(filePath.Length > 260);
LogHelper.Log("FabricDirectory.Create {0}", folderPath);
FabricDirectory.CreateDirectory(folderPath);
LogHelper.Log("FabricFile.Create {0}", filePath);
using (StreamWriter streamWriter = new StreamWriter(FabricFile.Create(filePath)))
{
LogHelper.Log("Write {0}", this.testString);
streamWriter.WriteLine(this.testString);
}
LogHelper.Log("FabricDirectory.GetDirectories {0}", this.testPath);
var result = FabricDirectory.GetDirectories(this.testPath);
Assert.AreEqual(1, result.Length);
LogHelper.Log("FabricDirectory.GetFiles {0}", this.testPath);
result = FabricDirectory.GetFiles(this.testPath);
Assert.AreEqual(0, result.Length);
LogHelper.Log("FabricDirectory.GetFiles {0}, AllDirectories", this.testPath);
result = FabricDirectory.GetFiles(this.testPath, "*", SearchOption.AllDirectories);
Assert.AreEqual(1, result.Length);
LogHelper.Log("FabricDirectory.GetDirectories {0}", folderPath);
result = FabricDirectory.GetDirectories(folderPath);
Assert.AreEqual(0, result.Length);
LogHelper.Log("FabricDirectory.GetFiles {0}", folderPath);
result = FabricDirectory.GetFiles(folderPath);
Assert.AreEqual(1, result.Length);
LogHelper.Log("FabricFile.Open {0}", filePath);
using (StreamReader streamReader = new StreamReader(FabricFile.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.None)))
{
string actual = streamReader.ReadLine();
LogHelper.Log("Read {0}", actual);
Assert.AreEqual(this.testString, actual);
}
LogHelper.Log("FabricFile.GetSize {0}", filePath);
long size = FabricFile.GetSize(filePath);
Assert.IsTrue(size > 0);
LogHelper.Log("FabricPath.GetDirectoryName {0}", filePath);
string directoryName = FabricPath.GetDirectoryName(filePath);
Assert.AreEqual(folderPath, directoryName);
LogHelper.Log("FabricFile.GetLastWriteTime {0}", filePath);
DateTime oldTime = FabricFile.GetLastWriteTime(filePath);
Thread.Sleep(TimeSpan.FromSeconds(1));
using (StreamWriter streamWriter = new StreamWriter(FabricFile.Open(filePath, FileMode.Open, FileAccess.Write)))
{
LogHelper.Log("Write {0}", this.testString);
streamWriter.WriteLine(this.testString);
}
DateTime newTime = FabricFile.GetLastWriteTime(filePath);
Assert.IsTrue(newTime > oldTime);
}
/// <summary>
/// Uploads the local source content to the image store at the remote destination.
/// </summary>
/// <param name="storeSource"> Location (relative to RootUri) from where to download the content. </param>
/// <param name="storeDestination"> Location relative to RootUri where the content needs to be uploaded. </param>
/// <param name="timeout">The timeout for copy content operation.</param>
/// <param name="skipFiles">Files that do not need to be copied.</param>
/// <param name="copyFlag">The copying control information to specify how file can be overwritten</param>
/// <param name="checkMarkFile">Flag the specified the checkmark file.</param>
public void CopyContent(string storeSource, string storeDestination, TimeSpan timeout, string[] skipFiles, CopyFlag copyFlag, bool checkMarkFile)
{
TimeoutHelper helper = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);
FileReaderLock readerLock = null;
try
{
string smbSourcePath = this.ConvertTagToSMBPath(storeSource);
string smbDestinationPath = this.ConvertTagToSMBPath(storeDestination);
readerLock = new FileReaderLock(smbSourcePath);
if (!readerLock.Acquire())
{
throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, FabricErrorCode.ImageStoreAcquireFileLockFailed);
}
if (helper != null)
{
helper.ThrowIfExpired();
}
#if !DotNetCoreClr
using (WindowsImpersonationContext impersonationContext = this.GetImpersonationContext())
#endif
{
bool fabricDirectoryExists = FabricDirectory.Exists(smbSourcePath);
if (fabricDirectoryExists && skipFiles.Any())
{
string[] fullFileNames = FabricDirectory.GetFiles(smbSourcePath, "*", true, SearchOption.AllDirectories);
var relativeFilePath = from file in fullFileNames
select file.Replace(smbSourcePath, "").TrimStart(new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar });
string[] filtered = relativeFilePath.Where(file => !skipFiles.Contains <string>(Path.GetFileName(file))).ToArray <string>();
if (filtered.Count() < fullFileNames.Count())
{
foreach (string file in filtered)
{
CopyCallerHoldsReaderLock(Path.Combine(smbSourcePath, file), Path.Combine(smbDestinationPath, file), copyFlag, helper);
}
}
else
{
CopyCallerHoldsReaderLock(smbSourcePath, smbDestinationPath, copyFlag, helper);
}
}
else
{
bool fabricFileExist = FabricFile.Exists(smbSourcePath);
if ((!FabricFile.Exists(smbSourcePath)) &&
(!FabricDirectory.Exists(smbSourcePath)))
{
throw new IOException(string.Format(CultureInfo.CurrentCulture, StringResources.ImageStoreError_DoesNotExistError, smbSourcePath));
}
CopyCallerHoldsReaderLock(smbSourcePath, smbDestinationPath, copyFlag, helper);
}
}
}
catch (IOException exception)
{
if (exception.GetType() == typeof(IOException))
{
throw new FabricImageStoreIOException(exception);
}
else
{
throw;
}
}
finally
{
if (readerLock != null)
{
readerLock.Dispose();
}
}
}
/// <summary>
/// Retrieves the last event index processed.
/// </summary>
/// <param name="bookmarkFolder"></param>
/// <param name="bytesRead"></param>
/// <returns></returns>
internal EventIndex ReadBookmarkFile(
string bookmarkFolder,
out long bytesRead)
{
bytesRead = 0;
EventIndex lastEventIndex = new EventIndex();
lastEventIndex.Set(DateTime.MinValue, -1);
string bookmarkFile = Path.Combine(bookmarkFolder, BookmarkFileName);
if (false == FabricFile.Exists(bookmarkFile))
{
// Bookmark file doesn't exist
return(lastEventIndex);
}
StreamReader reader = null;
try
{
// Open the file
try
{
Utility.PerformIOWithRetries(
() =>
{
FileStream fileStream = FabricFile.Open(bookmarkFile, FileMode.Open, FileAccess.Read);
Helpers.SetIoPriorityHint(fileStream.SafeFileHandle, Kernel32Types.PRIORITY_HINT.IoPriorityHintVeryLow);
reader = new StreamReader(fileStream);
},
this.methodExecutionInitialRetryIntervalMs,
this.methodExecutionMaxRetryCount,
this.methodExecutionMaxRetryIntervalMs);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Unable to open bookmark file {0}.",
bookmarkFile);
return(lastEventIndex);
}
long localBytesRead = 0;
// Get the version
string versionString = string.Empty;
try
{
Utility.PerformIOWithRetries(
() =>
{
versionString = reader.ReadLine();
localBytesRead += versionString.Length + 2;
},
this.methodExecutionInitialRetryIntervalMs,
this.methodExecutionMaxRetryCount,
this.methodExecutionMaxRetryIntervalMs);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Unable to read version information from bookmark file {0}.",
bookmarkFile);
return(lastEventIndex);
}
// Check the version
if (false == versionString.Equals(BookmarkFileFormatVersionString, StringComparison.Ordinal))
{
this.traceSource.WriteError(
this.logSourceId,
"Unexpected version string {0} encountered in bookmark file {1}.",
versionString,
bookmarkFile);
return(lastEventIndex);
}
// Get information about the last event that we read
string infoLine = string.Empty;
try
{
Utility.PerformIOWithRetries(
() =>
{
infoLine = reader.ReadLine();
localBytesRead += infoLine.Length + 2;
},
this.methodExecutionInitialRetryIntervalMs,
this.methodExecutionMaxRetryCount,
this.methodExecutionMaxRetryIntervalMs);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Unable to read information about last event read from bookmark file {0}.",
bookmarkFile);
return(lastEventIndex);
}
string[] infoLineParts = infoLine.Split(',');
if (infoLineParts.Length != (int)LastEventReadInfoParts.Count)
{
this.traceSource.WriteError(
this.logSourceId,
"The information in bookmark file {0} about the last event read is not in the expected format. {1}",
bookmarkFile,
infoLine);
return(lastEventIndex);
}
string lastEventTimestampString = infoLineParts[(int)LastEventReadInfoParts.LastEventTimestampLong].Trim();
long lastEventTimestampBinary;
if (false == long.TryParse(lastEventTimestampString, out lastEventTimestampBinary))
{
this.traceSource.WriteError(
this.logSourceId,
"Unable to retrieve timestamp of last event from bookmark file {0}.",
bookmarkFile);
return(lastEventIndex);
}
DateTime lastEventTimestamp = DateTime.FromBinary(lastEventTimestampBinary);
string lastEventTimestampDifferentiatorString = infoLineParts[(int)LastEventReadInfoParts.LastEventIndex].Trim();
int lastEventTimestampDifferentiator;
if (false == int.TryParse(lastEventTimestampDifferentiatorString, out lastEventTimestampDifferentiator))
{
this.traceSource.WriteError(
this.logSourceId,
"Unable to retrieve timestamp differentiator of last event from bookmark file {0}.",
bookmarkFile);
return(lastEventIndex);
}
// record bytes read
bytesRead = localBytesRead;
lastEventIndex.Set(lastEventTimestamp, lastEventTimestampDifferentiator);
}
finally
{
if (null != reader)
{
reader.Dispose();
}
}
return(lastEventIndex);
}
/// <summary>
/// Downloads content from the file image store to local destination.
/// </summary>
/// <param name="storeSource">The relative path of source file image store to be downloaded from.</param>
/// <param name="localDestination">The local destination path to download the content.</param>
/// <param name="handler">The image store progress handler which is not supported at the file image store.</param>
/// <param name="timeout">The timeout for performing the downloading operation.</param>
/// <param name="copyFlag">The copying control information to specify how file can be overwritten.</param>
public void DownloadContent(string storeSource, string localDestination, IImageStoreProgressHandler handler, TimeSpan timeout, CopyFlag copyFlag)
{
TimeoutHelper helper = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);
localDestination = this.GetLocalPath(localDestination);
string tempDirectory = null;
FileReaderLock readerLock = null;
try
{
string smbTag = this.ConvertTagToSMBPath(storeSource);
if ((!FabricFile.Exists(smbTag)) &&
(!FabricDirectory.Exists(smbTag)))
{
throw new IOException(string.Format(CultureInfo.CurrentCulture, StringResources.ImageStoreError_DoesNotExistError, smbTag));
}
readerLock = new FileReaderLock(smbTag);
if (!readerLock.Acquire())
{
throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, FabricErrorCode.ImageStoreAcquireFileLockFailed);
}
if (helper != null)
{
helper.ThrowIfExpired();
}
if (accessDescription != null && !accessDescription.HasWriteAccess)
{
// Copy the content from the remote Store to a temp location using the ImpersonationContext
string tempLocalResource = null;
#if !DotNetCoreClr
using (WindowsImpersonationContext impersonationContext = accessDescription.WindowsIdentity.Impersonate())
#endif
{
tempDirectory = CreateAndAclDirectory();
tempLocalResource = Path.Combine(tempDirectory, Path.GetRandomFileName());
CopyCallerHoldsReaderLock(smbTag, tempLocalResource, CopyFlag.AtomicCopy, helper);
}
readerLock.Release();
// Copy the content from the temp location to the local destination outside the ImpersonationContext
CopyCallerHoldsReaderLock(tempLocalResource, localDestination, copyFlag, helper);
}
#if !DotNetCoreClr
using (WindowsImpersonationContext impersonationContext = this.GetImpersonationContext())
#endif
{
CopyCallerHoldsReaderLock(smbTag, localDestination, copyFlag, helper);
}
}
catch (IOException exception)
{
if (exception.GetType() == typeof(IOException))
{
throw new FabricImageStoreException(StringResources.Error_ImageStoreIOException, exception);
}
else
{
throw;
}
}
finally
{
if (readerLock != null)
{
readerLock.Dispose();
}
if (tempDirectory != null && FabricDirectory.Exists(tempDirectory))
{
FabricDirectory.Delete(tempDirectory, recursive: true, deleteReadOnlyFiles: true);
}
}
}
/// <summary>
/// Updates the bookmark file with the last event read index.
/// </summary>
/// <param name="bookmarkFolder"></param>
/// <param name="lastReadEventIndexPosition"></param>
/// <param name="eventIndex"></param>
/// <param name="bytesWritten"></param>
/// <returns></returns>
internal bool UpdateBookmarkFile(
string bookmarkFolder,
long lastReadEventIndexPosition,
EventIndex eventIndex,
out long bytesWritten)
{
bytesWritten = 0;
FileStream fs = null;
StreamWriter writer = null;
string bookmarkFile = Path.Combine(bookmarkFolder, BookmarkFileName);
if (false == FabricFile.Exists(bookmarkFile))
{
return(false);
}
try
{
// Open the bookmark file
try
{
Utility.PerformIOWithRetries(
() =>
{
fs = new FileStream(bookmarkFile, FileMode.Open);
writer = new StreamWriter(fs);
},
this.methodExecutionInitialRetryIntervalMs,
this.methodExecutionMaxRetryCount,
this.methodExecutionMaxRetryIntervalMs);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Unable to open bookmark file {0} while processing events.",
bookmarkFile);
return(false);
}
// Move the stream position to the point where the event index needs to be written
fs.Position = lastReadEventIndexPosition;
long localBytesWritten = 0;
// Write the timestamp and index of the last event read and close the file
string lastEventReadString = string.Format("{0},{1},{2}", eventIndex.Timestamp.ToBinary(), eventIndex.Timestamp.ToString(), eventIndex.TimestampDifferentiator);
try
{
Utility.PerformIOWithRetries(
() =>
{
writer.WriteLine(lastEventReadString);
localBytesWritten = lastEventReadString.Length + 2;
writer.Dispose();
fs.Dispose();
},
this.methodExecutionInitialRetryIntervalMs,
this.methodExecutionMaxRetryCount,
this.methodExecutionMaxRetryIntervalMs);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Unable to write last event read {0} in bookmark file {1} while processing events.",
lastEventReadString,
bookmarkFile);
return(false);
}
// record bytes written
bytesWritten = localBytesWritten;
}
finally
{
if (null != writer)
{
writer.Dispose();
}
if (null != fs)
{
fs.Dispose();
}
}
return(true);
}
private void CopyCallerHoldsReaderLock(string source, string destination, CopyFlag copyFlag, TimeoutHelper helper)
{
string destinationDirectory = FabricPath.GetDirectoryName(destination);
if (!string.IsNullOrEmpty(destinationDirectory) && !FabricDirectory.Exists(destinationDirectory))
{
FabricDirectory.CreateDirectory(destinationDirectory);
}
using (FileWriterLock writerLock = new FileWriterLock(destination))
{
if (!writerLock.Acquire())
{
throw new FabricTransientException(StringResources.Error_ImageStoreAcquireFileLockFailed, FabricErrorCode.ImageStoreAcquireFileLockFailed);
}
if (helper != null)
{
helper.ThrowIfExpired();
}
if (FabricFile.Exists(source))
{
// This is a file copy
if (FabricFile.Exists(destination))
{
FabricFile.Delete(destination, deleteReadonly: true);
}
int retryCount = 0;
while (helper == null || !TimeoutHelper.HasExpired(helper))
{
try
{
bool shouldOverwrite = (copyFlag != CopyFlag.AtomicCopySkipIfExists);
FabricFile.Copy(source, destination, shouldOverwrite);
break;
}
catch (UnauthorizedAccessException)
{
TraceSource.WriteInfo(
TraceType,
"Uploading {0} to {1} caused UnauthorizedAccessException. RetryCount: {2}.",
source,
destination,
retryCount);
if (retryCount++ > 3)
{
throw;
}
// This could happen when a file is marked for delete and we try to delete
// it again or try to open the file. Retrying after sometime should fix the issue.
Thread.Sleep(TimeSpan.FromSeconds(retryCount));
}
if (helper != null)
{
helper.ThrowIfExpired();
}
}
}
else
{
// This is a folder copy
using (FolderCopy fc = new FolderCopy(copyFlag, null))
{
fc.Copy(source, destination);
}
}
}
}
private void ProcessEtlFilesWorker(
string fileNamePattern,
DateTime lastEndTime,
string friendlyName,
CancellationToken cancellationToken,
out DateTime endTime)
{
// By default, initialize endTime to lastEndTime. If we run into a
// situation where we don't have any ETL files to process, the
// default value will not be updated.
endTime = lastEndTime;
if (!Directory.Exists(this.traceDirectory) &&
this.traceDirectory.StartsWith(
ContainerEnvironment.ContainerLogRootDirectory,
StringComparison.InvariantCultureIgnoreCase))
{
// A temporary handling for the containers case, where DCA could be processing an instance's
// folder with a different older structure (if traces collectors or DCA itself got updated).
this.traceSource.WriteInfo(
this.logSourceId,
"Skipping a search cycle to match '{0}' as corresponding container instance directory '{1}' is not found.",
fileNamePattern,
this.traceDirectory);
return;
}
// Get all files in the traces directory that match the pattern
this.traceSource.WriteInfo(
this.logSourceId,
"Searching for ETL files whose names match '{0}' in directory '{1}'.",
fileNamePattern,
this.traceDirectory);
var dirInfo = new DirectoryInfo(this.traceDirectory);
var etlFiles = dirInfo.GetFiles(fileNamePattern);
this.traceSource.WriteInfo(
this.logSourceId,
"Found {0} ETL files whose names match '{1}' in directory '{2}'.",
etlFiles.Length,
fileNamePattern,
this.traceDirectory);
if (0 == etlFiles.Length)
{
// No ETL files found. Nothing more to do here.
return;
}
// We ignore files that are so old that they need to be deleted
var cutoffTime = DateTime.UtcNow.Add(-this.etlInMemoryProducerWorkerSettings.EtlDeletionAge);
// Reading in chronological order. Sort the files such that the
// file with the least recent creation time comes first.
etlFiles = etlFiles.OrderBy(e => e.CreationTime).ToArray();
// Ignore ETL files whose last-write time is older than the timestamp
// up to which we have already read all events. However, we never
// ignore the active ETL file because its timestamp is updated only
// after it has reached its full size.
var activeEtlFileName = etlFiles[etlFiles.Length - 1].Name;
etlFiles = etlFiles
.Where(f => f.Name.Equals(activeEtlFileName, StringComparison.OrdinalIgnoreCase) ||
(f.LastWriteTimeUtc.CompareTo(lastEndTime) >= 0))
.ToArray();
// Processing start time
var processingStartTime = DateTime.Now;
var processingEndTime = processingStartTime.Add(this.perProviderProcessingTime);
var filesProcessed = 0;
for (int i = 0; i < etlFiles.Length; i++)
{
if (etlFiles[i].LastWriteTimeUtc.CompareTo(cutoffTime) <= 0)
{
this.traceSource.WriteInfo(
this.logSourceId,
"Skipping ETL file {0} in directory {1} because it is too old.",
etlFiles[i],
this.traceDirectory);
continue;
}
// If the file has already been fully processed, then ignore
// that file.
if (FabricFile.Exists(Path.Combine(this.markerFileDirectory, etlFiles[i].Name)))
{
continue;
}
this.traceSource.WriteInfo(
this.logSourceId,
"Processing ETL file: {0}.",
etlFiles[i].Name);
// If we are processing Windows Fabric ETL files from their default location, then
// make sure that we have loaded the correct manifest for them.
if (this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation())
{
this.etlProcessor.EnsureCorrectWinFabManifestVersionLoaded(etlFiles[i].Name);
}
bool isActiveEtl = i == etlFiles.Length - 1;
if (isActiveEtl)
{
// process active etl file and then break so that we can process additional
// events in the next pass.
this.etlProcessor.ProcessActiveEtlFile(
etlFiles[i],
lastEndTime,
cancellationToken,
out endTime);
filesProcessed++;
if (false == this.checkpointManager.SetLastEndTime(etlFiles[i].Name, endTime))
{
if (0 != lastEndTime.CompareTo(endTime))
{
this.traceSource.WriteError(
this.logSourceId,
"Failed to write the last timestamp up to which {0} ETW events have been read to file {1}. Due to this error, although we have read events up to {2}, in our next pass we will start reading events from {3}.",
friendlyName,
etlFiles[i].Name,
endTime,
lastEndTime);
}
}
break;
}
else
{
this.etlProcessor.ProcessInactiveEtlFile(
etlFiles[i],
lastEndTime,
cancellationToken,
out endTime);
filesProcessed++;
if (false == this.checkpointManager.SetLastEndTime(etlFiles[i].Name, endTime))
{
if (0 != lastEndTime.CompareTo(endTime))
{
this.traceSource.WriteError(
this.logSourceId,
"Failed to write the last timestamp up to which {0} ETW events have been read to file {1}. Due to this error, although we have read events up to {2}, in our next pass we will start reading events from {3}.",
friendlyName,
etlFiles[i].Name,
endTime,
lastEndTime);
}
}
else
{
// set last end time so that the next iteration can calculate start
// time accurately
lastEndTime = endTime;
}
}
// check if time for this provider is up
if (0 > DateTime.Compare(processingEndTime, DateTime.Now))
{
this.traceSource.WriteInfo(
this.logSourceId,
"Due to time limit on etl processing, no more {0} files will be processed in this pass.",
fileNamePattern);
break;
}
if (cancellationToken.IsCancellationRequested)
{
this.traceSource.WriteInfo(
this.logSourceId,
"The producer is being stopped, so no more {0} files will be processed in this pass.",
fileNamePattern);
break;
}
}
// Write the length of the DCA's ETL file backlog. The performance test consumes this information.
this.perfHelper.RecordEtlPassBacklog(friendlyName, etlFiles.Length - filesProcessed);
}
private static void CopyFileWorker(object context)
{
CopyFileParameters copyParam = (CopyFileParameters)context;
FabricFile.Copy(copyParam.Source, copyParam.Destination, copyParam.Overwrite);
}
internal EtlInMemoryProducerWorker(
EtlInMemoryProducerWorkerParameters initParam,
DiskSpaceManager diskSpaceManager,
ITraceFileEventReaderFactory traceFileEventReaderFactory)
{
this.logSourceId = initParam.ProducerInstanceId;
this.traceSource = initParam.TraceSource;
this.cancellationTokenSource = new CancellationTokenSource();
this.perfHelper = new EtlInMemoryPerformance(this.traceSource);
this.diskSpaceManager = diskSpaceManager;
// Initialize the settings
this.etlInMemoryProducerWorkerSettings = EtlInMemoryProducerWorkerSettingsHelper.InitializeSettings(initParam);
if (WinFabricEtlType.DefaultEtl == this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType)
{
// If we're processing the default ETL files, we should keep track of
// whether or not we're on the FMM node. This information is used by
// some other plugin types.
Utility.LastFmmEventTimestamp = DateTime.MinValue;
}
// Initialize the sink list
this.sinks = initParam.EtlInMemoryProducer.ConsumerSinks.Cast <IEtlInMemorySink>().ToList().AsReadOnly();
this.etlToInMemoryBufferWriters = initParam.EtlInMemoryProducer.ConsumerSinks.OfType <EtlToInMemoryBufferWriter>().ToList().AsReadOnly();
this.etlToInMemoryBufferWriters.ForEach(e => e.SetEtlProducer(this));
// Figure out where the ETL files are located
this.traceDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeTraceDirectory(
this.etlInMemoryProducerWorkerSettings.EtlPath,
initParam.LogDirectory,
this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType);
this.markerFileDirectory = EtlInMemoryProducerWorkerSettingsHelper.InitializeMarkerFileDirectory(
this.etlInMemoryProducerWorkerSettings.EtlPath,
initParam.LogDirectory,
this.traceDirectory,
this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType);
this.providers = EtlInMemoryProducerWorkerSettingsHelper.InitializeProviders(
this.etlInMemoryProducerWorkerSettings.EtlPath,
this.etlInMemoryProducerWorkerSettings.EtlFilePatterns,
this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType,
message => this.traceSource.WriteError(this.logSourceId, message)).AsReadOnly();
if (0 == this.providers.Count)
{
// No ETL files to read, so return immediately
this.traceSource.WriteWarning(
this.logSourceId,
"No ETL files have been specified for processing.");
}
this.checkpointManager = new CheckpointManager(
this.providers,
this.etlInMemoryProducerWorkerSettings.EtlPath,
initParam.LogDirectory,
this.traceDirectory,
this.traceSource,
this.logSourceId);
this.etlReadInterval = this.etlInMemoryProducerWorkerSettings.EtlReadInterval;
if (this.etlReadInterval > TimeSpan.Zero)
{
// Create the directory that contains the marker files.
this.CreateDirectoriesForEtlProcessing();
// We need to collect bootstrap traces
this.bootstrapTraceProcessor = new BootstrapTraceProcessor(
this.traceDirectory,
this.markerFileDirectory,
this.etlToInMemoryBufferWriters,
this.etlInMemoryProducerWorkerSettings.EtlReadInterval,
this.traceSource,
this.logSourceId);
this.bootstrapTraceProcessor.Start();
// Create the ETL processor
this.etlProcessor = new EtlProcessor(
true,
this.IsProcessingWindowsFabricEtlFilesFromDefaultLocation(),
this.markerFileDirectory,
this.etlInMemoryProducerWorkerSettings.WindowsFabricEtlType,
this.traceSource,
this.logSourceId,
this.perfHelper,
this.sinks,
this.etlToInMemoryBufferWriters,
traceFileEventReaderFactory);
// Create a periodic timer to read ETL files
var timerId = string.Concat(
this.logSourceId,
EtlReadTimerIdSuffix);
this.etlReadTimer = new DcaTimer(
timerId,
state => this.EtlReadCallback(this.cancellationTokenSource.Token),
this.etlReadInterval);
this.etlReadTimer.Start();
// Figure out how much processing time is available to each provider.
this.ComputePerProviderEtlProcessingTimeSeconds();
}
// Disk manager set up for traces
foreach (var provider in this.providers)
{
var capturedProvider = provider;
this.diskSpaceManager.RegisterFolder(
this.logSourceId,
() => new DirectoryInfo(this.traceDirectory).EnumerateFiles(capturedProvider.EtlFileNamePattern),
f => FabricFile.Exists(Path.Combine(this.markerFileDirectory, f.Name)), // Safe to delete once marker file exists
f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
}
// Disk manager set up for marker files
this.diskSpaceManager.RegisterFolder(
this.logSourceId,
() => new DirectoryInfo(this.markerFileDirectory).EnumerateFiles(),
f => !FabricFile.Exists(Path.Combine(this.traceDirectory, f.Name)), // Safe to delete once original has been cleaned up
f => f.LastWriteTimeUtc >= DateTime.UtcNow.Add(-initParam.LatestSettings.EtlDeletionAgeMinutes));
}
private void DeliverEventsToConsumer(object state)
{
this.perfHelper.EventDeliveryPassBegin();
this.eventDeliveryPeriodAborted = false;
// Perform any tasks necessary at the beginning of the event delivery pass
this.eventSink.OnEtwEventDeliveryStart();
// Get the files from the buffered event directory
// We ignore files that are so old that they need to be deleted
DateTime cutoffTime = DateTime.UtcNow.Add(-this.eventDeletionAge);
DirectoryInfo dirInfo = new DirectoryInfo(this.etwEventCache);
FileInfo[] eventFilesInfo = dirInfo.GetFiles(CacheFileNameSearchPattern)
.Where(file => file.LastWriteTimeUtc.CompareTo(cutoffTime) > 0)
.ToArray();
// Sort the files such that the file with the most recent last-write
// time comes first. We'll process files in that order so that in
// case of huge backlogs the most recent (and hence likely to be
// most interesting) traces are processed first.
Array.Sort(eventFilesInfo, CompareFileLastWriteTimes);
// Process each of the files
int filesProcessed = 0;
DateTime processingEndTime = DateTime.Now.Add(this.eventDeliveryPassLength);
List <string> filesToDelete = new List <string>();
foreach (FileInfo eventFileInfo in eventFilesInfo)
{
// Process events from the current file
string eventFile = eventFileInfo.FullName;
this.ProcessEventsFromFile(eventFile);
filesProcessed++;
// If the event delivery pass is being aborted, then don't process
// any more files. Also, don't delete the current file because
// its processing may have been interrupted. This file will be
// processed again in the next pass.
if (this.eventDeliveryPeriodAborted)
{
this.TraceSource.WriteInfo(
this.LogSourceId,
"The event delivery pass is being aborted. Therefore, no more files in the buffered event directory will be processed.");
break;
}
// If we are in the process of stopping, then don't process any
// more files. Also, don't delete the current file because its
// processing may have been interrupted. This file will be
// processed again when we are restarted.
if (this.Stopping)
{
this.TraceSource.WriteInfo(
this.LogSourceId,
"The consumer is being stopped. Therefore, no more files in the buffered event directory will be processed.");
break;
}
// Add the file to the list of files that we delete at the end
// of the event delivery pass
filesToDelete.Add(eventFile);
if (0 > DateTime.Compare(processingEndTime, DateTime.Now))
{
this.TraceSource.WriteInfo(
this.LogSourceId,
"Due to time limit on backlog processing, no more files in the buffered event directory will be processed in this pass.");
break;
}
}
// Perform any tasks necessary at the end of the ETL processing pass
this.eventSink.OnEtwEventDeliveryStop();
// Delete the files that we successfully processed in this pass
foreach (string fileToDelete in filesToDelete)
{
try
{
Utility.PerformIOWithRetries(
ctx =>
{
string fileName = ctx;
FabricFile.Delete(fileName);
},
fileToDelete);
}
catch (Exception e)
{
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"Failed to delete file {0}.",
fileToDelete);
}
}
// Write performance-related information.
this.perfHelper.RecordEventDeliveryBacklog(eventFilesInfo.Length - filesProcessed);
this.perfHelper.EventDeliveryPassEnd();
// Schedule the next pass
this.eventDeliveryTimer.Start();
}
protected override bool CopyFileToDestination(string source, string sourceRelative, int retryCount, out bool fileSkipped, out bool fileCompressed)
{
// No compression
fileCompressed = false;
// Compute the destination path
string destination = Path.Combine(this.destinationPath, sourceRelative);
// Figure out the name of the directory at the destination and in the
// local map.
var destinationDir = FabricPath.GetDirectoryName(destination);
// If the directory at the destination doesn't exist, then create it
if ((false == string.IsNullOrEmpty(destinationDir)) &&
(false == this.PerformDestinationOperation(DirectoryExistsAtDestination, destinationDir)))
{
try
{
Utility.PerformIOWithRetries(
() =>
{
PerformDestinationOperation(
CreateDirectoryAtDestination,
destinationDir);
},
retryCount);
}
catch (Exception e)
{
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"Failed to create directory {0} for copying file {1}.",
destinationDir,
sourceRelative);
fileSkipped = true;
return false;
}
}
// Determine whether we need to first copy the file to the staging folder
string stagingFilePath = null;
if (false == string.IsNullOrEmpty(this.stagingFolderPath))
{
stagingFilePath = Path.Combine(this.stagingFolderPath, Path.GetFileName(sourceRelative));
}
// Copy the file over to its destination
DateTime sourceLastWriteTime = DateTime.MinValue;
bool sourceFileNotFound = false;
try
{
try
{
Utility.PerformIOWithRetries(
() =>
{
if (string.IsNullOrEmpty(stagingFilePath))
{
this.perfHelper.ExternalOperationBegin(
ExternalOperationTime.ExternalOperationType.FileShareCopy,
0);
}
try
{
// Copy the file
if (false == string.IsNullOrEmpty(stagingFilePath))
{
FabricFile.Copy(source, stagingFilePath, true);
}
else
{
FileCopyInfo copyInfo = new FileCopyInfo
{
Source = source,
Destination = destination
};
PerformDestinationOperation(CopyFileToDestination, copyInfo);
}
}
catch (FileNotFoundException)
{
sourceFileNotFound = true;
}
if (string.IsNullOrEmpty(stagingFilePath))
{
this.perfHelper.ExternalOperationEnd(
ExternalOperationTime.ExternalOperationType.FileShareCopy,
0);
this.perfHelper.FileUploaded();
}
},
retryCount);
}
catch (Exception e)
{
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"File copy failed. Source: {0}, destination: {1}",
source,
destination);
fileSkipped = true;
return false;
}
if (sourceFileNotFound)
{
// The source file was not found. Maybe it
// got deleted before we had a chance to copy
// it. Handle the error and move on.
this.TraceSource.WriteWarning(
this.LogSourceId,
"File {0} could not be uploaded to {1} because it was not found.",
source,
destination);
fileSkipped = true;
return true;
}
if (false == string.IsNullOrEmpty(stagingFilePath))
{
try
{
Utility.PerformIOWithRetries(
() =>
{
this.perfHelper.ExternalOperationBegin(
ExternalOperationTime.ExternalOperationType.FileShareCopy,
0);
FileCopyInfo copyInfo = new FileCopyInfo
{
Source = stagingFilePath,
Destination = destination
};
PerformDestinationOperation(CopyFileToDestination, copyInfo);
this.perfHelper.ExternalOperationEnd(
ExternalOperationTime.ExternalOperationType.FileShareCopy,
0);
this.perfHelper.FileUploaded();
},
retryCount);
}
catch (Exception e)
{
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"File copy failed. Source: {0}, destination: {1}",
source,
destination);
fileSkipped = true;
return false;
}
}
}
finally
{
if (false == string.IsNullOrEmpty(stagingFilePath))
{
try
{
Utility.PerformIOWithRetries(
() =>
{
FabricFile.Delete(stagingFilePath);
},
retryCount);
}
catch (Exception e)
{
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"Failed to delete file {0}",
stagingFilePath);
}
}
}
fileSkipped = false;
return true;
}
public void OnEtwEventProcessingPeriodStop()
{
if (null == this.streamWriter)
{
// An error occurred during the creation of the buffered event
// file. So just return immediately without doing anything.
return;
}
// Close the buffered event file that we are currently working on.
try
{
this.streamWriter.Dispose();
}
catch (System.Text.EncoderFallbackException ex)
{
// This can happen if the manifest file does not match the binary.
// Write an error message and move on.
this.TraceSource.WriteError(
this.LogSourceId,
"Exception occurred while closing buffered event file. Exception information: {0}",
ex);
}
// Make our buffered event files available for delivery to the consumer.
// This includes:
// - the buffered event file that we wrote in this pass
// - any old buffered event files that we wrote in previous passes
// (and got interrupted before we could rename them for delivery to
// the consumer).
string[] cacheFiles = FabricDirectory.GetFiles(
this.etwEventCache,
TempCacheFileNameSearchPattern);
foreach (string cacheFile in cacheFiles)
{
try
{
Utility.PerformIOWithRetries(
ctx =>
{
string fileName = ctx;
FabricFile.Move(
fileName,
Path.ChangeExtension(fileName, CacheFileNameExtension));
},
cacheFile);
this.TraceSource.WriteInfo(
this.LogSourceId,
"File containing filtered ETW events was renamed for delivery to consumer. Old name: {0}, new name: {1}.",
cacheFile,
Path.ChangeExtension(cacheFile, CacheFileNameExtension));
}
catch (Exception e)
{
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"Buffered event file {0} could not be renamed for delivery to consumer.",
cacheFile);
}
}
this.perfHelper.EtlReadPassEnd();
}
/// <summary>
/// Open an existing <see cref="BackupMetadataFile"/> from the given file path.
/// </summary>
/// <param name="backupMetadataFilePath">Path of the backup meta data file.</param>
/// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
/// <returns>The read <see cref="BackupMetadataFile"/>.</returns>
public static async Task <BackupMetadataFile> OpenAsync(
string backupMetadataFilePath,
CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(backupMetadataFilePath))
{
throw new ArgumentException(
string.Format(CultureInfo.CurrentCulture,
SR.Error_NullArgument_Formatted,
"backupMetadataFilePath"));
}
if (!FabricFile.Exists(backupMetadataFilePath))
{
throw new ArgumentException(
string.Format(
CultureInfo.CurrentCulture,
SR.Error_FilePath_Null,
backupMetadataFilePath),
"backupMetadataFilePath");
}
// Open the file with asynchronous flag and 4096 cache size (C# default).
// MCoskun: Default IoPriorityHint is used since this operation is called during restore.
using (var filestream = FabricFile.Open(
backupMetadataFilePath,
FileMode.Open,
FileAccess.Read,
FileShare.Read,
4096,
FileOptions.Asynchronous))
{
var backupMetadataFile = new BackupMetadataFile(backupMetadataFilePath);
// Read and validate the Footer section. The footer is always at the end of the stream, minus space for the checksum.
var footerHandle = new BlockHandle(
filestream.Length - FileFooter.SerializedSize - sizeof(ulong),
FileFooter.SerializedSize);
backupMetadataFile.footer =
await
FileBlock.ReadBlockAsync(
filestream,
footerHandle,
(reader, handle) => FileFooter.Read(reader, handle)).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
// Verify we know how to deserialize this version of the backup log file.
if (backupMetadataFile.footer.Version != Version)
{
throw new InvalidDataException(
string.Format(
CultureInfo.CurrentCulture,
SR.Error_BackupMetadata_Deserialized,
backupMetadataFile.footer.Version,
Version));
}
// Read and validate the properties section.
var propertiesHandle = backupMetadataFile.footer.PropertiesHandle;
backupMetadataFile.properties =
await
FileBlock.ReadBlockAsync(
filestream,
propertiesHandle,
(reader, handle) => FileProperties.Read <BackupMetadataFileProperties>(reader, handle)).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
return(backupMetadataFile);
}
}