public CheckpointManager(
bool isReadingFromApplicationManifest,
string etlPath,
string logDirectory,
string traceDirectory,
string producerInstanceId,
FabricEvents.ExtensionsEvents traceSource,
string logSourceId)
{
this.traceSource = traceSource;
this.logSourceId = logSourceId;
// Directory where the producer saves the file containing information
// about the point upto which we last read the ETL files
this.lastEtlFileReadFileDirectoryPath = InitializeLastEtlFileReadFilePath(
isReadingFromApplicationManifest,
etlPath,
logDirectory,
traceDirectory,
producerInstanceId);
// If the directory containing the last ETL read information is
// different from the log directory, then create it now.
if (false == this.lastEtlFileReadFileDirectoryPath.Equals(
logDirectory))
{
FabricDirectory.CreateDirectory(this.lastEtlFileReadFileDirectoryPath);
this.traceSource.WriteInfo(
this.logSourceId,
"Directory containing last ETL read info: {0}",
this.lastEtlFileReadFileDirectoryPath);
}
}
private void SetInformationTest(bool isLongPath, Kernel32Types.PRIORITY_HINT priorityHintOne, Kernel32Types.PRIORITY_HINT priorityHintTwo)
{
var folderPath = this.testPath;
if (isLongPath == true)
{
folderPath = this.ExtendPath(folderPath);
}
var filePath = Path.Combine(folderPath, this.testFileName);
Assert.IsTrue(!isLongPath || filePath.Length > 260, "file path must be greater than max path size.");
FabricDirectory.CreateDirectory(folderPath);
using (FileStream fileStream = FabricFile.Open(filePath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read))
{
var fileHandle = fileStream.SafeFileHandle;
// Verify that the default is Normal.
this.VerifyPriorityHint(fileHandle, Kernel32Types.PRIORITY_HINT.IoPriorityHintNormal);
this.SetPriorityHint(fileHandle, priorityHintOne);
this.VerifyPriorityHint(fileHandle, priorityHintOne);
if (priorityHintTwo != Kernel32Types.PRIORITY_HINT.MaximumIoPriorityHintType)
{
this.SetPriorityHint(fileHandle, priorityHintTwo);
this.VerifyPriorityHint(fileHandle, priorityHintTwo);
}
}
}
public static void CreateFolderIfNotExists(string directoryPath)
{
if (!FabricDirectory.Exists(directoryPath))
{
FabricDirectory.CreateDirectory(directoryPath);
}
}
public CheckpointManager(
IEnumerable <EtlInMemoryProducerWorker.ProviderInfo> providers,
string etlPath,
string logDirectory,
string traceDirectory,
FabricEvents.ExtensionsEvents traceSource,
string logSourceId)
{
this.traceSource = traceSource;
this.logSourceId = logSourceId;
// Directory where the producer saves the file containing information
// about the point upto which we last read the ETL files
var lastEtlFileReadFilePath = InitializeLastEtlFileReadFilePath(
etlPath,
logDirectory,
traceDirectory);
// If the directory containing the last ETL read information is
// different from the log directory, then create it now.
if (false == lastEtlFileReadFilePath.Equals(
logDirectory))
{
FabricDirectory.CreateDirectory(lastEtlFileReadFilePath);
this.traceSource.WriteInfo(
this.logSourceId,
"Directory containing last ETL read info: {0}",
lastEtlFileReadFilePath);
}
this.SetProviderCheckpointFileName(providers, lastEtlFileReadFilePath);
}
/// <summary>
/// Copy Folder method called by the CopyItem delegate to dequeue a folder copy item
/// This method enumerates the folder enqueues all its children and dequeues the current item
/// </summary>
/// <param name="sourceDir">
/// Source folder to be copied
/// </param>
/// <param name="destinationDir">
/// Destination folder
/// </param>
private void CopyFolder(string sourceDir, string destinationDir)
{
// create the destination folder if it doesn't exist
if (!FabricDirectory.Exists(destinationDir))
{
FabricDirectory.CreateDirectory(destinationDir);
}
// Enumerate source and queue up work items for each child folder
var directoryNames = FabricDirectory.GetDirectories(
sourceDir,
"*",
false, // full path
SearchOption.TopDirectoryOnly);
var fileNames = FabricDirectory.GetFiles(
sourceDir,
"*",
false, // full path
SearchOption.TopDirectoryOnly);
foreach (var directoryName in directoryNames)
{
this.QueueCopyItem(sourceDir, destinationDir, directoryName, true);
}
foreach (var fileName in fileNames)
{
this.QueueCopyItem(sourceDir, destinationDir, fileName, false);
}
}
private static void WriteStringToFile(string fileName, string value, bool writeLine = true, Encoding encoding = null)
{
var directory = FabricPath.GetDirectoryName(fileName);
if (!FabricDirectory.Exists(directory))
{
FabricDirectory.CreateDirectory(directory);
}
if (encoding == null)
{
#if DotNetCoreClrLinux
encoding = new UTF8Encoding(false);
#else
encoding = Encoding.GetEncoding(0);
#endif
}
using (StreamWriter writer = new StreamWriter(FabricFile.Open(fileName, FileMode.OpenOrCreate), encoding))
{
if (writeLine)
{
writer.WriteLine(value);
}
else
{
writer.Write(value);
}
#if DotNetCoreClrLinux
Helpers.UpdateFilePermission(fileName);
#endif
}
}
/// <summary>
/// Creates bookmark directory.
/// </summary>
/// <param name="sourceFolder"></param>
/// <param name="workDirectory"></param>
/// <param name="destinationKey"></param>
/// <param name="bookmarkFolder"></param>
/// <returns></returns>
private bool CreateBookmarkSubDirectory(
string sourceFolder,
string workDirectory,
string destinationKey,
out string bookmarkFolder)
{
string bookmarkParentFolder = Path.Combine(
workDirectory,
Utility.ShortWindowsFabricIdForPaths);
bool success = Utility.CreateWorkSubDirectory(
this.traceSource,
this.logSourceId,
destinationKey,
sourceFolder,
bookmarkParentFolder,
out bookmarkFolder);
if (success)
{
bookmarkFolder = Path.Combine(bookmarkFolder, BookmarkDirName);
}
FabricDirectory.CreateDirectory(bookmarkFolder);
return(success);
}
private void TestIntersectingHardLinkAndReplace(bool isLongPath, FileShare fileShare)
{
var folderPath = this.testPath;
if (true == isLongPath)
{
folderPath = this.ExtendPath(folderPath);
}
var filePath = Path.Combine(folderPath, this.testFileName);
Assert.IsTrue(!isLongPath || filePath.Length > 260, "file path must be greater than max path size.");
var hardLinkedFilePath = Path.Combine(folderPath, this.testHardLinkedFileName);
Assert.IsTrue(!isLongPath || hardLinkedFilePath.Length > 260, "hard linked file path must be greater than max path size.");
var replaceFilePath = Path.Combine(folderPath, this.testReplaceFileName);
Assert.IsTrue(!isLongPath || replaceFilePath.Length > 260, "replace file path must be greater than max path size.");
var backupFilePath = Path.Combine(folderPath, this.testBackupFileName);
Assert.IsTrue(!isLongPath || backupFilePath.Length > 260, "backup file path must be greater than max path size.");
LogHelper.Log("FabricDirectory.Create {0}", folderPath);
FabricDirectory.CreateDirectory(folderPath);
LogHelper.Log("FabricFile.Create {0}", filePath);
using (StreamWriter streamWriter = new StreamWriter(FabricFile.Create(filePath)))
{
LogHelper.Log("Write {0}", this.testString);
streamWriter.WriteLine(this.testString);
streamWriter.Flush();
}
using (StreamWriter streamWriter = new StreamWriter(FabricFile.Create(replaceFilePath)))
{
LogHelper.Log("Write {0}", this.testNewString);
streamWriter.WriteLine(this.testNewString);
streamWriter.Flush();
}
FabricFile.CreateHardLink(hardLinkedFilePath, filePath);
using (StreamReader hardLinkReader0 = new StreamReader(FabricFile.Open(hardLinkedFilePath, FileMode.Open, FileAccess.Read, fileShare)))
{
var hardLinkContent0 = hardLinkReader0.ReadLine();
Assert.AreEqual <string>(this.testString, hardLinkContent0, "after replace hard link file must have the old content.");
try
{
FabricFile.Replace(replaceFilePath, filePath, backupFilePath, false);
Assert.Fail();
}
catch (FileLoadException)
{
}
}
}
private bool CreateLocalMapFolder()
{
bool success = false;
if (FabricDirectory.Exists(this.localMap))
{
return(true);
}
try
{
Utility.PerformIOWithRetries(
() =>
{
FabricDirectory.CreateDirectory(this.localMap);
},
AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs,
AzureBlobEtwConstants.MethodExecutionMaxRetryCount,
AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs);
success = true;
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
string.Format("Unable to create directory {0}.", this.localMap));
}
return(success);
}
public void BootstrapTraceProcessorMoveOnWritableTest()
{
var mockTraceOutputWriter = TestUtility.MockRepository.Create <ITraceOutputWriter>();
var startEvent = new ManualResetEvent(false);
var stopEvent = new ManualResetEvent(false);
mockTraceOutputWriter.Setup(w => w.OnBootstrapTraceFileScanStart()).Callback(() => { startEvent.Set(); });
mockTraceOutputWriter.Setup(w => w.OnBootstrapTraceFileScanStop()).Callback(() => { stopEvent.Set(); });
var traceDirectory = Path.Combine(DefaultWorkDirectory, "MoveOnWritableTest.TraceDirectory");
var markerFileDirectory = Path.Combine(DefaultWorkDirectory, "MoveOnWritableTest.MarkerFileDirectory");
var testFilename = Path.Combine(traceDirectory, TestFileName);
var markerFilename = Path.Combine(markerFileDirectory, TestFileName);
mockTraceOutputWriter.Setup(w => w.OnBootstrapTraceFileAvailable(testFilename)).Returns(true);
FabricDirectory.CreateDirectory(traceDirectory);
FabricDirectory.CreateDirectory(markerFileDirectory);
var processor = new BootstrapTraceProcessor(
traceDirectory,
markerFileDirectory,
new [] { mockTraceOutputWriter.Object },
DefaultScanInterval,
new ErrorAndWarningFreeTraceEventSource(),
TestLogSourceId);
processor.Start();
var stream = File.Open(testFilename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.Read);
stream.WriteByte(0);
// Wait for at least one pass
startEvent.Reset();
startEvent.WaitOne();
stopEvent.Reset();
stopEvent.WaitOne();
// Assert marker directory empty
Assert.AreEqual(0, Directory.EnumerateFiles(markerFileDirectory).Count(), "File should not be copied to marker directory until writeable.");
stream.Close();
// Wait for at least one pass
startEvent.Reset();
startEvent.WaitOne();
stopEvent.Reset();
stopEvent.WaitOne();
// Assert marker directory empty
Assert.AreEqual(
markerFilename,
Directory.EnumerateFiles(markerFileDirectory).First(),
"File should be copied to marker directory when closed.");
}
private void CreateDirectoryPath(string path)
{
string directoryName = FabricPath.GetDirectoryName(path);
if ((directoryName != null) && (directoryName.Length > 0) && (!FabricDirectory.Exists(directoryName)))
{
FabricDirectory.CreateDirectory(directoryName);
}
}
internal async Task GenerateWithValueType <ValueType>(string logFolder, ValueType value, IStateSerializer <ValueType> valueSerializer)
{
// directory setup
if (FabricDirectory.Exists(logFolder))
{
FabricDirectory.Delete(logFolder, true);
}
FabricDirectory.CreateDirectory(logFolder);
var rand = new Random();
var reliabilitySimulator = new ReliabilitySimulator(
logFolder,
new Uri("fabric:/unittest/service" + rand.Next()), // random service name.
OnDataLossCallback, // we are never calling OnDataLossAsync on this ReliabilitySimulator.
CreateStateProvider);
reliabilitySimulator.CreateReplica(true, false);
var replicator = reliabilitySimulator.GetTransactionalReplicator();
replicator.TryAddStateSerializer <ValueType>(valueSerializer);
// Constants
var distributedDictionary = new DistributedDictionary <long, ValueType>();
var distributedQueue = new DistributedQueue <ValueType>();
var concurrentQueue = new ReliableConcurrentQueue <ValueType>();
// Setup
await reliabilitySimulator.OpenReplicaAsync(ReplicaOpenMode.New).ConfigureAwait(false);
await reliabilitySimulator.PromoteReplicaAsync(false).ConfigureAwait(false);
var result = replicator.CreateAsyncEnumerable(false, false);
Assert.AreEqual(0, result.ToEnumerable().Count(), "State Manager must be empty");
// Write data.
using (var txn = replicator.CreateTransaction())
{
await replicator.AddStateProviderAsync(txn, DictionaryName, distributedDictionary).ConfigureAwait(false);
await txn.CommitAsync().ConfigureAwait(false);
}
result = replicator.CreateAsyncEnumerable(false, false);
Assert.AreEqual(2, result.ToEnumerable().Count(), "State Manager must include all the state providers");
await PopulateDictionaryAsync(replicator, DictionaryName, 0, 8, 8, value).ConfigureAwait(false);
// Take backup
await replicator.BackupAsync(ComplexDataBackupCallbackAsync).ConfigureAwait(false);
// Clean up.
await reliabilitySimulator.DropReplicaAsync();
}
private void CreateWindowsFabricTraceSubFolders()
{
string subFolder = Path.Combine(this.etwLogDirName, BootstrapTracesFolderName);
FabricDirectory.CreateDirectory(subFolder);
subFolder = Path.Combine(this.etwLogDirName, FabricTracesFolderName);
FabricDirectory.CreateDirectory(subFolder);
subFolder = Path.Combine(this.etwLogDirName, LeaseTracesFolderName);
FabricDirectory.CreateDirectory(subFolder);
}
internal BufferedEtwEventProvider(
ITraceEventSourceFactory traceEventSourceFactory,
string logSourceId,
string traceBufferingFolder,
TimeSpan uploadInterval,
TimeSpan eventDeletionAgeIn,
IBufferedEtwEventSink eventSink)
: base(traceEventSourceFactory, logSourceId)
{
this.eventSink = eventSink;
this.bookmark = null;
this.perfHelper = new BufferedEtwEventPerformance(this.TraceSource, this.LogSourceId);
this.eventDeletionAge = eventDeletionAgeIn;
// Compute how much time we spend delivering buffered ETW events to
// the consumer in each pass. We define this to be roughly 1/4th of
// the interval between passes. This is an arbitrary choice.
this.eventDeliveryPassLength = TimeSpan.FromMilliseconds(uploadInterval.TotalMilliseconds / 4);
// Create the buffered event directory, in case it doesn't already exist.
this.etwEventCache = traceBufferingFolder;
FabricDirectory.CreateDirectory(this.etwEventCache);
this.TraceSource.WriteInfo(
this.LogSourceId,
"Directory containing buffered events: {0}",
this.etwEventCache);
// Create a timer to schedule the delivery of ETW events to the consumer
string timerId = string.Concat(
this.LogSourceId,
EventDeliveryTimerIdSuffix);
this.eventDeliveryTimer = new DcaTimer(
timerId,
this.DeliverEventsToConsumer,
uploadInterval);
this.eventDeliveryTimer.Start();
// Create a timer to delete old logs
timerId = string.Concat(
this.LogSourceId,
OldLogDeletionTimerIdSuffix);
var oldLogDeletionInterval =
(this.eventDeletionAge < TimeSpan.FromDays(1))
? EtlConsumerConstants.OldLogDeletionIntervalForTest
: EtlConsumerConstants.OldLogDeletionInterval;
this.oldLogDeletionTimer = new DcaTimer(
timerId,
this.DeleteOldLogsHandler,
oldLogDeletionInterval);
this.oldLogDeletionTimer.Start();
}
private void GetPerformanceCounterConfiguration()
{
// Figure out if performance counter collection is enabled. By default
// it is enabled and the 'IsEnabled' setting must be explicitly
// specified to disable it.
this.perfCounterCollectionEnabled = this.configReader.GetUnencryptedConfigValue(
PerformanceCounterCommon.PerformanceCounterSectionName,
IsEnabledParamName,
true);
if (this.perfCounterCollectionEnabled)
{
// Get the path to the performance counter binary files
string binaryFilePath = this.configReader.GetUnencryptedConfigValue(
PerformanceCounterCommon.PerformanceCounterSectionName,
TestOnlyCounterFilePathParamName,
string.Empty);
if (false == string.IsNullOrEmpty(binaryFilePath))
{
// Non-default path was specified
this.perfCounterBinaryFolder = binaryFilePath;
this.perfCounterBinaryArchiveFolder = Path.Combine(
binaryFilePath,
FolderProducerConstants.PerformanceCountersBinaryArchiveDirectoryName);
this.archiveFolderIsUnderBinaryFolder = true;
}
else
{
// Use default path
this.perfCounterBinaryFolder = Path.Combine(
this.logDirectory,
FolderProducerConstants.PerformanceCountersBinaryDirectoryName);
this.perfCounterBinaryArchiveFolder = Path.Combine(
this.logDirectory,
FolderProducerConstants.PerformanceCountersBinaryArchiveDirectoryName);
}
// Create the directory where the performance counter binary files will be archived.
FabricDirectory.CreateDirectory(this.perfCounterBinaryArchiveFolder);
this.traceSource.WriteInfo(
this.logSourceId,
"Performance counter data collection is enabled. Performance counter binary folder: {0}, performance counter binary archive folder: {1}, performance counter output folder: {2}.",
this.perfCounterBinaryFolder,
this.perfCounterBinaryArchiveFolder,
this.outputFolderPath);
}
else
{
this.traceSource.WriteInfo(
this.logSourceId,
"Performance counter data collection has been disabled via the {0} parameter in the {1} section of the cluster manifest.",
IsEnabledParamName,
PerformanceCounterCommon.PerformanceCounterSectionName);
}
}
private void CreateDirectoriesForEtlProcessing()
{
// Create the directory to hold marker files for fully
// processed ETL files
FabricDirectory.CreateDirectory(this.markerFileDirectory);
this.traceSource.WriteInfo(
this.logSourceId,
"Marker file directory: {0}",
this.markerFileDirectory);
}
private void GenerateWorkFolders()
{
string workFolder = this.nodeSettings.DeploymentFoldersInfo.WorkFolder;
DeployerTrace.WriteNoise("Deploying work folder {0}", workFolder);
#if DotNetCoreClrLinux
FabricDirectory.CreateDirectory(workFolder);
#else
Directory.CreateDirectory(workFolder);
#endif
}
private static ulong GetNewContainerInstanceIdAtomic(
ContainerDescription containerDesc)
{
#if DotNetCoreClrLinux
// There is a file called containerid in
// /log/application/partitionid/servicepackageactivationid/codepackagename/
// This file contains the number corresponding to the latest folder for logs
// /mnt/log/application/partitionid/servicepackageactivationid/codepackagename/number/application.log
//
// For the first instance of thecontainer, create a new file and start at 0,
// For every container afterwards, check the file exists, read from the file
// and add 1 to the value in the file in order to get the new current instance id
// and save the value to the file.
var applicationNamePart = containerDesc.ApplicationName;
var applicationNameExtraPart = string.Format("{0}/", RootNamingUriString);
if (applicationNamePart.StartsWith(applicationNameExtraPart))
{
applicationNamePart = applicationNamePart.Substring(applicationNameExtraPart.Length);
}
applicationNamePart = applicationNamePart.Replace('/', '_');
var digestedApplicationName = applicationNamePart + DigestedApplicationNameDockerSuffix;
var logRoots = "/mnt/logs";
var logPath = Path.Combine(logRoots, digestedApplicationName, containerDesc.PartitionId, containerDesc.ServicePackageActivationId, containerDesc.CodePackageName);
if (!FabricDirectory.Exists(logPath))
{
FabricDirectory.CreateDirectory(logPath);
}
ulong containerId = 0;
var containerInstanceIdFilePath = Path.Combine(logPath, CodePackageInstanceIdFileName);
if (!FabricFile.Exists(containerInstanceIdFilePath))
{
using (var containerIdFile = FabricFile.Create(containerInstanceIdFilePath))
{
containerId = 0;
}
}
else
{
containerId = ReadContainerIdFromFile(containerInstanceIdFilePath) + 1;
}
WriteContainerIdToFile(containerInstanceIdFilePath, containerId);
return(containerId);
#else
return(0);
#endif
}
private void DeployConfig(SettingsType settings)
{
string configDeploymentPath = this.nodeSettings.DeploymentFoldersInfo.ConfigDeploymentDirectory;
#if DotNetCoreClrLinux
FabricDirectory.CreateDirectory(configDeploymentPath);
#else
Directory.CreateDirectory(configDeploymentPath);
#endif
string configFilePath = Path.Combine(configDeploymentPath, Constants.FileNames.Settings);
DeployerTrace.WriteNoise("Generating the settings file at {0}", configFilePath);
XmlHelper.WriteXmlExclusive <SettingsType>(configFilePath, settings);
}
public void FabricDirectory_CreateNegative()
{
try
{
LogHelper.Log("FabricDirectory.Create {0}", BadPath);
FabricDirectory.CreateDirectory(BadPath);
Assert.Fail("should never reach here");
}
catch (Exception e)
{
LogHelper.Log("caught exception {0}", e);
Assert.IsTrue(e is IOException);
}
}
private bool TryUpdateLocalMap(int retryCount, string localMapDestinationDir, string localMapDestination, DateTime sourceLastWriteTime)
{
// Record the source file's last write time based on the snapshot
// we took before copying the source file. We know that we copied
// over a version that it at least as new as the timestamp in the
// snapshot.
// Figure out in which directory in the local map we will
// record information about this file. Create that directory
// if it doesn't exist.
if ((false == string.IsNullOrEmpty(localMapDestinationDir)) &&
(false == FabricDirectory.Exists(localMapDestinationDir)))
{
try
{
Utility.PerformIOWithRetries(
() => { FabricDirectory.CreateDirectory(localMapDestinationDir); },
retryCount);
}
catch (Exception e)
{
this.TraceSource.WriteExceptionAsError(
this.LogSourceId,
e,
"Failed to create directory {0} in the local map of uploaded files.",
localMapDestinationDir);
return(false);
}
}
// If the file doesn't exist in the local map, create it.
if (false == FabricFile.Exists(localMapDestination))
{
if (!this.TryCreateLocalMapFile(retryCount, localMapDestination, sourceLastWriteTime))
{
return(false);
}
}
else
{
// File already exists in the local map. Set the last
// write time for the file in the local map to the last
// write time from the source file snapshot
// It's a Best Effort operation.
this.TryUpdateLastWriteTime(localMapDestination, sourceLastWriteTime);
}
return(true);
}
private void GenerateDataDeployment()
{
string dataDeploymentPath = this.nodeSettings.DeploymentFoldersInfo.DataDeploymentDirectory;
#if DotNetCoreClrLinux
FabricDirectory.CreateDirectory(dataDeploymentPath);
#else
Directory.CreateDirectory(dataDeploymentPath);
#endif
InfrastructureInformationType infrastructureManifest = this.infrastructure.GetInfrastructureManifest();
string filePath = this.nodeSettings.DeploymentFoldersInfo.InfrastructureManfiestFile;
DeployerTrace.WriteNoise("Writing infrastructure manifest file at {0}", filePath);
WriteManifestFile <InfrastructureInformationType>(filePath, infrastructureManifest);
CopyClusterSettingsToFabricData();
}
public static void UpdateFileLocation(string traceFolder)
{
string drive = Path.GetPathRoot(traceFolder);
#if DotNetCoreClrLinux
string traceFileName = string.Format(CultureInfo.InvariantCulture, "FabricDeployer-{0}.trace", DateTime.UtcNow.Ticks);
if (!FabricDirectory.Exists(traceFolder))
{
WriteInfo("Trace folder doesn't exist. Creating trace folder: {0}", traceFolder);
FabricDirectory.CreateDirectory(traceFolder);
}
#else
if (!Directory.Exists(drive))
{
string newTraceFolder = Path.Combine(Path.GetTempPath(), "FabricDeployerTraces");
WriteInfo("Default trace destination does not exist: {0}. Using directory instead: {1}.", traceFolder, newTraceFolder);
traceFolder = newTraceFolder;
}
string traceFileName = string.Format(CultureInfo.InvariantCulture, "FabricDeployer-{0}.trace", DateTime.Now.Ticks);
if (!Directory.Exists(traceFolder))
{
WriteInfo("Trace folder doesn't exist. Creating trace folder: {0}", traceFolder);
Directory.CreateDirectory(traceFolder);
}
#endif
else
{
WriteInfo("Trace folder already exists. Traces will be written to existing trace folder: {0}", traceFolder);
}
string traceFileFullPath = Path.Combine(traceFolder, traceFileName);
#if DotNetCoreClr
/* TBD: Fix following code.
* if (deployerFileTraces != null)
* {
* deployerFileTraces.Dispose();
* }
* deployerFileTraces = new FileEventListener(traceFileFullPath);
* deployerFileTraces.EnableEvents(FabricEvents.Events, EventLevel.Informational);
*/
#else
TraceTextFileSink.SetPath(traceFileFullPath);
TraceConfig.SetDefaultLevel(TraceSinkType.TextFile, EventLevel.Informational);
#endif
}
//
// Fabric Runtime environment is not made visible inside the container if RemoveServiceFabricRuntimeAccess is specified.
//
private static void PopulateFabricRuntimeEnvironment(
ContainerConfig containerConfig,
ProcessDescription processDesc,
ContainerDescription containerDesc)
{
var packageFilePath = string.Empty;
if (processDesc.EnvVars.ContainsKey(FabricPackageFileNameEnvironment))
{
packageFilePath = processDesc.EnvVars[FabricPackageFileNameEnvironment];
}
var fabricContainerLogRoot = Path.Combine(Utility.FabricLogRoot, "Containers");
var fabricContainerRoot = Path.Combine(fabricContainerLogRoot, containerDesc.ContainerName);
FabricDirectory.CreateDirectory(fabricContainerRoot);
var fabricContainerTraceRoot = Path.Combine(fabricContainerRoot, "Traces");
FabricDirectory.CreateDirectory(fabricContainerTraceRoot);
var fabricContainerQueryTraceRoot = Path.Combine(fabricContainerRoot, "QueryTraces");
FabricDirectory.CreateDirectory(fabricContainerQueryTraceRoot);
var fabricContainerOperationalTraceRoot = Path.Combine(fabricContainerRoot, "OperationalTraces");
FabricDirectory.CreateDirectory(fabricContainerOperationalTraceRoot);
// Set path to UT file
// TODO: Bug#9728016 - Disable the bind until windows supports mounting file onto container
#if DotNetCoreClrLinux
PopulateFabricRuntimeEnvironmentForLinux(
containerConfig,
containerDesc,
processDesc,
packageFilePath,
fabricContainerRoot);
#else
PopulateFabricRuntimeEnvironmentForWindows(
containerConfig,
containerDesc,
processDesc,
packageFilePath,
fabricContainerRoot);
#endif
}
private static void AddContainerLogMount(
ContainerConfig containerConfig,
ContainerDescription containerDesc)
{
// Create mounting Path
// Pattern is Root/ApplicationName/PartitionId/ServicePackageActivationId/CodePackageName,
// we will mount up to Root/ApplicationName/PartitionId/ServicePackageActivationId
// ApplicationName might come as fabric:/MyAppName. User really only cares about the MyAppName portion.
// The / character in the app name will be converted to _ for the directory
// so we can keep only one directory instead of multiple (1 for each / in the name)
// If the name is fabric:/MyAppName/AnotherPartOfTheName, we will convert it to MyAppName_AnotherPartOfTheName
// On linux the mount root is /mnt/logs/
#if DotNetCoreClrLinux
var applicationNamePart = containerDesc.ApplicationName;
var applicationNameExtraPart = string.Format("{0}/", RootNamingUriString);
if (applicationNamePart.StartsWith(applicationNameExtraPart))
{
applicationNamePart = applicationNamePart.Substring(applicationNameExtraPart.Length);
}
applicationNamePart = applicationNamePart.Replace('/', '_');
containerConfig.Labels.Add(SimpleApplicationNameLabelKeyName, applicationNamePart);
var digestedApplicationName = applicationNamePart + DigestedApplicationNameDockerSuffix;
containerConfig.Labels.Add(DigestedApplicationNameLabelKeyName, digestedApplicationName);
var logRoots = "/mnt/logs";
var logPath = Path.Combine(logRoots, digestedApplicationName, containerDesc.PartitionId, containerDesc.ServicePackageActivationId);
if (Path.GetFullPath(logPath).Equals(Path.GetFullPath(logRoots)))
{
// We do no want to mount the root path to a container
return;
}
// Create path
FabricDirectory.CreateDirectory(logPath);
// Add to environment label
containerConfig.Env.Add(string.Format("{0}={1}", UserLogsDirectoryEnvVarName, logPath));
// Add to mount options
containerConfig.HostConfig.Binds.Add(string.Format("{0}:{0}", logPath));
#endif
}
public static void WriteXml <T>(string fileName, T value)
{
var directoryName = FabricPath.GetDirectoryName(fileName);
if (!FabricDirectory.Exists(directoryName))
{
FabricDirectory.CreateDirectory(directoryName);
}
using (var stream = FabricFile.Open(fileName, FileMode.Create, FileAccess.Write, FileShare.Read))
using (var writer = XmlWriter.Create(stream))
{
var serializer = new XmlSerializer(typeof(T));
serializer.Serialize(writer, value);
}
#if DotNetCoreClrLinux
Helpers.UpdateFilePermission(fileName);
#endif
}
public void TestMethodSetup()
{
// Initialization
bool result = StartDtrRead.Reset() && DtrReadCompleted.Reset() && EndOfTest.Reset();
Verify.IsTrue(result, "Successfully reset all events at the start of the test");
// Parse the configuration file
this.testStartTime = DateTime.UtcNow;
ParseConfig(testStartTime);
// Create the log and work directories
Utility.LogDirectory = Path.Combine(testDataDirectory, this.testStartTime.Ticks.ToString(), LogFolderName);
Utility.TraceSource.WriteInfo(
TraceType,
"Log folder: {0}",
Utility.LogDirectory);
Utility.InitializeWorkDirectory();
// Create the deleted items folder
this.deletedItemsFolder = Path.Combine(testDataDirectory, this.testStartTime.Ticks.ToString(), DeletedItemsFolderName);
FabricDirectory.CreateDirectory(deletedItemsFolder);
Utility.TraceSource.WriteInfo(
TraceType,
"Deleted items folder: {0}",
this.deletedItemsFolder);
// Create the MDS upload consumer
ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null);
var initParam = new ConsumerInitializationParameters(
Utility.WindowsFabricApplicationInstanceId,
TestConfigSectionName,
TestFabricNodeId,
TestFabricNodeName,
Utility.LogDirectory,
Utility.DcaWorkFolder,
new DiskSpaceManager());
this.uploader = new MdsEtwEventUploader(initParam);
EtwCsvFolder = this.uploader.EtwCsvFolder;
}
public void FabricFile_GetSize_FileOpenInReadShare_Succeeds()
{
string filePath = Path.Combine(this.testPath, this.testFileName);
LogHelper.Log("FabricDirectory.Create {0}", this.testPath);
FabricDirectory.CreateDirectory(this.testPath);
LogHelper.Log("FabricFile.Create {0}", filePath);
using (var writer = new StreamWriter(FabricFile.Create(filePath)))
{
writer.WriteLine(this.testString);
}
LogHelper.Log("FabricFile.Open {0}", filePath);
using (var filestream = FabricFile.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
{
long fileSize = FabricFile.GetSize(filePath);
LogHelper.Log("FabricFile.GetSize {0}", fileSize);
Assert.IsTrue(fileSize > 0, "FabricFile.GetSize should return non-zero file size.");
}
}
private bool UpdateLocalMap(string localMapDestinationDir, string localMapDestination)
{
// Figure out in which directory in the local map we will
// record information about this file. Create that directory
// if it doesn't exist.
if ((false == string.IsNullOrEmpty(localMapDestinationDir)) &&
(false == FabricDirectory.Exists(localMapDestinationDir)))
{
try
{
Utility.PerformIOWithRetries(
() => { FabricDirectory.CreateDirectory(localMapDestinationDir); },
AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs,
AzureBlobEtwConstants.MethodExecutionMaxRetryCount,
AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs);
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
"Failed to create directory {0} in the local map of uploaded files.",
localMapDestinationDir);
return(false);
}
}
// If the file doesn't exist in the local map, create it.
if (false == FabricFile.Exists(localMapDestination))
{
if (false == this.CreateLocalMapFile(localMapDestination))
{
return(false);
}
}
return(true);
}
private void CreateTraceFiles(int pass)
{
for (int i = 0; i < (int)TraceFileType.Count; i++)
{
// The MDS uploader creates a sub-folder at the destination with the same name as the
// trace subfolder name at the source. Therefore, we include the test start time in
// the trace subfolder name so that we get a different sub-folder at the destination
// for each test.
string traceSubFolderName = Path.Combine(
EtwCsvFolder,
String.Concat(
traceSubFolderNames[(TraceFileType)i],
this.testStartTime.Ticks.ToString()));
FabricDirectory.CreateDirectory(traceSubFolderName);
IEnumerable <TraceFileInfo> traceFilesToCreate = this.traceFileInfo[(TraceFileType)i]
.Where(t => (t.CreationPass == pass));
foreach (TraceFileInfo traceFileToCreate in traceFilesToCreate)
{
CreateTraceFile(traceFileToCreate, traceSubFolderName);
}
}
}