/// <summary>
/// All files at the given folder
/// </summary>
/// <param name="storeSource">Location (relative to root)</param>
/// <param name="timeout">List timeout</param>
/// <returns>File name of each files within the given location</returns>
public IEnumerable <string> ListContent(string storeSource, TimeSpan timeout)
{
List <string> files = new List <string>();
try
{
string smbTag = this.ConvertTagToSMBPath(storeSource);
if (!FabricDirectory.Exists(smbTag))
{
TraceSource.WriteWarning(
TraceType,
"Directory {0} doesn't exist",
smbTag);
}
else
{
this.GetAllFileFullNames(smbTag, timeout).ForEach <string>(fullName => files.Add(fullName.Substring(this.rootUri.Length + 1)));
}
}
catch (IOException exception)
{
if (exception.GetType() == typeof(IOException))
{
throw new FabricImageStoreException(StringResources.Error_ImageStoreIOException, exception);
}
else
{
throw;
}
}
return(files);
}
private Task <bool> OnFullBackupCallback(BackupInfo backupInfo, CancellationToken cancellationToken)
{
Assert.IsFalse(cancellationToken.IsCancellationRequested);
FullBackupFolderPath = Path.Combine(ClassTestPath, BackupContainerFolderName, Guid.NewGuid().ToString("N"));
FabricDirectory.Copy(backupInfo.Directory, FullBackupFolderPath, false);
return(Task.FromResult(true));
}
public static string ComputeHash(string resourcePath, bool imageStoreServiceEnabled)
{
byte[] hash = null;
if (FabricFile.Exists(resourcePath))
{
hash = ComputeHashOnFile(resourcePath);
}
else if (FabricDirectory.Exists(resourcePath))
{
hash = ComputeHashOnDirectory(resourcePath, imageStoreServiceEnabled);
}
else
{
System.Fabric.Interop.Utility.ReleaseAssert(true, "{0} is not found.");
}
StringBuilder hashStringBuilder = new StringBuilder();
for (int i = 0; i < hash.Length; i++)
{
hashStringBuilder.Append(string.Format(CultureInfo.InvariantCulture, "{0:X2}", hash[i]));
}
return(hashStringBuilder.ToString());
}
private bool IsStagingFolderNeededForCopy(string sourceFolder)
{
if (FileShareAccessAccountType.None != this.accessInfo.AccountType)
{
// We are impersonating in order to copy files to destination.
// Check if we can access the source folder while impersonating.
bool sourceAccessDeniedUnderImpersonation = false;
using (WindowsIdentity identity = this.GetIdentityToImpersonate())
{
#if DotNetCoreClr
WindowsIdentity.RunImpersonated(identity.AccessToken, () =>
#else
using (WindowsImpersonationContext impersonationCtx = identity.Impersonate())
#endif
{
try
{
// Use Directory.GetFiles directly instead of using FabricDirectory.GetFiles
// because the latter fails silently and returns an empty array if access is
// denied.
FabricDirectory.GetFiles(sourceFolder);
}
catch (UnauthorizedAccessException)
{
// We are unable to access the source folder while impersonating.
// Hence we need a staging folder, which we can access while
// impersonating.
sourceAccessDeniedUnderImpersonation = true;
}
}
/// <summary>
/// Creates bookmark directory.
/// </summary>
/// <param name="sourceFolder"></param>
/// <param name="workDirectory"></param>
/// <param name="destinationKey"></param>
/// <param name="bookmarkFolder"></param>
/// <returns></returns>
private bool CreateBookmarkSubDirectory(
string sourceFolder,
string workDirectory,
string destinationKey,
out string bookmarkFolder)
{
string bookmarkParentFolder = Path.Combine(
workDirectory,
Utility.ShortWindowsFabricIdForPaths);
bool success = Utility.CreateWorkSubDirectory(
this.traceSource,
this.logSourceId,
destinationKey,
sourceFolder,
bookmarkParentFolder,
out bookmarkFolder);
if (success)
{
bookmarkFolder = Path.Combine(bookmarkFolder, BookmarkDirName);
}
FabricDirectory.CreateDirectory(bookmarkFolder);
return(success);
}
private void TestIntersectingHardLinkAndReplace(bool isLongPath, FileShare fileShare)
{
var folderPath = this.testPath;
if (true == isLongPath)
{
folderPath = this.ExtendPath(folderPath);
}
var filePath = Path.Combine(folderPath, this.testFileName);
Assert.IsTrue(!isLongPath || filePath.Length > 260, "file path must be greater than max path size.");
var hardLinkedFilePath = Path.Combine(folderPath, this.testHardLinkedFileName);
Assert.IsTrue(!isLongPath || hardLinkedFilePath.Length > 260, "hard linked file path must be greater than max path size.");
var replaceFilePath = Path.Combine(folderPath, this.testReplaceFileName);
Assert.IsTrue(!isLongPath || replaceFilePath.Length > 260, "replace file path must be greater than max path size.");
var backupFilePath = Path.Combine(folderPath, this.testBackupFileName);
Assert.IsTrue(!isLongPath || backupFilePath.Length > 260, "backup file path must be greater than max path size.");
LogHelper.Log("FabricDirectory.Create {0}", folderPath);
FabricDirectory.CreateDirectory(folderPath);
LogHelper.Log("FabricFile.Create {0}", filePath);
using (StreamWriter streamWriter = new StreamWriter(FabricFile.Create(filePath)))
{
LogHelper.Log("Write {0}", this.testString);
streamWriter.WriteLine(this.testString);
streamWriter.Flush();
}
using (StreamWriter streamWriter = new StreamWriter(FabricFile.Create(replaceFilePath)))
{
LogHelper.Log("Write {0}", this.testNewString);
streamWriter.WriteLine(this.testNewString);
streamWriter.Flush();
}
FabricFile.CreateHardLink(hardLinkedFilePath, filePath);
using (StreamReader hardLinkReader0 = new StreamReader(FabricFile.Open(hardLinkedFilePath, FileMode.Open, FileAccess.Read, fileShare)))
{
var hardLinkContent0 = hardLinkReader0.ReadLine();
Assert.AreEqual <string>(this.testString, hardLinkContent0, "after replace hard link file must have the old content.");
try
{
FabricFile.Replace(replaceFilePath, filePath, backupFilePath, false);
Assert.Fail();
}
catch (FileLoadException)
{
}
}
}
/// <summary>
/// Post-processing of copying (from XStore to SMB)
/// </summary>
/// <param name="task">The task to end.</param>
private static void EndCopyFromXStoreToSMB(XStoreFileOperationTask task)
{
if (task.IsSucceeded)
{
// Step 3, 4 and 5 for atomic copy
// 3. delete dstFolder.old
// 4. rename dstFolder -> dstFolder.old
// 5. rename dstFolder.new -> dstFolder
// get the original destination path
string newDstRootUri = task.DstUri + NewExtension;
string oldDstRootUri = task.DstUri + OldExtension;
// Step 3, 4 & 5
if (FabricDirectory.Exists(oldDstRootUri))
{
// delete dstFolder.old
FabricDirectory.Delete(oldDstRootUri, recursive: true, deleteReadOnlyFiles: true);
}
if (FabricDirectory.Exists(task.DstUri))
{
// rename dstFolder -> dstFolder.old
RenameFolder(task.DstUri, oldDstRootUri);
}
// rename dstFolder.new -> dstFolder; and clean up
RenameFolder(newDstRootUri, task.DstUri);
if (FabricDirectory.Exists(oldDstRootUri))
{
FabricDirectory.Delete(oldDstRootUri, recursive: true, deleteReadOnlyFiles: true);
}
}
}
/// <summary>
/// Check if content is present in the store.
/// </summary>
/// <param name="tag"> Location (relative to RootUri) where to check the presence of the content. </param>
/// <param name="timeout">Check existence timeout</param>
/// <returns> True if the content exists, false otherwise. </returns>
public bool DoesContentExist(string tag, TimeSpan timeout)
{
TimeoutHelper helper = timeout == TimeSpan.MaxValue ? null : new TimeoutHelper(timeout);
#if !DotNetCoreClr
using (WindowsImpersonationContext impersonationContext = this.GetImpersonationContext())
#endif
{
string smbTag = this.ConvertTagToSMBPath(tag);
if (FabricFile.Exists(smbTag))
{
return(!FileLock.DoesWriterLockExist(smbTag));
}
if (helper != null)
{
helper.ThrowIfExpired();
}
if (FabricDirectory.Exists(smbTag))
{
return(!FileLock.DoesWriterLockExist(smbTag));
}
}
return(false);
}
private bool EnumerateFiles(object context)
{
var fileEnumerateInfo = (FileEnumerateInfo)context;;
fileEnumerateInfo.Files = FabricDirectory.GetFiles(fileEnumerateInfo.DirectoryPath, fileEnumerateInfo.FileExtension, fileEnumerateInfo.SearchOptionEnum);
return(true);
}
private static void WriteStringToFile(string fileName, string value, bool writeLine = true, Encoding encoding = null)
{
var directory = FabricPath.GetDirectoryName(fileName);
if (!FabricDirectory.Exists(directory))
{
FabricDirectory.CreateDirectory(directory);
}
if (encoding == null)
{
#if DotNetCoreClrLinux
encoding = new UTF8Encoding(false);
#else
encoding = Encoding.GetEncoding(0);
#endif
}
using (StreamWriter writer = new StreamWriter(FabricFile.Open(fileName, FileMode.OpenOrCreate), encoding))
{
if (writeLine)
{
writer.WriteLine(value);
}
else
{
writer.Write(value);
}
#if DotNetCoreClrLinux
Helpers.UpdateFilePermission(fileName);
#endif
}
}
public CheckpointManager(
IEnumerable <EtlInMemoryProducerWorker.ProviderInfo> providers,
string etlPath,
string logDirectory,
string traceDirectory,
FabricEvents.ExtensionsEvents traceSource,
string logSourceId)
{
this.traceSource = traceSource;
this.logSourceId = logSourceId;
// Directory where the producer saves the file containing information
// about the point upto which we last read the ETL files
var lastEtlFileReadFilePath = InitializeLastEtlFileReadFilePath(
etlPath,
logDirectory,
traceDirectory);
// If the directory containing the last ETL read information is
// different from the log directory, then create it now.
if (false == lastEtlFileReadFilePath.Equals(
logDirectory))
{
FabricDirectory.CreateDirectory(lastEtlFileReadFilePath);
this.traceSource.WriteInfo(
this.logSourceId,
"Directory containing last ETL read info: {0}",
lastEtlFileReadFilePath);
}
this.SetProviderCheckpointFileName(providers, lastEtlFileReadFilePath);
}
public void FabricDirectory_GetFilesNegative()
{
LogHelper.Log("FabricDirectory.GetFiles {0}", BadPath);
var res = FabricDirectory.GetFiles(BadPath);
Assert.AreEqual(0, res.Length);
}
private void RenameFolder(string srcFolderName, string dstFolderName)
{
if (string.IsNullOrEmpty(srcFolderName) || !FabricDirectory.Exists(srcFolderName))
{
this.traceSource.WriteError(ClassName, "SourceFolder {0} doesn't exist", srcFolderName);
throw new ArgumentException(StringResources.ImageStoreError_InvalidSourceFolderSpecified);
}
if (string.IsNullOrEmpty(dstFolderName) || FabricDirectory.Exists(dstFolderName))
{
this.traceSource.WriteError(ClassName, "Destination folder {0} does exist", dstFolderName);
throw new ArgumentException(StringResources.ImageStoreError_InvalidDestinationFolderSpecified);
}
try
{
FabricFile.Move(srcFolderName, dstFolderName);
}
catch (Exception ex)
{
this.traceSource.WriteError(
ClassName,
"Failed to rename folder {0} to {1} with error {2}, and detail exception {3}",
srcFolderName,
dstFolderName,
ex.Message,
ex);
throw;
}
}
private void GenerateCodeDeployment()
{
if (!this.nodeSettings.DeploymentFoldersInfo.IsCodeDeploymentNeeded)
{
return;
}
foreach (string service in this.servicesToBeDeployed)
{
string destinationFolder = this.nodeSettings.DeploymentFoldersInfo.GetCodeDeploymentDirectory(service);
string sourceFolder = this.nodeSettings.DeploymentFoldersInfo.GetInstalledBinaryDirectory(service);
if (IsCodeDeploymentNeeded(service, sourceFolder, destinationFolder))
{
try
{
FabricDirectory.Copy(sourceFolder, destinationFolder, true);
}
catch (Exception e)
{
DeployerTrace.WriteError("Code deployment failed because: {0}. Source folder: {1}. destinationFolder: {2}", e, sourceFolder, destinationFolder);
throw;
}
}
}
}
public static void ClassSetup(object testContext)
{
if (FabricDirectory.Exists(DefaultWorkDirectory))
{
FabricDirectory.Delete(DefaultWorkDirectory, true);
}
}
/// <summary>
/// Copy Folder method called by the CopyItem delegate to dequeue a folder copy item
/// This method enumerates the folder enqueues all its children and dequeues the current item
/// </summary>
/// <param name="sourceDir">
/// Source folder to be copied
/// </param>
/// <param name="destinationDir">
/// Destination folder
/// </param>
private void CopyFolder(string sourceDir, string destinationDir)
{
// create the destination folder if it doesn't exist
if (!FabricDirectory.Exists(destinationDir))
{
FabricDirectory.CreateDirectory(destinationDir);
}
// Enumerate source and queue up work items for each child folder
var directoryNames = FabricDirectory.GetDirectories(
sourceDir,
"*",
false, // full path
SearchOption.TopDirectoryOnly);
var fileNames = FabricDirectory.GetFiles(
sourceDir,
"*",
false, // full path
SearchOption.TopDirectoryOnly);
foreach (var directoryName in directoryNames)
{
this.QueueCopyItem(sourceDir, destinationDir, directoryName, true);
}
foreach (var fileName in fileNames)
{
this.QueueCopyItem(sourceDir, destinationDir, fileName, false);
}
}
public void TestCleanup()
{
if (FabricDirectory.Exists(this.testPath))
{
FabricDirectory.Delete(this.testPath, true);
}
}
public CheckpointManager(
bool isReadingFromApplicationManifest,
string etlPath,
string logDirectory,
string traceDirectory,
string producerInstanceId,
FabricEvents.ExtensionsEvents traceSource,
string logSourceId)
{
this.traceSource = traceSource;
this.logSourceId = logSourceId;
// Directory where the producer saves the file containing information
// about the point upto which we last read the ETL files
this.lastEtlFileReadFileDirectoryPath = InitializeLastEtlFileReadFilePath(
isReadingFromApplicationManifest,
etlPath,
logDirectory,
traceDirectory,
producerInstanceId);
// If the directory containing the last ETL read information is
// different from the log directory, then create it now.
if (false == this.lastEtlFileReadFileDirectoryPath.Equals(
logDirectory))
{
FabricDirectory.CreateDirectory(this.lastEtlFileReadFileDirectoryPath);
this.traceSource.WriteInfo(
this.logSourceId,
"Directory containing last ETL read info: {0}",
this.lastEtlFileReadFileDirectoryPath);
}
}
private static void PopulateVolumes(
ContainerConfig containerConfig,
ContainerDescription containerDesc)
{
foreach (var volume in containerDesc.Volumes)
{
#if !DotNetCoreClrLinux
if (string.IsNullOrEmpty(volume.Driver) && !FabricDirectory.Exists(volume.Source))
{
throw new FabricException(
string.Format(
"The source location specified for container volume does not exist. Source:{0}",
volume.Source),
FabricErrorCode.DirectoryNotFound);
}
#endif
var bindFormat = volume.IsReadOnly ? "{0}:{1}:ro" : "{0}:{1}";
containerConfig.HostConfig.Binds.Add(string.Format(bindFormat, volume.Source, volume.Destination));
}
foreach (var bind in containerDesc.BindMounts)
{
containerConfig.HostConfig.Binds.Add(
string.Format("{0}:{1}:ro", bind.Value, bind.Key));
}
}
private void SetInformationTest(bool isLongPath, Kernel32Types.PRIORITY_HINT priorityHintOne, Kernel32Types.PRIORITY_HINT priorityHintTwo)
{
var folderPath = this.testPath;
if (isLongPath == true)
{
folderPath = this.ExtendPath(folderPath);
}
var filePath = Path.Combine(folderPath, this.testFileName);
Assert.IsTrue(!isLongPath || filePath.Length > 260, "file path must be greater than max path size.");
FabricDirectory.CreateDirectory(folderPath);
using (FileStream fileStream = FabricFile.Open(filePath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read))
{
var fileHandle = fileStream.SafeFileHandle;
// Verify that the default is Normal.
this.VerifyPriorityHint(fileHandle, Kernel32Types.PRIORITY_HINT.IoPriorityHintNormal);
this.SetPriorityHint(fileHandle, priorityHintOne);
this.VerifyPriorityHint(fileHandle, priorityHintOne);
if (priorityHintTwo != Kernel32Types.PRIORITY_HINT.MaximumIoPriorityHintType)
{
this.SetPriorityHint(fileHandle, priorityHintTwo);
this.VerifyPriorityHint(fileHandle, priorityHintTwo);
}
}
}
/// <summary>
/// Initializes a new instance of the FileImageStore class.
/// </summary>
/// <param name="uri">The root URI (the root folder) of the image store.</param>
/// <param name="localRoot">The default path for the file image store root.</param>
/// <param name="accessDescription">Access rights to specific users or groups.</param>
public FileImageStore(
string uri,
string localRoot = null,
ImageStoreAccessDescription accessDescription = null)
{
if (uri == null)
{
throw new ArgumentException(StringResources.ImageStoreError_InvalidParameterSpecified, "uri");
}
this.localRoot = localRoot;
this.accessDescription = accessDescription;
this.storeParams = new FileProviderParameters(uri);
if (!this.storeParams.RootUri.StartsWith(SchemeTag, StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException(string.Format(CultureInfo.CurrentCulture, StringResources.ImageStoreError_InvalidImageStoreUriScheme, this.storeParams.RootUri));
}
this.rootUri = this.storeParams.RootUri.TrimStart().Substring(SchemeTag.Length).Trim();
#if !DotNetCoreClr
using (WindowsImpersonationContext impersonationContext = this.GetImpersonationContext())
#endif
{
if (!FabricDirectory.Exists(this.rootUri))
{
throw new DirectoryNotFoundException(string.Format(CultureInfo.CurrentCulture, StringResources.ImageStoreError_UnaccessibleImageStore, this.rootUri));
}
}
}
private bool EnumerateFolders(object context)
{
var folderEnumerateInfo = (FolderEnumerateInfo)context;
folderEnumerateInfo.SubFolders = FabricDirectory.GetDirectories(folderEnumerateInfo.DirectoryPath, "*", folderEnumerateInfo.GetFullPath, SearchOption.TopDirectoryOnly).Where(x => x.CompareTo(folderEnumerateInfo.DirectoryPathContinuationToken) >= 0).ToList();
return(true);
}
private bool CreateLocalMapFolder()
{
bool success = false;
if (FabricDirectory.Exists(this.localMap))
{
return(true);
}
try
{
Utility.PerformIOWithRetries(
() =>
{
FabricDirectory.CreateDirectory(this.localMap);
},
AzureBlobEtwConstants.MethodExecutionInitialRetryIntervalMs,
AzureBlobEtwConstants.MethodExecutionMaxRetryCount,
AzureBlobEtwConstants.MethodExecutionMaxRetryIntervalMs);
success = true;
}
catch (Exception e)
{
this.traceSource.WriteExceptionAsError(
this.logSourceId,
e,
string.Format("Unable to create directory {0}.", this.localMap));
}
return(success);
}
/// <summary>
/// Simply reads the folder to populate the backup folder.
/// This API does not trim or validate the backup folder.
/// Note: Using List instead of SortedList to allow same keys added to the List, this is due to
/// replicate backup log record may throw transient exception, then user may re-try the
/// BackupAsync call and leads to same backups uploaded again. In this case, add to SortedList
/// will fail since same key is not allowed.
/// </summary>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Task that represents the asynchronous work.</returns>
/// <exception cref="FabricMissingFullBackupException">A valid full backup folder to be the head of the backup chain could not be found.</exception>
/// <exception cref="ArgumentException">The relevant folder is not a valid full backup folder.</exception>
/// <exception cref="InvalidDataException">There is a corruption in the file.</exception>
internal async Task ReadAsync(CancellationToken cancellationToken)
{
string[] list = FabricDirectory.GetFiles(
this.backupFolder,
Replicator.Constants.ReplicatorBackupMetadataFileName,
SearchOption.AllDirectories);
if (list.Length != 1)
{
// Note that we are returning full backup missing if there is no full backup or if there is more than full backup.
// If this is confusing, we could change this to ArgumentException since it already is in the set of possible exceptions.
throw new FabricMissingFullBackupException(SR.Error_BackupFolderInfo_MissingFullBackup);
}
string fullBackupDirectoryName = Path.GetDirectoryName(list[0]);
await this.AddFullBackupFolderAsync(fullBackupDirectoryName, cancellationToken).ConfigureAwait(false);
string[] incrementalBackupMetadataFilePaths = FabricDirectory.GetFiles(
this.backupFolder,
Replicator.Constants.ReplicatorIncrementalBackupMetadataFileName,
SearchOption.AllDirectories);
foreach (string path in incrementalBackupMetadataFilePaths)
{
string incrementalBackupDirectory = Path.GetDirectoryName(path);
await this.AddIncrementalBackupFolderAsync(incrementalBackupDirectory, cancellationToken).ConfigureAwait(false);
}
}
public static void CreateFolderIfNotExists(string directoryPath)
{
if (!FabricDirectory.Exists(directoryPath))
{
FabricDirectory.CreateDirectory(directoryPath);
}
}
public void FabricDirectory_DeleteNonRecursiveEmptyFolder()
{
Directory.CreateDirectory(this.testPath);
LogHelper.Log("FabricDirectory.Delete {0}", this.testPath);
FabricDirectory.Delete(this.testPath, false, false);
Assert.IsFalse(FabricDirectory.Exists(this.testPath));
}
internal Task <bool> ComplexDataBackupCallbackAsync(Data.BackupInfo backupInfo, CancellationToken cancellationToken)
{
Assert.IsFalse(cancellationToken.IsCancellationRequested);
var complexDataBackupFolderPath = Path.Combine(logFolder, "BackupContainer", Guid.NewGuid().ToString("N"));
FabricDirectory.Copy(backupInfo.Directory, complexDataBackupFolderPath, false);
return(Task.FromResult(true));
}
bool CheckIfBackupExists(string fileOrFolderPath)
{
if (fileOrFolderPath.EndsWith(ZipFileExtension) || fileOrFolderPath.EndsWith(RecoveryPointMetadataFileExtension))
{
return(FabricFile.Exists(fileOrFolderPath));
}
return(FabricDirectory.Exists(fileOrFolderPath));
}
/// <summary>
/// Retrieve files and folders under the current folder from SMB
/// </summary>
/// <param name="srcPath"> The SMB path </param>
/// <param name="files">String array of file names in the folder</param>
/// <param name="folders">String array of subfolder names in the folder</param>
private static void GetFilesAndSubfoldersFromSMB(
string srcPath,
out string[] files,
out string[] folders)
{
files = FabricDirectory.GetFiles(srcPath);
folders = FabricDirectory.GetDirectories(srcPath);
}
private void CreateDirectoryPath(string path)
{
string directoryName = FabricPath.GetDirectoryName(path);
if ((directoryName != null) && (directoryName.Length > 0) && (!FabricDirectory.Exists(directoryName)))
{
FabricDirectory.CreateDirectory(directoryName);
}
}