/// <inheritdoc />
public Task <PlaceFileResult> PlaceFileAsync(Context context, FullPath path, VfsFilePlacementData placementData, CancellationToken token)
{
return(WithOperationContext(context, token, async operationContext =>
{
var result = await operationContext.PerformOperationAsync(
Tracer,
async() =>
{
var virtualPath = ToVirtualPath(placementData, path);
var node = Tree.AddFileNode(virtualPath, placementData, path.Path);
var vfsPath = _configuration.VfsRootPath / virtualPath;
// TODO: Faster path for getting size of file. Also would be good to batch somehow.
// Maybe have FileContentManager batch pin files
var pinResult = await _placer.PinAsync(context, node.Hash, operationContext.Token);
if (pinResult.ContentSize > 0)
{
node.Size = pinResult.ContentSize;
}
else
{
return new PlaceFileResult($"Pin file size = {pinResult.ContentSize}");
}
if (placementData.AccessMode == FileAccessMode.ReadOnly)
{
// NOTE: This should cause the placeholder to get created under vfs root
_fileSystem.DenyFileWrites(vfsPath);
}
var result = _fileSystem.CreateHardLink(vfsPath, path, replaceExisting: true);
if (result != CreateHardLinkResult.Success)
{
if (result == CreateHardLinkResult.FailedDestinationDirectoryDoesNotExist)
{
_fileSystem.CreateDirectory(path.Parent);
result = _fileSystem.CreateHardLink(vfsPath, path, replaceExisting: true);
}
if (result != CreateHardLinkResult.Success)
{
return new PlaceFileResult($"Failed to create hardlink: {result}");
}
}
return new PlaceFileResult(GetPlaceResultCode(placementData.RealizationMode, placementData.AccessMode), fileSize: pinResult.ContentSize /* Unknown */);
},
caller: "PlaceVirtualFile",
extraEndMessage: r => $"Hash={placementData.Hash}");
if (!result.Succeeded)
{
return await _placer.PlaceFileAsync(context, path, placementData, token);
}
return result;
}));
}
/// <inheritdoc />
protected override async Task <PlaceFileResult> PlaceFileCoreAsync(OperationContext operationContext, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter)
{
if (replacementMode != FileReplacementMode.ReplaceExisting && _fileSystem.FileExists(path))
{
if (replacementMode == FileReplacementMode.SkipIfExists)
{
return(new PlaceFileResult(PlaceFileResult.ResultCode.NotPlacedAlreadyExists));
}
else if (replacementMode == FileReplacementMode.FailIfExists)
{
return(new PlaceFileResult(
PlaceFileResult.ResultCode.Error,
$"File exists at destination {path} with FailIfExists specified"));
}
}
var placementData = new VfsFilePlacementData(contentHash, realizationMode, accessMode);
var virtualPath = _store.Configuration.UseSymlinks
? _contentManager.TryCreateSymlink(operationContext, path, placementData, replace: replacementMode == FileReplacementMode.ReplaceExisting).ThrowIfFailure()
: _contentManager.ToVirtualPath(path);
if (virtualPath == null)
{
return(await _innerSession.PlaceFileAsync(operationContext, contentHash, path, accessMode, replacementMode, realizationMode, operationContext.Token, urgencyHint));
}
_contentManager.Tree.AddFileNode(virtualPath, placementData);
return(new PlaceFileResult(GetPlaceResultCode(realizationMode, accessMode), fileSize: -1 /* Unknown */));
}
/// <inheritdoc />
protected override async Task <PlaceFileResult> PlaceFileCoreAsync(OperationContext operationContext, ContentHash contentHash, AbsolutePath path, FileAccessMode accessMode, FileReplacementMode replacementMode, FileRealizationMode realizationMode, UrgencyHint urgencyHint, Counter retryCounter)
{
if (!path.IsVirtual)
{
return(await _innerSession.PlaceFileAsync(
operationContext,
contentHash,
path,
accessMode,
replacementMode,
realizationMode,
operationContext.Token,
urgencyHint));
}
if (replacementMode != FileReplacementMode.ReplaceExisting && _fileSystem.FileExists(path))
{
if (replacementMode == FileReplacementMode.SkipIfExists)
{
return(new PlaceFileResult(PlaceFileResult.ResultCode.NotPlacedAlreadyExists));
}
else if (replacementMode == FileReplacementMode.FailIfExists)
{
return(new PlaceFileResult(
PlaceFileResult.ResultCode.Error,
$"File exists at destination {path} with FailIfExists specified"));
}
}
var placementData = new VfsFilePlacementData(contentHash, realizationMode, accessMode);
return(await _contentManager.PlaceFileAsync(operationContext, path, placementData, operationContext.Token));
}
/// <inheritdoc />
public Task <PlaceFileResult> PlaceFileAsync(Context context, FullPath path, VfsFilePlacementData placementData, CancellationToken token)
{
return(WithOperationContext(context, token, async operationContext =>
{
var virtualPath = _configuration.UseSymlinks
? TryCreateSymlink(operationContext, path, placementData, replaceExisting: true).ThrowIfFailure()
: ToVirtualPath(path);
if (virtualPath == null)
{
return await _placer.PlaceFileAsync(context, path, placementData, token);
}
Tree.AddFileNode(virtualPath, placementData, path.Path);
return new PlaceFileResult(GetPlaceResultCode(placementData.RealizationMode, placementData.AccessMode), fileSize: -1 /* Unknown */);
}));
}
/// <summary>
/// Converts the full path to a VFS root relative path
/// </summary>
internal VirtualPath ToVirtualPath(VfsFilePlacementData data, FullPath path)
{
// Use the same index (0) for ReadOnly/Hardlink files
var shouldHardlink = FileSystemContentStoreInternal.ShouldAttemptHardLink(path, data.AccessMode, data.RealizationMode);
// All hardlinks of a hash share the same file under the vfs root so just
// use index 0 to represent that file. Otherwise, create a unique index so
// that copies get unique files
var index = shouldHardlink
? 0
: Interlocked.Increment(ref _nextVfsCasTargetFileUniqueId);
VirtualPath casRelativePath = VfsUtilities.CreateCasRelativePath(data, index);
var virtualPath = _configuration.VfsCasRelativeRoot / casRelativePath;
return(virtualPath.Path);
}
/// <summary>
/// Adds a file node (and any needed parent directory nodes) at the VFS root relative path
/// </summary>
public VfsFileNode AddFileNode(string relativePath, VfsFilePlacementData data, string realPath)
{
var timestamp = DateTime.UtcNow;
if (_nodeMap.TryGetValue(relativePath, out var node))
{
return((VfsFileNode)node);
}
else
{
var parent = GetOrAddDirectoryNode(Path.GetDirectoryName(relativePath), allowAdd: true);
var result = _nodeMap.GetOrAdd(relativePath, (parent, timestamp, data), (l_relativePath, l_data) =>
{
return(new VfsFileNode(Path.GetFileName(l_relativePath), l_data.timestamp, l_data.parent, l_data.data, RealPath.Create(PathTable, realPath)));
});
node = result.Item.Value;
return((VfsFileNode)node);
}
}
/// <summary>
/// Places a hydrated file at the given VFS root relative path
/// </summary>
/// <param name="relativePath">the vfs root relative path</param>
/// <param name="data">the content and placement data for the file</param>
/// <param name="token">the cancellation token</param>
/// <returns>a task which completes when the operation is complete or throws an exception if error is encountered during operation</returns>
public Task PlaceHydratedFileAsync(VirtualPath relativePath, VfsFilePlacementData data, CancellationToken token)
{
var context = new OperationContext(new Context(_logger), token);
return(context.PerformOperationAsync(
_tracer,
async() =>
{
var tempFilePath = _tempDirectory.CreateRandomFileName();
var result = await _contentSession.PlaceFileAsync(
context,
data.Hash,
tempFilePath,
data.AccessMode,
FileReplacementMode.ReplaceExisting,
data.RealizationMode,
token).ThrowIfFailure();
var fullPath = ToFullPath(relativePath);
_fileSystem.MoveFile(tempFilePath, fullPath, true);
if (result.FileSize >= 0)
{
Counters[VfsCounters.PlaceHydratedFileBytes].Add(result.FileSize);
}
else
{
Counters[VfsCounters.PlaceHydratedFileUnknownSizeCount].Increment();
}
return BoolResult.Success;
},
extraStartMessage: $"RelativePath={relativePath}, Hash={data.Hash}",
counter: Counters[VfsCounters.PlaceHydratedFile]));
}
public VfsFileNode(string name, DateTime timestamp, VfsDirectoryNode parent, VfsFilePlacementData data, RealPath realPath)
: base(name, timestamp, parent)
{
Data = data;
RealPath = realPath;
}
/// <inheritdoc />
public static Task <PlaceFileResult> PlaceFileAsync(this IReadOnlyContentSession session, Context context, AbsolutePath path, VfsFilePlacementData placementData, CancellationToken token)
{
return(session.PlaceFileAsync(
context,
placementData.Hash,
path,
placementData.AccessMode,
FileReplacementMode.ReplaceExisting,
placementData.RealizationMode,
token));
}
internal Result <VirtualPath> TryCreateSymlink(OperationContext context, AbsolutePath sourcePath, VfsFilePlacementData data, bool replace)
{
return(context.PerformOperation(
_tracer,
() =>
{
_fileSystem.CreateDirectory(sourcePath.Parent);
if (replace)
{
FileUtilities.DeleteFile(sourcePath.Path);
}
var index = Interlocked.Increment(ref _nextVfsCasTargetFileUniqueId);
VirtualPath casRelativePath = VfsUtilities.CreateCasRelativePath(data, index);
var virtualPath = _configuration.VfsCasRelativeRoot / casRelativePath;
Tree.AddFileNode(virtualPath.Path, data);
// Ensure existence of the virtual directory in the VFS CAS root
//Tree.GetOrAddDirectoryNode((_configuration.VfsCasRelativeRoot / casRelativePath).Parent.Path);
var fullTargetPath = _configuration.VfsCasRootPath / casRelativePath;
var result = FileUtilities.TryCreateSymbolicLink(symLinkFileName: sourcePath.Path, targetFileName: fullTargetPath.Path, isTargetFile: true);
if (result.Succeeded)
{
return Result.Success(virtualPath.Path);
}
else
{
return Result.FromErrorMessage <VirtualPath>(result.Failure.DescribeIncludingInnerFailures());
}
},
extraStartMessage: $"SourcePath={sourcePath}, Hash={data.Hash}",
messageFactory: r => $"SourcePath={sourcePath}, Hash={data.Hash}, TargetPath={r.GetValueOrDefault()}",
counter: Counters[VfsCounters.TryCreateSymlink]));
}
/// <summary>
/// Attempts to create a symlink which points to a virtualized file materialized with the given data
/// </summary>
public Result <VirtualPath> TryCreateSymlink(OperationContext context, AbsolutePath sourcePath, VfsFilePlacementData data, bool replaceExisting)
{
return(context.PerformOperation(
Tracer,
() =>
{
_fileSystem.CreateDirectory(sourcePath.Parent);
if (replaceExisting)
{
FileUtilities.DeleteFile(sourcePath.Path);
}
var index = Interlocked.Increment(ref _nextVfsCasTargetFileUniqueId);
VirtualPath casRelativePath = VfsUtilities.CreateCasRelativePath(data, index);
var virtualPath = _configuration.VfsCasRelativeRoot / casRelativePath;
var fullTargetPath = _configuration.VfsCasRootPath / casRelativePath;
var now = DateTime.UtcNow;
var tempFilePath = _tempDirectory.CreateRandomFileName();
var result = FileUtilities.TryCreateSymbolicLink(symLinkFileName: tempFilePath.Path, targetFileName: fullTargetPath.Path, isTargetFile: true);
if (result.Succeeded)
{
var attributes = FileUtilities.GetFileAttributes(tempFilePath.Path);
attributes |= FileAttributes.Offline;
FileUtilities.SetFileAttributes(tempFilePath.Path, attributes);
_fileSystem.MoveFile(tempFilePath, sourcePath, replaceExisting: replaceExisting);
return Result.Success(virtualPath.Path);
}
else
{
return Result.FromErrorMessage <VirtualPath>(result.Failure.DescribeIncludingInnerFailures());
}
},
extraStartMessage: $"SourcePath={sourcePath}, Hash={data.Hash}",
messageFactory: r => $"SourcePath={sourcePath}, Hash={data.Hash}, TargetPath={r.GetValueOrDefault()}",
counter: Counters[VfsCounters.TryCreateSymlink]));
}
