public CachingCentralStorage(
DistributedCentralStoreConfiguration configuration,
CentralStorage fallbackStorage,
IAbsFileSystem fileSystem)
{
Configuration = configuration;
_fallbackStorage = fallbackStorage;
var maxRetentionMb = (int)Math.Ceiling(configuration.MaxRetentionGb * 1024);
var softRetentionMb = (int)(maxRetentionMb * 0.8);
var cacheFolder = configuration.CacheRoot / CacheSubFolderName;
// Create a private CAS for storing checkpoint data
// Avoid introducing churn into primary CAS
PrivateCas = new FileSystemContentStoreInternal(
fileSystem,
SystemClock.Instance,
cacheFolder,
new ConfigurationModel(
new ContentStoreConfiguration(new MaxSizeQuota(hardExpression: maxRetentionMb + "MB", softExpression: softRetentionMb + "MB")),
ConfigurationSelection.RequireAndUseInProcessConfiguration),
settings: new ContentStoreSettings()
{
TraceFileSystemContentStoreDiagnosticMessages = Configuration.TraceFileSystemContentStoreDiagnosticMessages,
SelfCheckSettings = Configuration.SelfCheckSettings,
});
}
protected override bool TraceErrorsOnly => true; // This type adds nothing in terms of tracing. So configure it to trace errors only.
/// <summary>
/// Initializes a new instance of the <see cref="ReadOnlyFileSystemContentSession" /> class.
/// </summary>
public ReadOnlyFileSystemContentSession(string name, FileSystemContentStoreInternal store, ImplicitPin implicitPin)
: base(name)
{
Store = store;
_pinContext = Store.CreatePinContext();
_implicitPin = implicitPin;
}
/// <nodoc />
public DeploymentProxyService(
ProxyServiceConfiguration configuration,
HostParameters hostParameters,
IAbsFileSystem fileSystem = null,
IClock clock = null,
IDeploymentServiceClient client = null)
{
clock ??= SystemClock.Instance;
Configuration = configuration;
Root = new AbsolutePath(configuration.RootPath);
Clock = clock;
ContentCacheRequests = new VolatileMap <(string, string), AsyncLazy <BoolResult> >(Clock);
ProxyAddress = new VolatileMap <UnitValue, AsyncLazy <string> >(Clock);
Client = client ?? DeploymentLauncherHost.Instance.CreateServiceClient();
HostParameters = hostParameters;
DownloadQueue = new ActionQueue(configuration.DownloadConcurrency ?? Environment.ProcessorCount);
Store = new FileSystemContentStoreInternal(
fileSystem ?? new PassThroughFileSystem(),
Clock,
DeploymentUtilities.GetCasRootPath(Root),
new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota($"{Configuration.RetentionSizeGb}GB"))),
settings: new ContentStoreSettings()
{
TraceFileSystemContentStoreDiagnosticMessages = true,
});
}
private async Task TestSuccessfulPutStreamAndGet(FileSystemContentStoreInternal store, byte[] content)
{
ContentHash actualContentHash = content.CalculateHash(ContentHashType);
using (var contentStream = new MemoryStream(content))
{
var context = new Context(Logger);
var result = await store.PutStreamAsync(context, contentStream, actualContentHash);
ContentHash hashFromPut = result.ContentHash;
long sizeFromPut = result.ContentSize;
Assert.True(actualContentHash.Equals(hashFromPut));
Assert.Equal(content.Length, sizeFromPut);
// Get the content from the store
using (var pinContext = store.CreatePinContext())
{
var r2 = await store.PinAsync(context, hashFromPut, pinContext);
r2.ShouldBeSuccess();
var r3 = await store.OpenStreamAsync(context, hashFromPut);
r3.ShouldBeSuccess();
byte[] bytes = await r3.Stream.GetBytes();
Assert.True(content.SequenceEqual(bytes));
}
}
}
protected async Task <PutResult> PutStreamAsync(FileSystemContentStoreInternal store, MemoryStream content)
{
var r = await store.PutStreamAsync(Context, content, ContentHashType);
Clock.Increment();
return(r);
}
/// <summary>
/// Initializes a new instance of the <see cref="FileSystemContentStore" /> class.
/// </summary>
public FileSystemContentStore(
IAbsFileSystem fileSystem,
IClock clock,
AbsolutePath rootPath,
ConfigurationModel?configurationModel,
IDistributedLocationStore?distributedStore,
ContentStoreSettings?settings)
{
Contract.Requires(fileSystem != null);
Contract.Requires(clock != null);
Contract.Requires(rootPath != null);
int singleInstanceTimeoutSeconds = ContentStoreConfiguration.DefaultSingleInstanceTimeoutSeconds;
if (configurationModel?.InProcessConfiguration != null)
{
// TODO: Stop using the configurationModel's SingleInstanceTimeout (bug 1365340)
// because FileSystemContentStore doesn't respect the config file's value
singleInstanceTimeoutSeconds = configurationModel.InProcessConfiguration.SingleInstanceTimeoutSeconds;
}
// FileSystemContentStore implicitly uses a null component name for compatibility with older versions' directory locks.
_directoryLock = new DirectoryLock(rootPath, fileSystem, TimeSpan.FromSeconds(singleInstanceTimeoutSeconds));
Store = new FileSystemContentStoreInternal(
fileSystem,
clock,
rootPath,
configurationModel,
settings,
distributedStore);
}
public DistributedCentralStorage(
DistributedCentralStoreConfiguration configuration,
ILocationStore locationStore,
IDistributedContentCopier copier,
CentralStorage fallbackStorage)
{
_configuration = configuration;
_copier = copier;
_fallbackStorage = fallbackStorage;
_locationStore = locationStore;
var maxRetentionMb = configuration.MaxRetentionGb * 1024;
var softRetentionMb = (int)(maxRetentionMb * 0.8);
var cacheFolder = configuration.CacheRoot / CacheSubFolderName;
_copierWorkingDirectory = new DisposableDirectory(copier.FileSystem, cacheFolder / "Temp");
// Create a private CAS for storing checkpoint data
// Avoid introducing churn into primary CAS
_privateCas = new FileSystemContentStoreInternal(
copier.FileSystem,
SystemClock.Instance,
cacheFolder,
new ConfigurationModel(
new ContentStoreConfiguration(new MaxSizeQuota(hardExpression: maxRetentionMb + "MB", softExpression: softRetentionMb + "MB")),
ConfigurationSelection.RequireAndUseInProcessConfiguration),
settings: new ContentStoreSettings()
{
TraceFileSystemContentStoreDiagnosticMessages = _configuration.TraceFileSystemContentStoreDiagnosticMessages,
SelfCheckSettings = _configuration.SelfCheckSettings,
});
}
private async Task TestSuccessfulPutAbsolutePathAndGet(FileSystemContentStoreInternal store, byte[] content, FileRealizationMode ingressModes)
{
using (var tempDirectory = new DisposableDirectory(FileSystem))
{
AbsolutePath pathToFile = CreateRandomFile(tempDirectory.Path, content);
using (var contentStream = new MemoryStream(content))
{
ContentHash actualContentHash = await contentStream.WithLength().CalculateHashAsync(ContentHashType);
var context = new Context(Logger);
var result = await store.PutFileAsync(context, pathToFile, ingressModes, ContentHashType);
ContentHash hashFromPut = result.ContentHash;
long sizeFromPut = result.ContentSize;
Assert.True(actualContentHash.Equals(hashFromPut));
Assert.Equal(content.Length, sizeFromPut);
// Get the content from the store
using (var pinContext = store.CreatePinContext())
{
var r2 = await store.PinAsync(context, hashFromPut, pinContext);
r2.ShouldBeSuccess();
var r3 = await store.OpenStreamAsync(context, hashFromPut);
var bytes = await r3.Stream.GetBytes();
Assert.True(content.SequenceEqual(bytes));
}
}
}
}
/// <inheritdoc />
public virtual Task <FileExistenceResult> CheckFileExistsAsync(OperationContext context, ContentLocation sourceLocation)
{
var path = new AbsolutePath(sourceLocation.Machine.Path) / FileSystemContentStoreInternal.GetPrimaryRelativePath(sourceLocation.Hash, includeSharedFolder: false);
var resultCode = FileUtilities.Exists(path.Path) ? FileExistenceResult.ResultCode.FileExists : FileExistenceResult.ResultCode.FileNotFound;
return(Task.FromResult(new FileExistenceResult(resultCode)));
}
protected async Task <bool> ContainsAsync(FileSystemContentStoreInternal store, ContentHash contentHash)
{
var r = await store.ContainsAsync(Context, contentHash);
Clock.Increment();
return(r);
}
private async Task <bool> ValidateContentDirectoryAsync(Context context)
{
_tracer.Always(context, "Validating local CAS content directory");
int contentDirectoryMismatchCount = 0;
var fileSystemContentDirectory = _enumerateBlobPathsFromDisk()
.Select(blobPath => FileSystemContentStoreInternal.TryGetHashFromPath(blobPath.FullPath, out var hash) ? (ContentHash?)hash : null)
.Where(hash => hash != null)
.GroupBy(hash => hash !.Value)
.ToDictionary(replicaGroup => replicaGroup.Key, replicaGroup => replicaGroup.Count());
foreach (var x in fileSystemContentDirectory.Keys)
{
var fileSystemHash = x;
int fileSystemHashReplicaCount = fileSystemContentDirectory[fileSystemHash];
await _contentDirectory.UpdateAsync(fileSystemHash, false, _clock, fileInfo =>
{
if (fileInfo == null)
{
contentDirectoryMismatchCount++;
_tracer.Always(context, $"Cache content directory for hash {fileSystemHash.ToShortString()} from disk does not exist.");
}
else if (fileInfo.ReplicaCount != fileSystemHashReplicaCount)
{
contentDirectoryMismatchCount++;
_tracer.Always(
context,
$"Directory for hash {fileSystemHash.ToShortString()} describes {fileInfo.ReplicaCount} replicas, but {fileSystemHashReplicaCount} replicas exist on disk.");
}
return(null);
});
}
foreach (var x in (await _contentDirectory.EnumerateContentHashesAsync())
.Where(hash => !fileSystemContentDirectory.ContainsKey(hash)))
{
var missingHash = x;
contentDirectoryMismatchCount++;
await _contentDirectory.UpdateAsync(missingHash, false, _clock, fileInfo =>
{
if (fileInfo != null)
{
_tracer.Always(
context,
$"Directory for hash {missingHash.ToShortString()} describes {fileInfo.ReplicaCount} replicas, but no replicas exist on disk.");
}
return(null);
});
}
_tracer.Always(
context, $"{contentDirectoryMismatchCount} mismatches between cache content directory and content files on disk.");
return(contentDirectoryMismatchCount == 0);
}
public void PathWithTildaShouldNotCauseArgumentException()
{
var path = PathGeneratorUtilities.GetAbsolutePath("e", @".BuildXLCache\Shared\VSO0\364\~DE-1");
var context = new Context(Logger);
var tracer = new Tracer(string.Empty);
FileSystemContentStoreInternal.TryGetHashFromPath(context, tracer, new AbsolutePath(path), out _).Should().BeFalse();
}
/// <summary>
/// Put a randomly-sized piece of content into the store.
/// </summary>
public static async Task <PutResult> PutRandomAsync(
this FileSystemContentStoreInternal store, Context context, int size, HashType hashType = HashType.Vso0, PinRequest pinRequest = default(PinRequest))
{
var data = ThreadSafeRandom.GetBytes(size);
using (var stream = new MemoryStream(data))
{
return(await store.PutStreamAsync(context, stream, hashType, pinRequest));
}
}
protected async Task PutRandomAndPinAsync(FileSystemContentStoreInternal store, int contentSize, PinContext pinContext)
{
PutResult putResult = await store.PutRandomAsync(Context, contentSize);
putResult.ShouldBeSuccess();
PinResult pinResult = await store.PinAsync(Context, putResult.ContentHash, pinContext);
pinResult.ShouldBeSuccess();
}
public Task ContainsFalse()
{
var context = new Context(Logger);
return(TestStore(context, _clock, async store =>
{
FileSystemContentStoreInternal cas = store;
var contentHash = ContentHash.Random();
(await cas.ContainsAsync(context, contentHash)).Should().BeFalse();
}));
}
private async Task <bool> ValidateNameHashesMatchContentHashesAsync(Context context)
{
int mismatchedParentDirectoryCount = 0;
int mismatchedContentHashCount = 0;
_tracer.Always(context, "Validating local CAS content hashes...");
await TaskSafetyHelpers.WhenAll(_enumerateBlobPathsFromDisk().Select(
async blobPath =>
{
var contentFile = blobPath.FullPath;
if (!contentFile.FileName.StartsWith(contentFile.GetParent().FileName, StringComparison.OrdinalIgnoreCase))
{
mismatchedParentDirectoryCount++;
_tracer.Debug(
context,
$"The first {FileSystemContentStoreInternal.HashDirectoryNameLength} characters of the name of content file at {contentFile}" +
$" do not match the name of its parent directory {contentFile.GetParent().FileName}.");
}
if (!FileSystemContentStoreInternal.TryGetHashFromPath(contentFile, out var hashFromPath))
{
_tracer.Debug(
context,
$"The path '{contentFile}' does not contain a well-known hash name.");
return;
}
var hasher = ContentHashers.Get(hashFromPath.HashType);
ContentHash hashFromContents;
using (var contentStream = await _fileSystem.OpenSafeAsync(
contentFile, FileAccess.Read, FileMode.Open, FileShare.Read | FileShare.Delete, FileOptions.SequentialScan, HashingExtensions.HashStreamBufferSize))
{
hashFromContents = await hasher.GetContentHashAsync(contentStream);
}
if (hashFromContents != hashFromPath)
{
mismatchedContentHashCount++;
_tracer.Debug(
context,
$"Content at {contentFile} content hash {hashFromContents.ToShortString()} did not match expected value of {hashFromPath.ToShortString()}.");
}
}));
_tracer.Always(context, $"{mismatchedParentDirectoryCount} mismatches between content file name and parent directory.");
_tracer.Always(context, $"{mismatchedContentHashCount} mismatches between content file name and file contents.");
return(mismatchedContentHashCount == 0 && mismatchedParentDirectoryCount == 0);
}
public static QuotaKeeper Create(
IAbsFileSystem fileSystem,
ContentStoreInternalTracer tracer,
CancellationToken token,
FileSystemContentStoreInternal store,
IDistributedLocationStore?distributedStore,
QuotaKeeperConfiguration configuration)
{
Contract.RequiresNotNull(fileSystem);
Contract.RequiresNotNull(tracer);
Contract.RequiresNotNull(configuration);
return(new QuotaKeeper(fileSystem, tracer, configuration, token, store, distributedStore));
}
public FileSystemContentStoreInternalChecker(
IAbsFileSystem fileSystem,
IClock clock,
AbsolutePath rootPath,
Tracer tracer,
SelfCheckSettings?settings,
FileSystemContentStoreInternal contentStoreInternal)
{
_fileSystem = fileSystem;
_clock = clock;
_contentStoreInternal = contentStoreInternal;
_tracer = tracer;
_settings = CreateOrAdjustSelfCheckSettings(settings, rootPath);
_selfCheckFilePath = rootPath / "selfCheckMarker.txt";
}
/// <nodoc />
public FileSystemContentStoreInternalChecker(
IAbsFileSystem fileSystem,
IClock clock,
AbsolutePath rootPath,
Tracer tracer,
ContentStoreSettings settings,
FileSystemContentStoreInternal contentStoreInternal)
{
_fileSystem = fileSystem;
_clock = clock;
_contentStoreInternal = contentStoreInternal;
_tracer = tracer;
_settings = settings;
_selfCheckFilePath = rootPath / "selfCheckMarker.txt";
}
/// <inheritdoc />
public virtual async Task <CopyFileResult> CopyToAsync(OperationContext context, ContentLocation sourceLocation, Stream destinationStream, CopyOptions options)
{
var sourcePath = new AbsolutePath(sourceLocation.Machine.Path) / FileSystemContentStoreInternal.GetPrimaryRelativePath(sourceLocation.Hash, includeSharedFolder: false);
if (!FileUtilities.Exists(sourcePath.Path))
{
return(new CopyFileResult(CopyResultCode.FileNotFoundError, $"Source file {sourcePath} doesn't exist."));
}
long startPosition = destinationStream.Position;
using (Stream s = FileUtilities.CreateAsyncFileStream(sourcePath.Path, FileMode.Open, FileAccess.Read, FileShare.Read, FileOptions.Asynchronous | FileOptions.SequentialScan))
{
return(await s.CopyToAsync(destinationStream, 81920, context.Token).ContinueWith(_ => CopyFileResult.SuccessWithSize(destinationStream.Position - startPosition)));
}
}
protected async Task Replicate(FileSystemContentStoreInternal store, ContentHash contentHash, DisposableDirectory directory, int numberOfFiles = 1500)
{
// ReSharper disable once UnusedVariable
foreach (var x in Enumerable.Range(0, numberOfFiles))
{
var result = await store.PlaceFileAsync(
Context,
contentHash,
directory.CreateRandomFileName(),
FileAccessMode.ReadOnly,
FileReplacementMode.FailIfExists,
FileRealizationMode.HardLink);
result.Code.Should().Be(PlaceFileResult.ResultCode.PlacedWithHardLink);
Clock.Increment();
}
}
/// <summary>
/// Converts the full path to a VFS root relative path
/// </summary>
internal VirtualPath ToVirtualPath(VfsFilePlacementData data, FullPath path)
{
// Use the same index (0) for ReadOnly/Hardlink files
var shouldHardlink = FileSystemContentStoreInternal.ShouldAttemptHardLink(path, data.AccessMode, data.RealizationMode);
// All hardlinks of a hash share the same file under the vfs root so just
// use index 0 to represent that file. Otherwise, create a unique index so
// that copies get unique files
var index = shouldHardlink
? 0
: Interlocked.Increment(ref _nextVfsCasTargetFileUniqueId);
VirtualPath casRelativePath = VfsUtilities.CreateCasRelativePath(data, index);
var virtualPath = _configuration.VfsCasRelativeRoot / casRelativePath;
return(virtualPath.Path);
}
public QuotaKeeper(
IAbsFileSystem fileSystem,
ContentStoreInternalTracer tracer,
QuotaKeeperConfiguration configuration,
CancellationToken token,
FileSystemContentStoreInternal store,
IDistributedLocationStore?distributedStore)
{
_contentStoreTracer = tracer;
Tracer = new Tracer(name: $"{Component}({store.RootPath})");
_allContentSize = configuration.ContentDirectorySize;
_token = token;
_store = store;
_distributedStore = distributedStore;
_reserveQueue = new BlockingCollection <QuotaRequest>();
_evictionQueue = new ConcurrentQueue <ReserveSpaceRequest>();
_rules = CreateRules(fileSystem, configuration, store);
Counters = new CounterCollection <QuotaKeeperCounters>();
}
protected async Task <ContentHash> PutAsync
(
FileSystemContentStoreInternal store,
MemoryStream content,
bool useFile = false,
HashType hashType = ContentHashType,
bool sync = false
)
{
ContentHash contentHash;
if (useFile)
{
using (var directory = new DisposableDirectory(FileSystem))
{
var path = directory.CreateRandomFileName();
using (var stream = FileSystem.TryOpen(
path, FileAccess.Write, FileMode.CreateNew, FileShare.Delete))
{
content.WriteTo(stream);
}
var r = await store.PutFileAsync(Context, path, FileRealizationMode.Any, hashType).ShouldBeSuccess();
contentHash = r.ContentHash;
}
}
else
{
var r = await store.PutStreamAsync(Context, content, hashType).ShouldBeSuccess();
contentHash = r.ContentHash;
}
Clock.Increment();
if (sync)
{
await store.SyncAsync(Context);
}
return(contentHash);
}
public async Task ConstructorSucceedsWithLargestPossibleCacheRootLength()
{
var context = new Context(Logger);
using (var testDirectory = new DisposableDirectory(FileSystem))
{
int extraLength = FileSystemConstants.LongPathsSupported ? 100 : 0;
int maxAllowedCacheRootLength = FileSystemConstants.MaxShortPath - 1 -
FileSystemContentStoreInternal.GetMaxContentPathLengthRelativeToCacheRoot() -
testDirectory.Path.Length - 1 +
extraLength;
var cacheRoot = new AbsolutePath(testDirectory.Path + new string('a', maxAllowedCacheRootLength));
using (var store = Create(cacheRoot, _clock))
{
await store.StartupAsync(context).ShouldBeSuccess();
await store.ShutdownAsync(context).ShouldBeSuccess();
}
}
}
/// <nodoc />
public DistributedCentralStorage(
DistributedCentralStoreConfiguration configuration,
IDistributedContentCopier copier,
CentralStorage fallbackStorage)
{
_configuration = configuration;
_copier = copier;
_fallbackStorage = fallbackStorage;
var maxRetentionMb = configuration.MaxRetentionGb * 1024;
var softRetentionMb = (int)(maxRetentionMb * 0.8);
// Create a private CAS for storing checkpoint data
// Avoid introducing churn into primary CAS
_privateCas = new FileSystemContentStoreInternal(
new PassThroughFileSystem(),
SystemClock.Instance,
configuration.CacheRoot / CacheSubFolderName,
new ConfigurationModel(
new ContentStoreConfiguration(new MaxSizeQuota(hardExpression: maxRetentionMb + "MB", softExpression: softRetentionMb + "MB")),
ConfigurationSelection.RequireAndUseInProcessConfiguration));
}
private List <IQuotaRule> CreateRules(
IAbsFileSystem fileSystem,
QuotaKeeperConfiguration configuration,
FileSystemContentStoreInternal store)
{
var rules = new List <IQuotaRule>();
if (configuration.EnableElasticity)
{
var elasticSizeRule = new ElasticSizeRule(
configuration.HistoryWindowSize,
configuration.InitialElasticSize,
() => CurrentSize,
store.ReadPinSizeHistory,
fileSystem,
store.RootPath);
rules.Add(elasticSizeRule);
}
else
{
if (configuration.MaxSizeQuota != null)
{
rules.Add(new MaxSizeRule(configuration.MaxSizeQuota, () => CurrentSize));
}
if (configuration.DiskFreePercentQuota != null)
{
rules.Add(new DiskFreePercentRule(configuration.DiskFreePercentQuota, fileSystem, store.RootPath));
}
}
if (!rules.Any())
{
throw new CacheException("At least one quota rule must be defined");
}
return(rules);
}
/// <summary>
/// Gets the relative from deployment root to the file with given hash in CAS
/// </summary>
public static RelativePath GetContentRelativePath(ContentHash hash)
{
return(CasRelativeRoot / FileSystemContentStoreInternal.GetPrimaryRelativePath(hash));
}
/// <summary>
/// Initializes a new instance of the <see cref="FileSystemContentSession" /> class.
/// </summary>
public FileSystemContentSession(string name, ImplicitPin implicitPin, FileSystemContentStoreInternal store)
: base(name, store, implicitPin)
{
}
protected async Task AssertDoesNotContain(FileSystemContentStoreInternal store, ContentHash contentHash)
{
var contains = await store.ContainsAsync(Context, contentHash);
contains.Should().BeFalse($"Expected hash={contentHash.ToShortString()} to not be present but was");
}
