public Task TestColdStorage()
{
return(RunTestAsync(async(context, store, directory) =>
{
var originalPath = directory.Path / "original.txt";
var fileContents = GetRandomFileContents();
// Create the file and hardlink it into the cache.
FileSystem.WriteAllText(originalPath, fileContents);
var contentHasher = HashInfoLookup.GetContentHasher(HashType.MD5);
var contentHash = contentHasher.GetContentHash(Encoding.UTF8.GetBytes(fileContents));
await store.PutFileAsync(context, contentHash, new DisposableFile(context, FileSystem, originalPath), context.Token).ShouldBeSuccess();
// Hardlink back to original location trying to replace existing.
var copyResult = await store.PlaceFileAsync(
context,
contentHash,
originalPath,
FileAccessMode.ReadOnly,
FileReplacementMode.ReplaceExisting,
FileRealizationMode.Any,
CancellationToken.None).ShouldBeSuccess();
// The file is intact.
FileSystem.ReadAllText(originalPath).Should().Be(fileContents);
}));
}
public ColdStorage(IAbsFileSystem fileSystem, ColdStorageSettings coldStorageSettings, DistributedContentCopier distributedContentCopier)
{
_fileSystem = fileSystem;
_copier = distributedContentCopier;
_rootPath = coldStorageSettings.GetAbsoulutePath();
ConfigurationModel configurationModel
= new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota(coldStorageSettings.CacheSizeQuotaString !)));
ContentStoreSettings contentStoreSettings = FromColdStorageSettings(coldStorageSettings);
_store = new FileSystemContentStore(fileSystem, SystemClock.Instance, _rootPath, configurationModel, null, contentStoreSettings, null);
HashType hashType;
if (!Enum.TryParse <HashType>(coldStorageSettings.ConsistentHashingHashType, true, out hashType))
{
hashType = HashType.SHA256;
}
_contentHasher = HashInfoLookup.GetContentHasher(hashType);
_copiesQuantity = coldStorageSettings.ConsistentHashingCopiesQuantity;
_maxParallelPlaces = coldStorageSettings.MaxParallelPlaces;
// Starts empty and is created during the first update
_ring = new RingNode[0];
}
public Task SimpleBlobPutAndGetTest()
{
return(RunTest(async(context, service, iteration) =>
{
// First heartbeat lets the service know its master, so it's willing to process requests
await service.OnRoleUpdatedAsync(context, Role.Master);
var machineId = new MachineId(0);
var data = ThreadSafeRandom.GetBytes((int)100);
var contentHash = HashInfoLookup.GetContentHasher(HashType.Vso0).GetContentHash(data);
var putResponse = await service.PutBlobAsync(new PutBlobRequest()
{
ContextId = Guid.NewGuid().ToString(),
ContentHash = contentHash,
Blob = data
});
putResponse.Succeeded.Should().BeTrue();
var getResponse = await service.GetBlobAsync(new GetBlobRequest()
{
ContextId = Guid.NewGuid().ToString(),
ContentHash = contentHash,
});
getResponse.Succeeded.Should().BeTrue();
getResponse.Blob.Should().NotBeNull();
getResponse.Blob.Should().BeEquivalentTo(data);
}));
}
public async Task HashBytesOfVariousSizes(int size)
{
using var fileSystem = new PassThroughFileSystem();
using var tempDirectory = new DisposableDirectory(fileSystem);
var content = ThreadSafeRandom.GetBytes(size);
var path = tempDirectory.Path / "1.txt";
fileSystem.WriteAllBytes(path, content);
var hashType = HashType.Vso0;
var contentHasher = HashInfoLookup.GetContentHasher(hashType);
var h1 = contentHasher.GetContentHash(content);
var h2 = CalculateHashWithMemoryMappedFile(fileSystem, path, hashType);
Assert.Equal(h1, h2);
#if NET_COREAPP
h2 = contentHasher.GetContentHash(content.AsSpan());
Assert.Equal(h1, h2);
#endif
using var memoryStream = new MemoryStream(content);
var h3 = await contentHasher.GetContentHashAsync(memoryStream);
Assert.Equal(h1, h3);
// Using an old style hashing to make sure it works the same as the new and optimized version.
using var fileStream = fileSystem.OpenForHashing(path);
var h4 = await contentHasher.GetContentHashAsync(fileStream);
Assert.Equal(h1, h4);
}
/// <summary>
/// Put a randomly-sized content from a file into the store.
/// </summary>
public static Task <PutResult> PutRandomFileAsync(
this IContentSession session,
Context context,
IAbsFileSystem fileSystem,
AbsolutePath path,
HashType hashType,
bool provideHash,
long size,
CancellationToken ct)
{
Contract.RequiresNotNull(session);
Contract.RequiresNotNull(context);
Contract.RequiresNotNull(fileSystem);
var c = context.CreateNested(nameof(ContentSessionExtensions));
// TODO: Fix this to work with size > int.Max (bug 1365340)
var data = ThreadSafeRandom.GetBytes((int)size);
fileSystem.WriteAllBytes(path, data);
if (!provideHash)
{
return(session.PutFileAsync(c, hashType, path, FileRealizationMode.Any, ct));
}
var hash = HashInfoLookup.GetContentHasher(hashType).GetContentHash(data);
return(session.PutFileAsync(c, hash, path, FileRealizationMode.Any, ct));
}
/// <summary>
/// Put a randomly-sized piece of content into the store.
/// </summary>
public static async Task <PutResult> PutRandomAsync(
this IContentSession session,
Context context,
HashType hashType,
bool provideHash,
long size,
CancellationToken ct)
{
Contract.RequiresNotNull(session);
Contract.RequiresNotNull(context);
var c = context.CreateNested(nameof(ContentSessionExtensions));
// TODO: Fix this to work with size > int.Max (bug 1365340)
var data = ThreadSafeRandom.GetBytes((int)size);
using (var stream = new MemoryStream(data))
{
if (!provideHash)
{
return(await session.PutStreamAsync(c, hashType, stream, ct).ConfigureAwait(false));
}
var hash = HashInfoLookup.GetContentHasher(hashType).GetContentHash(data);
return(await session.PutStreamAsync(c, hash, stream, ct).ConfigureAwait(false));
}
}
public async Task PushIsRejectedForTheSameHash()
{
await Task.Yield();
int numberOfPushes = 100;
await RunTestCase(async (rootPath, session, client) =>
{
var bytes = ThreadSafeRandom.GetBytes(1 + 42);
var input = Enumerable.Range(1, numberOfPushes)
.Select(data => (stream: new MemoryStream(bytes), hash: HashInfoLookup.GetContentHasher(HashType.Vso0).GetContentHash(bytes)))
.ToList();
var pushTasks = input.Select(
tpl =>
client.PushFileAsync(
new OperationContext(_context),
tpl.hash,
tpl.stream,
new CopyOptions(bandwidthConfiguration: null))).ToList();
var results = await Task.WhenAll(pushTasks);
results.Any(r => r.Status == CopyResultCode.Rejected_OngoingCopy).Should().BeTrue();
var result = await client.PushFileAsync(
new OperationContext(_context),
input[0].hash,
input[0].stream,
new CopyOptions(bandwidthConfiguration: null));
result.Status.Should().Be(CopyResultCode.Rejected_ContentAvailableLocally);
});
}
/// <nodoc />
internal static async Task <DedupNode> GetDedupNodeFromFileAsync(HashType hashType, string path)
{
var contentHasher = (DedupContentHasher <DedupNodeOrChunkHashAlgorithm>)HashInfoLookup.GetContentHasher(hashType);
using (var stream = FileStreamUtility.OpenFileStreamForAsync(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete))
{
return(await contentHasher.HashContentAndGetDedupNodeAsync(stream));
}
}
/// <inheritdoc />
protected override async Task <PutResult> PutFileCoreAsync(
OperationContext context,
HashType hashType,
AbsolutePath path,
FileRealizationMode realizationMode,
UrgencyHint urgencyHint,
Counter retryCounter)
{
if (hashType != RequiredHashType)
{
return(new PutResult(
new ContentHash(hashType),
$"BuildCache client requires HashType '{RequiredHashType}'. Cannot take HashType '{hashType}'."));
}
try
{
long contentSize;
ContentHash contentHash;
using (var hashingStream = new FileStream(
path.Path,
FileMode.Open,
FileAccess.Read,
FileShare.Read | FileShare.Delete,
StreamBufferSize))
{
contentSize = hashingStream.Length;
contentHash = await HashInfoLookup.GetContentHasher(hashType).GetContentHashAsync(hashingStream).ConfigureAwait(false);
}
using (var streamToPut = FileStreamUtils.OpenFileStreamForAsync(
path.Path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete))
{
BoolResult putSucceeded = await PutLazyStreamAsync(
context,
contentHash,
streamToPut,
urgencyHint).ConfigureAwait(false);
if (!putSucceeded.Succeeded)
{
return(new PutResult(
putSucceeded,
contentHash,
$"Failed to add a BlobStore reference to content with hash=[{contentHash}]"));
}
}
return(new PutResult(contentHash, contentSize));
}
catch (Exception e)
{
return(new PutResult(e, new ContentHash(hashType)));
}
}
internal void CopyFileTo(
[Required, Description("Machine to copy to")] string host,
[Required, Description("Path to source file")] string sourcePath,
[Description("File name where the GRPC port can be found when using cache service. 'CASaaS GRPC port' if not specified")] string grpcPortFileName,
[Description("The GRPC port"), DefaultValue(0)] int grpcPort)
{
Initialize();
var context = new Context(_logger);
var operationContext = new OperationContext(context, CancellationToken.None);
var retryPolicy = RetryPolicyFactory.GetLinearPolicy(ex => ex is ClientCanRetryException, (int)_retryCount, TimeSpan.FromSeconds(_retryIntervalSeconds));
if (grpcPort == 0)
{
grpcPort = Helpers.GetGrpcPortFromFile(_logger, grpcPortFileName);
}
var hasher = HashInfoLookup.GetContentHasher(HashType.MD5);
var bytes = File.ReadAllBytes(sourcePath);
var hash = hasher.GetContentHash(bytes);
try
{
var path = new AbsolutePath(sourcePath);
using Stream stream = File.OpenRead(path.Path);
var config = new GrpcCopyClientConfiguration();
using var clientCache = new GrpcCopyClientCache(context, new GrpcCopyClientCacheConfiguration()
{
GrpcCopyClientConfiguration = config
});
var copyFileResult = clientCache.UseAsync(operationContext, host, grpcPort, (nestedContext, rpcClient) =>
{
return(retryPolicy.ExecuteAsync(
() => rpcClient.PushFileAsync(nestedContext, hash, stream, new CopyOptions(bandwidthConfiguration: null)),
_cancellationToken));
}).GetAwaiter().GetResult();
if (!copyFileResult.Succeeded)
{
_tracer.Error(context, $"{copyFileResult}");
throw new CacheException(copyFileResult.ErrorMessage);
}
else
{
_tracer.Info(context, $"Copy of {sourcePath} was successful");
}
}
catch (Exception ex)
{
throw new CacheException(ex.ToString());
}
}
/// <inheritdoc />
protected override async Task <PutResult> PutStreamCoreAsync(
OperationContext context,
HashType hashType,
Stream stream,
UrgencyHint urgencyHint,
Counter retryCounter)
{
if (hashType != RequiredHashType)
{
return(new PutResult(
new ContentHash(hashType),
$"BuildCache client requires HashType '{RequiredHashType}'. Cannot take HashType '{hashType}'."));
}
try
{
StreamWithLength streamToPut;
// Can't assume we've been given a seekable stream.
if (stream.CanSeek)
{
streamToPut = stream.AssertHasLength();
}
else
{
streamToPut = await CreateSeekableStreamAsync(context, stream);
}
using (streamToPut)
{
Contract.Assert(streamToPut.Stream.CanSeek);
long contentSize = streamToPut.Length;
var contentHash = await HashInfoLookup.GetContentHasher(hashType)
.GetContentHashAsync(streamToPut)
.ConfigureAwait(false);
streamToPut.Stream.Seek(0, SeekOrigin.Begin);
var putResult =
await PutLazyStreamAsync(context, contentHash, streamToPut, urgencyHint).ConfigureAwait(false);
if (!putResult.Succeeded)
{
return(new PutResult(putResult, contentHash, $"Failed to add a BlobStore reference to content with hash=[{contentHash}]"));
}
return(new PutResult(contentHash, contentSize));
}
}
catch (Exception e)
{
return(new PutResult(e, new ContentHash(hashType)));
}
}
private async Task <bool> ValidateNameHashesMatchContentHashesAsync(Context context)
{
int mismatchedParentDirectoryCount = 0;
int mismatchedContentHashCount = 0;
_tracer.Always(context, "Validating local CAS content hashes...");
await TaskUtilities.SafeWhenAll(_enumerateBlobPathsFromDisk().Select(
async blobPath =>
{
var contentFile = blobPath.FullPath;
if (!contentFile.FileName.StartsWith(contentFile.GetParent().FileName, StringComparison.OrdinalIgnoreCase))
{
mismatchedParentDirectoryCount++;
_tracer.Debug(
context,
$"The first {FileSystemContentStoreInternal.HashDirectoryNameLength} characters of the name of content file at {contentFile}" +
$" do not match the name of its parent directory {contentFile.GetParent().FileName}.");
}
if (!FileSystemContentStoreInternal.TryGetHashFromPath(context, _tracer, contentFile, out var hashFromPath))
{
_tracer.Debug(
context,
$"The path '{contentFile}' does not contain a well-known hash name.");
return;
}
var hasher = HashInfoLookup.GetContentHasher(hashFromPath.HashType);
ContentHash hashFromContents;
using (var contentStream = _fileSystem.Open(
contentFile, FileAccess.Read, FileMode.Open, FileShare.Read | FileShare.Delete, FileOptions.SequentialScan, HashingExtensions.HashStreamBufferSize))
{
hashFromContents = await hasher.GetContentHashAsync(contentStream);
}
if (hashFromContents != hashFromPath)
{
mismatchedContentHashCount++;
_tracer.Debug(
context,
$"Content at {contentFile} content hash {hashFromContents.ToShortString()} did not match expected value of {hashFromPath.ToShortString()}.");
}
}));
_tracer.Always(context, $"{mismatchedParentDirectoryCount} mismatches between content file name and parent directory.");
_tracer.Always(context, $"{mismatchedContentHashCount} mismatches between content file name and file contents.");
return(mismatchedContentHashCount == 0 && mismatchedParentDirectoryCount == 0);
}
/// <summary>
/// Put a randomly-sized piece of content into the store.
/// </summary>
public static async Task <PutResult> PutContentAsync(
this IContentSession session, Context context, string content)
{
var c = context.CreateNested(nameof(ContentSessionExtensions));
var data = Encoding.UTF8.GetBytes(content);
var hashType = HashType.SHA256;
using (var stream = new MemoryStream(data))
{
var hash = HashInfoLookup.GetContentHasher(hashType).GetContentHash(data);
return(await session.PutStreamAsync(c, hash, stream, CancellationToken.None).ConfigureAwait(false));
}
}
public Task TestColdStorageWithBulkFunction()
{
return(RunTestAsync(async(context, store, directory) =>
{
var originalPath = directory.Path / "original.txt";
var fileContents = GetRandomFileContents();
// Build destination IContentSession
DisposableDirectory sessionDirectory = new DisposableDirectory(FileSystem);
ConfigurationModel configurationModel = new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota("10MB")));
FileSystemContentStore destination = new FileSystemContentStore(FileSystem, SystemClock.Instance, sessionDirectory.Path, configurationModel);
_ = await destination.StartupAsync(context);
IContentSession contentSession = destination.CreateSession(context, "test_session", BuildXL.Cache.ContentStore.Interfaces.Stores.ImplicitPin.None).Session;
_ = await contentSession.StartupAsync(context);
// Create the file and hardlink it into the cache.
FileSystem.WriteAllText(originalPath, fileContents);
var contentHasher = HashInfoLookup.GetContentHasher(HashType.MD5);
var contentHash = contentHasher.GetContentHash(Encoding.UTF8.GetBytes(fileContents));
await store.PutFileAsync(context, contentHash, new DisposableFile(context, FileSystem, originalPath), context.Token).ShouldBeSuccess();
FileSystem.DeleteFile(originalPath);
FileSystem.FileExists(originalPath).Should().Be(false);
ContentHashWithPath contentHashWithPath = new ContentHashWithPath(contentHash, originalPath);
List <ContentHashWithPath> listFile = new List <ContentHashWithPath>();
listFile.Add(contentHashWithPath);
// Hardlink back to original location trying to replace existing.
var copyTask = await store.FetchThenPutBulkAsync(
context,
listFile,
contentSession);
await copyTask.ToLookupAwait(r => { return r.Item.Succeeded; });
FileSystem.FileExists(originalPath).Should().Be(false);
// The file is in the destination.
await contentSession.PlaceFileAsync(context, contentHash, originalPath, FileAccessMode.Write, FileReplacementMode.FailIfExists, FileRealizationMode.Copy, CancellationToken.None).ShouldBeSuccess();
FileSystem.FileExists(originalPath).Should().Be(true);
FileSystem.ReadAllText(originalPath).Should().Be(fileContents);
_ = await contentSession.ShutdownAsync(context);
_ = await destination.ShutdownAsync(context);
}));
}
/// <summary>
/// Loads a configuration object from preprocessed json
/// </summary>
public static TConfig LoadPreprocessedConfig <TConfig>(string configurationPath, out string configHash, HostParameters hostParameters = null)
{
hostParameters ??= HostParameters.FromEnvironment();
var configJson = File.ReadAllText(configurationPath);
configHash = HashInfoLookup.GetContentHasher(HashType.Murmur).GetContentHash(Encoding.UTF8.GetBytes(configJson)).ToHex();
var preprocessor = DeploymentUtilities.GetHostJsonPreprocessor(hostParameters);
var preprocessedConfigJson = preprocessor.Preprocess(configJson);
var config = JsonSerializer.Deserialize <TConfig>(preprocessedConfigJson, DeploymentUtilities.ConfigurationSerializationOptions);
return(config);
}
public DeploymentManifest.FileSpec AddContent(string content)
{
var bytes = Encoding.UTF8.GetBytes(content);
var hash = HashInfoLookup.GetContentHasher(HashType.MD5).GetContentHash(bytes).ToString();
var downloadUrl = $"casaas://files?hash={hash}";
Content[downloadUrl] = bytes;
return(new DeploymentManifest.FileSpec()
{
Size = bytes.Length,
Hash = hash,
DownloadUrl = downloadUrl
});
}
/// <summary>
/// Extracts the debug entry for a javascript sourcemap file.
/// It will try to extract the client key from the sourcemap, so that
/// the tool that writes the sourcemap has control over they key.
/// It will fall back to the sha256 of the sourcemap as the client key
/// when it can't be found.
/// </summary>
internal static async Task <DebugEntryData[]> GetJsMapDebugEntryAsync(FileInfo file, bool calculateBlobId = false)
{
var fileName = file.FullName;
string clientKey = TryGetSymbolClientKeyFromJsMap(fileName);
if (clientKey == null)
{
// If the .js.map file file does not contain the proper info, use content hash as a fallback
try
{
using (var fileStream = FileStreamUtility.OpenFileStreamForAsync(fileName, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete))
{
var hash = await HashInfoLookup.GetContentHasher(HashType.SHA256).GetContentHashAsync(fileStream);
var clientId = hash.ToHex().ToLowerInvariant();
clientKey = CreateClientKey(clientId, Path.GetFileName(fileName));
}
}
catch (IOException)
{
return(new DebugEntryData[0]);
}
catch (UnauthorizedAccessException)
{
return(new DebugEntryData[0]);
}
}
BlobIdentifier blobId = null;
if (calculateBlobId)
{
var blobDescriptor = await FileBlobDescriptor.CalculateAsync(file.DirectoryName, chunkDedup : false, file.Name, FileBlobType.File, CancellationToken.None);
blobId = blobDescriptor.BlobIdentifier;
}
return(new[] {
new DebugEntryData()
{
BlobIdentifier = blobId,
ClientKey = clientKey,
InformationLevel = DebugInformationLevel.Private
}
});
}
public Task PushFileFailsWithUnknownError()
{
return(RunTestCase(async(server, rootPath, session, client) =>
{
var data = ThreadSafeRandom.GetBytes(1 + 42);
using var stream = new MemoryStream(data);
var hash = HashInfoLookup.GetContentHasher(HashType.Vso0).GetContentHash(data);
// Injecting failure.
server.GrpcContentServer.HandleRequestFailure = new Exception("Custom exception");
var pushResult = await client.PushFileAsync(
new OperationContext(_context),
hash,
stream,
new CopyOptions(bandwidthConfiguration: null));
pushResult.ShouldBeError("Custom exception");
}));
}
public async Task PushFileAsync(bool limitProactiveCopies)
{
await Task.Yield();
_proactivePushCountLimit = limitProactiveCopies ? 1 : 10000;
int numberOfPushes = 100;
await RunTestCase(async (rootPath, session, client) =>
{
var input = Enumerable.Range(1, numberOfPushes)
.Select(r => ThreadSafeRandom.GetBytes(1 + 42))
.Select(data => (stream: new MemoryStream(data), hash: HashInfoLookup.GetContentHasher(HashType.Vso0).GetContentHash(data)))
.ToList();
var pushTasks = input.Select(
tpl =>
client.PushFileAsync(
new OperationContext(_context),
tpl.hash,
tpl.stream,
new CopyOptions(bandwidthConfiguration: null))).ToList();
var results = await Task.WhenAll(pushTasks);
if (limitProactiveCopies)
{
// We're doing 100 simultaneous copies, at least some of them should fail, because we're not willing to wait for the response.
var error = results.FirstOrDefault(r => !r.Succeeded);
error.Should().NotBeNull("At least one copy operation should fail.");
error !.ErrorMessage.Should().Contain("CopyLimitReached");
}
else
{
// All operation should succeed!
results.All(r => r.ShouldBeSuccess().Succeeded).Should().BeTrue();
}
});
}
public Task TestPutToColdStorageWithRemoteLocations()
{
return(RunTestAsync(async(context, store, directory) =>
{
ClusterState clusterState = ClusterState.CreateForTest();
for (int i = 0; i <= 10; i++)
{
clusterState.AddMachine(new MachineId(i), new MachineLocation(i.ToString()));
}
var result = await store.UpdateRingAsync(context, clusterState).ShouldBeSuccess();
// Create file to put
var originalPath = directory.Path / "original.txt";
var fileContents = GetRandomFileContents();
FileSystem.WriteAllText(originalPath, fileContents);
var contentHasher = HashInfoLookup.GetContentHasher(HashType.MD5);
var contentHash = contentHasher.GetContentHash(Encoding.UTF8.GetBytes(fileContents));
await store.PutFileAsync(context, contentHash, new DisposableFile(context, FileSystem, originalPath), context.Token).ShouldBeSuccess();
}));
}
/// <summary>
/// Computes an hexidecimal content id for the given string
/// </summary>
public static string ComputeContentId(string value)
{
return(HashInfoLookup.GetContentHasher(HashType.Murmur).GetContentHash(Encoding.UTF8.GetBytes(value)).ToHex());
}