public Task TestUpdateRingAndGetLocations()
{
return(RunTestAsync(async(context, store, directory) =>
{
ClusterState clusterState = ClusterState.CreateForTest();
for (int i = 0; i <= 10; i++)
{
var machineId = new MachineId(i);
var machineLocation = new MachineLocation(i.ToString());
clusterState.AddMachine(machineId, machineLocation);
}
var contentHashes = new List <ContentHashWithPath>();
var contentHash1 = new ContentHashWithPath(new ContentHash("MD5:72F6F256239CC69B6FE9AF1C7489CFD1"), directory.Path / "hash1");
var contentHash2 = new ContentHashWithPath(new ContentHash("MD5:61C4F184221AD54A2FA4A92A6137AA42"), directory.Path / "hash2");
contentHashes.Add(contentHash1);
contentHashes.Add(contentHash2);
var result = await store.UpdateRingAsync(context, clusterState).ShouldBeSuccess();
var locations = store.GetBulkLocations(context, contentHashes).ShouldBeSuccess();
locations.Count.Should().Be(2);
locations.ContentHashesInfo[0].ContentHash.Serialize().Should().Be("MD5:72F6F256239CC69B6FE9AF1C7489CFD1");
locations.ContentHashesInfo[0].Locations[0].ToString().Should().Be("6");
locations.ContentHashesInfo[0].Locations[1].ToString().Should().Be("4");
locations.ContentHashesInfo[0].Locations[2].ToString().Should().Be("5");
locations.ContentHashesInfo[1].ContentHash.Serialize().Should().Be("MD5:61C4F184221AD54A2FA4A92A6137AA42");
locations.ContentHashesInfo[1].Locations[0].ToString().Should().Be("10");
locations.ContentHashesInfo[1].Locations[1].ToString().Should().Be("1");
locations.ContentHashesInfo[1].Locations[2].ToString().Should().Be("3");
}));
}
public Task TestColdStorageWithBulkFunction()
{
return(RunTestAsync(async(context, store, directory) =>
{
var originalPath = directory.Path / "original.txt";
var fileContents = GetRandomFileContents();
// Build destination IContentSession
DisposableDirectory sessionDirectory = new DisposableDirectory(FileSystem);
ConfigurationModel configurationModel = new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota("10MB")));
FileSystemContentStore destination = new FileSystemContentStore(FileSystem, SystemClock.Instance, sessionDirectory.Path, configurationModel);
_ = await destination.StartupAsync(context);
IContentSession contentSession = destination.CreateSession(context, "test_session", BuildXL.Cache.ContentStore.Interfaces.Stores.ImplicitPin.None).Session;
_ = await contentSession.StartupAsync(context);
// Create the file and hardlink it into the cache.
FileSystem.WriteAllText(originalPath, fileContents);
var contentHasher = HashInfoLookup.GetContentHasher(HashType.MD5);
var contentHash = contentHasher.GetContentHash(Encoding.UTF8.GetBytes(fileContents));
await store.PutFileAsync(context, contentHash, new DisposableFile(context, FileSystem, originalPath), context.Token).ShouldBeSuccess();
FileSystem.DeleteFile(originalPath);
FileSystem.FileExists(originalPath).Should().Be(false);
ContentHashWithPath contentHashWithPath = new ContentHashWithPath(contentHash, originalPath);
List <ContentHashWithPath> listFile = new List <ContentHashWithPath>();
listFile.Add(contentHashWithPath);
// Hardlink back to original location trying to replace existing.
var copyTask = await store.FetchThenPutBulkAsync(
context,
listFile,
contentSession);
await copyTask.ToLookupAwait(r => { return r.Item.Succeeded; });
FileSystem.FileExists(originalPath).Should().Be(false);
// The file is in the destination.
await contentSession.PlaceFileAsync(context, contentHash, originalPath, FileAccessMode.Write, FileReplacementMode.FailIfExists, FileRealizationMode.Copy, CancellationToken.None).ShouldBeSuccess();
FileSystem.FileExists(originalPath).Should().Be(true);
FileSystem.ReadAllText(originalPath).Should().Be(fileContents);
_ = await contentSession.ShutdownAsync(context);
_ = await destination.ShutdownAsync(context);
}));
}
public void TestRunWithThreeFallbacks()
{
List <ContentHashWithPath> listFile = new List <ContentHashWithPath>();
string[] dirs = new string[3] {
"temp", "CloudStore", "634ec8cad5be"
};
ContentHashWithPath contentHash1 = new ContentHashWithPath(new ContentHash("MD5:72F6F256239CC69B6FE9AF1C7489CFD1"), new AbsolutePath(PathGeneratorUtilities.GetAbsolutePath("A", dirs)) / "destination1.txt");
listFile.Add(contentHash1);
ContentHashWithPath contentHash2 = new ContentHashWithPath(new ContentHash("MD5:72F6F256239CC69B6FE9AF1C7489CFD2"), new AbsolutePath(PathGeneratorUtilities.GetAbsolutePath("A", dirs)) / "destination2.txt");
listFile.Add(contentHash2);
ContentHashWithPath contentHash3 = new ContentHashWithPath(new ContentHash("MD5:72F6F256239CC69B6FE9AF1C7489CFD3"), new AbsolutePath(PathGeneratorUtilities.GetAbsolutePath("A", dirs)) / "destination3.txt");
listFile.Add(contentHash3);
ContentHashWithPath contentHash4 = new ContentHashWithPath(new ContentHash("MD5:72F6F256239CC69B6FE9AF1C7489CFD4"), new AbsolutePath(PathGeneratorUtilities.GetAbsolutePath("A", dirs)) / "destination4.txt");
listFile.Add(contentHash4);
IEnumerable <Task <Indexed <PlaceFileResult> > > result = Workflows.RunWithFallback <ContentHashWithPath, PlaceFileResult>(
listFile,
initialFunc: args =>
{
Assert.Equal(args.Count, 4);
Assert.Equal(args[0], contentHash1);
Assert.Equal(args[1], contentHash2);
Assert.Equal(args[2], contentHash3);
Assert.Equal(args[3], contentHash4);
return(Task.FromResult(args.AsIndexed().Select(
p =>
{
// Only number 2 placed successfully
if (p.Index == 1)
{
return Task.FromResult(new Indexed <PlaceFileResult>(
new PlaceFileResult(PlaceFileResult.ResultCode.PlacedWithCopy),
p.Index));
}
return Task.FromResult(new Indexed <PlaceFileResult>(
new PlaceFileResult(PlaceFileResult.ResultCode.Error, "ERROR"),
p.Index));
})));
},
fallbackFunc: args =>
{
// First fallback should only receive 3 hashes
Assert.Equal(args.Count, 3);
Assert.Equal(args[0], contentHash1);
Assert.Equal(args[1], contentHash3);
Assert.Equal(args[2], contentHash4);
return(Task.FromResult(args.AsIndexed().Select(
p =>
{
// Only number 3 placed successfully
if (p.Index == 1)
{
return Task.FromResult(new Indexed <PlaceFileResult>(
new PlaceFileResult(PlaceFileResult.ResultCode.PlacedWithCopy),
p.Index));
}
return Task.FromResult(new Indexed <PlaceFileResult>(
new PlaceFileResult(PlaceFileResult.ResultCode.Error, "ERROR"),
p.Index));
})));
},
secondFallbackFunc: args =>
{
// Second fallback should only receive 2 hashes
Assert.Equal(args.Count, 2);
Assert.Equal(args[0], contentHash1);
Assert.Equal(args[1], contentHash4);
return(Task.FromResult(args.AsIndexed().Select(
p =>
{
return Task.FromResult(new Indexed <PlaceFileResult>(
new PlaceFileResult(PlaceFileResult.ResultCode.Error, "ERROR"),
p.Index));
})));
},
thirdFallbackFunc: args =>
{
// Second fallback should only receive 2 hashes
Assert.Equal(args.Count, 2);
Assert.Equal(args[0], contentHash1);
Assert.Equal(args[1], contentHash4);
return(Task.FromResult(args.AsIndexed().Select(
p =>
{
// Only number 4 placed successfully
if (p.Index == 1)
{
return Task.FromResult(new Indexed <PlaceFileResult>(
new PlaceFileResult(PlaceFileResult.ResultCode.PlacedWithCopy),
p.Index));
}
return Task.FromResult(new Indexed <PlaceFileResult>(
new PlaceFileResult(PlaceFileResult.ResultCode.Error, "ERROR"),
p.Index));
})));
},
arg =>
{
return(arg.Succeeded);
}).Result;
Assert.Equal(result.ToList().Count, 4);
result.ToList().ForEach(p => {
if (p.Result.Index == 0)
{
Assert.Equal(p.Result.Item.Code, PlaceFileResult.ResultCode.Error);
}
else
{
Assert.Equal(p.Result.Item.Code, PlaceFileResult.ResultCode.PlacedWithCopy);
}
});
}
