public async Task StartupWithCallerConfigurationWhenConfigurationFilesDoesNotExistCreatesIt()
{
using (var disposableDirectory = new DisposableDirectory(FileSystem))
{
var rootPath = disposableDirectory.Path;
var context = new Context(Logger);
var configuration = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1);
var configurationModel = new ConfigurationModel(
configuration,
ConfigurationSelection.UseFileAllowingInProcessFallback,
MissingConfigurationFileOption.WriteOnlyIfNotExists);
using (var store = new FileSystemContentStore(FileSystem, Clock, rootPath, configurationModel))
{
try
{
var r = await store.StartupAsync(context);
r.ShouldBeSuccess();
FileSystem.FileExists(rootPath / ContentStoreConfigurationExtensions.FileName).Should().BeTrue();
}
finally
{
await store.ShutdownAsync(context).ShouldBeSuccess();
}
}
}
}
// ReSharper disable once UnusedParameter.Local
static int Main(string[] args)
{
int resultCode;
using (var log = new ConsoleLog())
using (var logger = new Logger(log))
{
try
{
using (var fileSystem = new PassThroughFileSystem())
using (var directory = new DisposableDirectory(fileSystem))
using (var store = new FileSystemContentStore(fileSystem, logger, SystemClock.Instance, directory.Path))
{
var context = new Context(logger);
// ReSharper disable once AccessToDisposedClosure
resultCode = TaskSafetyHelpers.SyncResultOnThreadPool(() => RunStore(context, store));
}
}
catch (Exception exception)
{
logger.Error($"Unexpected error: {exception}");
resultCode = 1;
}
}
return(resultCode);
}
public static void Run(ContentSyncOptions options)
{
var metadataSource = Program.LoadMetadataSourceFromOptions(options as IMetadataSourceOptions);
if (metadataSource == null)
{
return;
}
var filter = FilterBuilder.MetadataFilterFromCommandLine(options as IMetadataFilterOptions);
if (filter == null)
{
return;
}
// Apply filters specified on the command line
var updatesToDownload = metadataSource.GetUpdates(filter);
if (updatesToDownload.Count == 0)
{
Console.WriteLine("No updates matched the filter");
return;
}
var filesToDownload = new List <UpdateFile>();
foreach (var update in updatesToDownload)
{
filesToDownload.AddRange(MetadataQuery.GetAllUpdateFiles(metadataSource, update));
}
var contentDestination = new FileSystemContentStore(options.ContentDestination);
contentDestination.Progress += LocalSource_OperationProgress;
var uniqueFiles = filesToDownload.GroupBy(f => f.DownloadUrl).Select(g => g.First()).ToList();
uniqueFiles.RemoveAll(f => contentDestination.Contains(f));
if (uniqueFiles.Count == 0)
{
ConsoleOutput.WriteGreen("The content matching the filter is up-to-date");
return;
}
var totalDownloadSize = uniqueFiles.Sum(f => (long)f.Size);
Console.Write($"Downloading {totalDownloadSize} bytes in {uniqueFiles.Count} files. Continue? (y/n)");
var response = Console.ReadKey();
if (response.Key == ConsoleKey.Y)
{
Console.WriteLine();
contentDestination.Add(uniqueFiles);
}
}
private ICache CreateBlockingPublishingCache(AbsolutePath path)
{
var configuration = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1);
var configurationModel = new ConfigurationModel(configuration);
var contentStore = new FileSystemContentStore(FileSystem, SystemClock.Instance, path, configurationModel);
var localCache = new OneLevelCache(
() => contentStore,
() => new MemoryMemoizationStore(Logger),
CacheDeterminism.NewCacheGuid());
return(new PublishingCacheToContentStore(new PublishingCache <OneLevelCache>(localCache, new BlockingPublishingStore(), Guid.NewGuid())));
}
protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration)
{
var rootPath = testDirectory.Path;
var configurationModel = new ConfigurationModel(configuration);
var fsStore = new FileSystemContentStore(FileSystem, SystemClock.Instance, rootPath / "fs", configurationModel);
_vfsStore = new VirtualizedContentStore(fsStore, Logger, new VfsCasConfiguration.Builder()
{
RootPath = rootPath / "vfs",
}.Build());
return(_vfsStore);
}
protected override Task RunTestAsync(
Context context, DisposableDirectory testDirectory, Func <IMemoizationStore, IMemoizationSession, Task> funcAsync, Func <DisposableDirectory, IMemoizationStore> createStoreFunc = null)
{
return(RunTestAsync(
context,
testDirectory,
async store =>
{
var configuration = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1);
var configurationModel = new ConfigurationModel(configuration);
using (var contentStore = new FileSystemContentStore(
FileSystem, SystemClock.Instance, testDirectory.Path, configurationModel))
{
try
{
var startupContentStoreResult = await contentStore.StartupAsync(context);
startupContentStoreResult.ShouldBeSuccess();
var contentSessionResult = contentStore.CreateSession(context, Name, ImplicitPin.None);
contentSessionResult.ShouldBeSuccess();
var sessionResult = store.CreateSession(context, Name, contentSessionResult.Session);
sessionResult.ShouldBeSuccess();
using (var cacheSession = new OneLevelCacheSession(Name, ImplicitPin.None, sessionResult.Session, contentSessionResult.Session))
{
try
{
var r = await cacheSession.StartupAsync(context);
r.ShouldBeSuccess();
await funcAsync(store, cacheSession);
}
finally
{
var r = await cacheSession.ShutdownAsync(context);
r.ShouldBeSuccess();
}
}
}
finally
{
var shutdownContentStoreResult = await contentStore.ShutdownAsync(context);
shutdownContentStoreResult.ShouldBeSuccess();
}
}
},
createStoreFunc));
}
public async Task StartupWithoutCallerConfigurationWhenConfigurationFileDoesNotExistGivesError()
{
using (var disposableDirectory = new DisposableDirectory(FileSystem))
{
var context = new Context(Logger);
using (var store = new FileSystemContentStore(FileSystem, Clock, disposableDirectory.Path))
{
var r = await store.StartupAsync(context);
r.ShouldBeError("ContentStoreConfiguration is missing");
}
}
}
public Task TestColdStorageWithBulkFunction()
{
return(RunTestAsync(async(context, store, directory) =>
{
var originalPath = directory.Path / "original.txt";
var fileContents = GetRandomFileContents();
// Build destination IContentSession
DisposableDirectory sessionDirectory = new DisposableDirectory(FileSystem);
ConfigurationModel configurationModel = new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota("10MB")));
FileSystemContentStore destination = new FileSystemContentStore(FileSystem, SystemClock.Instance, sessionDirectory.Path, configurationModel);
_ = await destination.StartupAsync(context);
IContentSession contentSession = destination.CreateSession(context, "test_session", BuildXL.Cache.ContentStore.Interfaces.Stores.ImplicitPin.None).Session;
_ = await contentSession.StartupAsync(context);
// Create the file and hardlink it into the cache.
FileSystem.WriteAllText(originalPath, fileContents);
var contentHasher = HashInfoLookup.GetContentHasher(HashType.MD5);
var contentHash = contentHasher.GetContentHash(Encoding.UTF8.GetBytes(fileContents));
await store.PutFileAsync(context, contentHash, new DisposableFile(context, FileSystem, originalPath), context.Token).ShouldBeSuccess();
FileSystem.DeleteFile(originalPath);
FileSystem.FileExists(originalPath).Should().Be(false);
ContentHashWithPath contentHashWithPath = new ContentHashWithPath(contentHash, originalPath);
List <ContentHashWithPath> listFile = new List <ContentHashWithPath>();
listFile.Add(contentHashWithPath);
// Hardlink back to original location trying to replace existing.
var copyTask = await store.FetchThenPutBulkAsync(
context,
listFile,
contentSession);
await copyTask.ToLookupAwait(r => { return r.Item.Succeeded; });
FileSystem.FileExists(originalPath).Should().Be(false);
// The file is in the destination.
await contentSession.PlaceFileAsync(context, contentHash, originalPath, FileAccessMode.Write, FileReplacementMode.FailIfExists, FileRealizationMode.Copy, CancellationToken.None).ShouldBeSuccess();
FileSystem.FileExists(originalPath).Should().Be(true);
FileSystem.ReadAllText(originalPath).Should().Be(fileContents);
_ = await contentSession.ShutdownAsync(context);
_ = await destination.ShutdownAsync(context);
}));
}
private async Task RunTestAsync(Context context, Func <IContentSession, Task> funcAsync)
{
using (var testDirectory = new DisposableDirectory(FileSystem))
{
var rootPath = testDirectory.Path;
var configuration = ContentStoreConfiguration.CreateWithMaxSizeQuotaMB(1);
var configurationModel = new ConfigurationModel(configuration);
using (var store = new FileSystemContentStore(
FileSystem, SystemClock.Instance, rootPath, configurationModel))
{
try
{
var startupStoreResult = await store.StartupAsync(context);
startupStoreResult.ShouldBeSuccess();
var createResult = store.CreateSession(context, Name, ImplicitPin.None);
createResult.ShouldBeSuccess();
using (var session = createResult.Session)
{
try
{
var startupSessionResult = await session.StartupAsync(context);
startupSessionResult.ShouldBeSuccess();
await funcAsync(session);
}
finally
{
var shutdownSessionResult = await session.ShutdownAsync(context);
shutdownSessionResult.ShouldBeSuccess();
}
}
}
finally
{
var shutdownStoreResult = await store.ShutdownAsync(context);
shutdownStoreResult.ShouldBeSuccess();
}
}
}
}
public async Task HardlinkTestAsync()
{
var clock = new MemoryClock();
var configuration = new ContentStoreConfiguration();
var configurationModel = new ConfigurationModel(inProcessConfiguration: configuration, ConfigurationSelection.RequireAndUseInProcessConfiguration);
var root1 = TestRootDirectoryPath / "Store1";
var store1 = new FileSystemContentStore(FileSystem, clock, root1, configurationModel);
var fakeDrive = new AbsolutePath(@"X:\");
var root2 = fakeDrive / "Store2";
var redirectedFileSystem = new RedirectionFileSystem(FileSystem, fakeDrive, TestRootDirectoryPath);
var store2 = new FileSystemContentStore(redirectedFileSystem, clock, fakeDrive, configurationModel);
var stores = new Dictionary <string, IContentStore>
{
{ root1.GetPathRoot(), store1 },
{ root2.GetPathRoot(), store2 },
};
var multiplexed = new MultiplexedContentStore(stores, preferredCacheDrive: root1.GetPathRoot(), tryAllSessions: true);
var context = new Context(Logger);
await multiplexed.StartupAsync(context).ShouldBeSuccess();
var sessionResult = multiplexed.CreateSession(context, "Default", ImplicitPin.None).ShouldBeSuccess();
var session = sessionResult.Session;
// Put random content which should go to preferred drive
var putResult = await session.PutRandomAsync(context, ContentStore.Hashing.HashType.MD5, provideHash : true, size : 1024, CancellationToken.None)
.ShouldBeSuccess();
// Should be able to place it with hardlink in primary drive
var destination1 = TestRootDirectoryPath / "destination1.txt";
var placeResult1 = await session.PlaceFileAsync(
context,
putResult.ContentHash,
destination1,
FileAccessMode.ReadOnly,
FileReplacementMode.FailIfExists,
FileRealizationMode.HardLink,
CancellationToken.None)
.ShouldBeSuccess();
placeResult1.Code.Should().Be(PlaceFileResult.ResultCode.PlacedWithHardLink);
// Should be able to place it with hardlink in secondary drive.
// The cache should copy the contents internally, and then place from the correct drive.
var destination2 = fakeDrive / "destination2.txt";
var placeResult2 = await session.PlaceFileAsync(
context,
putResult.ContentHash,
destination2,
FileAccessMode.ReadOnly,
FileReplacementMode.FailIfExists,
FileRealizationMode.HardLink,
CancellationToken.None)
.ShouldBeSuccess();
placeResult2.Code.Should().Be(PlaceFileResult.ResultCode.PlacedWithHardLink);
}