protected override TestServiceClientContentStore CreateStore(
AbsolutePath rootPath,
ContentStoreConfiguration configuration,
LocalServerConfiguration localContentServerConfiguration,
TimeSpan?heartbeatOverride)
{
configuration.Write(FileSystem, rootPath);
var grpcPortFileName = Guid.NewGuid().ToString();
var serviceConfiguration = new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
},
rootPath,
GracefulShutdownSeconds,
PortExtensions.GetNextAvailablePort(),
grpcPortFileName);
return(new TestServiceClientContentStore(
Logger,
FileSystem,
new ServiceClientContentStoreConfiguration(CacheName, null, Scenario),
heartbeatOverride,
serviceConfiguration,
localContentServerConfiguration: localContentServerConfiguration
));
}
protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration)
{
var rootPath = testDirectory.Path;
configuration.Write(FileSystem, rootPath).Wait();
var grpcPortFileName = Guid.NewGuid().ToString();
var serviceConfiguration = new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
},
rootPath,
MaxConnections,
GracefulShutdownSeconds,
PortExtensions.GetNextAvailablePort(),
grpcPortFileName);
return(new TestInProcessServiceClientContentStore(
FileSystem,
Logger,
CacheName,
Scenario,
null,
serviceConfiguration
));
}
public void WriteNoExistDirectoryThrows()
{
var configuration = new ContentStoreConfiguration();
var rootPath = _rootPath / "noexist";
Action func = () => configuration.Write(_fileSystem, rootPath);
Assert.Throws <CacheException>(func);
}
public Task WriteNoExistDirectoryThrows()
{
var configuration = new ContentStoreConfiguration();
var rootPath = _rootPath / "noexist";
Func <Task> funcAsync = async() => await configuration.Write(_fileSystem, rootPath);
return(Assert.ThrowsAsync <CacheException>(funcAsync));
}
private static void WriteContentStoreConfigFile(string cacheSizeQuotaString, AbsolutePath rootPath, IAbsFileSystem fileSystem)
{
fileSystem.CreateDirectory(rootPath);
var maxSizeQuota = new MaxSizeQuota(cacheSizeQuotaString);
var casConfig = new ContentStoreConfiguration(maxSizeQuota);
casConfig.Write(fileSystem, rootPath).GetAwaiter().GetResult();
}
private async Task RunServerTestAsync(Context context, string scenario, Func <Context, ServiceConfiguration, IRpcClient, Task> funcAsync)
{
using (var directory = new DisposableDirectory(FileSystem))
{
var storeConfig = new ContentStoreConfiguration(new MaxSizeQuota($"{1 * 1024 * 1024}"));
await storeConfig.Write(FileSystem, directory.Path).ConfigureAwait(false);
var serviceConfig =
new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ CacheName, directory.Path }
},
directory.Path,
ServiceConfiguration.DefaultMaxConnections,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
PortExtensions.GetNextAvailablePort(),
Guid.NewGuid().ToString())
{
TraceGrpcOperation = true
};
using (var server = new LocalContentServer(FileSystem, Logger, scenario, path => new FileSystemContentStore(FileSystem, SystemClock.Instance, path), new LocalServerConfiguration(serviceConfig)))
{
BoolResult r = await server.StartupAsync(context).ConfigureAwait(false);
r.ShouldBeSuccess();
var configuration = new ServiceClientRpcConfiguration()
{
GrpcPort = (int)serviceConfig.GrpcPort,
};
using (var rpcClient = new GrpcContentClient(new ServiceClientContentSessionTracer(scenario), FileSystem, configuration, scenario))
{
try
{
var createSessionResult = await rpcClient.CreateSessionAsync(new OperationContext(context), SessionName, CacheName, ImplicitPin.None);
createSessionResult.ShouldBeSuccess();
await funcAsync(context, serviceConfig, rpcClient);
}
finally
{
(await rpcClient.ShutdownAsync(context)).ShouldBeSuccess();
}
}
r = await server.ShutdownAsync(context);
r.ShouldBeSuccess();
}
}
}
protected override IContentStore CreateStore(
Context context,
TestFileCopier fileCopier,
ICopyRequester copyRequester,
DisposableDirectory testDirectory,
int index,
bool enableDistributedEviction,
int?replicaCreditInMinutes,
bool enableRepairHandling,
bool emptyFileHashShortcutEnabled,
object additionalArgs)
{
if (index == 0)
{
return(base.CreateStore(context, fileCopier, copyRequester, testDirectory, index, enableDistributedEviction, replicaCreditInMinutes, enableRepairHandling, emptyFileHashShortcutEnabled, additionalArgs));
}
var grpcPortsByStoreIndex = additionalArgs as int[];
var configuration = new ContentStoreConfiguration();
var rootPath = testDirectory.Path;
configuration.Write(FileSystem, rootPath).Wait();
var grpcPortFileName = Guid.NewGuid().ToString();
var serviceConfiguration = new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ "Default", rootPath }
},
rootPath,
ServiceConfiguration.DefaultMaxConnections,
ServiceConfiguration.DefaultGracefulShutdownSeconds,
grpcPortsByStoreIndex[index],
grpcPortFileName);
// HACK: This will only work for 3 machines. We know that master is machine 0, and workes are 1 and 2.
copyRequester = index == 0 || grpcPortsByStoreIndex.Length < 3
? copyRequester // Master does not need a copy requester.
: new GrpcFileCopier(context, grpcPortsByStoreIndex[index == 1 ? 2 : 1], 16, 1, 1); // Point workers towards each other.
return(new TestInProcessServiceClientContentStore(
FileSystem,
Logger,
"Default",
grpcPortFileName,
null,
serviceConfiguration,
// Ignore path since we configured only one cache in the testDirectory root.
contentStoreFactory: _ => new SessionCapturingStore(base.CreateStore(context, fileCopier, copyRequester, testDirectory, index, enableDistributedEviction, replicaCreditInMinutes, enableRepairHandling, emptyFileHashShortcutEnabled, additionalArgs))
));
}
public void FileRoundtripSucceeds()
{
var configuration1 = new ContentStoreConfiguration(
new MaxSizeQuota("10GB", "7GB"), new DiskFreePercentQuota("10", "13"));
configuration1.Write(_fileSystem, _rootPath);
var result = _fileSystem.ReadContentStoreConfiguration(_rootPath);
var configuration2 = result.Value;
configuration2.MaxSizeQuota.Hard.Should().Be(10L * 1024 * 1024 * 1024);
configuration2.MaxSizeQuota.Soft.Should().Be(7L * 1024 * 1024 * 1024);
configuration2.DiskFreePercentQuota.Hard.Should().Be(10);
configuration2.DiskFreePercentQuota.Soft.Should().Be(13);
}
protected override IContentStore CreateStore(AbsolutePath rootPath, string cacheName, ContentStoreConfiguration configuration)
{
configuration.Write(FileSystem, rootPath);
var grpcPortFileName = Guid.NewGuid().ToString();
var serviceConfig = new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ cacheName, rootPath }
},
rootPath,
GracefulShutdownSeconds,
PortExtensions.GetNextAvailablePort(),
grpcPortFileName);
return(new TestInProcessServiceClientContentStore(
FileSystem, Logger, cacheName, _scenario, null, serviceConfig));
}
internal void Quota
(
[Required, Description("CAS root directory")] string root,
[Description(MaxSizeDescription)] string maxSize,
[Description(DiskFreePercentDescription)] string diskFreePercent,
[DefaultValue(false), Description("Bring cache immediately into new quota")] bool purge,
[DefaultValue(false), Description("Display json format")] bool json
)
{
Initialize();
var rootPath = new AbsolutePath(root);
if (string.IsNullOrEmpty(maxSize) && string.IsNullOrEmpty(diskFreePercent))
{
Result <ContentStoreConfiguration> result = _fileSystem.ReadContentStoreConfigurationAsync(rootPath).Result;
if (result.Succeeded)
{
ShowConfiguration(result.Value, json);
}
else
{
_logger.Error("Failed to read configuration, result=[{result}]");
return;
}
}
else
{
var configuration = new ContentStoreConfiguration(maxSize, diskFreePercent);
configuration.Write(_fileSystem, rootPath).Wait();
ShowConfiguration(configuration, json);
}
if (purge)
{
RunFileSystemContentStoreInternal(rootPath, async(context, store) =>
{
await store.SyncAsync(context).ConfigureAwait(false);
});
}
}
internal void Create
(
[Required, Description("Content cache root directory")] string root,
[Required, Description(MaxSizeDescription)] string maxSize,
[Description(HashTypeDescription)] string hashType,
[DefaultValue(0), Description("Percentage of total size to populate. Mutually exclusive with fileCount.")] int percent,
[DefaultValue(0), Description("Number of files to populate. Mutually exclusive with percent.")] int fileCount,
[DefaultValue("4KB"), Description("Size of each populated file.")] string fileSize,
[DefaultValue(false), Description("Whether to treat fileSize as exact or maximum.")] bool useExactSize
)
{
Initialize();
var ht = GetHashTypeByNameOrDefault(hashType);
var rootPath = new AbsolutePath(root);
if (_fileSystem.DirectoryExists(rootPath))
{
throw new CacheException($"Root path=[{rootPath}] must not already exist.");
}
_fileSystem.CreateDirectory(rootPath);
var configuration = new ContentStoreConfiguration(maxSize, null);
configuration.Write(_fileSystem, new AbsolutePath(root)).Wait();
RunFileSystemContentStore(rootPath, async(context, session) =>
{
if (percent > 0)
{
var size = configuration.MaxSizeQuota.Soft;
await session.PutRandomAsync(context, ht, false, size, percent, fileSize.ToSize(), useExactSize).ConfigureAwait(false);
}
else if (fileCount > 0)
{
await session.PutRandomAsync(context, ht, false, fileCount, fileSize.ToSize(), useExactSize).ConfigureAwait(false);
}
});
}
protected override IContentStore CreateStore(AbsolutePath rootPath, ContentStoreConfiguration configuration)
{
configuration.Write(FileSystem, rootPath).Wait();
var grpcPortFileName = Guid.NewGuid().ToString();
var serviceConfiguration = new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
},
rootPath,
MaxConnections,
GracefulShutdownSeconds,
PortExtensions.GetNextAvailablePort(),
grpcPortFileName);
return(new TestServiceClientContentStore(
Logger,
FileSystem,
new ServiceClientContentStoreConfiguration(CacheName, null, Scenario),
null,
serviceConfiguration));
}
protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration)
{
var rootPath = testDirectory.Path;
configuration.Write(FileSystem, rootPath);
var grpcPortFileName = Guid.NewGuid().ToString();
var serviceConfig = new ServiceConfiguration(
new Dictionary <string, AbsolutePath> {
{ CacheName, rootPath }
},
rootPath,
GracefulShutdownSeconds,
PortExtensions.GetNextAvailablePort(),
grpcPortFileName);
return(new TestServiceClientContentStore(
Logger,
FileSystem,
new ServiceClientContentStoreConfiguration(CacheName, rpcConfiguration: null, scenario: Scenario),
heartbeatInterval: null,
serviceConfiguration: serviceConfig));
}
