public void AddOrUpdateMachineLocation(MachineLocation machineLocation)
{
MachineLocation gotMachineLocatoin = null;
machineLocationDictionary.TryGetValue(machineLocation.MachineName, out gotMachineLocatoin);
if (gotMachineLocatoin != null)
{
gotMachineLocatoin.Latitude = machineLocation.Latitude;
gotMachineLocatoin.Longitude = machineLocation.Longitude;
}
else
{
machineLocations.Add(machineLocation);
machineLocationDictionary.Add(machineLocation.MachineName, machineLocation);
}
}
public async Task CopyRetriesWithRestrictions(int retries)
{
var context = new Context(Logger);
var copyAttemptsWithRestrictedReplicas = 2;
var restrictedCopyReplicaCount = 3;
using (var directory = new DisposableDirectory(FileSystem))
{
var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.Zero, retries, copyAttemptsWithRestrictedReplicas, restrictedCopyReplicaCount);
var machineLocations = new MachineLocation[] { new MachineLocation(""), new MachineLocation(""), new MachineLocation(""), new MachineLocation(""), new MachineLocation("") };
var hash = ContentHash.Random();
var hashWithLocations = new ContentHashWithSizeAndLocations(
hash,
size: 99,
machineLocations);
mockFileCopier.CopyToAsyncResult = new CopyFileResult(CopyResultCode.UnknownServerError);
var result = await distributedCopier.TryCopyAndPutAsync(
new OperationContext(context),
hashWithLocations,
handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99)));
result.ShouldBeError();
int copyAttempts = 0;
for (var attemptCount = 0; attemptCount < retries; attemptCount++)
{
var maxReplicaCount = attemptCount < copyAttemptsWithRestrictedReplicas
? restrictedCopyReplicaCount
: int.MaxValue;
copyAttempts += Math.Min(maxReplicaCount, machineLocations.Length);
}
if (copyAttempts < distributedCopier.Settings.MaxRetryCount)
{
mockFileCopier.CopyAttempts.Should().Be(copyAttempts);
result.ErrorMessage.Should().NotContain("Maximum total retries");
}
else
{
mockFileCopier.CopyAttempts.Should().Be(distributedCopier.Settings.MaxRetryCount);
result.ErrorMessage.Should().Contain("Maximum total retries");
}
}
}
public async Task ReconciliationOverRealStorage()
{
var checkpointsKey = Guid.NewGuid().ToString();
// Copy and paste a real connection string here.
var storageConnectionString = string.Empty;
// Consider updating this directory if you want to keep data between invocations.
var workingDirectory = TestRootDirectoryPath;
var configuration = new LocalDiskCentralStoreConfiguration(
workingDirectory,
checkpointsKey);
var blobStoreConfiguration = new BlobCentralStoreConfiguration(
credentials: new AzureBlobStorageCredentials(storageConnectionString),
containerName: "checkpoints",
checkpointsKey: checkpointsKey);
var producerMachineLocation = new MachineLocation();
ConfigureWithOneMaster(s =>
{
s.ReconcileMode = ReconciliationMode.Once.ToString();
s.AzureStorageSecretName = Host.StoreSecret("StorageName", storageConnectionString);
});
await RunTestAsync(
new Context(Logger),
2,
async context =>
{
var master = context.GetMaster();
var worker = context.GetFirstWorker();
var workerId = worker.LocalLocationStore.ClusterState.PrimaryMachineId;
var workerSession = context.Sessions[context.GetFirstWorkerIndex()];
var checkpointState = new CheckpointState(
Role.Worker,
EventSequencePoint.Parse("24382354"),
"MD5:8C4856EA13F6AD59B65D8F6781D2A2F9||DCS||incrementalCheckpoints/24382354.10a0ca0f-d63f-4992-a088-f67bd00abd8a.checkpointInfo.txt|Incremental",
DateTime.Now,
producerMachineLocation);
// Next heartbeat workers to restore checkpoint
await worker.LocalLocationStore.ProcessStateAsync(new OperationContext(context), checkpointState, inline: true, forceRestore: true).ShouldBeSuccess();
var reconcileResult = await worker.ReconcileAsync(context).ShouldBeSuccess();
Output.WriteLine($"Reconcile result: {reconcileResult}");
});
}
/// <nodoc />
public DistributedContentSession(
string name,
IContentSession inner,
IContentLocationStore contentLocationStore,
DistributedContentCopier contentCopier,
IDistributedContentCopierHost copierHost,
MachineLocation localMachineLocation,
DistributedContentStoreSettings settings = default)
: base(
name,
inner,
contentLocationStore,
contentCopier,
copierHost,
localMachineLocation,
settings)
{
}
/// <nodoc />
public DistributedContentStore(
MachineLocation localMachineLocation,
AbsolutePath localCacheRoot,
Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
DistributedContentStoreSettings settings,
DistributedContentCopier <T> distributedCopier,
IClock clock = null,
ContentStoreSettings contentStoreSettings = null)
{
Contract.Requires(settings != null);
LocalMachineLocation = localMachineLocation;
_contentLocationStoreFactory = contentLocationStoreFactory;
_clock = clock;
_distributedCopier = distributedCopier;
_copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp");
contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings;
_settings = settings;
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup.
_evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted(
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
_settings.LocationStoreBatchSize);
_enableDistributedEviction = _settings.ReplicaCreditInMinutes != null;
var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(_settings.ReplicaCreditInMinutes, _settings.LocationStoreBatchSize) : null;
var enableTouch = _settings.ContentHashBumpTime.HasValue;
if (enableTouch)
{
_contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, _settings.ContentHashBumpTime.Value, clock: _clock);
}
TrimBulkAsync trimBulkAsync = null;
InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync);
}
/// <nodoc />
public DistributedContentSession(
string name,
IContentSession inner,
IContentLocationStore contentLocationStore,
DistributedContentCopier contentCopier,
DistributedContentStore distributedStore,
MachineLocation localMachineLocation,
ColdStorage coldStorage,
DistributedContentStoreSettings settings = default)
: base(
name,
inner,
contentLocationStore,
contentCopier,
distributedStore,
localMachineLocation,
coldStorage,
settings)
{
}
/// <nodoc />
public TransitioningContentLocationStore(
RedisContentLocationStoreConfiguration configuration,
RedisContentLocationStore redisContentLocationStore,
LocalLocationStore localLocationStore,
MachineLocation localMachineLocation,
ILocalContentStore localContentStore)
{
Contract.Requires(configuration != null);
Contract.Requires(localMachineLocation.IsValid);
LocalContentStore = localContentStore;
_configuration = configuration;
_localLocationStore = localLocationStore;
LocalMachineLocation = localMachineLocation;
_redisContentLocationStore = redisContentLocationStore;
Contract.Assert(!_configuration.HasReadOrWriteMode(ContentLocationMode.Redis) || _redisContentLocationStore != null);
Contract.Assert(!_configuration.HasReadOrWriteMode(ContentLocationMode.LocalLocationStore) || _localLocationStore != null);
}
public void RegisterNewMachineUsesCorrectDefaults()
{
var clusterState = new ClusterStateMachine();
var nowUtc = _clock.UtcNow;
MachineId machineId;
var machineLocation = new MachineLocation("node1");
(clusterState, machineId) = clusterState.RegisterMachine(machineLocation, nowUtc);
machineId.Index.Should().Be(MachineId.MinValue);
var record = clusterState.GetStatus(machineId).ThrowIfFailure();
record.Should().BeEquivalentTo(new MachineRecord()
{
Id = new MachineId(MachineId.MinValue),
Location = machineLocation,
State = ClusterStateMachine.InitialState,
LastHeartbeatTimeUtc = nowUtc,
});
}
/// <summary>
/// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class.
/// </summary>
public DistributedContentSession(
string name,
IContentSession inner,
IContentLocationStore contentLocationStore,
DistributedContentCopier <T> contentCopier,
MachineLocation localMachineLocation,
PinCache pinCache = null,
ContentTrackerUpdater contentTrackerUpdater = null,
DistributedContentStoreSettings settings = default)
: base(
name,
inner,
contentLocationStore,
contentCopier,
localMachineLocation,
pinCache: pinCache,
contentTrackerUpdater: contentTrackerUpdater,
settings)
{
_putFileGate = new SemaphoreSlim(settings.MaximumConcurrentPutFileOperations);
}
public async Task <IContentLocationStore> CreateAsync(MachineLocation machineLocation)
{
var connection = MockRedisDatabaseFactory.CreateConnection(RedisDatabase);
RedisDatabaseAdapter = RedisDatabaseAdapter ?? new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), connection), RedisContentLocationStoreFactory.DefaultKeySpace);
var machineLocationConnection = MockRedisDatabaseFactory.CreateConnection(MachineLocationRedisDatabase);
MachineRedisDatabaseAdapter = MachineRedisDatabaseAdapter ?? new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), machineLocationConnection), RedisContentLocationStoreFactory.DefaultKeySpace);
IContentLocationStore store = new RedisContentLocationStore(
RedisDatabaseAdapter,
MachineRedisDatabaseAdapter,
_mockClock,
BumpTime,
machineLocation.Data,
_configuration);
var redisStore = (RedisContentLocationStore)store;
redisStore.DisableReplica = true;
return(store);
}
public DistributedContentStore(
MachineLocation localMachineLocation,
AbsolutePath localCacheRoot,
Func <IDistributedLocationStore, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
DistributedContentStoreSettings settings,
DistributedContentCopier distributedCopier,
IClock?clock = null)
{
Contract.Requires(settings != null);
LocalMachineLocation = localMachineLocation;
_contentLocationStoreFactory = contentLocationStoreFactory;
_clock = clock ?? SystemClock.Instance;
_distributedCopier = distributedCopier;
_copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp");
_settings = settings;
InnerContentStore = innerContentStoreFunc(this);
}
private MachineLocation TranslateLocation(MachineLocation other)
{
if (_machineLocationTranslationMap.TryGetValue(other, out var translated))
{
return translated;
}
var otherPath = other.Path;
bool hasTrailingSlash = otherPath.EndsWith(@"\");
// Add dcs subfolder to the path
otherPath = Path.Combine(otherPath, hasTrailingSlash ? CacheSubFolderNameWithTrailingSlash : CacheSubFolderName);
// If other already ended with shared, this will rearrange so that the shared folder is under the dcs sub folder
otherPath = otherPath.ReplaceIgnoreCase(CacheSharedSubFolderToReplace, CacheSharedSubFolder);
var location = new MachineLocation(otherPath);
_machineLocationTranslationMap[other] = location;
return location;
}
protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration)
{
var rootPath = testDirectory.Path / "Root";
var configurationModel = new ConfigurationModel(configuration);
var fileCopier = new TestFileCopier();
var localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var localMachineDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var localMachineLocation = new MachineLocation(rootPath.Path);
var storeFactory = new MockRedisContentLocationStoreFactory(localDatabase, localMachineDatabase, rootPath);
var settings = CreateSettings();
var distributedCopier = new DistributedContentCopier <AbsolutePath>(
settings,
FileSystem,
fileCopier,
fileCopier,
copyRequester: null,
storeFactory.PathTransformer,
SystemClock.Instance);
return(new DistributedContentStore <AbsolutePath>(
localMachineLocation,
rootPath,
(nagleBlock, distributedEvictionSettings, contentStoreSettings, trimBulkAsync) =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance,
rootPath,
configurationModel,
nagleQueue: nagleBlock,
distributedEvictionSettings: distributedEvictionSettings,
settings: contentStoreSettings,
trimBulkAsync: trimBulkAsync),
storeFactory,
settings: settings,
distributedCopier: distributedCopier));
}
protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration)
{
var rootPath = testDirectory.Path / "Root";
var tempPath = testDirectory.Path / "Temp";
var configurationModel = new ConfigurationModel(configuration);
var fileCopier = new TestFileCopier();
var localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var localMachineDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance);
var localMachineLocation = new MachineLocation(rootPath.Path);
var storeFactory = new MockRedisContentLocationStoreFactory(localDatabase, localMachineDatabase, rootPath);
return(new DistributedContentStore <AbsolutePath>(
localMachineLocation.Data,
(nagleBlock, distributedEvictionSettings, contentStoreSettings, trimBulkAsync) =>
new FileSystemContentStore(
FileSystem,
SystemClock.Instance, rootPath,
configurationModel,
nagleQueue: nagleBlock,
distributedEvictionSettings: distributedEvictionSettings,
settings: contentStoreSettings,
trimBulkAsync: trimBulkAsync),
storeFactory,
fileCopier,
fileCopier,
storeFactory.PathTransformer,
copyRequester: null,
ReadOnlyDistributedContentSession <AbsolutePath> .ContentAvailabilityGuarantee.FileRecordsExist,
tempPath,
FileSystem,
RedisContentLocationStoreConstants.DefaultBatchSize,
settings: new DistributedContentStoreSettings
{
RetryIntervalForCopies = DefaultRetryIntervalsForTest,
PinConfiguration = new PinConfiguration()
},
setPostInitializationCompletionAfterStartup: true));
}
/// <summary>
/// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class.
/// </summary>
public DistributedContentSession(
string name,
IContentSession inner,
IContentLocationStore contentLocationStore,
DistributedContentCopier <T> contentCopier,
IDistributedContentCopierHost copierHost,
MachineLocation localMachineLocation,
PinCache pinCache = null,
ContentTrackerUpdater contentTrackerUpdater = null,
DistributedContentStoreSettings settings = default)
: base(
name,
inner,
contentLocationStore,
contentCopier,
copierHost,
localMachineLocation,
pinCache: pinCache,
contentTrackerUpdater: contentTrackerUpdater,
settings)
{
}
/// <nodoc />
public DistributedContentStore(
MachineLocation localMachineLocation,
AbsolutePath localCacheRoot,
Func <ContentStoreSettings, IDistributedLocationStore, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
DistributedContentStoreSettings settings,
DistributedContentCopier <T> distributedCopier,
IClock clock = null,
ContentStoreSettings contentStoreSettings = null)
{
Contract.Requires(settings != null);
LocalMachineLocation = localMachineLocation;
_contentLocationStoreFactory = contentLocationStoreFactory;
_clock = clock;
_distributedCopier = distributedCopier;
_copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp");
contentStoreSettings ??= ContentStoreSettings.DefaultSettings;
_settings = settings;
InnerContentStore = innerContentStoreFunc(contentStoreSettings, this);
}
private (string host, int port) ExtractHostInfo(MachineLocation machineLocation)
{
var path = machineLocation.Path;
if (path.StartsWith(GrpcUriSchemePrefix))
{
// This is a uri format machine location
var uri = new Uri(path);
return(uri.Host, uri.Port);
}
var sourcePath = new AbsolutePath(path);
// TODO: Keep the segments in the AbsolutePath object?
// TODO: Indexable structure?
var segments = sourcePath.GetSegments();
Contract.Assert(segments.Count >= 4);
string host = GetHostName(sourcePath.IsLocal, segments);
return(host, _configuration.GrpcPort);
}
public Task SimpleGetUpdatesTests()
{
return(RunTest(async(context, clock, storage) =>
{
var m1 = await storage.RegisterMachineAsync(context, MachineLocation.Create("A", 1)).ThrowIfFailureAsync();
var m2 = await storage.RegisterMachineAsync(context, MachineLocation.Create("B", 1)).ThrowIfFailureAsync();
var r = await storage.GetClusterUpdatesAsync(context, new GetClusterUpdatesRequest()
{
MaxMachineId = 0,
}).ThrowIfFailureAsync();
r.MaxMachineId.Should().Be(2);
r.UnknownMachines.Contains(new KeyValuePair <MachineId, MachineLocation>(m1.Id, m1.Location)).Should().BeTrue();
r.UnknownMachines.Contains(new KeyValuePair <MachineId, MachineLocation>(m2.Id, m2.Location)).Should().BeTrue();
r = await storage.GetClusterUpdatesAsync(context, new GetClusterUpdatesRequest()
{
MaxMachineId = 1,
}).ThrowIfFailureAsync();
r.MaxMachineId.Should().Be(2);
r.UnknownMachines.Contains(new KeyValuePair <MachineId, MachineLocation>(m2.Id, m2.Location)).Should().BeTrue();
}));
}
private async Task PushFileToRemoteLocationAsync(
OperationContext operationContext,
ContentHashWithSize contentHashWithSize,
MachineLocation machine)
{
var streamResult = await _session !.OpenStreamAsync(operationContext, contentHashWithSize.Hash, CancellationToken.None);
// If the OpenStream fails, we do not copy the file
if (streamResult.Succeeded)
{
using (streamResult.Stream)
{
await _copier.PushFileAsync(
operationContext,
contentHashWithSize,
machine,
streamResult.Stream,
isInsideRing : false,
CopyReason.ColdStorage,
ProactiveCopyLocationSource.DesignatedLocation,
attempt : 1).IgnoreFailure();
}
}
}
public Task <BoolResult> RequestCopyFileAsync(OperationContext context, ContentHash hash, MachineLocation targetMachine)
{
return(CopyHandlersByLocation[targetMachine].HandleCopyFileRequestAsync(context, hash));
}
public async Task <BoolResult> PushFileAsync(OperationContext context, ContentHash hash, Func <Task <Stream> > source, MachineLocation targetMachine)
{
var tempFile = AbsolutePath.CreateRandomFileName(WorkingDirectory);
using var stream = await source();
using (var file = File.OpenWrite(tempFile.Path))
{
await stream.CopyToAsync(file);
}
var result = await PushHandlersByLocation[targetMachine].HandlePushFileAsync(context, hash, tempFile, CancellationToken.None);
File.Delete(tempFile.Path);
return(result ? BoolResult.Success : new BoolResult(result));
}
void IDistributedContentCopierHost.ReportReputation(MachineLocation location, MachineReputation reputation)
{
// Don't report reputation as this component modifies machine locations so they won't be recognized
// by the machine reputation tracker
}
public MachineLocation reportMachineLocation(MachineLocation MachineLocation)
{
return MachineLocation;
}
/// <inheritdoc /> public bool IsMachineActive(MachineLocation machine) => false;
/// <summary>
/// Tries to resolve <see cref="MachineId"/> by <paramref name="machineLocation"/>.
/// </summary>
public bool TryResolveMachineId(MachineLocation machineLocation, out MachineId machineId)
{
return(_idByLocationMap.TryGetValue(machineLocation, out machineId));
}
/// <nodoc />
public DistributedContentStore(
byte[] localMachineLocation,
Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
IFileExistenceChecker <T> fileExistenceChecker,
IFileCopier <T> fileCopier,
IPathTransformer <T> pathTransform,
ICopyRequester copyRequester,
ReadOnlyDistributedContentSession <T> .ContentAvailabilityGuarantee contentAvailabilityGuarantee,
AbsolutePath tempFolderForCopies,
IAbsFileSystem fileSystem,
int locationStoreBatchSize,
DistributedContentStoreSettings settings,
int?replicaCreditInMinutes = null,
IClock clock = null,
bool enableRepairHandling = false,
TimeSpan?contentHashBumpTime = null,
ContentStoreSettings contentStoreSettings = null)
{
Contract.Requires(settings != null);
LocalMachineLocation = new MachineLocation(localMachineLocation);
_enableRepairHandling = enableRepairHandling;
_contentLocationStoreFactory = contentLocationStoreFactory;
_contentAvailabilityGuarantee = contentAvailabilityGuarantee;
_locationStoreBatchSize = locationStoreBatchSize;
contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings;
_settings = settings;
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup.
_evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted(
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
_locationStoreBatchSize);
_distributedCopierFactory = (contentLocationStore) =>
{
return(new DistributedContentCopier <T>(
tempFolderForCopies,
_settings,
fileSystem,
fileCopier,
fileExistenceChecker,
copyRequester,
pathTransform,
contentLocationStore));
};
_enableDistributedEviction = replicaCreditInMinutes != null;
var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(replicaCreditInMinutes, locationStoreBatchSize) : null;
var enableTouch = contentHashBumpTime.HasValue;
if (enableTouch)
{
_contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, contentHashBumpTime.Value, clock: clock);
}
TrimBulkAsync trimBulkAsync = null;
InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync);
if (settings.PinConfiguration?.UsePinCache == true)
{
_pinCache = new PinCache(clock: clock);
}
}
/// <inheritdoc />
public void ReportReputation(MachineLocation location, MachineReputation reputation)
{
}
/// <inheritdoc /> public void ReportReputation(MachineLocation location, MachineReputation reputation) => MachineReputationTracker.ReportReputation(location, reputation);
private (string host, int port) ExtractHostInfo(MachineLocation machineLocation)
{
var info = machineLocation.ExtractHostInfo();
return(info.host, info.port ?? _configuration.GrpcPort);
}
/// <inheritdoc />
public async Task <DeleteResult> DeleteFileAsync(OperationContext context, ContentHash hash, MachineLocation targetMachine)
{
(string host, int port) = ExtractHostInfo(targetMachine);
using (var client = new GrpcContentClient(
new ServiceClientContentSessionTracer(nameof(ServiceClientContentSessionTracer)),
new PassThroughFileSystem(),
new ServiceClientRpcConfiguration(port)
{
GrpcHost = host
},
scenario: string.Empty))
{
return(await client.DeleteContentAsync(context, hash, deleteLocalOnly : true));
}
}
/// <inheritdoc />
public Task <BoolResult> RequestCopyFileAsync(OperationContext context, ContentHash hash, MachineLocation targetMachine)
{
(string host, int port) = ExtractHostInfo(targetMachine);
return(_clientCache.UseAsync(context, host, port, (nestedContext, client) => client.RequestCopyFileAsync(nestedContext, hash)));
}
/// <inheritdoc />
public Task <PushFileResult> PushFileAsync(OperationContext context, ContentHash hash, Stream stream, MachineLocation targetMachine, CopyOptions options)
{
(string host, int port) = ExtractHostInfo(targetMachine);
return(_clientCache.UseAsync(context, host, port, (nestedContext, client) => client.PushFileAsync(nestedContext, hash, stream, options)));
}