public void AddOrUpdateMachineLocation(MachineLocation machineLocation) { MachineLocation gotMachineLocatoin = null; machineLocationDictionary.TryGetValue(machineLocation.MachineName, out gotMachineLocatoin); if (gotMachineLocatoin != null) { gotMachineLocatoin.Latitude = machineLocation.Latitude; gotMachineLocatoin.Longitude = machineLocation.Longitude; } else { machineLocations.Add(machineLocation); machineLocationDictionary.Add(machineLocation.MachineName, machineLocation); } }
public async Task CopyRetriesWithRestrictions(int retries) { var context = new Context(Logger); var copyAttemptsWithRestrictedReplicas = 2; var restrictedCopyReplicaCount = 3; using (var directory = new DisposableDirectory(FileSystem)) { var(distributedCopier, mockFileCopier) = CreateMocks(FileSystem, directory.Path, TimeSpan.Zero, retries, copyAttemptsWithRestrictedReplicas, restrictedCopyReplicaCount); var machineLocations = new MachineLocation[] { new MachineLocation(""), new MachineLocation(""), new MachineLocation(""), new MachineLocation(""), new MachineLocation("") }; var hash = ContentHash.Random(); var hashWithLocations = new ContentHashWithSizeAndLocations( hash, size: 99, machineLocations); mockFileCopier.CopyToAsyncResult = new CopyFileResult(CopyResultCode.UnknownServerError); var result = await distributedCopier.TryCopyAndPutAsync( new OperationContext(context), hashWithLocations, handleCopyAsync : tpl => Task.FromResult(new PutResult(hash, 99))); result.ShouldBeError(); int copyAttempts = 0; for (var attemptCount = 0; attemptCount < retries; attemptCount++) { var maxReplicaCount = attemptCount < copyAttemptsWithRestrictedReplicas ? restrictedCopyReplicaCount : int.MaxValue; copyAttempts += Math.Min(maxReplicaCount, machineLocations.Length); } if (copyAttempts < distributedCopier.Settings.MaxRetryCount) { mockFileCopier.CopyAttempts.Should().Be(copyAttempts); result.ErrorMessage.Should().NotContain("Maximum total retries"); } else { mockFileCopier.CopyAttempts.Should().Be(distributedCopier.Settings.MaxRetryCount); result.ErrorMessage.Should().Contain("Maximum total retries"); } } }
public async Task ReconciliationOverRealStorage() { var checkpointsKey = Guid.NewGuid().ToString(); // Copy and paste a real connection string here. var storageConnectionString = string.Empty; // Consider updating this directory if you want to keep data between invocations. var workingDirectory = TestRootDirectoryPath; var configuration = new LocalDiskCentralStoreConfiguration( workingDirectory, checkpointsKey); var blobStoreConfiguration = new BlobCentralStoreConfiguration( credentials: new AzureBlobStorageCredentials(storageConnectionString), containerName: "checkpoints", checkpointsKey: checkpointsKey); var producerMachineLocation = new MachineLocation(); ConfigureWithOneMaster(s => { s.ReconcileMode = ReconciliationMode.Once.ToString(); s.AzureStorageSecretName = Host.StoreSecret("StorageName", storageConnectionString); }); await RunTestAsync( new Context(Logger), 2, async context => { var master = context.GetMaster(); var worker = context.GetFirstWorker(); var workerId = worker.LocalLocationStore.ClusterState.PrimaryMachineId; var workerSession = context.Sessions[context.GetFirstWorkerIndex()]; var checkpointState = new CheckpointState( Role.Worker, EventSequencePoint.Parse("24382354"), "MD5:8C4856EA13F6AD59B65D8F6781D2A2F9||DCS||incrementalCheckpoints/24382354.10a0ca0f-d63f-4992-a088-f67bd00abd8a.checkpointInfo.txt|Incremental", DateTime.Now, producerMachineLocation); // Next heartbeat workers to restore checkpoint await worker.LocalLocationStore.ProcessStateAsync(new OperationContext(context), checkpointState, inline: true, forceRestore: true).ShouldBeSuccess(); var reconcileResult = await worker.ReconcileAsync(context).ShouldBeSuccess(); Output.WriteLine($"Reconcile result: {reconcileResult}"); }); }
/// <nodoc /> public DistributedContentSession( string name, IContentSession inner, IContentLocationStore contentLocationStore, DistributedContentCopier contentCopier, IDistributedContentCopierHost copierHost, MachineLocation localMachineLocation, DistributedContentStoreSettings settings = default) : base( name, inner, contentLocationStore, contentCopier, copierHost, localMachineLocation, settings) { }
/// <nodoc /> public DistributedContentStore( MachineLocation localMachineLocation, AbsolutePath localCacheRoot, Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc, IContentLocationStoreFactory contentLocationStoreFactory, DistributedContentStoreSettings settings, DistributedContentCopier <T> distributedCopier, IClock clock = null, ContentStoreSettings contentStoreSettings = null) { Contract.Requires(settings != null); LocalMachineLocation = localMachineLocation; _contentLocationStoreFactory = contentLocationStoreFactory; _clock = clock; _distributedCopier = distributedCopier; _copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp"); contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings; _settings = settings; // Queue is created in unstarted state because the eviction function // requires the context passed at startup. _evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted( Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism, Redis.RedisContentLocationStoreConstants.BatchInterval, _settings.LocationStoreBatchSize); _enableDistributedEviction = _settings.ReplicaCreditInMinutes != null; var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(_settings.ReplicaCreditInMinutes, _settings.LocationStoreBatchSize) : null; var enableTouch = _settings.ContentHashBumpTime.HasValue; if (enableTouch) { _contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, _settings.ContentHashBumpTime.Value, clock: _clock); } TrimBulkAsync trimBulkAsync = null; InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync); }
/// <nodoc /> public DistributedContentSession( string name, IContentSession inner, IContentLocationStore contentLocationStore, DistributedContentCopier contentCopier, DistributedContentStore distributedStore, MachineLocation localMachineLocation, ColdStorage coldStorage, DistributedContentStoreSettings settings = default) : base( name, inner, contentLocationStore, contentCopier, distributedStore, localMachineLocation, coldStorage, settings) { }
/// <nodoc /> public TransitioningContentLocationStore( RedisContentLocationStoreConfiguration configuration, RedisContentLocationStore redisContentLocationStore, LocalLocationStore localLocationStore, MachineLocation localMachineLocation, ILocalContentStore localContentStore) { Contract.Requires(configuration != null); Contract.Requires(localMachineLocation.IsValid); LocalContentStore = localContentStore; _configuration = configuration; _localLocationStore = localLocationStore; LocalMachineLocation = localMachineLocation; _redisContentLocationStore = redisContentLocationStore; Contract.Assert(!_configuration.HasReadOrWriteMode(ContentLocationMode.Redis) || _redisContentLocationStore != null); Contract.Assert(!_configuration.HasReadOrWriteMode(ContentLocationMode.LocalLocationStore) || _localLocationStore != null); }
public void RegisterNewMachineUsesCorrectDefaults() { var clusterState = new ClusterStateMachine(); var nowUtc = _clock.UtcNow; MachineId machineId; var machineLocation = new MachineLocation("node1"); (clusterState, machineId) = clusterState.RegisterMachine(machineLocation, nowUtc); machineId.Index.Should().Be(MachineId.MinValue); var record = clusterState.GetStatus(machineId).ThrowIfFailure(); record.Should().BeEquivalentTo(new MachineRecord() { Id = new MachineId(MachineId.MinValue), Location = machineLocation, State = ClusterStateMachine.InitialState, LastHeartbeatTimeUtc = nowUtc, }); }
/// <summary> /// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class. /// </summary> public DistributedContentSession( string name, IContentSession inner, IContentLocationStore contentLocationStore, DistributedContentCopier <T> contentCopier, MachineLocation localMachineLocation, PinCache pinCache = null, ContentTrackerUpdater contentTrackerUpdater = null, DistributedContentStoreSettings settings = default) : base( name, inner, contentLocationStore, contentCopier, localMachineLocation, pinCache: pinCache, contentTrackerUpdater: contentTrackerUpdater, settings) { _putFileGate = new SemaphoreSlim(settings.MaximumConcurrentPutFileOperations); }
public async Task <IContentLocationStore> CreateAsync(MachineLocation machineLocation) { var connection = MockRedisDatabaseFactory.CreateConnection(RedisDatabase); RedisDatabaseAdapter = RedisDatabaseAdapter ?? new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), connection), RedisContentLocationStoreFactory.DefaultKeySpace); var machineLocationConnection = MockRedisDatabaseFactory.CreateConnection(MachineLocationRedisDatabase); MachineRedisDatabaseAdapter = MachineRedisDatabaseAdapter ?? new RedisDatabaseAdapter(await RedisDatabaseFactory.CreateAsync(new EnvironmentConnectionStringProvider("TestConnectionString"), machineLocationConnection), RedisContentLocationStoreFactory.DefaultKeySpace); IContentLocationStore store = new RedisContentLocationStore( RedisDatabaseAdapter, MachineRedisDatabaseAdapter, _mockClock, BumpTime, machineLocation.Data, _configuration); var redisStore = (RedisContentLocationStore)store; redisStore.DisableReplica = true; return(store); }
public DistributedContentStore( MachineLocation localMachineLocation, AbsolutePath localCacheRoot, Func <IDistributedLocationStore, IContentStore> innerContentStoreFunc, IContentLocationStoreFactory contentLocationStoreFactory, DistributedContentStoreSettings settings, DistributedContentCopier distributedCopier, IClock?clock = null) { Contract.Requires(settings != null); LocalMachineLocation = localMachineLocation; _contentLocationStoreFactory = contentLocationStoreFactory; _clock = clock ?? SystemClock.Instance; _distributedCopier = distributedCopier; _copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp"); _settings = settings; InnerContentStore = innerContentStoreFunc(this); }
private MachineLocation TranslateLocation(MachineLocation other) { if (_machineLocationTranslationMap.TryGetValue(other, out var translated)) { return translated; } var otherPath = other.Path; bool hasTrailingSlash = otherPath.EndsWith(@"\"); // Add dcs subfolder to the path otherPath = Path.Combine(otherPath, hasTrailingSlash ? CacheSubFolderNameWithTrailingSlash : CacheSubFolderName); // If other already ended with shared, this will rearrange so that the shared folder is under the dcs sub folder otherPath = otherPath.ReplaceIgnoreCase(CacheSharedSubFolderToReplace, CacheSharedSubFolder); var location = new MachineLocation(otherPath); _machineLocationTranslationMap[other] = location; return location; }
protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration) { var rootPath = testDirectory.Path / "Root"; var configurationModel = new ConfigurationModel(configuration); var fileCopier = new TestFileCopier(); var localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance); var localMachineDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance); var localMachineLocation = new MachineLocation(rootPath.Path); var storeFactory = new MockRedisContentLocationStoreFactory(localDatabase, localMachineDatabase, rootPath); var settings = CreateSettings(); var distributedCopier = new DistributedContentCopier <AbsolutePath>( settings, FileSystem, fileCopier, fileCopier, copyRequester: null, storeFactory.PathTransformer, SystemClock.Instance); return(new DistributedContentStore <AbsolutePath>( localMachineLocation, rootPath, (nagleBlock, distributedEvictionSettings, contentStoreSettings, trimBulkAsync) => new FileSystemContentStore( FileSystem, SystemClock.Instance, rootPath, configurationModel, nagleQueue: nagleBlock, distributedEvictionSettings: distributedEvictionSettings, settings: contentStoreSettings, trimBulkAsync: trimBulkAsync), storeFactory, settings: settings, distributedCopier: distributedCopier)); }
protected override IContentStore CreateStore(DisposableDirectory testDirectory, ContentStoreConfiguration configuration) { var rootPath = testDirectory.Path / "Root"; var tempPath = testDirectory.Path / "Temp"; var configurationModel = new ConfigurationModel(configuration); var fileCopier = new TestFileCopier(); var localDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance); var localMachineDatabase = LocalRedisProcessDatabase.CreateAndStartEmpty(_redis, TestGlobal.Logger, SystemClock.Instance); var localMachineLocation = new MachineLocation(rootPath.Path); var storeFactory = new MockRedisContentLocationStoreFactory(localDatabase, localMachineDatabase, rootPath); return(new DistributedContentStore <AbsolutePath>( localMachineLocation.Data, (nagleBlock, distributedEvictionSettings, contentStoreSettings, trimBulkAsync) => new FileSystemContentStore( FileSystem, SystemClock.Instance, rootPath, configurationModel, nagleQueue: nagleBlock, distributedEvictionSettings: distributedEvictionSettings, settings: contentStoreSettings, trimBulkAsync: trimBulkAsync), storeFactory, fileCopier, fileCopier, storeFactory.PathTransformer, copyRequester: null, ReadOnlyDistributedContentSession <AbsolutePath> .ContentAvailabilityGuarantee.FileRecordsExist, tempPath, FileSystem, RedisContentLocationStoreConstants.DefaultBatchSize, settings: new DistributedContentStoreSettings { RetryIntervalForCopies = DefaultRetryIntervalsForTest, PinConfiguration = new PinConfiguration() }, setPostInitializationCompletionAfterStartup: true)); }
/// <summary> /// Initializes a new instance of the <see cref="DistributedContentSession{T}"/> class. /// </summary> public DistributedContentSession( string name, IContentSession inner, IContentLocationStore contentLocationStore, DistributedContentCopier <T> contentCopier, IDistributedContentCopierHost copierHost, MachineLocation localMachineLocation, PinCache pinCache = null, ContentTrackerUpdater contentTrackerUpdater = null, DistributedContentStoreSettings settings = default) : base( name, inner, contentLocationStore, contentCopier, copierHost, localMachineLocation, pinCache: pinCache, contentTrackerUpdater: contentTrackerUpdater, settings) { }
/// <nodoc /> public DistributedContentStore( MachineLocation localMachineLocation, AbsolutePath localCacheRoot, Func <ContentStoreSettings, IDistributedLocationStore, IContentStore> innerContentStoreFunc, IContentLocationStoreFactory contentLocationStoreFactory, DistributedContentStoreSettings settings, DistributedContentCopier <T> distributedCopier, IClock clock = null, ContentStoreSettings contentStoreSettings = null) { Contract.Requires(settings != null); LocalMachineLocation = localMachineLocation; _contentLocationStoreFactory = contentLocationStoreFactory; _clock = clock; _distributedCopier = distributedCopier; _copierWorkingDirectory = new DisposableDirectory(distributedCopier.FileSystem, localCacheRoot / "Temp"); contentStoreSettings ??= ContentStoreSettings.DefaultSettings; _settings = settings; InnerContentStore = innerContentStoreFunc(contentStoreSettings, this); }
private (string host, int port) ExtractHostInfo(MachineLocation machineLocation) { var path = machineLocation.Path; if (path.StartsWith(GrpcUriSchemePrefix)) { // This is a uri format machine location var uri = new Uri(path); return(uri.Host, uri.Port); } var sourcePath = new AbsolutePath(path); // TODO: Keep the segments in the AbsolutePath object? // TODO: Indexable structure? var segments = sourcePath.GetSegments(); Contract.Assert(segments.Count >= 4); string host = GetHostName(sourcePath.IsLocal, segments); return(host, _configuration.GrpcPort); }
public Task SimpleGetUpdatesTests() { return(RunTest(async(context, clock, storage) => { var m1 = await storage.RegisterMachineAsync(context, MachineLocation.Create("A", 1)).ThrowIfFailureAsync(); var m2 = await storage.RegisterMachineAsync(context, MachineLocation.Create("B", 1)).ThrowIfFailureAsync(); var r = await storage.GetClusterUpdatesAsync(context, new GetClusterUpdatesRequest() { MaxMachineId = 0, }).ThrowIfFailureAsync(); r.MaxMachineId.Should().Be(2); r.UnknownMachines.Contains(new KeyValuePair <MachineId, MachineLocation>(m1.Id, m1.Location)).Should().BeTrue(); r.UnknownMachines.Contains(new KeyValuePair <MachineId, MachineLocation>(m2.Id, m2.Location)).Should().BeTrue(); r = await storage.GetClusterUpdatesAsync(context, new GetClusterUpdatesRequest() { MaxMachineId = 1, }).ThrowIfFailureAsync(); r.MaxMachineId.Should().Be(2); r.UnknownMachines.Contains(new KeyValuePair <MachineId, MachineLocation>(m2.Id, m2.Location)).Should().BeTrue(); })); }
private async Task PushFileToRemoteLocationAsync( OperationContext operationContext, ContentHashWithSize contentHashWithSize, MachineLocation machine) { var streamResult = await _session !.OpenStreamAsync(operationContext, contentHashWithSize.Hash, CancellationToken.None); // If the OpenStream fails, we do not copy the file if (streamResult.Succeeded) { using (streamResult.Stream) { await _copier.PushFileAsync( operationContext, contentHashWithSize, machine, streamResult.Stream, isInsideRing : false, CopyReason.ColdStorage, ProactiveCopyLocationSource.DesignatedLocation, attempt : 1).IgnoreFailure(); } } }
public Task <BoolResult> RequestCopyFileAsync(OperationContext context, ContentHash hash, MachineLocation targetMachine) { return(CopyHandlersByLocation[targetMachine].HandleCopyFileRequestAsync(context, hash)); }
public async Task <BoolResult> PushFileAsync(OperationContext context, ContentHash hash, Func <Task <Stream> > source, MachineLocation targetMachine) { var tempFile = AbsolutePath.CreateRandomFileName(WorkingDirectory); using var stream = await source(); using (var file = File.OpenWrite(tempFile.Path)) { await stream.CopyToAsync(file); } var result = await PushHandlersByLocation[targetMachine].HandlePushFileAsync(context, hash, tempFile, CancellationToken.None); File.Delete(tempFile.Path); return(result ? BoolResult.Success : new BoolResult(result)); }
void IDistributedContentCopierHost.ReportReputation(MachineLocation location, MachineReputation reputation) { // Don't report reputation as this component modifies machine locations so they won't be recognized // by the machine reputation tracker }
public MachineLocation reportMachineLocation(MachineLocation MachineLocation) { return MachineLocation; }
/// <inheritdoc /> public bool IsMachineActive(MachineLocation machine) => false;
/// <summary> /// Tries to resolve <see cref="MachineId"/> by <paramref name="machineLocation"/>. /// </summary> public bool TryResolveMachineId(MachineLocation machineLocation, out MachineId machineId) { return(_idByLocationMap.TryGetValue(machineLocation, out machineId)); }
/// <nodoc /> public DistributedContentStore( byte[] localMachineLocation, Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc, IContentLocationStoreFactory contentLocationStoreFactory, IFileExistenceChecker <T> fileExistenceChecker, IFileCopier <T> fileCopier, IPathTransformer <T> pathTransform, ICopyRequester copyRequester, ReadOnlyDistributedContentSession <T> .ContentAvailabilityGuarantee contentAvailabilityGuarantee, AbsolutePath tempFolderForCopies, IAbsFileSystem fileSystem, int locationStoreBatchSize, DistributedContentStoreSettings settings, int?replicaCreditInMinutes = null, IClock clock = null, bool enableRepairHandling = false, TimeSpan?contentHashBumpTime = null, ContentStoreSettings contentStoreSettings = null) { Contract.Requires(settings != null); LocalMachineLocation = new MachineLocation(localMachineLocation); _enableRepairHandling = enableRepairHandling; _contentLocationStoreFactory = contentLocationStoreFactory; _contentAvailabilityGuarantee = contentAvailabilityGuarantee; _locationStoreBatchSize = locationStoreBatchSize; contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings; _settings = settings; // Queue is created in unstarted state because the eviction function // requires the context passed at startup. _evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted( Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism, Redis.RedisContentLocationStoreConstants.BatchInterval, _locationStoreBatchSize); _distributedCopierFactory = (contentLocationStore) => { return(new DistributedContentCopier <T>( tempFolderForCopies, _settings, fileSystem, fileCopier, fileExistenceChecker, copyRequester, pathTransform, contentLocationStore)); }; _enableDistributedEviction = replicaCreditInMinutes != null; var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(replicaCreditInMinutes, locationStoreBatchSize) : null; var enableTouch = contentHashBumpTime.HasValue; if (enableTouch) { _contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, contentHashBumpTime.Value, clock: clock); } TrimBulkAsync trimBulkAsync = null; InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync); if (settings.PinConfiguration?.UsePinCache == true) { _pinCache = new PinCache(clock: clock); } }
/// <inheritdoc /> public void ReportReputation(MachineLocation location, MachineReputation reputation) { }
/// <inheritdoc /> public void ReportReputation(MachineLocation location, MachineReputation reputation) => MachineReputationTracker.ReportReputation(location, reputation);
private (string host, int port) ExtractHostInfo(MachineLocation machineLocation) { var info = machineLocation.ExtractHostInfo(); return(info.host, info.port ?? _configuration.GrpcPort); }
/// <inheritdoc /> public async Task <DeleteResult> DeleteFileAsync(OperationContext context, ContentHash hash, MachineLocation targetMachine) { (string host, int port) = ExtractHostInfo(targetMachine); using (var client = new GrpcContentClient( new ServiceClientContentSessionTracer(nameof(ServiceClientContentSessionTracer)), new PassThroughFileSystem(), new ServiceClientRpcConfiguration(port) { GrpcHost = host }, scenario: string.Empty)) { return(await client.DeleteContentAsync(context, hash, deleteLocalOnly : true)); } }
/// <inheritdoc /> public Task <BoolResult> RequestCopyFileAsync(OperationContext context, ContentHash hash, MachineLocation targetMachine) { (string host, int port) = ExtractHostInfo(targetMachine); return(_clientCache.UseAsync(context, host, port, (nestedContext, client) => client.RequestCopyFileAsync(nestedContext, hash))); }
/// <inheritdoc /> public Task <PushFileResult> PushFileAsync(OperationContext context, ContentHash hash, Stream stream, MachineLocation targetMachine, CopyOptions options) { (string host, int port) = ExtractHostInfo(targetMachine); return(_clientCache.UseAsync(context, host, port, (nestedContext, client) => client.PushFileAsync(nestedContext, hash, stream, options))); }