private async Task <Dictionary <string, string> > TryRetrieveKeyVaultSecretsAsync(CancellationToken token, StringBuilder errorBuilder)
{
var keyVaultSettingsString = _distributedSettings.KeyVaultSettingsString;
if (keyVaultSettingsString is null && _redisContentSecretNames.KeyVaultSecretName is null)
{
errorBuilder.Append(
$"Either {nameof(DistributedContentSettings)}.{nameof(DistributedContentSettings.KeyVaultSettingsString)} or {nameof(RedisContentSecretNames)}.{nameof(RedisContentSecretNames.KeyVaultSecretName)} should be specified. ");
return(null);
}
if (keyVaultSettingsString is null && _redisContentSecretNames.KeyVaultSecretName != null)
{
var keyVaultSettingsStringProvider = new HostConnectionStringProvider(_arguments.Host, _redisContentSecretNames.KeyVaultSecretName, _logger);
var keyVaultSettingsStringResult = await keyVaultSettingsStringProvider.GetConnectionString();
if (!keyVaultSettingsStringResult.Succeeded)
{
errorBuilder.Append($"Unable to get key vault settings string: {keyVaultSettingsStringResult}. ");
return(null);
}
keyVaultSettingsString = keyVaultSettingsStringResult.ConnectionString;
}
_logger.Debug($"KeyVaultSettingsString: <non-null secret>, "
+ $"EventHubSecretName: {_distributedSettings.EventHubSecretName}, AzureStorageSecretName: {_distributedSettings.AzureStorageSecretName}, GlobalRedisSecretName: {_distributedSettings.GlobalRedisSecretName}, SecondaryGlobalRedisSecretName: {_distributedSettings.SecondaryGlobalRedisSecretName}.");
bool invalidConfiguration = appendIfNull(_distributedSettings.EventHubSecretName, $"{nameof(DistributedContentSettings)}.{nameof(DistributedContentSettings.EventHubSecretName)}");
invalidConfiguration |= appendIfNull(_distributedSettings.GlobalRedisSecretName, $"{nameof(DistributedContentSettings)}.{nameof(DistributedContentSettings.GlobalRedisSecretName)}");
if (invalidConfiguration)
{
return(null);
}
var storageSecretNames = GetAzureStorageSecretNames(errorBuilder);
if (storageSecretNames == null)
{
return(null);
}
if (_distributedSettings.EventHubSecretName != null &&
_distributedSettings.GlobalRedisSecretName != null)
{
var retryPolicy = CreateKeyVaultRetryPolicy(_distributedSettings);
return(await retryPolicy.ExecuteAsync(
async() =>
{
return await _arguments.Host.RetrieveKeyVaultSecretsAsync(
new List <string>(storageSecretNames)
{
_distributedSettings.EventHubSecretName,
_distributedSettings.GlobalRedisSecretName,
_distributedSettings.SecondaryGlobalRedisSecretName
}.Where(s => s != null).ToList(),
token);
},
token));
}
return(null);
bool appendIfNull(object value, string propertyName)
{
if (value is null)
{
errorBuilder.Append($"{propertyName} should be provided. ");
return(true);
}
return(false);
}
}
public IContentStore CreateContentStore(
AbsolutePath localCacheRoot,
NagleQueue <ContentHash> evictionAnnouncer = null,
ProactiveReplicationArgs replicationSettings = null,
DistributedEvictionSettings distributedEvictionSettings = null,
bool checkLocalFiles = true,
TrimBulkAsync trimBulkAsync = null)
{
var contentConnectionStringProvider = new HostConnectionStringProvider(_arguments.Host, _redisContentSecretNames.RedisContentSecretName, _logger);
var machineLocationsConnectionStringProvider = new HostConnectionStringProvider(_arguments.Host, _redisContentSecretNames.RedisMachineLocationsSecretName, _logger);
var redisContentLocationStoreConfiguration = new RedisContentLocationStoreConfiguration
{
RedisBatchPageSize = _distributedSettings.RedisBatchPageSize,
BlobExpiryTimeMinutes = _distributedSettings.BlobExpiryTimeMinutes,
MaxBlobCapacity = _distributedSettings.MaxBlobCapacity,
MaxBlobSize = _distributedSettings.MaxBlobSize
};
ApplyIfNotNull(_distributedSettings.ReplicaCreditInMinutes, v => redisContentLocationStoreConfiguration.ReplicaPenaltyInMinutes = v);
ApplyIfNotNull(_distributedSettings.LocationEntryExpiryMinutes, v => redisContentLocationStoreConfiguration.LocationEntryExpiry = TimeSpan.FromMinutes(v));
ApplyIfNotNull(_distributedSettings.MachineExpiryMinutes, v => redisContentLocationStoreConfiguration.MachineExpiry = TimeSpan.FromMinutes(v));
redisContentLocationStoreConfiguration.ReputationTrackerConfiguration.Enabled = _distributedSettings.IsMachineReputationEnabled;
if (_distributedSettings.IsContentLocationDatabaseEnabled)
{
var dbConfig = new RocksDbContentLocationDatabaseConfiguration(localCacheRoot / "LocationDb")
{
StoreClusterState = _distributedSettings.StoreClusterStateInDatabase
};
redisContentLocationStoreConfiguration.Database = dbConfig;
if (_distributedSettings.ContentLocationDatabaseGcIntervalMinutes != null)
{
dbConfig.LocalDatabaseGarbageCollectionInterval = TimeSpan.FromMinutes(_distributedSettings.ContentLocationDatabaseGcIntervalMinutes.Value);
}
ApplyKeyVaultSettingsForLlsAsync(redisContentLocationStoreConfiguration, localCacheRoot).GetAwaiter().GetResult();
}
if (_distributedSettings.IsRedisGarbageCollectionEnabled)
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = new RedisGarbageCollectionConfiguration()
{
MaximumEntryLastAccessTime = TimeSpan.FromMinutes(30)
};
}
else
{
redisContentLocationStoreConfiguration.GarbageCollectionConfiguration = null;
}
var localMachineLocation = _arguments.PathTransformer.GetLocalMachineLocation(localCacheRoot);
var contentHashBumpTime = TimeSpan.FromMinutes(_distributedSettings.ContentHashBumpTimeMinutes);
var redisContentLocationStoreFactory = new RedisContentLocationStoreFactory(
contentConnectionStringProvider,
machineLocationsConnectionStringProvider,
SystemClock.Instance,
contentHashBumpTime,
_keySpace,
localMachineLocation,
configuration: redisContentLocationStoreConfiguration
);
ReadOnlyDistributedContentSession <AbsolutePath> .ContentAvailabilityGuarantee contentAvailabilityGuarantee;
if (string.IsNullOrEmpty(_distributedSettings.ContentAvailabilityGuarantee))
{
contentAvailabilityGuarantee =
ReadOnlyDistributedContentSession <AbsolutePath> .ContentAvailabilityGuarantee
.FileRecordsExist;
}
else if (!Enum.TryParse(_distributedSettings.ContentAvailabilityGuarantee, true, out contentAvailabilityGuarantee))
{
throw new ArgumentException($"Unable to parse {nameof(_distributedSettings.ContentAvailabilityGuarantee)}: [{_distributedSettings.ContentAvailabilityGuarantee}]");
}
PinConfiguration pinConfiguration = null;
if (_distributedSettings.IsPinBetterEnabled)
{
pinConfiguration = new PinConfiguration();
if (_distributedSettings.PinRisk.HasValue)
{
pinConfiguration.PinRisk = _distributedSettings.PinRisk.Value;
}
if (_distributedSettings.MachineRisk.HasValue)
{
pinConfiguration.MachineRisk = _distributedSettings.MachineRisk.Value;
}
if (_distributedSettings.FileRisk.HasValue)
{
pinConfiguration.FileRisk = _distributedSettings.FileRisk.Value;
}
if (_distributedSettings.MaxIOOperations.HasValue)
{
pinConfiguration.MaxIOOperations = _distributedSettings.MaxIOOperations.Value;
}
pinConfiguration.UsePinCache = _distributedSettings.IsPinCachingEnabled;
if (_distributedSettings.PinCacheReplicaCreditRetentionMinutes.HasValue)
{
pinConfiguration.PinCacheReplicaCreditRetentionMinutes = _distributedSettings.PinCacheReplicaCreditRetentionMinutes.Value;
}
if (_distributedSettings.PinCacheReplicaCreditRetentionDecay.HasValue)
{
pinConfiguration.PinCacheReplicaCreditRetentionDecay = _distributedSettings.PinCacheReplicaCreditRetentionDecay.Value;
}
}
var lazyTouchContentHashBumpTime = _distributedSettings.IsTouchEnabled ? (TimeSpan?)contentHashBumpTime : null;
if (redisContentLocationStoreConfiguration.ReadMode == ContentLocationMode.LocalLocationStore)
{
// LocalLocationStore has its own internal notion of lazy touch/registration. We disable the lazy touch in distributed content store
// because it can conflict with behavior of the local location store.
lazyTouchContentHashBumpTime = null;
}
var contentStoreSettings = FromDistributedSettings(_distributedSettings);
ConfigurationModel configurationModel = null;
if (_arguments.Configuration.LocalCasSettings.CacheSettingsByCacheName.TryGetValue(_arguments.Configuration.LocalCasSettings.CasClientSettings.DefaultCacheName, out var namedCacheSettings))
{
configurationModel = new ConfigurationModel(new ContentStoreConfiguration(new MaxSizeQuota(namedCacheSettings.CacheSizeQuotaString)));
}
_logger.Debug("Creating a distributed content store for Autopilot");
var contentStore =
new DistributedContentStore <AbsolutePath>(
localMachineLocation,
(announcer, evictionSettings, checkLocal, trimBulk) =>
ContentStoreFactory.CreateContentStore(_fileSystem, localCacheRoot, announcer, distributedEvictionSettings: evictionSettings,
contentStoreSettings: contentStoreSettings, trimBulkAsync: trimBulk, configurationModel: configurationModel),
redisContentLocationStoreFactory,
_arguments.Copier,
_arguments.Copier,
_arguments.PathTransformer,
contentAvailabilityGuarantee,
localCacheRoot,
_fileSystem,
_distributedSettings.RedisBatchPageSize,
new DistributedContentStoreSettings()
{
UseTrustedHash = _distributedSettings.UseTrustedHash,
CleanRandomFilesAtRoot = _distributedSettings.CleanRandomFilesAtRoot,
TrustedHashFileSizeBoundary = _distributedSettings.TrustedHashFileSizeBoundary,
ParallelHashingFileSizeBoundary = _distributedSettings.ParallelHashingFileSizeBoundary,
MaxConcurrentCopyOperations = _distributedSettings.MaxConcurrentCopyOperations,
PinConfiguration = pinConfiguration,
},
replicaCreditInMinutes: _distributedSettings.IsDistributedEvictionEnabled?_distributedSettings.ReplicaCreditInMinutes: null,
enableRepairHandling: _distributedSettings.IsRepairHandlingEnabled,
contentHashBumpTime: lazyTouchContentHashBumpTime,
contentStoreSettings: contentStoreSettings);
_logger.Debug("Created Distributed content store.");
return(contentStore);
}
private async Task <Dictionary <string, string> > TryRetrieveKeyVaultSecretsAsync(CancellationToken token, StringBuilder errorBuilder)
{
var keyVaultSettingsString = _distributedSettings.KeyVaultSettingsString;
if (keyVaultSettingsString is null && _redisContentSecretNames.KeyVaultSecretName is null)
{
errorBuilder.Append(
$"Either {nameof(DistributedContentSettings)}.{nameof(DistributedContentSettings.KeyVaultSettingsString)} or {nameof(RedisContentSecretNames)}.{nameof(RedisContentSecretNames.KeyVaultSecretName)} should be specified. ");
return(null);
}
if (keyVaultSettingsString is null && _redisContentSecretNames.KeyVaultSecretName != null)
{
var keyVaultSettingsStringProvider = new HostConnectionStringProvider(_arguments.Host, _redisContentSecretNames.KeyVaultSecretName, _logger);
var keyVaultSettingsStringResult = await keyVaultSettingsStringProvider.GetConnectionString();
if (!keyVaultSettingsStringResult.Succeeded)
{
errorBuilder.Append($"Unable to get key vault settings string: {keyVaultSettingsStringResult}. ");
return(null);
}
keyVaultSettingsString = keyVaultSettingsStringResult.ConnectionString;
}
_logger.Debug($"KeyVaultSettingsString: <non-null secret>, "
+ $"EventHubSecretName: {_distributedSettings.EventHubSecretName}, AzureStorageSecretName: {_distributedSettings.AzureStorageSecretName}, GlobalRedisSecretName: {_distributedSettings.GlobalRedisSecretName}, SecondaryGlobalRedisSecretName: {_distributedSettings.SecondaryGlobalRedisSecretName}.");
bool invalidConfiguration = appendIfNull(_distributedSettings.EventHubSecretName, $"{nameof(DistributedContentSettings)}.{nameof(DistributedContentSettings.EventHubSecretName)}");
invalidConfiguration |= appendIfNull(_distributedSettings.GlobalRedisSecretName, $"{nameof(DistributedContentSettings)}.{nameof(DistributedContentSettings.GlobalRedisSecretName)}");
if (invalidConfiguration)
{
return(null);
}
var storageSecretNames = GetAzureStorageSecretNames(errorBuilder);
if (storageSecretNames == null)
{
return(null);
}
if (_distributedSettings.EventHubSecretName != null &&
_distributedSettings.GlobalRedisSecretName != null)
{
try
{
return(await _arguments.Host.RetrieveKeyVaultSecretsAsync(
new List <string>(storageSecretNames)
{
_distributedSettings.EventHubSecretName,
_distributedSettings.GlobalRedisSecretName,
_distributedSettings.SecondaryGlobalRedisSecretName
}.Where(s => s != null).ToList(),
token));
}
// In some cases, KeyVault provider may fail with HttpRequestException with an inner exception like 'The remote name could not be resolved: 'login.windows.net'.
// Theoretically, this should be handled by the host, but to make error handling simple and consistent (this method throws one exception type) the handling is happening here.
catch (Exception e) when(e.Message.Contains("The remote name could not be resolved"))
{
errorBuilder.Append($"Unable to get key vault settings because the key vault server is unavailable. Exception: {e}. ");
return(null);
}
}
return(null);
bool appendIfNull(object value, string propertyName)
{
if (value is null)
{
errorBuilder.Append($"{propertyName} should be provided. ");
return(true);
}
return(false);
}
}