/// <summary>
/// Initializes a new instance of the <see cref="BlobReadOnlyContentSession"/> class.
/// </summary>
/// <param name="fileSystem">Filesystem used to read/write files.</param>
/// <param name="name">Session name.</param>
/// <param name="implicitPin">Policy determining whether or not content should be automatically pinned on adds or gets.</param>
/// <param name="blobStoreHttpClient">Backing BlobStore http client.</param>
/// <param name="timeToKeepContent">Minimum time-to-live for accessed content.</param>
/// <param name="downloadBlobsThroughBlobStore">If true, gets blobs through BlobStore. If false, gets blobs from the Azure Uri.</param>
/// <param name="counterTracker">Parent counters to track the session.</param>
public BlobReadOnlyContentSession(
IAbsFileSystem fileSystem,
string name,
ImplicitPin implicitPin,
IBlobStoreHttpClient blobStoreHttpClient,
TimeSpan timeToKeepContent,
bool downloadBlobsThroughBlobStore,
CounterTracker counterTracker = null)
: base(name, counterTracker)
{
Contract.Requires(fileSystem != null);
Contract.Requires(name != null);
Contract.Requires(blobStoreHttpClient != null);
ImplicitPin = implicitPin;
BlobStoreHttpClient = blobStoreHttpClient;
TimeToKeepContent = timeToKeepContent;
_downloadBlobsThroughBlobStore = downloadBlobsThroughBlobStore;
_parallelSegmentDownloadConfig = new ParallelHttpDownload
.Configuration(
segmentDownloadTimeout: TimeSpan.FromMinutes(int.Parse(
Environment.GetEnvironmentVariable(EnvironmentVariablePrefix + "SegmentDownloadTimeoutInMinutes") ??
DefaultSegmentDownloadTimeoutInMinutes.ToString())),
segmentSizeInBytes: int.Parse(
Environment.GetEnvironmentVariable(EnvironmentVariablePrefix + "ParallelDownloadSegmentSizeInBytes") ??
DefaultParallelDownloadSegmentSizeInBytes.ToString()),
maxParallelSegmentDownloadsPerFile: int.Parse(
Environment.GetEnvironmentVariable(EnvironmentVariablePrefix + "MaxParallelSegmentDownloadsPerFile") ??
DefaultMaxParallelSegmentDownloadsPerFile.ToString()),
maxSegmentDownloadRetries:
int.Parse(
Environment.GetEnvironmentVariable(EnvironmentVariablePrefix + "MaxSegmentDownloadRetries") ??
DefaultMaxSegmentDownloadRetries.ToString()));
TempDirectory = new DisposableDirectory(fileSystem);
Native.IO.FileUtilities.CreateDirectory(TempDirectory.Path.Path);
_counters = CounterTracker.CreateCounterCollection <BackingContentStore.SessionCounters>(counterTracker);
_blobCounters = CounterTracker.CreateCounterCollection <Counters>(counterTracker);
}
/// <summary>
/// Creates an ICache that can communicate with a VSTS Build Cache Service.
/// </summary>
public static ICache Create(
IAbsFileSystem fileSystem,
ILogger logger,
VssCredentialsFactory vssCredentialsFactory,
BuildCacheServiceConfiguration cacheConfig,
Func <IContentStore> writeThroughContentStoreFunc)
{
Contract.Requires(fileSystem != null);
Contract.Requires(cacheConfig != null);
var domain = new ByteDomainId(cacheConfig.DomainId);
return(new BuildCacheCache(
fileSystem,
cacheConfig.CacheNamespace,
new BuildCacheHttpClientFactory(new Uri(cacheConfig.CacheServiceFingerprintEndpoint), vssCredentialsFactory, TimeSpan.FromMinutes(cacheConfig.HttpSendTimeoutMinutes), cacheConfig.UseAad),
new BackingContentStoreHttpClientFactory(new Uri(cacheConfig.CacheServiceContentEndpoint), vssCredentialsFactory, TimeSpan.FromMinutes(cacheConfig.HttpSendTimeoutMinutes), domain, cacheConfig.UseAad),
cacheConfig.MaxFingerprintSelectorsToFetch,
TimeSpan.FromDays(cacheConfig.DaysToKeepUnreferencedContent),
TimeSpan.FromMinutes(cacheConfig.PinInlineThresholdMinutes),
TimeSpan.FromHours(cacheConfig.IgnorePinThresholdHours),
TimeSpan.FromDays(cacheConfig.DaysToKeepContentBags),
TimeSpan.FromDays(cacheConfig.RangeOfDaysToKeepContentBags),
logger,
cacheConfig.FingerprintIncorporationEnabled,
cacheConfig.MaxDegreeOfParallelismForIncorporateRequests,
cacheConfig.MaxFingerprintsPerIncorporateRequest,
domain,
cacheConfig.ForceUpdateOnAddContentHashList,
writeThroughContentStoreFunc,
cacheConfig.SealUnbackedContentHashLists,
cacheConfig.UseBlobContentHashLists,
cacheConfig.UseDedupStore,
cacheConfig.OverrideUnixFileAccessMode,
cacheConfig.EnableEagerFingerprintIncorporation,
TimeSpan.FromHours(cacheConfig.InlineFingerprintIncorporationExpiryHours),
TimeSpan.FromMinutes(cacheConfig.EagerFingerprintIncorporationNagleIntervalMinutes),
cacheConfig.EagerFingerprintIncorporationNagleBatchSize,
downloadBlobsUsingHttpClient: cacheConfig.DownloadBlobsUsingHttpClient));
}
/// <summary>
/// Serialize hibernated session information to the standard filename in the given directory.
/// </summary>
public static async Task WriteAsync(this HibernatedSessions sessions, IAbsFileSystem fileSystem, AbsolutePath rootPath)
{
Contract.Requires(fileSystem != null);
Contract.Requires(rootPath != null);
// Due to abnormal process termination, the file that we'll be writing can be corrupted.
// To prevent this issue we first write the file into a temporary location and then we "move it" into a final location.
using (var tempFolder = new DisposableDirectory(fileSystem, rootPath / "Temp"))
{
var jsonTempPath = tempFolder.CreateRandomFileName();
var jsonPath = rootPath / FileName;
using (var stream =
await fileSystem.OpenSafeAsync(jsonTempPath, FileAccess.Write, FileMode.Create, FileShare.None))
{
sessions.SerializeToJSON(stream);
}
fileSystem.MoveFile(jsonTempPath, jsonPath, replaceExisting: true);
}
}
/// <inheritdoc />
public CheckpointManager(
ContentLocationDatabase database,
ICheckpointRegistry checkpointRegistry,
CentralStorage storage,
CheckpointManagerConfiguration configuration,
CounterCollection <ContentLocationStoreCounters> counters,
ICheckpointObserver checkpointObserver = null)
{
Database = database;
CheckpointRegistry = checkpointRegistry;
Storage = storage;
_configuration = configuration;
_fileSystem = new PassThroughFileSystem();
_checkpointStagingDirectory = configuration.WorkingDirectory / "staging";
_checkpointObserver = checkpointObserver;
Counters = counters;
LinkLifetime(Database);
LinkLifetime(CheckpointRegistry);
LinkLifetime(_checkpointObserver);
LinkLifetime(Storage);
}
/// <summary>
/// Initializes a new instance of the <see cref="BlobReadOnlyContentSession"/> class.
/// </summary>
/// <param name="fileSystem">Filesystem used to read/write files.</param>
/// <param name="name">Session name.</param>
/// <param name="implicitPin">Policy determining whether or not content should be automatically pinned on adds or gets.</param>
/// <param name="dedupStoreHttpClient">Backing DedupStore http client.</param>
/// <param name="timeToKeepContent">Minimum time-to-live for accessed content.</param>
/// <param name="tracer">A tracer for tracking blob content session calls.</param>
/// <param name="maxConnections">The maximum number of outboud connections to VSTS.</param>
public DedupReadOnlyContentSession(
IAbsFileSystem fileSystem,
string name,
ImplicitPin implicitPin,
IDedupStoreHttpClient dedupStoreHttpClient,
TimeSpan timeToKeepContent,
BackingContentStoreTracer tracer,
int maxConnections = DefaultMaxConnections)
{
Contract.Requires(fileSystem != null);
Contract.Requires(name != null);
Contract.Requires(dedupStoreHttpClient != null);
Name = name;
ImplicitPin = implicitPin;
DedupStoreClient = new DedupStoreClient(dedupStoreHttpClient, DefaultMaxParallelism);
Tracer = tracer;
FileSystem = fileSystem;
TempDirectory = new DisposableDirectory(fileSystem);
ConnectionGate = new SemaphoreSlim(maxConnections);
EndDateTime = DateTime.UtcNow + timeToKeepContent;
}
/// <summary>
/// Initializes a new instance of the <see cref="Application"/> class.
/// </summary>
public Application(CancellationToken cancellationToken)
{
_cancellationToken = cancellationToken;
_consoleLog = new ConsoleLog(Severity.Warning);
_logger = new Logger(true, _consoleLog);
_fileSystem = new PassThroughFileSystem(_logger);
_tracer = new Tracer(nameof(Application));
var kustoConnectionString = Environment.GetEnvironmentVariable(KustoConnectionStringEnvVarName);
_kustoUploader = string.IsNullOrWhiteSpace(kustoConnectionString)
? null
: new KustoUploader
(
kustoConnectionString,
database: KustoDatabase,
table: KustoTable,
deleteFilesOnSuccess: true,
checkForIngestionErrors: true,
log: _consoleLog
);
}
public static Result <ServiceOfflineDurationTracker> Create(
OperationContext context,
IClock clock,
IAbsFileSystem fileSystem,
int?logIntervalSeconds,
AbsolutePath logFilePath)
{
return(context.PerformOperation(Tracer,
() =>
{
if (logIntervalSeconds == null)
{
return new Result <ServiceOfflineDurationTracker>($"{nameof(ServiceOfflineDurationTracker)} is disabled");
}
var serviceTracker = new ServiceOfflineDurationTracker(context, clock, fileSystem, logIntervalSeconds.Value, logFilePath);
// We can't log current timestamp here, because it will mess up with the following GetOfflineDuration call.
// Instead, the caller of GetOfflineDuration may specify whether to update the file or not.
return new Result <ServiceOfflineDurationTracker>(serviceTracker);
}));
}
private List <IQuotaRule> CreateRules(
IAbsFileSystem fileSystem,
QuotaKeeperConfiguration configuration,
IContentStoreInternal store)
{
var rules = new List <IQuotaRule>();
var distributedEvictionSettings = configuration.DistributedEvictionSettings;
if (configuration.EnableElasticity)
{
var elasticSizeRule = new ElasticSizeRule(
configuration.HistoryWindowSize,
configuration.InitialElasticSize,
() => CurrentSize,
store.ReadPinSizeHistory,
fileSystem,
store.RootPath);
rules.Add(elasticSizeRule);
}
else
{
if (configuration.MaxSizeQuota != null)
{
rules.Add(new MaxSizeRule(configuration.MaxSizeQuota, () => CurrentSize));
}
if (configuration.DiskFreePercentQuota != null)
{
rules.Add(new DiskFreePercentRule(configuration.DiskFreePercentQuota, fileSystem, store.RootPath));
}
}
if (!rules.Any())
{
throw new CacheException("At least one quota rule must be defined");
}
return(rules);
}
public DistributedContentCopier(
DistributedContentStoreSettings settings,
IAbsFileSystem fileSystem,
IRemoteFileCopier fileCopier,
IContentCommunicationManager copyRequester,
IClock clock,
ILogger logger)
{
Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);
_settings = settings;
_remoteFileCopier = fileCopier;
CommunicationManager = copyRequester;
FileSystem = fileSystem;
_clock = clock;
Context = new Context(logger);
_copyScheduler = settings.CopyScheduler.Create(Context);
_retryIntervals = settings.RetryIntervalForCopies;
_maxRetryCount = settings.MaxRetryCount;
}
/// <summary>
/// Initializes a new instance of the <see cref="BuildCacheSession"/> class.
/// </summary>
/// <param name="fileSystem">Filesystem used to read/write files.</param>
/// <param name="name">Session name.</param>
/// <param name="implicitPin">Policy determining whether or not content should be automatically pinned on adds or gets.</param>
/// <param name="cacheNamespace">The namespace of the cache being communicated with.</param>
/// <param name="cacheId">The id of the cache being communicated with.</param>
/// <param name="contentHashListAdapter">Backing BuildCache http client.</param>
/// <param name="backingContentSession">Backing BlobStore content session.</param>
/// <param name="maxFingerprintSelectorsToFetch">Maximum number of selectors to enumerate for a GetSelectors call.</param>
/// <param name="minimumTimeToKeepContentHashLists">Minimum time-to-live for created or referenced ContentHashLists.</param>
/// <param name="rangeOfTimeToKeepContentHashLists">Range of time beyond the minimum for the time-to-live of created or referenced ContentHashLists.</param>
/// <param name="fingerprintIncorporationEnabled">Feature flag to enable fingerprints incorporation</param>
/// <param name="maxDegreeOfParallelismForIncorporateRequests">Throttle the number of fingerprints chunks sent in parallel</param>
/// <param name="maxFingerprintsPerIncorporateRequest">Max fingerprints allowed per chunk</param>
/// <param name="writeThroughContentSession">Optional write-through session to allow writing-behind to BlobStore</param>
/// <param name="sealUnbackedContentHashLists">If true, the client will attempt to seal any unbacked ContentHashLists that it sees.</param>
/// <param name="overrideUnixFileAccessMode">If true, overrides default Unix file access modes when placing files.</param>
/// <param name="tracer">A tracer for logging calls</param>
public BuildCacheSession(
IAbsFileSystem fileSystem,
string name,
ImplicitPin implicitPin,
string cacheNamespace,
Guid cacheId,
IContentHashListAdapter contentHashListAdapter,
IContentSession backingContentSession,
int maxFingerprintSelectorsToFetch,
TimeSpan minimumTimeToKeepContentHashLists,
TimeSpan rangeOfTimeToKeepContentHashLists,
bool fingerprintIncorporationEnabled,
int maxDegreeOfParallelismForIncorporateRequests,
int maxFingerprintsPerIncorporateRequest,
IContentSession writeThroughContentSession,
bool sealUnbackedContentHashLists,
bool overrideUnixFileAccessMode,
BuildCacheCacheTracer tracer)
: base(
fileSystem,
name,
implicitPin,
cacheNamespace,
cacheId,
contentHashListAdapter,
backingContentSession,
maxFingerprintSelectorsToFetch,
minimumTimeToKeepContentHashLists,
rangeOfTimeToKeepContentHashLists,
fingerprintIncorporationEnabled,
maxDegreeOfParallelismForIncorporateRequests,
maxFingerprintsPerIncorporateRequest,
writeThroughContentSession,
sealUnbackedContentHashLists,
overrideUnixFileAccessMode,
tracer)
{
}
public DistributedContentCopier(
DistributedContentStoreSettings settings,
IAbsFileSystem fileSystem,
IRemoteFileCopier fileCopier,
IContentCommunicationManager copyRequester,
IClock clock,
ILogger logger)
{
Contract.Requires(settings != null);
Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);
_settings = settings;
_remoteFileCopier = fileCopier;
_copyRequester = copyRequester;
FileSystem = fileSystem;
_clock = clock;
_ioGate = new OrderedSemaphore(_settings.MaxConcurrentCopyOperations, _settings.OrderForCopies, new Context(logger));
_proactiveCopyIoGate = new OrderedSemaphore(_settings.MaxConcurrentProactiveCopyOperations, _settings.OrderForProactiveCopies, new Context(logger));
_retryIntervals = settings.RetryIntervalForCopies;
_maxRetryCount = settings.MaxRetryCount;
_ioGateTimeoutForProactiveCopies = settings.ProactiveCopyIOGateTimeout;
}
/// <summary>
/// Open the named file asynchronously for reading and sets the expected file length for performance reasons if available.
/// </summary>
/// <remarks>
/// Setting the file length up-front leads to 20-30% performance improvements.
/// </remarks>
public static StreamWithLength?TryOpenForWrite(
this IAbsFileSystem fileSystem,
AbsolutePath path,
long?expectingLength,
FileMode fileMode,
FileShare share,
FileOptions options = FileOptions.None,
int bufferSize = FileSystemDefaults.DefaultFileStreamBufferSize)
{
var stream = fileSystem.TryOpen(path, FileAccess.Write, fileMode, share, options, bufferSize);
if (stream == null)
{
return(null);
}
if (expectingLength != null)
{
stream.Value.Stream.SetLength(expectingLength.Value);
}
return(stream.Value);
}
public TestFileSystemContentStoreInternal(
IAbsFileSystem fileSystem,
IClock clock,
AbsolutePath rootPath,
ContentStoreConfiguration configuration,
Action <ContentHashWithSize> onContentAdded = null,
Action <ContentHashWithSize> onContentEvicted = null,
NagleQueue <ContentHash> nagleQueue = null)
: base(fileSystem, clock, rootPath, new ConfigurationModel(configuration), nagleQueue: nagleQueue)
{
Contract.Requires(fileSystem != null);
Contract.Requires(clock != null);
Contract.Requires(rootPath != null);
Contract.Requires(configuration != null);
_onContentAdded = onContentAdded;
_onContentEvicted = onContentEvicted;
if (_onContentAdded != null || _onContentEvicted != null)
{
Announcer = this;
}
}
/// <nodoc />
public GrpcContentServer(
ILogger logger,
Capabilities serviceCapabilities,
ISessionHandler <IContentSession> sessionHandler,
IReadOnlyDictionary <string, IContentStore> storesByName,
LocalServerConfiguration localServerConfiguration = null)
{
Contract.Requires(storesByName != null);
_serviceCapabilities = serviceCapabilities;
_contentStoreByCacheName = storesByName;
_bufferSize = localServerConfiguration?.BufferSizeForGrpcCopies ?? ContentStore.Grpc.CopyConstants.DefaultBufferSize;
_gzipSizeBarrier = localServerConfiguration?.GzipBarrierSizeForGrpcCopies ?? (_bufferSize * 8);
_pool = new ByteArrayPool(_bufferSize);
_sessionHandler = sessionHandler;
_fileSystem = localServerConfiguration?.FileSystem ?? new PassThroughFileSystem();
_workingDirectory = (localServerConfiguration?.DataRootPath ?? _fileSystem.GetTempPath()) / "GrpcContentServer";
GrpcAdapter = new ContentServerAdapter(this);
Logger = logger;
}
/// <nodoc />
public LocalServerConfiguration(
AbsolutePath dataRootPath,
IReadOnlyDictionary <string, AbsolutePath> namedCacheRoots,
int grpcPort,
IAbsFileSystem fileSystem,
int?bufferSizeForGrpcCopies = null,
int?gzipBarrierSizeForGrpcCopies = null,
int?proactivePushCountLimit = null,
TimeSpan?logIncrementalStatsInterval = null,
TimeSpan?logMachineStatsInterval = null
)
{
DataRootPath = dataRootPath;
NamedCacheRoots = namedCacheRoots;
GrpcPort = grpcPort;
BufferSizeForGrpcCopies = bufferSizeForGrpcCopies;
GzipBarrierSizeForGrpcCopies = gzipBarrierSizeForGrpcCopies;
ProactivePushCountLimit = proactivePushCountLimit;
FileSystem = fileSystem;
LogIncrementalStatsInterval = logIncrementalStatsInterval ?? DefaultLogIncrementalStatsInterval;
LogMachineStatsInterval = logMachineStatsInterval ?? DefaultLogMachineStatsInterval;
}
public QuotaKeeper(
IAbsFileSystem fileSystem,
ContentStoreInternalTracer tracer,
QuotaKeeperConfiguration configuration,
CancellationToken token,
FileSystemContentStoreInternal store,
IDistributedLocationStore?distributedStore)
{
Contract.RequiresNotNull(fileSystem);
Contract.RequiresNotNull(tracer);
Contract.RequiresNotNull(configuration);
_contentStoreTracer = tracer;
Tracer = new Tracer(name: $"{Component}({store.RootPath})");
_allContentSize = configuration.ContentDirectorySize;
_token = token;
_store = store;
_distributedStore = distributedStore;
_reserveQueue = new BlockingCollection <QuotaRequest>();
_evictionQueue = new ConcurrentQueue <ReserveSpaceRequest>();
_rules = CreateRules(fileSystem, configuration, store);
Counters = new CounterCollection <QuotaKeeperCounters>();
}
/// <summary>
/// Try getting the attributes of a file.
/// </summary>
/// <returns>Returns false if the file doesn't exist.</returns>
public static bool TryGetFileAttributes(this IAbsFileSystem fileSystem, AbsolutePath path, out FileAttributes attributes)
{
if (!fileSystem.FileExists(path))
{
attributes = default;
return(false);
}
try
{
attributes = fileSystem.GetFileAttributes(path);
}
catch (FileNotFoundException)
{
// We checked file existence at the top of the method, but due to a race condition, the file can be gone
// at the point when we tried getting attributes.
// Current implementation tries it best to avoid unnecessary first chance exceptions, but can't eliminate them completely.
attributes = default;
return(false);
}
return(true);
}
private LocalCache(
ILogger logger,
AbsolutePath rootPathForStream,
AbsolutePath rootPathForPath,
IAbsFileSystem fileSystem,
IClock clock,
ConfigurationModel configurationModelForStream,
ConfigurationModel configurationModelForPath,
SQLiteMemoizationStoreConfiguration memoConfig,
bool checkLocalFiles,
bool emptyFileHashShortcutEnabled)
: base(
() => new StreamPathContentStore(
() => new FileSystemContentStore(fileSystem, clock ?? SystemClock.Instance, rootPathForStream, configurationModelForStream, settings: new ContentStoreSettings() { CheckFiles = checkLocalFiles, UseEmptyFileHashShortcut = emptyFileHashShortcutEnabled }),
() => new FileSystemContentStore(fileSystem, clock ?? SystemClock.Instance, rootPathForPath, configurationModelForPath, settings: new ContentStoreSettings() { CheckFiles = checkLocalFiles, UseEmptyFileHashShortcut = emptyFileHashShortcutEnabled })),
() => new SQLiteMemoizationStore(
logger,
clock ?? SystemClock.Instance,
memoConfig),
PersistentId.Load(fileSystem, rootPathForPath / IdFileName))
{
_fileSystem = fileSystem;
}
public static Result <ServiceOfflineDurationTracker> Create(
OperationContext context,
IClock clock,
IAbsFileSystem fileSystem,
DistributedCacheServiceConfiguration configuration)
{
return(context.PerformOperation(Tracer,
() =>
{
var logIntervalSeconds = configuration.DistributedContentSettings.ServiceRunningLogInSeconds;
if (logIntervalSeconds == null)
{
return new Result <ServiceOfflineDurationTracker>($"{nameof(ServiceOfflineDurationTracker)} is disabled");
}
var logFilePath = configuration.LocalCasSettings.GetCacheRootPathWithScenario(LocalCasServiceSettings.DefaultCacheName) / FileName;
var serviceTracker = new ServiceOfflineDurationTracker(context, clock, fileSystem, logIntervalSeconds.Value, logFilePath);
serviceTracker.LogCurrentTimeStampToFile(context);
return new Result <ServiceOfflineDurationTracker>(serviceTracker);
}));
}
/// <summary>
/// Initializes a new instance of the <see cref="ElasticSizeRule" /> class.
/// </summary>
public ElasticSizeRule(
int?historyWindowSize,
MaxSizeQuota initialElasticSize,
EvictAsync evictAsync,
Func <long> getCurrentSizeFunc,
Func <int, PinSizeHistory.ReadHistoryResult> getPinnedSizeHistoryFunc,
IAbsFileSystem fileSystem,
AbsolutePath rootPath,
double?calibrationCoefficient = default(double?),
DistributedEvictionSettings distributedEvictionSettings = null)
: base(evictAsync, OnlyUnlinkedValue, distributedEvictionSettings)
{
Contract.Requires(!historyWindowSize.HasValue || historyWindowSize.Value >= 0);
Contract.Requires(evictAsync != null);
Contract.Requires(getCurrentSizeFunc != null);
Contract.Requires(getPinnedSizeHistoryFunc != null);
Contract.Requires(fileSystem != null);
Contract.Requires(rootPath != null);
Contract.Requires(!calibrationCoefficient.HasValue || calibrationCoefficient.Value > 1.0);
_historyWindowSize = historyWindowSize ?? DefaultHistoryWindowSize;
_calibrationCoefficient = calibrationCoefficient ?? DefaultCalibrationCoefficient;
_getPinnedSizeHistoryFunc = getPinnedSizeHistoryFunc;
_getCurrentSizeFunc = getCurrentSizeFunc;
_fileSystem = fileSystem;
_rootPath = rootPath;
_initialQuota = initialElasticSize ?? SmallQuota;
var loadQuotaResult = LoadOrCreateNewAsync(_fileSystem, _initialQuota, _rootPath).GetAwaiter().GetResult();
_quota = loadQuotaResult.Quota;
_historyTimestampInTick = loadQuotaResult.HistoryTimestampInTick;
// TODO: CalibrateAsync method fails in tests all the time. Bug #1331905
CalibrateAsync().GetAwaiter().GetResult().IgnoreFailure();
Contract.Assert(IsInsideHardLimit(0, checkIfQuotaEnabled: false).Succeeded);
}
/// <summary>
/// Initializes a new instance of the <see cref="FileSystemContentStore" /> class.
/// </summary>
public FileSystemContentStore(
IAbsFileSystem fileSystem,
IClock clock,
AbsolutePath rootPath,
ConfigurationModel configurationModel = null,
NagleQueue <ContentHash> nagleQueue = null,
DistributedEvictionSettings distributedEvictionSettings = null,
TrimBulkAsync trimBulkAsync = null,
ContentStoreSettings settings = null)
{
Contract.Requires(fileSystem != null);
Contract.Requires(clock != null);
Contract.Requires(rootPath != null);
int singleInstanceTimeoutSeconds = ContentStoreConfiguration.DefaultSingleInstanceTimeoutSeconds;
if (configurationModel?.InProcessConfiguration != null)
{
// TODO: Stop using the configurationModel's SingleInstanceTimeout (bug 1365340)
// because FileSystemContentStore doesn't respect the config file's value
singleInstanceTimeoutSeconds = configurationModel.InProcessConfiguration.SingleInstanceTimeoutSeconds;
}
// FileSystemContentStore implicitly uses a null component name for compatibility with older versions' directory locks.
_directoryLock = new DirectoryLock(rootPath, fileSystem, TimeSpan.FromSeconds(singleInstanceTimeoutSeconds));
Store = new FileSystemContentStoreInternal(
fileSystem,
clock,
rootPath,
configurationModel,
nagleQueue,
distributedEvictionSettings,
settings);
_trimBulkAsync = trimBulkAsync;
}
private LoadQuotaResult LoadOrCreateNew(IAbsFileSystem fileSystem, MaxSizeQuota initialElasticQuota, AbsolutePath rootPath)
{
var filePath = rootPath / BinaryFileName;
try
{
if (!fileSystem.FileExists(filePath))
{
return(CreateNew(fileSystem, initialElasticQuota, rootPath));
}
using (var stream = fileSystem.OpenReadOnly(filePath, FileShare.Delete))
{
using (var reader = new BinaryReader(stream))
{
return(new LoadQuotaResult(new MaxSizeQuota(reader.ReadInt64(), reader.ReadInt64()), reader.ReadInt64()));
}
}
}
catch (IOException)
{
return(CreateNew(fileSystem, initialElasticQuota, rootPath));
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ReadOnlyServiceClientContentSession"/> class.
/// </summary>
public ReadOnlyServiceClientContentSession(
string name,
ImplicitPin implicitPin,
ILogger logger,
IAbsFileSystem fileSystem,
ServiceClientContentSessionTracer sessionTracer,
ServiceClientContentStoreConfiguration configuration,
Func <IRpcClient>?rpcClientFactory = null)
: base(name)
{
Contract.Requires(name != null);
Contract.Requires(logger != null);
Contract.Requires(fileSystem != null);
_implicitPin = implicitPin;
SessionTracer = sessionTracer;
Logger = logger;
FileSystem = fileSystem;
Configuration = configuration;
TempFileStreamFactory = new TempFileStreamFactory(FileSystem);
RpcClient = (rpcClientFactory ?? GetRpcClient)();
RetryPolicy = configuration.RetryPolicy;
}
public static (TestDistributedContentCopier, MockFileCopier) CreateMocks(
IAbsFileSystem fileSystem,
AbsolutePath rootDirectory,
TimeSpan retryInterval,
int retries = 1)
{
var mockFileCopier = new MockFileCopier();
var existenceChecker = new TestFileCopier();
var contentCopier = new TestDistributedContentCopier(
rootDirectory,
// Need to use exactly one retry.
new DistributedContentStoreSettings()
{
RetryIntervalForCopies = Enumerable.Range(0, retries).Select(r => retryInterval).ToArray(),
TrustedHashFileSizeBoundary = long.MaxValue // Disable trusted hash because we never actually move bytes and thus the hasher thinks there is a mismatch.
},
fileSystem,
mockFileCopier,
existenceChecker,
copyRequester: null,
new TestDistributedContentCopier.NoOpPathTransformer(rootDirectory));
return(contentCopier, mockFileCopier);
}
/// <summary>
/// Open the named file asynchronously for reading and sets the expected file length for performance reasons if available.
/// </summary>
/// <exception cref="FileNotFoundException">Throws if the file is not found.</exception>
/// <exception cref="DirectoryNotFoundException">Throws if the directory is not found.</exception>
/// <remarks>
/// Setting the file length up-front leads to 20-30% performance improvements.
/// </remarks>
public static StreamWithLength OpenForWrite(
this IAbsFileSystem fileSystem,
AbsolutePath path,
long?expectingLength,
FileMode fileMode,
FileShare share,
FileOptions options = FileOptions.None,
int bufferSize = FileSystemDefaults.DefaultFileStreamBufferSize)
{
var stream = fileSystem.TryOpenForWrite(path, expectingLength, fileMode, share, options, bufferSize);
if (stream == null)
{
// Stream is null when the file or a directory is not found.
if (fileSystem.DirectoryExists(path.Parent !))
{
throw new FileNotFoundException($"The file '{path}' does not exist.");
}
throw new DirectoryNotFoundException($"The directory '{path.Parent}' does not exist.");
}
return(stream.Value);
}
/// <summary>
/// Initializes a new instance of the <see cref="RedisContentLocationStoreFactory"/> class.
/// </summary>
public RedisContentLocationStoreFactory(
/*CanBeNull*/ IConnectionStringProvider contentConnectionStringProvider,
/*CanBeNull*/ IConnectionStringProvider machineLocationConnectionStringProvider,
IClock clock,
TimeSpan contentHashBumpTime,
string keySpace,
IAbsFileSystem fileSystem = null,
RedisContentLocationStoreConfiguration configuration = null)
{
Contract.Requires(!string.IsNullOrWhiteSpace(keySpace));
_contentConnectionStringProvider = contentConnectionStringProvider;
_machineConnectionStringProvider = machineLocationConnectionStringProvider;
Clock = clock;
_contentHashBumpTime = contentHashBumpTime;
KeySpace = keySpace + Salt;
Configuration = configuration ?? RedisContentLocationStoreConfiguration.Default;
if (Configuration.HasReadOrWriteMode(ContentLocationMode.Redis))
{
Contract.Assert(contentConnectionStringProvider != null, "When ReadFromRedis is on 'contentConnectionStringProvider' must not be null.");
Contract.Assert(machineLocationConnectionStringProvider != null, "When ReadFromRedis is on 'machineLocationConnectionStringProvider' must not be null.");
}
}
public TestInProcessServiceClientContentStore(
IAbsFileSystem fileSystem,
ILogger logger,
string cacheName,
string scenario,
TimeSpan?heartbeatInterval,
ServiceConfiguration serviceConfiguration,
uint retryIntervalSeconds = DefaultRetryIntervalSeconds,
uint retryCount = DefaultRetryCount,
LocalServerConfiguration localContentServerConfiguration = null)
: base(logger, fileSystem, CreateConfiguration(cacheName, scenario + TestBase.ScenarioSuffix, serviceConfiguration, retryIntervalSeconds, retryCount))
{
_fileSystem = fileSystem;
_logger = logger;
_heartbeatInterval = heartbeatInterval;
_configuration = serviceConfiguration;
_server = new LocalContentServer(
_fileSystem,
_logger,
Configuration.Scenario,
path => new FileSystemContentStore(FileSystem, SystemClock.Instance, path),
localContentServerConfiguration?.OverrideServiceConfiguration(_configuration) ?? TestConfigurationHelper.CreateLocalContentServerConfiguration(_configuration));
SetThreadPoolSizes();
}
public TestFileSystemContentStoreInternal(
IAbsFileSystem fileSystem,
IClock clock,
AbsolutePath rootPath,
ContentStoreConfiguration configuration,
Action <ContentHashWithSize> onContentAdded = null,
Action <ContentHashWithSize> onContentEvicted = null,
ContentStoreSettings settings = null,
IDistributedLocationStore distributedStore = null)
: base(fileSystem, clock, rootPath, new ConfigurationModel(configuration), settings: settings, distributedStore: distributedStore)
{
Contract.Requires(fileSystem != null);
Contract.Requires(clock != null);
Contract.Requires(rootPath != null);
Contract.Requires(configuration != null);
_onContentAdded = onContentAdded;
_onContentEvicted = onContentEvicted;
if (_onContentAdded != null || _onContentEvicted != null)
{
Announcer = this;
}
}
protected virtual ICache CreateCache(
DisposableDirectory testDirectory, string cacheNamespace, IAbsFileSystem fileSystem, ILogger logger, BackingOption backingOption, StorageOption storageOption, TimeSpan?expiryMinimum = null, TimeSpan?expiryRange = null)
{
return(CreateBareBuildCache(testDirectory, cacheNamespace, fileSystem, logger, backingOption, storageOption, expiryMinimum, expiryRange));
}
protected ContentSessionTestsBase(Func <IAbsFileSystem> createFileSystemFunc, ILogger logger, ITestOutputHelper output = null)
: base(output)
{
FileSystem = createFileSystemFunc();
Logger = logger;
}
