/// <summary>
/// Initializes a new instance of the
/// <see cref="ChangeSetPerformanceScope"/> class.
/// </summary>
/// <param name="instanceName">The instance name.</param>
/// <param name="changeSet">The change set.</param>
public ChangeSetPerformanceScope(string instanceName, Bulk.ChangeSet changeSet)
{
if (!s_enabled)
return;
_counters = s_counters.GetOrAdd(instanceName, CounterCollection.Create);
_size = changeSet.TotalItemsCount;
_startTicks = Stopwatch.GetTimestamp();
}
public PayoffParser(Economy e, string s)
{
var cc = new CounterCollection();
var eco = e;
NodeOps = new NodeOperation[] { new Add(), new Subtract(), new Multiply(), new Divide(), new Power(),
new Sqrt(), new Exp(), new Log(), new Ln(), new Abs(), new Max(), new Max(cc), new Min(), new Min(cc),
new Int(), new Pos(), new Rate(eco), new ATM(eco), new SwapRate(eco), new Discount(eco), new PV(eco),
new Sum(), new Sum(cc), new Counter(cc) };
// Warning: The order of the first 5 cannot be changed!
MasterNode = new Node(s, NodeOps);
}
private FileChangeTracker(LoggingContext loggingContext)
{
Contract.Requires(loggingContext != null);
m_loggingContext = loggingContext;
m_volumeMap = null;
m_journal = null;
m_changeTrackingSet = null;
m_trackingStateValue = (int)FileChangeTrackingState.DisabledSinceTrackingIsIncomplete;
Counters = new CounterCollection <FileChangeTrackingCounter>();
m_buildEngineFingerprint = null;
}
/// <summary>
/// Appends <paramref name="other"/> into a current collection instance.
/// </summary>
public void Append(CounterCollection <TEnum> other)
{
foreach (var value in EnumTraits <TEnum> .EnumerateValues())
{
if (IsStopwatch(value))
{
AddToCounter(value, other.GetElapsedTime(value));
}
AddToCounter(value, other.GetCounterValue(value));
}
}
/// <inheritdoc />
protected override async Task <BoolResult> SendEventsCoreAsync(
OperationContext context,
ContentLocationEventData[] events,
CounterCollection <ContentLocationEventStoreCounters> counters)
{
IReadOnlyList <EventData> eventDatas;
using (counters[Serialization].Start())
{
eventDatas = SerializeEventData(context, events);
}
var operationId = Guid.NewGuid();
context = context.CreateNested(operationId);
for (var eventNumber = 0; eventNumber < eventDatas.Count; eventNumber++)
{
var eventData = eventDatas[eventNumber];
eventData.Properties[EventFilterKey] = _configuration.Epoch;
eventData.Properties[SenderMachineKey] = _localMachineName;
counters[SentEventBatchCount].Increment();
Tracer.Debug(
context,
$"{Tracer.Name}: Sending {eventNumber}/{events.Length} event. OpId={operationId}, Epoch='{_configuration.Epoch}', Size={eventData.Body.Count}.");
counters[SentMessagesTotalSize].Add(eventData.Body.Count);
eventData.Properties[OperationIdKey] = operationId.ToString();
// Even though event hub client has it's own built-in retry strategy, we have to wrap all the calls into a separate
// one to cover a few more important cases that the default strategy misses.
await _extraEventHubClientRetryPolicy.ExecuteAsync(async() =>
{
try
{
await _partitionSender.SendAsync(eventData);
}
catch (ServerBusyException exception)
{
// TODO: Verify that the HResult is 50002. Documentation shows that this should be the error code for throttling,
// but documentation is done for Microsoft.ServiceBus.Messaging.ServerBusyException and not Microsoft.Azure.EventHubs.ServerBusyException
// https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-messaging-exceptions#serverbusyexception
Tracer.Debug(context, $"{Tracer.Name}: OpId={operationId} was throttled by EventHub. HResult={exception.HResult}");
Tracer.TrackMetric(context, "EventHubThrottle", 1);
throw;
}
});
}
return(BoolResult.Success);
}
/// <nodoc />
public PipTwoPhaseCache(LoggingContext loggingContext, EngineCache cache, PipExecutionContext context, PathExpander pathExpander)
{
Contract.Requires(loggingContext != null);
Contract.Requires(cache != null);
Contract.Requires(context != null);
LoggingContext = loggingContext;
ArtifactContentCache = cache.ArtifactContentCache;
TwoPhaseFingerprintStore = cache.TwoPhaseFingerprintStore;
Context = context;
Counters = new CounterCollection <PipCachingCounter>();
m_pathExpander = pathExpander;
}
static Inner()
{
PassThroughOperationKind = CreateOperationKind("[Pass-Through]", false);
var prefix = "PipExecutorCounter.";
foreach (var counter in EnumTraits <PipExecutorCounter> .EnumerateValues())
{
int trackedCacheCounterId = IsTrackedCacheCounter(counter) ? s_trackedCacheCounterKinds.Count : -1;
if (!CounterCollection <PipExecutorCounter> .IsStopwatch(counter))
{
s_pipExecutorCounterKinds.Add(Invalid);
continue;
}
var counterName = counter.ToString();
if (counterName.EndsWith(DurationSuffix, StringComparison.OrdinalIgnoreCase))
{
counterName = counterName.Substring(0, counterName.Length - DurationSuffix.Length);
}
counterName = prefix + counterName;
var operationKind = CreateOperationKind(counterName, /* hasPipTypeSpecialization */ IsPipTypeSpecificCounter(counter), trackedCacheCounterId: trackedCacheCounterId);
s_pipExecutorCounterKinds.Add(operationKind);
if (trackedCacheCounterId != -1)
{
s_trackedCacheCounterKinds.Add(operationKind);
}
}
prefix = "PipExecutionStep.";
foreach (var counter in EnumTraits <PipExecutionStep> .EnumerateValues())
{
var counterName = counter.ToString();
counterName = prefix + counterName;
var operationKind = CreateOperationKind(counterName, /* hasPipTypeSpecialization */ true);
s_pipExecutionStepKinds.Add(operationKind);
}
prefix = "Operation.";
foreach (var counter in EnumTraits <OperationCounter> .EnumerateValues())
{
var counterName = counter.ToString();
counterName = prefix + counterName;
var operationKind = CreateOperationKind(counterName, /* hasPipTypeSpecialization */ false);
s_operationCounterKinds.Add(operationKind);
}
}
public ActionResult Save(Bam.Net.Analytics.Counter[] values)
{
try
{
CounterCollection saver = new CounterCollection();
saver.AddRange(values);
saver.Save();
return(Json(new { Success = true, Message = "", Dao = "" }));
}
catch (Exception ex)
{
return(GetErrorResult(ex));
}
}
/// <summary>
/// Constructor for build set calculator.
/// </summary>
/// <param name="loggingContext">Logging context.</param>
/// <param name="graph">Pip graph.</param>
/// <param name="dirtyNodeTracker">Dirty node tracker.</param>
/// <param name="counters">Counter collection.</param>
protected BuildSetCalculator(
LoggingContext loggingContext,
DirectedGraph graph,
DirtyNodeTracker dirtyNodeTracker,
CounterCollection<PipExecutorCounter> counters)
{
Contract.Requires(loggingContext != null);
Contract.Requires(graph != null);
Contract.Requires(counters != null);
m_loggingContext = loggingContext;
m_graph = graph;
m_visitor = new NodeVisitor(graph);
m_dirtyNodeTracker = dirtyNodeTracker;
m_counters = counters;
}
/// <nodoc />
public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient)
{
m_logger = logger;
m_apiClient = apiClient;
m_config = config;
m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior;
m_cancellationSource = new CancellationTokenSource();
m_counters = new CounterCollection <SymbolClientCounter>();
m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}"));
m_symbolClient = new ReloadingSymbolClient(
logger: logger,
clientConstructor: CreateSymbolServiceClient);
}
public static CounterCollection <TEnum> operator +(CounterCollection <TEnum> x, CounterCollection <TEnum> y)
{
var result = new CounterCollection <TEnum>();
foreach (var value in EnumTraits <TEnum> .EnumerateValues())
{
if (IsStopwatch(value))
{
result.AddToCounter(value, x.GetElapsedTime(value) + y.GetElapsedTime(value));
}
result.AddToCounter(value, x.GetCounterValue(value) + y.GetCounterValue(value));
}
return(result);
}
public void SingleCounterUnderflow()
{
var collection = new CounterCollection(1);
XAssert.AreEqual(0, collection.GetCounterValueInternal(0));
collection.AddToCounterInternal(0, long.MinValue + 3);
collection.AddToCounterInternal(0, -3);
ExpectOverflow(
() =>
{
// Since the counter is sharded, overflow may manifest in Add (single shard overflow) or Get (overflow of shards only in aggregate).
collection.AddToCounterInternal(0, -1);
collection.GetCounterValueInternal(0);
});
}
/// <inheritdoc />
public CheckpointManager(
ContentLocationDatabase database,
ICheckpointRegistry checkpointRegistry,
CentralStorage storage,
CheckpointConfiguration configuration,
CounterCollection <ContentLocationStoreCounters> counters)
{
_database = database;
_checkpointRegistry = checkpointRegistry;
_storage = storage;
_configuration = configuration;
_fileSystem = new PassThroughFileSystem();
_checkpointStagingDirectory = configuration.WorkingDirectory / "staging";
Counters = counters;
}
public async Task ResourceInvalidationRespectsReferenceCountBeforeShutdown()
{
var stopLatch = TaskUtilities.CreateMutex(taken: true);
Task?outstandingTask = null;
CounterCollection <ResourcePoolV2Counters>?counters = null;
await RunTest <Key, Resource>(async (context, pool) =>
{
counters = pool.Counter;
var key = new Key(0);
await pool.UseAsync(context, key, async wrapper =>
{
outstandingTask = pool.UseAsync(context, key, async wrapper2 =>
{
wrapper2.ReferenceCount.Should().Be(2);
stopLatch.Release();
try
{
await Task.Delay(Timeout.InfiniteTimeSpan, wrapper2.ShutdownToken);
}
catch (TaskCanceledException) { }
wrapper2.ReferenceCount.Should().Be(1);
counters[ResourcePoolV2Counters.CreatedResources].Value.Should().Be(1);
counters[ResourcePoolV2Counters.ReleasedResources].Value.Should().Be(0);
counters[ResourcePoolV2Counters.ShutdownAttempts].Value.Should().Be(0);
return(BoolResult.Success);
}).ShouldBeSuccess();
using var token = await stopLatch.AcquireAsync();
wrapper.Invalidate(context);
return(BoolResult.Success);
}).ShouldBeSuccess();
});
Contract.AssertNotNull(counters);
Contract.AssertNotNull(outstandingTask);
// This should throw any exceptions that haven't been caught
await outstandingTask;
}
/// <nodoc />
public static CounterSet ToCounterSet(this CounterCollection counters)
{
var counterSet = new CounterSet();
foreach ((var counter, var counterName) in counters.GetCounters())
{
counterSet.Add($"{counterName}.Count", (long)counter.Value, counter.Name);
if (counter.IsStopwatch)
{
counterSet.Add($"{counterName}.AverageMs", counter.Value != 0 ? (long)counter.Duration.TotalMilliseconds / counter.Value : 0);
counterSet.Add($"{counterName}.DurationMs", (long)counter.Duration.TotalMilliseconds);
}
}
return(counterSet);
}
public QuotaKeeper(
IAbsFileSystem fileSystem,
ContentStoreInternalTracer tracer,
QuotaKeeperConfiguration configuration,
CancellationToken token,
IContentStoreInternal store)
{
_contentStoreTracer = tracer;
_tracer = new Tracer(name: Component);
_allContentSize = configuration.ContentDirectorySize;
_token = token;
_store = store;
_reserveQueue = new BlockingCollection <QuotaRequest>();
_evictionQueue = new ConcurrentQueue <ReserveSpaceRequest>();
_distributedEvictionSettings = configuration.DistributedEvictionSettings;
_rules = CreateRules(fileSystem, configuration, store);
Counters = new CounterCollection <QuotaKeeperCounters>();
}
private static void ValidateCounters(CounterCollection <ResourcePoolV2Counters> counters)
{
counters[ResourcePoolV2Counters.CreatedResources].Value.Should().BeGreaterOrEqualTo(counters[ResourcePoolV2Counters.ResourceInitializationAttempts].Value);
counters[ResourcePoolV2Counters.ResourceInitializationAttempts].Value.Should().BeGreaterOrEqualTo(counters[ResourcePoolV2Counters.ReleasedResources].Value);
counters[ResourcePoolV2Counters.ReleasedResources].Value.Should().BeGreaterOrEqualTo(counters[ResourcePoolV2Counters.ShutdownAttempts].Value);
counters[ResourcePoolV2Counters.ResourceInitializationSuccesses].Value.Should().Be(counters[ResourcePoolV2Counters.ShutdownAttempts].Value);
counters[ResourcePoolV2Counters.ResourceInitializationAttempts].Value.Should().Be(counters[ResourcePoolV2Counters.ReleasedResources].Value);
counters[ResourcePoolV2Counters.ResourceInitializationAttempts].Value.Should().Be(counters[ResourcePoolV2Counters.ResourceInitializationSuccesses].Value + counters[ResourcePoolV2Counters.ResourceInitializationFailures].Value);
counters[ResourcePoolV2Counters.ShutdownAttempts].Value.Should().Be(counters[ResourcePoolV2Counters.ShutdownSuccesses].Value + counters[ResourcePoolV2Counters.ShutdownFailures].Value);
counters[ResourcePoolV2Counters.ReleasedResources].Value.Should().Be(counters[ResourcePoolV2Counters.ShutdownAttempts].Value + counters[ResourcePoolV2Counters.ResourceInitializationFailures].Value);
counters[ResourcePoolV2Counters.GarbageCollectionAttempts].Value.Should().BeGreaterOrEqualTo(1);
counters[ResourcePoolV2Counters.GarbageCollectionSuccesses].Value.Should().BeGreaterOrEqualTo(1);
}
public async Task FailingStartupThrowsOnThreadAndDoesntCache()
{
CounterCollection <ResourcePoolV2Counters>?counters = null;
var configuration = new ResourcePoolConfiguration()
{
MaximumAge = TimeSpan.FromSeconds(1),
GarbageCollectionPeriod = Timeout.InfiniteTimeSpan,
};
await RunTest <Key, FailingResource>(async (context, pool) =>
{
counters = pool.Counter;
var key = new Key(0);
_ = await Assert.ThrowsAsync <ResultPropagationException>(() =>
{
return(pool.UseAsync <BoolResult>(context, key, wrapper =>
{
throw new NotImplementedException("This should not happen!");
}));
});
// This is just to ensure that retrying effectively does a retry
_ = await Assert.ThrowsAsync <ResultPropagationException>(() =>
{
return(pool.UseAsync <BoolResult>(context, key, wrapper =>
{
throw new NotImplementedException("This should not happen!");
}));
});
},
_ => new FailingResource(startupFailure: true),
configuration);
Contract.AssertNotNull(counters);
// We do 2 attempts to create a failing resource, so 4 total
counters[ResourcePoolV2Counters.ResourceInitializationAttempts].Value.Should().Be(2);
counters[ResourcePoolV2Counters.ResourceInitializationFailures].Value.Should().Be(2);
counters[ResourcePoolV2Counters.CreatedResources].Value.Should().Be(2);
counters[ResourcePoolV2Counters.ReleasedResources].Value.Should().Be(2);
}
public QuotaKeeper(
IAbsFileSystem fileSystem,
ContentStoreInternalTracer tracer,
QuotaKeeperConfiguration configuration,
CancellationToken token,
FileSystemContentStoreInternal store,
IDistributedLocationStore?distributedStore)
{
_contentStoreTracer = tracer;
Tracer = new Tracer(name: $"{Component}({store.RootPath})");
_allContentSize = configuration.ContentDirectorySize;
_token = token;
_store = store;
_distributedStore = distributedStore;
_reserveQueue = new BlockingCollection <QuotaRequest>();
_evictionQueue = new ConcurrentQueue <ReserveSpaceRequest>();
_rules = CreateRules(fileSystem, configuration, store);
Counters = new CounterCollection <QuotaKeeperCounters>();
}
public void SingleCounterWithContention()
{
var collection = new CounterCollection(1);
XAssert.AreEqual(0, collection.GetCounterValueInternal(0));
var threads = new Thread[Environment.ProcessorCount];
long expected = 0;
bool[] allStarted = new[] { false };
for (int i = 0; i < threads.Length; i++)
{
threads[i] = new Thread(
() =>
{
long thisThreadAdded = 0;
while (!Volatile.Read(ref allStarted[0]))
{
collection.AddToCounterInternal(0, 1);
thisThreadAdded++;
}
for (int j = 0; j < 30; j++)
{
collection.AddToCounterInternal(0, 1);
thisThreadAdded++;
}
Interlocked.Add(ref expected, thisThreadAdded);
});
threads[i].Start();
}
Volatile.Write(ref allStarted[0], true);
for (int i = 0; i < threads.Length; i++)
{
threads[i].Join();
}
XAssert.AreEqual(expected, collection.GetCounterValueInternal(0));
}
/// <inheritdoc />
public CheckpointManager(
ContentLocationDatabase database,
ICheckpointRegistry checkpointRegistry,
CentralStorage storage,
CheckpointConfiguration configuration,
CounterCollection <ContentLocationStoreCounters> counters)
{
_database = database;
_checkpointRegistry = checkpointRegistry;
_storage = storage;
_configuration = configuration;
_fileSystem = new PassThroughFileSystem();
_checkpointStagingDirectory = configuration.WorkingDirectory / "staging";
_incrementalCheckpointDirectory = configuration.WorkingDirectory / "incremental";
_lastCheckpointFile = configuration.WorkingDirectory / "lastCheckpoint.txt";
_fileSystem.CreateDirectory(_incrementalCheckpointDirectory);
_incrementalCheckpointInfoFile = _incrementalCheckpointDirectory / "checkpointInfo.txt";
Counters = counters;
}
/// <summary>
/// Представить исходный код, содержащийся в файле, в виде набора синтаксических единиц
/// </summary>
/// <param name="args"></param>
/// <returns></returns>
public override CSyntacticUnitsCollection Syntacticize(CTokenizerParms args)
{
CSyntacticUnitsCollection syntactic_collection = new CSyntacticUnitsCollection();
Parser parser = new Parser(new Scanner(args.GetPath()), string.Empty);
CounterCollection counters = parser.Parse();
if (parser.errors.count > 0)
{
//throw new System.Exception(args.GetPath() + " parse error");
}
else
{
SyntUnitCreator synt_creator = new SyntUnitCreator(args);
foreach (MethodBlock mb in counters.MethodCounters)
{
syntactic_collection.Add(synt_creator.FromMethodBlock(mb));
}
}
return(syntactic_collection);
}
/// <nodoc />
public static CounterSet ToCounterSet <TEnum>(this CounterCollection <TEnum> counters)
where TEnum : struct
{
CounterSet counterSet = new CounterSet();
foreach (var counterEnum in EnumTraits <TEnum> .EnumerateValues())
{
var counter = counters[counterEnum];
var counterName = counterEnum.ToString();
counterSet.Add($"{counterName}.Count", (long)counter.Value, counter.Name);
if (counter.IsStopwatch)
{
counterSet.Add($"{counterName}.AverageMs", counter.Value != 0 ? (long)counter.Duration.TotalMilliseconds / counter.Value : 0);
counterSet.Add($"{counterName}.DurationMs", (long)counter.Duration.TotalMilliseconds);
}
}
return(counterSet);
}
public static IRemoteProcessManager GetOrCreate(
LoggingContext loggingContext,
PipExecutionContext executionContext,
IConfiguration configuration,
CounterCollection <SandboxedProcessFactory.SandboxedProcessCounters> counters)
{
Interlocked.CompareExchange(
ref RemoteProcessManager,
new Lazy <IRemoteProcessManager>(() =>
{
#if FEATURE_ANYBUILD_PROCESS_REMOTING
IRemoteProcessManager remoteProcessManager = new AnyBuildRemoteProcessManager(loggingContext, executionContext, configuration, counters);
#else
IRemoteProcessManager remoteProcessManager = new NoRemotingRemoteProcessManager();
#endif
return(remoteProcessManager);
}),
null);
return(RemoteProcessManager.Value);
}
/// <summary>
/// Check if an exception is trancient and is worth retrying.
/// </summary>
/// <param name="ex">The exception thrown by Bond code.</param>
/// <param name="verifierCounters">counters use to track transient error type occurrences</param>
/// <returns>True is the retry makes sense; false otherwise.</returns>
internal static bool IsTransientBondException(Exception ex, CounterCollection <DistributionCounter> verifierCounters)
{
// Unwrap if its an aggregate exception
AggregateException aggregateException = ex as AggregateException;
if (aggregateException != null && aggregateException.InnerExceptions.Count == 1)
{
ex = aggregateException.InnerExceptions[0];
}
// SocketException is thrown when something goes wrong in the TCP channel.
// InvalidOperationException is thrown when a previous call has closed the connection.
// In the second case a retry will reopen the connection.
// BondTcpClient.OnConnectionComplete throws IOException on failure instead of SocketException.
// Sometimes we get TimeoutException
if (ex is SocketException || ex is InvalidOperationException || ex is IOException || ex is TimeoutException)
{
return(true);
}
#if !DISABLE_FEATURE_BOND_RPC
// 'No such method' probably means the buffer was corrupted somehow
// Retry and see if it succeeds next time
if (ex is Microsoft.Bond.RpcException && ex.Message.Contains("No such method"))
{
verifierCounters.IncrementCounter(DistributionCounter.ClientNoSuchMethodErrorCount);
return(true);
}
#endif
// If the connection gets broken while waiting for response, we can see a bug in the NetlibConnection
// where it tries to initialize an array section with null. It throws an unhandled and unwrapped
// ArgumentNullException.
if ((ex is ArgumentNullException || ex is NullReferenceException) && ex.StackTrace.Contains("Netlib"))
{
return(true);
}
return(false);
}
/// <summary>
/// Constructor.
/// </summary>
private FingerprintStoreExecutionLogTarget(
LoggingContext loggingContext,
PipExecutionContext context,
PipTable pipTable,
PipContentFingerprinter pipContentFingerprinter,
FingerprintStore fingerprintStore,
FingerprintStore cacheLookupFingerprintStore,
IConfiguration configuration,
EngineCache cache,
IReadonlyDirectedGraph graph,
CounterCollection <FingerprintStoreCounters> counters,
IDictionary <PipId, RunnablePipPerformanceInfo> runnablePipPerformance)
{
m_context = context;
m_pipTable = pipTable;
LoggingContext = loggingContext;
PipContentFingerprinter = pipContentFingerprinter;
ExecutionFingerprintStore = fingerprintStore;
CacheLookupFingerprintStore = cacheLookupFingerprintStore;
// Cache lookup store is per-build state and doesn't need to be garbage collected (vs. execution fignerprint store which is persisted build-over-build)
CacheLookupFingerprintStore?.GarbageCollectCancellationToken.Cancel();
m_pipCacheMissesQueue = new ConcurrentQueue <PipCacheMissInfo>();
Counters = counters;
FingerprintStoreMode = configuration.Logging.FingerprintStoreMode;
m_runtimeCacheMissAnalyzerTask = RuntimeCacheMissAnalyzer.TryCreateAsync(
this,
loggingContext,
context,
configuration,
cache,
graph,
runnablePipPerformance);
m_fingerprintStoreEventProcessor = new FingerprintStoreEventProcessor(Environment.ProcessorCount);
m_weakFingerprintSerializationTransientCache = new ConcurrentDictionary <PipId, string>();
Contract.Assume(
FingerprintStoreMode == FingerprintStoreMode.ExecutionFingerprintsOnly || CacheLookupFingerprintStore != null,
"Unless /storeFingerprints flag is set to /storeFingerprints:ExecutionFingerprintsOnly, the cache lookup FingerprintStore must exist.");
}
/// <summary>
/// Initializes a new instance of the <see cref="BlobReadOnlyContentSession"/> class.
/// </summary>
/// <param name="fileSystem">Filesystem used to read/write files.</param>
/// <param name="name">Session name.</param>
/// <param name="implicitPin">Policy determining whether or not content should be automatically pinned on adds or gets.</param>
/// <param name="blobStoreHttpClient">Backing BlobStore http client.</param>
/// <param name="timeToKeepContent">Minimum time-to-live for accessed content.</param>
/// <param name="downloadBlobsThroughBlobStore">If true, gets blobs through BlobStore. If false, gets blobs from the Azure Uri.</param>
/// <param name="counterTracker">Parent counters to track the session.</param>
public BlobReadOnlyContentSession(
IAbsFileSystem fileSystem,
string name,
ImplicitPin implicitPin,
IBlobStoreHttpClient blobStoreHttpClient,
TimeSpan timeToKeepContent,
bool downloadBlobsThroughBlobStore,
CounterTracker counterTracker = null)
: base(name, counterTracker)
{
Contract.Requires(fileSystem != null);
Contract.Requires(name != null);
Contract.Requires(blobStoreHttpClient != null);
ImplicitPin = implicitPin;
BlobStoreHttpClient = blobStoreHttpClient;
TimeToKeepContent = timeToKeepContent;
_downloadBlobsThroughBlobStore = downloadBlobsThroughBlobStore;
_parallelSegmentDownloadConfig = new ParallelHttpDownload
.Configuration(
segmentDownloadTimeout: TimeSpan.FromMinutes(int.Parse(
Environment.GetEnvironmentVariable(EnvironmentVariablePrefix + "SegmentDownloadTimeoutInMinutes") ??
DefaultSegmentDownloadTimeoutInMinutes.ToString())),
segmentSizeInBytes: int.Parse(
Environment.GetEnvironmentVariable(EnvironmentVariablePrefix + "ParallelDownloadSegmentSizeInBytes") ??
DefaultParallelDownloadSegmentSizeInBytes.ToString()),
maxParallelSegmentDownloadsPerFile: int.Parse(
Environment.GetEnvironmentVariable(EnvironmentVariablePrefix + "MaxParallelSegmentDownloadsPerFile") ??
DefaultMaxParallelSegmentDownloadsPerFile.ToString()),
maxSegmentDownloadRetries:
int.Parse(
Environment.GetEnvironmentVariable(EnvironmentVariablePrefix + "MaxSegmentDownloadRetries") ??
DefaultMaxSegmentDownloadRetries.ToString()));
TempDirectory = new DisposableDirectory(fileSystem);
_counters = CounterTracker.CreateCounterCollection <BackingContentStore.SessionCounters>(counterTracker);
_blobCounters = CounterTracker.CreateCounterCollection <Counters>(counterTracker);
}
/// <inheritdoc />
public CheckpointManager(
ContentLocationDatabase database,
ICheckpointRegistry checkpointRegistry,
CentralStorage storage,
CheckpointManagerConfiguration configuration,
CounterCollection <ContentLocationStoreCounters> counters,
ICheckpointObserver checkpointObserver = null)
{
Database = database;
CheckpointRegistry = checkpointRegistry;
Storage = storage;
_configuration = configuration;
_fileSystem = new PassThroughFileSystem();
_checkpointStagingDirectory = configuration.WorkingDirectory / "staging";
_checkpointObserver = checkpointObserver;
Counters = counters;
LinkLifetime(Database);
LinkLifetime(CheckpointRegistry);
LinkLifetime(_checkpointObserver);
LinkLifetime(Storage);
}
/// <nodoc />
public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient)
{
m_logger = logger;
m_apiClient = apiClient;
m_config = config;
m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior;
m_cancellationSource = new CancellationTokenSource();
m_counters = new CounterCollection <SymbolClientCounter>();
m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}"));
m_symbolClient = new ReloadingSymbolClient(
logger: logger,
clientConstructor: CreateSymbolServiceClient);
m_nagleQueue = NagleQueue <BatchedSymbolFile> .Create(
maxDegreeOfParallelism : m_config.MaxParallelUploads,
batchSize : m_config.BatchSize,
interval : m_config.NagleTime,
processBatch : ProcessBatchedFilesAsync);
m_fileUploadQueue = new ActionQueue(m_config.MaxParallelUploads);
}
public BlobReadOnlyContentSession(
BackingContentStoreConfiguration configuration,
string name,
ImplicitPin implicitPin,
IBlobStoreHttpClient blobStoreHttpClient,
CounterTracker?counterTracker = null)
: base(name, counterTracker)
{
Contract.Requires(configuration != null);
Contract.Requires(configuration.FileSystem != null);
Contract.Requires(name != null);
Contract.Requires(blobStoreHttpClient != null);
Configuration = configuration;
ImplicitPin = implicitPin;
BlobStoreHttpClient = blobStoreHttpClient;
_parallelSegmentDownloadConfig = ParallelHttpDownload.DownloadConfiguration.ReadFromEnvironment(EnvironmentVariablePrefix);
TempDirectory = new DisposableDirectory(configuration.FileSystem);
_counters = CounterTracker.CreateCounterCollection <BackingContentStore.SessionCounters>(counterTracker);
_blobCounters = CounterTracker.CreateCounterCollection <Counters>(counterTracker);
}
public void Temporal()
{
var collection = new CounterCollection <TestCounters>();
Stopwatch sw = Stopwatch.StartNew();
while (sw.ElapsedMilliseconds < 300)
{
using (collection.StartStopwatch(TestCounters.SomeTime))
{
Thread.Sleep(30);
}
}
sw.Stop();
// These times won't be exactly the same since the counter stopwatch is only measuring the sleep, not the loop and counter manipulation etc.
// In any of those steps we can get unlucky and be descheduled. But a large discrepancy is in high likelihood some serious fault.
TimeSpan delta = sw.Elapsed - collection.GetElapsedTime(TestCounters.SomeTime);
XAssert.IsTrue(
delta < 100.MillisecondsToTimeSpan(),
"Difference between System.Diagnostics.Stopwatch and the Counter stopwatch was very large");
}
