/// <inheritdoc />
public override void PipExecutionStepPerformanceReported(PipExecutionStepPerformanceEventData data)
{
// Remove the counts after the pip completes
if (data.Step == PipExecutionStep.HandleResult && PipGraph.PipTable.GetPipType(data.PipId) == PipType.Process)
{
m_observedInputCounts.RemoveKey(data.PipId);
}
}
/// <summary>
/// Needs to take the flushing lock. Called only from <see cref="FlushAsync(OperationContext)"/>. Refactored
/// out for clarity.
/// </summary>
private void PerformFlush(OperationContext context)
{
_database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCacheFlushes].Increment();
using (_database.Counters[ContentLocationDatabaseCounters.CacheFlush].Start())
{
using (_exchangeLock.AcquireWriteLock())
{
_flushingCache = _cache;
_cache = new ConcurrentBigMap <ShortHash, ContentLocationEntry>();
}
if (_configuration.FlushSingleTransaction)
{
_database.PersistBatch(context, _flushingCache);
}
else
{
var actionBlock = new ActionBlockSlim <KeyValuePair <ShortHash, ContentLocationEntry> >(_configuration.FlushDegreeOfParallelism, kv =>
{
// Do not lock on GetLock here, as it will cause a deadlock with
// SetMachineExistenceAndUpdateDatabase. It is correct not do take any locks as well, because
// no Store can happen while flush is running.
_database.Persist(context, kv.Key, kv.Value);
});
foreach (var kv in _flushingCache)
{
actionBlock.Post(kv);
}
actionBlock.Complete();
actionBlock.CompletionAsync().Wait();
}
_database.Counters[ContentLocationDatabaseCounters.NumberOfPersistedEntries].Add(_flushingCache.Count);
if (_configuration.FlushPreservePercentInMemory > 0)
{
int targetFlushingSize = (int)(_flushingCache.Count * _configuration.FlushPreservePercentInMemory);
int removeAmount = _flushingCache.Count - targetFlushingSize;
foreach (var key in _flushingCache.Keys.Take(removeAmount))
{
_flushingCache.RemoveKey(key);
}
}
else
{
using (_exchangeLock.AcquireWriteLock())
{
_flushingCache = new ConcurrentBigMap <ShortHash, ContentLocationEntry>();
}
}
_database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCompletedCacheFlushes].Increment();
}
}
/// <summary>
/// Needs to take the flushing lock. Called only from <see cref="FlushAsync(OperationContext)"/>. Refactored
/// out for clarity.
/// </summary>
private CounterCollection <FlushableCacheCounters> PerformFlush(OperationContext context)
{
_database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCacheFlushes].Increment();
var counters = new CounterCollection <FlushableCacheCounters>();
using (_database.Counters[ContentLocationDatabaseCounters.CacheFlush].Start())
{
using (_exchangeLock.AcquireWriteLock())
{
_flushingCache = _cache;
_cache = new ConcurrentBigMap <ShortHash, ContentLocationEntry>();
}
using (counters[FlushableCacheCounters.FlushingTime].Start()) {
var threads = _configuration.FlushDegreeOfParallelism;
if (threads <= 0)
{
threads = Environment.ProcessorCount;
}
if (_configuration.FlushSingleTransaction)
{
if (_configuration.FlushDegreeOfParallelism == 1 || _flushingCache.Count <= _configuration.FlushTransactionSize)
{
_database.PersistBatch(context, _flushingCache);
}
else
{
var actionBlock = new ActionBlockSlim <IEnumerable <KeyValuePair <ShortHash, ContentLocationEntry> > >(threads, kvs =>
{
_database.PersistBatch(context, kvs);
});
foreach (var kvs in _flushingCache.GetPages(_configuration.FlushTransactionSize))
{
actionBlock.Post(kvs);
}
actionBlock.Complete();
actionBlock.CompletionAsync().Wait();
}
}
else
{
var actionBlock = new ActionBlockSlim <KeyValuePair <ShortHash, ContentLocationEntry> >(threads, kv =>
{
// Do not lock on GetLock here, as it will cause a deadlock with
// SetMachineExistenceAndUpdateDatabase. It is correct not do take any locks as well, because
// no Store can happen while flush is running.
_database.Persist(context, kv.Key, kv.Value);
});
foreach (var kv in _flushingCache)
{
actionBlock.Post(kv);
}
actionBlock.Complete();
actionBlock.CompletionAsync().Wait();
}
}
counters[FlushableCacheCounters.Persisted].Add(_flushingCache.Count);
_database.Counters[ContentLocationDatabaseCounters.NumberOfPersistedEntries].Add(_flushingCache.Count);
using (counters[FlushableCacheCounters.CleanupTime].Start())
{
if (_configuration.FlushPreservePercentInMemory > 0)
{
int targetFlushingSize = (int)(_flushingCache.Count * _configuration.FlushPreservePercentInMemory);
int removeAmount = _flushingCache.Count - targetFlushingSize;
foreach (var key in _flushingCache.Keys.Take(removeAmount))
{
_flushingCache.RemoveKey(key);
}
}
else
{
using (_exchangeLock.AcquireWriteLock())
{
_flushingCache = new ConcurrentBigMap <ShortHash, ContentLocationEntry>();
}
}
}
counters[FlushableCacheCounters.Leftover].Add(_flushingCache.Count);
_database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCompletedCacheFlushes].Increment();
}
counters[FlushableCacheCounters.Growth].Add(_cache.Count);
return(counters);
}