public async Task <ReserveSpaceResult> TryReserveSpace(CacheEntry cacheEntry, string contentType, int byteCount,
bool allowEviction, AsyncLockProvider writeLocks,
CancellationToken cancellationToken)
{
var shard = Database.GetShardForKey(cacheEntry.RelativePath);
var shardSizeLimit = Options.MaxCacheBytes / Database.GetShardCount();
// When we're okay with deleting the database entry even though the file isn't written
var farFuture = DateTime.UtcNow.AddHours(1);
var maxAttempts = 30;
for (var attempts = 0; attempts < maxAttempts; attempts++)
{
var recordCreated =
await Database.CreateRecordIfSpace(shard, cacheEntry.RelativePath,
contentType,
EstimateEntryBytesWithOverhead(byteCount),
farFuture,
AccessCounter.GetHash(cacheEntry.Hash),
shardSizeLimit);
// Return true if we created the record
if (recordCreated)
{
return new ReserveSpaceResult()
{
Success = true
}
}
;
// We need to evict but we are not permitted
if (!allowEviction)
{
return new ReserveSpaceResult()
{
Success = false, Message = "Eviction disabled in sync mode"
}
}
;
var entryDiskSpace = EstimateEntryBytesWithOverhead(byteCount) +
Database.EstimateRecordDiskSpace(cacheEntry.RelativePath.Length + (contentType?.Length ?? 0));
var missingSpace = Math.Max(0, await Database.GetShardSize(shard) + entryDiskSpace - shardSizeLimit);
// Evict space
var evictResult = await EvictSpace(shard, missingSpace, writeLocks, cancellationToken);
if (!evictResult.Success)
{
return(evictResult); //We failed to evict enough space from the cache
}
}
return(new ReserveSpaceResult()
{
Success = false, Message = $"Eviction worked but CreateRecordIfSpace failed {maxAttempts} times."
});
}
public async void TestActiveLockCount()
{
var provider = new AsyncLockProvider();
var task = provider.TryExecuteAsync("1", 1500, CancellationToken.None, async() =>
{
await Task.Delay(50);
});
var task2 = provider.TryExecuteAsync("2", 1500, CancellationToken.None, async() =>
{
await Task.Delay(50);
});
var task2B = provider.TryExecuteAsync("2", 1500, CancellationToken.None, async() =>
{
await Task.Delay(50);
});
var task3 = provider.TryExecuteAsync("3", 1500, CancellationToken.None, async() =>
{
await Task.Delay(50);
throw new Exception();
});
Assert.Equal(3, provider.GetActiveLockCount());
await task;
await task2;
await task2B;
await Assert.ThrowsAsync <Exception>(async() => await task3);
Assert.Equal(0, provider.GetActiveLockCount());
}
public AsyncCache(AsyncCacheOptions options, ICacheCleanupManager cleanupManager, HashBasedPathBuilder pathBuilder, ILogger logger)
{
Options = options;
PathBuilder = pathBuilder;
CleanupManager = cleanupManager;
Logger = logger;
FileWriteLocks = new AsyncLockProvider();
QueueLocks = new AsyncLockProvider();
EvictAndWriteLocks = new AsyncLockProvider();
CurrentWrites = new AsyncWriteCollection(options.MaxQueuedBytes);
FileWriter = new CacheFileWriter(FileWriteLocks, Options.MoveFileOverwriteFunc, Options.MoveFilesIntoPlace);
}
public void TestConcurrency()
{
var provider = new AsyncLockProvider();
int sharedValue = 0;
var tasks = Enumerable.Range(0, 10).Select(async unused =>
Assert.True(await provider.TryExecuteAsync("1", 15000, CancellationToken.None, async() =>
{
var oldValue = sharedValue;
sharedValue++;
await Task.Delay(5);
sharedValue = oldValue;
}))).ToArray();
Assert.True(provider.MayBeLocked("1"));
Assert.Equal(1, provider.GetActiveLockCount());
Task.WaitAll(tasks);
Assert.Equal(0, sharedValue);
Assert.Equal(0, provider.GetActiveLockCount());
}
public EntityStore(IKeyValueStore <TK, TV> dbStore, string entityName, int keyShardCount = 1)
{
this.dbStore = Preconditions.CheckNotNull(dbStore, nameof(dbStore));
this.keyLockProvider = new AsyncLockProvider <TK>(Preconditions.CheckRange(keyShardCount, 1, nameof(keyShardCount)));
this.EntityName = Preconditions.CheckNonWhiteSpace(entityName, nameof(entityName));
}
private async Task <ReserveSpaceResult> EvictSpace(int shard, long diskSpace, AsyncLockProvider writeLocks, CancellationToken cancellationToken)
{
var bytesToDeleteOptimally = Math.Max(Options.MinCleanupBytes, diskSpace);
var bytesToDeleteMin = diskSpace;
long bytesDeleted = 0;
while (bytesDeleted < bytesToDeleteOptimally)
{
var deletionCutoff = DateTime.UtcNow.Subtract(Options.RetryDeletionAfter);
var creationCutoff = DateTime.UtcNow.Subtract(Options.MinAgeToDelete);
var records =
(await Database.GetDeletionCandidates(shard, deletionCutoff, creationCutoff, Options.CleanupSelectBatchSize, AccessCounter.Get))
.Select(r =>
new Tuple <ushort, ICacheDatabaseRecord>(
AccessCounter.Get(r.AccessCountKey), r))
.OrderByDescending(r => r.Item1)
.Select(r => r.Item2).ToArray();
foreach (var record in records)
{
if (cancellationToken.IsCancellationRequested)
{
throw new OperationCanceledException(cancellationToken);
}
if (bytesDeleted >= bytesToDeleteOptimally)
{
break;
}
var deletedBytes = await TryDeleteRecord(shard, record, writeLocks);
bytesDeleted += deletedBytes;
}
// Unlikely to find more records in the next iteration
if (records.Length < Options.CleanupSelectBatchSize)
{
// If we hit the bare minimum, return OK
if (bytesDeleted >= bytesToDeleteMin)
{
return(new ReserveSpaceResult()
{
Success = true
});
}
else
{
return(new ReserveSpaceResult()
{
Success = false, Message = $"Failed to evict enough space using {records.Length} candidates"
});
}
}
}
return(new ReserveSpaceResult()
{
Success = true
});
}
/// <summary>
/// Skips the record if there is delete contention for a file.
/// Only counts bytes as deleted if the physical file is deleted successfully.
/// Deletes db record whether file exists or not.
/// </summary>
/// <param name="shard"></param>
/// <param name="record"></param>
/// <param name="writeLocks"></param>
/// <returns></returns>
private async Task <long> TryDeleteRecord(int shard, ICacheDatabaseRecord record, AsyncLockProvider writeLocks)
{
long bytesDeleted = 0;
var unused = await writeLocks.TryExecuteAsync(record.RelativePath, 0, CancellationToken.None, async() =>
{
var physicalPath = PathBuilder.GetPhysicalPathFromRelativePath(record.RelativePath);
try
{
if (File.Exists(physicalPath))
{
File.Delete(physicalPath);
await Database.DeleteRecord(shard, record);
bytesDeleted = record.DiskSize;
}
else
{
await Database.DeleteRecord(shard, record);
}
}
catch (IOException ioException)
{
if (physicalPath.Contains(".moving_"))
{
// We already moved it. All we can do is update the last deletion attempt
await Database.UpdateLastDeletionAttempt(shard, record.RelativePath, DateTime.UtcNow);
return;
}
var movedRelativePath = record.RelativePath + ".moving_" +
new Random().Next(int.MaxValue).ToString("x", CultureInfo.InvariantCulture);
var movedPath = PathBuilder.GetPhysicalPathFromRelativePath(movedRelativePath);
try
{
//Move it so it usage will decrease and it can be deleted later
//TODO: This is not transactional, as the db record is written *after* the file is moved
//This should be split up into create and delete
(Options.MoveFileOverwriteFunc ?? File.Move)(physicalPath, movedPath);
await Database.ReplaceRelativePathAndUpdateLastDeletion(shard, record, movedRelativePath,
DateTime.UtcNow);
Logger?.LogError(ioException, "HybridCache: Error deleting file, moved for eventual deletion - {Path}", record.RelativePath);
}
catch (IOException ioException2)
{
await Database.UpdateLastDeletionAttempt(shard, record.RelativePath, DateTime.UtcNow);
Logger?.LogError(ioException2, "HybridCache: Failed to move file for eventual deletion - {Path}", record.RelativePath);
}
}
});
return(bytesDeleted);
}
public CacheFileWriter(AsyncLockProvider writeLocks, Action <string, string> moveFileOverwriteFunc, bool moveIntoPlace)
{
this.moveIntoPlace = moveIntoPlace;
WriteLocks = writeLocks;
this.moveFileOverwriteFunc = moveFileOverwriteFunc ?? File.Move;
}
public Task <ReserveSpaceResult> TryReserveSpace(CacheEntry cacheEntry, string contentType, int byteCount, bool allowEviction,
AsyncLockProvider writeLocks,
CancellationToken cancellationToken) => Task.FromResult(new ReserveSpaceResult()
{
Success = true
});
