public async Task <IEnumerable <Task <CacheCandidate> > > OpenReadAsync(string stateName)
{
var innerEnumerable = _inner == null
? Array.Empty <Task <CacheCandidate> >()
: await _inner.OpenReadAsync(stateName).ConfigureAwait(false);
return(Enumerate(innerEnumerable));
IEnumerable <Task <CacheCandidate> >
Enumerate(IEnumerable <Task <CacheCandidate> > fromInner)
{
// First, attempt to use local blobs
var inDirectory = Directory.EnumerateFiles(Path.Combine(_directory, stateName))
.OrderByDescending(path => path);
foreach (var fileName in inDirectory)
{
// Wrap in Task.Run so that exceptions do not kill the enumeration
var filePath = Path.Combine(_directory, stateName, fileName);
yield return(Task.Run(() => MemoryMap(filePath, null)));
// If still enumerating, then the file did not contain a valid cache, and
// so we delete it to avoid reading it again.
DeleteFile(filePath);
}
// If not, fall back to proxy blobs
foreach (var task in innerEnumerable)
{
var copyToLocal = Task.Run(async() =>
{
var result = await task.ConfigureAwait(false);
using (var remote = result.Contents)
{
var open = OpenWriteLocal(stateName);
using (var local = open.stream)
{
await remote.CopyToAsync(local).ConfigureAwait(false);
}
return(open.path, result.Name);
}
});
yield return(Task.Run(async() =>
{
var(localPath, remoteName) = await copyToLocal.ConfigureAwait(false);
return MemoryMap(localPath, remoteName);
}));
// If still enumerating, then the file did not contain a valid cache, and
// so we delete it to avoid reading it again.
Task.Run(async() =>
{
var(localPath, _) = await copyToLocal.ConfigureAwait(false);
DeleteFile(localPath);
});
}
}
}
/// <summary>
/// Attempt to load this projection from the source, updating its
/// <see cref="Current"/> and <see cref="Sequence"/>.
/// </summary>
/// <remarks>
/// Object is unchanged if loading fails.
///
/// Obviously, as this object does not support multi-threaded access,
/// it should NOT be accessed in any way before the task has completed.
/// </remarks>
public async Task TryLoadAsync(CancellationToken cancel = default)
{
if (_cacheProvider == null)
{
_log?.Warning($"[{Name}] no read cache provider !");
return;
}
var sw = Stopwatch.StartNew();
IEnumerable <Task <CacheCandidate> > candidates;
try
{
candidates = await _cacheProvider.OpenReadAsync(Name);
}
catch (Exception ex)
{
_log?.Warning($"[{Name}] error when opening cache.", ex);
return;
}
foreach (var candidateTask in candidates)
{
CacheCandidate candidate;
try
{
candidate = await candidateTask;
}
catch (Exception ex)
{
_log?.Warning($"[{Name}] error when opening cache.", ex);
continue;
}
_log?.Info($"[{Name}] reading cache {candidate.Name}");
var stream = candidate.Contents;
try
{
// Load the sequence number from the input
uint seq;
using (var br = new BinaryReader(stream, Encoding.UTF8, true))
seq = br.ReadUInt32();
_log?.Debug($"[{Name}] cache is at seq {seq}.");
// Create a new stream to hide the write of the sequence numbers
// (at the top and the bottom of the stream).
var boundedStream = new BoundedStream(stream, stream.Length - 8);
// Load the state, which advances the stream
var state = await _projection.TryLoadAsync(boundedStream, cancel)
.ConfigureAwait(false);
if (state == null)
{
_log?.Warning($"[{Name}] projection could not parse cache {candidate.Name}");
continue;
}
// Sanity check: is the same sequence number found at the end ?
uint endseq;
using (var br = new BinaryReader(stream, Encoding.UTF8, true))
endseq = br.ReadUInt32();
if (endseq != seq)
{
_log?.Warning($"[{Name}] sanity-check seq is {endseq} in cache {candidate.Name}");
continue;
}
_log?.Info($"[{Name}] loaded {stream.Length} bytes in {sw.Elapsed:mm':'ss'.'fff} from cache {candidate.Name}");
Current = state;
Sequence = seq;
return;
// Do NOT set _possiblyInconsistent to false here !
// Inconsistency can have external causes, e.g. event read
// failure, that are not automagically solved by loading from cache.
}
catch (EndOfStreamException)
{
_log?.Warning($"[{Name}] incomplete cache {candidate.Name}");
// Incomplete streams are simply treated as missing
}
catch (Exception ex)
{
_log?.Warning($"[{Name}] could not parse cache {candidate.Name}", ex);
// If a cache file cannot be parsed, try the next one
}
finally
{
stream.Dispose();
}
}
}
public async Task <IEnumerable <Task <CacheCandidate> > > OpenReadAsync(string stateName)
{
var innerEnumerable = _inner == null
? Array.Empty <Task <CacheCandidate> >()
: await _inner.OpenReadAsync(stateName).ConfigureAwait(false);
return(Enumerate(innerEnumerable));
IEnumerable <Task <CacheCandidate> >
Enumerate(IEnumerable <Task <CacheCandidate> > fromInner)
{
// First, attempt to use local blobs
var inDirectory = Directory.EnumerateFiles(Path.Combine(_directory, stateName))
.OrderByDescending(path => path);
foreach (var fileName in inDirectory)
{
// Wrap in Task.Run so that exceptions do not kill the enumeration
yield return(Task.Run(() => MemoryMap(fileName, null)));
// If still enumerating, then the file did not contain a valid cache, and
// so we delete it to avoid reading it again.
DeleteFile(fileName);
}
// If not, fall back to proxy blobs
foreach (var task in innerEnumerable)
{
var copyToLocal = Task.Run(async() =>
{
var result = await task.ConfigureAwait(false);
using (var remote = result.Contents)
{
var open = OpenWriteLocal(stateName);
using (var local = open.stream)
{
await remote.CopyToAsync(local).ConfigureAwait(false);
}
return(open.filename, result.Name);
}
});
yield return(Task.Run(async() =>
{
var(localName, remoteName) = await copyToLocal.ConfigureAwait(false);
return MemoryMap(localName, remoteName);
}));
// If still enumerating, then the file did not contain a valid cache, and
// so we delete it to avoid reading it again.
Task.Run(async() =>
{
var(localName, _) = await copyToLocal.ConfigureAwait(false);
DeleteFile(localName);
});
}
}
/// Memory-maps the file with the specified name
CacheCandidate MemoryMap(string fileName, string from = null)
{
var path = Path.Combine(_directory, stateName, fileName);
var length = new FileInfo(path).Length;
return(new CacheCandidate(
"mmap:" + path + (from == null ? "" : " from " + from),
new BigMemoryStream(new MemoryMapper(
MemoryMappedFile.CreateFromFile(path, FileMode.Open),
0, length))));
}
void DeleteFile(string fileName)
{
var path = Path.Combine(_directory, stateName, fileName);
try { File.Delete(path); }
catch { /* Ignored silently. */ }
}
}
/// <summary>
/// Attempt to load this projection from the source, updating its
/// <see cref="Current"/> and <see cref="Sequence"/>.
/// </summary>
/// <remarks>
/// Object is unchanged if loading fails.
///
/// Obviously, as this object does not support multi-threaded access,
/// it should NOT be accessed in any way before the task has completed.
/// </remarks>
public async Task TryLoadAsync(CancellationToken cancel = default(CancellationToken))
{
if (_cacheProvider == null)
{
_log?.Warning($"[{Name}] no read cache provider !");
return;
}
Stream source;
var sw = Stopwatch.StartNew();
try
{
source = await _cacheProvider.OpenReadAsync(Name);
}
catch (Exception ex)
{
_log?.Warning($"[{Name}] error when opening cache.", ex);
return;
}
if (source == null)
{
_log?.Info($"[{Name}] no cached data found.");
return;
}
try
{
// Load the sequence number from the input
uint seq;
using (var br = new BinaryReader(source, Encoding.UTF8, true))
seq = br.ReadUInt32();
_log?.Debug($"[{Name}] cache is at seq {seq}.");
// Load the state, which advances the stream
var state = await _projection.TryLoadAsync(source, cancel).ConfigureAwait(false);
if (state == null)
{
_log?.Warning($"[{Name}] projection could not parse cache.");
return;
}
// Sanity check: is the same sequence number found at the end ?
uint endseq;
using (var br = new BinaryReader(source, Encoding.UTF8, true))
endseq = br.ReadUInt32();
if (endseq != seq)
{
_log?.Warning($"[{Name}] sanity-check seq is {endseq}.");
return;
}
_log?.Info($"[{Name}] loaded from cache in {sw.Elapsed:mm':'ss'.'fff}.");
Current = state;
Sequence = seq;
// Do NOT set _possiblyInconsistent to false here !
// Inconsistency can have external causes, e.g. event read
// failure, that are not automagically solved by loading from cache.
}
catch (EndOfStreamException)
{
_log?.Warning($"[{Name}] cache is incomplete.");
// Incomplete streams are simply treated as missing
}
catch (Exception ex)
{
_log?.Warning($"[{Name}] could not parse cache.", ex);
throw;
}
}