Exemple #1
0
        public async Task <IEnumerable <Task <CacheCandidate> > > OpenReadAsync(string stateName)
        {
            var innerEnumerable = _inner == null
                ? Array.Empty <Task <CacheCandidate> >()
                : await _inner.OpenReadAsync(stateName).ConfigureAwait(false);

            return(Enumerate(innerEnumerable));

            IEnumerable <Task <CacheCandidate> >
            Enumerate(IEnumerable <Task <CacheCandidate> > fromInner)
            {
                // First, attempt to use local blobs
                var inDirectory = Directory.EnumerateFiles(Path.Combine(_directory, stateName))
                                  .OrderByDescending(path => path);

                foreach (var fileName in inDirectory)
                {
                    // Wrap in Task.Run so that exceptions do not kill the enumeration
                    var filePath = Path.Combine(_directory, stateName, fileName);
                    yield return(Task.Run(() => MemoryMap(filePath, null)));

                    // If still enumerating, then the file did not contain a valid cache, and
                    // so we delete it to avoid reading it again.
                    DeleteFile(filePath);
                }

                // If not, fall back to proxy blobs
                foreach (var task in innerEnumerable)
                {
                    var copyToLocal = Task.Run(async() =>
                    {
                        var result = await task.ConfigureAwait(false);
                        using (var remote = result.Contents)
                        {
                            var open = OpenWriteLocal(stateName);
                            using (var local = open.stream)
                            {
                                await remote.CopyToAsync(local).ConfigureAwait(false);
                            }

                            return(open.path, result.Name);
                        }
                    });

                    yield return(Task.Run(async() =>
                    {
                        var(localPath, remoteName) = await copyToLocal.ConfigureAwait(false);
                        return MemoryMap(localPath, remoteName);
                    }));

                    // If still enumerating, then the file did not contain a valid cache, and
                    // so we delete it to avoid reading it again.
                    Task.Run(async() =>
                    {
                        var(localPath, _) = await copyToLocal.ConfigureAwait(false);
                        DeleteFile(localPath);
                    });
                }
            }
        }
        /// <summary>
        /// Attempt to load this projection from the source, updating its
        /// <see cref="Current"/> and <see cref="Sequence"/>.
        /// </summary>
        /// <remarks>
        /// Object is unchanged if loading fails.
        ///
        /// Obviously, as this object does not support multi-threaded access,
        /// it should NOT be accessed in any way before the task has completed.
        /// </remarks>
        public async Task TryLoadAsync(CancellationToken cancel = default)
        {
            if (_cacheProvider == null)
            {
                _log?.Warning($"[{Name}] no read cache provider !");
                return;
            }

            var sw = Stopwatch.StartNew();

            IEnumerable <Task <CacheCandidate> > candidates;

            try
            {
                candidates = await _cacheProvider.OpenReadAsync(Name);
            }
            catch (Exception ex)
            {
                _log?.Warning($"[{Name}] error when opening cache.", ex);
                return;
            }

            foreach (var candidateTask in candidates)
            {
                CacheCandidate candidate;
                try
                {
                    candidate = await candidateTask;
                }
                catch (Exception ex)
                {
                    _log?.Warning($"[{Name}] error when opening cache.", ex);
                    continue;
                }

                _log?.Info($"[{Name}] reading cache {candidate.Name}");

                var stream = candidate.Contents;
                try
                {
                    // Load the sequence number from the input
                    uint seq;
                    using (var br = new BinaryReader(stream, Encoding.UTF8, true))
                        seq = br.ReadUInt32();

                    _log?.Debug($"[{Name}] cache is at seq {seq}.");

                    // Create a new stream to hide the write of the sequence numbers
                    // (at the top and the bottom of the stream).
                    var boundedStream = new BoundedStream(stream, stream.Length - 8);

                    // Load the state, which advances the stream
                    var state = await _projection.TryLoadAsync(boundedStream, cancel)
                                .ConfigureAwait(false);

                    if (state == null)
                    {
                        _log?.Warning($"[{Name}] projection could not parse cache {candidate.Name}");
                        continue;
                    }

                    // Sanity check: is the same sequence number found at the end ?
                    uint endseq;
                    using (var br = new BinaryReader(stream, Encoding.UTF8, true))
                        endseq = br.ReadUInt32();

                    if (endseq != seq)
                    {
                        _log?.Warning($"[{Name}] sanity-check seq is {endseq} in cache {candidate.Name}");
                        continue;
                    }

                    _log?.Info($"[{Name}] loaded {stream.Length} bytes in {sw.Elapsed:mm':'ss'.'fff} from cache {candidate.Name}");

                    Current  = state;
                    Sequence = seq;
                    return;

                    // Do NOT set _possiblyInconsistent to false here !
                    // Inconsistency can have external causes, e.g. event read
                    // failure, that are not automagically solved by loading from cache.
                }
                catch (EndOfStreamException)
                {
                    _log?.Warning($"[{Name}] incomplete cache {candidate.Name}");
                    // Incomplete streams are simply treated as missing
                }
                catch (Exception ex)
                {
                    _log?.Warning($"[{Name}] could not parse cache {candidate.Name}", ex);
                    // If a cache file cannot be parsed, try the next one
                }
                finally
                {
                    stream.Dispose();
                }
            }
        }
        public async Task <IEnumerable <Task <CacheCandidate> > > OpenReadAsync(string stateName)
        {
            var innerEnumerable = _inner == null
                ? Array.Empty <Task <CacheCandidate> >()
                : await _inner.OpenReadAsync(stateName).ConfigureAwait(false);

            return(Enumerate(innerEnumerable));

            IEnumerable <Task <CacheCandidate> >
            Enumerate(IEnumerable <Task <CacheCandidate> > fromInner)
            {
                // First, attempt to use local blobs
                var inDirectory = Directory.EnumerateFiles(Path.Combine(_directory, stateName))
                                  .OrderByDescending(path => path);

                foreach (var fileName in inDirectory)
                {
                    // Wrap in Task.Run so that exceptions do not kill the enumeration
                    yield return(Task.Run(() => MemoryMap(fileName, null)));

                    // If still enumerating, then the file did not contain a valid cache, and
                    // so we delete it to avoid reading it again.
                    DeleteFile(fileName);
                }

                // If not, fall back to proxy blobs
                foreach (var task in innerEnumerable)
                {
                    var copyToLocal = Task.Run(async() =>
                    {
                        var result = await task.ConfigureAwait(false);
                        using (var remote = result.Contents)
                        {
                            var open = OpenWriteLocal(stateName);
                            using (var local = open.stream)
                            {
                                await remote.CopyToAsync(local).ConfigureAwait(false);
                            }

                            return(open.filename, result.Name);
                        }
                    });

                    yield return(Task.Run(async() =>
                    {
                        var(localName, remoteName) = await copyToLocal.ConfigureAwait(false);
                        return MemoryMap(localName, remoteName);
                    }));

                    // If still enumerating, then the file did not contain a valid cache, and
                    // so we delete it to avoid reading it again.
                    Task.Run(async() =>
                    {
                        var(localName, _) = await copyToLocal.ConfigureAwait(false);
                        DeleteFile(localName);
                    });
                }
            }

            /// Memory-maps the file with the specified name
            CacheCandidate MemoryMap(string fileName, string from = null)
            {
                var path   = Path.Combine(_directory, stateName, fileName);
                var length = new FileInfo(path).Length;

                return(new CacheCandidate(
                           "mmap:" + path + (from == null ? "" : " from " + from),
                           new BigMemoryStream(new MemoryMapper(
                                                   MemoryMappedFile.CreateFromFile(path, FileMode.Open),
                                                   0, length))));
            }

            void DeleteFile(string fileName)
            {
                var path = Path.Combine(_directory, stateName, fileName);

                try { File.Delete(path); }
                catch { /* Ignored silently. */ }
            }
        }
Exemple #4
0
        /// <summary>
        /// Attempt to load this projection from the source, updating its
        /// <see cref="Current"/> and <see cref="Sequence"/>.
        /// </summary>
        /// <remarks>
        /// Object is unchanged if loading fails.
        ///
        /// Obviously, as this object does not support multi-threaded access,
        /// it should NOT be accessed in any way before the task has completed.
        /// </remarks>
        public async Task TryLoadAsync(CancellationToken cancel = default(CancellationToken))
        {
            if (_cacheProvider == null)
            {
                _log?.Warning($"[{Name}] no read cache provider !");
                return;
            }

            Stream source;

            var sw = Stopwatch.StartNew();

            try
            {
                source = await _cacheProvider.OpenReadAsync(Name);
            }
            catch (Exception ex)
            {
                _log?.Warning($"[{Name}] error when opening cache.", ex);
                return;
            }

            if (source == null)
            {
                _log?.Info($"[{Name}] no cached data found.");
                return;
            }

            try
            {
                // Load the sequence number from the input
                uint seq;
                using (var br = new BinaryReader(source, Encoding.UTF8, true))
                    seq = br.ReadUInt32();

                _log?.Debug($"[{Name}] cache is at seq {seq}.");

                // Load the state, which advances the stream
                var state = await _projection.TryLoadAsync(source, cancel).ConfigureAwait(false);

                if (state == null)
                {
                    _log?.Warning($"[{Name}] projection could not parse cache.");
                    return;
                }

                // Sanity check: is the same sequence number found at the end ?
                uint endseq;
                using (var br = new BinaryReader(source, Encoding.UTF8, true))
                    endseq = br.ReadUInt32();

                if (endseq != seq)
                {
                    _log?.Warning($"[{Name}] sanity-check seq is {endseq}.");
                    return;
                }

                _log?.Info($"[{Name}] loaded from cache in {sw.Elapsed:mm':'ss'.'fff}.");

                Current  = state;
                Sequence = seq;

                // Do NOT set _possiblyInconsistent to false here !
                // Inconsistency can have external causes, e.g. event read
                // failure, that are not automagically solved by loading from cache.
            }
            catch (EndOfStreamException)
            {
                _log?.Warning($"[{Name}] cache is incomplete.");
                // Incomplete streams are simply treated as missing
            }
            catch (Exception ex)
            {
                _log?.Warning($"[{Name}] could not parse cache.", ex);
                throw;
            }
        }