Example #1
0
        public bool TryCacheInMemory(bool shouldCacheNextChunk)
        {
            lock (_cacheSyncObj)
            {
                if (!_chunkConfig.EnableCache || _isMemoryChunk || !_isCompleted || _memoryChunk != null)
                {
                    _cachingChunk = 0;
                    return(false);
                }

                try
                {
                    var chunkSize = (ulong)GetChunkSize(_chunkHeader);
                    if (!ChunkUtil.IsMemoryEnoughToCacheChunk(chunkSize, (uint)_chunkConfig.ChunkCacheMaxPercent))
                    {
                        return(false);
                    }
                    _memoryChunk = FromCompletedFile(_filename, _chunkManager, _chunkConfig, true);
                    if (shouldCacheNextChunk)
                    {
                        Task.Factory.StartNew(() => _chunkManager.TryCacheNextChunk(this));
                    }
                    return(true);
                }
                catch (OutOfMemoryException) { return(false); }
                catch (Exception ex)
                {
                    _logger.Error(string.Format("Failed to cache completed chunk {0}", this), ex);
                    return(false);
                }
                finally
                {
                    _cachingChunk = 0;
                }
            }
        }
Example #2
0
        private void InitNew(int chunkNumber)
        {
            var chunkDataSize = 0;

            if (_chunkConfig.ChunkDataSize > 0)
            {
                chunkDataSize = _chunkConfig.ChunkDataSize;
            }
            else
            {
                chunkDataSize = _chunkConfig.ChunkDataUnitSize * _chunkConfig.ChunkDataCount;
            }

            _chunkHeader = new ChunkHeader(chunkNumber, chunkDataSize);

            _isCompleted = false;

            var fileSize = ChunkHeader.Size + _chunkHeader.ChunkDataTotalSize + ChunkFooter.Size;

            var writeStream    = default(Stream);
            var tempFilename   = string.Format("{0}.{1}.tmp", _filename, Guid.NewGuid());
            var tempFileStream = default(FileStream);

            try
            {
                if (_isMemoryChunk)
                {
                    _cachedLength = fileSize;
                    _cachedData   = Marshal.AllocHGlobal(_cachedLength);
                    writeStream   = new UnmanagedMemoryStream((byte *)_cachedData, _cachedLength, _cachedLength, FileAccess.ReadWrite);
                    writeStream.Write(_chunkHeader.AsByteArray(), 0, ChunkHeader.Size);
                }
                else
                {
                    var fileInfo = new FileInfo(_filename);
                    if (fileInfo.Exists)
                    {
                        File.SetAttributes(_filename, FileAttributes.Normal);
                        File.Delete(_filename);
                    }

                    tempFileStream = new FileStream(tempFilename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.Read, _chunkConfig.ChunkWriteBuffer, FileOptions.None);
                    tempFileStream.SetLength(fileSize);
                    tempFileStream.Write(_chunkHeader.AsByteArray(), 0, ChunkHeader.Size);
                    tempFileStream.Flush(true);
                    tempFileStream.Close();

                    File.Move(tempFilename, _filename);

                    writeStream = new FileStream(_filename, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, _chunkConfig.ChunkWriteBuffer, FileOptions.SequentialScan);
                    SetFileAttributes();
                }

                writeStream.Position = ChunkHeader.Size;

                _dataPosition        = 0;
                _flushedDataPosition = 0;
                _writerWorkItem      = new WriterWorkItem(new ChunkFileStream(writeStream));

                InitializeReaderWorkItems();

                if (!_isMemoryChunk)
                {
                    if (_chunkConfig.EnableCache)
                    {
                        var chunkSize = (ulong)GetChunkSize(_chunkHeader);
                        if (ChunkUtil.IsMemoryEnoughToCacheChunk(chunkSize, (uint)_chunkConfig.ChunkCacheMaxPercent))
                        {
                            try
                            {
                                _memoryChunk = CreateNew(_filename, chunkNumber, _chunkManager, _chunkConfig, true);
                            }
                            catch (OutOfMemoryException)
                            {
                                _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                            }
                            catch (Exception ex)
                            {
                                _logger.Error(string.Format("Failed to cache new chunk {0}", this), ex);
                                _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                            }
                        }
                        else
                        {
                            _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                        }
                    }
                    else
                    {
                        _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                    }
                }
            }
            catch
            {
                if (!_isMemoryChunk)
                {
                    if (tempFileStream != null)
                    {
                        Helper.EatException(() => tempFileStream.Close());
                    }
                    if (File.Exists(tempFilename))
                    {
                        Helper.EatException(() =>
                        {
                            File.SetAttributes(tempFilename, FileAttributes.Normal);
                            File.Delete(tempFilename);
                        });
                    }
                }
                throw;
            }

            _lastActiveTime = DateTime.Now;
        }
Example #3
0
        private void InitOngoing <T>(Func <byte[], T> readRecordFunc) where T : ILogRecord
        {
            var fileInfo = new FileInfo(_filename);

            if (!fileInfo.Exists)
            {
                throw new ChunkFileNotExistException(_filename);
            }

            _isCompleted = false;

            if (!TryParsingDataPosition(readRecordFunc, out _chunkHeader, out _dataPosition))
            {
                throw new ChunkBadDataException(string.Format("Failed to parse chunk data, chunk file: {0}", _filename));
            }

            _flushedDataPosition = _dataPosition;

            var writeStream = default(Stream);

            if (_isMemoryChunk)
            {
                var fileSize = ChunkHeader.Size + _chunkHeader.ChunkDataTotalSize + ChunkFooter.Size;
                _cachedLength = fileSize;
                _cachedData   = Marshal.AllocHGlobal(_cachedLength);
                writeStream   = new UnmanagedMemoryStream((byte *)_cachedData, _cachedLength, _cachedLength, FileAccess.ReadWrite);

                writeStream.Write(_chunkHeader.AsByteArray(), 0, ChunkHeader.Size);

                if (_dataPosition > 0)
                {
                    using (var fileStream = new FileStream(_filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 8192, FileOptions.SequentialScan))
                    {
                        fileStream.Seek(ChunkHeader.Size, SeekOrigin.Begin);
                        var buffer      = new byte[65536];
                        int toReadBytes = _dataPosition;

                        while (toReadBytes > 0)
                        {
                            int read = fileStream.Read(buffer, 0, Math.Min(toReadBytes, buffer.Length));
                            if (read == 0)
                            {
                                break;
                            }
                            toReadBytes -= read;
                            writeStream.Write(buffer, 0, read);
                        }
                    }
                }

                if (writeStream.Position != GetStreamPosition(_dataPosition))
                {
                    throw new InvalidOperationException(string.Format("UnmanagedMemoryStream position incorrect, expect: {0}, but: {1}", _dataPosition + ChunkHeader.Size, writeStream.Position));
                }
            }
            else
            {
                writeStream          = new FileStream(_filename, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, _chunkConfig.ChunkWriteBuffer, FileOptions.SequentialScan);
                writeStream.Position = GetStreamPosition(_dataPosition);
                SetFileAttributes();
            }

            _writerWorkItem = new WriterWorkItem(new ChunkFileStream(writeStream));

            InitializeReaderWorkItems();

            if (!_isMemoryChunk)
            {
                if (_chunkConfig.EnableCache)
                {
                    var chunkSize = (ulong)GetChunkSize(_chunkHeader);
                    if (ChunkUtil.IsMemoryEnoughToCacheChunk(chunkSize, (uint)_chunkConfig.ChunkCacheMaxPercent))
                    {
                        try
                        {
                            _memoryChunk = FromOngoingFile(_filename, _chunkManager, _chunkConfig, readRecordFunc, true);
                        }
                        catch (OutOfMemoryException)
                        {
                            _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                        }
                        catch (Exception ex)
                        {
                            _logger.Error(string.Format("Failed to cache ongoing chunk {0}", this), ex);
                            _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                        }
                    }
                    else
                    {
                        _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                    }
                }
                else
                {
                    _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                }
            }

            _lastActiveTime = DateTime.Now;

            if (!_isMemoryChunk)
            {
                _logger.InfoFormat("Ongoing chunk {0} initialized, _dataPosition: {1}", this, _dataPosition);
            }
        }
Example #4
0
        private int UncacheChunks(int maxUncacheCount = 10)
        {
            var uncachedCount = 0;

            if (Interlocked.CompareExchange(ref _uncachingChunks, 1, 0) == 0)
            {
                try
                {
                    var usedMemoryPercent = ChunkUtil.GetUsedMemoryPercent();
                    if (usedMemoryPercent <= (ulong)_config.ChunkCacheMinPercent)
                    {
                        return(0);
                    }

                    if (_logger.IsDebugEnabled)
                    {
                        _logger.DebugFormat("Current memory usage {0}% exceed the chunkCacheMinPercent {1}%, try to uncache chunks.", usedMemoryPercent, _config.ChunkCacheMinPercent);
                    }

                    var chunks = _chunks.Values.Where(x => x.IsCompleted && !x.IsMemoryChunk && x.HasCachedChunk).OrderBy(x => x.LastActiveTime).ToList();

                    foreach (var chunk in chunks)
                    {
                        if ((DateTime.Now - chunk.LastActiveTime).TotalSeconds >= _config.ChunkInactiveTimeMaxSeconds)
                        {
                            if (chunk.UnCacheFromMemory())
                            {
                                Thread.Sleep(100); //即便有内存释放了,由于通过API读取到的内存使用数可能不会立即更新,所以等待一定时间后检查内存是否足够
                                uncachedCount++;
                                if (uncachedCount >= maxUncacheCount || ChunkUtil.GetUsedMemoryPercent() <= (ulong)_config.ChunkCacheMinPercent)
                                {
                                    break;
                                }
                            }
                        }
                    }

                    if (_logger.IsDebugEnabled)
                    {
                        if (uncachedCount > 0)
                        {
                            _logger.DebugFormat("Uncached {0} chunks, current memory usage: {1}%", uncachedCount, ChunkUtil.GetUsedMemoryPercent());
                        }
                        else
                        {
                            _logger.Debug("No chunks uncached.");
                        }
                    }
                }
                catch (Exception ex)
                {
                    _logger.Error("Uncaching chunks has exception.", ex);
                }
                finally
                {
                    Interlocked.Exchange(ref _uncachingChunks, 0);
                }
            }

            return(uncachedCount);
        }