Example #1
0
        public void Close()
        {
            lock (_writeSyncObj)
            {
                if (!_isCompleted)
                {
                    Flush();
                }

                if (_writerWorkItem != null)
                {
                    ActionHelper.EatException(() => _writerWorkItem.Dispose());
                    _writerWorkItem = null;
                }

                if (!_isMemoryChunk)
                {
                    if (_cacheItems != null)
                    {
                        _cacheItems = null;
                    }
                }
                CloseAllReaderWorkItems();
                FreeMemory();
            }
        }
Example #2
0
        public void Complete()
        {
            lock (_writeSyncObj)
            {
                if (_isCompleted)
                {
                    return;
                }

                _chunkFooter = WriteFooter();
                if (!_isMemoryChunk)
                {
                    Flush();
                }

                _isCompleted = true;

                if (_writerWorkItem != null)
                {
                    ActionHelper.EatException(() => _writerWorkItem.Dispose());
                    _writerWorkItem = null;
                }

                if (!_isMemoryChunk)
                {
                    if (_cacheItems != null)
                    {
                        _cacheItems = null;
                    }

                    SetFileAttributes();
                    if (_memoryChunk != null)
                    {
                        _memoryChunk.Complete();
                    }
                }
            }
        }
Example #3
0
        private void InitOngoing <T>(Func <byte[], T> readRecordFunc) where T : ILogRecord
        {
            var fileInfo = new FileInfo(_filename);

            if (!fileInfo.Exists)
            {
                throw new ChunkFileNotExistException(_filename);
            }

            _isCompleted = false;

            if (!TryParsingDataPosition(readRecordFunc, out _chunkHeader, out _dataPosition))
            {
                throw new ChunkBadDataException(string.Format("Failed to parse chunk data, chunk file: {0}", _filename));
            }

            _flushedDataPosition = _dataPosition;

            var writeStream = default(Stream);

            if (_isMemoryChunk)
            {
                var fileSize = ChunkHeader.Size + _chunkHeader.ChunkDataTotalSize + ChunkFooter.Size;
                _cachedLength = fileSize;
                _cachedData   = Marshal.AllocHGlobal(_cachedLength);
                writeStream   = new UnmanagedMemoryStream((byte *)_cachedData, _cachedLength, _cachedLength, FileAccess.ReadWrite);

                writeStream.Write(_chunkHeader.AsByteArray(), 0, ChunkHeader.Size);

                if (_dataPosition > 0)
                {
                    using (var fileStream = new FileStream(_filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 8192, FileOptions.SequentialScan))
                    {
                        fileStream.Seek(ChunkHeader.Size, SeekOrigin.Begin);
                        var buffer      = new byte[65536];
                        int toReadBytes = _dataPosition;

                        while (toReadBytes > 0)
                        {
                            int read = fileStream.Read(buffer, 0, Math.Min(toReadBytes, buffer.Length));
                            if (read == 0)
                            {
                                break;
                            }
                            toReadBytes -= read;
                            writeStream.Write(buffer, 0, read);
                        }
                    }
                }

                if (writeStream.Position != GetStreamPosition(_dataPosition))
                {
                    throw new InvalidOperationException(string.Format("UnmanagedMemoryStream position incorrect, expect: {0}, but: {1}", _dataPosition + ChunkHeader.Size, writeStream.Position));
                }
            }
            else
            {
                var fileStream = new FileStream(_filename, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, _chunkConfig.ChunkWriteBuffer, FileOptions.SequentialScan);
                fileStream.Position = GetStreamPosition(_dataPosition);
                writeStream         = new BufferedStream(fileStream, _chunkConfig.ChunkWriteBuffer);
                SetFileAttributes();
            }

            _writerWorkItem = new WriterWorkItem(writeStream);

            InitializeReaderWorkItems();

            if (!_isMemoryChunk)
            {
                if (_chunkConfig.EnableCache)
                {
                    var chunkSize = (ulong)GetChunkSize(_chunkHeader);
                    if (ChunkUtil.IsMemoryEnoughToCacheChunk(chunkSize, (uint)_chunkConfig.ChunkCacheMaxPercent))
                    {
                        try
                        {
                            _memoryChunk = FromOngoingFile(_filename, _chunkManager, _chunkConfig, readRecordFunc, true);
                        }
                        catch (OutOfMemoryException)
                        {
                            _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                        }
                        catch (Exception ex)
                        {
                            _logger.Error(string.Format("Failed to cache ongoing chunk {0}", this), ex);
                            _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                        }
                    }
                    else
                    {
                        _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                    }
                }
                else
                {
                    _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                }
            }

            _lastActiveTime = DateTime.Now;

            if (!_isMemoryChunk)
            {
                _logger.InfoFormat("Ongoing chunk {0} initialized, _dataPosition: {1}", this, _dataPosition);
            }
        }
Example #4
0
        private void InitNew(int chunkNumber)
        {
            var chunkDataSize = 0;

            if (_chunkConfig.ChunkDataSize > 0)
            {
                chunkDataSize = _chunkConfig.ChunkDataSize;
            }
            else
            {
                chunkDataSize = _chunkConfig.ChunkDataUnitSize * _chunkConfig.ChunkDataCount;
            }

            _chunkHeader = new ChunkHeader(chunkNumber, chunkDataSize);

            _isCompleted = false;

            var fileSize = ChunkHeader.Size + _chunkHeader.ChunkDataTotalSize + ChunkFooter.Size;

            var writeStream    = default(Stream);
            var tempFilename   = string.Format("{0}.{1}.tmp", _filename, Guid.NewGuid());
            var tempFileStream = default(FileStream);

            try
            {
                if (_isMemoryChunk)
                {
                    _cachedLength = fileSize;
                    _cachedData   = Marshal.AllocHGlobal(_cachedLength);
                    writeStream   = new UnmanagedMemoryStream((byte *)_cachedData, _cachedLength, _cachedLength, FileAccess.ReadWrite);
                    writeStream.Write(_chunkHeader.AsByteArray(), 0, ChunkHeader.Size);
                }
                else
                {
                    var fileInfo = new FileInfo(_filename);
                    if (fileInfo.Exists)
                    {
                        File.SetAttributes(_filename, FileAttributes.Normal);
                        File.Delete(_filename);
                    }

                    tempFileStream = new FileStream(tempFilename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.Read, _chunkConfig.ChunkWriteBuffer, FileOptions.None);
                    tempFileStream.SetLength(fileSize);
                    tempFileStream.Write(_chunkHeader.AsByteArray(), 0, ChunkHeader.Size);
                    tempFileStream.Flush(true);
                    tempFileStream.Close();

                    File.Move(tempFilename, _filename);

                    var fileStream = new FileStream(_filename, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, _chunkConfig.ChunkWriteBuffer, FileOptions.SequentialScan);
                    writeStream = new BufferedStream(fileStream, _chunkConfig.ChunkWriteBuffer);
                    SetFileAttributes();
                }

                writeStream.Position = ChunkHeader.Size;

                _dataPosition        = 0;
                _flushedDataPosition = 0;
                _writerWorkItem      = new WriterWorkItem(writeStream);

                InitializeReaderWorkItems();

                if (!_isMemoryChunk)
                {
                    if (_chunkConfig.EnableCache)
                    {
                        var chunkSize = (ulong)GetChunkSize(_chunkHeader);
                        if (ChunkUtil.IsMemoryEnoughToCacheChunk(chunkSize, (uint)_chunkConfig.ChunkCacheMaxPercent))
                        {
                            try
                            {
                                _memoryChunk = CreateNew(_filename, chunkNumber, _chunkManager, _chunkConfig, true);
                            }
                            catch (OutOfMemoryException)
                            {
                                _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                            }
                            catch (Exception ex)
                            {
                                _logger.Error(string.Format("Failed to cache new chunk {0}", this), ex);
                                _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                            }
                        }
                        else
                        {
                            _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                        }
                    }
                    else
                    {
                        _cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
                    }
                }
            }
            catch
            {
                if (!_isMemoryChunk)
                {
                    if (tempFileStream != null)
                    {
                        ActionHelper.EatException(() => tempFileStream.Close());
                    }
                    if (File.Exists(tempFilename))
                    {
                        ActionHelper.EatException(() =>
                        {
                            File.SetAttributes(tempFilename, FileAttributes.Normal);
                            File.Delete(tempFilename);
                        });
                    }
                }
                throw;
            }

            _lastActiveTime = DateTime.Now;
        }