public bool TryCacheInMemory(bool shouldCacheNextChunk)
{
lock (_cacheSyncObj)
{
if (!_chunkConfig.EnableCache || _isMemoryChunk || !_isCompleted || _memoryChunk != null)
{
_cachingChunk = 0;
return(false);
}
try
{
var chunkSize = (ulong)GetChunkSize(_chunkHeader);
if (!ChunkUtil.IsMemoryEnoughToCacheChunk(chunkSize, (uint)_chunkConfig.ChunkCacheMaxPercent))
{
return(false);
}
_memoryChunk = FromCompletedFile(_filename, _chunkManager, _chunkConfig, true);
if (shouldCacheNextChunk)
{
Task.Factory.StartNew(() => _chunkManager.TryCacheNextChunk(this));
}
return(true);
}
catch (OutOfMemoryException) { return(false); }
catch (Exception ex)
{
_logger.Error(string.Format("Failed to cache completed chunk {0}", this), ex);
return(false);
}
finally
{
_cachingChunk = 0;
}
}
}
private void InitNew(int chunkNumber)
{
var chunkDataSize = 0;
if (_chunkConfig.ChunkDataSize > 0)
{
chunkDataSize = _chunkConfig.ChunkDataSize;
}
else
{
chunkDataSize = _chunkConfig.ChunkDataUnitSize * _chunkConfig.ChunkDataCount;
}
_chunkHeader = new ChunkHeader(chunkNumber, chunkDataSize);
_isCompleted = false;
var fileSize = ChunkHeader.Size + _chunkHeader.ChunkDataTotalSize + ChunkFooter.Size;
var writeStream = default(Stream);
var tempFilename = string.Format("{0}.{1}.tmp", _filename, Guid.NewGuid());
var tempFileStream = default(FileStream);
try
{
if (_isMemoryChunk)
{
_cachedLength = fileSize;
_cachedData = Marshal.AllocHGlobal(_cachedLength);
writeStream = new UnmanagedMemoryStream((byte *)_cachedData, _cachedLength, _cachedLength, FileAccess.ReadWrite);
writeStream.Write(_chunkHeader.AsByteArray(), 0, ChunkHeader.Size);
}
else
{
var fileInfo = new FileInfo(_filename);
if (fileInfo.Exists)
{
File.SetAttributes(_filename, FileAttributes.Normal);
File.Delete(_filename);
}
tempFileStream = new FileStream(tempFilename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.Read, _chunkConfig.ChunkWriteBuffer, FileOptions.None);
tempFileStream.SetLength(fileSize);
tempFileStream.Write(_chunkHeader.AsByteArray(), 0, ChunkHeader.Size);
tempFileStream.Flush(true);
tempFileStream.Close();
File.Move(tempFilename, _filename);
var fileStream = new FileStream(_filename, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, _chunkConfig.ChunkWriteBuffer, FileOptions.SequentialScan);
writeStream = new BufferedStream(fileStream, _chunkConfig.ChunkWriteBuffer);
SetFileAttributes();
}
writeStream.Position = ChunkHeader.Size;
_dataPosition = 0;
_flushedDataPosition = 0;
_writerWorkItem = new WriterWorkItem(writeStream);
InitializeReaderWorkItems();
if (!_isMemoryChunk)
{
if (_chunkConfig.EnableCache)
{
var chunkSize = (ulong)GetChunkSize(_chunkHeader);
if (ChunkUtil.IsMemoryEnoughToCacheChunk(chunkSize, (uint)_chunkConfig.ChunkCacheMaxPercent))
{
try
{
_memoryChunk = CreateNew(_filename, chunkNumber, _chunkManager, _chunkConfig, true);
}
catch (OutOfMemoryException)
{
_cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
}
catch (Exception ex)
{
_logger.Error(string.Format("Failed to cache new chunk {0}", this), ex);
_cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
}
}
else
{
_cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
}
}
else
{
_cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
}
}
}
catch
{
if (tempFileStream != null)
{
Helper.EatException(() => tempFileStream.Close());
}
if (File.Exists(tempFilename))
{
Helper.EatException(() =>
{
File.SetAttributes(tempFilename, FileAttributes.Normal);
File.Delete(tempFilename);
});
}
throw;
}
_lastActiveTime = DateTime.Now;
}
private void InitOngoing <T>(Func <byte[], T> readRecordFunc) where T : ILogRecord
{
var fileInfo = new FileInfo(_filename);
if (!fileInfo.Exists)
{
throw new ChunkFileNotExistException(_filename);
}
_isCompleted = false;
if (!TryParsingDataPosition(readRecordFunc, out _chunkHeader, out _dataPosition))
{
throw new ChunkBadDataException(string.Format("Failed to parse chunk data, chunk file: {0}", _filename));
}
_flushedDataPosition = _dataPosition;
var writeStream = default(Stream);
if (_isMemoryChunk)
{
var fileSize = ChunkHeader.Size + _chunkHeader.ChunkDataTotalSize + ChunkFooter.Size;
_cachedLength = fileSize;
_cachedData = Marshal.AllocHGlobal(_cachedLength);
writeStream = new UnmanagedMemoryStream((byte *)_cachedData, _cachedLength, _cachedLength, FileAccess.ReadWrite);
writeStream.Write(_chunkHeader.AsByteArray(), 0, ChunkHeader.Size);
if (_dataPosition > 0)
{
using (var fileStream = new FileStream(_filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 8192, FileOptions.None))
{
fileStream.Seek(ChunkHeader.Size, SeekOrigin.Begin);
var buffer = new byte[65536];
int toReadBytes = _dataPosition;
while (toReadBytes > 0)
{
int read = fileStream.Read(buffer, 0, Math.Min(toReadBytes, buffer.Length));
if (read == 0)
{
break;
}
toReadBytes -= read;
writeStream.Write(buffer, 0, read);
}
}
}
if (writeStream.Position != GetStreamPosition(_dataPosition))
{
throw new InvalidOperationException(string.Format("UnmanagedMemoryStream position incorrect, expect: {0}, but: {1}", _dataPosition + ChunkHeader.Size, writeStream.Position));
}
}
else
{
var fileStream = new FileStream(_filename, FileMode.Open, FileAccess.ReadWrite, FileShare.Read, _chunkConfig.ChunkWriteBuffer, FileOptions.SequentialScan);
fileStream.Position = GetStreamPosition(_dataPosition);
writeStream = new BufferedStream(fileStream, _chunkConfig.ChunkWriteBuffer);
SetFileAttributes();
}
_writerWorkItem = new WriterWorkItem(writeStream);
InitializeReaderWorkItems();
if (!_isMemoryChunk)
{
if (_chunkConfig.EnableCache)
{
var chunkSize = (ulong)GetChunkSize(_chunkHeader);
if (ChunkUtil.IsMemoryEnoughToCacheChunk(chunkSize, (uint)_chunkConfig.ChunkCacheMaxPercent))
{
try
{
_memoryChunk = Chunk.FromOngoingFile <T>(_filename, _chunkManager, _chunkConfig, readRecordFunc, true);
}
catch (OutOfMemoryException)
{
_cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
}
catch (Exception ex)
{
_logger.Error(string.Format("Failed to cache ongoing chunk {0}", this), ex);
_cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
}
}
else
{
_cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
}
}
else
{
_cacheItems = new CacheItem[_chunkConfig.ChunkLocalCacheSize];
}
}
_lastActiveTime = DateTime.Now;
if (!_isMemoryChunk)
{
_logger.InfoFormat("Ongoing chunk {0} initialized, _dataPosition: {1}", this, _dataPosition);
}
}
private int UncacheChunks(int maxUncacheCount = 10)
{
var uncachedCount = 0;
if (Interlocked.CompareExchange(ref _uncachingChunks, 1, 0) == 0)
{
try
{
var usedMemoryPercent = ChunkUtil.GetUsedMemoryPercent();
if (usedMemoryPercent <= (ulong)_config.ChunkCacheMinPercent)
{
return(0);
}
if (_logger.IsDebugEnabled)
{
_logger.DebugFormat("Current memory usage {0}% exceed the chunkCacheMinPercent {1}%, try to uncache chunks.", usedMemoryPercent, _config.ChunkCacheMinPercent);
}
var chunks = _chunks.Values.Where(x => x.IsCompleted && !x.IsMemoryChunk && x.HasCachedChunk).OrderBy(x => x.LastActiveTime).ToList();
foreach (var chunk in chunks)
{
if ((DateTime.Now - chunk.LastActiveTime).TotalSeconds >= _config.ChunkInactiveTimeMaxSeconds)
{
if (chunk.UnCacheFromMemory())
{
Thread.Sleep(100); //即便有内存释放了，由于通过API读取到的内存使用数可能不会立即更新，所以等待一定时间后检查内存是否足够
uncachedCount++;
if (uncachedCount >= maxUncacheCount || ChunkUtil.GetUsedMemoryPercent() <= (ulong)_config.ChunkCacheMinPercent)
{
break;
}
}
}
}
if (_logger.IsDebugEnabled)
{
if (uncachedCount > 0)
{
_logger.DebugFormat("Uncached {0} chunks, current memory usage: {1}%", uncachedCount, ChunkUtil.GetUsedMemoryPercent());
}
else
{
_logger.Debug("No chunks uncached.");
}
}
}
catch (Exception ex)
{
_logger.Error("Uncaching chunks has exception.", ex);
}
finally
{
Interlocked.Exchange(ref _uncachingChunks, 0);
}
}
return(uncachedCount);
}