public RedisReceiver(IBuffersPool buffersPool, IAsyncSocket asyncSocket)
{
_buffersPool = buffersPool;
_asyncSocket = asyncSocket;
_socketArgs = new AsyncSocketEventArgs();
_lineBuffer = new StringBuilder();
_offset = 0;
}
public RedisReceiver(IBuffersPool buffersPool, IAsyncSocket asyncSocket)
{
_buffersPool = buffersPool;
_asyncSocket = asyncSocket;
_socketArgs = new AsyncSocketEventArgs();
_lineBuffer = new StringBuilder();
_offset = 0;
}
public GroupsLoaderFactory(
IBuffersPool buffersPool,
IIoService ioService,
IConfig config)
{
_buffersPool = buffersPool;
_ioService = ioService;
_config = config;
}
public GroupsLinesOutputFactory(
IIoService ioService,
ITasksQueue tasksQueue,
IBuffersPool buffersPool,
IConfig config)
{
_ioService = ioService;
_tasksQueue = tasksQueue;
_buffersPool = buffersPool;
_config = config;
}
public InputReaderFactory(
IIoService ioService,
ITasksQueue tasksQueue,
IBuffersPool buffersPool,
IConfig config)
{
_ioService = ioService;
_tasksQueue = tasksQueue;
_buffersPool = buffersPool;
_config = config;
}
public RedisSender(IBuffersPool buffersPool, IAsyncSocket asyncSocket, bool autoFlush)
{
_buffersPool = buffersPool;
_asyncSocket = asyncSocket;
_autoFlush = autoFlush;
_senderContext = new SenderContext();
_flushArgs = new AsyncSocketEventArgs();
_flushArgs.Completed = FlushCallBack;
_sendingQueue = new ConcurrentQueue <ArraySegment <byte> >();
}
public RedisSender(IBuffersPool buffersPool, IAsyncSocket asyncSocket, bool autoFlush)
{
_buffersPool = buffersPool;
_asyncSocket = asyncSocket;
_autoFlush = autoFlush;
_senderContext = new SenderContext();
_flushArgs = new AsyncSocketEventArgs();
_flushArgs.Completed = FlushCallBack;
_sendingQueue = new ConcurrentQueue<ArraySegment<byte>>();
}
public LinesWriter(
long fileOffset,
IBuffersPool buffersPool,
ITasksQueue tasksQueue,
IIoService ioService,
IConfig config)
{
_buffersPool = buffersPool;
_tasksQueue = tasksQueue;
_ioService = ioService;
_writingPosition = fileOffset;
_usingBufferLength = config.UsingBufferLength;
_groupsFilePath = config.GroupsFilePath;
_groupsStorage = new Group[Consts.MaxGroupsCount];
_writers = new ConcurrentBag <IFileWriter>();
}
public IoService(IBuffersPool buffersPool)
{
_buffersPool = buffersPool;
}
public InputReader(
long readingOffset,
long readingLength,
IBuffersPool buffersPool,
ITasksQueue tasksQueue,
IIoService ioService,
IConfig config)
{
_buffersPool = buffersPool;
_tasksQueue = tasksQueue;
_readingOut = readingOffset + readingLength;
// -1 - for set "Buffer End" Symbol to last cell of buffer without data lose
var readingBufferLength = config.UsingBufferLength - 1;
using (var reader = ioService.OpenRead(config.InputFilePath))
{
var firstBufferHandle = _buffersPool.Get();
var length = (int)Math.Min(
readingBufferLength - Consts.EndLineBytesCount,
readingLength);
reader.Position = readingOffset;
length = reader.Read(firstBufferHandle.Value,
Consts.EndLineBytesCount,
length);
_firstBufferItem = new Item(
length + Consts.EndLineBytesCount,
firstBufferHandle);
readingOffset += length;
readingLength -= length;
}
_capacity = (int)Math.Min(
Math.Ceiling((double)readingLength / readingBufferLength),
InitCapacity);
_readed = new ConcurrentDictionary <int, Item>(
concurrencyLevel: config.MaxRunningTasksCount,
capacity: Math.Max(1, _capacity));
if (_capacity == 0)
{
_readed.TryAdd(0, new Item(0, Handle <byte[]> .Zero));
_readNext = new Action[] { () => _readed.TryAdd(0, new Item(0, Handle <byte[]> .Zero)) };
_readingIndex = Enumerable.Repeat(0, int.MaxValue).GetEnumerator();
return;
}
var readerStep = (_capacity - 1) * readingBufferLength;
_readingIndex = ReadingIndexes.GetEnumerator();
_readingIndex.MoveNext();
_readNext = new Action[_capacity];
for (int i = 0; i < _capacity; i++)
{
var reader = ioService.OpenRead(config.InputFilePath,
position: readingOffset + i * readingBufferLength);
int buffIndex = i;
Action read = delegate
{
var pooledBuff = _buffersPool.Get();
var length = (int)Math.Min(_readingOut - reader.Position, readingBufferLength);
length = reader.Read(pooledBuff.Value, 0, length);
_readed.TryAdd(buffIndex, new Item(length, pooledBuff));
};
_readNext[i] = delegate
{
read();
_readNext[buffIndex] = delegate
{
var position = reader.Position + readerStep;
if (position < _readingOut)
{
reader.Position = position;
read();
}
else
{
_readed.TryAdd(buffIndex, new Item(0, Handle <byte[]> .Zero));
reader.Dispose();
}
};
};
}
for (int i = 0; i < _readNext.Length; i++)
{
_tasksQueue.Enqueue(_readNext[i]);
}
}
public GroupsLoader(
GroupInfo[] groupsInfo,
IGroup[] output,
IBuffersPool buffersPool,
IIoService ioService,
IConfig config)
{
_groupsInfo = groupsInfo;
_output = output;
_ioService = ioService;
_buffersPool = buffersPool;
int maxGroupBytesCount = 0, maxGroupLinesCount = 0;
for (int i = 0; i < Consts.MaxGroupsCount; i++)
{
var info = groupsInfo[i];
if (!GroupInfo.IsZero(info))
{
maxGroupBytesCount = Math.Max(maxGroupBytesCount, info.BytesCount);
maxGroupLinesCount = Math.Max(maxGroupLinesCount, info.LinesCount);
}
}
var memoryUsedForBuffers = (long)
_buffersPool.Count *
config.PhysicalBufferLength;
var maxGroupBuffersCount = (int)Math.Ceiling((double)
maxGroupBytesCount /
config.UsingBufferLength);
var maxGroupSize =
maxGroupBuffersCount *
config.PhysicalBufferLength;
var lineSize =
Marshal.SizeOf <LineIndexes>() +
sizeof(ulong);
var maxSizeForGroupLines =
lineSize *
maxGroupLinesCount;
var maxLoadedGroupsCount =
memoryUsedForBuffers /
(maxGroupSize + maxSizeForGroupLines);
var memoryForLines = (int)
maxLoadedGroupsCount *
maxSizeForGroupLines;
_reservedLinesCount =
memoryForLines /
lineSize;
var buffersCountForFree = (int)Math.Ceiling((double)
memoryForLines /
config.PhysicalBufferLength);
var buffersCount =
_buffersPool.Count -
buffersCountForFree;
var allBuffers = _buffersPool.ExtractAll();
Array.Resize(ref allBuffers, buffersCount);
_buffers = allBuffers;
_linesIndexes = new LineIndexes[_reservedLinesCount];
_sortingSegments = new ulong[_reservedLinesCount];
_groupsFilePath = config.GroupsFilePath;
_usingBufferLength = config.UsingBufferLength;
_readers = Enumerable
.Range(0, Consts.MaxRunningTasksCount)
.Select(_ => _ioService.OpenRead(_groupsFilePath))
.ToArray();
var tempBuffersHandles = Enumerable
.Range(0, Consts.MaxRunningTasksCount)
.Select(_ => _buffersPool.Get())
.ToArray();
_tempBuffers = tempBuffersHandles
.Select(o => o.Value)
.ToArray();
_dispose = Enumerable
.Concat <IDisposable>(_readers, tempBuffersHandles)
.Select(o => new Action(o.Dispose))
.Aggregate((a, b) => a + b);
}
