public Engine(
ITasksQueue tasksQueue,
IGrouperIO io,
int usingBufferLength)
{
_io = io;
_tasksQueue = tasksQueue;
_usingBufferLength = usingBufferLength;
}
public Grouper(
IGroupsInfoMarger groupsInfoMarger,
IGrouperIOs grouperIOs,
ITasksQueue tasksQueue,
IConfig config)
{
_groupsInfoMarger = groupsInfoMarger;
_ios = grouperIOs;
_tasksQueue = tasksQueue;
_usingBufferLength = config.UsingBufferLength;
}
public GroupsLinesOutputFactory(
IIoService ioService,
ITasksQueue tasksQueue,
IBuffersPool buffersPool,
IConfig config)
{
_ioService = ioService;
_tasksQueue = tasksQueue;
_buffersPool = buffersPool;
_config = config;
}
public InputReaderFactory(
IIoService ioService,
ITasksQueue tasksQueue,
IBuffersPool buffersPool,
IConfig config)
{
_ioService = ioService;
_tasksQueue = tasksQueue;
_buffersPool = buffersPool;
_config = config;
}
public LinesWriter(
long fileOffset,
IBuffersPool buffersPool,
ITasksQueue tasksQueue,
IIoService ioService,
IConfig config)
{
_buffersPool = buffersPool;
_tasksQueue = tasksQueue;
_ioService = ioService;
_writingPosition = fileOffset;
_usingBufferLength = config.UsingBufferLength;
_groupsFilePath = config.GroupsFilePath;
_groupsStorage = new Group[Consts.MaxGroupsCount];
_writers = new ConcurrentBag <IFileWriter>();
}
public InputReader(
long readingOffset,
long readingLength,
IBuffersPool buffersPool,
ITasksQueue tasksQueue,
IIoService ioService,
IConfig config)
{
_buffersPool = buffersPool;
_tasksQueue = tasksQueue;
_readingOut = readingOffset + readingLength;
// -1 - for set "Buffer End" Symbol to last cell of buffer without data lose
var readingBufferLength = config.UsingBufferLength - 1;
using (var reader = ioService.OpenRead(config.InputFilePath))
{
var firstBufferHandle = _buffersPool.Get();
var length = (int)Math.Min(
readingBufferLength - Consts.EndLineBytesCount,
readingLength);
reader.Position = readingOffset;
length = reader.Read(firstBufferHandle.Value,
Consts.EndLineBytesCount,
length);
_firstBufferItem = new Item(
length + Consts.EndLineBytesCount,
firstBufferHandle);
readingOffset += length;
readingLength -= length;
}
_capacity = (int)Math.Min(
Math.Ceiling((double)readingLength / readingBufferLength),
InitCapacity);
_readed = new ConcurrentDictionary <int, Item>(
concurrencyLevel: config.MaxRunningTasksCount,
capacity: Math.Max(1, _capacity));
if (_capacity == 0)
{
_readed.TryAdd(0, new Item(0, Handle <byte[]> .Zero));
_readNext = new Action[] { () => _readed.TryAdd(0, new Item(0, Handle <byte[]> .Zero)) };
_readingIndex = Enumerable.Repeat(0, int.MaxValue).GetEnumerator();
return;
}
var readerStep = (_capacity - 1) * readingBufferLength;
_readingIndex = ReadingIndexes.GetEnumerator();
_readingIndex.MoveNext();
_readNext = new Action[_capacity];
for (int i = 0; i < _capacity; i++)
{
var reader = ioService.OpenRead(config.InputFilePath,
position: readingOffset + i * readingBufferLength);
int buffIndex = i;
Action read = delegate
{
var pooledBuff = _buffersPool.Get();
var length = (int)Math.Min(_readingOut - reader.Position, readingBufferLength);
length = reader.Read(pooledBuff.Value, 0, length);
_readed.TryAdd(buffIndex, new Item(length, pooledBuff));
};
_readNext[i] = delegate
{
read();
_readNext[buffIndex] = delegate
{
var position = reader.Position + readerStep;
if (position < _readingOut)
{
reader.Position = position;
read();
}
else
{
_readed.TryAdd(buffIndex, new Item(0, Handle <byte[]> .Zero));
reader.Dispose();
}
};
};
}
for (int i = 0; i < _readNext.Length; i++)
{
_tasksQueue.Enqueue(_readNext[i]);
}
}
