public void SingleGroupTest(
[ValueSource(nameof(Blocks_s))] LongRanges blocks,
[ValueSource(nameof(BufferSizes))] int usingBufferSize)
{
Assume.That(usingBufferSize >= blocks.Value.Max(o => o.Length));
const string groupsFilePath = "ZZZZzzzzZZZZzzzzZZZ";
const int buffersMemoryLimit = 1024 * 1024;
var configMock = new Mock <IConfig>();
var physicalBufferLength = usingBufferSize
+ Consts.BufferReadingEnsurance;
configMock
.SetupGet(o => o.PhysicalBufferLength)
.Returns(physicalBufferLength);
configMock
.SetupGet(o => o.UsingBufferLength)
.Returns(usingBufferSize);
configMock
.SetupGet(o => o.GroupsFilePath)
.Returns(groupsFilePath);
var bytesCount = blocks.Value.Sum(o => o.Length);
var linesCount = bytesCount / 4; // max
var buffersPool = new InfinityBuffersPool(
physicalBufferLength,
buffersMemoryLimit);
var lastBlock = blocks.Value[blocks.Value.Length - 1];
var inputSize = lastBlock.Offset + lastBlock.Length;
var input = new byte[inputSize];
var random = new Random();
foreach (var block in blocks.Value)
{
var blockContent = new byte[block.Length];
random.NextBytes(blockContent);
Array.Copy(blockContent, 0,
input, block.Offset,
block.Length);
}
var ioServiceMock = new Mock <IIoService>();
ioServiceMock
.Setup(o => o.OpenRead(groupsFilePath, 0L))
.Returns(() => new MemoryReader(new MemoryStream(input)));
IGroupsLoaderFactory loaderMaker =
new GroupsLoaderFactory(
buffersPool,
ioServiceMock.Object,
configMock.Object);
var groupIndex = random.Next(Consts.MaxGroupsCount);
var infos = new GroupInfo[Consts.MaxGroupsCount];
var groups = new IGroup[Consts.MaxGroupsCount];
infos[groupIndex] = new GroupInfo
{
BytesCount = bytesCount,
LinesCount = linesCount,
Mapping = blocks.Value
};
var loader = loaderMaker.Create(infos, groups);
var range = loader.LoadNextGroups();
Assert.AreEqual(0, range.Offset);
Assert.AreEqual(Consts.MaxGroupsCount, range.Length);
Assert.IsTrue(infos
.Where((_, i) => i != groupIndex)
.All(GroupInfo.IsZero));
Assert.IsTrue(groups
.Where((_, i) => i != groupIndex)
.All(o => o == null));
var actualGroup = groups[groupIndex];
Assert.IsNotNull(actualGroup);
Assert.AreEqual(0, actualGroup.Lines.Offset);
Assert.AreEqual(linesCount, actualGroup.Lines.Count);
Assert.AreEqual(0, actualGroup.SortingSegments.Offset);
Assert.AreEqual(linesCount, actualGroup.SortingSegments.Count);
Assert.GreaterOrEqual(actualGroup.Lines.Array.Length, linesCount);
Assert.GreaterOrEqual(actualGroup.SortingSegments.Array.Length, linesCount);
var actualGroupBytes = new byte[actualGroup.BytesCount];
var buffers = actualGroup.Buffers;
int j = 0;
for (; j < buffers.Count - 1; j++)
{
Array.Copy(buffers.Array[buffers.Offset + j], 0,
actualGroupBytes, j * usingBufferSize,
usingBufferSize);
}
Array.Copy(buffers.Array[buffers.Offset + j], 0,
actualGroupBytes, j * usingBufferSize,
actualGroup.BytesCount - j * usingBufferSize);
var expectedGroupBytes = blocks.Value
.Select(block => input.Skip((int)block.Offset)
.Take(block.Length))
.Aggregate(Enumerable.Concat)
.ToArray();
// Console.WriteLine(string.Join(" ", expectedGroupBytes));
// Console.WriteLine(string.Join(" ", actualGroupBytes));
CollectionAssert.AreEqual(
expectedGroupBytes,
actualGroupBytes);
}
public void Test(
string inputFileSettings,
string groupsFile,
int bufferSize,
int enginesCount,
int maxThreadsCount,
bool clear)
{
string inputFilePath = null;
try
{
if (inputFileSettings.StartsWith(UseExistanceFile))
{
inputFilePath = SplitString(inputFileSettings, ": ")[1];
}
else
{
var fileGenerationSettings = SplitString(inputFileSettings, " ");
Generator.Generate(sizeData: fileGenerationSettings[0],
lineSettings: fileGenerationSettings[1],
path: fileGenerationSettings[2]);
inputFilePath = fileGenerationSettings[2];
}
var configMock = new Mock <IConfig>();
var physicalBufferLength = bufferSize + 1;
configMock
.SetupGet(o => o.PhysicalBufferLength)
.Returns(physicalBufferLength);
configMock
.SetupGet(o => o.UsingBufferLength)
.Returns(bufferSize);
configMock
.SetupGet(o => o.MaxRunningTasksCount)
.Returns(maxThreadsCount);
configMock
.SetupGet(o => o.GrouperEnginesCount)
.Returns(enginesCount);
configMock
.SetupGet(o => o.InputFilePath)
.Returns(inputFilePath);
configMock
.SetupGet(o => o.GroupsFilePath)
.Returns(groupsFile);
IGroupsInfoMarger groupsSummaryInfoMarger =
new GroupsInfoMarger();
ITasksQueue tasksQueue =
new TasksQueue(configMock.Object);
IBuffersPool buffersPool =
new InfinityBuffersPool(physicalBufferLength);
IIoService ioService =
new IoService(
buffersPool);
IInputReaderFactory inputReaderMaker =
new InputReaderFactory(
ioService,
tasksQueue,
buffersPool,
configMock.Object);
IGroupsLinesOutputFactory linesWriterFactory =
new GroupsLinesOutputFactory(
ioService,
tasksQueue,
buffersPool,
configMock.Object);
IGrouperIOs grouperIOs =
new GrouperIOs(
inputReaderMaker,
linesWriterFactory,
ioService,
configMock.Object);
ILinesIndexesExtractor linesIndexesExtractor =
new LinesIndexesExtractor(
configMock.Object);
IGroupsLoaderFactory groupsLoaderMaker =
new GroupsLoaderFactory(
buffersPool,
ioService,
configMock.Object);
var grouper = new Grouper(
groupsSummaryInfoMarger,
grouperIOs,
tasksQueue,
configMock.Object);
var trivialGrouper = new TrivialGrouper();
var expectedGroups = trivialGrouper.SplitToGroups(
ReadAllLinesFrom(inputFilePath));
var groupsInfo = grouper.SeparateInputToGroups();
var output = new IGroup[Consts.MaxGroupsCount];
var loader = groupsLoaderMaker.Create(groupsInfo, output);
loader.LoadNextGroups();
var expectedGroupIds = expectedGroups
.Select(o => o.Id)
.ToArray();
var actualGroupIds = groupsInfo
.Select((group, id) => new { group, id })
.Where(o => !GroupInfo.IsZero(o.group))
.Select(o => o.id)
.ToArray();
#region DEBUG
// #if DEBUG
// var expectedGroupPrefixes = expectedGroupIds
// .Select(ToPrefix)
// .ToArray();
//
// var actualGroupPrefixes = actualGroupIds
// .Select(ToPrefix)
// .ToArray();
//
// var expectedGroupPrefixesInLine =
// string.Join(" | ", expectedGroupPrefixes);
//
// var actualGroupPrefixesInLine =
// string.Join(" | ", actualGroupPrefixes);
//
// var actualIdsOnly = actualGroupIds
// .Except(expectedGroupIds)
// .Select(ToPrefix);
//
// var actualIdsOnlyInLine =
// string.Join(" | ", actualIdsOnly);
//
// var expectedIdsOnly = expectedGroupIds
// .Except(actualGroupIds)
// .Select(ToPrefix);
//
// var allPrefixes =
// new[]
// {
// new[] {string.Empty},
//
// Enumerable.Range(' ', '~' - ' ' + 1)
// .Select(o => ((char) o).ToString()),
//
// Enumerable.Join(
// Enumerable.Range(' ', '~' - ' ' + 1),
// Enumerable.Range(' ', '~' - ' ' + 1),
// _ => true,
// _ => true,
// (c1, c2) => new string(new []{(char)c1, (char)c2}))
// }
// .Aggregate(Enumerable.Concat)
// .ToArray();
//
// var allCalculatedIds = allPrefixes
// .Select(ToId)
// .OrderBy(o => o)
// .ToArray();
//
// var allCalculatedIdsDistinct =
// allCalculatedIds.Distinct().ToArray();
//
// var allCalculatedPrefixes = Enumerable
// .Range(0, Consts.MaxGroupsCount)
// .Select(ToPrefix)
// .ToArray();
//
// var allCalculatedPrefixesDistinct =
// allCalculatedPrefixes.Distinct().ToArray();
// #endif
#endregion
CollectionAssert.AreEqual(
expectedGroupIds,
actualGroupIds);
int j = 0;
for (int i = 0; i < Consts.MaxGroupsCount; i++)
{
var info = groupsInfo[i];
if (GroupInfo.IsZero(info))
{
continue;
}
var expectedInfo = expectedGroups[j];
Assert.AreEqual(expectedInfo.BytesCount, info.BytesCount);
Assert.AreEqual(expectedInfo.LinesCount, info.LinesCount);
linesIndexesExtractor.ExtractIndexes(output[i]);
var expectedLines = expectedInfo.Lines
.Select(o => o.Content)
.ToArray();
foreach (var line in expectedLines)
{
line[0] = Consts.EndLineByte1;
}
var expectedLinesDictionary =
new Dictionary <HashedBytesArray, int>(info.LinesCount);
for (int k = 0; k < info.LinesCount; k++)
{
var hashedLine = Hash(expectedLines[k]);
if (expectedLinesDictionary.ContainsKey(hashedLine))
{
++expectedLinesDictionary[hashedLine];
}
else
{
expectedLinesDictionary.Add(hashedLine, 1);
}
}
#region DEBUG
// #if DEBUG
// var linesCountInDictionary = expectedLinesDictionary
// .Values.Sum(o => o);
// #endif
#endregion
var lines = output[i].Lines;
for (int k = 0; k < info.LinesCount; k++)
{
var lineIndexes = lines.Array[lines.Offset + k];
var lineLength = lineIndexes.LettersCount
+ lineIndexes.DigitsCount
+ 3;
var buffers = output[i].Buffers;
var bufferIndex = lineIndexes.Start / bufferSize;
var indexInBuffer = lineIndexes.Start % bufferSize;
var line = new byte[lineLength];
if (indexInBuffer + lineLength <= bufferSize)
{
Array.Copy(buffers.Array[buffers.Offset + bufferIndex], indexInBuffer,
line, 0,
lineLength);
}
else
{
var bufferRightLength = bufferSize - indexInBuffer;
Array.Copy(buffers.Array[buffers.Offset + bufferIndex], indexInBuffer,
line, 0,
bufferRightLength);
Array.Copy(buffers.Array[buffers.Offset + bufferIndex + 1], 0,
line, bufferRightLength,
lineLength - bufferRightLength);
}
var actualHashedLine = Hash(line);
Assert.IsTrue(expectedLinesDictionary.ContainsKey(actualHashedLine));
--expectedLinesDictionary[actualHashedLine];
if (expectedLinesDictionary[actualHashedLine] == 0)
{
expectedLinesDictionary.Remove(actualHashedLine);
}
}
Assert.AreEqual(0, expectedLinesDictionary.Count);
++j;
}
Assert.AreEqual(expectedGroups.Length, j);
loader.Dispose();
}
finally
{
if (clear)
{
if (!inputFileSettings.StartsWith(UseExistanceFile) &&
inputFilePath != null &&
File.Exists(inputFilePath))
{
File.Delete(inputFilePath);
}
if (File.Exists(groupsFile))
{
File.Delete(groupsFile);
}
}
}
}
public static void Sort(string input, string output)
{
IConfig config =
new Config(
input,
output);
IGroupsInfoMarger groupsInfoMarger =
new GroupsInfoMarger();
IBuffersPool buffersPool =
new BuffersPool(
config);
IIoService ioService =
new IoService(
buffersPool);
ITasksQueue tasksQueue =
new TasksQueue(
config);
IInputReaderFactory inputReaderFactory =
new InputReaderFactory(
ioService,
tasksQueue,
buffersPool,
config);
IGroupsLinesOutputFactory groupsLinessOutputFactory =
new GroupsLinesOutputFactory(
ioService,
tasksQueue,
buffersPool,
config);
IGrouperIOs grouperIOs =
new GrouperIOs(
inputReaderFactory,
groupsLinessOutputFactory,
ioService,
config);
IGrouper grouper =
new Grouper(
groupsInfoMarger,
grouperIOs,
tasksQueue,
config);
IGroupsLoaderFactory groupsLoaderFactory =
new GroupsLoaderFactory(
buffersPool,
ioService,
config);
ISortingSegmentsSupplier sortingSegmentsSupplier =
new SortingSegmentsSupplier(
config);
ILinesIndexesExtractor linesIndexesExtractor =
new LinesIndexesExtractor(
config);
IGroupSorter groupSorter =
new GroupSorter(
sortingSegmentsSupplier,
linesIndexesExtractor);
ISortedGroupWriterFactory sortedGroupWriterFactory =
new SortedGroupWriterFactory(
ioService,
config);
ISorter sorter =
new Sorter(
ioService,
grouper,
groupsLoaderFactory,
groupSorter,
sortedGroupWriterFactory,
config);
sorter.Sort();
}
