private void _getFileInput()
{
var path = _fileInput.GetFileInput();
var extension = Path.GetExtension(path);
// Get input reader based on extension of given path
InputReaderFactory inputReaderFactory = new InputReaderFactory();
IInputReader inputReader = inputReaderFactory.GetInputReader(extension);
// Get node and edge list from input reader
inputReader.Read(path);
_nodes = inputReader.CreateNodeList();
_edges = inputReader.CreateEdgeList();
}
public void Test(
[ValueSource(nameof(Cases))] TestCase testCase,
[ValueSource(nameof(BufferSizes))] BufferSize bufferSize,
[ValueSource(nameof(EnginesCount))] int enginesCount,
[ValueSource(nameof(MaxThreadsCount))] int maxThreadsCount)
{
const string inputPath = "ZZZZZzzzzZzZZzzzZZZzzz",
groupsPath = "WWwwwWWwwwWWWwwwwwwwww";
var groupsFileLength = testCase.Lines
.Sum(o => o.Length + Consts.EndLineBytesCount);
byte[] groupsFileContent = new byte[groupsFileLength];
var inputSource = testCase.Lines
.SelectMany(line => new[] { line.Select(c => (byte)c), Consts.EndLineBytes })
.Aggregate(Enumerable.Concat)
.ToArray();
var ioServiceMock = new Mock <IIoService>();
var configMock = new Mock <IConfig>();
ioServiceMock
.Setup(o => o.SizeOfFile(inputPath))
.Returns(groupsFileLength);
ioServiceMock
.Setup(o => o.OpenRead(inputPath, It.IsAny <long>()))
.Returns((string _, long position) =>
{
var inputStream = new MemoryStream(inputSource);
var inputReaderMock = new Mock <IFileReader>();
inputReaderMock
.SetupGet(o => o.Length)
.Returns(() => inputStream.Length);
inputReaderMock
.SetupGet(o => o.Position)
.Returns(() => inputStream.Position);
inputReaderMock
.SetupSet(o => o.Position = It.IsAny <long>())
.Callback((long value) => inputStream.Position = value);
inputReaderMock
.Setup(o => o.ReadByte())
.Returns(() => inputStream.ReadByte());
inputReaderMock
.Setup(o => o.Read(It.IsAny <byte[]>(), It.IsAny <int>(), It.IsAny <int>()))
.Returns((byte[] buff, int offset, int count) =>
inputStream.Read(buff, offset, count));
inputReaderMock
.Setup(o => o.Dispose())
.Callback(() => inputStream.Dispose());
inputReaderMock.Object.Position = position;
return(inputReaderMock.Object);
});
ioServiceMock
.Setup(o => o.OpenWrite(groupsPath, It.IsAny <long>(), false))
.Returns((string _, long position, bool __) =>
{
var groupsStream = new MemoryStream(groupsFileContent);
var writerMock = new Mock <IFileWriter>();
writerMock
.SetupGet(o => o.Position)
.Returns(() => groupsStream.Position);
writerMock
.SetupSet(o => o.Position = It.IsAny <long>())
.Callback((long value) => groupsStream.Position = value);
writerMock
.SetupGet(o => o.Length)
.Returns(() => groupsStream.Length);
writerMock
.Setup(o => o.Write(It.IsAny <byte[]>(), It.IsAny <int>(), It.IsAny <int>()))
.Callback((byte[] buff, int offset, int count) =>
groupsStream.Write(buff, offset, count));
writerMock
.Setup(o => o.Dispose())
.Callback(() => groupsStream.Close());
return(writerMock.Object);
});
var maxLineLength = testCase.Lines
.Max(line => line.Length + Consts.EndLineBytesCount);
int buffSize = new Dictionary <BufferSize, int>
{
{ BufferSize.Min, maxLineLength + 1 },
{ BufferSize.Small, maxLineLength + 2 },
{ BufferSize.Medium, groupsFileLength + 1 },
{ BufferSize.Large, groupsFileLength * 2 }
}[bufferSize];
configMock
.SetupGet(o => o.PhysicalBufferLength)
.Returns(buffSize + sizeof(ulong));
configMock
.SetupGet(o => o.UsingBufferLength)
.Returns(buffSize);
configMock
.SetupGet(o => o.MaxRunningTasksCount)
.Returns(maxThreadsCount);
configMock
.SetupGet(o => o.GrouperEnginesCount)
.Returns(enginesCount);
configMock
.SetupGet(o => o.InputFilePath)
.Returns(inputPath);
configMock
.SetupGet(o => o.GroupsFilePath)
.Returns(groupsPath);
IGroupsInfoMarger groupsSummaryInfoMarger =
new GroupsInfoMarger();
ITasksQueue tasksQueue =
new TasksQueue(configMock.Object);
IBuffersPool buffersPool =
new BuffersPool(configMock.Object);
IInputReaderFactory inputReaderMaker =
new InputReaderFactory(
ioServiceMock.Object,
tasksQueue,
buffersPool,
configMock.Object);
IGroupsLinesOutputFactory linesWriterFactory =
new GroupsLinesOutputFactory(
ioServiceMock.Object,
tasksQueue,
buffersPool,
configMock.Object);
IGrouperIOs grouperIOs =
new GrouperIOs(
inputReaderMaker,
linesWriterFactory,
ioServiceMock.Object,
configMock.Object);
var grouper = new Grouper(
groupsSummaryInfoMarger,
grouperIOs,
tasksQueue,
configMock.Object);
var trivialGrouper = new TrivialGrouper();
var expectedGroups = trivialGrouper
.SplitToGroups(testCase.Lines);
var groupsInfo = grouper.SeparateInputToGroups();
var resultGroups = ExtractGroups(groupsInfo, groupsFileContent);
Assert.IsTrue(resultGroups.Select(Group.IsValid).All(o => o));
CollectionAssert.AreEqual(
expectedGroups,
resultGroups);
}
public void Test(
string inputFileSettings,
string groupsFile,
int bufferSize,
int enginesCount,
int maxThreadsCount,
bool clear)
{
string inputFilePath = null;
try
{
if (inputFileSettings.StartsWith(UseExistanceFile))
{
inputFilePath = SplitString(inputFileSettings, ": ")[1];
}
else
{
var fileGenerationSettings = SplitString(inputFileSettings, " ");
Generator.Generate(sizeData: fileGenerationSettings[0],
lineSettings: fileGenerationSettings[1],
path: fileGenerationSettings[2]);
inputFilePath = fileGenerationSettings[2];
}
var configMock = new Mock <IConfig>();
var physicalBufferLength = bufferSize + 1;
configMock
.SetupGet(o => o.PhysicalBufferLength)
.Returns(physicalBufferLength);
configMock
.SetupGet(o => o.UsingBufferLength)
.Returns(bufferSize);
configMock
.SetupGet(o => o.MaxRunningTasksCount)
.Returns(maxThreadsCount);
configMock
.SetupGet(o => o.GrouperEnginesCount)
.Returns(enginesCount);
configMock
.SetupGet(o => o.InputFilePath)
.Returns(inputFilePath);
configMock
.SetupGet(o => o.GroupsFilePath)
.Returns(groupsFile);
IGroupsInfoMarger groupsSummaryInfoMarger =
new GroupsInfoMarger();
ITasksQueue tasksQueue =
new TasksQueue(configMock.Object);
IBuffersPool buffersPool =
new InfinityBuffersPool(physicalBufferLength);
IIoService ioService =
new IoService(
buffersPool);
IInputReaderFactory inputReaderMaker =
new InputReaderFactory(
ioService,
tasksQueue,
buffersPool,
configMock.Object);
IGroupsLinesOutputFactory linesWriterFactory =
new GroupsLinesOutputFactory(
ioService,
tasksQueue,
buffersPool,
configMock.Object);
IGrouperIOs grouperIOs =
new GrouperIOs(
inputReaderMaker,
linesWriterFactory,
ioService,
configMock.Object);
ILinesIndexesExtractor linesIndexesExtractor =
new LinesIndexesExtractor(
configMock.Object);
IGroupsLoaderFactory groupsLoaderMaker =
new GroupsLoaderFactory(
buffersPool,
ioService,
configMock.Object);
var grouper = new Grouper(
groupsSummaryInfoMarger,
grouperIOs,
tasksQueue,
configMock.Object);
var trivialGrouper = new TrivialGrouper();
var expectedGroups = trivialGrouper.SplitToGroups(
ReadAllLinesFrom(inputFilePath));
var groupsInfo = grouper.SeparateInputToGroups();
var output = new IGroup[Consts.MaxGroupsCount];
var loader = groupsLoaderMaker.Create(groupsInfo, output);
loader.LoadNextGroups();
var expectedGroupIds = expectedGroups
.Select(o => o.Id)
.ToArray();
var actualGroupIds = groupsInfo
.Select((group, id) => new { group, id })
.Where(o => !GroupInfo.IsZero(o.group))
.Select(o => o.id)
.ToArray();
#region DEBUG
// #if DEBUG
// var expectedGroupPrefixes = expectedGroupIds
// .Select(ToPrefix)
// .ToArray();
//
// var actualGroupPrefixes = actualGroupIds
// .Select(ToPrefix)
// .ToArray();
//
// var expectedGroupPrefixesInLine =
// string.Join(" | ", expectedGroupPrefixes);
//
// var actualGroupPrefixesInLine =
// string.Join(" | ", actualGroupPrefixes);
//
// var actualIdsOnly = actualGroupIds
// .Except(expectedGroupIds)
// .Select(ToPrefix);
//
// var actualIdsOnlyInLine =
// string.Join(" | ", actualIdsOnly);
//
// var expectedIdsOnly = expectedGroupIds
// .Except(actualGroupIds)
// .Select(ToPrefix);
//
// var allPrefixes =
// new[]
// {
// new[] {string.Empty},
//
// Enumerable.Range(' ', '~' - ' ' + 1)
// .Select(o => ((char) o).ToString()),
//
// Enumerable.Join(
// Enumerable.Range(' ', '~' - ' ' + 1),
// Enumerable.Range(' ', '~' - ' ' + 1),
// _ => true,
// _ => true,
// (c1, c2) => new string(new []{(char)c1, (char)c2}))
// }
// .Aggregate(Enumerable.Concat)
// .ToArray();
//
// var allCalculatedIds = allPrefixes
// .Select(ToId)
// .OrderBy(o => o)
// .ToArray();
//
// var allCalculatedIdsDistinct =
// allCalculatedIds.Distinct().ToArray();
//
// var allCalculatedPrefixes = Enumerable
// .Range(0, Consts.MaxGroupsCount)
// .Select(ToPrefix)
// .ToArray();
//
// var allCalculatedPrefixesDistinct =
// allCalculatedPrefixes.Distinct().ToArray();
// #endif
#endregion
CollectionAssert.AreEqual(
expectedGroupIds,
actualGroupIds);
int j = 0;
for (int i = 0; i < Consts.MaxGroupsCount; i++)
{
var info = groupsInfo[i];
if (GroupInfo.IsZero(info))
{
continue;
}
var expectedInfo = expectedGroups[j];
Assert.AreEqual(expectedInfo.BytesCount, info.BytesCount);
Assert.AreEqual(expectedInfo.LinesCount, info.LinesCount);
linesIndexesExtractor.ExtractIndexes(output[i]);
var expectedLines = expectedInfo.Lines
.Select(o => o.Content)
.ToArray();
foreach (var line in expectedLines)
{
line[0] = Consts.EndLineByte1;
}
var expectedLinesDictionary =
new Dictionary <HashedBytesArray, int>(info.LinesCount);
for (int k = 0; k < info.LinesCount; k++)
{
var hashedLine = Hash(expectedLines[k]);
if (expectedLinesDictionary.ContainsKey(hashedLine))
{
++expectedLinesDictionary[hashedLine];
}
else
{
expectedLinesDictionary.Add(hashedLine, 1);
}
}
#region DEBUG
// #if DEBUG
// var linesCountInDictionary = expectedLinesDictionary
// .Values.Sum(o => o);
// #endif
#endregion
var lines = output[i].Lines;
for (int k = 0; k < info.LinesCount; k++)
{
var lineIndexes = lines.Array[lines.Offset + k];
var lineLength = lineIndexes.LettersCount
+ lineIndexes.DigitsCount
+ 3;
var buffers = output[i].Buffers;
var bufferIndex = lineIndexes.Start / bufferSize;
var indexInBuffer = lineIndexes.Start % bufferSize;
var line = new byte[lineLength];
if (indexInBuffer + lineLength <= bufferSize)
{
Array.Copy(buffers.Array[buffers.Offset + bufferIndex], indexInBuffer,
line, 0,
lineLength);
}
else
{
var bufferRightLength = bufferSize - indexInBuffer;
Array.Copy(buffers.Array[buffers.Offset + bufferIndex], indexInBuffer,
line, 0,
bufferRightLength);
Array.Copy(buffers.Array[buffers.Offset + bufferIndex + 1], 0,
line, bufferRightLength,
lineLength - bufferRightLength);
}
var actualHashedLine = Hash(line);
Assert.IsTrue(expectedLinesDictionary.ContainsKey(actualHashedLine));
--expectedLinesDictionary[actualHashedLine];
if (expectedLinesDictionary[actualHashedLine] == 0)
{
expectedLinesDictionary.Remove(actualHashedLine);
}
}
Assert.AreEqual(0, expectedLinesDictionary.Count);
++j;
}
Assert.AreEqual(expectedGroups.Length, j);
loader.Dispose();
}
finally
{
if (clear)
{
if (!inputFileSettings.StartsWith(UseExistanceFile) &&
inputFilePath != null &&
File.Exists(inputFilePath))
{
File.Delete(inputFilePath);
}
if (File.Exists(groupsFile))
{
File.Delete(groupsFile);
}
}
}
}
public static void Sort(string input, string output)
{
IConfig config =
new Config(
input,
output);
IGroupsInfoMarger groupsInfoMarger =
new GroupsInfoMarger();
IBuffersPool buffersPool =
new BuffersPool(
config);
IIoService ioService =
new IoService(
buffersPool);
ITasksQueue tasksQueue =
new TasksQueue(
config);
IInputReaderFactory inputReaderFactory =
new InputReaderFactory(
ioService,
tasksQueue,
buffersPool,
config);
IGroupsLinesOutputFactory groupsLinessOutputFactory =
new GroupsLinesOutputFactory(
ioService,
tasksQueue,
buffersPool,
config);
IGrouperIOs grouperIOs =
new GrouperIOs(
inputReaderFactory,
groupsLinessOutputFactory,
ioService,
config);
IGrouper grouper =
new Grouper(
groupsInfoMarger,
grouperIOs,
tasksQueue,
config);
IGroupsLoaderFactory groupsLoaderFactory =
new GroupsLoaderFactory(
buffersPool,
ioService,
config);
ISortingSegmentsSupplier sortingSegmentsSupplier =
new SortingSegmentsSupplier(
config);
ILinesIndexesExtractor linesIndexesExtractor =
new LinesIndexesExtractor(
config);
IGroupSorter groupSorter =
new GroupSorter(
sortingSegmentsSupplier,
linesIndexesExtractor);
ISortedGroupWriterFactory sortedGroupWriterFactory =
new SortedGroupWriterFactory(
ioService,
config);
ISorter sorter =
new Sorter(
ioService,
grouper,
groupsLoaderFactory,
groupSorter,
sortedGroupWriterFactory,
config);
sorter.Sort();
}
