public void Test( string inputFileSettings, string groupsFile, int bufferSize, int enginesCount, int maxThreadsCount, bool clear) { string inputFilePath = null; try { if (inputFileSettings.StartsWith(UseExistanceFile)) { inputFilePath = SplitString(inputFileSettings, ": ")[1]; } else { var fileGenerationSettings = SplitString(inputFileSettings, " "); Generator.Generate(sizeData: fileGenerationSettings[0], lineSettings: fileGenerationSettings[1], path: fileGenerationSettings[2]); inputFilePath = fileGenerationSettings[2]; } var configMock = new Mock <IConfig>(); var physicalBufferLength = bufferSize + 1; configMock .SetupGet(o => o.PhysicalBufferLength) .Returns(physicalBufferLength); configMock .SetupGet(o => o.UsingBufferLength) .Returns(bufferSize); configMock .SetupGet(o => o.MaxRunningTasksCount) .Returns(maxThreadsCount); configMock .SetupGet(o => o.GrouperEnginesCount) .Returns(enginesCount); configMock .SetupGet(o => o.InputFilePath) .Returns(inputFilePath); configMock .SetupGet(o => o.GroupsFilePath) .Returns(groupsFile); IGroupsInfoMarger groupsSummaryInfoMarger = new GroupsInfoMarger(); ITasksQueue tasksQueue = new TasksQueue(configMock.Object); IBuffersPool buffersPool = new InfinityBuffersPool(physicalBufferLength); IIoService ioService = new IoService( buffersPool); IInputReaderFactory inputReaderMaker = new InputReaderFactory( ioService, tasksQueue, buffersPool, configMock.Object); IGroupsLinesOutputFactory linesWriterFactory = new GroupsLinesOutputFactory( ioService, tasksQueue, buffersPool, configMock.Object); IGrouperIOs grouperIOs = new GrouperIOs( inputReaderMaker, linesWriterFactory, ioService, configMock.Object); ILinesIndexesExtractor linesIndexesExtractor = new LinesIndexesExtractor( configMock.Object); IGroupsLoaderFactory groupsLoaderMaker = new GroupsLoaderFactory( buffersPool, ioService, configMock.Object); var grouper = new Grouper( groupsSummaryInfoMarger, grouperIOs, tasksQueue, configMock.Object); var trivialGrouper = new TrivialGrouper(); var expectedGroups = trivialGrouper.SplitToGroups( ReadAllLinesFrom(inputFilePath)); var groupsInfo = grouper.SeparateInputToGroups(); var output = new IGroup[Consts.MaxGroupsCount]; var loader = groupsLoaderMaker.Create(groupsInfo, output); loader.LoadNextGroups(); var expectedGroupIds = expectedGroups .Select(o => o.Id) .ToArray(); var actualGroupIds = groupsInfo .Select((group, id) => new { group, id }) .Where(o => !GroupInfo.IsZero(o.group)) .Select(o => o.id) .ToArray(); #region DEBUG // #if DEBUG // var expectedGroupPrefixes = expectedGroupIds // .Select(ToPrefix) // .ToArray(); // // var actualGroupPrefixes = actualGroupIds // .Select(ToPrefix) // .ToArray(); // // var expectedGroupPrefixesInLine = // string.Join(" | ", expectedGroupPrefixes); // // var actualGroupPrefixesInLine = // string.Join(" | ", actualGroupPrefixes); // // var actualIdsOnly = actualGroupIds // .Except(expectedGroupIds) // .Select(ToPrefix); // // var actualIdsOnlyInLine = // string.Join(" | ", actualIdsOnly); // // var expectedIdsOnly = expectedGroupIds // .Except(actualGroupIds) // .Select(ToPrefix); // // var allPrefixes = // new[] // { // new[] {string.Empty}, // // Enumerable.Range(' ', '~' - ' ' + 1) // .Select(o => ((char) o).ToString()), // // Enumerable.Join( // Enumerable.Range(' ', '~' - ' ' + 1), // Enumerable.Range(' ', '~' - ' ' + 1), // _ => true, // _ => true, // (c1, c2) => new string(new []{(char)c1, (char)c2})) // } // .Aggregate(Enumerable.Concat) // .ToArray(); // // var allCalculatedIds = allPrefixes // .Select(ToId) // .OrderBy(o => o) // .ToArray(); // // var allCalculatedIdsDistinct = // allCalculatedIds.Distinct().ToArray(); // // var allCalculatedPrefixes = Enumerable // .Range(0, Consts.MaxGroupsCount) // .Select(ToPrefix) // .ToArray(); // // var allCalculatedPrefixesDistinct = // allCalculatedPrefixes.Distinct().ToArray(); // #endif #endregion CollectionAssert.AreEqual( expectedGroupIds, actualGroupIds); int j = 0; for (int i = 0; i < Consts.MaxGroupsCount; i++) { var info = groupsInfo[i]; if (GroupInfo.IsZero(info)) { continue; } var expectedInfo = expectedGroups[j]; Assert.AreEqual(expectedInfo.BytesCount, info.BytesCount); Assert.AreEqual(expectedInfo.LinesCount, info.LinesCount); linesIndexesExtractor.ExtractIndexes(output[i]); var expectedLines = expectedInfo.Lines .Select(o => o.Content) .ToArray(); foreach (var line in expectedLines) { line[0] = Consts.EndLineByte1; } var expectedLinesDictionary = new Dictionary <HashedBytesArray, int>(info.LinesCount); for (int k = 0; k < info.LinesCount; k++) { var hashedLine = Hash(expectedLines[k]); if (expectedLinesDictionary.ContainsKey(hashedLine)) { ++expectedLinesDictionary[hashedLine]; } else { expectedLinesDictionary.Add(hashedLine, 1); } } #region DEBUG // #if DEBUG // var linesCountInDictionary = expectedLinesDictionary // .Values.Sum(o => o); // #endif #endregion var lines = output[i].Lines; for (int k = 0; k < info.LinesCount; k++) { var lineIndexes = lines.Array[lines.Offset + k]; var lineLength = lineIndexes.LettersCount + lineIndexes.DigitsCount + 3; var buffers = output[i].Buffers; var bufferIndex = lineIndexes.Start / bufferSize; var indexInBuffer = lineIndexes.Start % bufferSize; var line = new byte[lineLength]; if (indexInBuffer + lineLength <= bufferSize) { Array.Copy(buffers.Array[buffers.Offset + bufferIndex], indexInBuffer, line, 0, lineLength); } else { var bufferRightLength = bufferSize - indexInBuffer; Array.Copy(buffers.Array[buffers.Offset + bufferIndex], indexInBuffer, line, 0, bufferRightLength); Array.Copy(buffers.Array[buffers.Offset + bufferIndex + 1], 0, line, bufferRightLength, lineLength - bufferRightLength); } var actualHashedLine = Hash(line); Assert.IsTrue(expectedLinesDictionary.ContainsKey(actualHashedLine)); --expectedLinesDictionary[actualHashedLine]; if (expectedLinesDictionary[actualHashedLine] == 0) { expectedLinesDictionary.Remove(actualHashedLine); } } Assert.AreEqual(0, expectedLinesDictionary.Count); ++j; } Assert.AreEqual(expectedGroups.Length, j); loader.Dispose(); } finally { if (clear) { if (!inputFileSettings.StartsWith(UseExistanceFile) && inputFilePath != null && File.Exists(inputFilePath)) { File.Delete(inputFilePath); } if (File.Exists(groupsFile)) { File.Delete(groupsFile); } } } }
public void SingleGroupTest( [ValueSource(nameof(Blocks_s))] LongRanges blocks, [ValueSource(nameof(BufferSizes))] int usingBufferSize) { Assume.That(usingBufferSize >= blocks.Value.Max(o => o.Length)); const string groupsFilePath = "ZZZZzzzzZZZZzzzzZZZ"; const int buffersMemoryLimit = 1024 * 1024; var configMock = new Mock <IConfig>(); var physicalBufferLength = usingBufferSize + Consts.BufferReadingEnsurance; configMock .SetupGet(o => o.PhysicalBufferLength) .Returns(physicalBufferLength); configMock .SetupGet(o => o.UsingBufferLength) .Returns(usingBufferSize); configMock .SetupGet(o => o.GroupsFilePath) .Returns(groupsFilePath); var bytesCount = blocks.Value.Sum(o => o.Length); var linesCount = bytesCount / 4; // max var buffersPool = new InfinityBuffersPool( physicalBufferLength, buffersMemoryLimit); var lastBlock = blocks.Value[blocks.Value.Length - 1]; var inputSize = lastBlock.Offset + lastBlock.Length; var input = new byte[inputSize]; var random = new Random(); foreach (var block in blocks.Value) { var blockContent = new byte[block.Length]; random.NextBytes(blockContent); Array.Copy(blockContent, 0, input, block.Offset, block.Length); } var ioServiceMock = new Mock <IIoService>(); ioServiceMock .Setup(o => o.OpenRead(groupsFilePath, 0L)) .Returns(() => new MemoryReader(new MemoryStream(input))); IGroupsLoaderFactory loaderMaker = new GroupsLoaderFactory( buffersPool, ioServiceMock.Object, configMock.Object); var groupIndex = random.Next(Consts.MaxGroupsCount); var infos = new GroupInfo[Consts.MaxGroupsCount]; var groups = new IGroup[Consts.MaxGroupsCount]; infos[groupIndex] = new GroupInfo { BytesCount = bytesCount, LinesCount = linesCount, Mapping = blocks.Value }; var loader = loaderMaker.Create(infos, groups); var range = loader.LoadNextGroups(); Assert.AreEqual(0, range.Offset); Assert.AreEqual(Consts.MaxGroupsCount, range.Length); Assert.IsTrue(infos .Where((_, i) => i != groupIndex) .All(GroupInfo.IsZero)); Assert.IsTrue(groups .Where((_, i) => i != groupIndex) .All(o => o == null)); var actualGroup = groups[groupIndex]; Assert.IsNotNull(actualGroup); Assert.AreEqual(0, actualGroup.Lines.Offset); Assert.AreEqual(linesCount, actualGroup.Lines.Count); Assert.AreEqual(0, actualGroup.SortingSegments.Offset); Assert.AreEqual(linesCount, actualGroup.SortingSegments.Count); Assert.GreaterOrEqual(actualGroup.Lines.Array.Length, linesCount); Assert.GreaterOrEqual(actualGroup.SortingSegments.Array.Length, linesCount); var actualGroupBytes = new byte[actualGroup.BytesCount]; var buffers = actualGroup.Buffers; int j = 0; for (; j < buffers.Count - 1; j++) { Array.Copy(buffers.Array[buffers.Offset + j], 0, actualGroupBytes, j * usingBufferSize, usingBufferSize); } Array.Copy(buffers.Array[buffers.Offset + j], 0, actualGroupBytes, j * usingBufferSize, actualGroup.BytesCount - j * usingBufferSize); var expectedGroupBytes = blocks.Value .Select(block => input.Skip((int)block.Offset) .Take(block.Length)) .Aggregate(Enumerable.Concat) .ToArray(); // Console.WriteLine(string.Join(" ", expectedGroupBytes)); // Console.WriteLine(string.Join(" ", actualGroupBytes)); CollectionAssert.AreEqual( expectedGroupBytes, actualGroupBytes); }