/// <summary> /// The method sorts the source file and saves the result to the destination file. /// </summary> public void Sort(string sourceFile, string destinationFile) { // Validate arguments if (string.IsNullOrWhiteSpace(sourceFile)) { throw new ArgumentException("sourceFile cannot be empty."); } if (string.IsNullOrWhiteSpace(destinationFile)) { throw new ArgumentException("destinationFile cannot be empty."); } // Initialise paramets var fileWorker = new XFileWorker(destinationFile); var blockLimit = (int)(getAvailableMemory() * 0.3 / Environment.ProcessorCount); var blockCapacity = blockLimit / DefaultXRowLength; long currentblockSize = 0; XRow row = null; var rows = new List <XRow>(blockCapacity); // Read source file using (var file = File.OpenText(sourceFile)) { while ((row = file.ReadXRow()) != null) { currentblockSize += row.String.Length; rows.Add(row); // Split tha data on blocks if (currentblockSize > blockLimit) { // Sort block and save it to a temporary file fileWorker.SortAndSave(rows); // Clean lines buffer currentblockSize = 0; rows = new List <XRow>(blockCapacity); // Start merging the chanks to load CPU and I/O in parallel fileWorker.Merge(); } } } // Sort and save the last block if (rows.Count > 0) { fileWorker.SortAndSave(rows, true); } // Merge all blocks fileWorker.Merge(true); }
private void merge(string source1, string source2, string destination) { long s1 = 0; long s2 = 0; using (var f1 = File.OpenText(source1)) { XRow l1 = f1.ReadXRow(); s1 += l1.String.Length; using (var f2 = File.OpenText(source2)) { XRow l2 = f2.ReadXRow(); s2 += l2.String.Length; using (var d = new StreamWriter(destination, false, Encoding.Default)) { while (l1 != null || l2 != null) { if (l1 == null) { d.WriteLine(l2); l2 = f2.ReadXRow(); s2 += l2 != null ? l2.String.Length : 0; continue; } if (l2 == null) { d.WriteLine(l1); l1 = f1.ReadXRow(); s1 += l1 != null ? l1.String.Length : 0; continue; } if (l1.CompareTo(l2) <= 0) { d.WriteLine(l1); l1 = f1.ReadXRow(); s1 += l1 != null ? l1.String.Length : 0; continue; } d.WriteLine(l2); l2 = f2.ReadXRow(); s2 += l2 != null ? l2.String.Length : 0; } d.Close(); } } } var ex = false; using (var t1 = File.Open(source1, FileMode.Open)) { using (var t2 = File.Open(source2, FileMode.Open)) { using (var t3 = File.Open(destination, FileMode.Open)) { if (t3.Length != (t1.Length + t2.Length)) { ex = true; } } } } if (ex) { ex = false; } File.Delete(source1); File.Delete(source2); }