static void SortLevel2(string outputPath, string separator) { var sort1Path = outputPath + "\\" + "sort1"; var sort2Path = outputPath + "\\" + "sort2"; var textFiles = Directory.GetFiles(sort1Path, "*.txt", SearchOption.TopDirectoryOnly); foreach (var textFile in textFiles) { var subdirname = Path.GetFileNameWithoutExtension(textFile); if (!Directory.Exists(sort2Path + "\\" + subdirname)) { Directory.CreateDirectory(sort2Path + "\\" + subdirname); } } int filecount = 0; foreach (var textFile in textFiles) { ++filecount; var subdirname = Path.GetFileNameWithoutExtension(textFile); var sort2SubPath = sort2Path + "\\" + subdirname; OutPutWriter ow = new OutPutWriter(sort2SubPath, 0); int cnt = 0; using (FileStream fs = new FileStream(textFile, FileMode.Open)) using (BufferedStream bfs = new BufferedStream(fs)) using (StreamReader sr = new StreamReader(bfs)) { while (!sr.EndOfStream) { ++cnt; Console.WriteLine(cnt.ToString() + ":" + filecount.ToString() + "\\" + textFiles.Length); var line = sr.ReadLine(); ow.AddLevel1(line, separator); } } ow.FlushLevel1(0); } }
static void SortCorpus(string inputPath, string outputPath) { var textFiles = Directory.GetFiles(inputPath, "*.*", SearchOption.AllDirectories).Where(x => x.ToLower().EndsWith(".txt")); var sort1Path = outputPath + "\\" + "sort1"; if (!Directory.Exists(sort1Path)) { Directory.CreateDirectory(sort1Path); } OutPutWriter ow = new OutPutWriter(sort1Path, 1000); foreach (var textFile in textFiles) { int i = 0; using (FileStream fs = new FileStream(textFile, FileMode.Open)) using (BufferedStream bfs = new BufferedStream(fs)) using (StreamReader sr = new StreamReader(bfs)) { while (!sr.EndOfStream) { ++i; ow.Add(sr.ReadLine()); if (i % 100 == 0) { //Console.Clear(); Console.WriteLine(i.ToString()); } } } } ow.Flush(0); }