static void SortLevel2(string outputPath, string separator)
{
var sort1Path = outputPath + "\\" + "sort1";
var sort2Path = outputPath + "\\" + "sort2";
var textFiles = Directory.GetFiles(sort1Path, "*.txt", SearchOption.TopDirectoryOnly);
foreach (var textFile in textFiles)
{
var subdirname = Path.GetFileNameWithoutExtension(textFile);
if (!Directory.Exists(sort2Path + "\\" + subdirname))
{
Directory.CreateDirectory(sort2Path + "\\" + subdirname);
}
}
int filecount = 0;
foreach (var textFile in textFiles)
{
++filecount;
var subdirname = Path.GetFileNameWithoutExtension(textFile);
var sort2SubPath = sort2Path + "\\" + subdirname;
OutPutWriter ow = new OutPutWriter(sort2SubPath, 0);
int cnt = 0;
using (FileStream fs = new FileStream(textFile, FileMode.Open))
using (BufferedStream bfs = new BufferedStream(fs))
using (StreamReader sr = new StreamReader(bfs))
{
while (!sr.EndOfStream)
{
++cnt;
Console.WriteLine(cnt.ToString() + ":" + filecount.ToString() + "\\" + textFiles.Length);
var line = sr.ReadLine();
ow.AddLevel1(line, separator);
}
}
ow.FlushLevel1(0);
}
}
static void SortCorpus(string inputPath, string outputPath)
{
var textFiles =
Directory.GetFiles(inputPath, "*.*", SearchOption.AllDirectories).Where(x => x.ToLower().EndsWith(".txt"));
var sort1Path = outputPath + "\\" + "sort1";
if (!Directory.Exists(sort1Path))
{
Directory.CreateDirectory(sort1Path);
}
OutPutWriter ow = new OutPutWriter(sort1Path, 1000);
foreach (var textFile in textFiles)
{
int i = 0;
using (FileStream fs = new FileStream(textFile, FileMode.Open))
using (BufferedStream bfs = new BufferedStream(fs))
using (StreamReader sr = new StreamReader(bfs))
{
while (!sr.EndOfStream)
{
++i;
ow.Add(sr.ReadLine());
if (i % 100 == 0)
{
//Console.Clear();
Console.WriteLine(i.ToString());
}
}
}
}
ow.Flush(0);
}