static int compareChunkInfo(ChunkInfo a, ChunkInfo b)
{
int cmp = compareItems(a.items, b.items);
if (cmp != 0)
{
return(cmp);
}
long aindex = LongFromBase64(a.items.Last());
long bindex = LongFromBase64(b.items.Last());
return(aindex <bindex ? -1 : aindex> bindex ? 1 : 0);
}
static void ReadFEC()
{
Compte cpt = new Compte("", "");
Func <StreamReader> GetSource;
GetSource = GetStreamFromFlatFile;
string filename = Path.Combine(OutputDir, Path.GetFileNameWithoutExtension(Source) + "~s" + Path.GetExtension(Source));
List <LineInfo> lines = new List <LineInfo>(3000000);
Directory.CreateDirectory(Path.Combine(OutputDir, "chunks"));
int chunkNum = 0;
using (StreamWriter sw = new StreamWriter(Path.Combine(OutputDir, "result.txt"), false, encISO))
{
// using (StreamReader sr = new StreamReader(Source, encISO))
using (StreamReader sr = GetSource())
{
string header = sr.ReadLine();
AnalyseHeader(header);
using (StreamWriter swResult = new StreamWriter(filename, false, encISO))
{
swResult.WriteLine(header + Separator[0] + "index");
}
evtSortDone.Set();
evtWriteDone.Set();
string line;
int chunkSize = 0;
row = 0;
while (null != (line = sr.ReadLine()))
{
string[] items = line.Split(Separator);
row++;
if (chunkSize + (line.Length + 2) >= MaxChunkSize)
{
++chunkNum;
SortChunk(chunkNum, lines);
lines = new List <LineInfo>(400000);
chunkSize = 0;
}
chunkSize += (line.Length + 2);
lines.Add(new LineInfo(items, row));
}
++chunkNum;
SortChunk(chunkNum, lines);
}
}
evtSortDone.WaitOne();
evtWriteDone.WaitOne();
// merge sorts
// open chunks
List <ChunkInfo> chunks = new List <ChunkInfo>(chunkNum);
for (int num = 1; num <= chunkNum; ++num)
{
ChunkInfo ci = new ChunkInfo();
ci.FileName = Path.Combine(OutputDir, "chunks", "c" + num + ".txt");
ci.sr = new StreamReader(ci.FileName, encISO);
chunks.Add(ci);
string line;
line = ci.sr.ReadLine();
ci.items = line.Split(Separator);
}
//
try
{
chunks.Sort(compareChunkInfo);
ChunkInfo current = chunks[0];
chunks.RemoveAt(0);
rows = row;
row = 0;
using (StreamWriter sw = new StreamWriter(filename, true, encISO))
{
string separator = "" + Separator[0];
while (current != null)
{
sw.WriteLine(string.Join(separator, current.items));
row++;
string line = current.sr.ReadLine();
if (line == null)
{
current.sr.Close();
File.Delete(current.FileName);
if (chunks.Count > 0)
{
current = chunks[0];
chunks.RemoveAt(0);
}
else
{
current = null;
}
}
else
{
current.items = line.Split(Separator);
if (chunks.Count > 0 && compareChunkInfo(current, chunks[0]) > 0)
{
++changes;
// insert chunk
chunks.Add(current);
current = chunks[0];
chunks.RemoveAt(0);
chunks.Sort(compareChunkInfo);
}
}
}
}
}
finally
{
Console.WriteLine();
Console.WriteLine("suppression des chunks");
foreach (var chunk in chunks)
{
Console.WriteLine("fermeture de chunk {0}", chunk.FileName);
chunk.sr.Close();
Console.WriteLine("fermeture de chunk {0}", chunk.FileName);
File.Delete(chunk.FileName);
}
}
//
}