public static void read(wikistream wikimediastream, ref ConcurrentQueue<byte[]> pages, ref bool reader_done)
{
byte[] buffer = new byte[1000000];
PageProcessor pageproc = new PageProcessor();
while (true)
{
//read into buffer
int numbytes = 0;
switch (wikimediastream.type)
{
case "xml":
numbytes = wikimediastream.xmlstream.Read(buffer, 0, buffer.Length);
break;
case "gz":
numbytes = wikimediastream.gzipstream.Read(buffer, 0, buffer.Length);
break;
case "bz2":
numbytes = wikimediastream.bzip2stream.Read(buffer, 0, buffer.Length);
break;
}
if (numbytes <= 0)
{
break;
}
//now process buffer
pageproc.AppendBuffer(buffer, numbytes, ref pages);
//wait for other threads to catch up
while (pages.Count > 2000)
{
Thread.Sleep(100);
}
}
reader_done = true;
}