public static void read(wikistream wikimediastream, ref ConcurrentQueue<byte[]> pages, ref bool reader_done)
        {
            byte[] buffer = new byte[1000000];
            PageProcessor pageproc = new PageProcessor();

            while (true)
            {
                //read into buffer
                int numbytes = 0;
                switch (wikimediastream.type)
                {
                    case "xml":
                        numbytes = wikimediastream.xmlstream.Read(buffer, 0, buffer.Length);
                        break;
                    case "gz":
                        numbytes = wikimediastream.gzipstream.Read(buffer, 0, buffer.Length);
                        break;
                    case "bz2":
                        numbytes = wikimediastream.bzip2stream.Read(buffer, 0, buffer.Length);
                        break;
                }
                if (numbytes <= 0)
                {
                    break;
                }
                //now process buffer
                pageproc.AppendBuffer(buffer, numbytes, ref pages);

                //wait for other threads to catch up
                while (pages.Count > 2000)
                {
                    Thread.Sleep(100);
                }

            }
            reader_done = true;
        }