예제 #1
0
        private void MergePlain(List <string> filelist, string outputfilename, bool unswap)
        {
            int          buffersize = 50000;
            StreamWriter writer     = new StreamWriter(File.Open(outputfilename, FileMode.Create));

            StreamReader[]   srarr = new StreamReader[filelist.Count];
            Queue <string>[] q     = new Queue <string> [filelist.Count];

            for (int i = 0; i < filelist.Count; i++)
            {
                q[i] = new Queue <string>();
            }

            for (int i = 0; i < filelist.Count; i++)
            {
                srarr[i] = new StreamReader(filelist[i]);
            }

            for (int i = 0; i < filelist.Count; i++)
            {
                int qlength = 0;
                while (!srarr[i].EndOfStream && qlength < buffersize)
                {
                    string line = srarr[i].ReadLine();
                    q[i].Enqueue(line);
                    qlength++;
                }
            }

            var sd = new SortedDictionary <string, int>();

            for (int i = 0; i < q.Length; i++)
            {
                if (q[i].Count > 0)
                {
                    sd.AddAnyway(q[i].Dequeue(), i);
                }
            }

            if (sd.Any())
            {
                while (true)
                {
                    if (!sd.Any())
                    {
                        for (int i = 0; i < filelist.Count; i++)
                        {
                            int qlength = 0;
                            if (!q[i].Any())
                            {
                                while (!srarr[i].EndOfStream && qlength < buffersize)
                                {
                                    string line = srarr[i].ReadLine();
                                    q[i].Enqueue(line);
                                    qlength++;
                                }
                            }
                        }

                        for (int i = 0; i < q.Length; i++)
                        {
                            if (q[i].Count > 0)
                            {
                                sd.AddAnyway(q[i].Dequeue(), i);
                            }
                        }
                    }

                    if (!sd.Any())
                    {
                        break;
                    }

                    int    filenum = sd.First().Value;
                    string row     = sd.First().Key.TrimEnd('\t');
                    writer.WriteLine(unswap ? UnSwap(row) : row);
                    sd.Remove(sd.First().Key);
                    if (!q[filenum].Any())
                    {
                        int qlength = 0;
                        while (!srarr[filenum].EndOfStream && qlength < buffersize)
                        {
                            string line = srarr[filenum].ReadLine();
                            q[filenum].Enqueue(line);
                            qlength++;
                        }
                    }

                    if (q[filenum].Any())
                    {
                        string key = q[filenum].Dequeue();
                        sd.AddAnyway(key, filenum);
                    }
                }
            }

            for (int i = 0; i < filelist.Count; i++)
            {
                srarr[i].Dispose();
            }

            foreach (string filename in filelist)
            {
                if (File.Exists(filename))
                {
                    File.Delete(filename);
                }
            }

            writer.Dispose();
        }