コード例 #1
0
ファイル: Program.cs プロジェクト: chaitradangat/CorpusSort
        static void SortLevel2(string outputPath, string separator)
        {
            var sort1Path = outputPath + "\\" + "sort1";

            var sort2Path = outputPath + "\\" + "sort2";

            var textFiles = Directory.GetFiles(sort1Path, "*.txt", SearchOption.TopDirectoryOnly);

            foreach (var textFile in textFiles)
            {
                var subdirname = Path.GetFileNameWithoutExtension(textFile);

                if (!Directory.Exists(sort2Path + "\\" + subdirname))
                {
                    Directory.CreateDirectory(sort2Path + "\\" + subdirname);
                }
            }

            int filecount = 0;

            foreach (var textFile in textFiles)
            {
                ++filecount;

                var subdirname = Path.GetFileNameWithoutExtension(textFile);

                var sort2SubPath = sort2Path + "\\" + subdirname;

                OutPutWriter ow = new OutPutWriter(sort2SubPath, 0);

                int cnt = 0;

                using (FileStream fs = new FileStream(textFile, FileMode.Open))
                    using (BufferedStream bfs = new BufferedStream(fs))
                        using (StreamReader sr = new StreamReader(bfs))
                        {
                            while (!sr.EndOfStream)
                            {
                                ++cnt;
                                Console.WriteLine(cnt.ToString() + ":" + filecount.ToString() + "\\" + textFiles.Length);

                                var line = sr.ReadLine();

                                ow.AddLevel1(line, separator);
                            }
                        }

                ow.FlushLevel1(0);
            }
        }
コード例 #2
0
ファイル: Program.cs プロジェクト: chaitradangat/CorpusSort
        static void SortCorpus(string inputPath, string outputPath)
        {
            var textFiles =
                Directory.GetFiles(inputPath, "*.*", SearchOption.AllDirectories).Where(x => x.ToLower().EndsWith(".txt"));

            var sort1Path = outputPath + "\\" + "sort1";

            if (!Directory.Exists(sort1Path))
            {
                Directory.CreateDirectory(sort1Path);
            }

            OutPutWriter ow = new OutPutWriter(sort1Path, 1000);

            foreach (var textFile in textFiles)
            {
                int i = 0;

                using (FileStream fs = new FileStream(textFile, FileMode.Open))
                    using (BufferedStream bfs = new BufferedStream(fs))
                        using (StreamReader sr = new StreamReader(bfs))
                        {
                            while (!sr.EndOfStream)
                            {
                                ++i;

                                ow.Add(sr.ReadLine());

                                if (i % 100 == 0)
                                {
                                    //Console.Clear();
                                    Console.WriteLine(i.ToString());
                                }
                            }
                        }
            }

            ow.Flush(0);
        }