Beispiel #1
0
        private static void WriteDataTableToCSV()
        {
            #region WriteDataTableToCSV

            var table = new DataTable();
            table.Columns.Add("Name");
            table.Columns.Add("Age");
            table.Rows.Add("Kent", 33);
            table.Rows.Add("Belinda", 34);
            table.Rows.Add("Tempany", 8);
            table.Rows.Add("Xak", 0);

            using (var stringWriter = new StringWriter())
            {
                using (var writer = new CsvWriter(stringWriter))
                {
                    table.WriteCsv(writer);
                }

                Console.WriteLine("CSV: {0}", stringWriter);
            }

            #endregion
        }
        public void write_csv_returns_number_of_records_written()
        {
            using (var writer = new CsvWriter(new StringWriter()))
            {
                writer.WriteRecord("some", "record");

                var dataTable = new DataTable();
                dataTable.Columns.Add("First");
                dataTable.Columns.Add("Second");
                dataTable.Rows.Add("1", "2");
                dataTable.Rows.Add("3", "4");
                dataTable.Rows.Add("5", "6");

                Assert.Equal(3, dataTable.WriteCsv(writer, false));
                Assert.Equal(3, dataTable.WriteCsv(writer, true));
            }
        }
        public void write_csv_works_with_large_data_table_input()
        {
            var dataTable = new DataTable();
            dataTable.Columns.Add("First");
            dataTable.Columns.Add("Second");

            for (var i = 0; i < 1000; ++i)
            {
                dataTable.Rows.Add("value0_" + i, "value1_" + i);
            }

            // write less than all available records
            using (var writer = new CsvWriter(new StringWriter()))
            {
                Assert.Equal(913, dataTable.WriteCsv(writer, true, 913));
            }

            // write exactly available records
            using (var writer = new CsvWriter(new StringWriter()))
            {
                Assert.Equal(1000, dataTable.WriteCsv(writer, true, 1000));
            }

            // attempt to write more than available records
            using (var writer = new CsvWriter(new StringWriter()))
            {
                Assert.Equal(1000, dataTable.WriteCsv(writer, true, 1500));
            }
        }
        public void write_csv_uses_object_to_string_converter_to_convert_objects_in_data_row_to_string()
        {
            using (var stringWriter = new StringWriter())
            using (var writer = new CsvWriter(stringWriter))
            {
                writer.NewLine = "<EOL>";

                var dataTable = new DataTable();
                dataTable.Columns.Add("First");
                dataTable.Columns.Add("Second");
                dataTable.Rows.Add("1", 2);
                dataTable.Rows.Add(3, 4d);
                dataTable.Rows.Add(5m, 6f);

                Assert.Equal(3, dataTable.WriteCsv(writer, true, null, o => o.ToString() + "_SUFFIX"));
                Assert.Equal("First,Second<EOL>1_SUFFIX,2_SUFFIX<EOL>3_SUFFIX,4_SUFFIX<EOL>5_SUFFIX,6_SUFFIX<EOL>", stringWriter.ToString());
            }
        }
        public void write_csv_object_to_string_converter_converts_nulls_to_empty_string()
        {
            using (var stringWriter = new StringWriter())
            using (var writer = new CsvWriter(stringWriter))
            {
                writer.NewLine = "<EOL>";

                var dataTable = new DataTable();
                dataTable.Columns.Add("First");
                dataTable.Columns.Add("Second");
                dataTable.Rows.Add("1", 2);
                dataTable.Rows.Add(null, 4d);
                dataTable.Rows.Add(5m, null);

                Assert.Equal(3, dataTable.WriteCsv(writer));
                Assert.Equal("First,Second<EOL>1,2<EOL>,4<EOL>5,<EOL>", stringWriter.ToString());
            }
        }
        public void write_csv_writes_all_rows_as_data_records()
        {
            using (var stringWriter = new StringWriter())
            using (var writer = new CsvWriter(stringWriter))
            {
                writer.NewLine = "<EOL>";

                var dataTable = new DataTable();
                dataTable.Columns.Add("First");
                dataTable.Columns.Add("Second");
                dataTable.Rows.Add("1", "2");
                dataTable.Rows.Add("3", "4");
                dataTable.Rows.Add("5", "6");

                Assert.Equal(3, dataTable.WriteCsv(writer, true));
                Assert.Equal("First,Second<EOL>1,2<EOL>3,4<EOL>5,6<EOL>", stringWriter.ToString());
            }
        }
        public void write_csv_stops_if_it_reaches_maximum_rows()
        {
            using (var stringWriter = new StringWriter())
            using (var writer = new CsvWriter(stringWriter))
            {
                writer.NewLine = "<EOL>";

                var dataTable = new DataTable();
                dataTable.Columns.Add("First");
                dataTable.Columns.Add("Second");
                dataTable.Rows.Add("1", "2");
                dataTable.Rows.Add("3", "4");
                dataTable.Rows.Add("5", "6");

                Assert.Equal(2, dataTable.WriteCsv(writer, true, 2));
                Assert.Equal("First,Second<EOL>1,2<EOL>3,4<EOL>", stringWriter.ToString());
            }
        }
Beispiel #8
0
        private static void WebCrawling()
        {
            // 데이터 정의
            var dt = new DataTable();
            InitDataTable(Category, dt);

            // 웹크롤링 읽어오기 1 ~ 1236
            for (int uriCount = 1; uriCount <= 1236; uriCount++)
            {
                Uri targetUri = new Uri(GetUri(uriCount));

                var webRequest = HttpWebRequest.Create(targetUri) as HttpWebRequest;

                using (HttpWebResponse webResponse =
                    webRequest.GetResponse() as HttpWebResponse)
                using (Stream webResponseStream =
                    webResponse.GetResponseStream())
                {
                    var document = new HtmlDocument();
                    var targetEncoding = Encoding.UTF8;
                    document.Load(webResponseStream, targetEncoding, true);

                    int count = 0;
                    // 모든 tr 태그들을 불러온다
                    foreach (HtmlNode link in document.DocumentNode.SelectNodes("//tr"))
                    {
                        //"//a[@href]"

                        var row = dt.NewRow();
                        // 첫 3 Row 값을 무시한다.
                        count++;
                        if (count < 3) continue;

                        Console.WriteLine(link);
                        int foreachCount = 0;
                        foreach (var attribute in link.ChildNodes)
                        {
                            var str = attribute.InnerText.Replace("\t", "");
                            str = str.Replace("\n", "");
                            Console.WriteLine(str);

                            InputValue(foreachCount, row, str);
                            foreachCount++;
                        }
                        dt.Rows.Add(row);
                        Console.WriteLine("-----------------");
                    }
                }

                Console.WriteLine("==================");
                Console.WriteLine($"== {uriCount} 번째 완료 ==");
                Console.WriteLine("==================");
                Thread.Sleep(1000);
            }

            //StreamWriter sw = new StreamWriter("Martyrs.csv");
            //WriteToStream(sw, dt, false, false);
            dt.WriteCsv("Martyrs.csv");

            dt.TableName = "Martyrs";
            dt.WriteXml("Martyrs.xml", XmlWriteMode.WriteSchema);

            DataTable dt2 = new DataTable();
            string[] sortedCategory = new[]
            {
                "Name",
                "Status",
                "Adult",
                "Sex",
                "Province",
                "Area",
                "Date of death",
                "Cause of Death"
            };

            int sortedCound = 0;
            InitDataTable(sortedCategory, dt2);
            foreach (DataRow row in dt.Rows)
            {
                // 첫번재 값은 제외
                sortedCound++;
                if (sortedCound == 1)
                    continue;

                var dt2Row = dt2.NewRow();
                foreach (var cateName in sortedCategory)
                {
                    if (cateName != "Adult" && cateName != "Sex")
                    {
                        dt2Row[cateName] = row[cateName];
                    }
                    else
                    {
                        var str = row["Sex"] as string;
                        if (str == null)
                            break;
                        if (str.Contains("Adult"))
                        {
                            dt2Row["Adult"] = "Adult";
                        }
                        else if (str.Contains("Child"))
                        {
                            dt2Row["Adult"] = "Child";
                        }
                        else
                        {
                        }

                        if (str.Contains("Male"))
                        {
                            dt2Row["Sex"] = "Male";
                        }
                        else if (str.Contains("Female"))
                        {
                            dt2Row["Sex"] = "Female";
                        }
                        else
                        {
                        }
                    }
                }
                dt2.Rows.Add(dt2Row);
            }

            StreamWriter sw2 = new StreamWriter("Martyrs3.csv");
            WriteToStream(sw2, dt2, false, false);
        }
        public void write_csv_writes_a_header_record_based_on_data_table_column_names_if_requested()
        {
            using (var stringWriter = new StringWriter())
            using (var writer = new CsvWriter(stringWriter))
            {
                writer.NewLine = "<EOL>";

                var dataTable = new DataTable();
                dataTable.Columns.Add("First");
                dataTable.Columns.Add("Second");

                Assert.Equal(0, dataTable.WriteCsv(writer, true));
                Assert.Equal("First,Second<EOL>", stringWriter.ToString());
            }
        }
        //suicide attack db
        public static void Convert_SuicideAttackDb()
        {
            string[] attackCategory = new[]
            {
                "Attack Date",
                "Location",
                "Campaign",
                "Group",
                "Weapon",
                "Target",
                "Target type",
            };
            // Attack Date, Location, Campaign, Group, Weapon, Target, Target type
            var dt = new DataTable();
            dt.ReadCsv(@"origin\suicide_attack_db_2011-2015.06.csv");

            // start, end, label, description
            var makeDt = new DataTable();
            makeDt.SetColumns(Category);

            // 첫번재 값 제외
            foreach (DataRow row in dt.Rows)
            {
                var makeRow = makeDt.NewRow();
                makeRow[Category[0]] = row[attackCategory[0]];
                makeRow[Category[1]] = row[attackCategory[0]];
                makeRow[Category[2]] = row[attackCategory[1]];
                makeRow[Category[3]] = row[attackCategory[1]];
                makeDt.Rows.Add(makeRow);
            }
            makeDt.WriteCsv(@"visual\visual_suicde_attack_db.csv");
        }
        public static void Convert_peace()
        {
            // 시작 날짜,종료 날짜,사건 이름,설명,지역,위도,경도,행정구역,Refenence Link,올린사람,비고
            var dt = new DataTable();
            dt.ReadCsv(@"origin\peace.csv");

            // start, end, label, description
            var makeDt = new DataTable();
            makeDt.SetColumns(Category);

            // 첫번재 값 제외
            foreach (DataRow row in dt.Rows)
            {
                var makeRow = makeDt.NewRow();
                makeRow[Category[0]] = row[0];
                makeRow[Category[1]] = row[0];
                makeRow[Category[2]] = row[2];
                makeRow[Category[3]] = row[3];
                makeDt.Rows.Add(makeRow);
            }
            makeDt.WriteCsv(@"visual\visual_peace.csv");
        }
        public static void Convert_MartyrsAnalysis()
        {
            var originDt = new DataTable();
            originDt.ReadXml(@"origin\Martyrs.xml");

            DataTable dt2 = new DataTable();
            string[] sortedCategory = new[]
            {
                "Name",
                "Status",
                "Adult",
                "Sex",
                "Province",
                "Area",
                "Date of death",
                "Cause of Death"
            };

            int sortedCound = 0;
            dt2.SetColumns(sortedCategory);
            foreach (DataRow row in originDt.Rows)
            {
                // 첫번재 값은 제외
                sortedCound++;
                if (sortedCound == 1)
                    continue;

                var dt2Row = dt2.NewRow();
                foreach (var cateName in sortedCategory)
                {
                    if (cateName != "Adult" && cateName != "Sex")
                    {
                        dt2Row[cateName] = row[cateName];
                    }
                    else
                    {
                        var str = row["Sex"] as string;
                        if (str == null)
                            break;
                        if (str.Contains("Adult"))
                        {
                            dt2Row["Adult"] = "Adult";
                        }
                        else if (str.Contains("Child"))
                        {
                            dt2Row["Adult"] = "Child";
                        }
                        else
                        {
                        }

                        if (str.Contains("Male"))
                        {
                            dt2Row["Sex"] = "Male";
                        }
                        else if (str.Contains("Female"))
                        {
                            dt2Row["Sex"] = "Female";
                        }
                        else
                        {
                        }
                    }
                }
                dt2.Rows.Add(dt2Row);
            }

            dt2.WriteCsv(@"visual\Martyrs.csv");
        }
        public static void Converter_Classify()
        {
            var orgin = new DataTable();
            orgin.ReadCsv(@"visual\Martyrs3.csv");

            // 매 달마다 사망자수
            var deathClassfy = orgin.AsEnumerable().GroupBy(q => q[4]);

            List<string> deathListh = new List<string>();
            Console.WriteLine("Classfy Death");
            foreach (var death in deathClassfy)
            {
                Console.WriteLine($"Area.Key : {death.Key}, Area.Count : {death.Count()}");

                deathListh.Add(death.Key.ToString());
            }
            string str = String.Empty;
            foreach (var death in deathListh)
            {
                str += death + ",";
            }
            Console.WriteLine($"Total : {orgin.Rows.Count}");
            Console.WriteLine($"Classify Count : {deathClassfy.Count()}");
            Console.WriteLine("end");
            var makeDt = new DataTable();
            makeDt.SetColumns(Category);

            //start: 해당 달에 밝혀진 사망자 수
            //label : 전체 사망자 수
            //description : 전체 사망자 수

            // 월별로 정리
            // 모든 Day를 1로 정의
            foreach (DataRow row in orgin.Rows)
            {
                try
                {
                    DateTime date = Convert.ToDateTime(row[6]);
                    // .ToString("yyyy-MM-dd")
                    var day = date.Day;
                    if (day > 1)
                    {
                        var sub = 1 - day;
                        date = date.AddDays(sub);
                    }
                    row[6] = date.ToString("yyyy-MM-dd");
                }
                catch (Exception)
                {

                }
            }
            //orgin.WriteCsv(@"visual\Martyrs5.csv");
            Debug.WriteLine("end");

            // groupby를 통하 합침.
            // 이를 날짜별로 정리
            var dateClassfy = orgin.AsEnumerable().GroupBy(q => q[6]).OrderBy(q => q.Key);

            //foreach (var dateData in dateClassfy)
            //{
            //    var deathClassify = dateData.GroupBy(q => q[7]);
            //}

            // 사망 지역
            var makeDt3 = new DataTable();
            makeDt3.SetColumns(new string[]
            {
                "date",
                "total_count",
                "Hama",
                "Damascus Suburbs","Damascus",
                "Daraa","Deir Ezzor",
                "Idlib",
                "Aleppo",
                "Homs",
                "Lattakia",
                "Quneitra",
                "Other",
                "Hasakeh",
                "Raqqa",
                "Tartous",
                "Sweida"
            });
            foreach (var dateData in dateClassfy)
            {
                var row = makeDt3.NewRow();
                row[0] = dateData.Key;
                row[1] = dateData.Count();

                var deathClassify = dateData.GroupBy(q => q[4]);
                foreach (var classify in deathClassify)
                {
                    string name = classify.Key.ToString();
                    if (String.IsNullOrEmpty(name) == false)
                    {
                        if (String.IsNullOrEmpty(name))
                        {

                        }
                        else
                        {
                            int rowCount = 0;
                            int.TryParse(row[name] as string, out rowCount);
                            row[name] = rowCount + classify.Count();
                        }
                    }
                }

                makeDt3.Rows.Add(row);
            }
            makeDt3.WriteCsv(@"visual\MartyrsProvince.csv");

            // 사망자 원인
            var makeDt2 = new DataTable();
            makeDt2.SetColumns(new string[]
            {
                "date",
                "total_count",
                "Shooting",
                //"Warplane shelling",
                //"Detention - Torture",
                //"Kidnapping - Execution",
                "Shelling",
                "Other",
                "Field Execution",
                "Explosion",
                //"Detention - Execution",
                //"Detention - Torture - Execution",
                //"Kidnapping - Torture",
                "Un-allowed to seek Medical help",
                //"Kidnapping - Torture - Execution",
                "Chemical and toxic gases",
                "Kidnapping-Detention",
            });

            // 월별 사망자수
            foreach (var dateData in dateClassfy)
            {
                var row = makeDt2.NewRow();
                row[0] = dateData.Key;
                row[1] = dateData.Count();

                var deathClassify = dateData.GroupBy(q => q[7]);
                foreach (var classify in deathClassify)
                {
                    string name = classify.Key.ToString();
                    if (String.IsNullOrEmpty(name) == false)
                    {
                        if (name == "Warplane shelling")
                            name = "Shelling";
                        if (name.Contains("Kidnapping"))
                            name = "Kidnapping-Detention";
                        if (name.Contains("Detention"))
                            name = "Kidnapping-Detention";

                        int rowCount = 0;
                        int.TryParse(row[name] as string, out rowCount);
                        row[name] = rowCount + classify.Count();
                    }
                }

                makeDt2.Rows.Add(row);
            }
            makeDt2.WriteCsv(@"visual\MartyrsDeathCount.csv");

            // 월별 사망자 누적수
            makeDt2.Rows.Clear();
            int count = 0;
            foreach (var dateData in dateClassfy)
            {
                var row = makeDt2.NewRow();
                row[0] = dateData.Key;
                count += dateData.Count();
                row[1] = count;
                makeDt2.Rows.Add(row);
            }
            makeDt2.WriteCsv(@"visual\MartyrsDeathCount2.csv");

            Debug.WriteLine("end");
        }
        public static void Convert_Martyrs()
        {
            var orgin = new DataTable();
            orgin.ReadCsv(@"visual\Martyrs3.csv");

            //var deathClassfy = orgin.AsEnumerable().GroupBy(q => q[7]);

            //Console.WriteLine("Classfy Death");
            //foreach (var death in deathClassfy)
            //{
            //    Console.WriteLine($"death.Key : {death.Key}, death.Count : {death.Count()}");
            //}
            //Console.WriteLine($"Total : {orgin.Rows.Count}");
            //Console.WriteLine("end");

            // start, end, label, description
            var makeDt = new DataTable();
            makeDt.SetColumns(Category);

            //start: 해당 달에 밝혀진 사망자 수
            //label : 전체 사망자 수
            //description : 전체 사망자 수

            // 월별로 정리
            // 모든 Day를 1로 정의
            foreach (DataRow row in orgin.Rows)
            {
                try
                {
                    DateTime date = Convert.ToDateTime(row[6]);
                    // .ToString("yyyy-MM-dd")
                    var day = date.Day;
                    if (day > 1)
                    {
                        var sub = 1 - day;
                        date = date.AddDays(sub);
                    }
                    row[6] = date.ToString("yyyy-MM-dd");
                }
                catch (Exception)
                {

                }
            }
            orgin.WriteCsv(@"visual\Martyrs5.csv");
            Debug.WriteLine("end");

            // groupby를 통하 합침.
            // 이를 날짜별로 정리
            var dateClassfy = orgin.AsEnumerable().GroupBy(q => q[6]).OrderBy(q => q.Key);

            var makeDt2 = new DataTable();
            makeDt2.SetColumns(new string[] { "date", "num" });

            // 월별 사망자수
            foreach (var dateData in dateClassfy)
            {
                var row = makeDt2.NewRow();
                row[0] = dateData.Key;
                row[1] = dateData.Count();
                makeDt2.Rows.Add(row);
            }
            makeDt2.WriteCsv(@"visual\MartyrsCount.csv");

            // 월별 사망자 누적수
            makeDt2.Rows.Clear();
            int count = 0;
            foreach (var dateData in dateClassfy)
            {
                var row = makeDt2.NewRow();
                row[0] = dateData.Key;
                count += dateData.Count();
                row[1] = count;
                makeDt2.Rows.Add(row);
            }
            makeDt2.WriteCsv(@"visual\MartyrsCount2.csv");

            Debug.WriteLine("end");
        }