static void Main1(string[] args) { ConfigReset(); #region url /* * string filepath = @"D:\code\douban\res1page.txt"; * string str = File.ReadAllText(filepath); */ #endregion string str = ""; Stopwatch stopWatch = new Stopwatch(); for (int i = start; i <= 1740; i = i + 20) { try { stopWatch.Start(); start = i; str = GetRespose(start); //string filepath = @"D:\code\douban\1.json"; //string str = File.ReadAllText(filepath); TitleModel model = JsonConvert.DeserializeObject <TitleModel>(str); MysqlDapper mysql = new MysqlDapper(); List <BlindDate> li = new List <BlindDate>(); List <FullPages> pages = new List <FullPages>(); ObjectAddList(model, li, pages); var insertint = mysql.InsertObjList(li); if (pages.Count > 0) { int tt = mysql.InsertObjList(pages); } if (insertint == 0) { nlog.Error($"insert result 0,get one obj:({li[0].ToJson()})"); } stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; if (ts.TotalSeconds < 5) { nlog.Info($"once time(s):{ts.TotalSeconds}"); Random r = new Random(); int sec = r.Next(3000, 6000); Thread.Sleep(sec); } } catch (Exception e) { nlog.Error(e, $"str:{str}"); throw; } } }
private static void ObjectAddList(TitleModel model, List <BlindDate> li, List <FullPages> pages) { foreach (var item in model.items) { var target = item.target; BlindDate date = new BlindDate(); if (target.status == null) { if (int.TryParse(target.author.id, out int id)) { date.author_id = id; } if (int.TryParse(target.id, out int fullid)) { date.fullid = fullid; } else { nlog.Error($"原:{target.id},start:{start},target{item.target.ToJson()}"); break; } date.author_loc = target.author.loc == null ? "" : target.author.loc.name; date.author_name = target.author.name == null ? "" : target.author.name; date.author_reg = Convert.ToDateTime(target.author.reg_time); date.theabstract = item.@abstract; date.page_createtime = Convert.ToDateTime(target.create_time); li.Add(date); } else { FullPages page = new FullPages(); var status = target.status; if (long.TryParse(status.author.id, out long id)) { date.author_id = id; } if (long.TryParse(status.id, out long fullid)) { date.fullid = fullid; page.fullid = fullid; } else { nlog.Error($"原:{status.id},start:{start},target{item.target.ToJson()}"); break; } date.author_loc = status.author.loc == null ? "" : status.author.loc.name; date.author_name = status.author.name; date.author_reg = Convert.ToDateTime(status.author.reg_time); date.theabstract = item.@abstract; date.page_createtime = Convert.ToDateTime(status.create_time); li.Add(date); page.detailstext = status.text; string imgs = ""; if (status.images.Length > 0) { foreach (var img in status.images) { imgs += img.large.url + ";"; } } page.imgs = imgs; pages.Add(page); } } }