public static void Insert() { //返回主键 Id int id = (int)context.Insert <User>(() => new User() { Name = "lu", Age = 18, Gender = Gender.Man, CityId = 1, OpTime = DateTime.Now }); /* * INSERT INTO [Users]([Name],[Age],[Gender],[CityId],[OpTime]) VALUES(N'lu',18,1,1,GETDATE());SELECT @@IDENTITY */ User user = new User(); user.Name = "lu"; user.Age = 18; user.Gender = Gender.Man; user.CityId = 1; user.OpTime = DateTime.Now; //会自动将自增 Id 设置到 user 的 Id 属性上 user = context.Insert(user); /* * String @P_0 = 'lu'; * Gender @P_1 = Man; * Int32 @P_2 = 18; * Int32 @P_3 = 1; * DateTime @P_4 = '2016/8/26 18:41:08'; * INSERT INTO [Users]([Name],[Gender],[Age],[CityId],[OpTime]) VALUES(@P_0,@P_1,@P_2,@P_3,@P_4);SELECT @@IDENTITY */ ConsoleHelper.WriteLineAndReadKey(); }
public static void InsertTest() { MsSqlContext context = new MsSqlContext(DbHelper.ConnectionString); User user = new User(); user.Id = 10; user.Name = "88888"; user.Age = 21; user.Gender = Gender.Man; var id = context.Insert <User>(() => new User() { Name = user.Name, Age = user.Age, Gender = Gender.Man, OpTime = DateTime.Now }); //var id = context.Insert<User>(() => new User() { Name = "lu", NickName = "so", Age = 18, Gender = Gender.Man, OpTime = DateTime.Now }); //var users = context.Query<User>().Where(a => a.Name == null).ToList(); //user = context.Query<User>().Where(a => a.Id == (int)id).First(); user.OpTime = DateTime.Now; var user1 = context.Insert(user); context.Insert(new User() { Name = "lu", Age = 18, Gender = Gender.Man, OpTime = DateTime.Now }); ConsoleHelper.WriteLineAndReadKey(); }
static void SpecifiedTableTest() { object ret = null; User user = null; ret = context.Query <User>().Where(a => a.Id > -1).TakePage(1, 20).ToList(); ret = context.Query <User>("Users_1").Where(a => a.Id > -1).TakePage(1, 20).ToList(); ret = context.Query <User>().Where(a => a.Id > -1).Select(a => a.Name).TakePage(1, 20).ToList(); ret = context.Query <User>("Users_1").Where(a => a.Id > -1).Select(a => a.Name).TakePage(1, 20).ToList(); ret = context.Query <User>().InnerJoin <User>(context.Query <User>("Users_1"), (u1, u2) => u1.Id == u2.Id).Select((u1, u2) => new { u1, u2 }).Where(a => a.u1.Id > -1 && a.u2.Id > -2).ToList(); user = new User() { Id = 100, Name = "Users_1", Gender = Gender.Man, Age = 180, OpTime = DateTime.Now }; context.Insert <User>(user); context.Insert <User>(user, "Users_1"); context.Insert <User>(() => new User() { Name = "Users_1_123", Gender = Gender.Woman, Age = 1800, OpTime = DateTime.Now }); context.Insert <User>(() => new User() { Name = "Users_1_123", Gender = Gender.Woman, Age = 1800, OpTime = DateTime.Now }, "Users_1"); user = context.Query <User>().Where(a => a.Id == 0).First(); user.Name = user.Name + "1"; context.Update(user); context.Update(user, "Users_1"); context.Update <User>(a => a.Id == 0, a => new User() { Age = a.Age + 1 }); context.Update <User>(a => a.Id == 0, a => new User() { Age = a.Age + 1 }, "Users_1"); user = new User() { Id = 2114 }; context.Delete(user); context.Delete(user, "Users_1"); context.Delete <User>(a => a.Id == 2115); context.Delete <User>(a => a.Id == 2115, "Users_1"); ConsoleHelper.WriteLineAndReadKey(); }
/*sql script: * CREATE TABLE [dbo].[ExtensionMappingType]( * [Id] [int] IDENTITY(1,1) NOT NULL, * [Name] [nvarchar](100) NULL, * [F_Char] [nvarchar](1) NULL, * [F_Time] [time](7) NULL, * CONSTRAINT [PK_ExtensionMappingType] PRIMARY KEY CLUSTERED * ( * [Id] ASC * )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] * ) ON [PRIMARY] */ public static void RunDemo() { //step 1: /* 原生 Chloe 不支持 char 和 TimeSpan 类型映射,需要我们自己注册,注册映射类必须在程序启动时进行 */ MappingTypeSystem.Configure(typeof(char), DbType.StringFixedLength); MappingTypeSystem.Configure(typeof(TimeSpan), DbType.Time); //step 2: /* 因为我们新增了 MappingType,所以需要对原生的 SqlConnection、SqlServerCommand、SqlServerDataReader、SqlServerParameter 包装处理,所以,我们需要自个儿实现 IDbConnectionFactory 工厂 */ SqlServerDbConnectionFactory sqlServerDbConnectionFactory = new SqlServerDbConnectionFactory(DbHelper.ConnectionString); MsSqlContext context = new MsSqlContext(sqlServerDbConnectionFactory); /* 经过上述封装,我们就可以支持 char 和 TimeSpan 类型映射了 */ ExtensionMappingType entity = new ExtensionMappingType(); entity.Name = "test"; entity.F_Char = 'A'; entity.F_Time = TimeSpan.FromHours(12); context.Insert(entity); Console.WriteLine(entity.Id); TimeSpan ts = TimeSpan.FromHours(12); ExtensionMappingType model = context.Query <ExtensionMappingType>().Where(a => a.F_Time == ts && a.Id == entity.Id).First(); Console.WriteLine(model.Id == entity.Id); Console.WriteLine("Yeah!!"); Console.ReadKey(); }
//[Benchmark] public override long Insert() { var entity = GetInsertEntity(); var newEntity = _chloeContext.Insert <T_Entity>(entity); return(newEntity.FLong); }
public static void Insert() { var id = (int)context.Insert <TestEntity>(() => new TestEntity() { F_Byte = 1 , F_Int16 = 16 , F_Int32 = 32 , F_Int64 = 64 , F_Double = 1.123456 , F_Float = 1.1234F , F_Decimal = 1.12345678M , F_Bool = true , F_DateTime = DateTime.Now , F_Guid = Guid.NewGuid() , F_String = "12345" }); var entity = new TestEntity(); entity.F_Byte = 1; entity.F_Int16 = 16; entity.F_Int32 = 32; entity.F_Int64 = 64; entity.F_Double = 1.123456; entity.F_Float = 1.1234F; entity.F_Decimal = 1.12345678M; entity.F_Bool = true; entity.F_DateTime = DateTime.Now; entity.F_Guid = Guid.NewGuid(); entity.F_String = "12345"; entity = context.Insert(entity); ConsoleHelper.WriteLineAndReadKey(); }
public void Execute(IJobExecutionContext context) { //IDbCommandInterceptor interceptor = new DbCommandInterceptor(); //dbcontext.Session.AddInterceptor(interceptor); DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); IQuery <Page> cpq = dbcontext.Query <Page>(); IQuery <Chapter> cq = dbcontext.Query <Chapter>(); List <Page> plst = cpq.Where(a => a.pagelocal.Length == 0).Take(20).ToList(); HttpWebHelper web = new HttpWebHelper(); foreach (var p in plst) { try { Stream stream = web.GetStream("http://cdn.sns.dongmanmanhua.cn/20150119_288/1421677325732TxLNo_JPEG/thumbnail_ipad.jpg"); Image img = Image.FromStream(stream); stream.Close(); string filePath = AppDomain.CurrentDomain.BaseDirectory + "DownLoadImgs/" + p.Id + ".jpg"; img.Save(filePath, System.Drawing.Imaging.ImageFormat.Jpeg); string localimg = UcHelper.uploadFile("Page/" + p.Id + ".jpg", filePath); p.pagelocal = localimg; p.modify = dt; dbcontext.Update(p); dbcontext.Update <Chapter>(a => a.Id == p.chapterid, a => new Chapter() { downstatus = DownChapter.完图片, modify = dt }); File.Delete(filePath); } catch (Exception ex) { logger.Error(ex.Message); //Chapter chapter = cq.Where(x => x.Id == p.chapterid).FirstOrDefault(); //chapter.retry = chapter.retry + 1; //chapter.modify = dt; //dbcontext.Update(chapter); dbcontext.Update <Chapter>(a => a.Id == p.chapterid, a => new Chapter() { retry = a.retry + 1, modify = dt }); Err_PageJob err = new Err_PageJob(); err.imgurl = p.pagesource; err.source = p.source; err.errtype = ErrPage.限制访问; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public static void InsertTest() { string name = "so88"; long longAge = 10; int? age = 18; name = null; int r = -1; MsSqlContext context = new MsSqlContext(DbHelper.ConnectionString); User user = new User(); user.Name = "lu"; user.Age = 21; user.Gender = Gender.Man; user.CityId = 1; user.OpTime = DateTime.Now; user.Id = 100; //var id = context.Insert<User>(() => new User() { Name = user.Name, NickName = user.Name, Age = user.Age, Gender = Gender.Man, OpTime = DateTime.Now }); ////var users = context.Query<User>().Where(a => a.Name == null).ToList(); ////user = context.Query<User>().Where(a => a.Id == (int)id).First(); //user.ByteArray = new byte[] { 1, 2, 3 }; //user.OpTime = DateTime.Now; var user1 = context.Insert(user); //返回主键 Id int id = (int)context.Insert <User>(() => new User() { Name = user.Name, Age = user.Age, Gender = Gender.Man, CityId = 1, OpTime = DateTime.Now }); ConsoleHelper.WriteLineAndReadKey(); }
public static void InsertTest1() { IDbContext context = new MsSqlContext(DbHelper.ConnectionString); //返回主键 Id int id = (int)context.Insert <User>(() => new User() { Name = "lu", Age = 18, Gender = Gender.Man, CityId = 1, OpTime = DateTime.Now }); /* * INSERT INTO [Users]([Name],[Age],[Gender],[CityId],[OpTime]) VALUES(N'lu',18,1,1,GETDATE());SELECT @@IDENTITY */ User user = new User(); user.Name = "lu"; //user.Age = 18; user.Gender = Gender.Man; user.CityId = 1; user.OpTime = DateTime.Now; //会自动将自增 Id 设置到 user 的 Id 属性上 user = context.Insert(user); /* * String @P_0 = "lu"; * Gender @P_1 = Man; * Int32 @P_2 = 18; * Int32 @P_3 = 1; * DateTime @P_4 = "1992/1/16 0:00:00"; * INSERT INTO [Users]([Name],[Gender],[Age],[CityId],[OpTime]) VALUES(@P_0,@P_1,@P_2,@P_3,@P_4);SELECT @@IDENTITY */ ConsoleHelper.WriteLineAndReadKey(); }
public void Execute(IJobExecutionContext context) { //IDbCommandInterceptor interceptor = new DbCommandInterceptor(); //dbcontext.Session.AddInterceptor(interceptor); DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); IQuery <Comic> comicq = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Chapter> plst = cpq.Where(a => a.chaptersource.Length != 0 && a.chapterlocal.Length == 0).TakePage(1, 20).ToList(); HttpWebHelper web = new HttpWebHelper(); foreach (var p in plst) { try { string refer = ""; if (p.source == Source.dongmanmanhua) { var comic = comicq.Where(x => x.comicid == p.comicid).FirstOrDefault(); refer = comic.bookurl; } Stream stream = web.GetStream(p.chaptersource, 300, "", null, refer, null); Image img = Image.FromStream(stream); stream.Close(); string filePath = AppDomain.CurrentDomain.BaseDirectory + "DownLoadImgs/" + p.Id + ".jpg"; img.Save(filePath, System.Drawing.Imaging.ImageFormat.Jpeg); string localimg = UcHelper.uploadFile("Chapter/" + p.Id + ".jpg", filePath); p.chapterlocal = localimg; p.modify = dt; dbcontext.Update(p); } catch (Exception ex) { Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = p.chapterurl; err.errtype = ErrChapter.图片出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public void Execute(IJobExecutionContext context) { //IDbCommandInterceptor interceptor = new DbCommandInterceptor(); //dbcontext.Session.AddInterceptor(interceptor); DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); IQuery <Comic> cpq = dbcontext.Query <Comic>(); List <Comic> plst = cpq.Where(a => a.comiccoverlocal.Length == 0).TakePage(1, 20).ToList(); HttpWebHelper web = new HttpWebHelper(); foreach (var p in plst) { try { Stream stream = web.GetStream(p.comiccoversource); Image img = Image.FromStream(stream); stream.Close(); string filePath = AppDomain.CurrentDomain.BaseDirectory + "DownLoadImgs/" + p.Id + ".jpg"; img.Save(filePath, System.Drawing.Imaging.ImageFormat.Jpeg); string localimg = UcHelper.uploadFile("Comic/" + p.Id + ".jpg", filePath); p.comiccoverlocal = localimg; p.modify = dt; dbcontext.Update(p); } catch (Exception ex) { Err_ComicJob err = new Err_ComicJob(); err.bookurl = p.bookurl; err.errtype = ErrComic.图片出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public void Execute(IJobExecutionContext context) { List <bkIm> bklst = ExcelHelper.Import(@"C:\ComicCrawer\GetComics\Excel\漫画抓取信息.xlsx").ToList(); List <Comic> comiclst = new List <Comic>(); IQuery <Comic> cq = dbcontext.Query <Comic>(); DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); foreach (var bk in bklst) { var comic = cq.Where(x => x.bookurl == bk.bookurl).FirstOrDefault(); if (comic != null) { continue; } if (bk.bookurl.StartsWith("http://ac.qq.com")) { try { HttpHelper _helper = new HttpHelper("http://ac.qq.com"); string bookurl = bk.bookurl.Replace("http://ac.qq.com/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("id=\"input_artistName\" value=\"(?<key1>.*?)\""); Match match1 = reg1.Match(bookdata); string authorname = match1.Groups["key1"].Value; Regex reg2 = new Regex("img src=\"(?<key1>.*?)\" alt=\"(?<key2>.*?)\" height=\"280\" width=\"210\""); Match match2 = reg2.Match(bookdata); string comiccover = match2.Groups["key1"].Value; string comicname = match2.Groups["key2"].Value; Regex reg3 = new Regex("<label class=\"works-intro-status\">(?<key1>.*?)</label>"); Match match3 = reg3.Match(bookdata); string isfinished = match3.Groups["key1"].Value; Regex reg4 = new Regex("<meta name=\"Description\" content=\"(?<key1>.*?),简介:(?<key2>.*?)《(?<key3>.*?)》的标签:(?<key4>.*?)\" />"); Match match4 = reg4.Match(bookdata); string bookdesc = match4.Groups["key2"].Value; string theme = match4.Groups["key4"].Value; comiclst.Add(new Comic() { authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = bk.bookurl.Split('/').LastOrDefault(), comicname = comicname, isfinished = isfinished, theme = theme, isvip = "0", source = Source.QQ, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } else if (bk.bookurl.StartsWith("https://www.dongmanmanhua.cn")) { try { HttpHelper _helper = new HttpHelper("https://www.dongmanmanhua.cn"); string bookurl = bk.bookurl.Replace("https://www.dongmanmanhua.cn/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("<meta property=\"og:image\" content=\"(?<key1>.*?)\" />"); Match match1 = reg1.Match(bookdata); string comiccover = match1.Groups["key1"].Value; Regex reg2 = new Regex("<meta property=\"com-dongman:webtoon:author\" content=\"(?<key1>.*?)\" />"); Match match2 = reg2.Match(bookdata); string authorname = match2.Groups["key1"].Value; Regex reg3 = new Regex("<p class=\"summary\">(?<key1>.*?)</p>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match3 = reg3.Match(bookdata); string bookdesc = match3.Groups["key1"].Value; Regex reg4 = new Regex("<h1 class=\"subj\">(?<key1>.*?)</h1>"); Match match4 = reg4.Match(bookdata); string comicname = match4.Groups["key1"].Value; Regex reg5 = new Regex("txt_ico_up"); Match match5 = reg5.Match(bookdata); string isfinished = match5.Length > 0?"连载中": "已完结"; Regex reg6 = new Regex("<h2 class=\"genre(?<key1>.*?)\">(?<key2>.*?)</h2>"); Match match6 = reg6.Match(bookdata); string theme = match6.Groups["key2"].Value; comiclst.Add(new Comic() { authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = bk.bookurl.Split('=').LastOrDefault(), comicname = comicname, isfinished = isfinished, theme = theme, isvip = "0", source = Source.dongmanmanhua, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { logger.Error(ex.Message); Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } if (comiclst.Count > 0) { dbcontext.BulkInsert(comiclst); } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Chapter> cplst = cpq.Where(x => x.source == Source._163 && x.downstatus == DownChapter.待处理链接 && x.isvip == "0").Take(200).ToList(); List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.处理完链接, modify = dt }); foreach (var cp in cplst) { try { string chapterpage = cp.chapterurl.Replace("https://manhua.163.com/", ""); var imgdata = _helper.Get(null, chapterpage); Regex rex1 = new Regex("imageId:"); var match1 = rex1.Match(imgdata); if (match1.Value.Trim() != "") { Regex rex2 = new Regex("url: window.IS_SUPPORT_WEBP (?<key1>.*?),"); var match2 = rex2.Matches(imgdata); List <Page> pglst = new List <Page>(); int sort = 0; foreach (var item in match2) { sort = sort + 1; var tt = ((Match)item).Groups["key1"].Value; var imgurl = tt.Split('"')[3].Substring(0, tt.Split('"')[3].IndexOf("%3D") + 3); pglst.Add(new Page() { pagesource = imgurl, chapterid = cp.chapterid, modify = dt, shortdate = shortdate, sort = sort, source = cp.source, pagelocal = "", }); } if (pglst.Count > 0) { dbcontext.BulkInsert(pglst); } } else { cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); } } catch (Exception ex) { logger.Error(ex.Message); cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public static void InsertTest() { string name = "so88"; long longAge = 10; int? age = 18; name = null; int r = -1; MsSqlContext context = new MsSqlContext(DbHelper.ConnectionString); User user = new User(); user.Id = 10; user.Name = "88888"; user.Age = 21; user.Gender = Gender.Man; var id = context.Insert<User>(() => new User() { Name = user.Name, NickName = user.Name, Age = user.Age, Gender = Gender.Man, OpTime = DateTime.Now }); var id = context.Insert<User>(() => new User() { Name = "lu", NickName = "so", Age = 18, Gender = Gender.Man, OpTime = DateTime.Now }); //var users = context.Query<User>().Where(a => a.Name == null).ToList(); //user = context.Query<User>().Where(a => a.Id == (int)id).First(); user.ByteArray = new byte[] { 1, 2, 3 }; user.OpTime = DateTime.Now; var user1 = context.Insert(user); context.Insert(new User() { Name = "lu", NickName = "so", Age = 18, Gender = Gender.Man, ByteArray = new byte[] { 1, 2 }, OpTime = DateTime.Now }); ConsoleHelper.WriteLineAndReadKey(); }
public void Execute(IJobExecutionContext context) { //List<bkIm> bklst = ExcelHelper.Import(@"C:\Users\Administrator\Desktop\7家书单抓取.xlsx").ToList(); //List<bkIm> bklst = ExcelHelper.Import2(@"C:\Users\Administrator\Desktop\漫画抓取信息.xlsx").ToList(); List <bkIm> bklst = new List <bkIm>(); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628520" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627844" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627229" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626970" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626968" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626920" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626916" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626652" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626455" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624495" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624347" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624308" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624012" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623715" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623595" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623115" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622968" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622674" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622498" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/621673" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/621253" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/620523" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/620493" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553632" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553579" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553055" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/552920" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/552918" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/552839" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/552104" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/547900" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/547343" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/545168" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/544521" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/543172" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/542724" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/539105" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/536658" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628956" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628566" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628481" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628198" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628180" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628109" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627912" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627383" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627300" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627019" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626907" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626527" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626362" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624871" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624601" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623795" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623201" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623200" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623199" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623196" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623195" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622985" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622973" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622971" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622754" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622694" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622561" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622160" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/621727" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/621051" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/620725" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553972" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628458" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553644" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553026" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/552065" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/551693" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/551405" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/551386" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/549847" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/549277" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/548731" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/547243" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/545320" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/542861" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/538969" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/537982" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/534826" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/534796" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/534422" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/533555" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/533395" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/529810" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/518008" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/512742" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626289" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626269" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622660" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553518" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553204" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/549673" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/549599" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/545131" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/537899" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628565" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628464" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628428" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628305" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/628269" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627823" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/627456" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626955" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626819" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626604" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626465" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626268" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/626267" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624341" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624152" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624135" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/624009" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623833" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623720" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623494" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623492" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623459" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623283" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623125" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/623009" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622887" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622856" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622855" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622854" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622565" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622365" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622216" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/622108" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/621020" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/620928" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/553939" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/550484" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/549278" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/549274" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/549125" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/547358" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/545404" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/541345" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/540523" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/539443" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/536042" }); bklst.Add(new bkIm() { bookurl = "http://ac.qq.com/Comic/comicInfo/id/535934" }); //IQuery<VIPFreeComic> vfcq = dbcontext.Query<VIPFreeComic>(); //List<VIPFreeComic> vfclst = vfcq.Where(x => true).ToList(); //vfclst.ForEach(x => //{ // bklst.Add(new bkIm() // { // bookurl = x.bookurl // }); //}); List <Comic> comiclst = new List <Comic>(); IQuery <Comic> cq = dbcontext.Query <Comic>(); DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); foreach (var bk in bklst) { if (!bk.bookurl.StartsWith("http://ac.qq.com")) { continue; } var comic = cq.Where(x => x.bookurl == bk.bookurl).FirstOrDefault(); if (comic != null) { continue; } if (bk.bookurl.StartsWith("http://ac.qq.com")) { try { HttpHelper _helper = new HttpHelper("http://ac.qq.com"); string bookurl = bk.bookurl.Replace("http://ac.qq.com/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("id=\"input_artistName\" value=\"(?<key1>.*?)\""); Match match1 = reg1.Match(bookdata); string authorname = match1.Groups["key1"].Value; Regex reg2 = new Regex("img src=\"(?<key1>.*?)\" alt=\"(?<key2>.*?)\" height=\"280\" width=\"210\""); Match match2 = reg2.Match(bookdata); string comiccover = match2.Groups["key1"].Value; string comicname = match2.Groups["key2"].Value; Regex reg3 = new Regex("<label class=\"works-intro-status\">(?<key1>.*?)</label>"); Match match3 = reg3.Match(bookdata); string isfinished = match3.Groups["key1"].Value; Regex reg4 = new Regex("<meta name=\"Description\" content=\"(?<key1>.*?),简介:(?<key2>.*?)《(?<key3>.*?)》的标签:(?<key4>.*?)\" />"); Match match4 = reg4.Match(bookdata); string bookdesc = match4.Groups["key2"].Value; string theme = match4.Groups["key4"].Value; comiclst.Add(new Comic() { authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = (int)Source.QQ + "_" + bk.bookurl.Split('/').LastOrDefault(), comicname = comicname, isfinished = isfinished, theme = theme, isvip = "0", source = Source.QQ, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } else if (bk.bookurl.StartsWith("https://www.dongmanmanhua.cn")) { try { HttpHelper _helper = new HttpHelper("https://www.dongmanmanhua.cn"); string bookurl = bk.bookurl.Replace("https://www.dongmanmanhua.cn/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("<meta property=\"og:image\" content=\"(?<key1>.*?)\" />"); Match match1 = reg1.Match(bookdata); string comiccover = match1.Groups["key1"].Value; Regex reg2 = new Regex("<meta property=\"com-dongman:webtoon:author\" content=\"(?<key1>.*?)\" />"); Match match2 = reg2.Match(bookdata); string authorname = match2.Groups["key1"].Value; Regex reg3 = new Regex("<p class=\"summary\">(?<key1>.*?)</p>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match3 = reg3.Match(bookdata); string bookdesc = match3.Groups["key1"].Value; Regex reg4 = new Regex("<h1 class=\"subj\">(?<key1>.*?)</h1>"); Match match4 = reg4.Match(bookdata); string comicname = match4.Groups["key1"].Value; Regex reg5 = new Regex("txt_ico_up"); Match match5 = reg5.Match(bookdata); string isfinished = match5.Length > 0 ? "连载中" : "已完结"; Regex reg6 = new Regex("<h2 class=\"genre(?<key1>.*?)\">(?<key2>.*?)</h2>"); Match match6 = reg6.Match(bookdata); string theme = match6.Groups["key2"].Value; comiclst.Add(new Comic() { authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = (int)Source.dongmanmanhua + "_" + bk.bookurl.Split('=').LastOrDefault(), comicname = comicname, isfinished = isfinished, theme = theme, isvip = "0", source = Source.dongmanmanhua, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { logger.Error(ex.Message); Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } else if (bk.bookurl.StartsWith("http://www.u17.com")) { try { HttpHelper _helper = new HttpHelper("http://www.u17.com"); string bookurl = bk.bookurl.Replace("http://www.u17.com/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("<title>(?<key1>.*?)</title>"); Match match1 = reg1.Match(bookdata); string comicname = match1.Groups["key1"].Value.Split('_')[0]; string authorname = match1.Groups["key1"].Value.Split('_')[1]; Regex reg2 = new Regex("cover_url = \"(?<key1>.*?)\""); Match match2 = reg2.Match(bookdata); string comiccover = match2.Groups["key1"].Value; Regex reg3 = new Regex("id=\"words\">(?<key1>.*?)</p>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match3 = reg3.Match(bookdata); string bookdesc = match3.Groups["key1"].Value.Trim(); Regex reg4 = new Regex("状态:(?<key1>.*?)<span(?<key2>.*?)>(?<key3>.*?)</span>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match4 = reg4.Match(bookdata); string isfinished = match4.Groups["key3"].Value.Trim(); Regex reg5 = new Regex("txt_ico_up"); Match match5 = reg5.Match(bookdata); string theme = ""; string pattern = "class=\"label(?<key1>.*?)\" title=\"(?<key3>.*?)\">(?<key2>.*?)</a>"; MatchCollection match6 = Regex.Matches(bookdata, pattern); if (match6.Count == 0) { pattern = "class=\"(?<key1>.*?)_tag \" id=\"tag_(?<key3>.*?)\">(?<key2>.*?)</a>"; match6 = Regex.Matches(bookdata, pattern); } for (int j = 0; j < match6.Count; j++) { if (j == 0) { theme = match6[j].Groups["key2"].Value.Trim(); } else { theme = theme + "," + match6[j].Groups["key2"].Value.Trim(); } } comiclst.Add(new Comic() { comicname = comicname, authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = (int)Source.U17 + "_" + bk.bookurl.Split('/').LastOrDefault().Replace(".html", ""), isfinished = isfinished, theme = theme, isvip = "0", source = Source.U17, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { logger.Error(ex.Message); Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } else if (bk.bookurl.StartsWith("http://www.zymk.cn")) { try { HttpHelper _helper = new HttpHelper("http://www.zymk.cn"); string bookurl = bk.bookurl.Replace("http://www.zymk.cn/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("<meta property=\"og:novel:book_name\" content=\"(?<key1>.*?)\">"); Match match1 = reg1.Match(bookdata); string comicname = match1.Groups["key1"].Value; Regex reg2 = new Regex("<meta property=\"og:novel:author\" content=\"(?<key1>.*?)\">"); Match match2 = reg2.Match(bookdata); string authorname = match2.Groups["key1"].Value; Regex reg3 = new Regex("<meta property=\"og:image\" content=\"(?<key1>.*?)\">"); Match match3 = reg3.Match(bookdata); string comiccover = match3.Groups["key1"].Value; Regex reg4 = new Regex("<div class=\"desc-con\">(?<key1>.*?)</div>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match4 = reg4.Match(bookdata); string bookdesc = match4.Groups["key1"].Value.Trim(); Regex reg5 = new Regex("<meta property=\"og:novel:status\" content=\"(?<key1>.*?)\">"); Match match5 = reg5.Match(bookdata); string isfinished = match5.Groups["key1"].Value == "连载" ? "连载中" : "已完结"; string theme = ""; Regex reg6 = new Regex("<meta property=\"og:novel:category\" content=\"(?<key1>.*?)\">"); Match match6 = reg6.Match(bookdata); theme = string.Join(",", match6.Groups["key1"].Value.Trim().Split(' ')); comiclst.Add(new Comic() { comicname = comicname, authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = (int)Source.Zymk + "_" + bk.bookurl.Split('/')[3], isfinished = isfinished, theme = theme, isvip = "0", source = Source.Zymk, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { logger.Error(ex.Message); Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } else if (bk.bookurl.StartsWith("http://www.manhuatai.com")) { try { HttpHelper _helper = new HttpHelper("http://www.manhuatai.com"); string bookurl = bk.bookurl.Replace("http://www.manhuatai.com/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("<meta property=\"og:novel:book_name\" content=\"(?<key1>.*?)\">"); Match match1 = reg1.Match(bookdata); string comicname = match1.Groups["key1"].Value; Regex reg2 = new Regex("<meta property=\"og:novel:author\" content=\"(?<key1>.*?)\">"); Match match2 = reg2.Match(bookdata); string authorname = match2.Groups["key1"].Value; Regex reg3 = new Regex("<meta property=\"og:image\" content=\"(?<key1>.*?)\">"); Match match3 = reg3.Match(bookdata); string comiccover = match3.Groups["key1"].Value; Regex reg4 = new Regex("<div class=\"wz clearfix t1\"><div>(?<key1>.*?)<a href=\"javascript:void(?<key2>.*?)\" target=\"_self\" class=\"wzrtitle\" onclick=\"openOrCloseSummary", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match4 = reg4.Match(bookdata); string bookdesc = match4.Groups["key1"].Value.Trim(); Regex reg5 = new Regex("<meta property=\"og:novel:status\" content=\"(?<key1>.*?)\">"); Match match5 = reg5.Match(bookdata); string isfinished = match5.Groups["key1"].Value == "连载中" ? "连载中" : "已完结"; string theme = ""; Regex reg6 = new Regex("<meta property=\"og:novel:category\" content=\"(?<key1>.*?)\">"); Match match6 = reg6.Match(bookdata); theme = string.Join(",", match6.Groups["key1"].Value.Trim().Split(' ')); comiclst.Add(new Comic() { comicname = comicname, authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = (int)Source.Manhuatai + "_" + bk.bookurl.Split('/')[3], isfinished = isfinished, theme = theme, isvip = "0", source = Source.Manhuatai, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { logger.Error(ex.Message); Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } else if (bk.bookurl.StartsWith("https://manhua.163.com")) { try { HttpHelper _helper = new HttpHelper("https://manhua.163.com"); string bookurl = bk.bookurl.Replace("https://manhua.163.com/", ""); var bookdata = _helper.Get(null, bookurl); Regex reg1 = new Regex("<meta name=\"keywords\" content=\"(?<key1>.*?)\">"); Match match1 = reg1.Match(bookdata); var arry = match1.Groups["key1"].Value.Split(',').ToList(); string comicname = arry[0]; string authorname = arry[arry.Count - 2]; arry.Remove(arry[0]); arry.Remove(arry[0]); arry.Remove(arry[arry.Count - 1]); arry.Remove(arry[arry.Count - 1]); string theme = string.Join(",", arry.Where(x => x.Length == 2).ToArray()); Regex reg3 = new Regex("<img class=\"sr-bcover\" src=\"(?<key1>.*?)\"/>"); Match match3 = reg3.Match(bookdata); string comiccover = match3.Groups["key1"].Value; Regex reg4 = new Regex("<dt>简介</dt>(?<key2>.*?)<dd>(?<key1>.*?)</dd>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match4 = reg4.Match(bookdata); string bookdesc = match4.Groups["key1"].Value.Trim(); Regex reg5 = new Regex("<dt>状态</dt>(?<key2>.*?)<dd><a href=\"(?<key3>.*?)\" target=\"_blank\">(?<key1>.*?)</a></dd>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match5 = reg5.Match(bookdata); string isfinished = match5.Groups["key1"].Value == "连载中" ? "连载中" : "已完结"; comiclst.Add(new Comic() { comicname = comicname, authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = (int)Source._163 + "_" + bk.bookurl.Split('/')[4], isfinished = isfinished, theme = theme, isvip = "0", source = Source._163, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { logger.Error(ex.Message); Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } else if (bk.bookurl.StartsWith("http://www.mh160.com")) { try { HttpWebHelper _helper = new HttpWebHelper(); var bookdata = _helper.Get(bk.bookurl, Encoding.GetEncoding("gb2312")); Regex reg1 = new Regex("<meta property=\"og:novel:book_name\" content=\"(?<key1>.*?)\">"); Match match1 = reg1.Match(bookdata); string comicname = match1.Groups["key1"].Value; Regex reg2 = new Regex("<meta property=\"og:novel:author\" content=\"(?<key1>.*?)\">"); Match match2 = reg2.Match(bookdata); string authorname = match2.Groups["key1"].Value; Regex reg3 = new Regex("<meta property=\"og:image\" content=\"(?<key1>.*?)\">"); Match match3 = reg3.Match(bookdata); string comiccover = match3.Groups["key1"].Value; Regex reg4 = new Regex("<div class=\"introduction\" id=\"intro1\"><p>(?<key1>.*?)</p>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match4 = reg4.Match(bookdata); string bookdesc = match4.Groups["key1"].Value.Trim(); Regex reg5 = new Regex("<meta property=\"og:novel:status\" content=\"(?<key1>.*?)\">"); Match match5 = reg5.Match(bookdata); string isfinished = match5.Groups["key1"].Value == "连载中" ? "连载中" : "已完结"; string theme = ""; Regex reg6 = new Regex("<meta property=\"og:novel:category\" content=\"(?<key1>.*?)\">"); Match match6 = reg6.Match(bookdata); theme = string.Join(",", match6.Groups["key1"].Value.Trim().Split(' ')); comiclst.Add(new Comic() { comicname = comicname, authorname = authorname, bookurl = bk.bookurl, comiccoversource = comiccover, comiccoverlocal = "", comicdesc = bookdesc, comicid = (int)Source.mh160 + "_" + bk.bookurl.Split('/')[4], isfinished = isfinished, theme = theme, isvip = "0", source = Source.mh160, stopcrawer = false, isoffline = false, recrawer = false, shortdate = shortdate, modify = dt, }); } catch (Exception ex) { logger.Error(ex.Message); Err_ComicJob err = new Err_ComicJob(); err.bookurl = bk.bookurl; err.errtype = ErrComic.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } if (comiclst.Count > 0) { //foreach (var item in comiclst) //{ // var comic = cq.Where(x => x.bookurl == item.bookurl).FirstOrDefault(); // comic.comicdesc = item.comicdesc; // dbcontext.Update(comic); //} dbcontext.BulkInsert(comiclst); } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Comic> comiclst = q.Where(a => a.source == Source._163 && a.shortdate == shortdate).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { shortdate = yesterday, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source._163).ToList(); if (cplst.Count == 0) { try { int sort = 0; var bookdata = _helper.Get <_163_Chapter_Api>(null, string.Format(chapterurl, comic.comicid.Replace("6_", ""))); if (bookdata.code == 200) { bookdata.catalog.sections.ForEach(x => { foreach (var item in x.sections) { sort = sort + 1; chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + item.sectionId, chaptername = item.fullTitle, chapterurl = $"https://manhua.163.com/reader/{item.bookId}/{item.sectionId}#imgIndex=0", sort = sort, comicid = comic.comicid, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = item.needPay == 0 ?"0":"1", chaptersource = "", chapterlocal = "", modify = dt, shortdate = shortdate, }); } }); } } catch (Exception ex) { logger.Error(ex.Message); comic.shortdate = shortdate; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string ticks = dt.Ticks.ToString(); string shortdate = dt.ToString("yyyy-MM-dd"); string updatedatetime = shortdate + " " + ((dt.Hour / 0.5 + 1) * 0.5).ToString(); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); IQuery <PageHis> phisq = dbcontext.Query <PageHis>(); IQuery <Page> pq = dbcontext.Query <Page>(); IQuery <Notice> nq = dbcontext.Query <Notice>(); List <Comic> comiclst = q.Where(a => a.source == Source._163 && (a.updatedatetime == null || a.updatedatetime != updatedatetime)).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { updatedatetime = updatedatetime, modify = dt }); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source._163).ToList(); List <Chapter> chapterlst = new List <Chapter>(); if (cplst.Count > 0) { try { int sort = 0; var bookdata = _helper.Get <_163_Chapter_Api>(null, string.Format(chapterurl, comic.comicid.Replace("6_", ""))); if (bookdata.code == 200) { bookdata.catalog.sections.ForEach(x => { foreach (var item in x.sections) { sort = sort + 1; chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + item.sectionId, chaptername = item.fullTitle, chapterurl = $"https://manhua.163.com/reader/{item.bookId}/{item.sectionId}#imgIndex=0", sort = sort, comicid = comic.comicid, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = item.needPay == 0 ? "0" : "1", chaptersource = "", chapterlocal = "", modify = dt, shortdate = shortdate, }); } }); } int delete = cplst.Except(chapterlst, new Chapter_Comparer()).Count(); // 删章 List <Chapter> add = chapterlst.Except(cplst, new Chapter_Comparer()).ToList(); // 新增 if (delete > 0) { List <string> idlst = cplst.Select(x => x.chapterid).ToList(); int phislstcount = phisq.Where(x => idlst.Contains(x.chapterid)).Count(); if (phislstcount == 0) { List <Page> pglst = pq.Where(x => idlst.Contains(x.chapterid)).ToList(); if (pglst.Count > 0) { List <PageHis> phislst = new List <PageHis>(); pglst.ForEach(x => { phislst.Add(new PageHis() { chapterid = x.chapterid, modify = x.modify, pagelocal = x.pagelocal, pagesource = x.pagesource, pagestate = x.pagestate, shortdate = x.shortdate, sort = x.sort, source = x.source, ticks = x.ticks }); }); dbcontext.BulkInsert(phislst); } } dbcontext.Delete <Page>(x => idlst.Contains(x.chapterid)); dbcontext.Delete <Chapter>(x => idlst.Contains(x.chapterid)); if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } Notice notice = new Notice(); notice.noticeid = comic.comicid; notice.noticestatus = NoticeStatus.等待处理; notice.noticetype = NoticeType.目录变更; notice.source = comic.source; notice.shortdate = shortdate; notice.modify = dt; var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); if (nqwait == null) { dbcontext.Insert(notice); } continue; } else { List <Chapter> mvadd = chapterlst.Except(add, new Chapter_Comparer()).ToList(); string cplststr = string.Join(",", cplst.Select(x => x.chapterid).ToArray()); string chapterlststr = string.Join(",", mvadd.Select(x => x.chapterid).ToArray()); if (cplststr != chapterlststr) // 调序 { List <string> idlst = cplst.Select(x => x.chapterid).ToList(); int phislstcount = phisq.Where(x => idlst.Contains(x.chapterid)).Count(); if (phislstcount == 0) { List <Page> pglst = pq.Where(x => idlst.Contains(x.chapterid)).ToList(); if (pglst.Count > 0) { List <PageHis> phislst = new List <PageHis>(); pglst.ForEach(x => { phislst.Add(new PageHis() { chapterid = x.chapterid, modify = x.modify, pagelocal = x.pagelocal, pagesource = x.pagesource, pagestate = x.pagestate, shortdate = x.shortdate, sort = x.sort, source = x.source, ticks = x.ticks }); }); dbcontext.BulkInsert(phislst); } } dbcontext.Delete <Page>(x => idlst.Contains(x.chapterid)); dbcontext.Delete <Chapter>(x => idlst.Contains(x.chapterid)); if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } Notice notice = new Notice(); notice.noticeid = comic.comicid; notice.noticestatus = NoticeStatus.等待处理; notice.noticetype = NoticeType.目录变更; notice.source = comic.source; notice.shortdate = shortdate; notice.modify = dt; var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); if (nqwait == null) { dbcontext.Insert(notice); } continue; } } if (add.Count > 0) { dbcontext.BulkInsert(add); } } catch (Exception ex) { logger.Error(ex.Message); comic.updatedatetime = ""; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Chapter> cplst = cpq.Where(x => x.source == Source.Zymk && x.downstatus == DownChapter.待处理链接).Take(200).ToList(); List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.处理完链接, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var cp in cplst) { try { string chapterpage = cp.chapterurl.Replace("http://www.zymk.cn/", ""); var imgdata = _helper.Get(null, chapterpage); Regex imgpath = new Regex("chapter_addr:\"(?<key1>.*?)\","); Regex starindex = new Regex("start_var:(?<key1>.*?),"); Regex totalimg = new Regex("end_var:(?<key1>.*?),"); Regex pageid = new Regex("chapter_id:(?<key1>.*?),"); Regex domain = new Regex("domain:\"(?<key1>.*?)\","); Regex comic_size = new Regex("middle:\"(?<key1>.*?)\""); var _imgpath = imgpath.Match(imgdata).Groups["key1"].Value.Replace("\\\\", "#").Replace("\\", "").Replace("#", "\\"); var _totalimg = totalimg.Match(imgdata).Groups["key1"].Value; var _pageid = pageid.Match(imgdata).Groups["key1"].Value; var _domain = domain.Match(imgdata).Groups["key1"].Value; var _comic_size = comic_size.Match(imgdata).Groups["key1"].Value; int start = int.Parse(starindex.Match(imgdata).Groups["key1"].Value.Trim()); List <Page> pglst = new List <Page>(); int imgcount = int.Parse(_totalimg); imgcount = imgcount - start; string imgdecodepath = DecodeHelper.Decode(_imgpath, int.Parse(_pageid)); for (int i = 0; i <= imgcount; i++) { string pgsource = "http://mhpic." + _domain + "/comic/" + imgdecodepath + start + ".jpg" + _comic_size; pglst.Add(new Page() { pagesource = pgsource, chapterid = cp.chapterid, modify = dt, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", }); start = start + 1; } dbcontext.BulkInsert(pglst); } catch (Exception ex) { logger.Error(ex.Message); cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Comic> comiclst = q.Where(a => a.source == Source.dongmanmanhua && a.shortdate == shortdate).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { shortdate = yesterday, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.Id && a.source == Source.dongmanmanhua).ToList(); if (cplst.Count == 0) { try { string bookurl = comic.bookurl.Replace("https://www.dongmanmanhua.cn/", ""); var bookdata = _helper.Get(null, bookurl); string pattern = "<li id=\"episode_(?<key1>.*?)\" data-episode-no=\"(?<key2>.*?)\">(?<key3>.*?)</li>"; MatchCollection matches = Regex.Matches(bookdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); int pagecount = (int)Math.Ceiling(int.Parse(matches[0].Groups[1].Value) / 10.0); for (int i = 0; i < matches.Count; i++) { var lihtml = matches[i].Groups["key3"].Value; Regex reg1 = new Regex("href=\"(?<key1>.*?)\""); Match match1 = reg1.Match(lihtml); string chapterurl = match1.Groups["key1"].Value; Regex reg2 = new Regex("<span class=\"tx\">#(?<key1>.*?)</span>"); Match match2 = reg2.Match(lihtml); int sort = int.Parse(match2.Groups["key1"].Value); Regex reg3 = new Regex("<span class=\"subj\"><span>(?<key1>.*?)</span></span>"); Match match3 = reg3.Match(lihtml); string chaptername = match3.Groups["key1"].Value; Regex reg4 = new Regex("src=\"(?<key1>.*?)\""); Match match4 = reg4.Match(lihtml); string chaptersource = match4.Groups["key1"].Value; chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + sort, chaptername = chaptername, chapterurl = "https:" + chapterurl, sort = sort, comicid = comic.Id, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = chaptersource, chapterlocal = "", modify = dt, shortdate = shortdate, }); } for (int i = 2; i <= pagecount; i++) { var bookdata2 = _helper.Get(null, bookurl + "&page=" + i); string pattern2 = "<li id=\"episode_(?<key1>.*?)\" data-episode-no=\"(?<key2>.*?)\">(?<key3>.*?)</li>"; MatchCollection matches2 = Regex.Matches(bookdata2, pattern2, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int j = 0; j < matches2.Count; j++) { var lihtml = matches2[j].Groups["key3"].Value; Regex reg1 = new Regex("href=\"(?<key1>.*?)\""); Match match1 = reg1.Match(lihtml); string chapterurl = match1.Groups["key1"].Value; Regex reg2 = new Regex("<span class=\"tx\">#(?<key1>.*?)</span>"); Match match2 = reg2.Match(lihtml); int sort = int.Parse(match2.Groups["key1"].Value); Regex reg3 = new Regex("<span class=\"subj\"><span>(?<key1>.*?)</span></span>"); Match match3 = reg3.Match(lihtml); string chaptername = match3.Groups["key1"].Value; Regex reg4 = new Regex("src=\"(?<key1>.*?)\""); Match match4 = reg4.Match(lihtml); string chaptersource = match4.Groups["key1"].Value; chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + sort, chaptername = chaptername, chapterurl = "https:" + chapterurl, sort = sort, comicid = comic.Id, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = chaptersource, chapterlocal = "", modify = dt, shortdate = shortdate, }); } } } catch (Exception ex) { logger.Error(ex.Message); comic.shortdate = shortdate; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Comic> comiclst = q.Where(a => a.source == Source.mh160 && a.shortdate == shortdate).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { shortdate = yesterday, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source.mh160).ToList(); if (cplst.Count == 0) { try { var bookdata = _helper.Get(comic.bookurl, Encoding.GetEncoding("gb2312")); bookdata = StringHelper.MergeSpace(bookdata); //string pattern = "<div class=\"plist pnormal\" id=\"play_0\"><li(?<key1>.*?)><a href=\"(?<key2>.*?)\" title=\"(?<key3>.*?)\" target=\"_blank\">(?<key4>.*?)</a></li>"; string pattern = "<div class=\"plist pnormal\" id=\"play_0\"> <ul>(?<key1>.*?)</ul> </div>"; MatchCollection matches = Regex.Matches(bookdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); string pattern2 = "<a href=\"(?<key1>.*?)\" title=\"(?<key2>.*?)\" target=\"_blank\">(?<key3>.*?)</a>"; MatchCollection matches2 = Regex.Matches(matches[0].Groups["key1"].Value, pattern2, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int i = 0; i < matches2.Count; i++) { string chapterid = comic.comicid + "_" + matches2[i].Groups["key1"].Value.Split('/').LastOrDefault().Replace(".html", ""); string chaptername = matches2[i].Groups["key2"].Value; string chapterurl = "http://www.mh160.com" + matches2[i].Groups["key1"].Value; int sort = matches.Count - i; chapterlst.Add(new Chapter() { chapterid = chapterid, chaptername = chaptername, chapterurl = chapterurl, sort = sort, comicid = comic.comicid, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = "", chapterlocal = "", modify = dt, shortdate = shortdate, }); } } catch (Exception ex) { logger.Error(ex.Message); comic.shortdate = shortdate; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } }
public void Execute(IJobExecutionContext context) { //logger.Info("QQ_page vip begin"); //string isStart = "IsStartBuyQQ".ValueOfAppSetting(); //if (isStart != null && isStart.Equals("1")) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>();//x.comicid == "1_524356" IQuery <PageHis> phisq = dbcontext.Query <PageHis>(); //List<Chapter> cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.待处理链接 && x.isvip.Equals("1")).Take(200).ToList(); List <Chapter> cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.完图片 && x.isvip.Equals("1")).Take(200).ToList(); List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.处理中, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); Regex rex = new Regex("var DATA = '(?<key1>.*?)',"); string errMsg; foreach (var cp in cplst) { try { IQuery <PageHis> cpHis = phisq.Where(x => x.chapterid == cp.chapterid); if (cpHis != null && cpHis.Count() > 0) { List <PageHis> cpHisList = cpHis.ToList(); List <Page> pglst = new List <Page>(); foreach (var page in cpHisList) { pglst.Add(new Page() { chapterid = page.chapterid, modify = dt, shortdate = shortdate, sort = page.sort, source = page.source, pagelocal = "", pagesource = page.pagesource }); } cp.downstatus = DownChapter.处理完链接; cp.modify = dt; dbcontext.Update(cp); dbcontext.BulkInsert(pglst); logger.Info("QQ_page vip syn history sucess:id=" + cp.Id); } else { errMsg = string.Empty; selenium.Navigate().GoToUrl(cp.chapterurl); IList <IWebElement> frames = selenium.FindElements(By.TagName("iframe")); IWebElement controlPanelFrame = null; foreach (var frame in frames) { if (frame.GetAttribute("id") == "iframeAll") { controlPanelFrame = frame; break; } } if (controlPanelFrame != null && controlPanelFrame.Displayed == true) //QQ登录 { try { selenium.SwitchTo().Frame(controlPanelFrame); IReadOnlyCollection <IWebElement> switchtoElement = selenium.FindElements(By.Id("switcher_plogin")); if (switchtoElement != null && switchtoElement.Count > 0 && switchtoElement.First().Displayed == true) { switchtoElement.First().Click(); selenium.FindElement(By.Id("u")).Clear(); selenium.FindElement(By.Id("u")).SendKeys("3283360259"); selenium.FindElement(By.Id("p")).Clear(); selenium.FindElement(By.Id("p")).SendKeys("xxxttt5544"); //selenium.FindElement(By.Id("u")).Clear(); //selenium.FindElement(By.Id("u")).SendKeys("1434299101"); //selenium.FindElement(By.Id("p")).Clear(); //selenium.FindElement(By.Id("p")).SendKeys("zhangyin123"); selenium.FindElement(By.Id("login_button")).Click(); } selenium.SwitchTo().DefaultContent(); } catch (Exception ex) { errMsg = "QQ 登录失败:" + ex.Message; logger.Error(errMsg); } } selenium.Navigate().GoToUrl("http://ac.qq.com/Home/buyList"); ICookieJar listCookie = selenium.Manage().Cookies; // IList<Cookie> listCookie = selenuim.Manage( ).Cookies.AllCookies;//只是显示 可以用Ilist对象 //显示初始Cookie的内容 Console.WriteLine("--------------------"); Console.WriteLine($"当前Cookie集合的数量:\t{listCookie.AllCookies.Count}"); for (int i = 0; i < listCookie.AllCookies.Count; i++) { Console.WriteLine($"Cookie的名称:{listCookie.AllCookies[i].Name}"); Console.WriteLine($"Cookie的值:{listCookie.AllCookies[i].Value}"); Console.WriteLine($"Cookie的所在域:{listCookie.AllCookies[i].Domain}"); Console.WriteLine($"Cookie的路径:{listCookie.AllCookies[i].Path}"); Console.WriteLine($"Cookie的过期时间:{listCookie.AllCookies[i].Expiry}"); Console.WriteLine("-----"); } frames = selenium.FindElements(By.TagName("iframe")); IWebElement checkVipFrame = null; foreach (var frame in frames) { if (frame.GetAttribute("id") == "checkVipFrame") { checkVipFrame = frame; break; } } if (checkVipFrame != null && checkVipFrame.Displayed == true) { try { //自动购买 selenium.SwitchTo().Frame(checkVipFrame); IReadOnlyCollection <IWebElement> checkAutoElement = selenium.FindElements(By.Id("check_auto_next")); IReadOnlyCollection <IWebElement> singlBbuyElement = selenium.FindElements(By.ClassName("single_buy")); if (checkAutoElement != null && singlBbuyElement != null && checkAutoElement.Count > 0 && singlBbuyElement.Count > 0 && checkAutoElement.First().Displayed == true) { if (singlBbuyElement.First().Text.IndexOf("点券不足") > -1) { //列表中未成功购买的数据还原成待处理 dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.待处理链接, modify = dt }); ////关闭购买,等待修改配置 //"IsStartBuyQQ".SetAppSettingValue("0"); if (isHasMoney) { Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = "点券不足,请去充值!"; err = dbcontext.Insert(err); } isHasMoney = false; //Thread.Sleep(3600000); continue; } else { isHasMoney = true; } checkAutoElement.First().Click(); singlBbuyElement.First().Click(); } selenium.SwitchTo().DefaultContent(); } catch (Exception ex) { errMsg = "自动购买失败:" + ex.Message; logger.Error(errMsg); } } Match match1 = rex.Match(selenium.PageSource); string key = match1.Groups["key1"].Value; if (string.IsNullOrEmpty(key) || errMsg != string.Empty) { cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = errMsg != string.Empty ? errMsg : "DATA解析失败"; err = dbcontext.Insert(err); continue; } string s = DecodeHelper.QQPageDecode(key.Substring(1)); var t = JsonHelper.DeserializeJsonToObject <QQ_Page_Api>(s); if (t.picture.Count < 1) { cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = "访问付费章节内容时只存在一张图片"; err = dbcontext.Insert(err); continue; } List <Page> pglst = new List <Page>(); for (int i = 0; i < t.picture.Count; i++) { if (pglst.Exists(x => x.pagesource == t.picture[i].url) == false) { pglst.Add(new Page() { chapterid = cp.chapterid, modify = DateTime.Now, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", pagesource = t.picture[i].url }); } else { Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = DateTime.Now; err.shortdate = shortdate; err.message = "存在重复图片"; err = dbcontext.Insert(err); } } cp.downstatus = DownChapter.处理完链接; cp.modify = dt; dbcontext.Update(cp); dbcontext.BulkInsert(pglst); ids.Remove(cp.Id); logger.Info("QQ_page vip buy sucess:id=" + cp.Id); } } catch (Exception ex) { if (ex.Message.IndexOf("Unexpected error. System.Net.WebException:") > -1) { InitChromeDriver(); } logger.Error(ex.Message); cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Chapter> cplst = cpq.Where(x => x.source == Source.U17 && x.downstatus == DownChapter.待处理链接 && x.isvip == "0").Take(200).ToList(); List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.处理完链接, modify = dt }); foreach (var cp in cplst) { try { var imgdata = _helper.Get <U17_Page_Api>(null, string.Format(pageurl, cp.chapterid.Replace(cp.comicid + "_", ""))); List <Page> pglst = new List <Page>(); int sort = 0; imgdata.image_list.ForEach(x => { sort = sort + 1; pglst.Add(new Page() { pagesource = x.src, chapterid = cp.chapterid, modify = dt, shortdate = shortdate, sort = sort, source = cp.source, pagelocal = "", }); }); if (pglst.Count > 0) { dbcontext.BulkInsert(pglst); } } catch (Exception ex) { logger.Error(ex.Message); cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Chapter> cplst = cpq.Where(x => x.source == Source.dongmanmanhua && x.downstatus == DownChapter.待处理链接).Take(200).ToList(); List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.处理完链接, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var cp in cplst) { List <Page> pglst = new List <Page>(); try { string chapterpage = cp.chapterurl.Replace("https://www.dongmanmanhua.cn", ""); var imgdata = _helper.Get(null, chapterpage); Regex reg1 = new Regex("documentURL: '(?<key1>.*?)'"); Match match1 = reg1.Match(imgdata); string docURl = match1.Groups["key1"].Value; if (string.IsNullOrEmpty(docURl)) { string pattern = "<div class=\"(?<key1>.*?)\" id=\"_imageList\">(?<key2>.*?)</div>"; MatchCollection matches = Regex.Matches(imgdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); string imghtml = matches[0].Groups["key2"].Value; string pattern2 = "data-url=\"(?<key1>.*?)\""; MatchCollection matches2 = Regex.Matches(imghtml, pattern2); for (int i = 0; i < matches2.Count; i++) { pglst.Add(new Page() { chapterid = cp.chapterid, modify = dt, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", pagesource = matches2[i].Groups["key1"].Value }); } } else { Regex reg2 = new Regex("stillcut: '(?<key1>.*?)'"); Match match2 = reg2.Match(imgdata); string stillcut = match2.Groups["key1"].Value; var bookdata = _helper.Get <dongmanmanhua_Page_Api>(null, "https://www.dongmanmanhua.cn" + docURl); string regstr = bookdata.assets.image.ToString(); string pattern3 = ": \"(?<key1>.*?)\""; MatchCollection matches3 = Regex.Matches(regstr, pattern3, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int i = 0; i < matches3.Count; i++) { pglst.Add(new Page() { chapterid = cp.chapterid, modify = dt, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", pagesource = string.Format(stillcut.Replace("=filename", "0"), matches3[0].Groups["key1"].Value) }); } } dbcontext.BulkInsert(pglst); } catch (Exception ex) { logger.Error(ex.Message); cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Chapter> cplst = cpq.Where(x => x.source == Source.dongmanmanhua && x.downstatus == DownChapter.待处理链接).Take(200).ToList(); List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.处理完链接, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var cp in cplst) { try { string chapterpage = cp.chapterurl.Replace("https://www.dongmanmanhua.cn", ""); var imgdata = _helper.Get(null, chapterpage); string pattern = "<div class=\"(?<key1>.*?)\" id=\"_imageList\">(?<key2>.*?)</div>"; MatchCollection matches = Regex.Matches(imgdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); string imghtml = matches[0].Groups["key2"].Value; string pattern2 = "data-url=\"(?<key1>.*?)\""; MatchCollection matches2 = Regex.Matches(imghtml, pattern2); List <Page> pglst = new List <Page>(); for (int i = 0; i < matches2.Count; i++) { pglst.Add(new Page() { chapterid = cp.Id, modify = dt, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", pagesource = matches2[i].Groups["key1"].Value }); } dbcontext.BulkInsert(pglst); } catch (Exception ex) { logger.Error(ex.Message); cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public bool BatchInsert(List <Product_Size> list) { bool v = false; string DbType = Globals.Configuration["db:DbType"].ToString(); string ConnString = Globals.Configuration["db:ConnString"].ToString(); if (DbType == "MySql") { using (MySqlContext context = new MySqlContext(new MySqlConnectionFactory(ConnString))) { try { context.Session.BeginTransaction(); foreach (var item in list) { //int row = context.Session.ExecuteNonQuery("select 1 from yd_fdtz where DKBH = '"+item.DKBH+"' limit 1;"); if (item.Id == "0") { item.Id = IdHelper.CreateStringSnowflakeId(); context.Insert <Product_Size>(item); } else { context.Update <Product_Size>(item); } } context.Session.CommitTransaction(); } catch { if (context.Session.IsInTransaction) { context.Session.RollbackTransaction(); } throw; } } } if (DbType == "SqlServer") { using (MsSqlContext context = new MsSqlContext(ConnString)) { try { context.Session.BeginTransaction(); foreach (var item in list) { //int row = context.Session.ExecuteNonQuery("select count(1) from yd_fdtz where DKBH = '" + item.DKBH + "'"); if (item.Id == "0") { item.Id = IdHelper.CreateStringSnowflakeId(); context.Insert <Product_Size>(item); } else { context.Update <Product_Size>(item); } } context.Session.CommitTransaction(); } catch { if (context.Session.IsInTransaction) { context.Session.RollbackTransaction(); } throw; } } } return(v); }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string ticks = dt.Ticks.ToString(); string shortdate = dt.ToString("yyyy-MM-dd"); string updatedatetime = shortdate + " " + ((dt.Hour / 0.5 + 1) * 0.5).ToString(); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); IQuery <PageHis> phisq = dbcontext.Query <PageHis>(); IQuery <Page> pq = dbcontext.Query <Page>(); IQuery <Notice> nq = dbcontext.Query <Notice>(); List <Comic> comiclst = q.Where(a => a.source == Source.QQ && a.isoffline == false && (a.updatedatetime == null || a.updatedatetime != updatedatetime)).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { updatedatetime = updatedatetime, modify = dt }); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source.QQ).ToList(); List <Chapter> chapterlst = new List <Chapter>(); if (cplst.Count > 0) { try { string bookurl = comic.bookurl.Replace("http://ac.qq.com/", ""); var bookdata = _helper.Get(null, bookurl); //class=\"chapter-page-all works-(?<key1>.*?)-list\" Regex reg1 = new Regex("<ol class=\"chapter-page-all works-chapter-list\">(?<key1>.*?)</ol>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match1 = reg1.Match(bookdata); string htmlSource = match1.Groups["key1"].Value; string pattern = "<a target=\"_blank\" title=\"(?<key1>.*?)\" href=\"(?<key2>.*?)\">(?<key3>.*?)</a>"; MatchCollection matches = Regex.Matches(htmlSource, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int i = 0; i < matches.Count; i++) { chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + matches[i].Groups["key2"].Value.Split('/').LastOrDefault(), chaptername = matches[i].Groups["key3"].Value.Trim(), chapterurl = "http://ac.qq.com" + matches[i].Groups["key2"].Value, sort = i + 1, comicid = comic.comicid, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = "", chapterlocal = "", modify = dt, shortdate = shortdate, ticks = ticks }); } int delete = cplst.Except(chapterlst, new Chapter_Comparer()).Count(); // 删章 List <Chapter> add = chapterlst.Except(cplst, new Chapter_Comparer()).ToList(); // 新增 //if (delete > 0) //{ // List<string> idlst = cplst.Select(x => x.chapterid).ToList(); // int phislstcount = phisq.Where(x => idlst.Contains(x.chapterid)).Count(); // if (phislstcount == 0) // { // List<Page> pglst = pq.Where(x => idlst.Contains(x.chapterid)).ToList(); // if (pglst.Count>0) // { // List<PageHis> phislst = new List<PageHis>(); // pglst.ForEach(x => // { // phislst.Add(new PageHis() // { // chapterid = x.chapterid, // modify = x.modify, // pagelocal = x.pagelocal, // pagesource = x.pagesource, // pagestate = x.pagestate, // shortdate = x.shortdate, // sort = x.sort, // source = x.source, // ticks = x.ticks // }); // }); // dbcontext.BulkInsert(phislst); // } // } // dbcontext.Delete<Page>(x => idlst.Contains(x.chapterid)); // dbcontext.Delete<Chapter>(x => idlst.Contains(x.chapterid)); // if (chapterlst.Count > 0) // { // dbcontext.BulkInsert(chapterlst); // } // Notice notice = new Notice(); // notice.noticeid = comic.comicid; // notice.noticestatus = NoticeStatus.等待处理; // notice.noticetype = NoticeType.目录变更; // notice.source = comic.source; // notice.shortdate = shortdate; // notice.modify = dt; // var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); // if (nqwait == null) // { // dbcontext.Insert(notice); // } // continue; //} //else //{ // List<Chapter> mvadd = chapterlst.Except(add, new Chapter_Comparer()).ToList(); // string cplststr = string.Join(",", cplst.Select(x => x.chapterid).ToArray()); // string chapterlststr = string.Join(",", mvadd.Select(x => x.chapterid).ToArray()); // if (cplststr != chapterlststr) // 调序 // { // List<string> idlst = cplst.Select(x => x.chapterid).ToList(); // int phislstcount = phisq.Where(x => idlst.Contains(x.chapterid)).Count(); // if (phislstcount == 0) // { // List<Page> pglst = pq.Where(x => idlst.Contains(x.chapterid)).ToList(); // if (pglst.Count > 0) // { // List<PageHis> phislst = new List<PageHis>(); // pglst.ForEach(x => // { // phislst.Add(new PageHis() // { // chapterid = x.chapterid, // modify = x.modify, // pagelocal = x.pagelocal, // pagesource = x.pagesource, // pagestate = x.pagestate, // shortdate = x.shortdate, // sort = x.sort, // source = x.source, // ticks = x.ticks // }); // }); // dbcontext.BulkInsert(phislst); // } // } // dbcontext.Delete<Page>(x => idlst.Contains(x.chapterid)); // dbcontext.Delete<Chapter>(x => idlst.Contains(x.chapterid)); // if (chapterlst.Count > 0) // { // dbcontext.BulkInsert(chapterlst); // } // Notice notice = new Notice(); // notice.noticeid = comic.comicid; // notice.noticestatus = NoticeStatus.等待处理; // notice.noticetype = NoticeType.目录变更; // notice.source = comic.source; // notice.shortdate = shortdate; // notice.modify = dt; // var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); // if (nqwait == null) // { // dbcontext.Insert(notice); // } // continue; // } //} if (add.Count > 0) { dbcontext.BulkInsert(add); } } catch (Exception ex) { logger.Error(ex.Message); comic.updatedatetime = ""; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } }
public void Execute(IJobExecutionContext context) { //IDbCommandInterceptor interceptor = new DbCommandInterceptor(); //dbcontext.Session.AddInterceptor(interceptor); DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); IQuery <Page> cpq = dbcontext.Query <Page>(); IQuery <Chapter> cq = dbcontext.Query <Chapter>(); List <Page> plst; if (m_DownTotal > 0) { plst = cpq.Where(a => a.pagestate == PageState.None && a.Id % m_DownTotal == m_DownIndex).Take(100).ToList(); } else { plst = cpq.Where(a => a.pagestate == PageState.None).Take(100).ToList(); } //if (plst.Count < pageSize) // pageIndex = 1; //else // pageIndex++; HttpWebHelper web = new HttpWebHelper(); foreach (var p in plst) { string filePath = AppDomain.CurrentDomain.BaseDirectory + "DownLoadImgs/" + p.Id + ".jpg"; try { string refer = ""; if (p.source == Source.dongmanmanhua || p.source == Source.dmzj) { var chapter = cq.Where(x => x.chapterid == p.chapterid).FirstOrDefault(); refer = chapter.chapterurl; Stream stream = web.GetStream(p.pagesource, 3000, "", null, refer, null); Image img = Image.FromStream(stream); stream.Close(); img.Save(filePath, System.Drawing.Imaging.ImageFormat.Jpeg); } else { WebClient myclient = new WebClient(); myclient.DownloadFile(p.pagesource, filePath); } string localimg = UcHelper.uploadFile("Page/" + p.Id + ".jpg", filePath); p.pagelocal = localimg; p.pagestate = PageState.成功; p.modify = dt; dbcontext.Update(p); File.Delete(filePath); } catch (Exception ex) { try { WebClient myclient = new WebClient(); myclient.DownloadFile(p.pagesource, filePath); Bitmap bytes; using (WebP webp = new WebP()) bytes = webp.Load(filePath); File.Delete(filePath); bytes.Save(filePath); string localimg = UcHelper.uploadFile("Page/" + p.Id + ".jpg", filePath); p.pagelocal = localimg; p.modify = dt; p.pagestate = PageState.成功; dbcontext.Update(p); File.Delete(filePath); continue; } catch (Exception) { logger.Error(ex.Message); Chapter chapter = cq.Where(x => x.chapterid == p.chapterid).FirstOrDefault(); chapter.retry = chapter.retry + 1; chapter.modify = dt; if (chapter.retry > 30) { p.pagestate = PageState.失败; p.modify = dt; dbcontext.Update(p); } dbcontext.Update(chapter); //dbcontext.Update<Chapter>(a => a.chapterid == p.chapterid, a => new Chapter() //{ // retry = a.retry + 1, // modify = dt //}); Err_PageJob err = new Err_PageJob(); err.imgurl = p.pagesource; err.source = p.source; err.errtype = ErrPage.限制访问; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } }
public void Execute(IJobExecutionContext context) { //string keystr = "meyJjb21pYyI6eyJpZCI6NTM4NzQzLCJ0aXRsZSI6Ilx1NGUwN1x1OTFjY1x1NjY3NFx1NWRkZCIsImNvbGxlY3QiOiI0NzA0MSIsImlzSmFwYW5Db21pYyI6ZmFsc2UsImlzTGlnaHROb3ZlbCI6ZmFsc2UsImlzTGlnaHRDb21pYyI6ZmFsc2UsImlzRmluaXNoIjpmYWxzZSwiaXNSb2FzdGFibGUiOnRydWUsImVJZCI6IktsQlBTVU5BVmxWV0Jnc2ZBUVlPQUF3S0hFUldOQT09In0sImNoYXB0ZXIiOnsiY2lkIjo0OSwiY1RpdGxlIjoiXHU3YjJjMjVcdTU2ZGUgXHU2NDk1XHU4OGMyMlx1ZmYwOFx1NGUwYVx1ZmYwOSIsImNTZXEiOiI0OSIsInZpcFN0YXR1cyI6MiwicHJldkNpZCI6NDgsIm5leHRDaWQiOjUwLCJibGFua0ZpcnN0IjoxLCJjYW5SZWFkIjp0cnVlfSwicGljdHVyZSI6W3sicGlkIjoiMjY1MCIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NTgmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF8wY2E3NjllMDk1YTU0NTYwYzQ5Nzk3ZmEwNGY1NzQ5N18yNjUwLmpwZyJ9LHsicGlkIjoiMjY1MSIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NzImZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF8yZmVmYmQ0Mzk3ODY5YmU5MGI4OTQxOTdmYTEwMGZmNF8yNjUxLmpwZyJ9LHsicGlkIjoiMjY1MiIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NzImZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9jODBiMTE0ZDIwOGIzZWMyZDkxNWM3MmRhODIwZDRhYl8yNjUyLmpwZyJ9LHsicGlkIjoiMjY1MyIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NzImZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9kYjI5MjI5MmM4NmVkN2U0MDIwN2Q2MDg4YjU2ODliYl8yNjUzLmpwZyJ9LHsicGlkIjoiMjY1NCIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NzImZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF81YjEyOWVmMWJjY2NlZDQ3OWU1NDE3NTliMTIzZWJkNF8yNjU0LmpwZyJ9LHsicGlkIjoiMjY1NSIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2NzImZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF85MGY2MTg2MjMyNzcxYTE5NzVmOTVlM2RiYTQ3ZGIxZV8yNjU1LmpwZyJ9LHsicGlkIjoiMjY1NiIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9mZGVmMjExZmIxZjRlYWIxNGQzNDE1MjM1ZDNjOWIxNV8yNjU2LmpwZyJ9LHsicGlkIjoiMjY1NyIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9lNmQxZjAwNWQ1OTkxNzRkMjJlODI1MDE3MGVhMTM4N18yNjU3LmpwZyJ9LHsicGlkIjoiMjY1OCIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF81NmI3YjI1YjVkZDM5YzdhNmU1OGNhYjY1Njc1MGY3MV8yNjU4LmpwZyJ9LHsicGlkIjoiMjY1OSIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9jODEyNGJiYzk2NTVlODVhMTk0ZTRiNDQwMzA2YTFjYV8yNjU5LmpwZyJ9LHsicGlkIjoiMjY2MCIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF85ZDE0ZDYzOTY1Mjg0ZDUzY2MwMDkwN2QxZGIzY2YzNV8yNjYwLmpwZyJ9LHsicGlkIjoiMjY2MSIsIndpZHRoIjo4MDAsImhlaWdodCI6MTIwMCwidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODYmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF9lZmU5OThmYjU1ZDJlZjYyMGFlMmIzYmM3ZjYyY2IzOV8yNjYxLmpwZyJ9LHsicGlkIjoiMjY2MiIsIndpZHRoIjo4NjAsImhlaWdodCI6MTQ1MywidXJsIjoiaHR0cDpcL1wvYWMudGMucXEuY29tXC9zdG9yZV9maWxlX2Rvd25sb2FkP2J1aWQ9MTUwMTcmdWluPTE1MTA3MDk2ODcmZGlyX3BhdGg9XC8mbmFtZT0xNV8wOV8zNF84NmYzY2QzMzgzYjA2N2JhOTRjYWQ4YjZlOThkMGY2NV8yNjYyLmpwZyJ9XSwiYWRzIjp7InRvcCI6eyJ0aXRsZSI6Ilx1NjVmNlx1N2E3YVx1NjA0Ylx1NGViYSIsInBpYyI6Imh0dHBzOlwvXC9tYW5odWEucXBpYy5jblwvb3BlcmF0aW9uXC8wXC8xM18wOV81NF9kOTU4YWYyOGU4NmYzNjhmYjhiN2EwMGMzMDZmODQ4OV8xNTEwNTM4MDQxNTMwLmpwZ1wvMCIsInVybCI6Imh0dHA6XC9cL2FjLnFxLmNvbVwvQ29taWNWaWV3XC9pbmRleFwvaWRcLzU1MDkyMVwvY2lkXC80Iiwid2lkdGgiOiI2NTAiLCJoZWlnaHQiOiIxMTAifSwibGVmdCI6W10sImJvdHRvbSI6eyJ0aXRsZSI6Ilx1NzNiMFx1NGUxNlx1NjBjNVx1NGViYVx1NjYyZlx1NWMzZVx1NzJkMCIsInBpYyI6Imh0dHBzOlwvXC9tYW5odWEucXBpYy5jblwvb3BlcmF0aW9uXC8wXC8xM18wOV81Ml84ZWJjZTJkZmJhMzZhZDhlYTA2YjEyODRkNzY1Mzc0Nl8xNTEwNTM3OTM3NDY2LmpwZ1wvMCIsInVybCI6Imh0dHA6XC9cL2FjLnFxLmNvbVwvQ29taWNWaWV3XC9pbmRleFwvaWRcLzYyNjg1MFwvY2lkXC8yIiwid2lkdGgiOiI2NTAiLCJoZWlnaHQiOiIxMTAifX0sImFydGlzdCI6eyJhdmF0YXIiOiJodHRwOlwvXC9xNC5xbG9nby5jblwvZz9iPXFxJms9d2g1c0ZOazAzUDV6RmlhOGljdGQ4ZW13JnM9NjQwJnQ9MTQ4MzMzOTY3NSIsIm5pY2siOiJcdTc3ZTVcdTk3ZjNcdTc5ZmJcdTUyYThcdTY1NzBcdTViNTciLCJ1aW5DcnlwdCI6IlVUSk1kVkpCZDAxTlFrSkVibGhzWmpOWlJqRjBaejA5In19"; //var ss = DecodeHelper.QQPageDecode(keystr.Substring(1)); //var tt = JsonHelper.DeserializeJsonToObject<QQ_Page_Api>(ss); //var redis = new RedisProxy(); //try //{ // redis.HashSet("QQ_Chapter", "185158", "185157"); //} //catch (Exception ex) //{ // throw; //} //var login_key = redis.Get<QQ_Login_Key>("QQ_1434299101"); DateTime dt = DateTime.Now; string ticks = dt.Ticks.ToString(); string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>();//x.comicid == "1_622585" IQuery <VIPFreeComic> vfcq = dbcontext.Query <VIPFreeComic>(); List <Chapter> cplst; if (QQChapterTotal > 0) { cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.待处理链接 && x.isvip == "0" && x.Id % QQChapterTotal == QQChapterIndex).Take(200).ToList(); } else { cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.待处理链接 && x.isvip == "0").Take(200).ToList(); } List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { ticks = ticks, downstatus = DownChapter.处理完链接, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var cp in cplst) { try { //Dictionary<string, string> headers = new Dictionary<string, string>() //{ // {"Cookie","uin=o1434299101; skey="+login_key.skey+";" } //}; var imgdata = _helper.Get(cp.chapterurl, Encoding.GetEncoding("UTF-8"), null, "", null, "", null, "application/x-www-form-urlencoded; charset=UTF-8"); Regex rex = new Regex("var DATA = '(?<key1>.*?)',"); Match match = rex.Match(imgdata); string key = match.Groups["key1"].Value; if (string.IsNullOrEmpty(key)) { cp.downstatus = DownChapter.待处理链接; cp.modify = dt; cp.ticks = ticks; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = "DATA解析失败"; err = dbcontext.Insert(err); continue; } var s = DecodeHelper.QQPageDecode(key.Substring(1)); var t = JsonHelper.DeserializeJsonToObject <QQ_Page_Api>(s); if (t.chapter.canRead) { List <Page> pglst = new List <Page>(); for (int i = 0; i < t.picture.Count; i++) { pglst.Add(new Page() { chapterid = cp.chapterid, modify = dt, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", ticks = ticks, pagesource = t.picture[i].url }); } cp.isvip = "0"; cp.ticks = ticks; cp.downstatus = DownChapter.处理完链接; cp.modify = dt; dbcontext.Update(cp); dbcontext.BulkInsert(pglst); } else { List <VIPFreeComic> vfclst = vfcq.Where(x => x.source == Source.QQ && x.comicid == cp.comicid).ToList(); cp.isvip = vfclst.Count == 0 ? "1" : "0"; cp.ticks = ticks; cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); } } catch (Exception ex) { cp.downstatus = DownChapter.待处理链接; cp.modify = dt; cp.ticks = ticks; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Chapter> cplst = cpq.Where(x => x.source == Source.QQ && x.downstatus == DownChapter.待处理链接).Take(200).ToList(); List <int> ids = cplst.Select(x => x.Id).ToList(); dbcontext.Update <Chapter>(a => ids.Contains(a.Id), a => new Chapter() { downstatus = DownChapter.处理完链接, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var cp in cplst) { try { string chapterpage = cp.chapterurl.Replace("http://ac.qq.com", ""); var imgdata = _helper.Get(null, chapterpage); Regex rex = new Regex("var DATA = '(?<key1>.*?)',"); Match match = rex.Match(imgdata); string key = match.Groups["key1"].Value; if (string.IsNullOrEmpty(key)) { cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(dt); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = "DATA解析失败"; err = dbcontext.Insert(err); continue; } var s = DecodeHelper.QQPageDecode(key.Substring(1)); var t = JsonHelper.DeserializeJsonToObject <QQ_Page_Api>(s); if (t.chapter.vipStatus == 1) { List <Page> pglst = new List <Page>(); for (int i = 0; i < t.picture.Count; i++) { pglst.Add(new Page() { chapterid = cp.Id, modify = dt, shortdate = shortdate, sort = i + 1, source = cp.source, pagelocal = "", pagesource = t.picture[i].url }); } dbcontext.Update(cp); dbcontext.BulkInsert(pglst); } } catch (Exception ex) { logger.Error(ex.Message); cp.downstatus = DownChapter.待处理链接; cp.modify = dt; dbcontext.Update(cp); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = cp.chapterurl; err.source = cp.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string ticks = dt.Ticks.ToString(); string shortdate = dt.ToString("yyyy-MM-dd"); string updatedatetime = shortdate + " " + ((dt.Hour / 0.5 + 1) * 0.5).ToString(); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); IQuery <PageHis> phisq = dbcontext.Query <PageHis>(); IQuery <Page> pq = dbcontext.Query <Page>(); IQuery <Notice> nq = dbcontext.Query <Notice>(); List <Comic> comiclst = q.Where(a => a.source == Source.dongmanmanhua && (a.updatedatetime == null || a.updatedatetime != updatedatetime)).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { updatedatetime = updatedatetime, modify = dt }); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.comicid && a.source == Source.dongmanmanhua).ToList(); List <Chapter> chapterlst = new List <Chapter>(); if (cplst.Count > 0) { try { string bookurl = comic.bookurl.Replace("https://www.dongmanmanhua.cn/", ""); var bookdata = _helper.Get(null, bookurl); string pattern = "<li id=\"episode_(?<key1>.*?)\" data-episode-no=\"(?<key2>.*?)\">(?<key3>.*?)</li>"; MatchCollection matches = Regex.Matches(bookdata, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); int pagecount = (int)Math.Ceiling(int.Parse(matches[0].Groups[1].Value) / 10.0); for (int i = 0; i < matches.Count; i++) { var lihtml = matches[i].Groups["key3"].Value; Regex reg1 = new Regex("href=\"(?<key1>.*?)\""); Match match1 = reg1.Match(lihtml); string chapterurl = match1.Groups["key1"].Value; Regex reg2 = new Regex("<span class=\"tx\">#(?<key1>.*?)</span>"); Match match2 = reg2.Match(lihtml); int sort = int.Parse(match2.Groups["key1"].Value); //Regex reg3 = new Regex("<span class=\"subj\"><span>(?<key1>.*?)</span></span>"); //Match match3 = reg3.Match(lihtml); //string chaptername = match3.Groups["key1"].Value; Regex reg3 = new Regex("alt=\"(?<key1>.*?)\""); Match match3 = reg3.Match(lihtml); string chaptername = match3.Groups["key1"].Value; Regex reg4 = new Regex("src=\"(?<key1>.*?)\""); Match match4 = reg4.Match(lihtml); string chaptersource = match4.Groups["key1"].Value; chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + sort, chaptername = chaptername, chapterurl = "https:" + chapterurl, sort = sort, comicid = comic.comicid, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = chaptersource, chapterlocal = "", modify = dt, shortdate = shortdate, }); } for (int i = 2; i <= pagecount; i++) { var bookdata2 = _helper.Get(null, bookurl + "&page=" + i); string pattern2 = "<li id=\"episode_(?<key1>.*?)\" data-episode-no=\"(?<key2>.*?)\">(?<key3>.*?)</li>"; MatchCollection matches2 = Regex.Matches(bookdata2, pattern2, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int j = 0; j < matches2.Count; j++) { var lihtml = matches2[j].Groups["key3"].Value; Regex reg1 = new Regex("href=\"(?<key1>.*?)\""); Match match1 = reg1.Match(lihtml); string chapterurl = match1.Groups["key1"].Value; Regex reg2 = new Regex("<span class=\"tx\">#(?<key1>.*?)</span>"); Match match2 = reg2.Match(lihtml); int sort = int.Parse(match2.Groups["key1"].Value); //Regex reg3 = new Regex("<span class=\"subj\"><span>(?<key1>.*?)</span></span>"); //Match match3 = reg3.Match(lihtml); //string chaptername = match3.Groups["key1"].Value; Regex reg3 = new Regex("alt=\"(?<key1>.*?)\""); Match match3 = reg3.Match(lihtml); string chaptername = match3.Groups["key1"].Value; Regex reg4 = new Regex("src=\"(?<key1>.*?)\""); Match match4 = reg4.Match(lihtml); string chaptersource = match4.Groups["key1"].Value; chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + sort, chaptername = chaptername, chapterurl = "https:" + chapterurl, sort = sort, comicid = comic.comicid, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = chaptersource, chapterlocal = "", modify = dt, shortdate = shortdate, }); } } bool contextChange = false; if (chapterlst.Count > 0) { chapterlst = chapterlst.OrderBy(x => x.sort).ToList(); int cmpCount = cplst.Count >= chapterlst.Count ? chapterlst.Count : cplst.Count; for (int i = 0; i < cmpCount; i++) { if (cplst[i].chapterid != chapterlst[i].chapterid) { contextChange = true; } } } else { continue; } if (contextChange) { dbcontext.Delete <Chapter>(x => x.comicid == comic.comicid); if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); Notice notice = new Notice(); notice.noticeid = comic.comicid; notice.noticestatus = NoticeStatus.等待处理; notice.noticetype = NoticeType.目录变更; notice.source = comic.source; notice.shortdate = shortdate; notice.modify = dt; //var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); //if (nqwait == null) //{ dbcontext.Insert(notice); //} } } else { if (cplst.Count == chapterlst.Count) { continue; } if (cplst.Count > chapterlst.Count) { List <Chapter> needrmvlst = cplst.Except(chapterlst, new Chapter_Comparer()).ToList(); List <int> rmvidlst = needrmvlst.Select(x => x.Id).ToList(); logger.Error(string.Join(",", needrmvlst.Select(x => x.chapterid).ToArray())); dbcontext.Delete <Chapter>(x => rmvidlst.Contains(x.Id)); } if (cplst.Count < chapterlst.Count) { List <Chapter> needaddlst = chapterlst.Except(cplst, new Chapter_Comparer()).ToList(); dbcontext.BulkInsert(needaddlst); } } //int delete = cplst.Except(chapterlst, new Chapter_Comparer()).Count(); // 删章 //List<Chapter> add = chapterlst.Except(cplst, new Chapter_Comparer()).ToList(); // 新增 //Chapter maxChapter = cplst.OrderBy(x => x.sort).Max(); //if (delete > 0) //{ // List<string> idlst = cplst.Select(x => x.chapterid).ToList(); // dbcontext.Delete<Page>(x => idlst.Contains(x.chapterid)); // dbcontext.Delete<Chapter>(x => idlst.Contains(x.chapterid)); // if (chapterlst.Count > 0) // { // dbcontext.BulkInsert(chapterlst); // } // Notice notice = new Notice(); // notice.noticeid = comic.comicid; // notice.noticestatus = NoticeStatus.等待处理; // notice.noticetype = NoticeType.目录变更; // notice.source = comic.source; // notice.shortdate = shortdate; // notice.modify = dt; // var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); // if (nqwait == null) // { // dbcontext.Insert(notice); // } // continue; //} //else //{ // List<Chapter> mvadd = chapterlst.Except(add, new Chapter_Comparer()).ToList(); // string cplststr = string.Join(",", cplst.Select(x => x.chapterid).ToArray()); // string chapterlststr = string.Join(",", mvadd.Select(x => x.chapterid).ToArray()); // if (cplststr != chapterlststr) // 调序 // { // List<string> idlst = cplst.Select(x => x.chapterid).ToList(); // dbcontext.Delete<Page>(x => idlst.Contains(x.chapterid)); // dbcontext.Delete<Chapter>(x => idlst.Contains(x.chapterid)); // if (chapterlst.Count > 0) // { // dbcontext.BulkInsert(chapterlst); // } // Notice notice = new Notice(); // notice.noticeid = comic.comicid; // notice.noticestatus = NoticeStatus.等待处理; // notice.noticetype = NoticeType.目录变更; // notice.source = comic.source; // notice.shortdate = shortdate; // notice.modify = dt; // var nqwait = nq.Where(x => x.noticeid == comic.comicid && x.noticestatus == NoticeStatus.等待处理 && x.noticetype == NoticeType.目录变更).FirstOrDefault(); // if (nqwait == null) // { // dbcontext.Insert(notice); // } // continue; // } //} //if (add.Count > 0) //{ // //int addsort = cplst.Count(); // //foreach (var item in add) // //{ // // item.sort = item.sort + addsort; // //} // dbcontext.BulkInsert(add); //} } catch (Exception ex) { logger.Error(ex.Message); comic.updatedatetime = ""; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } }
public void Execute(IJobExecutionContext context) { DateTime dt = DateTime.Now; string shortdate = dt.ToString("yyyy-MM-dd"); string yesterday = dt.AddDays(-1).ToString("yyyy-MM-dd"); IQuery <Comic> q = dbcontext.Query <Comic>(); IQuery <Chapter> cpq = dbcontext.Query <Chapter>(); List <Comic> comiclst = q.Where(a => a.source == Source.QQ && a.shortdate == shortdate).Take(200).ToList(); List <int> ids = comiclst.Select(x => x.Id).ToList(); dbcontext.Update <Comic>(a => ids.Contains(a.Id), a => new Comic() { shortdate = yesterday, modify = dt }); List <Chapter> chapterlst = new List <Chapter>(); foreach (var comic in comiclst) { List <Chapter> cplst = cpq.Where(a => a.comicid == comic.Id && a.source == Source.QQ).ToList(); if (cplst.Count == 0) { try { string bookurl = comic.bookurl.Replace("http://ac.qq.com/", ""); var bookdata = _helper.Get(null, bookurl); //class=\"chapter-page-all works-(?<key1>.*?)-list\" Regex reg1 = new Regex("<ol class=\"chapter-page-all works-chapter-list\">(?<key1>.*?)</ol>", RegexOptions.IgnoreCase | RegexOptions.Singleline); Match match1 = reg1.Match(bookdata); string htmlSource = match1.Groups["key1"].Value; string pattern = "<a target=\"_blank\" title=\"(?<key1>.*?)\" href=\"(?<key2>.*?)\">(?<key3>.*?)</a>"; MatchCollection matches = Regex.Matches(htmlSource, pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline); for (int i = 0; i < matches.Count; i++) { chapterlst.Add(new Chapter() { chapterid = comic.comicid + "_" + matches[i].Groups["key2"].Value.Split('/').LastOrDefault(), chaptername = matches[i].Groups["key3"].Value.Trim(), chapterurl = "http://ac.qq.com" + matches[i].Groups["key2"].Value, sort = i + 1, comicid = comic.Id, retry = 0, source = comic.source, downstatus = DownChapter.待处理链接, isvip = "0", chaptersource = "", chapterlocal = "", modify = dt, shortdate = shortdate, }); } } catch (Exception ex) { logger.Error(ex.Message); comic.shortdate = shortdate; comic.modify = dt; dbcontext.Update(comic); Err_ChapterJob err = new Err_ChapterJob(); err.bookurl = comic.bookurl; err.source = comic.source; err.errtype = ErrChapter.解析出错; err.modify = dt; err.shortdate = shortdate; err.message = ex.Message; err = dbcontext.Insert(err); continue; } } } if (chapterlst.Count > 0) { dbcontext.BulkInsert(chapterlst); } }