/// <summary>
/// 批量插入壁纸
/// </summary>
/// <param name="input"></param>
/// <returns></returns>
public async Task <ServiceResult <string> > BulkInsertWallpaperAsync(BulkInsertWallpaperInput input)
{
var result = new ServiceResult <string>();
if (!input.Wallpapers.Any())
{
result.IsFailed(ResponseText.DATA_IS_NONE);
return(result);
}
var urls = _wallpaperRepository.Where(x => x.Type == (int)input.Type).Select(x => x.Url).ToList();
var wallpapers = ObjectMapper.Map <IEnumerable <WallpaperDto>, IEnumerable <Domain.Wallpaper.Wallpaper> >(input.Wallpapers)
.Where(x => !urls.Contains(x.Url));
foreach (var item in wallpapers)
{
item.Type = (int)input.Type;
item.CreateTime = item.Url.Split("/").Last().Split("_").First().TryToDateTime();
}
await _wallpaperRepository.BulkInsertAsync(wallpapers);
result.IsSuccess(ResponseText.INSERT_SUCCESS);
return(result);
}
public async Task ExecuteAsync()
{
var wallpaperUrls = new List <WallpaperJobItem <string> >
{
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_1_1.html", Type = WallpaperEnum.Beauty
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_58_1.html", Type = WallpaperEnum.Sportsman
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_66_1.html", Type = WallpaperEnum.CuteBaby
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_4_1.html", Type = WallpaperEnum.Emotion
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_3_1.html", Type = WallpaperEnum.Landscape
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_9_1.html", Type = WallpaperEnum.Animal
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_13_1.html", Type = WallpaperEnum.Plant
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_64_1.html", Type = WallpaperEnum.Food
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_11_1.html", Type = WallpaperEnum.Movie
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_5_1.html", Type = WallpaperEnum.Anime
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_34_1.html", Type = WallpaperEnum.HandPainted
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_65_1.html", Type = WallpaperEnum.Text
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_2_1.html", Type = WallpaperEnum.Creative
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_10_1.html", Type = WallpaperEnum.Car
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_14_1.html", Type = WallpaperEnum.PhysicalEducation
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_63_1.html", Type = WallpaperEnum.Military
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_17_1.html", Type = WallpaperEnum.Festival
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_15_1.html", Type = WallpaperEnum.Game
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_12_1.html", Type = WallpaperEnum.Apple
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_7_1.html", Type = WallpaperEnum.Other
}
};
var web = new HtmlWeb();
var list_task = new List <Task <WallpaperJobItem <HtmlDocument> > >();
wallpaperUrls.AsParallel().ForAll(item =>
{
var task = Task.Run(async() =>
{
var htmlDocument = await web.LoadFromWebAsync(item.Result);
return(new WallpaperJobItem <HtmlDocument>
{
Result = htmlDocument,
Type = item.Type
});
});
list_task.Add(task);
});
Task.WaitAll(list_task.ToArray());
var wallpapers = new List <Domain.Wallpaper.Wallpaper>();
foreach (var list in list_task)
{
var item = await list;
var imgs = item.Result.DocumentNode.SelectNodes("//article[@id='wper']/div[@class='jbox']/div[@class='kbox']/div/a/img[1]").ToList();
imgs.AsParallel().ForAll(x =>
{
wallpapers.Add(new Domain.Wallpaper.Wallpaper
{
Url = x.GetAttributeValue("data-big", ""),
Title = x.GetAttributeValue("title", ""),
Type = (int)item.Type,
CreateTime = x.Attributes["data-big"].Value.Split("/").Last().Split("_").First().TryToDateTime()
});
});
}
var urls = (await _wallpaperRepository.GetListAsync()).Select(x => x.Url);
wallpapers = wallpapers.Where(x => !urls.Contains(x.Url)).ToList();
if (wallpapers.Any())
{
await _wallpaperRepository.BulkInsertAsync(wallpapers);
}
}
/// <summary>
/// 壁纸数据抓取
/// </summary>
/// <returns></returns>
public async Task RunAsync()
{
try
{
LoggerHelper.Write($"壁纸数据抓取 {DateTime.Now:yyyy-MM-dd HH:mm:ss}");
var wallpaperUrls = new List <WallpaperJobItem <string> >
{
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_1_1.html", Type = WallpaperEnum.Beauty
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_58_1.html", Type = WallpaperEnum.Sportsman
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_66_1.html", Type = WallpaperEnum.CuteBaby
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_4_1.html", Type = WallpaperEnum.Emotion
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_3_1.html", Type = WallpaperEnum.Landscape
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_9_1.html", Type = WallpaperEnum.Animal
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_13_1.html", Type = WallpaperEnum.Plant
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_64_1.html", Type = WallpaperEnum.Food
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_11_1.html", Type = WallpaperEnum.Movie
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_5_1.html", Type = WallpaperEnum.Anime
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_34_1.html", Type = WallpaperEnum.HandPainted
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_65_1.html", Type = WallpaperEnum.Text
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_2_1.html", Type = WallpaperEnum.Creative
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_10_1.html", Type = WallpaperEnum.Car
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_14_1.html", Type = WallpaperEnum.PhysicalEducation
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_63_1.html", Type = WallpaperEnum.Other
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_17_1.html", Type = WallpaperEnum.Military
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_15_1.html", Type = WallpaperEnum.Festival
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_12_1.html", Type = WallpaperEnum.Game
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_7_1.html", Type = WallpaperEnum.Apple
}
};
var web = new HtmlWeb();
var list_task = new List <Task <WallpaperJobItem <HtmlDocument> > >();
//var proxy = await _redis.ZRandomAsync();
//var hasProxy = !string.IsNullOrWhiteSpace(proxy);
wallpaperUrls.ForEach(item =>
{
var task = Task.Run(async() =>
{
var obj = new HtmlDocument();
//if (hasProxy)
//{
// WebClient wc = new WebClient();
// wc.Proxy = new WebProxy(proxy);
// obj.LoadHtml(await wc.DownloadStringTaskAsync(item.Result));
//}
//else
obj = await web.LoadFromWebAsync(item.Result);
return(new WallpaperJobItem <HtmlDocument>
{
Result = obj,
Type = item.Type
});
});
list_task.Add(task);
});
Task.WaitAll(list_task.ToArray());
var wallpapers = new List <Wallpaper>();
foreach (var list in list_task)
{
var item = await list;
var imgs = item.Result.DocumentNode.SelectNodes("//article[@id='wper']/div[@class='jbox']/div[@class='kbox']/div/a/img[1]").ToList();
imgs.ForEach(x =>
{
wallpapers.Add(new Wallpaper
{
Url = x.GetAttributeValue("data-big", ""),
Title = x.GetAttributeValue("title", ""),
Type = (int)item.Type,
CreateTime = x.Attributes["data-big"].Value.Split("/").Last().Split("_").First().TryToDateTime()
});
});
}
var urls = _wallpaperRepository.GetListAsync().Result.Select(x => x.Url);
wallpapers = wallpapers.Where(x => !urls.Contains(x.Url)).ToList();
if (wallpapers.Any())
{
await _wallpaperRepository.BulkInsertAsync(wallpapers);
_ = EmailAsync(wallpapers.Count());
}
LoggerHelper.Write($"壁纸数据抓取 hasProxy= 本次抓取到{wallpapers.Count()}条数据，时间:{DateTime.Now:yyyy-MM-dd HH:mm:ss}");
}
catch (Exception ex)
{
LoggerHelper.Write(ex, $"壁纸数据抓取 异常：WallpaperJob本次抓取异常, {DateTime.Now:yyyy -MM-dd HH:mm:ss}");
}
}
/// <summary>
/// 背景图片数据抓取
/// </summary>
/// <returns></returns>
public async Task RunAsync()
{
try
{
LoggerHelper.Write($"背景图片数据抓取 {DateTime.Now:yyyy-MM-dd HH:mm:ss}");
var per = 80;
var pageUrls = new List <string>
{
$"https://api.pexels.com/v1/search?query=sexy&per_page={per}&page={new Random().Next(1, 100)}",
$"https://api.pexels.com/v1/search?query=beauty&per_page={per}&page={new Random().Next(1, 100)}",
$"https://api.pexels.com/v1/search?query=basketball&per_page={per}&page={new Random().Next(1, 100)}",
$"https://api.pexels.com/v1/search?query=china&per_page={per}&page={new Random().Next(1, 100)}",
$"https://api.pexels.com/v1/search?query=dog&per_page={per}&page={new Random().Next(1, 100)}",
$"https://api.pexels.com/v1/search?query=science&per_page={per}&page={new Random().Next(1, 100)}",
$"https://api.pexels.com/v1/search?query=green&per_page={per}&page={new Random().Next(1, 100)}",
$"https://api.pexels.com/v1/search?query=sport&per_page={per}&page={new Random().Next(1, 100)}",
$"https://api.pexels.com/v1/search?query=phone&per_page={per}&page={new Random().Next(1, 100)}",
$"https://api.pexels.com/v1/search?query=car&per_page={per}&page={new Random().Next(1, 100)}",
$"https://api.pexels.com/v1/search?query=lingerie&per_page={per}&page={new Random().Next(1, 100)}",
$"https://api.pexels.com/v1/search?query=pretty&per_page={per}&page={new Random().Next(1, 100)}",
$"https://api.pexels.com/v1/search?query=couple&per_page={per}&page={new Random().Next(1, 100)}",
};
var list_task = new List <Task <HtmlDocument> >();
var web = new HtmlWeb();
pageUrls.ForEach(item =>
{
var task = Task.Run(async() =>
{
using var client = _httpClient.CreateClient();
client.DefaultRequestHeaders.Add("Authorization", "563492ad6f917000010000011c4e23f0bba54da2b5fe830bf7121898");
var httpResponse = await client.GetAsync(item);
var obj = await httpResponse.Content.ReadAsStringAsync();
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(obj);
return(doc);
});
list_task.Add(task);
});
Task.WaitAll(list_task.ToArray());
var wallpapers = new List <Wallpaper>();
foreach (var list in list_task)
{
var item = await list;
try
{
var obj = JObject.Parse(item.ParsedText);
var nodes = obj["photos"];
foreach (var node in nodes)
{
wallpapers.Add(new Wallpaper
{
Url = node["src"]["large2x"].ToString(),
Title = node["photographer"].ToString(),
Type = (int)WallpaperEnum.BgImage,
CreateTime = DateTime.Now
});
}
}
catch (Exception ex)
{
LoggerHelper.Write(ex, $"背景图片数据抓取 list_task异常：本次抓取异常 {item} ");
}
}
var urls = _wallpaperRepository.GetListAsync().Result.Select(x => x.Url);
wallpapers = wallpapers.Where(x => !urls.Contains(x.Url)).ToList();
if (wallpapers.Any())
{
await _wallpaperRepository.BulkInsertAsync(wallpapers);
}
_ = SendingAsync(wallpapers.Count());
LoggerHelper.Write($"背景图片数据抓取 本次抓取到{wallpapers.Count()}条数据，时间:{DateTime.Now:yyyy-MM-dd HH:mm:ss}.");
}
catch (Exception ex)
{
LoggerHelper.Write(ex, $"背景图片数据抓取 异常：HotNewsJob本次抓取异常, {DateTime.Now:yyyy -MM-dd HH:mm:ss}");
}
}
public async Task ExecuteAsync()
{
var wallpaperUrls = new List <WallpaperJobItem <string> >
{
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_1_1.html", Type = WallpaperEnum.Beauty
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_58_1.html", Type = WallpaperEnum.Sportsman
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_66_1.html", Type = WallpaperEnum.CuteBaby
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_4_1.html", Type = WallpaperEnum.Emotion
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_3_1.html", Type = WallpaperEnum.Landscape
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_9_1.html", Type = WallpaperEnum.Animal
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_13_1.html", Type = WallpaperEnum.Plant
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_64_1.html", Type = WallpaperEnum.Food
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_11_1.html", Type = WallpaperEnum.Movie
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_5_1.html", Type = WallpaperEnum.Anime
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_34_1.html", Type = WallpaperEnum.HandPainted
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_65_1.html", Type = WallpaperEnum.Text
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_2_1.html", Type = WallpaperEnum.Creative
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_10_1.html", Type = WallpaperEnum.Car
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_14_1.html", Type = WallpaperEnum.PhysicalEducation
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_63_1.html", Type = WallpaperEnum.Military
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_17_1.html", Type = WallpaperEnum.Festival
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_15_1.html", Type = WallpaperEnum.Game
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_12_1.html", Type = WallpaperEnum.Apple
},
new WallpaperJobItem <string> {
Result = "https://www.i4.cn/wper_4_19_7_1.html", Type = WallpaperEnum.Other
}
};
var web = new HtmlWeb();
var list_task = new List <Task <WallpaperJobItem <HtmlDocument> > >();
wallpaperUrls.ForEach(item =>
{
var task = Task.Run(async() =>
{
var htmlDocument = await web.LoadFromWebAsync(item.Result);
return(new WallpaperJobItem <HtmlDocument>
{
Result = htmlDocument,
Type = item.Type
});
});
list_task.Add(task);
});
Task.WaitAll(list_task.ToArray());
var wallpapers = new List <Domain.Wallpaper.Wallpaper>();
foreach (var list in list_task)
{
var item = await list;
var imgs = item.Result.DocumentNode.SelectNodes("//*[@id=\"wper\"]/div/div[3]/div/a/img[1]").ToList();
imgs.ForEach(x =>
{
wallpapers.Add(new Domain.Wallpaper.Wallpaper
{
Url = x.GetAttributeValue("data-big", ""),
Title = x.GetAttributeValue("title", ""),
Type = (int)item.Type,
CreateTime = x.Attributes["data-big"].Value.Split("/").Last().Split("_").First().TryToDateTime()
});
});
}
var urls = (await _wallpaperRepository.GetListAsync()).Select(x => x.Url);
wallpapers = wallpapers.Where(x => !urls.Contains(x.Url)).ToList();
if (wallpapers.Any())
{
await _wallpaperRepository.BulkInsertAsync(wallpapers);
}
// 发送Email
var message = new MimeMessage
{
Subject = "【定时任务】壁纸数据抓取任务推送",
Body = new BodyBuilder
{
HtmlBody = $"本次抓取到{wallpapers.Count()}条数据，时间:{DateTime.Now:yyyy-MM-dd HH:mm:ss}"
}.ToMessageBody()