private static async Task prepareImage(CommonDbContext dbContext, ContentEntry chapterContent, Stream imgStream, string imgName)
{
var mediaImg = chapterContent.MediaResource?.FirstOrDefault(x => x.Name == imgName);
if (mediaImg == null)
{
if (chapterContent.MediaResource == null)
{
chapterContent.MediaResource = new List <FileEntry>();
}
//图片判重
if (chapterContent.MediaResource.Any(x => x.Name == imgName))
{
return;
}
imgStream.Position = 0;
QiniuTool.UploadImage(imgStream, imgName).Wait();
chapterContent.MediaResource.Add(new FileEntry
{
ActualPath = "https://mioto.milbit.com/" + imgName,
Name = imgName,
CreateTime = DateTime.Now,
Id = Guid.NewGuid()
});
await dbContext.SaveChangesAsync();
}
}
static string ProcessFile(string filePath, string dirName)
{
var extention = Path.GetExtension(filePath);
var fileName = "cartoon/" + Path.GetRandomFileName();
switch (extention)
{
case ".zip":
var outpuDirPaht = ZipTool.Exact(filePath);
AnalyseDic(outpuDirPaht, dirName);
break;
case ".pdf":
var outpuStream = PDFTool.PDFToImage(filePath);
//QiniuTool.UploadImage(outpuStream, fileName).Wait();
break;
case ".jpg":
case ".jpeg":
QiniuTool.UploadImage(filePath, fileName).Wait();
break;
}
return(fileName);
}
public void Start()
{
var allNum = dbcontext.Set <FileEntry>().Where(x => x.ActualPath.Contains("img.fox800.xyz")).Count();
var wc = new WebClient();
for (int i = 0; i < allNum;)
{
var targetRow = dbcontext.Set <FileEntry>().Where(x => x.ActualPath.Contains("img.fox800.xyz")).Skip(i).Take(100).ToList();
foreach (var item in targetRow)
{
try
{
var imgBytes = wc.DownloadData(item.ActualPath);
var imgStream = new MemoryStream(imgBytes);
var savekey = item.ActualPath.Replace("img.fox800.xyz", "fox800/");
QiniuTool.UploadImage(imgStream, savekey).Wait();
item.ActualPath = "http://mioto.milbit.com/" + savekey;
item.Name = "fox800";
dbcontext.SaveChanges();
Console.WriteLine("process : " + i);
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
}
i += 100;
if (i > allNum)
{
i = allNum - 1;
}
}
}
private async Task <string> UploadToQiniu(Stream stream, string fileType = "image", string ext = "")
{
var key = GetFileName(fileType, ext);
await QiniuTool.UploadImage(stream, key);
return(key);
}
private void CreateContentAndUploadImage(string filePath, string cateName, List <Stream> streamList, int contentOrder)
{
var name = Path.GetFileNameWithoutExtension(filePath);
//var cateName = Path.GetDirectoryName(filePath).Split('\\').Last();
contentUploadCreator.CreateContent(name, cateName, contentOrder);
var index = 0;
streamList.ForEach(async imgStream =>
{
var fileName = "cartoon/toptoon/" + cateName + "/" + name + "_" + index + ".jpg";
QiniuTool.UploadImage(imgStream, fileName).Wait();
contentUploadCreator.SetContentIImg(name, cateName, fileName, index);
index++;
});
}
private static void ProcessListPage(IWebElement obj, Dictionary <string, string> State)
{
var exuctor = TaskExutor.Instance;
Console.WriteLine(obj.FindElement(By.CssSelector("body")).GetAttribute("innerHTML"));
var categories = obj.FindElements(By.CssSelector(".ajax-load-con.content"));
var categoryDbContext = GetDbContext();
foreach (var item in categories)
{
var nextUrl = item.FindElement(By.CssSelector("a")).GetAttribute("href");
Console.WriteLine(item.GetAttribute("innerHTML"));
var coverUrl = item.FindElement(By.CssSelector("li .image-item img")).GetAttribute("src");
var categoryTitle = item.FindElement(By.CssSelector(".posts-default-title a")).GetAttribute("title").Trim();
var creatTime = DateTime.Parse(item.FindElement(By.CssSelector(".ico-time")).GetAttribute("innerText"));
var imgStream = DownloadImg(coverUrl);
var coverImgPath = "cartoon/" + categoryTitle + "/coverimg.jpg";
QiniuTool.UploadImage(imgStream, coverImgPath).Wait();
var dbCategory = categoryDbContext.Categories.FirstOrDefault(x => x.Name == categoryTitle);
if (dbCategory == null)
{
var categoryModel = CreateCategory(categoryTitle, creatTime, coverImgPath);
categoryDbContext.Add(categoryModel);
}
var dbContent = categoryDbContext.ContentEntry.FirstOrDefault(x => x.Title == categoryTitle);
if (dbContent == null)
{
dbContent = CreateContent(dbCategory, categoryTitle, creatTime);
categoryDbContext.Add(dbContent);
}
if (dbContent.Category == null)
{
var subCate = categoryDbContext.Categories.FirstOrDefault(x => x.Name == categoryTitle);
dbContent.Category = subCate;
}
categoryDbContext.SaveChanges();
if (!string.IsNullOrEmpty(nextUrl))
{
var state = new Dictionary <string, string>();
var categoriesTaskSpider = new TaskSpider(nextUrl, state);
exuctor.SetTask(categoriesTaskSpider, ProcessImgPage);
}
}
}
private void CreateContentAndUploadImage(string contentDir, string cateName, int contentOrder)
{
var name = contentDir.Split('\\').Last();
var imgList = Directory.GetFiles(contentDir);
//var cateName = contentDir.Split('\\').Last();
contentUploadCreator.CreateContent(name, cateName, contentOrder);
var index = 0;
foreach (var imgItem in imgList)
{
var fileName = "cartoon/toptoon/" + cateName + "/" + name + "_" + index + ".jpg";
QiniuTool.UploadImage(imgItem, fileName).Wait();
contentUploadCreator.SetContentIImg(name, cateName, fileName, index);
index++;
}
}
public async Task <IActionResult> AssetPost(string key)
{
var fileStream = HttpContext.Request.Body;
var reader = await HttpContext.Request.BodyReader.ReadAsync();
var streamReader = new StreamReader(fileStream);
var stream = new MemoryStream();
await fileStream.CopyToAsync(stream);
await QiniuTool.UploadImage(stream, key);
var jobj = JObject.Parse("{}");
jobj.Add("content-name", key);
jobj.Add("status", "ok");
return(Content(jobj.ToString(), "application/json"));
}
public async Task SvaeChapterDetail(CommonDbContext dbCtx, string cateName, string contentName, string fileName, int order)
{
try
{
var content = await dbCtx.ContentEntry.Include(x => x.MediaResource).FirstOrDefaultAsync(x => x.Category.Name == cateName && x.Title == contentName);
var category = await dbCtx.Categories.FirstOrDefaultAsync(x => x.Name == cateName);
if (content.MediaResource == null)
{
content.MediaResource = new List <FileEntry> {
};
}
var name = $"{cateName}/{contentName}/{order}img.jpg";
if (!content.MediaResource.Any(x => x.Name == name))
{
await QiniuTool.UploadImage(fileName, name);
content.MediaResource.Add(new FileEntry
{
Name = name,
ActualPath = "https://mioto.milbit.com/" + name,
CreateTime = DateTime.Now,
Order = order,
Tag = cateName
});
}
await dbCtx.SaveChangesAsync();
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
}
public static async Task UploadQiniu(string url, string key)
{
var imgStream = DownloadImg(url);
var coverImgPath = key;
await QiniuTool.UploadImage(imgStream, coverImgPath);
}
private static void AddCartoon()
{
var dbcontext = buildDbContext();
Console.WriteLine("请输入路径, 并按回车: 然后等待程序退出;");
var path = Console.ReadLine();
//var path = @"C:\Users\v-huamli\Documents\WeChat Files\lihuaming4000\Files\花美男幼儿园1";
//var path = @"G:\SystemDiretory\Downloads\1-22";
var matchRegex = new Regex(@"第(\d+)话");
var topCategories = Directory.EnumerateDirectories(path).ToList();
var topCateEntryList = new List <Categories>();
foreach (var topCate in topCategories)
{
var topCatTmpArr = topCate.Split("\\");
var topCatName = topCatTmpArr[topCatTmpArr.Length - 1];
var topCateEntry = new Categories
{
CreateTime = DateTime.Now,
Id = Guid.NewGuid(),
ChildrenCategories = new List <Categories>(),
Name = topCatName
};
dbcontext.Categories.AddAsync(topCateEntry).Wait();
dbcontext.SaveChangesAsync().Wait();
//漫画名
var secPath = Path.Combine(path, topCate);
//章节名
var secondCategories = Directory.EnumerateDirectories(secPath).OrderBy(x =>
{
var matchVal = matchRegex.Match(x);
var intVal = matchVal.Groups[1].Value;
return(int.Parse(intVal));
}).ToList();
//var contentList = new List<ContentEntry>();
var order = 0;
foreach (var secCate in secondCategories)
{
var secCateTmpArr = secCate.Split("\\");
var secCateName = secCateTmpArr[secCateTmpArr.Length - 1];
//图片名
var thrPath = Path.Combine(secPath, secCate);
var fileNameList = Directory.EnumerateFiles(thrPath).ToList();
var fileEntryList = new List <FileEntry>();
var fileUploadTaskList = new List <Task>();
var contentEntry = new ContentEntry
{
Category = topCateEntry,
CreateTime = DateTime.Now,
Id = Guid.NewGuid(),
Title = secCateName,
Order = order++
};
foreach (var fileName in fileNameList)
{
var saveKey = "cartoon/" + topCatName
+ "-" + secCateName + "-" + Path.GetFileName(fileName);
var task = QiniuTool.UploadImage(fileName, saveKey);
fileEntryList.Add(new FileEntry
{
ActualPath = "https://mioto.milbit.com/" + saveKey,
CreateTime = DateTime.Now,
Name = saveKey,
Tag = saveKey
});
fileUploadTaskList.Add(task);
}
Task.WaitAll(fileUploadTaskList.ToArray());
contentEntry.MediaResource = fileEntryList;
dbcontext.ContentEntry.AddAsync(contentEntry).Wait();
dbcontext.SaveChangesAsync().Wait();
}
}
dbcontext.SaveChangesAsync().Wait();
}
private static void ProcessListPage(IWebElement webElement, Dictionary <string, string> state)
{
try
{
var body = webElement.FindElement(By.TagName("body")).GetAttribute("innerHTML");
Console.WriteLine(body);
var conentList = webElement.FindElements(By.CssSelector(".storyCell"));
//漫画列表
foreach (var contentItem in conentList)
{
var categoryTitle = contentItem.FindElement(By.CssSelector(".rightInfo .title")).GetAttribute("innerText");
if (!zeroList.Any(x => x.Name == categoryTitle))
{
continue;
}
var coverUrl = contentItem.FindElement(By.CssSelector(".leftCover img")).GetAttribute("src");
var url = contentItem.FindElement(By.CssSelector(".leftCover a")).GetAttribute("href");
var desc = contentItem.FindElement(By.CssSelector(".rightInfo .intr")).GetAttribute("innerText");
TaskExutor exuctor = TaskExutor.Instance;
//var state = new Dictionary<string, string>();
var curState = new Dictionary <string, string>();
foreach (var item in state)
{
curState.Add(item.Key, item.Value);
}
curState.Add("cateTitle", categoryTitle);
var contentTaskSpider = new TaskSpider(url, curState);
contentTaskSpider.SetCookie(_cookie);
//Console.WriteLine(title);
//upload qiniu
var imgStream = DownloadImg(coverUrl);
var coverImgPath = "cartoon/" + categoryTitle + "/coverimg.jpg";
QiniuTool.UploadImage(imgStream, coverImgPath).Wait();
var categoryDbContext = GetDbContext();
var hasCategory = categoryDbContext.Categories.AnyAsync(x => x.Name == categoryTitle).Result;
if (!hasCategory)
{
categoryDbContext.Categories.Add(new Categories
{
Id = Guid.NewGuid(),
CreateTime = DateTime.Now,
Name = categoryTitle,
Description = desc.Trim(),
MediaResource = new List <FileEntry> {
new FileEntry {
Id = Guid.NewGuid(),
ActualPath = "https://mioto.milbit.com/" + coverImgPath,
CreateTime = DateTime.Now,
Name = coverImgPath
}
},
Tags = new List <Tags> {
new Tags {
Id = Guid.NewGuid(),
Name = "恋爱"
},
new Tags {
Id = Guid.NewGuid(),
Name = "都市"
},
new Tags {
Id = Guid.NewGuid(),
Name = "Sex"
},
new Tags {
Id = Guid.NewGuid(),
Name = "SpiderUplaod2"
}
}
});
categoryDbContext.SaveChanges();
}
processChapter(categoryTitle, contentTaskSpider);
}
}
catch (Exception exc)
{
throw;
}
}