public void Start()
{
var allNum = dbcontext.Set <FileEntry>().Where(x => x.ActualPath.Contains("img.fox800.xyz")).Count();
var wc = new WebClient();
for (int i = 0; i < allNum;)
{
var targetRow = dbcontext.Set <FileEntry>().Where(x => x.ActualPath.Contains("img.fox800.xyz")).Skip(i).Take(100).ToList();
foreach (var item in targetRow)
{
try
{
var imgBytes = wc.DownloadData(item.ActualPath);
var imgStream = new MemoryStream(imgBytes);
var savekey = item.ActualPath.Replace("img.fox800.xyz", "fox800/");
QiniuTool.UploadImage(imgStream, savekey).Wait();
item.ActualPath = "http://mioto.milbit.com/" + savekey;
item.Name = "fox800";
dbcontext.SaveChanges();
Console.WriteLine("process : " + i);
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
}
i += 100;
if (i > allNum)
{
i = allNum - 1;
}
}
}
private static async Task prepareImage(CommonDbContext dbContext, ContentEntry chapterContent, Stream imgStream, string imgName)
{
var mediaImg = chapterContent.MediaResource?.FirstOrDefault(x => x.Name == imgName);
if (mediaImg == null)
{
if (chapterContent.MediaResource == null)
{
chapterContent.MediaResource = new List <FileEntry>();
}
//图片判重
if (chapterContent.MediaResource.Any(x => x.Name == imgName))
{
return;
}
imgStream.Position = 0;
QiniuTool.UploadImage(imgStream, imgName).Wait();
chapterContent.MediaResource.Add(new FileEntry
{
ActualPath = "https://mioto.milbit.com/" + imgName,
Name = imgName,
CreateTime = DateTime.Now,
Id = Guid.NewGuid()
});
await dbContext.SaveChangesAsync();
}
}
private async Task <object> ListFile(string prefix = "image")
{
var start = int.Parse(Request.Query["start"].ToString());
var size = int.Parse(Request.Query["size"].ToString());
//QiniuTool.GenerateToken();
//var mgr = new Qiniu.RS.BucketManager(macKey);
//await mgr.BatchDeleteAsync(bucket, saveKey);
//var rsf = new RSFClient(bucket);
//var files = await rsf.ListPrefixAsync(bucket, prefix);
//var list = files.Items.Select(p => string.Format("{0}/{1}{2}", domain, p.Key, prefix == "image" ? suffix : "")).OrderByDescending(p => p).Skip(start).Take(size).ToList();
var list = await QiniuTool.ListFile(prefix);
return(new
{
state = "SUCCESS",
list = list == null ? null : list.Select(x => new { url = x }),
start = start,
size = size,
total = list.Count
});
}
static string ProcessFile(string filePath, string dirName)
{
var extention = Path.GetExtension(filePath);
var fileName = "cartoon/" + Path.GetRandomFileName();
switch (extention)
{
case ".zip":
var outpuDirPaht = ZipTool.Exact(filePath);
AnalyseDic(outpuDirPaht, dirName);
break;
case ".pdf":
var outpuStream = PDFTool.PDFToImage(filePath);
//QiniuTool.UploadImage(outpuStream, fileName).Wait();
break;
case ".jpg":
case ".jpeg":
QiniuTool.UploadImage(filePath, fileName).Wait();
break;
}
return(fileName);
}
private async Task <string> UploadToQiniu(Stream stream, string fileType = "image", string ext = "")
{
var key = GetFileName(fileType, ext);
await QiniuTool.UploadImage(stream, key);
return(key);
}
public UtilitiesModule()
{
Get("/Image/Token", async ctx =>
{
var token = QiniuTool.GenerateToken();
await ctx.Ajax(new { token });
});
}
public IActionResult Token()
{
var token = QiniuTool.GenerateToken();
var result = new ApiMessage();
result.Data = token;
result.Message = "success!";
result.Success = true;
return(Json(result));
}
public IActionResult GetToken(string fileType)
{
var token = QiniuTool.GenerateToken();
return(Json(new
{
token = token,
suffix = suffix
}));
}
private static void RemoveContent(CommonDbContext dbcontext, Categories categories)
{
//dbcontext.Entry(contentList);
foreach (var contentItem in categories.ContentList)
{
QiniuTool.DeleteImage(contentItem.MediaResource.Select(x => x.ActualPath.Replace("https://mioto.milbit.com/", "")).ToArray()).Wait();
//dbcontext.Entry(contentItem.MediaResource);
//dbcontext.Remove(contentItem.MediaResource);
}
dbcontext.RemoveRange(categories.ContentList);
dbcontext.SaveChanges();
}
public GameController(QiniuTool _qiniutool,
VoteAppDbContext _dbContext,
IMemoryCache _memoryCache,
OryxWebSocketPool _wsPool,
IDistributedCache _distributedCache,
RabbitMQClient _rabbitMqClient
)
{
qiniutool = _qiniutool;
dbContext = _dbContext;
memoryCache = _memoryCache;
wsPool = _wsPool;
distributedCache = _distributedCache;
rabbitMqClient = _rabbitMqClient;
}
private void CreateContentAndUploadImage(string filePath, string cateName, List <Stream> streamList, int contentOrder)
{
var name = Path.GetFileNameWithoutExtension(filePath);
//var cateName = Path.GetDirectoryName(filePath).Split('\\').Last();
contentUploadCreator.CreateContent(name, cateName, contentOrder);
var index = 0;
streamList.ForEach(async imgStream =>
{
var fileName = "cartoon/toptoon/" + cateName + "/" + name + "_" + index + ".jpg";
QiniuTool.UploadImage(imgStream, fileName).Wait();
contentUploadCreator.SetContentIImg(name, cateName, fileName, index);
index++;
});
}
private static void ProcessListPage(IWebElement obj, Dictionary <string, string> State)
{
var exuctor = TaskExutor.Instance;
Console.WriteLine(obj.FindElement(By.CssSelector("body")).GetAttribute("innerHTML"));
var categories = obj.FindElements(By.CssSelector(".ajax-load-con.content"));
var categoryDbContext = GetDbContext();
foreach (var item in categories)
{
var nextUrl = item.FindElement(By.CssSelector("a")).GetAttribute("href");
Console.WriteLine(item.GetAttribute("innerHTML"));
var coverUrl = item.FindElement(By.CssSelector("li .image-item img")).GetAttribute("src");
var categoryTitle = item.FindElement(By.CssSelector(".posts-default-title a")).GetAttribute("title").Trim();
var creatTime = DateTime.Parse(item.FindElement(By.CssSelector(".ico-time")).GetAttribute("innerText"));
var imgStream = DownloadImg(coverUrl);
var coverImgPath = "cartoon/" + categoryTitle + "/coverimg.jpg";
QiniuTool.UploadImage(imgStream, coverImgPath).Wait();
var dbCategory = categoryDbContext.Categories.FirstOrDefault(x => x.Name == categoryTitle);
if (dbCategory == null)
{
var categoryModel = CreateCategory(categoryTitle, creatTime, coverImgPath);
categoryDbContext.Add(categoryModel);
}
var dbContent = categoryDbContext.ContentEntry.FirstOrDefault(x => x.Title == categoryTitle);
if (dbContent == null)
{
dbContent = CreateContent(dbCategory, categoryTitle, creatTime);
categoryDbContext.Add(dbContent);
}
if (dbContent.Category == null)
{
var subCate = categoryDbContext.Categories.FirstOrDefault(x => x.Name == categoryTitle);
dbContent.Category = subCate;
}
categoryDbContext.SaveChanges();
if (!string.IsNullOrEmpty(nextUrl))
{
var state = new Dictionary <string, string>();
var categoriesTaskSpider = new TaskSpider(nextUrl, state);
exuctor.SetTask(categoriesTaskSpider, ProcessImgPage);
}
}
}
public async Task <IActionResult> AssetPost(string key)
{
var fileStream = HttpContext.Request.Body;
var reader = await HttpContext.Request.BodyReader.ReadAsync();
var streamReader = new StreamReader(fileStream);
var stream = new MemoryStream();
await fileStream.CopyToAsync(stream);
await QiniuTool.UploadImage(stream, key);
var jobj = JObject.Parse("{}");
jobj.Add("content-name", key);
jobj.Add("status", "ok");
return(Content(jobj.ToString(), "application/json"));
}
private void CreateContentAndUploadImage(string contentDir, string cateName, int contentOrder)
{
var name = contentDir.Split('\\').Last();
var imgList = Directory.GetFiles(contentDir);
//var cateName = contentDir.Split('\\').Last();
contentUploadCreator.CreateContent(name, cateName, contentOrder);
var index = 0;
foreach (var imgItem in imgList)
{
var fileName = "cartoon/toptoon/" + cateName + "/" + name + "_" + index + ".jpg";
QiniuTool.UploadImage(imgItem, fileName).Wait();
contentUploadCreator.SetContentIImg(name, cateName, fileName, index);
index++;
}
}
public IActionResult Token()
{
var result = new ApiMessage();
try
{
var token = QiniuTool.GenerateToken();
result.Data = token;
result.Message = "success!";
result.Success = true;
}
catch (Exception exc)
{
result.SetFault(exc);
}
return(Json(result));
}
public async Task UpdateUserInfo(Guid userId, UpdateUserInfoViewModel updateModel)
{
var userInfoEntry = await userAccountAccessor.OneAsync <UserAccountEntry>(x => x.Id == userId);
/*remove old img from qiniu */
if (!string.IsNullOrEmpty(userInfoEntry.Avatar))
{
var avatarUri = new Uri(userInfoEntry.Avatar);
var saveKey = avatarUri.AbsolutePath.Remove(0);
await QiniuTool.DeleteImage(new string[] { saveKey });
}
/*end */
userInfoEntry.NickName = updateModel.NickName;
userInfoEntry.Avatar = updateModel.Avatar;
if (userInfoEntry.Profile == null)
{
userInfoEntry.Profile = new UserAccountProfile {
};
}
userInfoEntry.Profile.BirthDay = updateModel.BirthDay;
userInfoEntry.Profile.Location = updateModel.Location;
await userAccountAccessor.Update(userInfoEntry);
}
public async Task SvaeChapterDetail(CommonDbContext dbCtx, string cateName, string contentName, string fileName, int order)
{
try
{
var content = await dbCtx.ContentEntry.Include(x => x.MediaResource).FirstOrDefaultAsync(x => x.Category.Name == cateName && x.Title == contentName);
var category = await dbCtx.Categories.FirstOrDefaultAsync(x => x.Name == cateName);
if (content.MediaResource == null)
{
content.MediaResource = new List <FileEntry> {
};
}
var name = $"{cateName}/{contentName}/{order}img.jpg";
if (!content.MediaResource.Any(x => x.Name == name))
{
await QiniuTool.UploadImage(fileName, name);
content.MediaResource.Add(new FileEntry
{
Name = name,
ActualPath = "https://mioto.milbit.com/" + name,
CreateTime = DateTime.Now,
Order = order,
Tag = cateName
});
}
await dbCtx.SaveChangesAsync();
}
catch (Exception exc)
{
Console.WriteLine(exc.Message);
}
}
public string TokenByKey(string fileName)
{
return(QiniuTool.GenerateToken(fileName));
}
public string GenerateQiniu()
{
var qiniu = new QiniuTool(settingService.File.AppKey, settingService.File.AppSecret, settingService.File.Bucket, settingService.File.Zone);
return(qiniu.GenerateToken());
}
public static async Task UploadQiniu(string url, string key)
{
var imgStream = DownloadImg(url);
var coverImgPath = key;
await QiniuTool.UploadImage(imgStream, coverImgPath);
}
public void DeleteQIniu()
{
var imgKeyList = dbContext.FileEntry.Where(x => x.ActualPath.Contains("cartoon/toptoon/今天在哪嘞(无水印)")).Select(x => x.ActualPath.Replace("http://mioto.milbit.com/", "")).ToArray();
QiniuTool.DeleteImage(imgKeyList).Wait();
}
private static void AddCartoon()
{
var dbcontext = buildDbContext();
Console.WriteLine("请输入路径, 并按回车: 然后等待程序退出;");
var path = Console.ReadLine();
//var path = @"C:\Users\v-huamli\Documents\WeChat Files\lihuaming4000\Files\花美男幼儿园1";
//var path = @"G:\SystemDiretory\Downloads\1-22";
var matchRegex = new Regex(@"第(\d+)话");
var topCategories = Directory.EnumerateDirectories(path).ToList();
var topCateEntryList = new List <Categories>();
foreach (var topCate in topCategories)
{
var topCatTmpArr = topCate.Split("\\");
var topCatName = topCatTmpArr[topCatTmpArr.Length - 1];
var topCateEntry = new Categories
{
CreateTime = DateTime.Now,
Id = Guid.NewGuid(),
ChildrenCategories = new List <Categories>(),
Name = topCatName
};
dbcontext.Categories.AddAsync(topCateEntry).Wait();
dbcontext.SaveChangesAsync().Wait();
//漫画名
var secPath = Path.Combine(path, topCate);
//章节名
var secondCategories = Directory.EnumerateDirectories(secPath).OrderBy(x =>
{
var matchVal = matchRegex.Match(x);
var intVal = matchVal.Groups[1].Value;
return(int.Parse(intVal));
}).ToList();
//var contentList = new List<ContentEntry>();
var order = 0;
foreach (var secCate in secondCategories)
{
var secCateTmpArr = secCate.Split("\\");
var secCateName = secCateTmpArr[secCateTmpArr.Length - 1];
//图片名
var thrPath = Path.Combine(secPath, secCate);
var fileNameList = Directory.EnumerateFiles(thrPath).ToList();
var fileEntryList = new List <FileEntry>();
var fileUploadTaskList = new List <Task>();
var contentEntry = new ContentEntry
{
Category = topCateEntry,
CreateTime = DateTime.Now,
Id = Guid.NewGuid(),
Title = secCateName,
Order = order++
};
foreach (var fileName in fileNameList)
{
var saveKey = "cartoon/" + topCatName
+ "-" + secCateName + "-" + Path.GetFileName(fileName);
var task = QiniuTool.UploadImage(fileName, saveKey);
fileEntryList.Add(new FileEntry
{
ActualPath = "https://mioto.milbit.com/" + saveKey,
CreateTime = DateTime.Now,
Name = saveKey,
Tag = saveKey
});
fileUploadTaskList.Add(task);
}
Task.WaitAll(fileUploadTaskList.ToArray());
contentEntry.MediaResource = fileEntryList;
dbcontext.ContentEntry.AddAsync(contentEntry).Wait();
dbcontext.SaveChangesAsync().Wait();
}
}
dbcontext.SaveChangesAsync().Wait();
}
public static void DeleteCartoon()
{
var dbcontext = buildDbContext();
var targetCategoryList = dbcontext.Categories
.Include(x => x.Tags)
.Include(x => x.ChildrenCategories)
.Include(x => x.MediaResource)
.Include(x => x.ContentList)
.Include("ChildrenCategories.ContentList")
.Include("ContentList.MediaResource")
.Include("ChildrenCategories.MediaResource").Where(x => x.Name == "h-mate 完结(不含外传）").ToList();
//.Include("ChildrenCategories.MediaResource").Where(x => x.Tags.Any(c => c.Name == "新资源压缩")).ToList();
//.Include("ChildrenCategories.MediaResource").Where(x => x.Name == "妹控进行时" || x.Name == "多重人格侦探").ToList();
//.Include("ChildrenCategories.MediaResource").Where(x => x.Tags.Any(c => c.Name == "妖气漫画")).ToList();
foreach (var topCategory in targetCategoryList)
{
if (topCategory.ChildrenCategories != null && topCategory.ChildrenCategories.Count > 0)
{
foreach (var subCategory in topCategory.ChildrenCategories)
{
if (subCategory.ContentList != null && subCategory.ContentList.Count > 0)
{
foreach (var contentItem in subCategory.ContentList)
{
QiniuTool.DeleteImage(contentItem.MediaResource.Select(x => x.ActualPath.Replace("https://mioto.milbit.com/", "")).ToArray()).Wait();
dbcontext.Remove(contentItem.MediaResource);
dbcontext.SaveChanges();
}
dbcontext.RemoveRange(subCategory.ContentList);
}
dbcontext.SaveChanges();
}
dbcontext.RemoveRange(topCategory.ChildrenCategories);
dbcontext.SaveChanges();
}
else
{
foreach (var contentItem in topCategory.ContentList)
{
if (contentItem.MediaResource != null && contentItem.MediaResource.Count > 0)
{
QiniuTool.DeleteImage(contentItem.MediaResource.Select(x => x.ActualPath.Replace("https://mioto.milbit.com/", "")).ToArray()).Wait();
dbcontext.RemoveRange(contentItem.MediaResource);
dbcontext.SaveChanges();
}
}
dbcontext.RemoveRange(topCategory.ContentList);
}
dbcontext.Remove(topCategory);
if (topCategory.Tags != null && topCategory.Tags.Count > 0)
{
dbcontext.RemoveRange(topCategory.Tags);
}
dbcontext.SaveChanges();
}
//foreach (var item in dbcontext.ContentEntry.Include(x => x.MediaResource))
//{
// QiniuTool.DeleteImage(item.MediaResource.Select(x => x.ActualPath.Replace("https://mioto.milbit.com/", "")).ToArray()).Wait();
//}
}
private static void ProcessListPage(IWebElement webElement, Dictionary <string, string> state)
{
try
{
var body = webElement.FindElement(By.TagName("body")).GetAttribute("innerHTML");
Console.WriteLine(body);
var conentList = webElement.FindElements(By.CssSelector(".storyCell"));
//漫画列表
foreach (var contentItem in conentList)
{
var categoryTitle = contentItem.FindElement(By.CssSelector(".rightInfo .title")).GetAttribute("innerText");
if (!zeroList.Any(x => x.Name == categoryTitle))
{
continue;
}
var coverUrl = contentItem.FindElement(By.CssSelector(".leftCover img")).GetAttribute("src");
var url = contentItem.FindElement(By.CssSelector(".leftCover a")).GetAttribute("href");
var desc = contentItem.FindElement(By.CssSelector(".rightInfo .intr")).GetAttribute("innerText");
TaskExutor exuctor = TaskExutor.Instance;
//var state = new Dictionary<string, string>();
var curState = new Dictionary <string, string>();
foreach (var item in state)
{
curState.Add(item.Key, item.Value);
}
curState.Add("cateTitle", categoryTitle);
var contentTaskSpider = new TaskSpider(url, curState);
contentTaskSpider.SetCookie(_cookie);
//Console.WriteLine(title);
//upload qiniu
var imgStream = DownloadImg(coverUrl);
var coverImgPath = "cartoon/" + categoryTitle + "/coverimg.jpg";
QiniuTool.UploadImage(imgStream, coverImgPath).Wait();
var categoryDbContext = GetDbContext();
var hasCategory = categoryDbContext.Categories.AnyAsync(x => x.Name == categoryTitle).Result;
if (!hasCategory)
{
categoryDbContext.Categories.Add(new Categories
{
Id = Guid.NewGuid(),
CreateTime = DateTime.Now,
Name = categoryTitle,
Description = desc.Trim(),
MediaResource = new List <FileEntry> {
new FileEntry {
Id = Guid.NewGuid(),
ActualPath = "https://mioto.milbit.com/" + coverImgPath,
CreateTime = DateTime.Now,
Name = coverImgPath
}
},
Tags = new List <Tags> {
new Tags {
Id = Guid.NewGuid(),
Name = "恋爱"
},
new Tags {
Id = Guid.NewGuid(),
Name = "都市"
},
new Tags {
Id = Guid.NewGuid(),
Name = "Sex"
},
new Tags {
Id = Guid.NewGuid(),
Name = "SpiderUplaod2"
}
}
});
categoryDbContext.SaveChanges();
}
processChapter(categoryTitle, contentTaskSpider);
}
}
catch (Exception exc)
{
throw;
}
}
public string Token()
{
return(QiniuTool.GenerateToken());
}
public string TokenByParam(string fileName, string bucket, ZoneID zoneID)
{
return(QiniuTool.GenerateToken(fileName, bucket, zoneID));
}
