/// <summary> /// paralell parse chapters info /// </summary> /// <param name="threadsNumber">how many threads is running at the same time</param> /// <param name="match">add ChapterInfo to list if ShortChapterInfo match</param> protected void ParseChaptersInfo(Predicate <ShortChapterInfo> match, int threadsNumber) { IChapterJsonParser jsonParser = new ChapterJsonParser(); Thread[] threads = new Thread[threadsNumber]; int threadsIndex = 0; foreach (var chapter in MangaInfo.ShortChaptersInfo) { // check if it s match the request if (match(chapter)) { // we use post increament so we just compare this values if (threadsIndex == threadsNumber) { threadsIndex = 0; // start threads foreach (var thread in threads) { thread.Start(); } //// wait for thread's work complete //foreach (var thread in threads) //{ // thread.Join(); //} } // parse json func ThreadStart parseFunc = () => { // wait for calling thread wait for end of the parsing // parse data from site var chapterLocal = chapter; int id = Convert.ToInt32(chapterLocal.Id); IChapterInfo info = jsonParser.GetChapterInfo(id); ChaptersInfo.Add(info); }; threads[threadsIndex++] = new Thread(parseFunc); } } // if something left in threads array and yeah there only matched chapters if (threadsIndex > 0) { // threadsIndex is size of the array with useful parseFuncs // start threads for (int i = 0; i < threadsIndex; i++) { threads[i].Start(); } // wait for thread's work complete for (int i = 0; i < threadsIndex; i++) { threads[i].Join(); } } }
/// <summary> /// parse chapters info /// </summary> /// <param name="match">predicate for chapters</param> public List <IChapterInfo> ParseChaptersInfo(Predicate <ShortChapterInfo> match) { IChapterJsonParser jsonParser = new ChapterJsonParser(); List <IChapterInfo> chaptersInfo = new List <IChapterInfo>(); foreach (var chapter in MangaInfo.ShortChaptersInfo) { // check if it s match the user's request if (match(chapter)) { int id = Convert.ToInt32(chapter.Id); IChapterInfo info = jsonParser.GetChapterInfo(id); chaptersInfo.Add(info); } } return(chaptersInfo); }
/// <summary> /// parse pages into directory /// </summary> /// <param name="chapterInfo">chapterInfo for parsing</param> /// <param name="numberOfTry">number of try if while parsing something gone wrong it will try to parse this again this amount of time</param> public void Parse(IChapterInfo chapterInfo, int numberOfTry) { int numberOfPages = chapterInfo.Pages.Count; int parsedPages = 0; foreach (var page in chapterInfo.Pages) { string pageUrl = $"{chapterInfo.ServerUrl}/{chapterInfo.Hash}/{page.PageName}"; // parse try number of try for (int i = 0; i < numberOfTry; i++) { // parse one page try { ParsePage(page, chapterInfo); // call delagate string message = $"{DateTime.Now}: page \"{pageUrl}\" has parsed successfully into dir \"{Dir.FullName}\", page number {page.PageNumber}, chapter number: {chapterInfo.Chapter}, volume number: {chapterInfo.Volume}, chapter id: {chapterInfo.Id}"; var e = new OnProgressParserEventArgs() { NumberOfPages = numberOfPages, ParsedPages = ++parsedPages, Message = message }; OnProgress?.Invoke(this, e); #if DEBUG Trace.WriteLine($"{DateTime.Now}: page \"{pageUrl}\" has parsed successfully, page number {page.PageNumber}, chapter number: {chapterInfo.Chapter}, volume number: {chapterInfo.Volume}, chapter id: {chapterInfo.Id}"); #endif break; } catch (Exception exc) { #if DEBUG Trace.WriteLine($"{DateTime.Now}: FAIL page \"{pageUrl}\" has parsed unsuccessfully, page number {page.PageNumber}, chapter number: {chapterInfo.Chapter}, volume number: {chapterInfo.Volume}, chapter id: {chapterInfo.Id}, \n{exc.Message}{exc.StackTrace}\n"); #endif // if numberOfTry is already passed then we will throw exception if (i + 1 == numberOfTry) { throw; } } } } }
/// <summary> /// parse page /// </summary> /// <param name="page">page info in chapter inf</param> /// <param name="chapterInfo">chapterInfo</param> protected void ParsePage(ChapterInfo.Page page, IChapterInfo chapterInfo) { // page's names have string type string pageName = page.PageName; // local page name and full path to page // VOLUMENUMBER_CHAPTERNUMBER_PAGENUMBER string localPageName = $"{chapterInfo.Volume}_{chapterInfo.Chapter}_{page.PageNumber}"; string pageExtension = Path.GetExtension(pageName); char dirSeparatorChar = Path.DirectorySeparatorChar; string fullPath = $"{Dir.FullName}{dirSeparatorChar}{localPageName}{pageExtension}"; // get url to image page string pageUrl = $"{chapterInfo.ServerUrl}/{chapterInfo.Hash}/{pageName}"; WebRequest request = WebRequest.Create(pageUrl); #if DEBUG Trace.WriteLine($"{DateTime.Now}: web request has created to this url \"{pageUrl}\", chapter id: {chapterInfo.Id}"); #endif HttpWebResponse response = (HttpWebResponse)request.GetResponse(); #if DEBUG Trace.WriteLine($"{DateTime.Now}: recieve response from \"{pageUrl}\", chapter id: {chapterInfo.Id}"); #endif // open file BinaryReader reader = new BinaryReader(response.GetResponseStream()); FileStream fileStream = File.OpenWrite(fullPath); BinaryWriter writer = new BinaryWriter(fileStream); // make request to server containing pages, try { // write content to image file int bufferSize = 1024; byte[] buffer; while (true) { buffer = reader.ReadBytes(bufferSize); writer.Write(buffer); if (buffer.Length == 0) { break; } } // save writed data writer.Flush(); #if DEBUG Trace.WriteLine($"{DateTime.Now}: page has writed to file \"{fullPath}\", chapter id: {chapterInfo.Id}"); #endif } finally { // close all streams writer.Close(); reader.Close(); response.Close(); } }