public void PageTestPrepare()
{
// Setup
var pageName = "Page name";
var pageProfile = new PageProfile(pageName)
{
new LineProfile(),
new LineProfile(),
new LineProfile()
};
var profileList = new List <LineProfile>();
var lineMockList = new List <Mock <ILine> >();
var menuHandlerMock = SetUpMenuHandlerMock(profileList, (p, line) => lineMockList.Add(line));
// Act
this.pageUnderTest.Prepare(pageProfile, menuHandlerMock.Object);
// Assert
Assert.AreEqual(3, profileList.Count);
Assert.AreEqual(3, lineMockList.Count);
Assert.AreEqual(pageName, this.pageUnderTest.Name);
for (var index = 0; index < lineMockList.Count; index++)
{
var lineMock = lineMockList[index];
var lineProfile = pageProfile[index];
lineMock.Verify(l => l.Prepare(lineProfile, menuHandlerMock.Object), Times.Once());
}
}
public void PageTestSetPositionWithOnlyFlexible()
{
// Setup
var pageProfile = new PageProfile()
{
new LineProfile(),
new LineProfile()
};
var lineMockList = new List <Mock <ILine> >();
var menuHandlerMock = SetUpMenuHandlerMock(null, (p, line) => lineMockList.Add(line));
this.pageUnderTest.Prepare(pageProfile, menuHandlerMock.Object);
lineMockList[0].Setup(l => l.GetPreferredHeight()).Returns((double?)null);
lineMockList[0].Setup(l => l.GetPreferredWidth()).Returns(100);
lineMockList[1].Setup(l => l.GetPreferredHeight()).Returns((double?)null);
lineMockList[1].Setup(l => l.GetPreferredWidth()).Returns(90);
var parentMock = new Mock <IFrame>();
// Act
this.pageUnderTest.SetPosition(parentMock.Object, 0, 0, 100, 50);
// Assert
lineMockList[0].Verify(l => l.SetPosition(this.pageFrameMock.Object, 0, 0, 100, 20), Times.Once);
lineMockList[1].Verify(l => l.SetPosition(this.pageFrameMock.Object, 0, 30, 100, 20), Times.Once);
}
public void PageTestSetPositionWithNoFlexible()
{
// Setup
var pageProfile = new PageProfile()
{
new LineProfile(),
new LineProfile()
};
var lineMockList = new List <Mock <ILine> >();
var menuHandlerMock = SetUpMenuHandlerMock(null, (p, line) => lineMockList.Add(line));
this.pageUnderTest.Prepare(pageProfile, menuHandlerMock.Object);
lineMockList[0].Setup(l => l.GetPreferredHeight()).Returns(30);
lineMockList[0].Setup(l => l.GetPreferredWidth()).Returns(100);
lineMockList[1].Setup(l => l.GetPreferredHeight()).Returns(10);
lineMockList[1].Setup(l => l.GetPreferredWidth()).Returns(90);
var parentMock = new Mock <IFrame>();
// Act
this.pageUnderTest.SetPosition(parentMock.Object, 2, 3, 100, 75);
// Assert
this.pageFrameMock.Verify(f => f.SetParent(parentMock.Object));
this.pageFrameMock.Verify(f => f.SetWidth(100));
this.pageFrameMock.Verify(f => f.SetHeight(50));
this.pageFrameMock.Verify(f => f.SetPoint(FramePoint.TOPLEFT, parentMock.Object, FramePoint.TOPLEFT, 2, -3));
lineMockList[0].Verify(l => l.SetPosition(this.pageFrameMock.Object, 0, 0, 100, 30), Times.Once);
lineMockList[1].Verify(l => l.SetPosition(this.pageFrameMock.Object, 0, 40, 100, 10), Times.Once);
}
public void EditPersonalDetails(string lng)
{
#region Test Data
string login = "******";
string pas = "******";
#endregion
LoginHelper.Login(webDriver, lng, login, pas);
PageTradingAccountsReal pageTradingAccountsReal = new PageTradingAccountsReal(webDriver, lng);
pageTradingAccountsReal.BtnMenuProfile.Click();
pageTradingAccountsReal.LnkMenuMyProfile.Click();
PageProfile pageProfile = new PageProfile(webDriver, lng);
//pageTradingAccountsReal.WaitLoadPage(pageProfile);
pageProfile.LnkEditPersonalDetails.Click();
PageEditPersonalDetails pageEditPersonalDetails = new PageEditPersonalDetails(webDriver, lng);
//pageEditPersonalDetails.WaitLoadPage(pageEditPersonalDetails);
//check and save old values
Assert.AreEqual("TestNameUAE", pageEditPersonalDetails.TxtFirstName.GetAttribute("value"));
Assert.AreEqual("TestLastNameUAE", pageEditPersonalDetails.TxtLastName.GetAttribute("value"));
Assert.AreEqual("+971-50-000-0000", pageEditPersonalDetails.TxtPhone.GetAttribute("value"));
pageEditPersonalDetails.BtnSave.Click();
//pageTradingAccountsReal.WaitLoadPage(pageProfile);
pageProfile.LnkEditPersonalDetails.Click();
//check old values
Assert.AreEqual("TestNameUAE", pageEditPersonalDetails.TxtFirstName.GetAttribute("value"));
Assert.AreEqual("TestLastNameUAE", pageEditPersonalDetails.TxtLastName.GetAttribute("value"));
Assert.AreEqual("+971-50-000-0000", pageEditPersonalDetails.TxtPhone.GetAttribute("value"));
//input new values
pageEditPersonalDetails.TxtFirstName.Clear();
pageEditPersonalDetails.TxtFirstName.SendKeys("newTestNameUAE");
pageEditPersonalDetails.TxtLastName.Clear();
pageEditPersonalDetails.TxtLastName.SendKeys("newTestLastNameUAE");
pageEditPersonalDetails.TxtPhone.Clear();
pageEditPersonalDetails.TxtPhone.SendKeys("+971-50-000-0001");
pageEditPersonalDetails.BtnSave.Click();
//check successfully message
//pageTradingAccountsReal.WaitLoadPage(pageTradingAccountsReal);
Assert.AreEqual("Your request was successfully received. We will review it and let you know about the results. If it is necessary, we will contact you for details.", pageTradingAccountsReal.MsgSuccess.Text);
pageTradingAccountsReal.BtnMenuProfile.Click();
pageTradingAccountsReal.LnkMenuMyProfile.Click();
pageProfile.LnkEditPersonalDetails.Click();
//pageEditPersonalDetails.WaitLoadPage(pageEditPersonalDetails);
//check new values
Assert.AreEqual("TestNameUAE", pageEditPersonalDetails.TxtFirstName.GetAttribute("value"));
Assert.AreEqual("TestLastNameUAE", pageEditPersonalDetails.TxtLastName.GetAttribute("value"));
Assert.AreEqual("+971-50-000-0000", pageEditPersonalDetails.TxtPhone.GetAttribute("value"));
}
public override void Prepare(IElementProfile profile, IMenuHandler handler)
{
base.Prepare(profile, handler);
var panelProfile = (PanelProfile)profile;
var innerPageProfile = new PageProfile();
panelProfile.ForEach(innerPageProfile.Add);
this.innerPage = (IPage)handler.CreateRegion(innerPageProfile);
this.innerPage.Prepare(innerPageProfile, handler);
this.name = panelProfile.name;
}
public static async Task Crawler()
{
//Download chromium browser revision package
await new BrowserFetcher().DownloadAsync(ChromiumRevision);
using var browser = await Puppeteer.LaunchAsync(new LaunchOptions
{
// Headless = true
Headless = false
});
var optionsBuilder = new DbContextOptionsBuilder <ContentContext>();
var options = optionsBuilder
.UseSqlite("Data Source=D:\\WebCrawlerPrj\\Crawler\\DB\\content.db;",
providerOptions => providerOptions.CommandTimeout(60));
await using var _context = new ContentContext(options.Options);
var articles = _context.CrawledLinks.Select(c => new { c.Id, c.Url }).ToList();
foreach (var article in articles)
{
var page = await browser.NewPageAsync();
var pageContent = await GetPageContent(page, article.Url, 1024, 768);
var context = BrowsingContext.New(Configuration.Default);
var parser = context.GetService <IHtmlParser>();
var document = parser.ParseDocument(pageContent);
var articleContent = document.QuerySelector(".sidebar_1").InnerHtml;
var articleModel = CreatePageModel(article.Url, articleContent, article.Id);
var articleEntity = PageProfile.MapCreateModelToEntity(articleModel);
await _context.Pages.AddAsync(articleEntity);
await _context.SaveChangesAsync();
await page.CloseAsync();
}
await browser.CloseAsync();
}
public void PageGetPreferredWidthTestWithNoFlexible()
{
// Setup
var pageProfile = new PageProfile()
{
new LineProfile(),
new LineProfile(),
new LineProfile()
};
var lineMockList = new List <Mock <ILine> >();
var menuHandlerMock = SetUpMenuHandlerMock(null, (p, line) => lineMockList.Add(line));
this.pageUnderTest.Prepare(pageProfile, menuHandlerMock.Object);
lineMockList[0].Setup(l => l.GetPreferredWidth()).Returns(100);
lineMockList[1].Setup(l => l.GetPreferredWidth()).Returns(90);
lineMockList[2].Setup(l => l.GetPreferredWidth()).Returns(110);
// Act
var preferredWidth = this.pageUnderTest.GetPreferredWidth();
// Assert
Assert.AreEqual(110, preferredWidth);
}
private void init(Profile.ProfileType Type,string Name="")
{
if (profile == null)
{
profile = new Profile(Type);
profile.Name = Name;
SaveRequired = true;
}
UpdateTitle();
profile.PropertyChanged += profile_PropertyChanged;
pageProfile = new PageProfile(profile);
pageAllMarks = new PageAllMarks(profile);
AddTab(pageProfile , "Profile");
AddTab(pageAllMarks, "All Marks");
}
public void LegalDocumentation(string lng)
{
#region Test Data
#endregion
//Create user
LoginHelper.Login(ngWebDriver, lng);
PageTradingAccountsReal pageTradingAccountsReal = new PageTradingAccountsReal(ngWebDriver, lng);
pageTradingAccountsReal.WaitLoadPage(pageTradingAccountsReal);
pageTradingAccountsReal.BtnMenuProfile.Click();
pageTradingAccountsReal.LnkMenuMyProfile.Click();
PageProfile pageProfile = new PageProfile(ngWebDriver, lng);
pageTradingAccountsReal.WaitLoadPage(pageProfile);
pageProfile.LnkLegalDocumentation.Click();
PageLegalDocumentation pageLegalDocumentation = new PageLegalDocumentation(ngWebDriver, lng);
pageTradingAccountsReal.WaitLoadPage(pageLegalDocumentation);
pageLegalDocumentation.BtnClientAgreement.Click();
Assert.AreEqual(@"https://datarepo.fxpro.co.uk/legal/mena/Client_Agreement.pdf", ngWebDriver.WrappedDriver.Url);
ngWebDriver.WrappedDriver.Navigate().Back();
pageTradingAccountsReal.WaitLoadPage(pageLegalDocumentation);
pageLegalDocumentation.BtnClientAgreementAr.Click();
Assert.AreEqual(@"https://datarepo.fxpro.co.uk/legal/mena/Client_Agreement_ar.pdf", ngWebDriver.WrappedDriver.Url);
ngWebDriver.WrappedDriver.Navigate().Back();
pageTradingAccountsReal.WaitLoadPage(pageLegalDocumentation);
pageLegalDocumentation.BtnComplaintsHandlingProcedure.Click();
Assert.AreEqual(@"https://datarepo.fxpro.co.uk/legal/mena/Complaints_Handling_Procedure.pdf", ngWebDriver.WrappedDriver.Url);
ngWebDriver.WrappedDriver.Navigate().Back();
pageTradingAccountsReal.WaitLoadPage(pageLegalDocumentation);
pageLegalDocumentation.BtnOrderExecutionPolicy.Click();
Assert.AreEqual(@"https://datarepo.fxpro.co.uk/legal/mena/Order_Execution_Policy.pdf", ngWebDriver.WrappedDriver.Url);
ngWebDriver.WrappedDriver.Navigate().Back();
pageTradingAccountsReal.WaitLoadPage(pageLegalDocumentation);
pageLegalDocumentation.BtnOrderExecutionPolicyAr.Click();
Assert.AreEqual(@"https://datarepo.fxpro.co.uk/legal/mena/Order_Execution_Policy_ar.pdf", ngWebDriver.WrappedDriver.Url);
ngWebDriver.WrappedDriver.Navigate().Back();
pageTradingAccountsReal.WaitLoadPage(pageLegalDocumentation);
pageLegalDocumentation.BtnClientCategorisationNotice.Click();
Assert.AreEqual(@"https://datarepo.fxpro.co.uk/legal/mena/Client_Categorisation_Notice.pdf", ngWebDriver.WrappedDriver.Url);
ngWebDriver.WrappedDriver.Navigate().Back();
pageTradingAccountsReal.WaitLoadPage(pageLegalDocumentation);
pageLegalDocumentation.BtnConflictsOfInterestPolicy.Click();
Assert.AreEqual(@"https://datarepo.fxpro.co.uk/legal/mena/Conflict_of_Interest_Policy.pdf", ngWebDriver.WrappedDriver.Url);
ngWebDriver.WrappedDriver.Navigate().Back();
pageTradingAccountsReal.WaitLoadPage(pageLegalDocumentation);
pageLegalDocumentation.BtnFxProWallet.Click();
Assert.AreEqual(@"https://datarepo.fxpro.co.uk/legal/mena/FxPro_Wallet.pdf", ngWebDriver.WrappedDriver.Url);
ngWebDriver.WrappedDriver.Navigate().Back();
pageTradingAccountsReal.WaitLoadPage(pageLegalDocumentation);
pageLegalDocumentation.BtnRiskDisclosure.Click();
Assert.AreEqual(@"https://datarepo.fxpro.co.uk/legal/mena/Risk_Disclosure_Notice.pdf", ngWebDriver.WrappedDriver.Url);
ngWebDriver.WrappedDriver.Navigate().Back();
pageTradingAccountsReal.WaitLoadPage(pageLegalDocumentation);
pageLegalDocumentation.BtnRiskDisclosureAr.Click();
Assert.AreEqual(@"https://datarepo.fxpro.co.uk/legal/mena/Risk_Disclosure_Notice_ar.pdf", ngWebDriver.WrappedDriver.Url);
ngWebDriver.WrappedDriver.Navigate().Back();
pageTradingAccountsReal.WaitLoadPage(pageLegalDocumentation);
pageLegalDocumentation.BtnRetailRiskDisclosure.Click();
Assert.AreEqual(@"https://datarepo.fxpro.co.uk/legal/mena/Retail_FX_Risk_Disclosure_Statement.pdf", ngWebDriver.WrappedDriver.Url);
ngWebDriver.WrappedDriver.Navigate().Back();
pageTradingAccountsReal.WaitLoadPage(pageLegalDocumentation);
pageLegalDocumentation.BtnTermsAndConditionsForFixedSpreadAccount.Click();
Assert.AreEqual(@"https://datarepo.fxpro.co.uk/legal/mena/Terms_and_Conditions_for_Fixed_Spread_Account.pdf", ngWebDriver.WrappedDriver.Url);
ngWebDriver.WrappedDriver.Navigate().Back();
pageTradingAccountsReal.WaitLoadPage(pageLegalDocumentation);
}
public async Task Post(CreatePerks request)
{
var perks = new List <Perk>()
{
new Perk
{
Id = "Perk/vegan",
Name = "Веганство",
ImagePath = "https://github.com/Kibnet/Axiometr.Server/raw/master/resources/vegan.png",
KeyWords = new List <string>
{
"сыроед",
"веган",
"вегетариан",
}
},
new Perk
{
Id = "Perk/eatmeet",
Name = "Мясоедство",
ImagePath = "https://github.com/Kibnet/Axiometr.Server/raw/master/resources/eatmeet.png",
KeyWords = new List <string>
{
"мясо",
"мясоед",
}
},
new Perk
{
Id = "Perk/antivaccination",
Name = "Против прививок",
ImagePath = "https://github.com/Kibnet/Axiometr.Server/raw/master/resources/antivaccination.png",
KeyWords = new List <string>
{
"прививка",
"вакцина",
"грип",
"covid",
"ковид",
}
},
new Perk
{
Id = "Perk/vaccination",
Name = "За прививки",
ImagePath = "https://github.com/Kibnet/Axiometr.Server/raw/master/resources/vaccination.png",
KeyWords = new List <string>
{
"прививка",
"вакцина",
"грип",
"covid",
"ковид",
}
},
new Perk
{
Id = "Perk/krymru",
Name = "Крым - РФ",
ImagePath = "https://github.com/Kibnet/Axiometr.Server/raw/master/resources/krymru.png",
KeyWords = new List <string>
{
"крым",
}
},
new Perk
{
Id = "Perk/krymua",
Name = "Крым - Украина",
ImagePath = "https://github.com/Kibnet/Axiometr.Server/raw/master/resources/krymua.png",
KeyWords = new List <string>
{
"крым",
}
},
new Perk
{
Id = "Perk/aborting",
Name = "За аборты",
ImagePath = "https://github.com/Kibnet/Axiometr.Server/raw/master/resources/aborting.png",
KeyWords = new List <string>
{
"аборт",
}
},
new Perk
{
Id = "Perk/antiaborting",
Name = "Против абортов",
ImagePath = "https://github.com/Kibnet/Axiometr.Server/raw/master/resources/antiaborting.png",
KeyWords = new List <string>
{
"аборт",
}
},
new Perk
{
Id = "Perk/traditionsex",
Name = "Традиционные половые отношения",
ImagePath = "https://github.com/Kibnet/Axiometr.Server/raw/master/resources/traditionsex.png",
KeyWords = new List <string>
{
"семья",
"дети",
"ребенок",
"любов",
}
},
new Perk
{
Id = "Perk/nontraditionsex",
Name = "Не традиционные половые отношения",
ImagePath = "https://github.com/Kibnet/Axiometr.Server/raw/master/resources/nontraditionsex.png",
KeyWords = new List <string>
{
"геи",
"лесби",
"лгбт",
"транссексуал",
"прайд",
"любов",
}
},
new Perk
{
Id = "Perk/filthylanguage",
Name = "Мат",
ImagePath = "https://github.com/Kibnet/Axiometr.Server/raw/master/resources/filthylanguage.png",
KeyWords = new List <string>
{
"редиска",
"f**k",
}
},
new Perk
{
Id = "Perk/adult",
Name = "18+",
ImagePath = "https://github.com/Kibnet/Axiometr.Server/raw/master/resources/adult.png",
KeyWords = new List <string>
{
"порно",
"секс",
"убить",
"кровь",
}
}
};
foreach (var perk in perks)
{
await RavenSession.StoreAsync(perk);
}
var medusa = new PageProfile
{
Address = "https://meduza.io/"
};
var wonderzine = new PageProfile
{
Address = "https://www.wonderzine.com/"
};
var profiles = new List <PageProfile>
{
medusa,
wonderzine,
};
foreach (var profile in profiles)
{
await RavenSession.StoreAsync(profile);
}
await RavenSession.SaveChangesAsync();
DocumentCountersOperation operation1 = new DocumentCountersOperation
{
DocumentId = RavenSession.Advanced.GetDocumentId(medusa),
Operations = new List <CounterOperation>
{
new CounterOperation()
{
CounterName = "krymua",
Delta = 10,
Type = CounterOperationType.Increment,
},
}
};
CounterBatch counterBatch = new CounterBatch();
counterBatch.Documents.Add(operation1);
var counteBatch = new CounterBatchOperation(counterBatch);
RavenSession.Advanced.DocumentStore.Operations.Send(counteBatch);
}
public void LegalDocumentation(string lng)
{
#region Test Data
string login = "******";
string pas = "******";
#endregion
//Create user
LoginHelper.Login(webDriver, lng, login, pas);
PageTradingAccountsReal pageTradingAccountsReal = new PageTradingAccountsReal(webDriver, lng);
pageTradingAccountsReal.BtnMenuProfile.Click();
pageTradingAccountsReal.LnkMenuMyProfile.Click();
PageProfile pageProfile = new PageProfile(webDriver, lng);
pageProfile.LnkLegalDocumentation.Click();
PageLegalDocumentation pageLegalDocumentation = new PageLegalDocumentation(webDriver, lng);
pageLegalDocumentation.BtnClientAgreement.Click();
Wait.UrlContains(webDriver, @"https://s3.eu-central-1.amazonaws.com/files-mrkt/datarepo/legal/mena/Client_Agreement.pdf");
pageLegalDocumentation.GoToBack();
Wait.UrlContains(webDriver, pageLegalDocumentation.ExpectedUrl);
pageLegalDocumentation.BtnClientAgreementAr.Click();
Wait.UrlContains(webDriver, @"https://s3.eu-central-1.amazonaws.com/files-mrkt/datarepo/legal/mena/Client_Agreement_ar.pdf");
pageLegalDocumentation.GoToBack();
Wait.UrlContains(webDriver, pageLegalDocumentation.ExpectedUrl);
pageLegalDocumentation.BtnComplaintsHandlingProcedure.Click();
Wait.UrlContains(webDriver, @"https://s3.eu-central-1.amazonaws.com/files-mrkt/datarepo/legal/mena/Complaints_Handling_Procedure.pdf");
pageLegalDocumentation.GoToBack();
Wait.UrlContains(webDriver, pageLegalDocumentation.ExpectedUrl);
pageLegalDocumentation.BtnOrderExecutionPolicy.Click();
Wait.UrlContains(webDriver, @"https://s3.eu-central-1.amazonaws.com/files-mrkt/datarepo/legal/mena/Order_Execution_Policy.pdf");
pageLegalDocumentation.GoToBack();
Wait.UrlContains(webDriver, pageLegalDocumentation.ExpectedUrl);
pageLegalDocumentation.BtnOrderExecutionPolicyAr.Click();
Wait.UrlContains(webDriver, @"https://s3.eu-central-1.amazonaws.com/files-mrkt/datarepo/legal/mena/Order_Execution_Policy_ar.pdf");
pageLegalDocumentation.GoToBack();
Wait.UrlContains(webDriver, pageLegalDocumentation.ExpectedUrl);
pageLegalDocumentation.BtnClientCategorisationNotice.Click();
Wait.UrlContains(webDriver, @"https://s3.eu-central-1.amazonaws.com/files-mrkt/datarepo/legal/mena/Client_Categorisation_Notice.pdf");
pageLegalDocumentation.GoToBack();
Wait.UrlContains(webDriver, pageLegalDocumentation.ExpectedUrl);
pageLegalDocumentation.BtnConflictsOfInterestPolicy.Click();
Wait.UrlContains(webDriver, @"https://s3.eu-central-1.amazonaws.com/files-mrkt/datarepo/legal/mena/Conflict_of_Interest_Policy.pdf");
pageLegalDocumentation.GoToBack();
Wait.UrlContains(webDriver, pageLegalDocumentation.ExpectedUrl);
pageLegalDocumentation.BtnFxProWallet.Click();
Wait.UrlContains(webDriver, @"https://s3.eu-central-1.amazonaws.com/files-mrkt/datarepo/legal/mena/FxPro_Wallet.pdf");
pageLegalDocumentation.GoToBack();
Wait.UrlContains(webDriver, pageLegalDocumentation.ExpectedUrl);
pageLegalDocumentation.BtnRiskDisclosure.Click();
Wait.UrlContains(webDriver, @"https://s3.eu-central-1.amazonaws.com/files-mrkt/datarepo/legal/mena/Risk_Disclosure_Notice.pdf");
pageLegalDocumentation.GoToBack();
Wait.UrlContains(webDriver, pageLegalDocumentation.ExpectedUrl);
pageLegalDocumentation.BtnRiskDisclosureAr.Click();
Wait.UrlContains(webDriver, @"https://s3.eu-central-1.amazonaws.com/files-mrkt/datarepo/legal/mena/Risk_Disclosure_Notice_ar.pdf");
pageLegalDocumentation.GoToBack();
Wait.UrlContains(webDriver, pageLegalDocumentation.ExpectedUrl);
pageLegalDocumentation.BtnRetailRiskDisclosure.Click();
Wait.UrlContains(webDriver, @"https://s3.eu-central-1.amazonaws.com/files-mrkt/datarepo/legal/mena/Retail_FX_Risk_Disclosure_Statement.pdf");
pageLegalDocumentation.GoToBack();
Wait.UrlContains(webDriver, pageLegalDocumentation.ExpectedUrl);
pageLegalDocumentation.BtnTermsAndConditionsForFixedSpreadAccount.Click();
Wait.UrlContains(webDriver, @"https://s3.eu-central-1.amazonaws.com/files-mrkt/datarepo/legal/mena/Terms_and_Conditions_for_Fixed_Spread_Account.pdf");
pageLegalDocumentation.GoToBack();
Wait.UrlContains(webDriver, pageLegalDocumentation.ExpectedUrl);
}
private static async Task OnPageCrawlCompleted(PageCrawlCompletedArgs e, ContentContext _context)
{
if (e.CrawledPage.HttpRequestException != null ||
e.CrawledPage.HttpResponseMessage.StatusCode != HttpStatusCode.OK)
{
Console.WriteLine("Crawl of page failed {0}", e.CrawledPage.Uri.AbsoluteUri);
return;
}
if (string.IsNullOrEmpty(e.CrawledPage.Content.Text))
{
Console.WriteLine("Page had no content {0}", e.CrawledPage.Uri.AbsoluteUri);
return;
}
count++;
var httpStatus = e.CrawledPage.HttpResponseMessage.StatusCode;
var rawPageText = e.CrawledPage.Content.Text;
if (e.CrawledPage.Uri.AbsoluteUri == "https://www.webtoons.com/en/dailySchedule")
{
var context = BrowsingContext.New(Configuration.Default);
var parser = context.GetService <IHtmlParser>();
var document = parser.ParseDocument(rawPageText);
var comicCards = document.QuerySelectorAll(".daily_card_item");
foreach (var comicCard in comicCards)
{
var imageLink = new Uri(comicCard.QuerySelector("img").GetAttribute("src")).PathAndQuery;
var link = comicCard.GetAttribute("href");
var titleNo = HttpUtility.ParseQueryString(new Uri(link).Query).Get("title_no");
var genre = comicCard.QuerySelector(".genre").InnerHtml;
var subject = comicCard.QuerySelector(".subj").InnerHtml;
var author = comicCard.QuerySelector(".author").InnerHtml;
var cardModel = createWebToonModel(link, titleNo, imageLink, genre, subject, author);
var cardEntity = WebtoonProfile.MapCreateModelToEntity(cardModel);
var existingWeb = await _context.WebToons.FindAsync(titleNo);
if (existingWeb != null)
{
continue;
}
await _context.WebToons.AddAsync(cardEntity);
// await _context.SaveChangesAsync();
// Console.WriteLine(imageLink);
// await _context.SaveChangesAsync();
}
}
else if (e.CrawledPage.Uri.AbsoluteUri.Contains("list"))
{
var context = BrowsingContext.New(Configuration.Default);
var parser = context.GetService <IHtmlParser>();
var document = parser.ParseDocument(rawPageText);
var list = document.QuerySelectorAll("#_listUl li");
Console.WriteLine(list.Length);
foreach (var item in list)
{
var link = new Uri(e.CrawledPage.Uri.AbsoluteUri);
var titleNo = HttpUtility.ParseQueryString(link.Query).Get("title_no");
var episodeName = item.QuerySelector("a .subj").InnerHtml;
var episodeThumb = new Uri(item.QuerySelector("a .thmb img").GetAttribute("src")).PathAndQuery;
var episodeDate = item.QuerySelector("a .date").InnerHtml;
var episodeLink = item.QuerySelector("a").GetAttribute("href");
var episodeLinkHash = ComputeSha256Hash(episodeLink);
var episodeModel = createEpisodeModel(titleNo, episodeName, episodeThumb, episodeDate, episodeLink,
episodeLinkHash);
var episodeEntity = EpisodeProfile.MapCreateModelToEntity(episodeModel);
// Console.WriteLine(episodeThumb);
// var _context = Program.ServiceProvider.GetService<ContentContext>();
var existed = await _context.Episodes.FindAsync(episodeLinkHash);
if (existed != null)
{
continue;
}
await _context.AddAsync(episodeEntity);
// await _context.SaveChangesAsync();
// Console.WriteLine(episodeLinkHash);
}
}
else if (e.CrawledPage.Uri.AbsoluteUri.Contains("viewer"))
{
var context = BrowsingContext.New(Configuration.Default);
var parser = context.GetService <IHtmlParser>();
var document = parser.ParseDocument(rawPageText);
var content = document.QuerySelectorAll(".viewer_img img");
var contentLinks = content.Select(cont => new Uri(cont.GetAttribute("data-url")).PathAndQuery).ToList();
// foreach (var cont in contentLinks)
// {
// var linkData = new Uri(cont);
// Console.WriteLine(linkData.PathAndQuery);
// }
var episodeLink = e.CrawledPage.Uri.AbsoluteUri;
var epsiodeLinkHash = ComputeSha256Hash(episodeLink);
var contentJson = JsonSerializer.Serialize(contentLinks);
var pageModel = createPageModel(epsiodeLinkHash, contentJson);
var pageEntity = PageProfile.MapCreateModelToEntity(pageModel);
// var _context = Program.ServiceProvider.GetService<ContentContext>();
var existingPage = _context.Pages.Where(p => p.Content == contentJson).ToList();
if (existingPage.Count == 0)
{
await _context.AddAsync(pageEntity);
}
// await _context.SaveChangesAsync();
}
await _context.SaveChangesAsync();
Program.LogInfo(count.ToString());
}
