public async Task GetUsers()
{
await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision);
Browser browser = await Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = false
});
BrowserContext userContext = await browser.CreateIncognitoBrowserContextAsync();
Page communityPage = await userContext.NewPageAsync();
await communityPage.SetViewportAsync(new ViewPortOptions
{
Height = 768,
Width = 1024
});
string ver = await browser.GetVersionAsync();
string userAgent = await browser.GetUserAgentAsync();
await communityPage.SetUserAgentAsync("Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 74.0.3723.0 Safari / 537.36");
await communityPage.GoToAsync(CommunityUrl);
ElementHandle followersLoaded = await communityPage.WaitForSelectorAsync("#public_followers");
}
private async Task <VkBrwUser> InitializeAsync(VkBrwUser user)
{
await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision);
LaunchOptions browserOptions = new LaunchOptions {
Headless = false, Args = new string[] { "--lang=ru" }
};
Browser brw = await Puppeteer.LaunchAsync(browserOptions);
BrowserContext userContext = await brw.CreateIncognitoBrowserContextAsync();
Page userPage = await userContext.NewPageAsync();
await userPage.SetViewportAsync(new ViewPortOptions
{
Width = 1024,
Height = 768
});
await userPage.SetUserAgentAsync("Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 74.0.3723.0 Safari / 537.36");
user.Communitites = await GetCommunities(userContext, userPage, user);
bool successParse = long.TryParse(user.ProfileLink.Split('/').LastOrDefault()?.Substring(2), out long parseResult);
user.UserId = successParse ? parseResult : default;
await userPage.CloseAsync();
await userContext.CloseAsync();
return(user);
}
public async Task InitializeAsync()
{
var browserOptions = new BrowserTypeLaunchOptions
{
Headless = false,
SlowMo = SlowMo
};
// Add support to modify BaseUrl from outside.
string?isGitHubActions = Environment.GetEnvironmentVariable("IsGitHubActions");
string?baseUrl = Environment.GetEnvironmentVariable("BaseUrl");
if (isGitHubActions == "true")
{
// Assume it's in GitHub actions and we want to run it fast.
browserOptions.Headless = true;
browserOptions.SlowMo = null;
baseUrl ??= "https://autoscrum.jkdev.me";
}
if (!string.IsNullOrWhiteSpace(baseUrl))
{
BaseUrl = baseUrl;
}
PlaywrightInstance = await Playwright.CreateAsync();
Browser = await PlaywrightInstance.Chromium.LaunchAsync(browserOptions);
BrowserContext = await Browser.NewContextAsync();
Page = await BrowserContext.NewPageAsync();
await Page.GotoAsync(BaseUrl);
}
private async Task GetCommunityDetailsAsync(BrowserContext browserContext, VkBrwCommunity community)
{
Page communityPage = await browserContext.NewPageAsync();
////groupPage.EvaluateFunctionAsync($"() => window.open('{groupUrl}')").GetAwaiter().GetResult();
////Target newWindowTarget = browser.WaitForTargetAsync(target => target.Url == "https://www.example.com/").Result;
////Page newPage = newWindowTarget.PageAsync().Result;
try
{
await communityPage.GoToAsync(community.CommunityUrl);
}
catch (NavigationException)
{
return;
}
WaitForSelectorOptions waitSelectorOptions = new WaitForSelectorOptions {
Timeout = 10000
};
ElementHandle communityLoadedElement = await communityPage.WaitForSelectorAsync("div#page_wrap.scroll_fix_wrap._page_wrap", waitSelectorOptions);
if (communityLoadedElement != null)
{
ElementHandle communityBlockedElement = await communityPage.QuerySelectorAsync("div.groups_blocked");
if (communityBlockedElement != null)
{
community.Blocked = true;
}
else
{
community.Type = await GetCommunityTypeAsync(communityPage);
community.CommunityId = await GetCommunityIdAsync(communityPage, community.Type);
}
}
await communityPage.CloseAsync();
}
public async Task Run([QueueTrigger("%JUMBO_QUEUE_NAME%", Connection = "JUMBO_CONNECTION_STRING")] string searchTerm, ILogger log)
{
const string merchantName = "Jumbo";
var page = await BrowserContext.NewPageAsync().ConfigureAwait(false);
await page.GotoAsync($"https://www.jumbo.com/zoeken?searchTerms={HttpUtility.UrlEncode(searchTerm)}", new PageGotoOptions { Timeout = 0 });
var productElements = await page.QuerySelectorAllAsync("//*[@analytics-tag='product card']");
foreach (var productElement in productElements)
{
try
{
using Task <IElementHandle?> getProductTitle = productElement.QuerySelectorAsync("css=[class='title-link']");
using Task <IElementHandle?> getPriceUnits = productElement.QuerySelectorAsync("css=[class='whole']");
using Task <IElementHandle?> getPriceFraction = productElement.QuerySelectorAsync("css=[class='fractional']");
using Task <IElementHandle?> getUnitSizeAndMeasure = productElement.QuerySelectorAsync("css=[class='price-per-unit']");
using Task <IElementHandle?> getImageSource = productElement.QuerySelectorAsync("css=[class='image']");
using Task <IElementHandle?> getDetailsPageUrl = productElement.QuerySelectorAsync("css=a:first-child");
await Task.WhenAll(getImageSource, getPriceFraction, getUnitSizeAndMeasure, getPriceUnits, getProductTitle, getDetailsPageUrl);
IElementHandle?productTitleHandle = await getProductTitle.ConfigureAwait(false);
IElementHandle?productPriceUnitsHandle = await getPriceUnits.ConfigureAwait(false);
IElementHandle?productPriceFractionHandle = await getPriceFraction.ConfigureAwait(false);
IElementHandle?unitSizeAndMeasureHandle = await getUnitSizeAndMeasure.ConfigureAwait(false);
IElementHandle?imageElementHandle = await getImageSource.ConfigureAwait(false);
string?id = await productElement.GetAttributeAsync("data-product-Id").ConfigureAwait(false);
string?title = productTitleHandle is not null ? await productTitleHandle.TextContentAsync().ConfigureAwait(false) : "";
string?detailsPageLink = productTitleHandle is not null ? await productTitleHandle.GetAttributeAsync("href").ConfigureAwait(false) : "";
string?priceUnits = productPriceUnitsHandle is not null ? await productPriceUnitsHandle.TextContentAsync().ConfigureAwait(false) : "";
string?priceFraction = productPriceFractionHandle is not null ? await productPriceFractionHandle.TextContentAsync().ConfigureAwait(false) : "";
string?unitSize = "";
string?unitOfMeasure = "";
if (unitSizeAndMeasureHandle is not null)
{
var unitSizeAndMeasure = await unitSizeAndMeasureHandle.TextContentAsync().ConfigureAwait(false);
var parts = unitSizeAndMeasure is not null?unitSizeAndMeasure.Split('/') : new[] { "", "" };
unitSize = RemoveSpecialCharacters(parts[0]);
unitOfMeasure = RemoveSpecialCharacters(parts[1]);
}
string?imageSource = await imageElementHandle.GetAttributeAsync("src").ConfigureAwait(false);
string?description = "";
if (detailsPageLink is not null)
{
await using var detailsContext = await Browser.NewContextAsync(new BrowserNewContextOptions { }).ConfigureAwait(false);
var detailsPage = await detailsContext.NewPageAsync().ConfigureAwait(false);
await detailsPage.GotoAsync($"https://www.jumbo.com{detailsPageLink}", new PageGotoOptions { Timeout = 0 });
var descriptionElement = await detailsPage.QuerySelectorAsync("css=[analytics-tag='product description collapsible']");
description = descriptionElement is not null ? await descriptionElement.TextContentAsync() ?? string.Empty : "";
await detailsContext.CloseAsync().ConfigureAwait(false);
}
var validatedProduct = Product.Create(id, title, description, merchantName, priceUnits, priceFraction, imageSource, unitSize, unitOfMeasure);
switch (validatedProduct)
{
case Valid <Product>(var product):
var result = IndexDocumentsAction.MergeOrUpload(product.ToIndexableProduct());
await SearchClient.IndexDocumentsAsync(IndexDocumentsBatch.Create(result));
break;
case Invalid <Product>(var errors):
// TODO : proper tracing
foreach (var error in errors)
{
Console.WriteLine(error);
}
break;
default:
break;
}
}
catch (Exception ex)
{
Console.WriteLine(ex.StackTrace);
}
}
await page.CloseAsync();
}
private async Task DiscoverSystemAsync()
{
GeoDetector geoDetector = new GeoDetector();
CurrentCountry = await geoDetector.GetCountryAsync();
switch (CurrentCountry)
{
case Countries.RU:
Language = BrowsingLanguage.RU;
break;
case Countries.CN:
case Countries.US:
case Countries.DE:
case Countries.FR:
case Countries.IE:
case Countries.GB:
case Countries.SG:
Language = BrowsingLanguage.EN;
break;
default:
Language = BrowsingLanguage.RU;
break;
}
await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision);
LaunchOptions browserOptions = new LaunchOptions
{
Headless = true, Args = new string[] { "--lang=" + Language.ToString().ToLowerInvariant() }
};
Browser browser = await Puppeteer.LaunchAsync(browserOptions);
BrowserContext discoveryContext = await browser.CreateIncognitoBrowserContextAsync();
Page page = await discoveryContext.NewPageAsync();
await page.SetViewportAsync(new ViewPortOptions
{
Width = 1024,
Height = 768
});
await page.SetUserAgentAsync(UserAgent);
if (Mobile)
{
await DiscoverServiceMobile(page);
}
else
{
await DiscoverServiceDesktop(page);
}
if (!page.IsClosed)
{
await page.CloseAsync();
}
await discoveryContext.CloseAsync();
if (!browser.IsClosed)
{
await browser.CloseAsync();
}
}
/// <summary>
///
/// </summary>
/// <param name="urls"></param>
/// <param name="path"></param>
/// <param name="changeCountry">选择切换的国家</param>
/// <param name="randomCity">随机切换城市</param>
/// <param name="addText">添加页眉/页脚内容</param>
/// <returns></returns>
private async Task BulidPdfAsync(Uri[] urls, string path, string changeCountry, bool randomCity, string addText, bool saveHis, PaperFormat paperFormat)
{
try
{
await new BrowserFetcher(new BrowserFetcherOptions()
{
Host = "http://cdn.npm.taobao.org/dist"
}).DownloadAsync(BrowserFetcher.DefaultRevision);
var option = new LaunchOptions
{
UserDataDir = $"{System.IO.Directory.GetCurrentDirectory()}/UserData",
Headless = true,
DefaultViewport = null,
Args = new[] {
"--no-sandbox",
"--disable-setuid-sandbox",
"--lang=en-US,en",
"--start-maximized",
// "--disable-dev-shm-usage",
// "--disable-extensions",
// "--disable-gpu",
// "--disable-infobars",
//"--disable-local-storage",
// "--no-zygote",
// "--disable-bundled-ppapi-flash"
//$"--user-agent={_userAgent}"
},
IgnoreHTTPSErrors = true,
};
using (var browser = await Puppeteer.LaunchAsync(option))
{
BrowserContext browserContext = browser.DefaultContext;
//if (saveHis)
//{
// browserContext = await browser.CreateIncognitoBrowserContextAsync();
//}
var page = await browserContext.NewPageAsync();
await page.SetUserAgentAsync(_userAgent);
await page.SetViewportAsync(new ViewPortOptions
{
Width = Screen.PrimaryScreen.WorkingArea.Width,
Height = Screen.PrimaryScreen.WorkingArea.Height
});
if (!changeCountry.Contains("请选择") && _CountryModels.Any())
{
OnShowTooltipInfo("初始化首页内容,开始切换国家");
//var zips = Properties.Resources.zip.Split(new[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries);
//var zip = zips.OrderBy(m => Guid.NewGuid()).First();
await page.GoToAsync("https://www.amazon.com/");
var pagetitle = await page.GetTitleAsync();
try
{
var docid = _CountryModels.Find(m => m.Text.Contains(changeCountry));
if (docid != null)
{
await page.EvaluateFunctionAsync(Properties.Resources.oChange2, docid.Id);
}
}
catch (Exception ex)
{
MessageBox.Show("访问网址受限,自动切换地区失败,\r\n请等待亚马逊解封或切换新ip后再尝试\r\n" + ex.Message, pagetitle);
return;
}
await Task.Delay(8000);
}
else if (randomCity)//没有选择国家的情况，选择地区切换地区
{
OnShowTooltipInfo("初始化首页内容,开始切换地区");
var zips = Properties.Resources.zip.Split(new[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries);
var zip = zips.OrderBy(m => Guid.NewGuid()).First();
await page.GoToAsync("https://www.amazon.com/");
var pagetitle = await page.GetTitleAsync();
try
{
await InputMatchCaptch(page);
await page.EvaluateFunctionAsync(Properties.Resources.oChange, zip);
}
catch (Exception ex)
{
MessageBox.Show("访问网址受限,自动切换地区失败,\r\n请等待亚马逊解封或切换新ip后再尝试", pagetitle);
throw ex;
}
await Task.Delay(10000);
}
int index = 0;
int max = urls.Length;
var prelist = Properties.Resources.preload.Split(new[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries);
var maxwidth = Screen.PrimaryScreen.WorkingArea.Width;
foreach (var item in urls)
{
index += 1;
var ht = $"<div style=\"top:0px;font-size:10px;margin-left:10px;width:{maxwidth}px;\" ><span style=\"margin-left: 20px;\">{item}</span><span style=\"margin-left: 25%;\"></span><span style=\"margin-left:25%;\">{addText}</span></div>";
var ft = $"<div style=\"top:0px;font-size:10px;margin-left: 10px;width:{maxwidth}px;\"><span style=\"margin-left: 20px;\">{item}</span><span style=\"margin-left: 25%;\"></span><span style=\"margin-left:25%;\">{addText}</span></div>";
try
{
OnShowTooltipInfo($"开始加载页面{item}");
// progressBar.PerformStep();
//搜索记录
if (saveHis)
{
var preurl = prelist.OrderBy(m => Guid.NewGuid()).Take(6).ToList();
for (int i = 0; i < preurl.Count; i++)
{
await InputMatchCaptch(page);
var url = preurl[i];
OnShowTooltipInfo($"增加浏览记录 {(i + 1)}/{preurl.Count}");
await page.GoToAsync(url);
}
}
OnShowTooltipInfo("打开目标页面 " + item.ToString());
//目标页面
await page.GoToAsync(item.ToString());
await page.EvaluateFunctionAsync <string>("()=>{try{document.querySelector('#productTitle').click(); document.querySelector('.nav-signin-tt.nav-flyout').remove();return 1;}catch(ex){return 0;}}");
if (saveHis)
{
OnShowTooltipInfo("展示浏览记录");
var result = await page.EvaluateFunctionAsync <string>("()=>{try{window.scrollBy(0,document.querySelector('.navFooterBackToTopText').getBoundingClientRect().top-600);return 1;}catch(ex){console.log(ex);return 0;}}");
await Task.Delay(3000);
result = await page.EvaluateFunctionAsync <string>(Properties.Resources.oRandom);
await Task.Delay(5000);
}
var file = Path.Combine(path, item.AbsolutePath.Replace("/dp/", "").Replace("/", "") + ".pdf");
OnShowTooltipInfo("开始生成PDF " + file);
if (option.Headless)
{
await page.PdfAsync(file, new PdfOptions()
{
PrintBackground = true,
DisplayHeaderFooter = !string.IsNullOrEmpty(addText),
HeaderTemplate = ht,
FooterTemplate = ft,
MarginOptions = new PuppeteerSharp.Media.MarginOptions()
{
Top = "40",
Bottom = "40",
Left = "30",
Right = "30"
},
Format = paperFormat
});
}
OnSetProcessValue(index);
}
catch (Exception ex)
{
OnShowTooltipInfo(ex.Message + "操作失败 " + item);
continue;
// throw;
}
}
//if (saveHis)
//{
// var ck = await page.GetCookiesAsync("https://www.amazon.com");
// await page.DeleteCookieAsync(ck);
//}
OnShowTooltipInfo($"操作完成 {index}/{urls.Length}");
}
}
catch (Exception ex)
{
MessageBox.Show(ex.Message, "message", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
//throw;
}
}
private async Task <Page> NewWhatsappPage(BrowserContext context)
{
var page = await context.NewPageAsync();
await page.SetUserAgentAsync(USER_AGENT);
await page.SetViewportAsync(VIEW_PORT);
await page.GoToAsync("https://web.whatsapp.com/");
await page.ExposeFunctionAsync("messageReceived", (string text) =>
{
Console.WriteLine(text);
return(Task.CompletedTask);
});
await page.EvaluateFunctionAsync(@"
() => {
var observer = new MutationObserver((mutations) => {
for(var mutation of mutations) {
if(mutation.addedNodes.length) {
for(var node of mutation.addedNodes) {
var childIn = node.querySelector('div.message-in');
var childOut = node.querySelector('div.message-out');
if(childIn) {
var messageContainer = childIn.querySelector('div.copyable-text');
if(messageContainer) {
var nome = '';
if(messageContainer.getAttribute('data-pre-plain-text')) {
nome = messageContainer.getAttribute('data-pre-plain-text');
}
var texto = messageContainer.querySelector('span.selectable-text.copyable-text').innerText;
messageReceived(nome + texto);
}
} else if(childOut) {
var messageContainer = childOut.querySelector('div.copyable-text');
if(messageContainer) {
var nome = '';
if(messageContainer.getAttribute('data-pre-plain-text')) {
nome = messageContainer.getAttribute('data-pre-plain-text');
}
var texto = messageContainer.querySelector('span.selectable-text.copyable-text').innerText;
messageReceived(nome + texto);
}
}
}
}
}
});
for(var node of document.querySelectorAll('div.copyable-area>div[tabindex=\'0\']:nth-child(3)>div:nth-child(3)')) {
observer.observe(node, { childList: true, attributes: true });
}
}
");
/*
* @"
* () => {
* var observer = new MutationObserver((mutations) => {
* for(var mutation of mutations) {
* if(mutation.addedNodes.length) {
* for(var node in mutation.addedNodes) {
* if(node.tagName.toUpperCase() === 'DIV' && node.classList.contains('message-in') && node.classList.contains('tail')) {
* messageReceived(node.innerHTML);
* }
* }
* }
* }
* });
* observer.observe(document.querySelector('div#app'), { childList: true, attributes: true });
* }
* "
*/
return(page);
}
