static async Task Main(string[] args)
{
var style = new LaunchOptions {
Headless = true
};
await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision);
Console.WriteLine("\nGet information from CGV...");
using (var browser = await Puppeteer.LaunchAsync(style))
using (var page = await browser.NewPageAsync()) {
await page.GoToAsync("https://www.cgv.id/en/movies/now_playing");
var jsSelectAllAnchors = await page.QuerySelectorAllHandleAsync(".movie-list-body > ul >li > a").EvaluateFunctionAsync <string[]> ("elements => elements.map(a => a.href)");
Console.WriteLine("\n=========Now Playing=========");
HtmlWeb web = new HtmlWeb();
for (int i = 0; i < jsSelectAllAnchors.Length; i++)
{
var htmlDoc = web.Load(jsSelectAllAnchors[i]);
Console.WriteLine("\n\n========================================================================================================");
var Title = htmlDoc.DocumentNode.SelectNodes("//div[@class='movie-info-title']");
foreach (var node in Title)
{
Console.WriteLine("\nTITLE :\n" + node.InnerText.Trim() + "\n");
}
var Starring = htmlDoc.DocumentNode.SelectNodes("//div[@class='movie-add-info left']/ul /li");
foreach (var node in Starring)
{
Console.WriteLine("> " + node.InnerText.Trim());
}
var Trailer = htmlDoc.DocumentNode.SelectNodes("//div[@class='trailer-btn-wrapper']/img");
foreach (var node in Trailer)
{
var getLink = node.GetAttributeValue("onclick", string.Empty);
Console.WriteLine("\n> TRAILER LINK : " + getLink.Remove(0, 11));
}
var Synopsis = htmlDoc.DocumentNode.SelectNodes("//div[@class='movie-synopsis right']");
foreach (var node in Synopsis)
{
Console.WriteLine("\n> SYNOPIS : " + node.InnerText.Trim());
}
}
}
}
public static async Task StartAsync()
{
await new BrowserFetcher().DownloadAsync();
var browser = await Puppeteer.LaunchAsync(new LaunchOptions { Headless = true, IgnoreHTTPSErrors = true, Args = new string[] { "--no-sandbox", "--disable-web-security" } });
_page = await browser.NewPageAsync();
await _page.GoToAsync("https://discord.com");
var properties = new SuperProperties();
await _page.SetUserAgentAsync(properties.UserAgent);
_superProps = properties.ToBase64();
}
public async Task TargetPageShouldReturnABackgroundPage()
{
using (var browserWithExtension = await Puppeteer.LaunchAsync(
TestConstants.BrowserWithExtensionOptions(),
TestConstants.LoggerFactory))
{
var backgroundPageTarget = await WaitForBackgroundPageTargetAsync(browserWithExtension);
var page = await backgroundPageTarget.PageAsync();
Assert.Equal(6, await page.EvaluateFunctionAsync <int>("() => 2 * 3"));
Assert.Equal(42, await page.EvaluateFunctionAsync <int>("() => window.MAGIC"));
}
}
private async Task <ExportStatementPage> LoadExportStatementsPage()
{
await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultChromiumRevision);
_browser = await Puppeteer.LaunchAsync(new LaunchOptions { Headless = true });
_page = await _browser.NewPageAsync();
await _page.SetDownloadPath(_options.DownloadPath);
var loginPage = new LoginPage(_page);
var balancesPage = await loginPage.Login(_options.UserName, _options.Password);
return(await balancesPage.GoToExportStatementPage());
}
public async Task <Browser> GetBrowser()
{
await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision);
return(await Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = true,
Args = new []
{
"--no-sandbox",
"--disable-setuid-sandbox"
}
}, _loggerFactory));
}
public async Task StartAsync()
{
await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision);
Console.WriteLine("Got Chrome Driver");
// Create an instance of the browser and configure launch options
this.browser = await Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = true,
Timeout = (int)TimeSpan.FromSeconds(10).TotalMilliseconds
});
Console.WriteLine("Created Browser Instance");
}
public async Task ShouldDownloadAndExtractLinuxBinary()
{
var browserFetcher = Puppeteer.CreateBrowserFetcher(new BrowserFetcherOptions
{
Platform = Platform.Linux,
Path = _downloadsFolder,
Host = TestConstants.ServerUrl
});
var revisionInfo = browserFetcher.RevisionInfo("123456");
Server.SetRedirect(revisionInfo.Url.Substring(TestConstants.ServerUrl.Length), "/chromium-linux.zip");
Assert.False(revisionInfo.Local);
Assert.Equal(Platform.Linux, revisionInfo.Platform);
Assert.False(await browserFetcher.CanDownloadAsync("100000"));
Assert.True(await browserFetcher.CanDownloadAsync("123456"));
try
{
revisionInfo = await browserFetcher.DownloadAsync("123456");
Assert.True(revisionInfo.Local);
Assert.Equal("LINUX BINARY\n", File.ReadAllText(revisionInfo.ExecutablePath));
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
#if NETCOREAPP //This will not be run on net4x anyway.
Mono.Unix.FileAccessPermissions permissions = ConvertPermissions(LinuxSysCall.ExecutableFilePermissions);
Assert.Equal(permissions, UnixFileSystemInfo.GetFileSystemEntry(revisionInfo.ExecutablePath).FileAccessPermissions & permissions);
#endif
}
Assert.Equal(new[] { "123456" }, browserFetcher.LocalRevisions());
browserFetcher.Remove("123456");
Assert.Empty(browserFetcher.LocalRevisions());
//Download should return data from a downloaded version
//This section is not in the Puppeteer test.
await browserFetcher.DownloadAsync("123456");
Server.Reset();
revisionInfo = await browserFetcher.DownloadAsync("123456");
Assert.True(revisionInfo.Local);
Assert.Equal("LINUX BINARY\n", File.ReadAllText(revisionInfo.ExecutablePath));
}
finally
{
EnsureDownloadsFolderIsDeleted();
}
}
public async Task ShouldFilterOutIgnoredDefaultArguments()
{
var defaultArgs = Puppeteer.GetDefaultArgs();
var options = TestConstants.DefaultBrowserOptions();
options.IgnoredDefaultArgs = new[] { defaultArgs[0], defaultArgs[2] };
using (var browser = await Puppeteer.LaunchAsync(options))
{
var spawnargs = browser.Process.StartInfo.Arguments;
Assert.DoesNotContain(defaultArgs[0], spawnargs);
Assert.Contains(defaultArgs[1], spawnargs);
Assert.DoesNotContain(defaultArgs[2], spawnargs);
}
}
public async Task ShouldDownloadAndExtractLinuxBinary()
{
var browserFetcher = Puppeteer.CreateBrowserFetcher(new BrowserFetcherOptions
{
Platform = Platform.Linux,
Path = _downloadsFolder,
Host = TestConstants.ServerUrl
});
var revisionInfo = browserFetcher.RevisionInfo(123456);
Server.SetRedirect(revisionInfo.Url.Substring(TestConstants.ServerUrl.Length), "/chromium-linux.zip");
Assert.False(revisionInfo.Local);
Assert.Equal(Platform.Linux, revisionInfo.Platform);
Assert.False(await browserFetcher.CanDownloadAsync(100000));
Assert.True(await browserFetcher.CanDownloadAsync(123456));
try
{
revisionInfo = await browserFetcher.DownloadAsync(123456);
Assert.True(revisionInfo.Local);
Assert.Equal("LINUX BINARY\n", File.ReadAllText(revisionInfo.ExecutablePath));
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
#if NETCOREAPP //don't need to run this code if we're not netcore app since net471 won't run on NIX. And UnixFileSystemInfo is not available for net471
Assert.Equal(
LinuxPermissionsSetter.ExecutableFilePermissions,
UnixFileSystemInfo.GetFileSystemEntry(revisionInfo.ExecutablePath).FileAccessPermissions & LinuxPermissionsSetter.ExecutableFilePermissions);
#endif
}
Assert.Equal(new[] { 123456 }, browserFetcher.LocalRevisions());
browserFetcher.Remove(123456);
Assert.Empty(browserFetcher.LocalRevisions());
//Download should return data from a downloaded version
//This section is not in the Puppeteer test.
await browserFetcher.DownloadAsync(123456);
Server.Reset();
revisionInfo = await browserFetcher.DownloadAsync(123456);
Assert.True(revisionInfo.Local);
Assert.Equal("LINUX BINARY\n", File.ReadAllText(revisionInfo.ExecutablePath));
}
finally
{
EnsureDownloadsFolderIsDeleted();
}
}
public async Task <Browser> LaunchAsync(LaunchOptions options)
{
_plugins.ForEach(e => e.BeforeLaunch(options));
var browser = await Puppeteer.LaunchAsync(options);
_plugins.ForEach(e => e.AfterLaunch(browser));
await OnStart(new BrowserStartContext()
{
StartType = StartType.Launch,
IsHeadless = options.Headless
}, browser);
return(browser);
}
public async Task ShouldRejectIfExecutablePathIsInvalid()
{
var options = TestConstants.DefaultBrowserOptions();
options.ExecutablePath = "random-invalid-path";
var exception = await Assert.ThrowsAsync <FileNotFoundException>(() =>
{
return(Puppeteer.LaunchAsync(options, TestConstants.LoggerFactory));
});
Assert.Equal("Failed to launch chrome! path to executable does not exist", exception.Message);
Assert.Equal(options.ExecutablePath, exception.FileName);
}
public async Task ShouldCloseBrowserWithBeforeunloadPage()
{
var headfulOptions = TestConstants.DefaultBrowserOptions();
headfulOptions.Headless = false;
await using (var browser = await Puppeteer.LaunchAsync(headfulOptions))
await using (var page = await browser.NewPageAsync())
{
await page.GoToAsync(TestConstants.ServerUrl + "/beforeunload.html");
// We have to interact with a page so that 'beforeunload' handlers fire.
await page.ClickAsync("body");
}
}
public async Task ShouldOpenDevtoolsWhenDevtoolsTrueOptionIsGiven()
{
var headfulOptions = TestConstants.DefaultBrowserOptions();
headfulOptions.Devtools = true;
await using (var browser = await Puppeteer.LaunchAsync(headfulOptions))
{
var context = await browser.CreateIncognitoBrowserContextAsync();
await Task.WhenAll(
context.NewPageAsync(),
context.WaitForTargetAsync(target => target.Url.Contains("devtools://")));
}
}
public async Task ShouldSupportIgnoreHTTPSErrorsOption()
{
var options = TestConstants.DefaultBrowserOptions();
options.IgnoreHTTPSErrors = true;
using (var browser = await Puppeteer.LaunchAsync(options, TestConstants.ChromiumRevision, TestConstants.LoggerFactory))
using (var page = await browser.NewPageAsync())
{
var response = await page.GoToAsync(TestConstants.HttpsPrefix + "/empty.html");
Assert.Equal(HttpStatusCode.OK, response.Status);
}
}
public async Task ShouldWorkWithNoDefaultArguments()
{
var options = TestConstants.DefaultBrowserOptions();
options.IgnoreDefaultArgs = true;
using (var browser = await Puppeteer.LaunchAsync(options, TestConstants.LoggerFactory))
{
Assert.Single(await browser.PagesAsync());
using (var page = await browser.NewPageAsync())
{
Assert.Equal(121, await page.EvaluateExpressionAsync <int>("11 * 11"));
}
}
}
public async Task CanLogin()
{
// 1. Create a new account against the InMemory database
var httpResponse = await _client.SendAsync(new HttpRequestMessage(HttpMethod.Post, "/api/accounts")
{
Content = new StringContent(System.Text.Json.JsonSerializer.Serialize(_signupRequests[1]), Encoding.UTF8, "application/json")
});
httpResponse.EnsureSuccessStatusCode();
// 2. Ensure PuppeteerSharp has the browser downloaded
await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision);
using (var browser = await Puppeteer.LaunchAsync(new LaunchOptions {
Headless = true
}))
{
using (var page = await browser.NewPageAsync())
{
// 3. Navigate to the test client page
await page.GoToAsync($"http://{_webHostFixture.Host}/test-client/index.html");
var navigationTask = page.WaitForNavigationAsync();
await Task.WhenAll(navigationTask, page.ClickAsync("button"));
// 4. Fill out the login form
await page.TypeAsync("#Username", _signupRequests[1].Email);
await page.TypeAsync("#Password", _signupRequests[1].Password);
// 5. Hit the login button and wait for redirect navigation...
navigationTask = page.WaitForNavigationAsync(new NavigationOptions {
WaitUntil = new[] { WaitUntilNavigation.Networkidle0 }
});
await Task.WhenAll(navigationTask, page.ClickAsync(".btn-primary"));
var content = await page.GetContentAsync();
await page.CloseAsync();
// 6. Assert we have a logged-in state in the test client
Assert.Contains("User logged in", content);
Assert.Contains("Prescott Terrell", content);
Assert.Contains("*****@*****.**", content);
Assert.Contains("employer", content);
}
};
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
log.LogInformation("C# HTTP trigger function processed a request.");
await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision);
string requestBody = await new StreamReader(req.Body).ReadToEndAsync();
dynamic data = JsonConvert.DeserializeObject(requestBody);
dynamic html = data?.html;
string local = data.local;
using (Browser browser = await Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = true,
Args = new[] { "--no-sandbox", "--disable-setuid-sandbox" },
DefaultViewport = new ViewPortOptions
{
Width = 2000,
Height = 800
}
}))
using (Page page = await browser.NewPageAsync())
{
if (Convert.ToBoolean(local))
{
string BaseDirectory = AppDomain.CurrentDomain.BaseDirectory;
html = File.ReadAllText("test_html.html");
}
await page.SetContentAsync(html);
string result = await page.GetContentAsync();
await page.PdfAsync("test.pdf");
byte[] content = File.ReadAllBytes("test.pdf");
//SaveHtmlToDB(result);
System.Net.Mime.ContentDisposition cd = new System.Net.Mime.ContentDisposition
{
FileName = "PDF TEST",
Inline = false // false = prompt the user for downloading; true = browser to try to show the file inline
};
return(new FileContentResult(content, "application/pdf"));
}
}
public virtual async Task <Snippet[]> GetSnippetsAsync(string searchQuery, int count, CancellationToken cancellationToken)
{
if (searchQuery == null)
{
return(null);
}
var snippets = new List <Snippet>(); // итоговый список сниппетов
// Для запроса страницы используется библиотека Puppeteer sharp
// она позволяет обходить защиту поисковых систем от парсинга
// подробнее тут https://www.puppeteersharp.com/api/index.html
await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision); // грузит браузер через который будут осуществляться запросы
// запуск браузера
Browser browser = await Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = true
});
Page page = await browser.NewPageAsync();
var parser = new HtmlParser();
int leftToTake = count;
int pageId = 0;
while (snippets.Count != count)
{
if (cancellationToken.IsCancellationRequested)
{
return(snippets.ToArray());
}
leftToTake -= snippets.Count;
await page.GoToAsync($"{Settings.BaseUrl}{searchQuery}{Settings.Page}{pageId++}"); // собственно получение обьекта страницы
var html = await page.GetContentAsync();
var document = await parser.ParseDocumentAsync(html); // строим DOM модель
var containers = document.QuerySelectorAll(Settings.MainContainerClass).Take(leftToTake);
GetSnippetsFromContainers(containers, ref snippets);
}
return(snippets.ToArray());
}
public async Task Run([QueueTrigger("orders")] string myQueueItem, ILogger log)
{
log.LogInformation($"Browser path: {appInfo.BrowserExecutablePath}");
int orderId = int.Parse(myQueueItem);
var browser = await Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = true,
ExecutablePath = appInfo.BrowserExecutablePath
});
var page = await browser.NewPageAsync();
await page.GoToAsync($"http://*****:*****@InvoiceURL where OrderId = @OrderId";
var affectedRows = connection.Execute(sql, new { OrderId = orderId, InvoiceURL = sasUri });
}
}
public static async Task Main(string[] args)
{
var options = new LaunchOptions {
Headless = true
};
Console.WriteLine("Downloading Chromium");
await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision);
Console.WriteLine("Goes to CGV Now PLaying");
using (var browser = await Puppeteer.LaunchAsync(options))
using (var page = await browser.NewPageAsync())
{
await page.GoToAsync("https://www.cgv.id/en/movies/now_playing");
var list = await page.QuerySelectorAllHandleAsync(".movie-list-body > ul >li > a").EvaluateFunctionAsync <string[]>("elements => elements.map(a => a.href)");
Console.WriteLine("Movie List");
for (int i = 0; i < list.Length; i++)
{
Console.WriteLine("==========================================================================================================");
Console.WriteLine("");
HtmlWeb web = new HtmlWeb();
var htmlDoc = web.Load(list[i]);
var nodes1 = htmlDoc.DocumentNode.SelectNodes("//div[@class='movie-info-title']");
foreach (var x in nodes1)
{
Console.WriteLine(x.InnerHtml.Trim());
}
var nodes2 = htmlDoc.DocumentNode.SelectNodes("//div[@class='trailer-btn-wrapper']/img");
foreach (var x in nodes2)
{
Console.WriteLine("Trailer : " + x.GetAttributeValue("onclick", string.Empty).Substring(11));
}
Console.WriteLine("PRODUCER : -");
var nodes3 = htmlDoc.DocumentNode.SelectNodes("//div[@class='movie-add-info left']/ ul / li");
foreach (var x in nodes3)
{
Console.WriteLine(x.InnerHtml.Trim());
}
var nodes4 = htmlDoc.DocumentNode.SelectNodes("//div[@class='movie-synopsis right']");
foreach (var x in nodes4)
{
Console.WriteLine("SINOPSIS : " + x.InnerText.Trim());
}
Console.WriteLine();
}
}
}
public async Task <FileContentResult> Get(string handle)
{
var profilePage = $"https://www.twitter.com/{handle}";
var badgeTemplate = "http://*****:*****@"() => {
return {
Name: document.querySelector('.ProfileHeaderCard-nameLink').innerText,
Bio: document.querySelector('.ProfileHeaderCard-bio').innerText,
ImageUrl: document.querySelector('.ProfileAvatar-image').src
};
}");
profile.Handle = handle;
}
using (var page = await browser.NewPageAsync())
{
await page.GoToAsync(badgeTemplate);
await page.WaitForSelectorAsync("#name");
await page.EvaluateFunctionAsync <TwitterProfile>(@"(profile) => {
console.log(profile);
document.querySelector('#name').innerText = profile.Name;
document.querySelector('#handle').innerText = profile.Handle;
document.querySelector('#avatar').src = profile.ImageUrl;
document.querySelector('#bio').innerText = profile.Bio;
}", profile);
var element = await page.QuerySelectorAsync("#card");
return(File(await element.ScreenshotDataAsync(), "image/png"));
}
}
}
public async Task ExecuteAsync()
{
var path = Path.Combine(Path.GetTempPath(), "FatTiger.png");
await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision);
using var browser = await Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = true,
Args = new string[] { "--no-sandbox" }
});
using var page = await browser.NewPageAsync();
await page.SetViewportAsync(new ViewPortOptions
{
Width = 1920,
Height = 1080
});
var url = "https://www.baidu.com";
await page.GoToAsync(url, WaitUntilNavigation.Networkidle0);
var content = await page.GetContentAsync();
await page.PdfAsync("FatTiger.pdf");
await page.ScreenshotAsync(path, new ScreenshotOptions
{
FullPage = true,
Type = ScreenshotType.Png
});
// 发送带图片的Email
var builder = new BodyBuilder();
var image = builder.LinkedResources.Add(path);
image.ContentId = MimeUtils.GenerateMessageId();
builder.HtmlBody = "当前时间:{0}.<img src=\"cid:{1}\"/>".FormatWith(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), image.ContentId);
var message = new MimeMessage
{
Subject = "【定时任务】每日热点数据抓取任务推送",
Body = builder.ToMessageBody()
};
await EmailHelper.SendAsync(message);
}
private void BuildBrowser()
{
lock (_locker)
{
if (_browser != null && _browser.IsConnected)
{
return;
}
_browser = Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = _options.Headless,
IgnoreHTTPSErrors = true
}).GetAwaiter().GetResult();
}
}
//se abre el navegador Chromium
public async Task AbrirNavegador()
{
try
{
this.browser = await Puppeteer.LaunchAsync(options);
this.page = await browser.NewPageAsync();
}
catch (Exception ex)
{
Console.WriteLine(ex);
System.Windows.Forms.MessageBox.Show(ex.Message);
Console.ReadKey();
}
}
async Task <Browser> CreateBrowser(ILogger log, ProxyConnectionCfg proxy)
{
var browser = await Puppeteer.LaunchAsync(new LaunchOptions {
Headless = CollectCfg.Headless,
DefaultViewport = new ViewPortOptions {
Width = 1024, Height = 1200
},
Args = new[] {
"--disable-dev-shm-usage", "--no-sandbox",
proxy != null ? $"--proxy-server={proxy.Url}" : null
}.NotNull().ToArray()
}); // logging has so much inconsequential errors we ignore it
return(browser);
}
public static async Task Main(string[] args)
{
var options = new LaunchOptions
{
Headless = true,
DumpIO = false,
ExecutablePath = args[0]
};
using (var browser = await Puppeteer.LaunchAsync(options))
{
Console.WriteLine(browser.WebSocketEndpoint);
Console.ReadLine();
}
}
private static async Task <Browser> PrepareHeadlessBrowser(bool isHeadless)
{
var options = new LaunchOptions
{
Headless = isHeadless,
Timeout = 0,
};
Console.WriteLine("Downloading chromium (if needed)");
await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision);
var browser = await Puppeteer.LaunchAsync(options);
return(browser);
}
protected PuppeteerLoader(Microsoft.Extensions.Configuration.IConfiguration configuration, ILogger <PuppeteerLoader> logger, bool headless)
{
this.logger = logger;
this.configuration = configuration;
this.headless = headless;
_ = new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision).Result;
browser = Puppeteer.LaunchAsync(new LaunchOptions
{
Headless = headless,
}).Result;
logger.LogInformation($"Create {nameof(Browser)}");
}
public async Task ShouldRejectAllPromisesWhenBrowserIsClosed()
{
using (var browser = await Puppeteer.LaunchAsync(
TestConstants.DefaultBrowserOptions(),
TestConstants.LoggerFactory))
using (var page = await browser.NewPageAsync())
{
var neverResolves = page.EvaluateFunctionHandleAsync("() => new Promise(r => {})");
await browser.CloseAsync();
var exception = await Assert.ThrowsAsync <TargetClosedException>(() => neverResolves);
Assert.Contains("Protocol error", exception.Message);
}
}
public static async Task Main(string[] args)
{
var options = new LaunchOptions
{
Headless = true,
DumpIO = true,
ExecutablePath = args[1]
};
using (var browser = await Puppeteer.LaunchAsync(options))
using (var page = await browser.NewPageAsync())
{
await page.EvaluateFunctionAsync("_dumpioTextToLog => console.log(_dumpioTextToLog)", args[0]);
}
}
