public async Task <ActionResult> TwitterCallback(TwitterCallbackParameters p) { //直リンやTwitterの認証拒否はトップページに飛ばす if (p.oauth_token == null || p.oauth_verifier == null) { HttpContext.Response.Headers.Add("Location", "/"); return(StatusCode(StatusCodes.Status303SeeOther)); } await p.InitValidate(HttpContext).ConfigureAwait(false); OAuth.OAuthSession Session; if ((Session = p.OAuthSession()) == null) { HttpContext.Response.Headers.Add("Location", "/"); return(StatusCode(StatusCodes.Status303SeeOther)); } // tokenをDBに保存 Tokens token = Session.GetTokens(p.oauth_verifier); var VeryfyTokenResult = await p.StoreNewLogin(token).ConfigureAwait(false); //すでにサインインしてたユーザーならそいつのページに飛ばす if (VeryfyTokenResult == DBHandler.VerifytokenResult.Exist) { HttpContext.Response.Headers.Add("Location", "/users/" + token.UserId.ToString()); return(StatusCode(StatusCodes.Status303SeeOther)); } else { //新規ユーザーはツイート等を取得させる CrawlManager.Run(token.UserId); HttpContext.Response.Headers.Add("Location", "/auth/first"); return(StatusCode(StatusCodes.Status303SeeOther)); } }
public async Task <ActionResult> Wait() { var Params = new LoginParameters(); await Params.InitValidate(HttpContext).ConfigureAwait(false); if (Params.ID.HasValue) { await CrawlManager.WhenCrawled(Params.ID.Value).ConfigureAwait(false); HttpContext.Response.Headers.Add("Location", "/users/" + Params.ID.Value.ToString()); return(StatusCode(StatusCodes.Status303SeeOther)); } else { HttpContext.Response.Headers.Add("Location", "/"); return(StatusCode(StatusCodes.Status303SeeOther)); } }
static void Main(string[] args) { List <string> urlList = new List <string> { "https://www.youtube.com/watch?v=e8CLsYzE5wk" }; CrawlStatusManager.Init(); CrawlStatusManager.AddPendingWebsites(urlList); var crawledWebsites = new ESWriteWebsitesManager(); var pendingWebsites = new ESWritePendingWebsitesManager(); var suggestions = new ESWriteSuggestionsManager(); CrawlManager crawlManager = new CrawlManager(crawledWebsites, pendingWebsites, suggestions); crawlManager.StartCrawlingAsync(); Console.ReadLine(); }
public ExtraBladetListener(CrawlManager manager) { this.manager = manager; }
public DrListener(CrawlManager manager) { this.manager = manager; }
public TitleCrawler(CrawlManager manager) { this.manager = manager; }
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env, CrawlManager crawlManager) { if (env.IsDevelopment()) { app.UseDeveloperExceptionPage(); app.UseDatabaseErrorPage(); } else { app.UseExceptionHandler("/Error"); // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts. // app.UseHsts(); } app.UseCors(); // app.UseHttpsRedirection(); app.UseStaticFiles(); app.UseSpaStaticFiles(); // //Hangfire Service //app.UseHangfireDashboard("/hangfire", new DashboardOptions //{ // Authorization = new[] { new MyAuthorizationFilter() }, // IgnoreAntiforgeryToken = true // <--This //}); app.UseRouting(); app.UseHangfireServer(); app.UseAuthentication(); app.UseIdentityServer(); app.UseAuthorization(); app.UseEndpoints(endpoints => { endpoints.MapControllerRoute( name: "default", pattern: "{controller}/{action=Index}/{id?}"); endpoints.MapRazorPages(); endpoints.MapHangfireDashboard(new DashboardOptions { Authorization = new[] { new MyAuthorizationFilter() }, IgnoreAntiforgeryToken = true // <--This }); }); app.UseSpa(spa => { spa.Options.SourcePath = "ClientApp"; if (env.IsDevelopment()) { spa.Options.StartupTimeout = TimeSpan.FromSeconds(120); spa.UseReactDevelopmentServer(npmScript: "start"); // spa.UseProxyToSpaDevelopmentServer("https://localhost:3000/"); } }); // Apply Migration to Database InitializeDatabase(app); CreateRoles(app.ApplicationServices); // Add HangFire // backgroundJobClient.Enqueue(() => Console.WriteLine("Hello Hangfire job!!!!")); // recurringJobManager.AddOrUpdate("Run Every Minute", () => Console.WriteLine("Test Recuring Job !!!"), "* * * * *"); //CrawlUtils.Instance.createJobs(); crawlManager.CreateEnabledSourcesJobs(); }