public async Task Get_link_fail() { //Arrange var fakeLink = new Link { FoundOn = "http://www.google.com", Address = "http://www.test.com", Date = DateTime.Now, IsCrawled = true, Response = "Fake response" }; var options = new DbContextOptionsBuilder <CrawlerContext>() .UseInMemoryDatabase("Test") .Options; var context = new CrawlerContext(options); context.Add(fakeLink); context.SaveChanges(); //Act var linksController = new LinksController(context); var actionResult = await linksController.GetLink(5); //Assert Assert.IsInstanceOfType(actionResult.Result, typeof(NotFoundResult)); }
private void _btnStartFolder_Click(object sender, EventArgs e) { if (String.IsNullOrEmpty(folderBrowser.SelectedPath)) { _lblDir.Text = "Select a dir, dork"; return; } string[] files = Directory.GetFiles(folderBrowser.SelectedPath, "*.txt", SearchOption.AllDirectories); _resultAction = new Action <IRequest>(Write); _pageCounterAction = new Action(UpdateSiteCount); FileSaver.FileName = "SocialiteFindings-" + DateTime.Now.ToString("MM-dd-yyyy-hh-mm-ss") + ".txt"; foreach (string file in files) { string[] lines = File.ReadAllLines(file); foreach (string line in lines) { if (String.IsNullOrEmpty(line)) { continue; } CrawlerContext.Initialize(); CrawlerContext.LightMode = true; CrawlerContext.Depth = 4; CrawlerContext.SetURL(line); Crawler.CrawlSite(_resultAction, _pageCounterAction); } } }
public async Task InitializeAsync(CrawlerContext ctx) { if (ctx == null) { throw new ArgumentNullException(nameof(ctx)); } await this._initialize(ctx); }
private void Form1_Load(object sender, EventArgs e) { _radBrute.Checked = true; CrawlerContext.Initialize(); _pnlPortsSelect.Visible = false; _txtURL.Text = "blackdoorsec.net"; _chkScreenshot.Enabled = false; }
public Task <bool> ShouldTerminateAsync(CrawlerContext ctx) { if (ctx == null) { throw new ArgumentNullException(nameof(ctx)); } return(this._shouldTerminate(ctx)); }
private void Form1_Load(object sender, EventArgs e) { CrawlerContext.Initialize(); CrawlerContext.LightMode = true; _txtURL.Text = "https://www.buzzfeed.com";//"http://www.algolia.com/press";// foreach (string domain in Common.CommonDomainData.Select(x => x.DomainName)) { _lstTarget.Items.Add(domain); } }
private void _bgwScanner_DoWork(object sender, DoWorkEventArgs e) { try { // CrawlerContext.SessionFileName = "Scraper"; // Core.SQLite.Accessor.Settings.ConnectionString = "data source=" + CrawlerContext.SessionFileName; CrawlerContext.SetURL(_txtURL.Text); _resultAction = new Action <IRequest>(Write); _pageCounterAction = new Action(UpdateSiteCount); Crawler.CrawlSite(_resultAction, _pageCounterAction); //LinearCrawler.CrawlSite(DataGridWrite, SiteCounter); } catch (Exception ex) { MessageBox.Show("Scanner Error: " + ex.Message); } }
private TResult Execute <TResult>(Func <CrawlerContext, TResult> func) { try { TResult result = default(TResult); if (func is null) { return(result); } using (var context = new CrawlerContext()) { result = func.Invoke(context); } return(result); } catch (Exception ex) { throw ex; } }
protected override DbSet <ProxyWebApi> GetDbSet(CrawlerContext crawlerContext) { return(crawlerContext.ProxyWebApis); }
public SearchController(ILogger <SearchController> logger, CrawlerContext db, EsConnectionOptions esConnectionOptions) { _logger = logger; this.db = db; _esConnectionOptions = esConnectionOptions; }
public CofeeLandsController(CrawlerContext context, IConfiguration config) : base(context, config) { }
public SaigonsController(CrawlerContext context, IConfiguration config) : base(context, config) { }
public TiaSangController(CrawlerContext context, IConfiguration config) : base(context, config) { }
public ViettimesController(CrawlerContext context, IConfiguration config) : base(context, config) { }
protected override DbSet <Work> GetDbSet(CrawlerContext crawlerContext) { return(crawlerContext.Works); }
public NhadatController(CrawlerContext context, IConfiguration config) : base(context, config) { }
public CungcauController(CrawlerContext context, IConfiguration config) : base(context, config) { }
public UriPageController(ILogger <UriPageController> logger, CrawlerContext db) { _logger = logger; this.db = db; }
public EfBaseRepository(CrawlerContext context) { _context = context; _table = _context.Set <TEntity>(); }
public TopbankController(CrawlerContext context, IConfiguration config) : base(context, config) { }
protected override DbSet <Author> GetDbSet(CrawlerContext crawlerContext) { return(crawlerContext.Authors); }
public TuoitrecuoiController(CrawlerContext context, IConfiguration config) : base(context, config) { }
public VnexpressController(CrawlerContext context, IConfiguration config) : base(context, config) { }
public GenericRepository() { _dbContext = new CrawlerContext(); _entities = _dbContext.Set <TEntity>(); }
protected override DbSet <DownLoadFile> GetDbSet(CrawlerContext crawlerContext) { return(crawlerContext.DownLoadFiles); }
//private static BaseHelper<T> instance; //public static BaseHelper<T> Instance //{ // get // { // if (instance is null) // { // instance = new BaseHelper<T>(); // } // return instance; // } //} protected virtual DbSet <T> GetDbSet(CrawlerContext crawlerContext) { return(null); }
public SpidersController(CrawlerContext context, IConfiguration config) { _config = config; _context = context; }
public LinksController(CrawlerContext context) { _context = context; //_hubContext = hubContext; //uncomment to use SignalR }
public DantriController(CrawlerContext context, IConfiguration config) : base(context, config) { }
public ThoibaoController(CrawlerContext context, IConfiguration config) : base(context, config) { }