public async Task Get_link_fail()
{
//Arrange
var fakeLink = new Link
{
FoundOn = "http://www.google.com",
Address = "http://www.test.com",
Date = DateTime.Now,
IsCrawled = true,
Response = "Fake response"
};
var options = new DbContextOptionsBuilder <CrawlerContext>()
.UseInMemoryDatabase("Test")
.Options;
var context = new CrawlerContext(options);
context.Add(fakeLink);
context.SaveChanges();
//Act
var linksController = new LinksController(context);
var actionResult = await linksController.GetLink(5);
//Assert
Assert.IsInstanceOfType(actionResult.Result, typeof(NotFoundResult));
}
private void _btnStartFolder_Click(object sender, EventArgs e)
{
if (String.IsNullOrEmpty(folderBrowser.SelectedPath))
{
_lblDir.Text = "Select a dir, dork";
return;
}
string[] files = Directory.GetFiles(folderBrowser.SelectedPath, "*.txt", SearchOption.AllDirectories);
_resultAction = new Action <IRequest>(Write);
_pageCounterAction = new Action(UpdateSiteCount);
FileSaver.FileName = "SocialiteFindings-" + DateTime.Now.ToString("MM-dd-yyyy-hh-mm-ss") + ".txt";
foreach (string file in files)
{
string[] lines = File.ReadAllLines(file);
foreach (string line in lines)
{
if (String.IsNullOrEmpty(line))
{
continue;
}
CrawlerContext.Initialize();
CrawlerContext.LightMode = true;
CrawlerContext.Depth = 4;
CrawlerContext.SetURL(line);
Crawler.CrawlSite(_resultAction, _pageCounterAction);
}
}
}
public async Task InitializeAsync(CrawlerContext ctx)
{
if (ctx == null)
{
throw new ArgumentNullException(nameof(ctx));
}
await this._initialize(ctx);
}
private void Form1_Load(object sender, EventArgs e)
{
_radBrute.Checked = true;
CrawlerContext.Initialize();
_pnlPortsSelect.Visible = false;
_txtURL.Text = "blackdoorsec.net";
_chkScreenshot.Enabled = false;
}
public Task <bool> ShouldTerminateAsync(CrawlerContext ctx)
{
if (ctx == null)
{
throw new ArgumentNullException(nameof(ctx));
}
return(this._shouldTerminate(ctx));
}
private void Form1_Load(object sender, EventArgs e)
{
CrawlerContext.Initialize();
CrawlerContext.LightMode = true;
_txtURL.Text = "https://www.buzzfeed.com";//"http://www.algolia.com/press";//
foreach (string domain in Common.CommonDomainData.Select(x => x.DomainName))
{
_lstTarget.Items.Add(domain);
}
}
private void _bgwScanner_DoWork(object sender, DoWorkEventArgs e)
{
try
{
// CrawlerContext.SessionFileName = "Scraper";
// Core.SQLite.Accessor.Settings.ConnectionString = "data source=" + CrawlerContext.SessionFileName;
CrawlerContext.SetURL(_txtURL.Text);
_resultAction = new Action <IRequest>(Write);
_pageCounterAction = new Action(UpdateSiteCount);
Crawler.CrawlSite(_resultAction, _pageCounterAction);
//LinearCrawler.CrawlSite(DataGridWrite, SiteCounter);
}
catch (Exception ex)
{
MessageBox.Show("Scanner Error: " + ex.Message);
}
}
private TResult Execute <TResult>(Func <CrawlerContext, TResult> func)
{
try
{
TResult result = default(TResult);
if (func is null)
{
return(result);
}
using (var context = new CrawlerContext())
{
result = func.Invoke(context);
}
return(result);
}
catch (Exception ex)
{
throw ex;
}
}
protected override DbSet <ProxyWebApi> GetDbSet(CrawlerContext crawlerContext)
{
return(crawlerContext.ProxyWebApis);
}
public SearchController(ILogger <SearchController> logger, CrawlerContext db, EsConnectionOptions esConnectionOptions)
{
_logger = logger;
this.db = db;
_esConnectionOptions = esConnectionOptions;
}
public CofeeLandsController(CrawlerContext context, IConfiguration config) : base(context, config)
{
}
public SaigonsController(CrawlerContext context, IConfiguration config) : base(context, config)
{
}
public TiaSangController(CrawlerContext context, IConfiguration config) : base(context, config)
{
}
public ViettimesController(CrawlerContext context, IConfiguration config) : base(context, config)
{
}
protected override DbSet <Work> GetDbSet(CrawlerContext crawlerContext)
{
return(crawlerContext.Works);
}
public NhadatController(CrawlerContext context, IConfiguration config) : base(context, config)
{
}
public CungcauController(CrawlerContext context, IConfiguration config) : base(context, config)
{
}
public UriPageController(ILogger <UriPageController> logger, CrawlerContext db)
{
_logger = logger;
this.db = db;
}
public EfBaseRepository(CrawlerContext context)
{
_context = context;
_table = _context.Set <TEntity>();
}
public TopbankController(CrawlerContext context, IConfiguration config) : base(context, config)
{
}
protected override DbSet <Author> GetDbSet(CrawlerContext crawlerContext)
{
return(crawlerContext.Authors);
}
public TuoitrecuoiController(CrawlerContext context, IConfiguration config) : base(context, config)
{
}
public VnexpressController(CrawlerContext context, IConfiguration config) : base(context, config)
{
}
public GenericRepository()
{
_dbContext = new CrawlerContext();
_entities = _dbContext.Set <TEntity>();
}
protected override DbSet <DownLoadFile> GetDbSet(CrawlerContext crawlerContext)
{
return(crawlerContext.DownLoadFiles);
}
//private static BaseHelper<T> instance;
//public static BaseHelper<T> Instance
//{
// get
// {
// if (instance is null)
// {
// instance = new BaseHelper<T>();
// }
// return instance;
// }
//}
protected virtual DbSet <T> GetDbSet(CrawlerContext crawlerContext)
{
return(null);
}
public SpidersController(CrawlerContext context, IConfiguration config)
{
_config = config;
_context = context;
}
public LinksController(CrawlerContext context)
{
_context = context;
//_hubContext = hubContext; //uncomment to use SignalR
}
public DantriController(CrawlerContext context, IConfiguration config) : base(context, config)
{
}
public ThoibaoController(CrawlerContext context, IConfiguration config) : base(context, config)
{
}
