public async Task TestUnusualUris()
{
DatabasePool.Initialise(1);
var uri = new Uri("https://en.wikipedia.org/w/index.php?title=Talk:Event_Horizon_Telescope&action=edit");
Assert.IsTrue(await VerifyUrlIsBlocked(uri), uri.ToString());
uri = new Uri("http://omim.org/contact");
Assert.IsTrue(await VerifyUrlIsBlocked(uri), uri.ToString());
uri = new Uri("https://www.wikipedia.org/wiki/Burger");
Assert.IsTrue(!await VerifyUrlIsBlocked(uri), uri.ToString());
uri = new Uri("https://www.bbc.com/news/world-asia-40360168");
Assert.IsTrue(!await VerifyUrlIsBlocked(uri), uri.ToString());
uri = new Uri("https://wiki.dolphin-emu.org/index.php?title=Category:Japan_(Release_region)");
Assert.IsTrue(!await VerifyUrlIsBlocked(uri), uri.ToString());
}
public static async Task Main(string[] args)
{
if (args.Length < 2)
{
Usage();
return;
}
string paths = args[0];
int.TryParse(args[1], out int startPacketIndex);
// turn on log4net
log4net.Config.XmlConfigurator.Configure();
// catch all errors and log them
AppDomain.CurrentDomain.UnhandledException += (sender, eventArgs) => Utility.LogException(eventArgs.ExceptionObject as Exception);
// ignore all certificate validation issues
ServicePointManager.ServerCertificateValidationCallback = (sender, cert, chain, sslPolicyErrors) => true;
// console encoding will now be unicode
Console.OutputEncoding = System.Text.Encoding.UTF8;
// database init
DatabasePool.Initialise();
// configure fetcho
await SetupConfiguration(paths);
// upgrade the database
DatabaseUpgrader.Upgrade();
// buffers to connect the seperate tasks together
BufferBlock <IEnumerable <QueueItem> > prioritisationBuffer = CreateBufferBlock(DefaultBufferBlockLimit);
// really beef this buffers max size up since it takes for ever accumulate so we dont want to lose any
BufferBlock <IEnumerable <QueueItem> > fetchQueueBuffer = CreateBufferBlock(DefaultBufferBlockLimit * 1000);
//BufferBlock<IEnumerable<QueueItem>> requeueBuffer = CreateBufferBlock(DefaultBufferBlockLimit);
ITargetBlock <IEnumerable <QueueItem> > outboxWriter = CreateOutboxWriter();
BufferBlock <IWebResourceWriter> dataWriterPool = CreateDataWriterPool();
// fetcho!
var readLinko = new ReadLinko(prioritisationBuffer, startPacketIndex);
var queueo = new Queueo(prioritisationBuffer, fetchQueueBuffer, outboxWriter); // DataflowBlock.NullTarget<IEnumerable<QueueItem>>()
var fetcho = new Fetcho(fetchQueueBuffer, DataflowBlock.NullTarget <IEnumerable <QueueItem> >(), dataWriterPool);
var stato = new Stato("stats.csv", fetcho, queueo, readLinko);
var controlo = new Controlo(prioritisationBuffer, fetchQueueBuffer, dataWriterPool, () =>
{
readLinko.Shutdown();
queueo.Shutdown();
fetcho.Shutdown();
stato.Shutdown();
});
//var requeueWriter = new BufferBlockObjectFileWriter<IEnumerable<QueueItem>>(cfg.DataSourcePath, "requeue", requeueBuffer);
//var rejectsWriter = new BufferBlockObjectFileWriter<IEnumerable<QueueItem>>(cfg.DataSourcePath, "rejects", new NullTarget);
// execute
var tasks = new List <Task>();
tasks.Add(stato.Process());
tasks.Add(fetcho.Process());
await Task.Delay(1000);
tasks.Add(queueo.Process());
await Task.Delay(1000);
tasks.Add(readLinko.Process());
tasks.Add(controlo.Process());
await Task.WhenAll(tasks.ToArray()).ConfigureAwait(false);
CloseAllWriters(dataWriterPool);
DatabasePool.DestroyAll();
}