public async Task AsyncJobQueue_TestWithOwnToken()
{
IAsyncJobQueue <string> tq = new AsyncJobQueue <string>();
Guid token = Guid.NewGuid();
Assert.Equal(token, await tq.QueueJob(EmitStrings(), token));
(string val, bool valid) = await tq.GetNext(token);
Assert.Equal("Hello World", val);
Assert.True(valid);
(val, valid) = await tq.GetNext(token);
Assert.Equal("Goodbye World", val);
Assert.True(valid);
(val, valid) = await tq.GetNext(token);
Assert.False(valid);
Assert.Null(val);
}
public async Task AsyncJobQueue_HandlesCancellation()
{
IAsyncJobQueue <string> tq = new AsyncJobQueue <string>(false);
var token = await tq.QueueJob(EmitStrings());
int times = 0;
var ct = new CancellationTokenSource();
await foreach (string x in tq.AsEnumerable(token))
{
times++;
tq.RequestCancellation(token);
}
Assert.Equal(1, times);
var(val, next) = await tq.GetNext(token);
Assert.False(next);
Assert.Null(val);
Assert.True(tq.TryRemoveSource(token, out var _));
}
public async Task AsyncJobQueue_Test()
{
IAsyncJobQueue <string> tq = new AsyncJobQueue <string>();
var token = await tq.QueueJob(EmitStrings());
(string val, bool next) = await tq.GetNext(token);
Assert.Equal("Hello World", val);
Assert.True(next);
(val, next) = await tq.GetNext(token);
Assert.True(next);
Assert.Equal("Goodbye World", val);
(val, next) = await tq.GetNext(token);
Assert.False(next);
Assert.Null(val);
(val, next) = await tq.GetNext(token);
Assert.False(next);
Assert.Null(val);
}
public async Task AsyncJobQueue_ContextTest()
{
IAsyncJobQueue <string> tq = new AsyncJobQueue <string>(false);
var token = await tq.QueueJob(EmitStrings());
(string val, bool next) = await tq.GetNext(token);
Assert.Equal("Hello World", val);
Assert.False(tq.TryRemoveSource(token, out var _));
await foreach (string nextval in tq.AsEnumerable(token))
{
Assert.Equal("Goodbye World", nextval);
}
(val, next) = await tq.GetNext(token);
Assert.False(next);
Assert.Null(val);
var original = tq.GetSource(token);
await foreach (string nextval in original)
{
Assert.True(nextval == "Hello World" || nextval == "Goodbye World");
}
Assert.True(tq.TryRemoveSource(token, out var originalNext));
await foreach (string nextval in originalNext)
{
Assert.True(nextval == "Hello World" || nextval == "Goodbye World");
}
Assert.False(tq.TryRemoveSource(token, out var _));
}
static void Init()
{
//Coming from KSP V6 configs
int crawlerMaxQueuedJob = 50000;
int crawlerMaxConcurrentJob = 5;
int crawlerAsyncTimeout = 18000;
bool crawlerEnableTpsControl = true;
int crawlerTps = 8;
int crawlerSyncTimeout = 18000;
string proxyServer = "";
bool proxyBypassOnLocal = true;
int cacheSize = 5000;
string cacheExpireTimeSpanString = "1.00:00:00";
int connectionLimit = 50;
allowAutoRedirect = false;
userAgent = defaultUserAgent;
if (jobQueue == null)
{
jobQueue = new AsyncJobQueue<CrawlerState>(ProcessRequest, AbortJob);
}
else
{
jobQueue.StopProcess();
}
jobQueue.MaxQueuedJob = crawlerMaxQueuedJob;
int maxConcurrentJob = crawlerMaxConcurrentJob;
jobQueue.MaxConcurrentJob = maxConcurrentJob;
maxConcurrentJobControl = new Semaphore(maxConcurrentJob, maxConcurrentJob);
AsyncTimeout = crawlerAsyncTimeout;
jobQueue.AsyncTimeout = AsyncTimeout;
jobQueue.EnableTpsControl = crawlerEnableTpsControl;
if (jobQueue.EnableTpsControl)
{
jobQueue.Tps = crawlerTps;
}
jobQueue.StartProcess();
SyncTimeout = crawlerSyncTimeout;
if (!string.IsNullOrEmpty(proxyServer))
{
crawlerProxy = new WebProxy(proxyServer);
try
{
crawlerProxy.BypassProxyOnLocal = proxyBypassOnLocal;
}
catch
{
crawlerProxy.BypassProxyOnLocal = true;
}
}
try
{
lruCache = new LruCache<string, UrlContent>(cacheSize);
}
catch
{
lruCache = new LruCache<string, UrlContent>(5000);
}
try
{
cacheExpireTimeSpan = TimeSpan.Parse(cacheExpireTimeSpanString);
}
catch
{
cacheExpireTimeSpan = TimeSpan.FromDays(1);
}
// Set Connection limit
if (connectionLimit > ServicePointManager.DefaultConnectionLimit)
{
ServicePointManager.DefaultConnectionLimit = (int)connectionLimit;
}
}
static void Init()
{
//Coming from KSP V6 config
string msnWebServiceUrl = "http://api.bing.net:80/soap.asmx";
string appId = "67C2363425B169EE6ECD80C8AFD7C1F2387A8A47";
string msnProxyServer = "";
bool msnProxyBypassOnLocal = true;
int msnSyncTimeout = 180000;
int msnMaxQueuedJob = 50000;
int msnMaxConcurrentJob = 5;
bool msnEnableTpsControl = false;
int msnTps = 8;
int msnAsyncTimeout = 180000;
int cacheSize = 5000;
string cacheExpireTimeSpanString = "1.00:00:00";
int connectionLimit = 50;
serviceUrl = msnWebServiceUrl;
appID = appId;
string proxyServer = msnProxyServer;
if (!string.IsNullOrEmpty(proxyServer))
{
WebProxy msnProxy = new WebProxy(proxyServer);
try
{
msnProxy.BypassProxyOnLocal = msnProxyBypassOnLocal;
}
catch
{
msnProxy.BypassProxyOnLocal = true;
}
s = new BingService(serviceUrl, msnProxy);
}
else
{
s = new BingService(serviceUrl);
}
s.Timeout = msnSyncTimeout;
if (jobQueue == null)
{
jobQueue = new AsyncJobQueue<SnippetGatherState>(ProcessJob, AbortJob);
}
else
{
jobQueue.StopProcess();
}
jobQueue.MaxQueuedJob = msnMaxQueuedJob;
int maxConcurrentJob = msnMaxConcurrentJob;
jobQueue.MaxConcurrentJob = maxConcurrentJob;
maxConcurrentJobControl = new Semaphore(maxConcurrentJob, maxConcurrentJob);
jobQueue.EnableTpsControl = msnEnableTpsControl;
if (jobQueue.EnableTpsControl)
{
jobQueue.Tps = msnTps;
}
jobQueue.AsyncTimeout = msnAsyncTimeout;
jobQueue.StartProcess();
try
{
lruCache = new LruCache<string, QueryResponse>(cacheSize);
}
catch
{
lruCache = new LruCache<string, QueryResponse>(5000);
}
try
{
cacheExpireTimeSpan = TimeSpan.Parse(cacheExpireTimeSpanString);
}
catch
{
cacheExpireTimeSpan = TimeSpan.FromDays(1);
}
if (connectionLimit > ServicePointManager.DefaultConnectionLimit)
{
ServicePointManager.DefaultConnectionLimit = connectionLimit;
}
}
