// Dispose of WebBrowserPool
public void Dispose()
{
if (_apartment == null)
{
throw new ObjectDisposedException(this.GetType().Name);
}
// cancel and wait for all pending tasks
_cts.Cancel();
var task = _apartment.Run(() => Task.WhenAll(_pendingTasks.ToArray()));
try
{
task.Wait();
}
catch
{
if (!task.IsCanceled)
{
throw;
}
}
// dispose of WebBrowser objects
_apartment.Run(() =>
{
while (_browsers.Any())
{
_browsers.Dequeue().Dispose();
}
});
_apartment.Dispose();
_apartment = null;
}
// by Noseratio - http://stackoverflow.com/a/22262976/1768303
// main logic
static async Task ScrapSitesAsync(string[] urls, CancellationToken token)
{
using (var apartment = new MessageLoopApartment())
{
// create WebBrowser inside MessageLoopApartment
var webBrowser = apartment.Invoke(() => new WebBrowser());
try
{
foreach (var url in urls)
{
Console.WriteLine("URL:\n" + url);
// cancel in 30s or when the main token is signalled
var navigationCts = CancellationTokenSource.CreateLinkedTokenSource(token);
navigationCts.CancelAfter((int)TimeSpan.FromSeconds(30).TotalMilliseconds);
var navigationToken = navigationCts.Token;
// run the navigation task inside MessageLoopApartment
string html = await apartment.Run(() =>
webBrowser.NavigateAsync(url, navigationToken), navigationToken);
Console.WriteLine("HTML:\n" + html);
}
}
finally
{
// dispose of WebBrowser inside MessageLoopApartment
apartment.Invoke(() => webBrowser.Dispose());
}
}
}
CancellationTokenSource _cts; // global cancellation (for Dispose)
public WebBrowserPool(int maxParallel, CancellationToken token)
{
if (maxParallel < 1)
{
throw new ArgumentException("maxParallel");
}
_cts = CancellationTokenSource.CreateLinkedTokenSource(token);
_apartment = new MessageLoopApartment();
_semaphore = new SemaphoreSlim(maxParallel);
_browsers = new Queue <WebBrowser>();
_pendingTasks = new HashSet <Task>();
// init the pool of WebBrowser objects
_apartment.Invoke(() =>
{
while (--maxParallel >= 0)
{
_browsers.Enqueue(new WebBrowser());
}
});
}
