public static void AddConsoleLogging(ThreadedWebClientWorker downloader)
{
downloader.DownloaderThreadJobChanged += (sender, id, job) =>
{
if (job == null)
{
// Console.WriteLine("D[" + id + "] Inavtive");
}
else
{
Console.WriteLine("D[" + id + "] " + job);
}
};
downloader.DownloaderThreadStatus += (sender, id, status) =>
{
if (status)
{
//Console.WriteLine("D[" + id + "] active");
}
else
{
// Console.WriteLine("D[" + id + "] Inavtive");
}
};
downloader.WorkerThreadError += (sender, id, job, exp) =>
{
Console.WriteLine("W[" + id + "] Error: " + job);
Console.WriteLine(exp.Message);
};
downloader.WorkerThreadJobDone += (sender, id, job) =>
{
Console.WriteLine("W[" + id + "] Done: " + job);
};
downloader.WorkerThreadStatusChanged += (sender, id, job) =>
{
if (job == null)
{
// Console.WriteLine("W[" + id + "] Inavtive");
}
else
{
Console.WriteLine("W[" + id + "] " + job);
}
};
downloader.DownloaderBadProxyRemoved += (sender, id, proxy) =>
{
Console.WriteLine("D[" + id + "] Bad proxy remove " + proxy.Ip + ":" + proxy.Port + " from " +
proxy.Country);
};
downloader.DownloaderNoGoodProxyLeft += sender =>
{
Console.WriteLine("No good proxies left, have to terminate!");
};
}
public DownloadManagerGuiWindow(ThreadedWebClientWorker worker)
{
InitializeComponent();
this.viewModel = new MainViewModel();
this.DataContext = this.viewModel;
this.viewModel.WorkThreadCount = 2;
this.viewModel.DownloadThreadCount = 50;
this.viewModel.DownloadThreadCount = worker.GetDownloadThreadCount();
this.viewModel.WorkThreadCount = worker.GetWorkerThreadCount();
this.worker = worker;
}
public static void AttachGui(ThreadedWebClientWorker worker)
{
var task = new Thread(() =>
{
System.Windows.Application application = new System.Windows.Application();
application.Run(new DownloadManagerGuiWindow(worker));
});
task.SetApartmentState(ApartmentState.STA);
worker.WorkDone += sender =>
{
try
{
task.Abort();
}
catch (Exception)
{
}
};
task.Start();
}
public void Run()
{
logger.Info("Uk Data Scraper Started.");
logger.Info("Max holder queue: {0}.", MaxQueue);
logger.Info("Worker Threads: {0}.", WorkerThreads);
logger.Info("Download Threads: {0}.", DownloadThreads);
logger.Info("Use Proxies: {0} ({1}).", UseProxie, ProxieFile);
var initalJobs = new List<IThreadedWebClientJob>();
// SCS
var holderListSCS = new HolderXml<Sofa>("output/Sofa.{0}.xml", MaxQueue);
//initalJobs.Add(new SCSInital(holderListSCS));
var holderFurnitureVillage = new HolderXml<Sofa>("output/FurnitureVillage.{0}.xml", MaxQueue);
//initalJobs.Add(new FurnitureVillageInital(holderFurnitureVillage));
var holderHarveysFurnitureSofas = new HolderXml<Sofa>("output/HarveysFurnitureSofas.{0}.xml", MaxQueue);
//initalJobs.Add(new HarveysFurnitureSofas(holderHarveysFurnitureSofas));
var holderCWTE = new HolderXml<CWTEObject>("output/CWTE.{0}.xml", MaxQueue);
//initalJobs.Add(new CWTEInitial(holderCWTE));
var holderCTA = new HolderXml<CTAObject>("output/CTA.{0}.xml", MaxQueue);
//initalJobs.Add(new CTAInitial(holderCTA));
var holderMipiParticipants = new MipiParticipantHolder("output/MipiParticipants.csv", MaxQueue);
var holderMipiCompanies = new MipiCompanyHolder("output/MipiCompanies.csv", MaxQueue);
initalJobs.Add(new LoginJob(holderMipiParticipants, holderMipiCompanies));
logger.Trace("Got {0} initial jobs.", initalJobs.Count);
logger.Trace("Enabeling the threaded worker client.");
var worker = new ThreadedWebClientWorker(MaxQueue, true);
worker.SetThreads(WorkerThreads, DownloadThreads);
if (UseProxie)
{
var proxies = GetProxies();
logger.Info("Loaded {0} proxies.", proxies.Count);
worker.AddProxies(proxies);
}
logger.Trace("Adding the jobs to the worker.");
worker.AddJob(initalJobs);
if (UseGui)
{
logger.Trace("Starting the GUI interface.");
ScraperGUIHelper.AttachGui(worker);
}
logger.Info("Starting jobs.");
worker.Run();
logger.Info("Jobs finished.");
logger.Debug("Closing holders.");
if(holderCWTE != null)
holderCWTE.WriteCurrentObjects();
if (holderCTA != null)
holderCTA.WriteCurrentObjects();
if (holderHarveysFurnitureSofas != null)
holderHarveysFurnitureSofas.WriteCurrentObjects();
if (holderListSCS != null)
holderListSCS.WriteCurrentObjects();
if (holderFurnitureVillage != null)
holderFurnitureVillage.WriteCurrentObjects();
if (holderMipiParticipants != null)
holderMipiParticipants.WriteCurrentObjects();
if (holderMipiCompanies != null)
holderMipiCompanies.WriteCurrentObjects();
if (holderCWTE != null)
holderCWTE.Collet();
if (holderCTA != null)
holderCTA.Collet();
if (holderHarveysFurnitureSofas != null)
holderHarveysFurnitureSofas.Collet();
if (holderListSCS != null)
holderListSCS.Collet();
if (holderFurnitureVillage != null)
holderFurnitureVillage.Collet();
if (holderMipiParticipants != null)
holderMipiParticipants.Collet();
if (holderMipiCompanies != null)
holderMipiCompanies.Collet();
logger.Debug("Holder closing done.");
#if DEBUG
Console.WriteLine("Holder console open for debugging:");
Console.ReadLine();
#endif
}
