public static void AddConsoleLogging(ThreadedWebClientWorker downloader) { downloader.DownloaderThreadJobChanged += (sender, id, job) => { if (job == null) { // Console.WriteLine("D[" + id + "] Inavtive"); } else { Console.WriteLine("D[" + id + "] " + job); } }; downloader.DownloaderThreadStatus += (sender, id, status) => { if (status) { //Console.WriteLine("D[" + id + "] active"); } else { // Console.WriteLine("D[" + id + "] Inavtive"); } }; downloader.WorkerThreadError += (sender, id, job, exp) => { Console.WriteLine("W[" + id + "] Error: " + job); Console.WriteLine(exp.Message); }; downloader.WorkerThreadJobDone += (sender, id, job) => { Console.WriteLine("W[" + id + "] Done: " + job); }; downloader.WorkerThreadStatusChanged += (sender, id, job) => { if (job == null) { // Console.WriteLine("W[" + id + "] Inavtive"); } else { Console.WriteLine("W[" + id + "] " + job); } }; downloader.DownloaderBadProxyRemoved += (sender, id, proxy) => { Console.WriteLine("D[" + id + "] Bad proxy remove " + proxy.Ip + ":" + proxy.Port + " from " + proxy.Country); }; downloader.DownloaderNoGoodProxyLeft += sender => { Console.WriteLine("No good proxies left, have to terminate!"); }; }
public DownloadManagerGuiWindow(ThreadedWebClientWorker worker) { InitializeComponent(); this.viewModel = new MainViewModel(); this.DataContext = this.viewModel; this.viewModel.WorkThreadCount = 2; this.viewModel.DownloadThreadCount = 50; this.viewModel.DownloadThreadCount = worker.GetDownloadThreadCount(); this.viewModel.WorkThreadCount = worker.GetWorkerThreadCount(); this.worker = worker; }
public static void AttachGui(ThreadedWebClientWorker worker) { var task = new Thread(() => { System.Windows.Application application = new System.Windows.Application(); application.Run(new DownloadManagerGuiWindow(worker)); }); task.SetApartmentState(ApartmentState.STA); worker.WorkDone += sender => { try { task.Abort(); } catch (Exception) { } }; task.Start(); }
public void Run() { logger.Info("Uk Data Scraper Started."); logger.Info("Max holder queue: {0}.", MaxQueue); logger.Info("Worker Threads: {0}.", WorkerThreads); logger.Info("Download Threads: {0}.", DownloadThreads); logger.Info("Use Proxies: {0} ({1}).", UseProxie, ProxieFile); var initalJobs = new List<IThreadedWebClientJob>(); // SCS var holderListSCS = new HolderXml<Sofa>("output/Sofa.{0}.xml", MaxQueue); //initalJobs.Add(new SCSInital(holderListSCS)); var holderFurnitureVillage = new HolderXml<Sofa>("output/FurnitureVillage.{0}.xml", MaxQueue); //initalJobs.Add(new FurnitureVillageInital(holderFurnitureVillage)); var holderHarveysFurnitureSofas = new HolderXml<Sofa>("output/HarveysFurnitureSofas.{0}.xml", MaxQueue); //initalJobs.Add(new HarveysFurnitureSofas(holderHarveysFurnitureSofas)); var holderCWTE = new HolderXml<CWTEObject>("output/CWTE.{0}.xml", MaxQueue); //initalJobs.Add(new CWTEInitial(holderCWTE)); var holderCTA = new HolderXml<CTAObject>("output/CTA.{0}.xml", MaxQueue); //initalJobs.Add(new CTAInitial(holderCTA)); var holderMipiParticipants = new MipiParticipantHolder("output/MipiParticipants.csv", MaxQueue); var holderMipiCompanies = new MipiCompanyHolder("output/MipiCompanies.csv", MaxQueue); initalJobs.Add(new LoginJob(holderMipiParticipants, holderMipiCompanies)); logger.Trace("Got {0} initial jobs.", initalJobs.Count); logger.Trace("Enabeling the threaded worker client."); var worker = new ThreadedWebClientWorker(MaxQueue, true); worker.SetThreads(WorkerThreads, DownloadThreads); if (UseProxie) { var proxies = GetProxies(); logger.Info("Loaded {0} proxies.", proxies.Count); worker.AddProxies(proxies); } logger.Trace("Adding the jobs to the worker."); worker.AddJob(initalJobs); if (UseGui) { logger.Trace("Starting the GUI interface."); ScraperGUIHelper.AttachGui(worker); } logger.Info("Starting jobs."); worker.Run(); logger.Info("Jobs finished."); logger.Debug("Closing holders."); if(holderCWTE != null) holderCWTE.WriteCurrentObjects(); if (holderCTA != null) holderCTA.WriteCurrentObjects(); if (holderHarveysFurnitureSofas != null) holderHarveysFurnitureSofas.WriteCurrentObjects(); if (holderListSCS != null) holderListSCS.WriteCurrentObjects(); if (holderFurnitureVillage != null) holderFurnitureVillage.WriteCurrentObjects(); if (holderMipiParticipants != null) holderMipiParticipants.WriteCurrentObjects(); if (holderMipiCompanies != null) holderMipiCompanies.WriteCurrentObjects(); if (holderCWTE != null) holderCWTE.Collet(); if (holderCTA != null) holderCTA.Collet(); if (holderHarveysFurnitureSofas != null) holderHarveysFurnitureSofas.Collet(); if (holderListSCS != null) holderListSCS.Collet(); if (holderFurnitureVillage != null) holderFurnitureVillage.Collet(); if (holderMipiParticipants != null) holderMipiParticipants.Collet(); if (holderMipiCompanies != null) holderMipiCompanies.Collet(); logger.Debug("Holder closing done."); #if DEBUG Console.WriteLine("Holder console open for debugging:"); Console.ReadLine(); #endif }