public override void ProcessMessage(RabbitMQ.Client.Events.BasicDeliverEventArgs message) { JobCrlProperties jobDownloadHtml = JobCrlProperties.FromJson(UTF8Encoding.UTF8.GetString(Decompress(message.Body))); _handlerParserProperties.ProcessJob(jobDownloadHtml.ProductId, jobDownloadHtml.Html); this.GetChannel().BasicAck(message.DeliveryTag, true); Logger.InfoFormat("Processed {0}", jobDownloadHtml.ProductId); }
private void SaveToStorage(JobCrlProperties jobDownloadHtml, string html) { _storageHtml.SaveHtml(new HtmlProduct() { ProductId = jobDownloadHtml.ProductId, Html = html, Domain = jobDownloadHtml.Domain, Classification = jobDownloadHtml.Classification }); }
private string DownloadHtml(JobCrlProperties jobDownloadHtml) { WebExceptionStatus w = WebExceptionStatus.ConnectFailure; var html = this._downloadHtml.GetHtml(jobDownloadHtml.DetailUrl, 45, 2, out w); html = CommonConvert.RemoveScripTag(html); html = CommonConvert.RemoveCommentXML(html); return(html); }
public override void ProcessMessage(BasicDeliverEventArgs message) { try { JobCrlProperties jobDownloadHtml = JobCrlProperties.FromJson(Encoding.UTF8.GetString(message.Body)); if (jobDownloadHtml != null) { _h1.ProcessJob(jobDownloadHtml); } this.GetChannel().BasicAck(message.DeliveryTag, true); } catch (System.Exception ex1) { throw; } }
public void ProcessJob(JobCrlProperties jobDownloadHtml) { DateTime dtFrom = DateTime.Now; var html = DownloadHtml(jobDownloadHtml); if (!string.IsNullOrEmpty(html)) { //SaveToStorage(jobDownloadHtml, html); jobDownloadHtml.Html = html; byte[] mss = Compress(UTF8Encoding.UTF8.GetBytes(jobDownloadHtml.GetJson())); this._producerBasic.Publish(mss, true); Thread.Sleep(this._config.TimeDelay); } else { _logger.Info("Can't download html"); } _logger.Info(string.Format("Processed job {0} in {1}", jobDownloadHtml.ProductId, (DateTime.Now - dtFrom).Milliseconds)); }