コード例 #1
0
        public static void PushDownloadHtml(IEnumerable <string> domains)
        {
            SqlDb          sql            = new SqlDb(ConfigStatic.ProductConnection);
            ProductAdapter productAdapter = new ProductAdapter(sql);

            foreach (var domain in domains)
            {
                ProducerBasic producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
                                                                ConfigStatic.GetQueueWaitDownloadHtml(domain));
                string queryData = string.Format(@"
                        Select Id, DetailUrl 
                        From Product 
                        Where Company = {0}
                        Order by Id
                        "
                                                 , productAdapter.GetCompanyIdByDomain(domain));
                sql.ProcessDataTableLarge(queryData,
                                          10000, (Row, iRow) =>
                {
                    producerBasic.PublishString(new JobDownloadHtml()
                    {
                        ProductId = Convert.ToInt64(Row["Id"]),
                        DetailUrl = Convert.ToString(Row["DetailUrl"]),
                        Domain    = domain
                    }.GetJson());
                });
            }
        }
コード例 #2
0
        public WorkerParseData(string domain) : base(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
                                                     ConfigStatic.GetQueueParse(domain), false)
        {
            var domainModule = new DomainModule();
            var kernel       = new StandardKernel(domainModule);

            _handlerParserProperties = kernel.Get <IHandlerParserProperties>();
            _handlerParserProperties.Init(domain);
        }
コード例 #3
0
        public WorkerDownloadHtml(string domain) : base(
                RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
                ConfigStatic.GetQueueWaitDownloadHtml(domain), false)
        {
            var domainModule = new DomainModule();
            var kernel       = new StandardKernel(domainModule);

            _h1 = kernel.Get <HandlerDownloadHtml>();
            _h1.Init(domain);
        }
コード例 #4
0
        /// <summary>
        /// Public Constructor for WindowsService.
        /// - Put all of your Initialization code here.
        /// </summary>
        public WindowsService()
        {
            var allOk = false;

            this.ServiceName  = "StationMockService";
            this.EventLog.Log = "Application";

            // These Flags set whether or not to handle that specific
            //  type of event. Set to true if you need it, false otherwise.
            this.CanHandlePowerEvent         = true;
            this.CanHandleSessionChangeEvent = true;
            this.CanPauseAndContinue         = true;
            this.CanShutdown = true;
            this.CanStop     = true;

            var setting = ConfigStatic.GetConfigSetting();

            if (setting.UseConfigSetting)
            {
                socket = new ServerSocket(setting.HostIPAddress, setting.Port);
                log.Debug(LogHelp.LogText($"Socket Loaded {setting.HostIPAddress}:{setting.Port}"));
                allOk = true;
            }
            else if (setting.UseAutoIPconfig)
            {
                var ipAddresses = IPAddressResolver.GetList();
                foreach (var item in ipAddresses)
                {
                    ConfigStatic.HostIpAddress = item.Address;
                }
                ConfigStatic.HostPort         = "3000";
                ConfigStatic.UseConfigSetting = "true";
                socket = new ServerSocket(setting.HostIPAddress, setting.Port);
                log.Info(LogHelp.LogText($"Socket Loaded {setting.HostIPAddress}:{setting.Port}"));
                allOk = true;
            }

            if (allOk)
            {
                log.Debug(LogHelp.LogText($"Start... Initialize"));
                socket.Initialize();

                log.Debug(LogHelp.LogText($"Start... Stories"));

                log.Debug(LogHelp.LogText($"Start... Timer"));
                aTimer.Elapsed += new ElapsedEventHandler(OnTimedEvent);
                aTimer.Interval = 5000;
                aTimer.Enabled  = true;

                log.Debug(LogHelp.LogText($"Start... StartListening"));
                socket.StartListening();
                log.Debug(LogHelp.LogText($"Start... OK!"));
            }
        }
コード例 #5
0
        private static void PushJobDownload(string domain)
        {
            ProducerBasic producer = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties),
                                                       ConfigStatic.GetQueueWaitDownloadHtml(domain));
            IStorageProduct storageProduct = new StorageProduct();
            int             i         = 0;
            string          DetailUrl = "";
            Regex           regex     = new Regex(@"http://www.+html");

            storageProduct.ProcessProduct(domain, (sender, product) =>
            {
                if (domain == "lazada.vn")
                {
                    string urlencode        = product.DetailUrl;
                    string urldecode        = HttpUtility.UrlDecode(HttpUtility.UrlDecode(urlencode));
                    MatchCollection matches = regex.Matches(urldecode);

                    DetailUrl = matches[0].Value.ToString();

                    ////string urlencode = product.DetailUrl;
                    ////string urldecode = HttpUtility.UrlDecode(product.DetailUrl);
                    ////char charRange = '?';
                    ////int startIndex = urldecode.IndexOf(charRange) + 1;
                    ////int endIndex = urldecode.LastIndexOf(charRange) - 1;
                    ////int length = endIndex - startIndex + 1;
                    ////DetailUrl = urldecode.Substring(startIndex, length).Replace("url=", "");
                }
                else
                {
                    DetailUrl = product.DetailUrl;
                }
                producer.PublishString(new JobCrlProperties()
                {
                    ProductId        = product.Id,
                    DetailUrl        = UtilCrl.GetUrl(DetailUrl, domain),
                    Domain           = domain,
                    ClassificationId = product.ClassificationId,
                    Classification   = product.Classification
                }.GetJson());
                i++;
                //log.Info(string.Format("{0} {1}", i, product.Id));
                log.InfoFormat("{0}: {1}", i, product.Id);
            });
        }
コード例 #6
0
        public static void PushParseFromNoSql(IEnumerable <string> domains)
        {
            NoSqlAdapter noSqlAdapter = NoSqlAdapter.GetInstance();

            foreach (var domain in domains)
            {
                Task.Factory.StartNew(() =>
                {
                    int iCount = 0;
                    var producerBasicWaitPs = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties), ConfigStatic.GetQueueParse(domain));
                    noSqlAdapter.ProcessAllIdProductByCompany(domain, (obj, productId) =>
                    {
                        iCount++;
                        producerBasicWaitPs.PublishString(new JobParse()
                        {
                            Id = productId
                        }.GetJson());

                        if (iCount % 10 == 0)
                        {
                            log.Info(string.Format("Pushed {0} mss of {1}", iCount, domain));
                        }
                    });
                    log.Info(string.Format("Pushed all data for company {0} {1}", domain, iCount));
                });
            }
            Thread.Sleep(10000000);
        }
コード例 #7
0
 public void Init(string domain)
 {
     this._domain        = domain;
     this._producerBasic = new ProducerBasic(RabbitMQManager.GetRabbitMQServer(ConfigStatic.KeyRabbitMqCrlProductProperties), ConfigStatic.GetQueueParse(this._domain));
     this._config        = _storageConfigCrl.GetConfig(this._domain);
 }