Exemple #1
0
 private static void SaveQueue(UrlQueue url)
 {
     ThriftTool.AddColumn(url.Url, "UrlQueue", "Url", url.Url);
     ThriftTool.AddColumn(url.Url, "UrlQueue", "Title", url.Title);
     ThriftTool.AddColumn(url.Url, "UrlQueue", "Type", url.Type);
     ThriftTool.AddColumn(url.Url, "UrlQueue", "Status", url.Status);
 }
Exemple #2
0
        private static void QueuePage(string url, string title)
        {
            var urlQueue = new UrlQueue
            {
                Url    = url,
                Title  = title,
                Type   = "M01",
                Status = QueueType.Queue.ToString()
            };

            SaveQueue(urlQueue);
        }
Exemple #3
0
        public static void ProcessQueue(int limit)
        {
            CqlResult cqlResult = ThriftTool.GetByCql("Select * from UrlQueue where 'Status'='" + QueueType.Queue.ToString() + "' and 'Type'='M01' limit " + limit);

            if (cqlResult.Rows.Count < limit)
            {
                Utility.Logging("select result count:" + cqlResult.Rows.Count);
            }

            foreach (var row in cqlResult.Rows)
            {
                var url = new UrlQueue();
                foreach (var col in row.Columns)
                {
                    var name = ThriftTool.ToString(col.Name);

                    switch (name)
                    {
                    case "Title":
                        url.Title = ThriftTool.ToString(col.Value);
                        break;

                    case "Url":
                        url.Url = ThriftTool.ToString(col.Value);
                        break;

                    case "Status":
                        url.Status = ThriftTool.ToString(col.Value);
                        break;

                    case "Type":
                        url.Type = ThriftTool.ToString(col.Value);
                        break;
                    }
                }

                //Console.WriteLine("Title:" + url.Title);
                //Console.WriteLine("Url:" + url.Url);
                //Console.WriteLine("Status:" + url.Status);
                //Console.WriteLine("Type:" + url.Type);

                if (url.Url != null)
                {
                    Thread.Sleep(ChangeTopic);
                    ProcessUrl("http://www.mobile01.com/" + url.Url);
                }

                //mark as done
                ThriftTool.AddColumn(url.Url, "UrlQueue", "Status", QueueType.Done.ToString());
            }
        }
Exemple #4
0
 private static void SaveQueue(UrlQueue url)
 {
     ThriftTool.AddColumn(url.Url, "UrlQueue", "Url", url.Url);
     ThriftTool.AddColumn(url.Url, "UrlQueue", "Title", url.Title);
     ThriftTool.AddColumn(url.Url, "UrlQueue", "Type", url.Type);
     ThriftTool.AddColumn(url.Url, "UrlQueue", "Status", url.Status);
 }
Exemple #5
0
        private static void QueuePage(string url, string title)
        {

            var urlQueue = new UrlQueue
            {
                Url = url,
                Title = title,
                Type = "M01",
                Status = QueueType.Queue.ToString()
            };
            SaveQueue(urlQueue);
        }
Exemple #6
0
        public static void ProcessQueue(int limit)
        {
            CqlResult cqlResult = ThriftTool.GetByCql("Select * from UrlQueue where 'Status'='" + QueueType.Queue.ToString() + "' and 'Type'='M01' limit "+limit);

            if (cqlResult.Rows.Count < limit) Utility.Logging("select result count:" + cqlResult.Rows.Count);

            foreach (var row in cqlResult.Rows)
            {
                var url = new UrlQueue();
                foreach (var col in row.Columns)
                {
                    var name = ThriftTool.ToString(col.Name);

                    switch (name)
                    {
                        case "Title":
                            url.Title = ThriftTool.ToString(col.Value);
                            break;
                        case "Url":
                            url.Url = ThriftTool.ToString(col.Value);
                            break;
                        case "Status":
                            url.Status = ThriftTool.ToString(col.Value);
                            break;
                        case "Type":
                            url.Type = ThriftTool.ToString(col.Value);
                            break;
                    }
                }

                //Console.WriteLine("Title:" + url.Title);
                //Console.WriteLine("Url:" + url.Url);
                //Console.WriteLine("Status:" + url.Status);
                //Console.WriteLine("Type:" + url.Type);

                if (url.Url != null)
                {
                    Thread.Sleep(ChangeTopic);
                    ProcessUrl("http://www.mobile01.com/" + url.Url);
                }

                //mark as done
                ThriftTool.AddColumn(url.Url, "UrlQueue", "Status", QueueType.Done.ToString());
            }
        }