private static void SaveQueue(UrlQueue url) { ThriftTool.AddColumn(url.Url, "UrlQueue", "Url", url.Url); ThriftTool.AddColumn(url.Url, "UrlQueue", "Title", url.Title); ThriftTool.AddColumn(url.Url, "UrlQueue", "Type", url.Type); ThriftTool.AddColumn(url.Url, "UrlQueue", "Status", url.Status); }
private static void QueuePage(string url, string title) { var urlQueue = new UrlQueue { Url = url, Title = title, Type = "M01", Status = QueueType.Queue.ToString() }; SaveQueue(urlQueue); }
public static void ProcessQueue(int limit) { CqlResult cqlResult = ThriftTool.GetByCql("Select * from UrlQueue where 'Status'='" + QueueType.Queue.ToString() + "' and 'Type'='M01' limit " + limit); if (cqlResult.Rows.Count < limit) { Utility.Logging("select result count:" + cqlResult.Rows.Count); } foreach (var row in cqlResult.Rows) { var url = new UrlQueue(); foreach (var col in row.Columns) { var name = ThriftTool.ToString(col.Name); switch (name) { case "Title": url.Title = ThriftTool.ToString(col.Value); break; case "Url": url.Url = ThriftTool.ToString(col.Value); break; case "Status": url.Status = ThriftTool.ToString(col.Value); break; case "Type": url.Type = ThriftTool.ToString(col.Value); break; } } //Console.WriteLine("Title:" + url.Title); //Console.WriteLine("Url:" + url.Url); //Console.WriteLine("Status:" + url.Status); //Console.WriteLine("Type:" + url.Type); if (url.Url != null) { Thread.Sleep(ChangeTopic); ProcessUrl("http://www.mobile01.com/" + url.Url); } //mark as done ThriftTool.AddColumn(url.Url, "UrlQueue", "Status", QueueType.Done.ToString()); } }
public static void ProcessQueue(int limit) { CqlResult cqlResult = ThriftTool.GetByCql("Select * from UrlQueue where 'Status'='" + QueueType.Queue.ToString() + "' and 'Type'='M01' limit "+limit); if (cqlResult.Rows.Count < limit) Utility.Logging("select result count:" + cqlResult.Rows.Count); foreach (var row in cqlResult.Rows) { var url = new UrlQueue(); foreach (var col in row.Columns) { var name = ThriftTool.ToString(col.Name); switch (name) { case "Title": url.Title = ThriftTool.ToString(col.Value); break; case "Url": url.Url = ThriftTool.ToString(col.Value); break; case "Status": url.Status = ThriftTool.ToString(col.Value); break; case "Type": url.Type = ThriftTool.ToString(col.Value); break; } } //Console.WriteLine("Title:" + url.Title); //Console.WriteLine("Url:" + url.Url); //Console.WriteLine("Status:" + url.Status); //Console.WriteLine("Type:" + url.Type); if (url.Url != null) { Thread.Sleep(ChangeTopic); ProcessUrl("http://www.mobile01.com/" + url.Url); } //mark as done ThriftTool.AddColumn(url.Url, "UrlQueue", "Status", QueueType.Done.ToString()); } }