public static void Extract(Category category) { string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "Extract"); if(!Directory.Exists(path)) Directory.CreateDirectory(path); pset.Clear(); pdic.Clear(); string downPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "Down", category.Name); string fileName = string.Format(@"{0}\{1}.txt", path, category.Name); StreamWriter sw = new StreamWriter(fileName, false, Encoding.UTF8); for (int i = category.DownPageCount; i >= 1; i--) { string htmlFileName = string.Format(@"{0}\{1}.html", downPath, i); if (!File.Exists(htmlFileName)) Logger.Instance.Write(string.Format("{0}-{1}.html-not exist", category.Name, i)); StreamReader sr = new StreamReader(htmlFileName, Encoding.UTF8); string text = sr.ReadToEnd(); sr.Close(); var action = CreateAction(category.Type); if (action == null) continue; Extract(text, sw, category.Name,category.DbName, action); } sw.Close(); Console.WriteLine("{0}:Extract Data Finished!", category.Name); }
protected override void ExecuteCommand(Action<string> output, params string[] args) { WebClient wc = new WebClient(); wc.Encoding = Encoding.UTF8; string urlFormat = "http://www.pinble.com/Template/WebService1.asmx/Present3DList?pageindex={0}&lottory={1}&pl3={2}&name={3}&isgp={4}"; List<Category> categories = CategoryBiz.Instance.GetEnabledCategories(false); Dictionary<string, string> dict = new Dictionary<string, string>(categories.Count); string dataSourceName = ConfigHelper.GetAppSettings("dataSource"); DataSourceElement config = ConfigManager.DataSourceSection.DataSources[dataSourceName]; DwNumberBiz biz = new DwNumberBiz("shand11x5"); foreach (var category in categories) { if (category.ParentId == 0) continue; string url = string.Format(urlFormat, 1, category.Code, "", category.Name, category.IsGP); string htmlText = wc.DownloadString(url); int peroidCount = ConvertHelper.GetInt32(Regex.Match(htmlText, "共\\:(\\d+)条", RegexOptions.Singleline | RegexOptions.IgnoreCase).Groups[1].Value); int pageCount = ConvertHelper.GetInt32(Regex.Match(htmlText, "分页\\:1/(\\d+)页", RegexOptions.Singleline | RegexOptions.IgnoreCase).Groups[1].Value); Category entity = new Category() { Id = category.Id, PeroidCount = peroidCount, DownPageCount = pageCount }; CategoryBiz.Instance.Modify(entity, entity.Id, Category.C_DownPageCount, Category.C_PeroidCount); biz.DataAccessor.ConnectionString = config.Databases[category.DbName.Trim().ToLower()].ConnectionString; int maxSeq = biz.DataAccessor.SelectMaxWithCondition("Seq", 10, string.Empty); int downPeroids = biz.Count; string propmt = string.Format("{0}:{1}", category.Name, string.Format("{0}期,下载,{1}期,Max Seq,{2}", peroidCount, downPeroids, maxSeq)); output(propmt); if (category.Type.Equals("11X5")) this.CheckC5CX(output, category, biz.DataAccessor.ConnectionString); } }
public void Start(DateTime currentDateTime) { if (!this.IsUpdateTime(currentDateTime)) return; string urlFormat = "http://www.pinble.com/Template/WebService1.asmx/Present3DList?pageindex={0}&lottory={1}&pl3={2}&name={3}&isgp={4}"; List<Category> categories = CategoryBiz.Instance.GetCategories(); WebClient wc = new WebClient(); wc.Encoding = Encoding.UTF8; foreach (var category in categories) { if (category.ParentId == 0) continue; string url = string.Format(urlFormat, 1, category.Code, "", category.Name, category.IsGP); try { string htmlText = wc.DownloadString(url); int peroidCount = ConvertHelper.GetInt32(Regex.Match(htmlText, "共\\:(\\d+)条", RegexOptions.Singleline | RegexOptions.IgnoreCase).Groups[1].Value); int pageCount = ConvertHelper.GetInt32(Regex.Match(htmlText, "分页\\:1/(\\d+)页", RegexOptions.Singleline | RegexOptions.IgnoreCase).Groups[1].Value); Category entity = new Category() { Id = category.Id, PeroidCount = peroidCount, DownPageCount = pageCount }; CategoryBiz.Instance.Modify(entity, entity.Id, Category.C_DownPageCount, Category.C_PeroidCount); } catch(Exception ex) { Logger.Instance.Write(string.Format("url:{0},message:{1}", url, ex)); } } }
public static void LogFormat(Logging.Category category, string format, Logging.Level level = Logging.Level.LOG, params object[] args) { if (Self != null && Self._logging != null) { Self._logging.LogFormat(category, format, level, args); } }
//public static access (static global-access systems like logging, databases, etc...) public static void Log(Logging.Category category, string message, Logging.Level level = Logging.Level.LOG) { if (Self != null && Self._logging != null) { Self._logging.Log(category, message, level); } }
private void CheckC5CX(Action<string> output, Category category,string connectionString) { string[] c5cxs = new string[] { "C5C2", "C5C3", "C5C4", "C5C6", "C5C7", "C5C8" }; DwC5CXSpanBiz c5cxbiz = new DwC5CXSpanBiz(category.DbName); c5cxbiz.DataAccessor.ConnectionString = connectionString; foreach (string c5cx in c5cxs) { c5cxbiz.DataAccessor.TableName = ConfigHelper.GetDwSpanTableName(c5cx); int peroidCount = c5cxbiz.DataAccessor.Count(); int maxSeq = c5cxbiz.DataAccessor.SelectMaxWithCondition("Seq", 10, string.Empty); string propmt = string.Format("{0}:{1}", category.Name + "-" + c5cx, string.Format("{0}期,下载,Max Seq,{1}", peroidCount, maxSeq)); output(propmt); } }
private static void DownLeCaiKaiJiangData(Category category) { string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "KaiJiang", category.Name); if (!Directory.Exists(path)) Directory.CreateDirectory(path); WebClient wc = new WebClient(); wc.Encoding = Encoding.UTF8; DateTime startDate = new DateTime(2008,1,1); Queue<string> fileQueue= new Queue<string>(100); for (DateTime currDate = startDate; currDate <= DateTime.Now; currDate = currDate.AddDays(1)) { string url = string.Format(category.DownUrl, currDate.ToString("yyyy-MM-dd")); string fileName = string.Format("{0}\\{1}.html", path, currDate.ToString("yyyy-MM-dd")); fileQueue.Enqueue(fileName); try { while(fileQueue.Count > 0){ string tmpFileName = fileQueue.Dequeue(); if (!File.Exists(tmpFileName)) wc.DownloadFile(url, tmpFileName); } } catch { fileQueue.Enqueue(fileName); Logger.Instance.Write(url); } } Console.WriteLine("{0}:DownPage Finished!", category.Name); }
private static void DownPinbleKaiJiangData(Category category) { string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "KaiJiang", category.Name); if (!Directory.Exists(path)) Directory.CreateDirectory(path); WebClient wc = new WebClient(); wc.Encoding = Encoding.UTF8; for (int i = 1; i <= category.DownPageCount; i++) { string url = string.Format(category.DownUrl, i, category.Code, "", category.Name, category.IsGP); string fileName = string.Format("{0}\\{1}.html", path, i); try { string htmlText = wc.DownloadString(url); wc.DownloadFile(url, fileName); } catch { Logger.Instance.Write(url); } } Console.WriteLine("{0}:DownPage Finished!", category.Name); }
public void Log( Category category, Severity severity, String message ) { _verbosityHandler[ VerbosityLevel ][ (int) severity ]( category, severity, message ); }
private void PersistNone( Category category, Severity severity, String message ) { }
protected abstract void Persist( Category category, Severity severity, String message );
private static void DownPage(Category category) { string dataUrl = "http://www.pinble.com/Template/WebService1.asmx/Present3DList?pageindex={0}&lottory={1}&pl3={2}&name={3}&isgp={4}"; string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "Down", category.Name); if (!Directory.Exists(path)) Directory.CreateDirectory(path); WebClient wc = new WebClient(); wc.Encoding = Encoding.UTF8; for (int i = 1; i <= category.DownPageCount; i++) { string url = string.Format(dataUrl, i, category.Code, "", category.Name, category.IsGP); string fileName = string.Format("{0}\\{1}.html", path, i); try { string htmlText = wc.DownloadString(url); wc.DownloadFile(url, fileName); } catch { Logger.Instance.Write(url); } } Console.WriteLine("{0}:DownPage Finished!", category.Name); }