public static void Extract(Category category)
{
string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "Extract");
if(!Directory.Exists(path))
Directory.CreateDirectory(path);
pset.Clear();
pdic.Clear();
string downPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "Down", category.Name);
string fileName = string.Format(@"{0}\{1}.txt", path, category.Name);
StreamWriter sw = new StreamWriter(fileName, false, Encoding.UTF8);
for (int i = category.DownPageCount; i >= 1; i--)
{
string htmlFileName = string.Format(@"{0}\{1}.html", downPath, i);
if (!File.Exists(htmlFileName))
Logger.Instance.Write(string.Format("{0}-{1}.html-not exist", category.Name, i));
StreamReader sr = new StreamReader(htmlFileName, Encoding.UTF8);
string text = sr.ReadToEnd();
sr.Close();
var action = CreateAction(category.Type);
if (action == null) continue;
Extract(text, sw, category.Name,category.DbName, action);
}
sw.Close();
Console.WriteLine("{0}:Extract Data Finished!", category.Name);
}
protected override void ExecuteCommand(Action<string> output, params string[] args)
{
WebClient wc = new WebClient();
wc.Encoding = Encoding.UTF8;
string urlFormat = "http://www.pinble.com/Template/WebService1.asmx/Present3DList?pageindex={0}&lottory={1}&pl3={2}&name={3}&isgp={4}";
List<Category> categories = CategoryBiz.Instance.GetEnabledCategories(false);
Dictionary<string, string> dict = new Dictionary<string, string>(categories.Count);
string dataSourceName = ConfigHelper.GetAppSettings("dataSource");
DataSourceElement config = ConfigManager.DataSourceSection.DataSources[dataSourceName];
DwNumberBiz biz = new DwNumberBiz("shand11x5");
foreach (var category in categories)
{
if (category.ParentId == 0) continue;
string url = string.Format(urlFormat, 1, category.Code, "", category.Name, category.IsGP);
string htmlText = wc.DownloadString(url);
int peroidCount = ConvertHelper.GetInt32(Regex.Match(htmlText, "共\\：(\\d+)条", RegexOptions.Singleline | RegexOptions.IgnoreCase).Groups[1].Value);
int pageCount = ConvertHelper.GetInt32(Regex.Match(htmlText, "分页\\:1/(\\d+)页", RegexOptions.Singleline | RegexOptions.IgnoreCase).Groups[1].Value);
Category entity = new Category() { Id = category.Id, PeroidCount = peroidCount, DownPageCount = pageCount };
CategoryBiz.Instance.Modify(entity, entity.Id, Category.C_DownPageCount, Category.C_PeroidCount);
biz.DataAccessor.ConnectionString = config.Databases[category.DbName.Trim().ToLower()].ConnectionString;
int maxSeq = biz.DataAccessor.SelectMaxWithCondition("Seq", 10, string.Empty);
int downPeroids = biz.Count;
string propmt = string.Format("{0}:{1}",
category.Name, string.Format("{0}期,下载,{1}期,Max Seq,{2}", peroidCount, downPeroids, maxSeq));
output(propmt);
if (category.Type.Equals("11X5"))
this.CheckC5CX(output, category, biz.DataAccessor.ConnectionString);
}
}
public void Start(DateTime currentDateTime)
{
if (!this.IsUpdateTime(currentDateTime)) return;
string urlFormat = "http://www.pinble.com/Template/WebService1.asmx/Present3DList?pageindex={0}&lottory={1}&pl3={2}&name={3}&isgp={4}";
List<Category> categories = CategoryBiz.Instance.GetCategories();
WebClient wc = new WebClient();
wc.Encoding = Encoding.UTF8;
foreach (var category in categories)
{
if (category.ParentId == 0) continue;
string url = string.Format(urlFormat, 1, category.Code, "", category.Name, category.IsGP);
try
{
string htmlText = wc.DownloadString(url);
int peroidCount = ConvertHelper.GetInt32(Regex.Match(htmlText, "共\\：(\\d+)条",
RegexOptions.Singleline | RegexOptions.IgnoreCase).Groups[1].Value);
int pageCount = ConvertHelper.GetInt32(Regex.Match(htmlText, "分页\\:1/(\\d+)页",
RegexOptions.Singleline | RegexOptions.IgnoreCase).Groups[1].Value);
Category entity = new Category() { Id = category.Id, PeroidCount = peroidCount, DownPageCount = pageCount };
CategoryBiz.Instance.Modify(entity, entity.Id, Category.C_DownPageCount, Category.C_PeroidCount);
}
catch(Exception ex)
{
Logger.Instance.Write(string.Format("url:{0},message:{1}", url, ex));
}
}
}
public static void LogFormat(Logging.Category category, string format, Logging.Level level = Logging.Level.LOG, params object[] args)
{
if (Self != null && Self._logging != null)
{
Self._logging.LogFormat(category, format, level, args);
}
}
//public static access (static global-access systems like logging, databases, etc...)
public static void Log(Logging.Category category, string message, Logging.Level level = Logging.Level.LOG)
{
if (Self != null && Self._logging != null)
{
Self._logging.Log(category, message, level);
}
}
private void CheckC5CX(Action<string> output, Category category,string connectionString)
{
string[] c5cxs = new string[] { "C5C2", "C5C3", "C5C4", "C5C6", "C5C7", "C5C8" };
DwC5CXSpanBiz c5cxbiz = new DwC5CXSpanBiz(category.DbName);
c5cxbiz.DataAccessor.ConnectionString = connectionString;
foreach (string c5cx in c5cxs)
{
c5cxbiz.DataAccessor.TableName = ConfigHelper.GetDwSpanTableName(c5cx);
int peroidCount = c5cxbiz.DataAccessor.Count();
int maxSeq = c5cxbiz.DataAccessor.SelectMaxWithCondition("Seq", 10, string.Empty);
string propmt = string.Format("{0}:{1}", category.Name + "-" + c5cx,
string.Format("{0}期,下载,Max Seq,{1}", peroidCount, maxSeq));
output(propmt);
}
}
private static void DownLeCaiKaiJiangData(Category category)
{
string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "KaiJiang", category.Name);
if (!Directory.Exists(path))
Directory.CreateDirectory(path);
WebClient wc = new WebClient();
wc.Encoding = Encoding.UTF8;
DateTime startDate = new DateTime(2008,1,1);
Queue<string> fileQueue= new Queue<string>(100);
for (DateTime currDate = startDate; currDate <= DateTime.Now; currDate = currDate.AddDays(1))
{
string url = string.Format(category.DownUrl, currDate.ToString("yyyy-MM-dd"));
string fileName = string.Format("{0}\\{1}.html", path, currDate.ToString("yyyy-MM-dd"));
fileQueue.Enqueue(fileName);
try
{
while(fileQueue.Count > 0){
string tmpFileName = fileQueue.Dequeue();
if (!File.Exists(tmpFileName))
wc.DownloadFile(url, tmpFileName);
}
}
catch
{
fileQueue.Enqueue(fileName);
Logger.Instance.Write(url);
}
}
Console.WriteLine("{0}:DownPage Finished!", category.Name);
}
private static void DownPinbleKaiJiangData(Category category)
{
string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "KaiJiang", category.Name);
if (!Directory.Exists(path))
Directory.CreateDirectory(path);
WebClient wc = new WebClient();
wc.Encoding = Encoding.UTF8;
for (int i = 1; i <= category.DownPageCount; i++)
{
string url = string.Format(category.DownUrl, i, category.Code, "", category.Name, category.IsGP);
string fileName = string.Format("{0}\\{1}.html", path, i);
try
{
string htmlText = wc.DownloadString(url);
wc.DownloadFile(url, fileName);
}
catch
{
Logger.Instance.Write(url);
}
}
Console.WriteLine("{0}:DownPage Finished!", category.Name);
}
public void Log( Category category, Severity severity, String message )
{
_verbosityHandler[ VerbosityLevel ][ (int) severity ]( category, severity, message );
}
private void PersistNone( Category category, Severity severity, String message )
{
}
protected abstract void Persist( Category category, Severity severity, String message );
private static void DownPage(Category category)
{
string dataUrl = "http://www.pinble.com/Template/WebService1.asmx/Present3DList?pageindex={0}&lottory={1}&pl3={2}&name={3}&isgp={4}";
string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Data", "Down", category.Name);
if (!Directory.Exists(path))
Directory.CreateDirectory(path);
WebClient wc = new WebClient();
wc.Encoding = Encoding.UTF8;
for (int i = 1; i <= category.DownPageCount; i++)
{
string url = string.Format(dataUrl, i, category.Code, "", category.Name, category.IsGP);
string fileName = string.Format("{0}\\{1}.html", path, i);
try
{
string htmlText = wc.DownloadString(url);
wc.DownloadFile(url, fileName);
}
catch
{
Logger.Instance.Write(url);
}
}
Console.WriteLine("{0}:DownPage Finished!", category.Name);
}
