public SetDataSourceRes SetDataSource(Stream stream)
{
if (WebOperationContext.Current.IncomingRequest.Headers == null)
{
throw new Exception("Can not get current WebOpreationContext.");
}
SetDataSourceReq param = null;
using (var streamReader = new StreamReader(stream))
{
param = DataConverter.Deserializer <SetDataSourceReq>(streamReader.ReadToEnd());
}
var res = new SetDataSourceRes();
if (param != null && param.rawdata != null && param.rawdata.Count > 0 && !string.IsNullOrWhiteSpace(param.source) && !string.IsNullOrWhiteSpace(param.category))
{
var query = MariaQueryBuilder.SetDataSource(param);
MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", query);
res.code = "200";
res.message = "요청 접수 완료";
}
else
{
res.code = "400";
res.message = "잘못된 요청";
}
return(res);
}
public List <JsonDictionary> AllAnalysis(SetDataSourceReq data_source)
{
var key = "종가";
var time_key = "날짜";
var result = new List <JsonDictionary>();
for (var i = 1; i <= data_source.rawdata.Count; i++)
{
var origin = new List <JsonDictionary>(data_source.rawdata.GetRange(0, i));
var ret_data = SingleAnalysis(origin, key, i, time_key);
var test = new JsonDictionary(ret_data.GetDictionary());
result.Add(test);
}
return(result);
}
public static string SetDataSource(SetDataSourceReq param)
{
var query = string.Empty;
if (!SourceList.Contains(param.source))
{
query = CreateSourceTable(param.source);
lock (SourceList)
{
SourceList.Add(param.source.ToLower());
}
}
query = InsertSource(param.source, param.category, param.rawdata, param.collected_at, query);
return(query);
}
private bool EmptyInformation(string collectionName, Func <string, bool> callback)
{
var file = Path.Combine(System.AppDomain.CurrentDomain.BaseDirectory, "stocklist.json");
var stockText = File.ReadAllText(file);
var stockJson = JsonValue.Parse(stockText);
foreach (var stock in stockJson)
{
var result = new SetDataSourceReq();
result.rawdata = new List <JsonDictionary>();
result.source = collectionName;
result.category = "종목코드";
result.collected_at = "날짜";
var finance = new JsonDictionary();
var code = stock.Value["code"].ReadAs <string>();
var name = stock.Value["name"].ReadAs <string>();
var type = stock.Value["type"].ReadAs <string>();
var cnt = stock.Value["cnt"].ReadAs <string>();
finance.Add("종목코드", code);
finance.Add("종목유형", type);
finance.Add("종목명", name);
finance.Add("상장주식수", cnt);
result.rawdata.Add(finance);
ThreadPool.QueueUserWorkItem((a) =>
{
var setSourceQuery = MariaQueryBuilder.SetDataSource(result);
MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", setSourceQuery);
});
}
callback.DynamicInvoke("test");
return(true);
}
private bool KOSPI_KOSDAQ(string collectionName, Func <string, bool> callback)
{
Console.WriteLine("{0} Collector Start : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
var codeArr = new List <string>()
{
"KOSPI", "KOSDAQ"
};
foreach (var code in codeArr)
{
try
{
var googleUrl = "http://www.google.com/finance/getprices?q={code}&i=86400&p=40Y&f=d,c,v,k,o,h,l&df=cpct&auto=0".Replace("{code}", code);
var reqParam = new RequestParameter()
{
Url = googleUrl,
ContentType = "application/json",
EncodingOption = "UTF8",
Method = "GET"
};
var stockData = this.DecodeHex(HttpsRequest.Instance.GetResponseByHttps(reqParam));
var histroyCsv = Regex.Split(stockData, @"\n");
var columnInfo = new string[] { "날짜", "종가", "고가", "저가", "시가", "거래량", "전일비", "전일비율" };
var result = new SetDataSourceReq();
result.rawdata = new List <JsonDictionary>();
result.source = collectionName;
result.category = "종목코드";
result.collected_at = "날짜";
var standardTime = string.Empty;
for (int i = 7; i < histroyCsv.Length - 1; i++)
{
var sise = new JsonDictionary();
var row = histroyCsv[i].Split(',');
sise.Add("종목코드", code);
sise.Add("종목명", code);
sise.Add("종목유형", "SOSOK");
if (row[0].Trim().Contains("a"))
{
standardTime = row[0].Trim().Replace("a", "");
sise.Add(columnInfo[0], standardTime);
}
else
{
sise.Add(columnInfo[0], (int.Parse(standardTime) + (86400 * int.Parse(row[0]))).ToString());
}
sise.Add(columnInfo[1], row[1].Trim());
sise.Add(columnInfo[2], row[2].Trim());
sise.Add(columnInfo[3], row[3].Trim());
sise.Add(columnInfo[4], row[4].Trim());
sise.Add(columnInfo[5], row[5].Trim());
if (result.rawdata.Count > 0)
{
var prevPrice = double.Parse(result.rawdata[result.rawdata.Count - 1]["종가"].ToString());
if (prevPrice > 0)
{
var diff = double.Parse(row[1].Trim()) - prevPrice;
sise.Add(columnInfo[6], diff);
sise.Add(columnInfo[7], (double)diff / prevPrice * 100);
}
}
result.rawdata.Add(sise);
}
if (result.rawdata.Count > 0)
{
ThreadPool.QueueUserWorkItem((a) =>
{
var setSourceQuery = MariaQueryBuilder.SetDataSource(result);
MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", setSourceQuery);
});
}
}
catch (Exception ex)
{
Console.WriteLine("All Stock Collector Error");
}
}
callback.DynamicInvoke("test");
Console.WriteLine("{0} Collector End : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
return(true);
}
private bool FinanceInformation(string collectionName, Func <string, bool> callback)
{
Console.WriteLine("{0} Collector Start : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
var file = Path.Combine(System.AppDomain.CurrentDomain.BaseDirectory, "stocklist.json");
var stockText = File.ReadAllText(file);
var stockJson = JsonValue.Parse(stockText);
foreach (var stock in stockJson)
{
try
{
var result = new SetDataSourceReq();
result.rawdata = new List <JsonDictionary>();
result.source = collectionName;
result.category = "종목코드";
result.collected_at = "날짜";
var code = stock.Value["code"].ReadAs <string>();
var name = stock.Value["name"].ReadAs <string>();
var type = stock.Value["type"].ReadAs <string>();
var cnt = stock.Value["cnt"].ReadAs <string>();
var url = "http://companyinfo.stock.naver.com/v1/company/ajax/cF1001.aspx?cmp_cd={code}&fin_typ=0&freq_typ=Y";
var reqParam = new RequestParameter()
{
Url = url.Replace("{code}", code),
ContentType = "text/html",
EncodingOption = "UTF8",
Method = "GET"
};
var htmlCode = HttpsRequest.Instance.GetResponseByHttps(reqParam);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(htmlCode);
var titleNodes = doc.DocumentNode.SelectNodes("//th[contains(@class,'bg txt title')]");
var dateNodes = doc.DocumentNode.SelectNodes("//th[contains(@class,' bg')]");
var dataNodes = doc.DocumentNode.SelectNodes("//td[contains(@class,'num')]");
for (int i = 1; i < dateNodes.Count; i++)
{
var finance = new JsonDictionary();
var dateNode = dateNodes[i];
var dateText = dateNode.InnerText.Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace(" ", "");
if (string.IsNullOrWhiteSpace(dateText))
{
continue;
}
var dateValue = dateText.Substring(0, 7);
var date = DateTime.Parse(dateValue);
var unixtime = EnvironmentHelper.GetUnixTime(date) / 1000;
finance.Add("종목코드", code);
finance.Add("종목유형", type);
finance.Add("종목명", name);
finance.Add("상장주식수", cnt);
finance.Add("날짜", unixtime);
for (int j = 0; j < titleNodes.Count; j++)
{
var titleNode = titleNodes[j];
var dataNode = dataNodes[j * (dateNodes.Count - 1) + (i - 1)];
var key = titleNode.InnerText.Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace(" ", "");
var value = dataNode.InnerText.Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace(" ", "").Replace(",", "");
if (string.IsNullOrWhiteSpace(value))
{
continue;
}
finance.Add(key, value);
}
if (finance.GetDictionary().Keys.Count < 5)
{
continue;
}
result.rawdata.Add(finance);
}
if (result.rawdata.Count > 0)
{
ThreadPool.QueueUserWorkItem((a) =>
{
var setSourceQuery = MariaQueryBuilder.SetDataSource(result);
MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", setSourceQuery);
});
}
}
catch (Exception ex)
{
Console.WriteLine("Finance Collector Error");
}
}
callback.DynamicInvoke("test");
Console.WriteLine("{0} Collector End : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
return(true);
}
private bool CurrentStockInformation(string collectionName, Func <string, bool> callback)
{
Console.WriteLine("{0} Collector Start : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
var file = Path.Combine(System.AppDomain.CurrentDomain.BaseDirectory, "stocklist.json");
var stockText = File.ReadAllText(file);
var stockJson = JsonValue.Parse(stockText);
var progress = 1;
foreach (var stock in stockJson)
{
try
{
var result = new SetDataSourceReq();
result.rawdata = new List <JsonDictionary>();
result.source = collectionName;
result.category = "종목코드";
result.collected_at = "날짜";
var code = stock.Value["code"].ReadAs <string>();
var name = stock.Value["name"].ReadAs <string>();
var type = stock.Value["type"].ReadAs <string>();
var cnt = stock.Value["cnt"].ReadAs <string>();
var state = stock.Value.ContainsKey("state") ? stock.Value["state"].ReadAs <bool>() : false;
var nvParser = new nvParser(code);
string[] siseInfo;
var date = this.config["CurrentStockInformation"]["date"].ReadAs <string>();
if (string.IsNullOrWhiteSpace(date))
{
siseInfo = nvParser.getSise(2);
}
else
{
siseInfo = nvParser.getSise(date, 2);
}
if (siseInfo.Length < 7)
{
continue;
}
var columnInfo = new string[] { "날짜", "종가", "전일비", "시가", "고가", "저가", "거래량", "전일비율" };
var sise = new JsonDictionary();
var siseDate = DateTime.Parse(siseInfo[0]).AddHours(16);
var siseUnix = EnvironmentHelper.GetUnixTime(siseDate) / 1000;
sise.Add("종목코드", code);
sise.Add("종목명", name);
sise.Add("종목유형", type);
sise.Add("상장주식수", cnt);
sise.Add(columnInfo[0], siseUnix);
sise.Add(columnInfo[1], siseInfo[1]);
var sign = string.Empty;
if (1 + 7 < siseInfo.Length)
{
if (int.Parse(siseInfo[1]) < int.Parse(siseInfo[1 + 7]))
{
sign = "-";
}
}
sise.Add(columnInfo[2], sign + siseInfo[2]);
sise.Add(columnInfo[3], siseInfo[3]);
sise.Add(columnInfo[4], siseInfo[4]);
sise.Add(columnInfo[5], siseInfo[5]);
sise.Add(columnInfo[6], siseInfo[6]);
var diff = double.Parse(sign + siseInfo[2]);
var prevPrice = siseInfo.Length > 8 ? int.Parse(siseInfo[1 + 7]) : int.Parse(siseInfo[1]);
if (prevPrice > 0)
{
sise.Add(columnInfo[7], diff / prevPrice * 100);
}
else
{
sise.Add(columnInfo[7], 0);
}
if (sise.ContainsKey("종가"))
{
Task.Factory.StartNew(() =>
{
var analysis_sise = StockAnalysis.Instance.AutoAnalysis("day", code, siseUnix, sise);
result.rawdata.Add(analysis_sise);
var setSourceQuery = MariaQueryBuilder.SetDataSource(result);
MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", setSourceQuery);
});
}
}
catch (Exception ex)
{
Console.WriteLine("Current Stock Collector Error");
}
EnvironmentHelper.ProgressBar(progress, stockJson.Count);
progress++;
}
callback.DynamicInvoke("test");
Console.WriteLine("{0} Collector End : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
return(true);
}
private bool AllStockInformation(string collectionName, Func <string, bool> callback)
{
Console.WriteLine("{0} Collector Start : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
var file = Path.Combine(System.AppDomain.CurrentDomain.BaseDirectory, "stocklist.json");
var stockText = File.ReadAllText(file);
var stockJson = JsonValue.Parse(stockText);
var progress = 1;
foreach (var stock in stockJson)
{
try
{
var result = new SetDataSourceReq();
result.rawdata = new List <JsonDictionary>();
result.source = collectionName;
result.category = "종목코드";
result.collected_at = "날짜";
var code = stock.Value["code"].ReadAs <string>();
var name = stock.Value["name"].ReadAs <string>();
var type = stock.Value["type"].ReadAs <string>();
var cnt = stock.Value["cnt"].ReadAs <string>();
var url = "http://finance.naver.com/item/sise_day.nhn?code={code}&page={page}";
var reqParam = new RequestParameter()
{
Url = url.Replace("{code}", code).Replace("{page}", "1"),
ContentType = "text/html",
EncodingOption = "Default",
Method = "GET"
};
var htmlCode = HttpsRequest.Instance.GetResponseByHttps(reqParam);
var lastPattern = "<td class=\"pgRR\"[^>]*>(.*?)</td>";
var lastMatches = Regex.Match(htmlCode, lastPattern, RegexOptions.Singleline | RegexOptions.Multiline | RegexOptions.IgnoreCase);
var pagePattern = "page=(.*?)\"";
var page = Regex.Match(lastMatches.Value, pagePattern);
var lastNumber = 1;
if (!string.IsNullOrWhiteSpace(page.Value))
{
lastNumber = int.Parse(page.Value.Replace("page=", "").Replace("\"", ""));
lastNumber = lastNumber > 50 ? 50 : lastNumber;
}
var columnInfo = new string[] { "날짜", "종가", "전일비", "시가", "고가", "저가", "거래량", "전일비율" };
for (int i = lastNumber; i >= 1; i--)
{
reqParam.Url = url.Replace("{code}", code).Replace("{page}", i.ToString());
htmlCode = HttpsRequest.Instance.GetResponseByHttps(reqParam);
MatchCollection tableMatches = Regex.Matches(WithoutComments(htmlCode), TablePattern, ExpressionOptions);
string tableHtmlWithoutComments = WithoutComments(tableMatches[0].Value);
MatchCollection rowMatches = Regex.Matches(tableHtmlWithoutComments, RowPattern, ExpressionOptions);
for (int j = rowMatches.Count - 1; j >= 0; j--)
{
Match rowMatch = rowMatches[j];
if (!rowMatch.Value.Contains("<th"))
{
MatchCollection cellMatches = Regex.Matches(rowMatch.Value, CellPattern, ExpressionOptions);
if (cellMatches.Count == 7)
{
var sise = new JsonDictionary();
sise.Add("종목코드", code);
sise.Add("종목명", name);
sise.Add("종목유형", type);
sise.Add("상장주식수", cnt);
var index = 0;
foreach (Match cellMatch in cellMatches)
{
var key = columnInfo[index];
var valuePattern = "<span [^>]*>(.*?)</span>";
var valueMatch = Regex.Match(cellMatch.Groups[1].Value.Replace("\n", "").Replace("\t", ""), valuePattern);
var value = valueMatch.Groups[1].Value;
if (index == 0)
{
if (string.IsNullOrWhiteSpace(value))
{
break;
}
var siseDate = DateTime.Parse(value).AddHours(16);
value = (EnvironmentHelper.GetUnixTime(siseDate) / 1000).ToString();
}
if (index == 2)
{
value = cellMatch.Value.Contains("하락") ? "-" + value : value;
}
sise.Add(key, value.Replace(",", string.Empty));
index++;
}
if (index == 7)
{
if (result.rawdata.Count > 0)
{
var diff = double.Parse(sise["전일비"].ToString());
var prevPrice = double.Parse(result.rawdata[result.rawdata.Count - 1]["종가"].ToString());
sise.Add(columnInfo[index], prevPrice > 0 ? diff / prevPrice * 100 : 0);
}
result.rawdata.Add(sise);
}
}
}
}
}
if (result.rawdata.Count > 0)
{
result.rawdata = StockAnalysis.Instance.AllAnalysis(result);
ThreadPool.QueueUserWorkItem((a) =>
{
var setSourceQuery = MariaQueryBuilder.SetDataSource(result);
MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", setSourceQuery);
});
}
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
EnvironmentHelper.ProgressBar(progress, stockJson.Count);
progress++;
}
callback.DynamicInvoke("test");
Console.WriteLine("{0} Collector End : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
return(true);
}
private void ExecuteAnalysis(JsonValue analysis)
{
var analysis_name = analysis["name"].ReadAs <string>();
var target_source = analysis["target_source"].ReadAs <string>();
var analysis_query = analysis["analysis_query"].ReadAs <string>();
var analysis_options = analysis["options"];
Console.WriteLine("{0} Analysis Start : {1}", analysis_name, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
if (analysis_query.Contains("{category}"))
{
var categories_query = "SELECT category FROM current_" + target_source;
var categories = MariaDBConnector.Instance.GetJsonArray(categories_query);
foreach (var row in categories)
{
var category = row["category"].ReadAs <string>();
var query = analysis_query.Replace("{category}", category).Replace("{analysis_name}", analysis_name);
foreach (var kv in analysis_options)
{
var key = "{" + kv.Key.ToLower() + "}";
query = query.Replace(key, kv.Value.ReadAs <string>());
}
var data = MariaDBConnector.Instance.GetQuery("DynamicQueryExecuter", query);
if (data != null)
{
var setSource = new SetDataSourceReq()
{
rawdata = data,
category = category,
source = target_source,
collected_at = "날짜"
};
if (setSource.rawdata.Count > 0)
{
ThreadPool.QueueUserWorkItem((a) =>
{
var setSourceQuery = MariaQueryBuilder.SetDataSource(setSource);
MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", setSourceQuery);
});
}
}
}
}
else
{
var query = analysis_query.Replace("{analysis_name}", analysis_name);
foreach (var kv in analysis_options)
{
var key = "{" + kv.Key.ToLower() + "}";
query = query.Replace(key, kv.Value.ReadAs <string>());
}
var data = MariaDBConnector.Instance.GetQuery("DynamicQueryExecuter", query);
if (data != null)
{
var setSource = new SetDataSourceReq()
{
rawdata = data,
category = "카테고리",
source = target_source,
collected_at = "날짜"
};
if (setSource.rawdata.Count > 0)
{
var setSourceQuery = MariaQueryBuilder.SetDataSource(setSource);
MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", setSourceQuery);
}
}
}
Console.WriteLine("{0} Analysis End : {1}", analysis_name, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
}
public void AutoAnalysis(string p, ref SetDataSourceReq result)
{
throw new NotImplementedException();
}
