public SetDataSourceRes SetDataSource(Stream stream) { if (WebOperationContext.Current.IncomingRequest.Headers == null) { throw new Exception("Can not get current WebOpreationContext."); } SetDataSourceReq param = null; using (var streamReader = new StreamReader(stream)) { param = DataConverter.Deserializer <SetDataSourceReq>(streamReader.ReadToEnd()); } var res = new SetDataSourceRes(); if (param != null && param.rawdata != null && param.rawdata.Count > 0 && !string.IsNullOrWhiteSpace(param.source) && !string.IsNullOrWhiteSpace(param.category)) { var query = MariaQueryBuilder.SetDataSource(param); MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", query); res.code = "200"; res.message = "요청 접수 완료"; } else { res.code = "400"; res.message = "잘못된 요청"; } return(res); }
public List <JsonDictionary> AllAnalysis(SetDataSourceReq data_source) { var key = "종가"; var time_key = "날짜"; var result = new List <JsonDictionary>(); for (var i = 1; i <= data_source.rawdata.Count; i++) { var origin = new List <JsonDictionary>(data_source.rawdata.GetRange(0, i)); var ret_data = SingleAnalysis(origin, key, i, time_key); var test = new JsonDictionary(ret_data.GetDictionary()); result.Add(test); } return(result); }
public static string SetDataSource(SetDataSourceReq param) { var query = string.Empty; if (!SourceList.Contains(param.source)) { query = CreateSourceTable(param.source); lock (SourceList) { SourceList.Add(param.source.ToLower()); } } query = InsertSource(param.source, param.category, param.rawdata, param.collected_at, query); return(query); }
private bool EmptyInformation(string collectionName, Func <string, bool> callback) { var file = Path.Combine(System.AppDomain.CurrentDomain.BaseDirectory, "stocklist.json"); var stockText = File.ReadAllText(file); var stockJson = JsonValue.Parse(stockText); foreach (var stock in stockJson) { var result = new SetDataSourceReq(); result.rawdata = new List <JsonDictionary>(); result.source = collectionName; result.category = "종목코드"; result.collected_at = "날짜"; var finance = new JsonDictionary(); var code = stock.Value["code"].ReadAs <string>(); var name = stock.Value["name"].ReadAs <string>(); var type = stock.Value["type"].ReadAs <string>(); var cnt = stock.Value["cnt"].ReadAs <string>(); finance.Add("종목코드", code); finance.Add("종목유형", type); finance.Add("종목명", name); finance.Add("상장주식수", cnt); result.rawdata.Add(finance); ThreadPool.QueueUserWorkItem((a) => { var setSourceQuery = MariaQueryBuilder.SetDataSource(result); MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", setSourceQuery); }); } callback.DynamicInvoke("test"); return(true); }
private bool KOSPI_KOSDAQ(string collectionName, Func <string, bool> callback) { Console.WriteLine("{0} Collector Start : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); var codeArr = new List <string>() { "KOSPI", "KOSDAQ" }; foreach (var code in codeArr) { try { var googleUrl = "http://www.google.com/finance/getprices?q={code}&i=86400&p=40Y&f=d,c,v,k,o,h,l&df=cpct&auto=0".Replace("{code}", code); var reqParam = new RequestParameter() { Url = googleUrl, ContentType = "application/json", EncodingOption = "UTF8", Method = "GET" }; var stockData = this.DecodeHex(HttpsRequest.Instance.GetResponseByHttps(reqParam)); var histroyCsv = Regex.Split(stockData, @"\n"); var columnInfo = new string[] { "날짜", "종가", "고가", "저가", "시가", "거래량", "전일비", "전일비율" }; var result = new SetDataSourceReq(); result.rawdata = new List <JsonDictionary>(); result.source = collectionName; result.category = "종목코드"; result.collected_at = "날짜"; var standardTime = string.Empty; for (int i = 7; i < histroyCsv.Length - 1; i++) { var sise = new JsonDictionary(); var row = histroyCsv[i].Split(','); sise.Add("종목코드", code); sise.Add("종목명", code); sise.Add("종목유형", "SOSOK"); if (row[0].Trim().Contains("a")) { standardTime = row[0].Trim().Replace("a", ""); sise.Add(columnInfo[0], standardTime); } else { sise.Add(columnInfo[0], (int.Parse(standardTime) + (86400 * int.Parse(row[0]))).ToString()); } sise.Add(columnInfo[1], row[1].Trim()); sise.Add(columnInfo[2], row[2].Trim()); sise.Add(columnInfo[3], row[3].Trim()); sise.Add(columnInfo[4], row[4].Trim()); sise.Add(columnInfo[5], row[5].Trim()); if (result.rawdata.Count > 0) { var prevPrice = double.Parse(result.rawdata[result.rawdata.Count - 1]["종가"].ToString()); if (prevPrice > 0) { var diff = double.Parse(row[1].Trim()) - prevPrice; sise.Add(columnInfo[6], diff); sise.Add(columnInfo[7], (double)diff / prevPrice * 100); } } result.rawdata.Add(sise); } if (result.rawdata.Count > 0) { ThreadPool.QueueUserWorkItem((a) => { var setSourceQuery = MariaQueryBuilder.SetDataSource(result); MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", setSourceQuery); }); } } catch (Exception ex) { Console.WriteLine("All Stock Collector Error"); } } callback.DynamicInvoke("test"); Console.WriteLine("{0} Collector End : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); return(true); }
private bool FinanceInformation(string collectionName, Func <string, bool> callback) { Console.WriteLine("{0} Collector Start : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); var file = Path.Combine(System.AppDomain.CurrentDomain.BaseDirectory, "stocklist.json"); var stockText = File.ReadAllText(file); var stockJson = JsonValue.Parse(stockText); foreach (var stock in stockJson) { try { var result = new SetDataSourceReq(); result.rawdata = new List <JsonDictionary>(); result.source = collectionName; result.category = "종목코드"; result.collected_at = "날짜"; var code = stock.Value["code"].ReadAs <string>(); var name = stock.Value["name"].ReadAs <string>(); var type = stock.Value["type"].ReadAs <string>(); var cnt = stock.Value["cnt"].ReadAs <string>(); var url = "http://companyinfo.stock.naver.com/v1/company/ajax/cF1001.aspx?cmp_cd={code}&fin_typ=0&freq_typ=Y"; var reqParam = new RequestParameter() { Url = url.Replace("{code}", code), ContentType = "text/html", EncodingOption = "UTF8", Method = "GET" }; var htmlCode = HttpsRequest.Instance.GetResponseByHttps(reqParam); HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(htmlCode); var titleNodes = doc.DocumentNode.SelectNodes("//th[contains(@class,'bg txt title')]"); var dateNodes = doc.DocumentNode.SelectNodes("//th[contains(@class,' bg')]"); var dataNodes = doc.DocumentNode.SelectNodes("//td[contains(@class,'num')]"); for (int i = 1; i < dateNodes.Count; i++) { var finance = new JsonDictionary(); var dateNode = dateNodes[i]; var dateText = dateNode.InnerText.Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace(" ", ""); if (string.IsNullOrWhiteSpace(dateText)) { continue; } var dateValue = dateText.Substring(0, 7); var date = DateTime.Parse(dateValue); var unixtime = EnvironmentHelper.GetUnixTime(date) / 1000; finance.Add("종목코드", code); finance.Add("종목유형", type); finance.Add("종목명", name); finance.Add("상장주식수", cnt); finance.Add("날짜", unixtime); for (int j = 0; j < titleNodes.Count; j++) { var titleNode = titleNodes[j]; var dataNode = dataNodes[j * (dateNodes.Count - 1) + (i - 1)]; var key = titleNode.InnerText.Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace(" ", ""); var value = dataNode.InnerText.Replace("\r", "").Replace("\n", "").Replace("\t", "").Replace(" ", "").Replace(",", ""); if (string.IsNullOrWhiteSpace(value)) { continue; } finance.Add(key, value); } if (finance.GetDictionary().Keys.Count < 5) { continue; } result.rawdata.Add(finance); } if (result.rawdata.Count > 0) { ThreadPool.QueueUserWorkItem((a) => { var setSourceQuery = MariaQueryBuilder.SetDataSource(result); MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", setSourceQuery); }); } } catch (Exception ex) { Console.WriteLine("Finance Collector Error"); } } callback.DynamicInvoke("test"); Console.WriteLine("{0} Collector End : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); return(true); }
private bool CurrentStockInformation(string collectionName, Func <string, bool> callback) { Console.WriteLine("{0} Collector Start : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); var file = Path.Combine(System.AppDomain.CurrentDomain.BaseDirectory, "stocklist.json"); var stockText = File.ReadAllText(file); var stockJson = JsonValue.Parse(stockText); var progress = 1; foreach (var stock in stockJson) { try { var result = new SetDataSourceReq(); result.rawdata = new List <JsonDictionary>(); result.source = collectionName; result.category = "종목코드"; result.collected_at = "날짜"; var code = stock.Value["code"].ReadAs <string>(); var name = stock.Value["name"].ReadAs <string>(); var type = stock.Value["type"].ReadAs <string>(); var cnt = stock.Value["cnt"].ReadAs <string>(); var state = stock.Value.ContainsKey("state") ? stock.Value["state"].ReadAs <bool>() : false; var nvParser = new nvParser(code); string[] siseInfo; var date = this.config["CurrentStockInformation"]["date"].ReadAs <string>(); if (string.IsNullOrWhiteSpace(date)) { siseInfo = nvParser.getSise(2); } else { siseInfo = nvParser.getSise(date, 2); } if (siseInfo.Length < 7) { continue; } var columnInfo = new string[] { "날짜", "종가", "전일비", "시가", "고가", "저가", "거래량", "전일비율" }; var sise = new JsonDictionary(); var siseDate = DateTime.Parse(siseInfo[0]).AddHours(16); var siseUnix = EnvironmentHelper.GetUnixTime(siseDate) / 1000; sise.Add("종목코드", code); sise.Add("종목명", name); sise.Add("종목유형", type); sise.Add("상장주식수", cnt); sise.Add(columnInfo[0], siseUnix); sise.Add(columnInfo[1], siseInfo[1]); var sign = string.Empty; if (1 + 7 < siseInfo.Length) { if (int.Parse(siseInfo[1]) < int.Parse(siseInfo[1 + 7])) { sign = "-"; } } sise.Add(columnInfo[2], sign + siseInfo[2]); sise.Add(columnInfo[3], siseInfo[3]); sise.Add(columnInfo[4], siseInfo[4]); sise.Add(columnInfo[5], siseInfo[5]); sise.Add(columnInfo[6], siseInfo[6]); var diff = double.Parse(sign + siseInfo[2]); var prevPrice = siseInfo.Length > 8 ? int.Parse(siseInfo[1 + 7]) : int.Parse(siseInfo[1]); if (prevPrice > 0) { sise.Add(columnInfo[7], diff / prevPrice * 100); } else { sise.Add(columnInfo[7], 0); } if (sise.ContainsKey("종가")) { Task.Factory.StartNew(() => { var analysis_sise = StockAnalysis.Instance.AutoAnalysis("day", code, siseUnix, sise); result.rawdata.Add(analysis_sise); var setSourceQuery = MariaQueryBuilder.SetDataSource(result); MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", setSourceQuery); }); } } catch (Exception ex) { Console.WriteLine("Current Stock Collector Error"); } EnvironmentHelper.ProgressBar(progress, stockJson.Count); progress++; } callback.DynamicInvoke("test"); Console.WriteLine("{0} Collector End : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); return(true); }
private bool AllStockInformation(string collectionName, Func <string, bool> callback) { Console.WriteLine("{0} Collector Start : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); var file = Path.Combine(System.AppDomain.CurrentDomain.BaseDirectory, "stocklist.json"); var stockText = File.ReadAllText(file); var stockJson = JsonValue.Parse(stockText); var progress = 1; foreach (var stock in stockJson) { try { var result = new SetDataSourceReq(); result.rawdata = new List <JsonDictionary>(); result.source = collectionName; result.category = "종목코드"; result.collected_at = "날짜"; var code = stock.Value["code"].ReadAs <string>(); var name = stock.Value["name"].ReadAs <string>(); var type = stock.Value["type"].ReadAs <string>(); var cnt = stock.Value["cnt"].ReadAs <string>(); var url = "http://finance.naver.com/item/sise_day.nhn?code={code}&page={page}"; var reqParam = new RequestParameter() { Url = url.Replace("{code}", code).Replace("{page}", "1"), ContentType = "text/html", EncodingOption = "Default", Method = "GET" }; var htmlCode = HttpsRequest.Instance.GetResponseByHttps(reqParam); var lastPattern = "<td class=\"pgRR\"[^>]*>(.*?)</td>"; var lastMatches = Regex.Match(htmlCode, lastPattern, RegexOptions.Singleline | RegexOptions.Multiline | RegexOptions.IgnoreCase); var pagePattern = "page=(.*?)\""; var page = Regex.Match(lastMatches.Value, pagePattern); var lastNumber = 1; if (!string.IsNullOrWhiteSpace(page.Value)) { lastNumber = int.Parse(page.Value.Replace("page=", "").Replace("\"", "")); lastNumber = lastNumber > 50 ? 50 : lastNumber; } var columnInfo = new string[] { "날짜", "종가", "전일비", "시가", "고가", "저가", "거래량", "전일비율" }; for (int i = lastNumber; i >= 1; i--) { reqParam.Url = url.Replace("{code}", code).Replace("{page}", i.ToString()); htmlCode = HttpsRequest.Instance.GetResponseByHttps(reqParam); MatchCollection tableMatches = Regex.Matches(WithoutComments(htmlCode), TablePattern, ExpressionOptions); string tableHtmlWithoutComments = WithoutComments(tableMatches[0].Value); MatchCollection rowMatches = Regex.Matches(tableHtmlWithoutComments, RowPattern, ExpressionOptions); for (int j = rowMatches.Count - 1; j >= 0; j--) { Match rowMatch = rowMatches[j]; if (!rowMatch.Value.Contains("<th")) { MatchCollection cellMatches = Regex.Matches(rowMatch.Value, CellPattern, ExpressionOptions); if (cellMatches.Count == 7) { var sise = new JsonDictionary(); sise.Add("종목코드", code); sise.Add("종목명", name); sise.Add("종목유형", type); sise.Add("상장주식수", cnt); var index = 0; foreach (Match cellMatch in cellMatches) { var key = columnInfo[index]; var valuePattern = "<span [^>]*>(.*?)</span>"; var valueMatch = Regex.Match(cellMatch.Groups[1].Value.Replace("\n", "").Replace("\t", ""), valuePattern); var value = valueMatch.Groups[1].Value; if (index == 0) { if (string.IsNullOrWhiteSpace(value)) { break; } var siseDate = DateTime.Parse(value).AddHours(16); value = (EnvironmentHelper.GetUnixTime(siseDate) / 1000).ToString(); } if (index == 2) { value = cellMatch.Value.Contains("하락") ? "-" + value : value; } sise.Add(key, value.Replace(",", string.Empty)); index++; } if (index == 7) { if (result.rawdata.Count > 0) { var diff = double.Parse(sise["전일비"].ToString()); var prevPrice = double.Parse(result.rawdata[result.rawdata.Count - 1]["종가"].ToString()); sise.Add(columnInfo[index], prevPrice > 0 ? diff / prevPrice * 100 : 0); } result.rawdata.Add(sise); } } } } } if (result.rawdata.Count > 0) { result.rawdata = StockAnalysis.Instance.AllAnalysis(result); ThreadPool.QueueUserWorkItem((a) => { var setSourceQuery = MariaQueryBuilder.SetDataSource(result); MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", setSourceQuery); }); } } catch (Exception ex) { Console.WriteLine(ex.ToString()); } EnvironmentHelper.ProgressBar(progress, stockJson.Count); progress++; } callback.DynamicInvoke("test"); Console.WriteLine("{0} Collector End : {1}", collectionName, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); return(true); }
private void ExecuteAnalysis(JsonValue analysis) { var analysis_name = analysis["name"].ReadAs <string>(); var target_source = analysis["target_source"].ReadAs <string>(); var analysis_query = analysis["analysis_query"].ReadAs <string>(); var analysis_options = analysis["options"]; Console.WriteLine("{0} Analysis Start : {1}", analysis_name, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); if (analysis_query.Contains("{category}")) { var categories_query = "SELECT category FROM current_" + target_source; var categories = MariaDBConnector.Instance.GetJsonArray(categories_query); foreach (var row in categories) { var category = row["category"].ReadAs <string>(); var query = analysis_query.Replace("{category}", category).Replace("{analysis_name}", analysis_name); foreach (var kv in analysis_options) { var key = "{" + kv.Key.ToLower() + "}"; query = query.Replace(key, kv.Value.ReadAs <string>()); } var data = MariaDBConnector.Instance.GetQuery("DynamicQueryExecuter", query); if (data != null) { var setSource = new SetDataSourceReq() { rawdata = data, category = category, source = target_source, collected_at = "날짜" }; if (setSource.rawdata.Count > 0) { ThreadPool.QueueUserWorkItem((a) => { var setSourceQuery = MariaQueryBuilder.SetDataSource(setSource); MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", setSourceQuery); }); } } } } else { var query = analysis_query.Replace("{analysis_name}", analysis_name); foreach (var kv in analysis_options) { var key = "{" + kv.Key.ToLower() + "}"; query = query.Replace(key, kv.Value.ReadAs <string>()); } var data = MariaDBConnector.Instance.GetQuery("DynamicQueryExecuter", query); if (data != null) { var setSource = new SetDataSourceReq() { rawdata = data, category = "카테고리", source = target_source, collected_at = "날짜" }; if (setSource.rawdata.Count > 0) { var setSourceQuery = MariaQueryBuilder.SetDataSource(setSource); MariaDBConnector.Instance.SetQuery("DynamicQueryExecuter", setSourceQuery); } } } Console.WriteLine("{0} Analysis End : {1}", analysis_name, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); }
public void AutoAnalysis(string p, ref SetDataSourceReq result) { throw new NotImplementedException(); }