// load existing processed data // compare pearson signature // look for status.txt - Append or FirstDate operation / Append.txt, FirstDate.txt // load JSONTemp data // save in Compiled dir public void ProcessAppend(string id) { string dir = "compiled\\" + id + "\\"; string datafile = dir + id + ".txt"; string eventf = dir + id + ".events.txt"; if (!File.Exists(datafile) || !File.Exists(eventf)) { ErrorAppend("ProcessIDAppend: data or event file does not exist " + id); // should be done by AddID / QueueRestart return; } // status.txt string jsontemp = "JSONTemp\\" + id + "\\"; string statusf = jsontemp + "status.txt"; if (!File.Exists(statusf)) { ErrorAppend("ProcessIDAppend: status file does not exist"); return; } if (File.ReadAllText(statusf) == "Ok") return; if (File.ReadAllText(statusf) == "Skip") return; bool specialcase = false; if (File.ReadAllText(statusf).IndexOf("JSON") != -1 && !File.Exists(dir + id + ".0.txt")) { specialcase = true; File.WriteAllText(statusf, "Append"); } string whattodo = File.ReadAllText(statusf); if (whattodo.IndexOf("Append") == -1 && specialcase == false) return; // append.txt string append = jsontemp + "Append.txt"; if (!File.Exists(append)) { Console.WriteLine("append.txt data file not found - " + id); return; } // load yahoo and get last updated date ShareData yahoosharedata = new ShareData(id); yahoosharedata.LoadYahooData(File.ReadAllText(datafile), File.ReadAllText(eventf)); string yahoolastline = yahoosharedata[0]; // [0] is latest data string[] yahoosplit = yahoolastline.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); string yahoolastdate = yahoosplit[0]; //if (id != "205778" && yahoolastdate == msftdate) // return; // load append data ShareData appenddata = new ShareData(id); string appendlastline = ""; appenddata.LoadJSONData(File.ReadAllText(append)); appenddata.Sort(); if (yahoolastline != appenddata[yahoolastdate]) { // dividend, split occured ErrorAppend("dividend/split occured " + id); QueueRestart(id); return; } // look for pearson signature appendlastline = appenddata[0]; // [0] latest if (id != "234871") { string[] appendsplit = appendlastline.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); string appendlastdate = appendsplit[0]; if (appendlastline == m_pearson[appendlastdate]) { // delisted ErrorAppend("delisted - " + id); return; } } // combine if (yahoosharedata.LoadJSONData(File.ReadAllText(append)) == 0) { //ErrorAppend("no datapoints " + id); File.WriteAllText(statusf, "Ok"); return; } File.WriteAllText(statusf, "Ok"); yahoosharedata.Save(); if (FinishedProcessingEvent != null) FinishedProcessingEvent(id); }
public void ProcessJSON(string id) { string dir = "JSONTemp\\" + id + "\\"; string statusf = dir + "status.txt"; if (!File.Exists(statusf)) return; string content = File.ReadAllText(statusf); if (content.IndexOf("JSON") == -1) return; int count = 0; ShareData sd = new ShareData(id); for (; ; ) { string path = dir + id + "." + count + ".txt"; if (!System.IO.File.Exists(path)) break; JSONIntoShareData(id, File.ReadAllText(path), sd); count++; } sd.Save(); File.WriteAllText(statusf, "Ok"); if (FinishedProcessingEvent != null) FinishedProcessingEvent(id); }
public void ProcessJSON(string id, string symbol) { string dir = "JSON"; int count = 0; ShareData sd = new ShareData(id, symbol); for (; ; ) { string path = dir + "\\" + id + "." + count + ".txt"; if (!System.IO.File.Exists(path)) break; string filecontent = System.IO.File.ReadAllText(path); Match chartpointmatch = Regex.Match(filecontent, @"WSOD.InteractiveChart.SetChartPoints\((.+?)\)"); if (chartpointmatch.Success == false) { ErrorCombine("no datapoints in " + path); continue; } string outerdata = chartpointmatch.Groups[1].Value; bool success; MatchCollection mm = Regex.Matches(outerdata, @"\\""(.+?)}"); foreach (Match mmm in mm) { string extractedline = mmm.Groups[1].Value; MatchCollection line = Regex.Matches(extractedline, @":\\""(.+?)\\"""); string date = line[0].Groups[1].Value; string open = line[2].Groups[1].Value; // [1] is UK date, [2] is descriptive date string high = line[3].Groups[1].Value; string low = line[4].Groups[1].Value; string close = line[5].Groups[1].Value; // is adjusted string volume = line[6].Groups[1].Value; success = sd.AddDataPoint(date, open, high, low, close, volume); if (success == false) ErrorCombine("duplicate data " + date + " " + sd.symbol); } mm = Regex.Matches(filecontent, @"~overlayEvent_~(.+?)\.replace(.+?)\.replace"); foreach (Match mmm in mm) { Match datematch = Regex.Match(mmm.Groups[2].Value, @"~overlayDate_~\\u003e(.+?)\\u003c"); string date = datematch.Groups[1].Value; Match opmatch = Regex.Match(mmm.Groups[2].Value, @"~header_~\\u003e(.+?)\\u003c"); string op = opmatch.Groups[1].Value; //MatchCollection values = Regex.Matches(mmm.Groups[1].Value, @"\\u003e(.+?)\.\\u003c"); // \u003e\u003cdiv\u003e1.00 in PCT.\u003c, \u003e\u003cdiv\u003e0.3812 in USD.\u003c // earnings // split // dividend // stock dividend MatchCollection values; if (op == "Earnings" || op == "Split") values = Regex.Matches(mmm.Groups[1].Value, @"\\u003e(.+?)\\u003c"); else values = Regex.Matches(mmm.Groups[1].Value, @"\\u003e(.+?)\.\\u003c"); foreach (Match value in values) { string opext = value.Groups[1].Value; opext = opext.Replace(@"\u003c", ""); opext = opext.Replace(@"\u003e", ""); opext = opext.Replace(@"div", ""); string[] splits = opext.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); if (splits.Length == 3) // ex 1.00 in PCT. { if (splits[2].IndexOf("PCT") != -1) success = sd.AddEvent(date, op, splits[0], "PCT"); else success = sd.AddEvent(date, op, splits[0], splits[2].Replace(".", "")); } else // no currency - earnings - probably in USD success = sd.AddEvent(date, op, splits[0], "N/A"); if (success == false) ErrorCombine("duplicate event " + date + " " + sd.symbol); } } count++; } sd.Save(); }