// load existing processed data // compare pearson signature // look for status.txt - Append or FirstDate operation / Append.txt, FirstDate.txt // load JSONTemp data // save in Compiled dir public void ProcessAppend(string id) { string dir = "compiled\\" + id + "\\"; string datafile = dir + id + ".txt"; string eventf = dir + id + ".events.txt"; if (!File.Exists(datafile) || !File.Exists(eventf)) { ErrorAppend("ProcessIDAppend: data or event file does not exist " + id); // should be done by AddID / QueueRestart return; } // status.txt string jsontemp = "JSONTemp\\" + id + "\\"; string statusf = jsontemp + "status.txt"; if (!File.Exists(statusf)) { ErrorAppend("ProcessIDAppend: status file does not exist"); return; } if (File.ReadAllText(statusf) == "Ok") return; if (File.ReadAllText(statusf) == "Skip") return; bool specialcase = false; if (File.ReadAllText(statusf).IndexOf("JSON") != -1 && !File.Exists(dir + id + ".0.txt")) { specialcase = true; File.WriteAllText(statusf, "Append"); } string whattodo = File.ReadAllText(statusf); if (whattodo.IndexOf("Append") == -1 && specialcase == false) return; // append.txt string append = jsontemp + "Append.txt"; if (!File.Exists(append)) { Console.WriteLine("append.txt data file not found - " + id); return; } // load yahoo and get last updated date ShareData yahoosharedata = new ShareData(id); yahoosharedata.LoadYahooData(File.ReadAllText(datafile), File.ReadAllText(eventf)); string yahoolastline = yahoosharedata[0]; // [0] is latest data string[] yahoosplit = yahoolastline.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); string yahoolastdate = yahoosplit[0]; //if (id != "205778" && yahoolastdate == msftdate) // return; // load append data ShareData appenddata = new ShareData(id); string appendlastline = ""; appenddata.LoadJSONData(File.ReadAllText(append)); appenddata.Sort(); if (yahoolastline != appenddata[yahoolastdate]) { // dividend, split occured ErrorAppend("dividend/split occured " + id); QueueRestart(id); return; } // look for pearson signature appendlastline = appenddata[0]; // [0] latest if (id != "234871") { string[] appendsplit = appendlastline.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); string appendlastdate = appendsplit[0]; if (appendlastline == m_pearson[appendlastdate]) { // delisted ErrorAppend("delisted - " + id); return; } } // combine if (yahoosharedata.LoadJSONData(File.ReadAllText(append)) == 0) { //ErrorAppend("no datapoints " + id); File.WriteAllText(statusf, "Ok"); return; } File.WriteAllText(statusf, "Ok"); yahoosharedata.Save(); if (FinishedProcessingEvent != null) FinishedProcessingEvent(id); }