// load existing processed data
// compare pearson signature
// look for status.txt - Append or FirstDate operation / Append.txt, FirstDate.txt
// load JSONTemp data
// save in Compiled dir
public void ProcessAppend(string id)
{
string dir = "compiled\\" + id + "\\";
string datafile = dir + id + ".txt";
string eventf = dir + id + ".events.txt";
if (!File.Exists(datafile) || !File.Exists(eventf))
{
ErrorAppend("ProcessIDAppend: data or event file does not exist " + id); // should be done by AddID / QueueRestart
return;
}
// status.txt
string jsontemp = "JSONTemp\\" + id + "\\";
string statusf = jsontemp + "status.txt";
if (!File.Exists(statusf))
{
ErrorAppend("ProcessIDAppend: status file does not exist");
return;
}
if (File.ReadAllText(statusf) == "Ok") return;
if (File.ReadAllText(statusf) == "Skip") return;
bool specialcase = false;
if (File.ReadAllText(statusf).IndexOf("JSON") != -1 && !File.Exists(dir + id + ".0.txt"))
{
specialcase = true;
File.WriteAllText(statusf, "Append");
}
string whattodo = File.ReadAllText(statusf);
if (whattodo.IndexOf("Append") == -1 && specialcase == false) return;
// append.txt
string append = jsontemp + "Append.txt";
if (!File.Exists(append))
{
Console.WriteLine("append.txt data file not found - " + id);
return;
}
// load yahoo and get last updated date
ShareData yahoosharedata = new ShareData(id);
yahoosharedata.LoadYahooData(File.ReadAllText(datafile), File.ReadAllText(eventf));
string yahoolastline = yahoosharedata[0]; // [0] is latest data
string[] yahoosplit = yahoolastline.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
string yahoolastdate = yahoosplit[0];
//if (id != "205778" && yahoolastdate == msftdate)
// return;
// load append data
ShareData appenddata = new ShareData(id);
string appendlastline = "";
appenddata.LoadJSONData(File.ReadAllText(append));
appenddata.Sort();
if (yahoolastline != appenddata[yahoolastdate])
{
// dividend, split occured
ErrorAppend("dividend/split occured " + id);
QueueRestart(id);
return;
}
// look for pearson signature
appendlastline = appenddata[0]; // [0] latest
if (id != "234871")
{
string[] appendsplit = appendlastline.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
string appendlastdate = appendsplit[0];
if (appendlastline == m_pearson[appendlastdate])
{
// delisted
ErrorAppend("delisted - " + id);
return;
}
}
// combine
if (yahoosharedata.LoadJSONData(File.ReadAllText(append)) == 0)
{
//ErrorAppend("no datapoints " + id);
File.WriteAllText(statusf, "Ok");
return;
}
File.WriteAllText(statusf, "Ok");
yahoosharedata.Save();
if (FinishedProcessingEvent != null)
FinishedProcessingEvent(id);
}