// load existing processed data
// compare pearson signature
// look for status.txt - Append or FirstDate operation / Append.txt, FirstDate.txt
// load JSONTemp data
// save in Compiled dir
public void ProcessAppend(string id)
{
string dir = "compiled\\" + id + "\\";
string datafile = dir + id + ".txt";
string eventf = dir + id + ".events.txt";
if (!File.Exists(datafile) || !File.Exists(eventf))
{
ErrorAppend("ProcessIDAppend: data or event file does not exist " + id); // should be done by AddID / QueueRestart
return;
}
// status.txt
string jsontemp = "JSONTemp\\" + id + "\\";
string statusf = jsontemp + "status.txt";
if (!File.Exists(statusf))
{
ErrorAppend("ProcessIDAppend: status file does not exist");
return;
}
if (File.ReadAllText(statusf) == "Ok") return;
if (File.ReadAllText(statusf) == "Skip") return;
bool specialcase = false;
if (File.ReadAllText(statusf).IndexOf("JSON") != -1 && !File.Exists(dir + id + ".0.txt"))
{
specialcase = true;
File.WriteAllText(statusf, "Append");
}
string whattodo = File.ReadAllText(statusf);
if (whattodo.IndexOf("Append") == -1 && specialcase == false) return;
// append.txt
string append = jsontemp + "Append.txt";
if (!File.Exists(append))
{
Console.WriteLine("append.txt data file not found - " + id);
return;
}
// load yahoo and get last updated date
ShareData yahoosharedata = new ShareData(id);
yahoosharedata.LoadYahooData(File.ReadAllText(datafile), File.ReadAllText(eventf));
string yahoolastline = yahoosharedata[0]; // [0] is latest data
string[] yahoosplit = yahoolastline.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
string yahoolastdate = yahoosplit[0];
//if (id != "205778" && yahoolastdate == msftdate)
// return;
// load append data
ShareData appenddata = new ShareData(id);
string appendlastline = "";
appenddata.LoadJSONData(File.ReadAllText(append));
appenddata.Sort();
if (yahoolastline != appenddata[yahoolastdate])
{
// dividend, split occured
ErrorAppend("dividend/split occured " + id);
QueueRestart(id);
return;
}
// look for pearson signature
appendlastline = appenddata[0]; // [0] latest
if (id != "234871")
{
string[] appendsplit = appendlastline.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
string appendlastdate = appendsplit[0];
if (appendlastline == m_pearson[appendlastdate])
{
// delisted
ErrorAppend("delisted - " + id);
return;
}
}
// combine
if (yahoosharedata.LoadJSONData(File.ReadAllText(append)) == 0)
{
//ErrorAppend("no datapoints " + id);
File.WriteAllText(statusf, "Ok");
return;
}
File.WriteAllText(statusf, "Ok");
yahoosharedata.Save();
if (FinishedProcessingEvent != null)
FinishedProcessingEvent(id);
}
public void ProcessJSON(string id)
{
string dir = "JSONTemp\\" + id + "\\";
string statusf = dir + "status.txt";
if (!File.Exists(statusf)) return;
string content = File.ReadAllText(statusf);
if (content.IndexOf("JSON") == -1) return;
int count = 0;
ShareData sd = new ShareData(id);
for (; ; )
{
string path = dir + id + "." + count + ".txt";
if (!System.IO.File.Exists(path))
break;
JSONIntoShareData(id, File.ReadAllText(path), sd);
count++;
}
sd.Save();
File.WriteAllText(statusf, "Ok");
if (FinishedProcessingEvent != null)
FinishedProcessingEvent(id);
}
public void ProcessJSON(string id, string symbol)
{
string dir = "JSON";
int count = 0;
ShareData sd = new ShareData(id, symbol);
for (; ; )
{
string path = dir + "\\" + id + "." + count + ".txt";
if (!System.IO.File.Exists(path))
break;
string filecontent = System.IO.File.ReadAllText(path);
Match chartpointmatch = Regex.Match(filecontent, @"WSOD.InteractiveChart.SetChartPoints\((.+?)\)");
if (chartpointmatch.Success == false)
{
ErrorCombine("no datapoints in " + path);
continue;
}
string outerdata = chartpointmatch.Groups[1].Value;
bool success;
MatchCollection mm = Regex.Matches(outerdata, @"\\""(.+?)}");
foreach (Match mmm in mm)
{
string extractedline = mmm.Groups[1].Value;
MatchCollection line = Regex.Matches(extractedline, @":\\""(.+?)\\""");
string date = line[0].Groups[1].Value;
string open = line[2].Groups[1].Value; // [1] is UK date, [2] is descriptive date
string high = line[3].Groups[1].Value;
string low = line[4].Groups[1].Value;
string close = line[5].Groups[1].Value; // is adjusted
string volume = line[6].Groups[1].Value;
success = sd.AddDataPoint(date, open, high, low, close, volume);
if (success == false)
ErrorCombine("duplicate data " + date + " " + sd.symbol);
}
mm = Regex.Matches(filecontent, @"~overlayEvent_~(.+?)\.replace(.+?)\.replace");
foreach (Match mmm in mm)
{
Match datematch = Regex.Match(mmm.Groups[2].Value, @"~overlayDate_~\\u003e(.+?)\\u003c");
string date = datematch.Groups[1].Value;
Match opmatch = Regex.Match(mmm.Groups[2].Value, @"~header_~\\u003e(.+?)\\u003c");
string op = opmatch.Groups[1].Value;
//MatchCollection values = Regex.Matches(mmm.Groups[1].Value, @"\\u003e(.+?)\.\\u003c"); // \u003e\u003cdiv\u003e1.00 in PCT.\u003c, \u003e\u003cdiv\u003e0.3812 in USD.\u003c
// earnings
// split
// dividend
// stock dividend
MatchCollection values;
if (op == "Earnings" || op == "Split") values = Regex.Matches(mmm.Groups[1].Value, @"\\u003e(.+?)\\u003c");
else values = Regex.Matches(mmm.Groups[1].Value, @"\\u003e(.+?)\.\\u003c");
foreach (Match value in values)
{
string opext = value.Groups[1].Value;
opext = opext.Replace(@"\u003c", "");
opext = opext.Replace(@"\u003e", "");
opext = opext.Replace(@"div", "");
string[] splits = opext.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
if (splits.Length == 3) // ex 1.00 in PCT.
{
if (splits[2].IndexOf("PCT") != -1)
success = sd.AddEvent(date, op, splits[0], "PCT");
else
success = sd.AddEvent(date, op, splits[0], splits[2].Replace(".", ""));
}
else // no currency - earnings - probably in USD
success = sd.AddEvent(date, op, splits[0], "N/A");
if (success == false)
ErrorCombine("duplicate event " + date + " " + sd.symbol);
}
}
count++;
}
sd.Save();
}