//Creates from scratch, or 'tops up', or reloads a single day of, an All-Table for a single share for a period
//spanning tradingSpan trading days starting at startDate.
//If topupOnly, then a leading number of days get 'skipped' (since the AllTable is assumed to already have the data in it).
//If reloadOffset is non zero, we skip that number of AllTable records into the AllTable file and then overwrite
//the succeeding 104 bands (i.e. one days worth)
//The passed in tradeHash holds the raw trading information needed to create each AllTable record.
//tradeHash: Key: 'ShareName,YYMMDD,bandNum' e.g. "B+S BANKSYSTEME AG O.N.,180722,1" band 1 is from 09:00:00 to 09:04:59
// Value: Trade object with properties shareName,shareNum,tradeDate,line ... but shareNum at this stage may be 0
private static void GenerateSingleAllTable(
CancellationToken ct, string allTableFile, int shareNum, string shareName, DateTime startDate, int tradingSpan,
ref Dictionary <string, Trade> tradeHash, bool topUpOnly, TopupInformation topupInfo, string reloadDate)
{
LogHeaderForGenerateSingleAllTable(allTableFile, startDate, topUpOnly, reloadDate);
AllTable atRec;
var runDate = startDate.AddDays(0);
//Create or Append to AllTable file?
FileMode mode;
if (reloadDate.Length == 6)
{
// we are reloading... reloadDate = YYMMDD
mode = FileMode.Open;
}
else
{
//more usual case
mode = topUpOnly ? FileMode.Append : FileMode.Create;
}
using (FileStream fs = new FileStream(allTableFile, mode))
{
int rowNum = 0;
if (reloadDate.Length == 6)
{
//RELOAD of a single day
// advance the seek position to the start of the day we want to overwrite
var bf = new BinaryFormatter();
while (fs.Position != fs.Length)
{
var lastPos = fs.Position;
var testAt = (AllTable)bf.Deserialize(fs);
rowNum++;
if (testAt.Date == reloadDate)
{
fs.Position = lastPos; // back up
rowNum--;
break; // and go on to writing out 104 bands
}
}
if (fs.Position == fs.Length)
{
Helper.LogStatus("Error", $"Could not find reloadDate '{reloadDate}' records in All-Table {allTableFile}");
fs.Dispose();
return;
}
}
else
{
if (topUpOnly)
{
//we're appending...
//no need to write rows 1 and 2 if this is a topup
//get starting RowNum to use from passed in TopupInformation
rowNum = topupInfo.LastRow[shareNum] + 1;
}
else
{
//we're creating from scratch
//do the first 2 rows right away (they are special)
atRec = AllTableFactory.InitialRow(rowNum++, "YYMMDD", "Day", "TimeFrom", "TimeTo");
Helper.SerializeAllTableRecord(fs, atRec);
atRec = AllTableFactory.InitialRow(rowNum++, "", "", "", "");
Helper.SerializeAllTableRecord(fs, atRec);
}
}
//now consider every day in the trading span, but if topUpOnly, skip over those we already have.
int tradingDays = 0; double yesterPrice = 0; double lastPrice = 0;
while (!ct.IsCancellationRequested && tradingDays < tradingSpan)
{
if (Helper.IsTradingDay(runDate))
{
//var rowDate = runDate.ToShortDateString().Replace("/", "").Substring(2); //YYMMDD in en-ZA culture
var rowDate = runDate.ToString("yyMMdd"); // culture independent
var topupInfoKey = $"{rowDate},{shareNum}";
if (!topUpOnly || !topupInfo.DatesData[topupInfoKey].AlreadyHave)
{
//each day has 104 five-minute bands from 09h00 to 17h40 - save each one as an 'at' record to disk
var rowDay = runDate.DayOfWeek.ToString().Substring(0, 3);
lastPrice = yesterPrice;
for (int timeBand = 0; timeBand < 104; timeBand++)
{
int minsIntoDay = 9 * 60 + 5 * timeBand;
int hr = minsIntoDay / 60;
int min = minsIntoDay % (60 * hr);
string timeFrom = hr.ToString("00") + ":" + min.ToString("00") + ":00";
string timeTo = hr.ToString("00") + ":" + (min + 4).ToString("00") + ":59";
atRec = AllTableFactory.InitialRow(rowNum++, rowDate, rowDay, timeFrom, timeTo);
atRec.FP = lastPrice;
// now fill from passed in tradeHash
lastPrice = FillAllTableRowFromTradehash(shareNum, atRec, timeBand + 1, tradeHash, lastPrice);
//save AllTable row record to disk - this will be overwriting of reloadDate has been set
Helper.SerializeAllTableRecord(fs, atRec);
}
yesterPrice = lastPrice;
}
tradingDays++;
}
runDate = runDate.AddDays(1);
}
}
if (ct.IsCancellationRequested)
{
//delete the All-Table file just saved
try
{
File.Delete(allTableFile);
}
catch (Exception ex)
{
}
}
}
// Sweep thru the the shares either deleting early part of existing All-Tables or
// the entire file (depending on passed in topUp parameter)
// If topUpOnly is true, data in the All-Tables is effectively 'shuffled up'.
// If topUpOnly is false, each All-Table is deleted.
// This will leave an All-Table to which data must be appended (or else which must be written anew,
// which is akin to appending to a zero size file)
// Also, once complete, there will in general be a tail bit of the toupInfo object for whose dates have
// values of 'alreadyHave' remaining false.
// These will be the dates for which new All-Table records will need to be appended
private static void PrepareAllTables(ref TopupInformation topupInfo, DateTime startDate,
int tradingSpan, string[] allShares, bool topUpOnly, Action <string> progress)
{
var atPath = Helper.UserSettings().AllTablesFolder;
var msg = $"Preparing all *.at files";
progress(msg);
Helper.Log("Info", msg);
//var shares = allShares;
foreach (string share in allShares)
{
Match m = Regex.Match(share, @"(.+)\s(\d+)$");
if (m.Success)
{
var shareName = m.Groups[1].Value.TrimEnd();
var shareNum = Convert.ToInt32(m.Groups[2].Value);
var allTableFile = atPath + @"\" + $"alltable_{shareNum}.at";
var tmpFile = allTableFile + ".tmp";
if (File.Exists(allTableFile))
{
if (topUpOnly)
{
//prepare a 'new' all-table file by skipping over data in the existing all-table which has fallen
//out of range, retaining the run of records to the end (for subsequent appending to with new data)
//essentially chopping off the first bit.
using (FileStream fs1 = new FileStream(allTableFile, FileMode.Open))
{
//read entire existing alltable file into memory (can be 10400 records!)
var oldRows = Helper.DeserializeAllTable <AllTable>(fs1).Skip(2).ToList();
//skip to first record in oldRows holding the first wanted date, then start writing to a NEW version
using (FileStream fs2 = new FileStream(tmpFile, FileMode.Create, FileAccess.Write, FileShare.None, 131072)) // 128K
{
//do the first 2 rows right away (they are special)
Helper.SerializeAllTableRecord(fs2, AllTableFactory.InitialRow(0, "YYMMDD", "Day", "TimeFrom", "TimeTo"));
Helper.SerializeAllTableRecord(fs2, AllTableFactory.InitialRow(1, "", "", "", ""));
int rowNum = 2;
foreach (AllTable at in oldRows)
{
// must this old row be kept?
var topupInfoKey = $"{at.Date},{shareNum}";
// key may not be found if previously the date was processed and subsequently was
// classified a public holiday. (it would have had no trading, all bands empty anyway)
if (topupInfo.DatesData.ContainsKey(topupInfoKey) && topupInfo.DatesData[topupInfoKey].Wanted)
{
//yes, so append it to tmp file
at.Row = rowNum;
at.F = rowNum - 1;
Helper.SerializeAllTableRecord(fs2, at);
//note that we now have data for the date (will be repeatedly done for 104 such bands)
topupInfo.DatesData[topupInfoKey].AlreadyHave = true;
topupInfo.LastRow[shareNum] = at.Row;
rowNum++;
}
}
}
}
//delete old alltable and replace with smaller new one, to which new data will be appended
if (File.Exists(tmpFile))
{
File.Delete(allTableFile);
File.Move(tmpFile, allTableFile);
progress($"AllTable prepared, (share {shareNum})");
}
}
else
{
File.Delete(allTableFile);
progress($"{allTableFile} deleted...");
}
}
var auditFile = atPath + @"\Audit\" + $"{shareNum.ToString("000")}.txt";
if (File.Exists(auditFile))
{
if (topUpOnly)
{
//TODO: decide if Audit file should be peserved and allowed to grow indefinitely
File.Delete(auditFile);
}
else
{
File.Delete(auditFile);
}
}
}
}
Helper.Log("Info", $"{allShares.Count()} '.at' files participating");
}