// should only be called within write lock
private void populateConstructWithDayAtIndex(Day day_, RollGeneration gen_)
{
if (day_ == null || day_.Lines == null) return;
#if DEBUG
var sw = new Stopwatch();
sw.Start();
#endif
foreach (var line in day_.Lines)
{
var relativeDate = new RelativeDate(day_.Dates, line.GmtDateAsDate().ToTZfromGMT(day_.DispTimeZone));
if (relativeDate.Index == -1) // this shouldn't happen
continue;
m_data.SetValue(relativeDate, (int)gen_, line);
}
#if DEBUG
sw.Stop();
Logger.Info(string.Format("Backfilling {0} took {1}ms (Lines={2})", day_, sw.ElapsedMilliseconds, day_.Lines.Count),
typeof (LiveDayComparison));
#endif
}
public LiveDayComparison(Day currentRoll_, Day priorRoll_)
{
m_priorRoll = priorRoll_;
m_currentRoll = currentRoll_;
m_data = new ConstructGenGen<RelativeDate, Line>(new[] {"Current", "PriorRoll"});
}
public static async Task PopulateHistoricMinutes(Day rollDay_, CarbonClient cc_)
{
var dictionary = new SortedDictionary<RelativeDate, Line>();
//FileHelper.GetFront(rollDay_, dictionary);
//FileHelper.GetBack(rollDay_, dictionary);
//FileHelper.GetSpread(rollDay_, dictionary);
await CarbonHelper.Get(rollDay_, dictionary, cc_);
rollDay_.Lines = dictionary.Keys.Select(x => dictionary[x]).ToList();
// quick check to see if the sorting on rds is working as expected
{
RelativeDate prior=default(RelativeDate);
int count =0;
foreach (var rd in dictionary.Keys)
{
try
{
if (count==0) continue;
if (prior.UnderlyingDate > rd.UnderlyingDate)
{
System.Diagnostics.Debugger.Break();
var a = new RelativeDate(prior.IndexDates, prior.UnderlyingDate, prior.DispTimeZone);
var b = new RelativeDate(rd.IndexDates, rd.UnderlyingDate, rd.DispTimeZone);
}
}
finally
{
prior = rd;
++count;
}
}
}
// post process
for (int i = 0; i < rollDay_.Lines.Count; ++i)
{
if (i > 0)
{
if (rollDay_.Lines[i].BackContractValue.IsZero())
rollDay_.Lines[i].BackContractValue = rollDay_.Lines[i - 1].BackContractValue;
if (rollDay_.Lines[i].FrontContractValue.IsZero())
rollDay_.Lines[i].FrontContractValue = rollDay_.Lines[i - 1].FrontContractValue;
if (rollDay_.Lines[i].SpreadValue.IsZero())
rollDay_.Lines[i].SpreadValue = rollDay_.Lines[i - 1].SpreadValue;
}
rollDay_.Lines[i].CalcSeriesValue(rollDay_.DefaultTail);
}
}
private static IEnumerable<DateTime> getFileDates(Day rollDay_)
{
var fileDates = new[]
{
rollDay_.RollDate.AddMonths(-1),
rollDay_.RollDate,
};
if (rollDay_.LastTrade > rollDay_.RollDate)
fileDates = fileDates.AppendWith(rollDay_.LastTrade).ToArray();
return fileDates;
}
public static bool GetFront(Day day_, IDictionary<RelativeDate, Line> values_)
{
var filenames =
getFileDates(day_)
.Select(
x =>
string.Format("*{0}-{1}-{2}.csv{3}", x.Year, x.Month.ToString("00"), day_.FrontContractRIC(),
CanGoToMonths(day_) ? ".gz" : string.Empty)).ToArray();
var dirs = GetDirs(day_);
return LoadFromFile(dirs, filenames, DayField.FrontValue, DayField.FrontVolume, values_, day_);
}
public static async Task Get(Day day_, IDictionary<RelativeDate, Line> values_, CarbonClient cc_)
{
// front
{
await getFromCarbon(
possibleIdentifiers_:
new[] {day_.FrontContractRIC(), string.Format("{0}^{1}", day_.FrontContractRIC(), day_.LastTrade.YearTen())},
values_: values_,
dateIndexes_: day_.IndexedDates,
valueField_: DayField.FrontValue,
volumeField_: DayField.FrontVolume,
zone_:day_.DispTimeZone,
cc_: cc_
);
}
// back
{
await getFromCarbon(
possibleIdentifiers_:
new[] {day_.BackContractRIC(), string.Format("{0}^{1}", day_.BackContractRIC(), day_.LastTrade.YearTen())},
values_: values_,
dateIndexes_: day_.IndexedDates,
valueField_: DayField.BackValue,
volumeField_: DayField.BackVolume,
zone_:day_.DispTimeZone,
cc_: cc_
);
}
// spread
{
await getFromCarbon(
possibleIdentifiers_:
new[]
{
day_.SpreadContractRIC(
AttributeHelper.GetSingleAttribute<RICContractAttribute>(day_.Set).SpreadTickerFormation)
},
values_: values_,
dateIndexes_:day_.IndexedDates,
valueField_: DayField.SpreadValue,
volumeField_: DayField.SpreadVolume,
zone_:day_.DispTimeZone,
cc_: cc_
);
}
}
public static async Task Populate(Day rollDay_, CarbonClient cc_)
{
PopulateDates(rollDay_);
if (rollDay_.IndexedDates != null && rollDay_.IndexedDates.Select(x=>x.Date).Min() > DateTime.Today)
return;
Logger.Info(
string.Format("First IndexDate for [{0}] is [{1}]", rollDay_,
rollDay_.IndexedDates.Select(x => x.Date).Min().ToString("dd-MMM-yyyy")), typeof (DayHelper));
await PopulateHistoricMinutes(rollDay_,cc_);
BackfillEveryMinute(rollDay_);
PopulateOI(rollDay_);
}
private static string[] GetDirs(Day day_)
{
var list = new List<string>();
//if (CanGoToMonths(day_))
{
var dt = day_.LastTrade;
for (int i = 0; i < 3; ++i, dt = dt.AddMonths(-1))
{
string dir = string.Format(@"{0}\{1}\{2}\{3}", MONTHLY_ROOT, dt.Year, dt.Month.ToString("00"), day_.RICBase.ToUpper().Substring(0, 1));
if (Directory.Exists(dir)) list.Add(dir);
dir = string.Format(@"{0}\{1}\{2}\{3}", MONTHLY_ROOT, dt.Year, dt.Month.ToString("00"), "1");
if (Directory.Exists(dir)) list.Add(dir);
}
}
//else
{
var endDate = day_.LastTrade.AddDays(3d);
for (int i = 0; i < 45; ++i)
{
try
{
var dir = string.Format(@"{0}\{1}", DAILY_ROOT, endDate.ToString("yyyyMMdd"));
if (Directory.Exists(dir) == false) continue;
var p = string.Format(@"{0}\{1}", dir, "processed");
if (Directory.Exists(p))
list.Add(p);
var nm = string.Format(@"{0}\{1}", dir, "nometa");
if (Directory.Exists(nm))
list.Add(nm);
}
finally
{
endDate = endDate.AddDays(-1d);
}
}
}
return list.ToArray();
}
public static void PopulateOI(Day rollDay_)
{
var bbgCode_from = string.Format("{0}{1} {2}",
rollDay_.BbgBase,
rollDay_.ContractFront,
rollDay_.BbgSuffix);
var bbgCode_to = string.Format("{0}{1} {2}",
rollDay_.BbgBase,
rollDay_.ContractBack,
rollDay_.BbgSuffix);
rollDay_.OI_Front = BbgTalk.HistoryRequester.GetHistory(rollDay_.RollDate.AddMonths(-2), bbgCode_from, "OPEN_INT", BbgTalk.RefreshOption.AlwaysRefresh);
rollDay_.OI_Back = BbgTalk.HistoryRequester.GetHistory(rollDay_.RollDate.AddMonths(-2), bbgCode_to, "OPEN_INT", BbgTalk.RefreshOption.AlwaysRefresh);
var att = AttributeHelper.GetSingleAttribute<OIOffsetAttribute>(rollDay_.Set);
if (att == null) return;
rollDay_.OI_Front = rollDay_.OI_Front.DoOffset(att.Offset);
rollDay_.OI_Back = rollDay_.OI_Back.DoOffset(att.Offset);
}
private static bool CanGoToMonths(Day day_)
{
return day_.LastTrade < DateTime.Today && day_.LastTrade.IsSameMonthAs(DateTime.Today) == false;
}
private static bool LoadFromFile(IEnumerable<string> dirs_, IEnumerable<string> filenames_, DayField valueField_, DayField volumeField_, IDictionary<RelativeDate, Line> values_, Day day_)
{
bool foundAFile = false;
foreach (var dir in dirs_)
{
foreach (var path in filenames_)
{
var files = Directory.GetFiles(dir, path);
if (files.Length == 0)
{
continue;
}
string file = files[0];
Logger.Info(string.Format("Processing file [{0}]", file), typeof(FileHelper));
if (file.EndsWith(".gz"))
file = getUnzippedPath(file);
try
{
foundAFile = true;
foreach (var p in CsvFile.Read<IntradayFileLineItem>(file))
{
if (!p.GmtDate.HasValue)
continue;
var rd = new RelativeDate(day_.IndexedDates, p.GmtDate.Value.ToTZfromGMT(day_.DispTimeZone), day_.DispTimeZone);
if (rd.Index == RelativeDate.INVALID_INDEX) continue;
if (!values_.ContainsKey(rd))
values_[rd] = new Line { GMTDateTime = p.GmtDate.Value.ToString(Line.MongoDateFormat) };
values_[rd].SetValue(valueField_, p.CloseMid);
values_[rd].SetValue(volumeField_, p.Volume);
}
}
catch (Exception ex_)
{
Logger.Error("Error", typeof (FileHelper), ex_);
}
}
}
return foundAFile;
}
public static bool GetSpread(Day day_, IDictionary<RelativeDate, Line> values_)
{
var ricFormation = AttributeHelper.GetSingleAttribute<RICContractAttribute>(day_.Set).SpreadTickerFormation;
var filenames = getFileDates(day_).Select(x =>
{
var fileName = string.Format("*{0}-{1}-{2}.csv{3}",
x.Year,
x.Month.ToString("00"),
day_.SpreadContractRIC(ricFormation),
CanGoToMonths(day_) ? ".gz" : string.Empty);
return fileName;
}).ToArray();
var dirs = GetDirs(day_);
return LoadFromFile(dirs, filenames, DayField.SpreadValue, DayField.SpreadVolume, values_, day_);
}
private static void BackfillEveryMinute(Day rollDay_)
{
if (rollDay_.Lines == null || rollDay_.Lines.Count < 2)
return;
var toAdd = new List<Line>();
var dates = rollDay_.Lines.Select(x => x.GmtDateAsDate()).OrderBy(x=>x).ToArray();
var date = dates[0];
var lastDate = dates[dates.Length - 1];
int index = 0;
while (date <= lastDate)
{
date = date.AddMinutes(1d);
var rd = new RelativeDate(rollDay_.IndexedDates, date.ToTZfromGMT(rollDay_.DispTimeZone), rollDay_.DispTimeZone);
if (rd.Index == RelativeDate.INVALID_INDEX)
continue;
while (index < rollDay_.Lines.Count-1 && rollDay_.Lines[index].GmtDateAsDate() < date)
++index;
// if we already have a line for this minute?
if (rollDay_.Lines[index].GmtDateAsDate() == date)
continue;
if (rollDay_.Lines[index].GmtDateAsDate() > date)
{
--index;
var copyThis = rollDay_.Lines[index];
toAdd.Add(new Line
{
SpreadValue=copyThis.SpreadValue,
BackContractValue = copyThis.BackContractValue,
FrontContractValue=copyThis.FrontContractValue,
GMTDateTime=date.ToString(Line.MongoDateFormat)
});
}
}
if (toAdd.Count > 0)
{
rollDay_.Lines.AddRange(toAdd);
rollDay_.Lines.Sort((a, b) => a.GmtDateAsDate().CompareTo(b.GmtDateAsDate()));
}
}
public static void PopulateDates(Day rollDay_)
{
var h = AttributeHelper.GetSingleAttribute<HolidayCalendarAttribute>(rollDay_.Set);
rollDay_.DispTimeZone = h.DisplayTimezone;
// need to build up a list of dates and their index relative to the rolldate
var listDateIndex = new List<Tuple<DateTime, int>>();
// start off by going backwards from the rolldate for 21 days
var date = rollDay_.RollDate;
for (int i = 0; i < 21; ++i)
{
listDateIndex.Add(new Tuple<DateTime, int>(date, -i));
date = MyCalendar.PrevTradeDate(date, h.Code);
}
// for some contractSets, our anchor point for rolling is firstNotice, but then we still want to monitor up to last trade on a 't+' notation
date = rollDay_.RollDate;
int plusIndex = 0;
while (date < rollDay_.LastTrade)
{
date = MyCalendar.NextTradeDate(date, h.Code);
++plusIndex;
if (date <= rollDay_.LastTrade)
{
listDateIndex.Add(new Tuple<DateTime, int>(date, plusIndex));
}
}
rollDay_.IndexedDates = listDateIndex.OrderByDescending(x => x.Item2).Select(x => new RDDateIndex(x.Item1, x.Item2)).ToArray();
}
private void subscribeToBbgLive(Day day_)
{
// need to subscribe to 3 things, the priorContract, the newContract, and the spread
// 1) new contract
m_bbgLive_NewContract =
BbgTalk.Core.Instance.GetLiveSecurityBarData(
ticker_: day_.CurrentContractBbgTicker(), barWidthInMinutes_: 1);
subscribeToNewContract();
// 2) old contract
m_bbgLive_PriorContract =
BbgTalk.Core.Instance.GetLiveSecurityBarData(
ticker_: day_.OldContractBbgTicker(), barWidthInMinutes_: 1);
subscribeToOldContract();
// 3) spread
m_bbgLive_Spread =
BbgTalk.Core.Instance.GetLiveSecurityBarData(
ticker_: day_.SpreadContractBbgTicker(), barWidthInMinutes_: 1);
subscribeToSpreadContract();
}
private void backFillFromBloombergToNow(Day day_)
{
{
var histCurrent = BbgTalk.HistoryRequester.GetIntradayBars(DateTime.Today.ToTZfromLocal(day_.DispTimeZone), m_currentRoll.CurrentContractBbgTicker(),
"LAST_PRICE", 1);
backFill(histCurrent, DayField.ToValue, DayField.ToVolume);
}
{
var histOld = BbgTalk.HistoryRequester.GetIntradayBars(DateTime.Today.ToTZfromLocal(day_.DispTimeZone), m_currentRoll.OldContractBbgTicker(),
"LAST_PRICE", 1);
backFill(histOld, DayField.PriorValue, DayField.PriorVolume);
}
{
var spread = BbgTalk.HistoryRequester.GetIntradayBars(DateTime.Today.ToTZfromLocal(day_.DispTimeZone), m_currentRoll.SpreadContractBbgTicker(),
"LAST_PRICE", 1);
backFill(spread, DayField.SpreadValue, DayField.SpreadVolume);
}
}
