/// <summary>
/// Convert contract expiry string to DateTime
/// </summary>
/// <param name="contract"></param>
/// <returns></returns>
internal static DateTime GetContractExpiryDateTime(Contract contract)
{
if (contract == null || string.IsNullOrEmpty(contract.LastTradeDateOrContractMonth))
{
return(DateTime.MaxValue);
}
try
{
if (contract.LastTradeDateOrContractMonth.Length == 8)
{
return(DateTime.ParseExact(contract.LastTradeDateOrContractMonth, "yyyyMMdd", CultureInfo.InvariantCulture));
}
if (contract.LastTradeDateOrContractMonth.Length == 6)
{
// format: "yyyyMM"
int year = int.Parse(contract.LastTradeDateOrContractMonth.Substring(0, 4));
int month = int.Parse(contract.LastTradeDateOrContractMonth.Substring(4, 2));
DateTime lastTradeDay = new DateTime(year, month, 1);
lastTradeDay = lastTradeDay.AddMonths(1);
lastTradeDay = lastTradeDay.AddDays(-1); // last day of the month
return(lastTradeDay);
}
LogAndMessage.LogAndQueue(MessageType.Error, "Program error. Unknown format of last trade data/contract month:" + contract.LastTradeDateOrContractMonth);
}
catch
{
LogAndMessage.LogAndQueue(MessageType.Error, "Program error. Cannot interpret format of last trade data/contract month:" + contract.LastTradeDateOrContractMonth);
}
return(DateTime.MinValue);
}
/// <summary>
/// This method creates a new or gets an existing TickerData object for a ticker
/// </summary>
/// <param name="ticker"></param>
/// <returns></returns>
/// <remarks>It looks for registered ticker object first and returns it. If ticker is not registered yet, it registers it and returns the new object. </remarks>
internal TickerData RegisterTickerData(string ticker)
{
lock (mapTickerToTickerData)
{
TickerData tickerData;
if (mapTickerToTickerData.TryGetValue(ticker, out tickerData))
{
return(tickerData);
}
tickerData = new TickerData();
tickerData.Ticker = ticker;
mapTickerToTickerData.Add(ticker, tickerData);
SymbolParts ftTicker;
try
{
ftTicker = new SymbolParts(ticker);
}
catch (Exception)
{
ftTicker = null;
tickerData.QuoteDataStatus = QuotationStatus.Failed;
LogAndMessage.LogAndQueue(tickerData, MessageType.Warning, "Invalid symbol.");
}
tickerData.SymbolParts = ftTicker;
return(tickerData);
}
}
private void SendBackfillRequest(FTController ibController)
{
if (downloadContract != null
// && (string.IsNullOrEmpty(contract.LastTradeDateOrContractMonth) || histStart.ToString("yyyyMMdd").CompareTo(contract.LastTradeDateOrContractMonth) <= 0)
&& (downloadStep > 24 * 60 || IBClientHelper.IsBankingHour(downloadStart) || IBClientHelper.IsBankingHour(downloadEnd))) //check if we need to queue a requests
{
//try to avoid intraday request for saturday-sunday
Id = IBClientHelper.GetNextReqId();
// add quotelist of subcontracts of continuous contract
if (TickerData.SymbolParts.IsContinuous)
{
if (!TickerData.ContinuousQuotesDictionary.ContainsKey(downloadContract.LocalSymbol))
{
TickerData.ContinuousQuotesDictionary.Add(downloadContract.LocalSymbol, new QuotationList(FTDataSource.Periodicity));
}
}
if (downloadPeriodicity <= Periodicity.FifteenSeconds) // download step is smaller than a day
{
LogAndMessage.LogAndQueue(MessageType.Info, TickerData.ToString(downloadContract) + ": Requesting data from " + downloadStart.ToShortDateString() + " " + downloadStart.ToShortTimeString() + " to " + downloadEnd.ToShortDateString() + " " + downloadEnd.ToShortTimeString() + " " + ToString(false, LogAndMessage.VerboseLog));
}
else
{
LogAndMessage.LogAndQueue(MessageType.Info, TickerData.ToString(downloadContract) + ": Requesting data from " + downloadStart.ToShortDateString() + " to " + downloadEnd.ToShortDateString() + " " + ToString(false, LogAndMessage.VerboseLog));
}
WaitingForResponse = true;
RequestTime = DateTime.Now;
ibController.SendHistoricalDataRequest(Id, downloadContract, downloadEnd, downloadStart, downloadPeriodicity, TickerData.SymbolParts.DataType);
}
else
{
// calc next download period
if (CalcNextBackfillRequest())
{
SendBackfillRequest(ibController);
}
}
}
internal override bool Process(FTController ibController, bool allowNewRequest)
{
const int requestTimeoutPeriod = 20;
// if no contract received yet
if (TickerData.ContractStatus == ContractStatus.SendRequest || TickerData.ContractStatus == ContractStatus.WaitForResponse)
{
return(allowNewRequest);
}
// if no contract found
if (TickerData.ContractStatus != ContractStatus.Ok)
{
TickerData.HeadTimestampStatus = HeadTimestampStatus.Failed;
IsFinished = true;
return(allowNewRequest);
}
lock (TickerData) // request handling
{
// if not waiting for response
switch (TickerData.HeadTimestampStatus)
{
// if marked to get headtimestamp
case HeadTimestampStatus.SendRequest:
if (allowNewRequest)
{
LogAndMessage.Log(TickerData, MessageType.Trace, "Requesting earliest data point. " + ToString(false, LogAndMessage.VerboseLog));
TickerData.HeadTimestampStatus = HeadTimestampStatus.WaitForResponse;
RequestTime = DateTime.Now;
ibController.SendHeadTimestampRequest(Id, TickerData);
}
return(false);
// if request is sent, but response has not arrived yet
// see ibClient_HeadTimestamp event handler
case HeadTimestampStatus.WaitForResponse:
// if no answer in time
if (RequestTime.AddSeconds(requestTimeoutPeriod) < DateTime.Now)
{
if (RequestTimeouts == 0)
{
LogAndMessage.LogAndQueue(TickerData, MessageType.Error, "Request of earliest data point timed out. Retrying. " + ToString(true, LogAndMessage.VerboseLog));
RequestTimeouts++;
TickerData.HeadTimestampStatus = HeadTimestampStatus.SendRequest;
Id = IBClientHelper.GetNextReqId();
goto case HeadTimestampStatus.SendRequest;
}
LogAndMessage.LogAndQueue(TickerData, MessageType.Error, "Request of earliest data point timed out. " + ToString(true, LogAndMessage.VerboseLog));
TickerData.HeadTimestampStatus = HeadTimestampStatus.Failed;
goto case HeadTimestampStatus.Failed;
}
return(allowNewRequest);
// if new, offline ticker
case HeadTimestampStatus.Offline:
// ticker's HeadTimestamp is updated
case HeadTimestampStatus.Ok:
// ticker's HeadTimestamp is NOT updated (we do not mark ticker as failed. it still may work!)
case HeadTimestampStatus.Failed:
IsFinished = true;
return(allowNewRequest);
// this is program error
default:
TickerData.HeadTimestampStatus = HeadTimestampStatus.Failed;
IsFinished = true;
return(allowNewRequest);
}
}
}
internal override bool Process(FTController ibController, bool allowNewRequest)
{
const int requestTimeoutPeriod = 10;
lock (TickerData) // request handling
{
// if not waiting for response
switch (TickerData.ContractStatus)
{
// if marked to get contract details
case ContractStatus.SendRequest:
if (allowNewRequest)
{
LogAndMessage.Log(TickerData, MessageType.Trace, "Getting contract. " + ToString(false, LogAndMessage.VerboseLog));
Contract contract = new Contract();
contract.Exchange = TickerData.SymbolParts.Exchange;
contract.SecType = TickerData.SymbolParts.SecurityType;
contract.Currency = TickerData.SymbolParts.Currency;
contract.IncludeExpired = true;
if (TickerData.SymbolParts.IsContinuous)
{
if (!string.IsNullOrEmpty(TickerData.SymbolParts.Underlying))
{
contract.Symbol = TickerData.SymbolParts.Underlying;
}
else
{
contract.Symbol = TickerData.SymbolParts.Symbol;
}
}
else
{
contract.LocalSymbol = TickerData.SymbolParts.Symbol;
}
TickerData.contractDetailsList.Clear();
TickerData.ContractStatus = ContractStatus.WaitForResponse;
RequestTime = DateTime.Now;
ibController.SendContractDetailsRequest(Id, contract);
}
return(false);
// if request is sent, but response has not arrived yet
// see ibclient_ContractDetails and ibclient_ContractDetailsEnd event handlers
case ContractStatus.WaitForResponse:
// if no answer in Time
if (RequestTime.AddSeconds(requestTimeoutPeriod) < DateTime.Now)
{
LogAndMessage.LogAndQueue(TickerData, MessageType.Error, "Getting contract info timed out, symbol is offline. " + ToString(true, LogAndMessage.VerboseLog));
TickerData.ContractStatus = ContractStatus.Failed;
goto case ContractStatus.Failed;
}
return(allowNewRequest);
// if new, offline ticker
case ContractStatus.Offline:
goto case ContractStatus.Failed;
// contract found
case ContractStatus.Ok:
goto case ContractStatus.Failed;
// no contract found
case ContractStatus.Failed:
IsFinished = true;
return(allowNewRequest);
default:
throw new ArgumentOutOfRangeException();
}
}
}
internal override bool Process(FTController ibController, bool allowNewRequest)
{
// if no contract received yet
if (TickerData.ContractStatus == ContractStatus.SendRequest || TickerData.ContractStatus == ContractStatus.WaitForResponse)
{
return(allowNewRequest);
}
// if GetQuotesEx was not yet called (stockInfo is not ready yet), symbol data cannot be updated
if (TickerData.StockInfo == null)
{
IsFinished = true;
return(allowNewRequest);
}
// if not waiting for response
switch (TickerData.SymbolStatus)
{
// if marked to update AB symbol's data
case SymbolStatus.WaitForContractUpdate:
// if no contract found
if (TickerData.ContractStatus != ContractStatus.Ok)
{
LogAndMessage.LogAndQueue(TickerData, MessageType.Error, "Getting contract info failed, AmiBroker symbol data cannot be updated.");
TickerData.SymbolStatus = SymbolStatus.Failed;
goto case SymbolStatus.Failed;
}
// plugin may call it when StockInfo is not available (E.g. start watchlist backfill)
if (TickerData.StockInfo == null)
{
LogAndMessage.Log(TickerData, MessageType.Trace, "StockInfo data is not available. AmiBroker symbol data is not updated.");
return(false);
}
try
{
// update AB's information
TickerData.StockInfo.AliasName = TickerData.ContractDetails.Contract.LocalSymbol + '/' + TickerData.ContractDetails.Contract.Symbol;
TickerData.StockInfo.FullName = TickerData.ContractDetails.LongName;
TickerData.StockInfo.PointValue = TickerData.ContractDetails.PriceMagnifier;
TickerData.StockInfo.TickSize = (float)TickerData.ContractDetails.MinTick;
TickerData.StockInfo.WebId = TickerData.ContractDetails.Contract.ConId.ToString();
TickerData.StockInfo.Currency = TickerData.ContractDetails.Contract.Currency;
TickerData.SymbolStatus = SymbolStatus.Ok;
LogAndMessage.LogAndQueue(TickerData, MessageType.Info, "AmiBroker symbol data is updated.");
}
catch (Exception ex)
{
TickerData.SymbolStatus = SymbolStatus.Failed;
LogAndMessage.LogAndQueue(TickerData, MessageType.Error, "AmiBroker symbol data update failed:" + ex);
return(false);
}
goto case SymbolStatus.Ok;
// if new ticker
case SymbolStatus.Offline:
// AB symbol's data are updated
case SymbolStatus.Ok:
// AB symbol's data NOT updated, e.g: no contract found
case SymbolStatus.Failed:
IsFinished = true;
return(allowNewRequest);
default:
TickerData.SymbolStatus = SymbolStatus.Failed;
IsFinished = true;
return(allowNewRequest);
}
}
/// <summary>
/// Timer event handler
/// </summary>
/// <param name="sender"></param>
/// <remarks>
/// If connection is broken, it tries to reconnect in every 30 secs.
/// If it reconnects, it starts a backfill of all tickers
/// If status is ready and autorefresh is configured, it starts the scheduled refresh
/// </remarks>
private void timer_Tick(object sender)
{
if (controller == null)
{
return;
}
// check and indicate thread entry
if (Interlocked.CompareExchange(ref inTimerTick, 1, 0) != 0)
{
return;
}
IBPluginState currPluginState = IBPluginState.Disconnected;
try
{
currPluginState = controller.GetIBPluginState();
if (currPluginState != prevPluginState)
{
LogAndMessage.Log(MessageType.Info, "Plugin status: " + currPluginState);
}
// if no connection between the data plugin and the TWS client
if (currPluginState == IBPluginState.Disconnected)
{
// if not manually disconnected, try to reconnect
if (manuallyDisconnected == false)
{
// if retry period has elapsed
if (connectionRetryTime < DateTime.Now)
{
// if connection has just got broken (prevent repeated log entries)
if (prevPluginState != IBPluginState.Disconnected)
{
LogAndMessage.LogAndQueue(MessageType.Warning, "Data plugin has been disconnected from TWS. Trying to reconnect in every " + ConnectionRetryInterval + " sec.");
}
// set new retry time and increase interval up to 30 secs
connectionRetryTime = DateTime.Now.AddSeconds(ConnectionRetryInterval);
// try to reconnect
controller.Connect(true);
}
}
}
// data plugin is connected to TWS
else
{
// reset connection retrying
connectionRetryTime = DateTime.MinValue;
// if an existing connection was disconnected
if (prevPluginState == IBPluginState.Disconnected && !firstConnection || controller.RestartStreaming)
{
LogAndMessage.LogAndQueue(MessageType.Warning, "TWS has been reconnected. Starting database refresh.");
// request refresh of all tickers, restart subscriptions, etc.
controller.RestartAfterReconnect(lastUptodateTime);
return;
}
// clear flag of first connection
firstConnection = false;
// check if it's time to run AutoRefresh
if (prevPluginState == IBPluginState.Ready) // Plugin works with up to date data
{
lastUptodateTime = DateTime.Now;
if (IBConfiguration != null && IBConfiguration.AutoRefreshEnabled && // Auto refresh is enabled
nextAutoRefreshTime < DateTime.Now) // The time of auto refresh has passed
{
DateTime refreshStartDate = DateTime.Now.Date.AddDays(-IBConfiguration.AutoRefreshDays);
LogAndMessage.LogAndQueue(MessageType.Warning, "Starting automatic database refresh (" + refreshStartDate.ToShortDateString() + ").");
controller.RefreshAllUsed(refreshStartDate);
CalcNextAutoRefreshTime();
}
}
}
}
catch (Exception ex)
{
LogAndMessage.Log(MessageType.Error, "Error in data source timer event handler: " + ex);
}
finally
{
// update plugin state
prevPluginState = currPluginState;
// indicate thread exit
Interlocked.Exchange(ref inTimerTick, 0);
}
}
private void mBackfillAll_Click(object sender, EventArgs e)
{
if (firstGetQuotesExCall) // db periodicity not known yet
{
return;
}
if (string.IsNullOrEmpty(currentTicker)) // no selected ticker
{
return;
}
DateTime refreshStartDate = GetRefreshStartDate(sender);
refreshStartDate = IBClientHelper.GetAdjustedStartDate(refreshStartDate, Periodicity, DateTime.MinValue, false);
//
// collecting all tickers
//
int stockCount = 0;
StringCollection tickersInDatabase = new StringCollection();
LogAndMessage.Log(MessageType.Info, "Manual backfill of all tickers from " + refreshStartDate.ToShortDateString() + ".");
try
{
Type abAppType = Type.GetTypeFromProgID("Broker.Application");
object abApp = Activator.CreateInstance(abAppType);
// access AB COM interface to get current ticker
if (abAppType != null && abApp != null)
{
object abStocks = abAppType.InvokeMember("Stocks", BindingFlags.GetProperty, null, abApp, null);
Type abStocksType = abStocks.GetType();
// get the number of stocks in the db
stockCount = (int)abStocksType.InvokeMember("Count", BindingFlags.GetProperty, null, abStocks, null);
if (stockCount > 0)
{
Type abStockType = abStocksType.InvokeMember("Item", BindingFlags.GetProperty, null, abStocks, new object[] { 0 }).GetType();
for (int i = 0; i < stockCount; i++)
{
object abStock = abStocksType.InvokeMember("Item", BindingFlags.GetProperty, null, abStocks, new object[] { i });
if (abStock != null)
{
string ticker = (string)abStockType.InvokeMember("Ticker", BindingFlags.GetProperty, null, abStock, null);
if (!tickersInDatabase.Contains(ticker))
{
tickersInDatabase.Add(ticker);
}
}
}
}
else
{
LogAndMessage.Log(MessageType.Trace, "Manual backfill of all symbols failed. Database has no symbols.");
}
}
else
{
LogAndMessage.Log(MessageType.Warning, "Manual backfill of all symbols failed. ActiveX interface error.");
}
}
catch (Exception ex)
{
LogAndMessage.LogAndQueue(MessageType.Error, "Manual backfill of all symbols failed. Exception: " + ex);
}
StartBackfills(refreshStartDate, tickersInDatabase);
}
private void mBackfillWL_Click(object sender, EventArgs e)
{
if (firstGetQuotesExCall) // db periodicity not known yet
{
return;
}
if (string.IsNullOrEmpty(currentTicker)) // no selected ticker
{
return;
}
DateTime refreshStartDate = GetRefreshStartDate(sender);
refreshStartDate = IBClientHelper.GetAdjustedStartDate(refreshStartDate, Periodicity, DateTime.MinValue, false);
WatchlistForm watchlistForm = new WatchlistForm(DatabasePath);
if (DialogResult.OK != watchlistForm.ShowDialog() || watchlistForm.SelectedWatchlistIndices == null)
{
return;
}
int[] selectedWatchlistIndices = watchlistForm.SelectedWatchlistIndices;
watchlistForm.Dispose();
StringCollection tickersInWatchlists = new StringCollection();
ulong watchlistBits = 0;
foreach (int watchlistId in selectedWatchlistIndices)
{
watchlistBits |= (ulong)1 << watchlistId;
}
LogAndMessage.Log(MessageType.Info, "Manual backfill of watchlists (" + watchlistBits.ToString("0x") + ") from " + refreshStartDate.ToShortDateString() + ".");
try
{
Type abAppType = Type.GetTypeFromProgID("Broker.Application");
object abApp = Activator.CreateInstance(abAppType);
// access AB COM interface to get current ticker
if (abAppType != null && abApp != null)
{
object abStocks = abAppType.InvokeMember("Stocks", BindingFlags.GetProperty, null, abApp, null);
Type abStocksType = abStocks.GetType();
// get the number of stocks in the db
int stockCount = (int)abStocksType.InvokeMember("Count", BindingFlags.GetProperty, null, abStocks, null);
if (stockCount > 0)
{
Type abStockType = abStocksType.InvokeMember("Item", BindingFlags.GetProperty, null, abStocks, new object[] { 0 }).GetType();
for (int i = 0; i < stockCount; i++)
{
object abStock = abStocksType.InvokeMember("Item", BindingFlags.GetProperty, null, abStocks, new object[] { i });
if (abStock != null)
{
string ticker = (string)abStockType.InvokeMember("Ticker", BindingFlags.GetProperty, null, abStock, null);
uint watchlistBits1 = (uint)(int)abStockType.InvokeMember("WatchListBits", BindingFlags.GetProperty, null, abStock, null);
uint watchlistBits2 = (uint)(int)abStockType.InvokeMember("WatchListBits2", BindingFlags.GetProperty, null, abStock, null);
ulong watchlistBitsCombined = (watchlistBits2 << 32) + watchlistBits1;
if ((watchlistBitsCombined & watchlistBits) != 0)
{
if (!tickersInWatchlists.Contains(ticker))
{
tickersInWatchlists.Add(ticker);
}
}
}
}
}
else
{
LogAndMessage.Log(MessageType.Trace, "Manual backfill of watchlists failed. Database has no symbols.");
}
}
else
{
LogAndMessage.Log(MessageType.Warning, "Manual backfill of watchlists failed. ActiveX interface error.");
}
}
catch (Exception ex)
{
LogAndMessage.LogAndQueue(MessageType.Error, "Manual backfill of watchlists failed. Exception: " + ex);
}
if (tickersInWatchlists.Count == 0)
{
LogAndMessage.Log(MessageType.Trace, "Manual backfill of watchlists failed. Selected watchlists have no symbols.");
}
else
{
StartBackfills(refreshStartDate, tickersInWatchlists);
}
}
private void StartBackfills(DateTime refreshStartDate, StringCollection tickersToBackfill)
{
if (tickersToBackfill.Count == 0)
{
return;
}
//
// checking if long download is accepted
//
int stepSize = IBClientHelper.GetDownloadStep(Periodicity);
TimeSpan ts = DateTime.Now.Subtract(refreshStartDate);
int requestsNo = (int)Math.Ceiling(ts.TotalMinutes / stepSize * tickersToBackfill.Count);
// if mixed database, add number of EOD requests
requestsNo = (Periodicity != Periodicity.EndOfDay && Workspace.AllowMixedEODIntra != 0) ? requestsNo + tickersToBackfill.Count : requestsNo;
//
// checking if database can accomodate the data
//
// quoteNum is aproximate value
int quoteNum = (int)(ts.TotalSeconds / (int)Periodicity); // total number of bars if traded 24 hours
if (Periodicity != Periodicity.EndOfDay)
{
if (IBConfiguration.RthOnly)
{
quoteNum /= 3;
}
}
//
// building and showing warning
//
bool tooManyRequest = requestsNo > 3;
bool tooManyQuotes = quoteNum / 1.2 > Workspace.NumBars;
if (tooManyRequest || tooManyQuotes)
{
TimeSpan mts = new TimeSpan(0, 0, requestsNo * 10);
StringBuilder msg = new StringBuilder(500);
msg.Append("The requested data refresh");
if (tooManyRequest)
{
msg.Append(" may download more bar data (~");
msg.Append((quoteNum / 1.2).ToString("N0"));
msg.Append(") than your database can accomodate (");
msg.Append(Workspace.NumBars.ToString());
msg.Append(")");
}
if (tooManyRequest)
{
if (tooManyRequest)
{
msg.Append(" and it");
}
msg.Append(" may start a long data download operation (~");
msg.Append(mts);
msg.Append(")");
}
msg.AppendLine(".");
msg.AppendLine("Do you still want it?");
if (MessageBox.Show(msg.ToString(),
"Warning", MessageBoxButtons.YesNo, MessageBoxIcon.Warning) == DialogResult.No)
{
return;
}
}
//
// start backfills of selected length
//
foreach (var ticker in tickersToBackfill)
{
if (!controller.RefreshTicker(ticker, refreshStartDate))
{
LogAndMessage.LogAndQueue(MessageType.Warning, ticker + ": Cannot start manual backfill because ticker is being backfilled.");
}
}
}
/// <summary>
/// Processing all queues/request
/// </summary>
/// <returns>true, if any request was sent</returns>
private bool ProcessQueues()
{
try
{
#if DEBUG
bool traceMode = backfillQueue.IsBusy || subscriptionQueue.IsBusy ||
symbolQueue.IsBusy || headTimestampQueue.IsBusy || contractQueue.IsBusy;
traceMode = false;
if (traceMode)
{
LogAndMessage.Log(MessageType.Trace, "Start of ProcessAllQueues");
}
#else
bool traceMode = false;
#endif
//
// sending notificaton to AB to get quotes from the plugin (GetQuotesEx will be called)
//
SendQuoteNotificationToAB();
if (!ibController.IsIbConnected())
{
return(true);
}
// remove timed out entries from request throlling queue and check if any request can be sent
bool throttling = ibAllRequestTimeQueue.IsThrottled();
if (throttling)
{
LogAndMessage.LogAndQueue(MessageType.Info, "Throttling all requests.");
}
// remove timed out entries from hist request throlling queue and check if historical request can be sent
bool histThrottling = ibHstRequestTimeQueue.IsThrottled() || ibHstThrottlingEndTime >= DateTime.Now;
//
// main queue processing logic
//
// check if general request can be sent
bool allowNewRequest = !throttling;
if (!histThrottling)
{
allowNewRequest &= backfillQueue.ProcessQueuedRequests(ibController, allowNewRequest, traceMode);
}
allowNewRequest &= subscriptionQueue.ProcessQueuedRequests(ibController, allowNewRequest, traceMode);
allowNewRequest &= headTimestampQueue.ProcessQueuedRequests(ibController, allowNewRequest, traceMode);
allowNewRequest &= contractQueue.ProcessQueuedRequests(ibController, allowNewRequest, traceMode);
allowNewRequest &= symbolQueue.ProcessQueuedRequests(ibController, allowNewRequest, traceMode);
if (allowNewRequest)
{
allowNewRequest &= ibController.SendCurrentTimeRequest();
}
#if DEBUG
if (traceMode)
{
LogAndMessage.Log(MessageType.Trace, "End of ProcessAllQueues");
}
#endif
// other scheduled jobs
ibController.StartContractRefresh();
ibController.UpdateFailedTickers();
return(!allowNewRequest); // indicate newly sent request -> longer wait time
}
catch (Exception ex)
{
LogAndMessage.LogAndQueue(MessageType.Error, "Program error. ProcessQueues exception: " + ex);
return(true);
}
}
internal override bool Process(FTController ibController, bool allowNewRequest)
{
int requestTimeoutPeriod = 75;
// if contract of the ticker is still being retrieved or headtimestamp of the ticker is needed (not Offline) AND not yet retrieved
if (TickerData.ContractStatus <= ContractStatus.WaitForResponse ||
((FTDataSource.Periodicity == Periodicity.EndOfDay || FTDataSource.AllowMixedEODIntra) && TickerData.HeadTimestampStatus <= HeadTimestampStatus.WaitForResponse))
{
return(allowNewRequest);
}
if (TickerData.ContractStatus == ContractStatus.Failed || TickerData.ContractStatus == ContractStatus.Offline ||
TickerData.HeadTimestampStatus == HeadTimestampStatus.Failed || (TickerData.HeadTimestampStatus == HeadTimestampStatus.Offline && (FTDataSource.Periodicity == Periodicity.EndOfDay || FTDataSource.AllowMixedEODIntra)))
{
TickerData.QuoteDataStatus = QuotationStatus.Failed;
IsFinished = true;
return(allowNewRequest);
}
lock (TickerData) // request handling
{
// if reqHistoricalData is send to IB and we are waiting for answer
if (WaitingForResponse)
{
// request is not yet timed out...
if (RequestTime.AddSeconds(requestTimeoutPeriod) > DateTime.Now)
{
return(allowNewRequest);
}
// no response arrived in time, request is timed out...
LogAndMessage.LogAndQueue(TickerData, MessageType.Info, "Historical data request has timed out. " + ToString(true, LogAndMessage.VerboseLog));
RequestTimeouts++;
WaitingForResponse = false;
// if there were too many reqHistoricalData timeouts
if (RequestTimeouts > 2)
{
// drop this ticker...
TickerData.QuoteDataStatus = QuotationStatus.Failed;
IsFinished = true;
return(allowNewRequest);
}
}
// if no new request can be sent (request pacing)
bool histThrottling = !allowNewRequest || TickerData.QuoteDataStatus > QuotationStatus.New && RequestTime.AddSeconds(6.5) > DateTime.Now;
// process the ticker depending on its state
switch (TickerData.QuoteDataStatus)
{
case QuotationStatus.Offline:
LogAndMessage.Log(MessageType.Error, "Program error. Offline ticker cannot get historical update.");
IsFinished = true;
return(allowNewRequest);
// All historical data requests are processed for the ticker
// (the last CalcNextHistoricalDataRequest call sets this state)
case QuotationStatus.DownloadedEod:
#region Merging and backadjusting downloaded quotes of different contracts/expiry into a simgle QuotationList of the continuous contract
if (TickerData.SymbolParts.IsContinuous)
{
QuotationList mergedQuotes = new QuotationList(FTDataSource.Periodicity);
int newQuoteIndex;
foreach (ContractDetails cd in TickerData.contractDetailsList)
{
// if there were no quotes receiced for this contract...
if (!TickerData.ContinuousQuotesDictionary.ContainsKey(cd.Contract.LocalSymbol))
{
continue;
}
newQuoteIndex = 0;
if (mergedQuotes.Count > 0)
{
int mergedQuoteIndex = mergedQuotes.Count - 1;
AmiDate mergedQuoteDateTime = mergedQuotes[mergedQuoteIndex].DateTime;
// move forward to the first quote not overlqapping with prev contract
while (newQuoteIndex < TickerData.ContinuousQuotesDictionary[cd.Contract.LocalSymbol].Count - 1 && TickerData.ContinuousQuotesDictionary[cd.Contract.LocalSymbol][newQuoteIndex].DateTime.Date < mergedQuoteDateTime.Date)
{
newQuoteIndex++;
}
// at this point newQuoteIndex points to a quote of the "same" date as mergedQuoteDateTime (if there are quotes for the same day, if not, then the next day)
// if daily database then we look for a day where volume on older contract is greater (switch over day)
if (FTDataSource.Periodicity == Periodicity.EndOfDay)
{
// find the quote that has a lower volume
while (newQuoteIndex > 0 && mergedQuoteIndex > 0 &&
TickerData.ContinuousQuotesDictionary[cd.Contract.LocalSymbol][newQuoteIndex].DateTime.Date == mergedQuotes[mergedQuoteIndex].DateTime.Date && // quotes are of same date
TickerData.ContinuousQuotesDictionary[cd.Contract.LocalSymbol][newQuoteIndex].Volume > mergedQuotes[mergedQuoteIndex].Volume) // new contract's volume is higher then old contract's volume
{
newQuoteIndex--;
mergedQuoteIndex--;
}
// at this point newQuoteIndex and lastQuoteDateTime point to quote at which contract is replaced
}
if (TickerData.ContinuousQuotesDictionary[cd.Contract.LocalSymbol][newQuoteIndex].DateTime.Date != mergedQuotes[mergedQuoteIndex].DateTime.Date)
{
LogAndMessage.Log(MessageType.Info, TickerData.ToString(cd.Contract) + ": No overlapping quote found. Used dates to change contracts: " + mergedQuotes[mergedQuoteIndex].DateTime + " and " + TickerData.ContinuousQuotesDictionary[cd.Contract.LocalSymbol][newQuoteIndex].DateTime + ".");
}
else
{
LogAndMessage.Log(MessageType.Info, TickerData.ToString(cd.Contract) + ": Switching to new contract on " + mergedQuotes[mergedQuoteIndex].DateTime + ".");
}
// get "closing prices" of the contract on the same day
float closeOfNewer = TickerData.ContinuousQuotesDictionary[cd.Contract.LocalSymbol][newQuoteIndex].Price;
float closeOfOlder = mergedQuotes[mergedQuoteIndex].Price;
double priceMult = closeOfNewer / closeOfOlder;
// back-adjust prev contracts' prices
QuotationList tempList = new QuotationList(FTDataSource.Periodicity);
for (int i = 0; i < mergedQuoteIndex; i++)
{
Quotation quote = mergedQuotes[i];
quote.Open = (float)(quote.Open * priceMult);
quote.High = (float)(quote.High * priceMult);
quote.Low = (float)(quote.Low * priceMult);
quote.Price = (float)(quote.Price * priceMult);
tempList.Merge(quote);
}
mergedQuotes.Clear();
mergedQuotes = tempList;
}
// add quotes of newer contract
for (; newQuoteIndex < TickerData.ContinuousQuotesDictionary[cd.Contract.LocalSymbol].Count; newQuoteIndex++)
{
mergedQuotes.Merge(TickerData.ContinuousQuotesDictionary[cd.Contract.LocalSymbol][newQuoteIndex]);
}
}
TickerData.Quotes = mergedQuotes;
}
#endregion
// this is not THROTTLED, but counted in general throttling queue
ibController.SendSubscriptionRequest(0, TickerData, false);
TickerData.QuoteDataStatus = QuotationStatus.Online;
return(allowNewRequest);
// this should never happen (ticker with online status should not be in the queue...)
case QuotationStatus.Online:
LogAndMessage.LogAndQueue(TickerData, MessageType.Info, "Backfill finished, symbol is ready. ");
IsFinished = true;
return(allowNewRequest);
// if any error happend
case QuotationStatus.Failed:
// if intraday download received no data response
if (errorCode == 162 && FTDataSource.Periodicity < Periodicity.EndOfDay && FTDataSource.Periodicity > Periodicity.FifteenSeconds)
{
errorCode = 0;
// move forward 4 periods to speed up download/find first valid period with available data
CalcNextBackfillRequest();
CalcNextBackfillRequest();
CalcNextBackfillRequest();
LogAndMessage.Log(TickerData, MessageType.Trace, "No data returned, fast forward download period.");
// start next download
TickerData.QuoteDataStatus = QuotationStatus.DownloadingIntra;
return(allowNewRequest);
}
else
{
LogAndMessage.LogAndQueue(TickerData, MessageType.Info, "Backfill failed, symbol is offline.");
IsFinished = true;
return(allowNewRequest);
}
// start historical data refresh
case QuotationStatus.New:
if (histThrottling)
{
return(false);
}
// calc download properties
downloadPeriodicity = FTDataSource.Periodicity;
downloadStep = IBClientHelper.GetDownloadStep(FTDataSource.Periodicity);
downloadInterval = IBClientHelper.GetDownloadInterval(FTDataSource.Periodicity);
downloadStart = IBClientHelper.GetAdjustedStartDate(TickerData.RefreshStartDate, FTDataSource.Periodicity, GetEarliestDownloadDate(), true);
downloadEnd = downloadStart.AddMinutes(downloadInterval);
downloadContract = GetCurrentContract(downloadStart);
// remove quotes already stored
TickerData.Quotes.Clear();
// set next state
if (FTDataSource.Periodicity == Periodicity.EndOfDay)
{
TickerData.QuoteDataStatus = QuotationStatus.DownloadingEod;
}
else
{
TickerData.QuoteDataStatus = QuotationStatus.DownloadingIntra;
}
// not to wait to send next request
RequestTime = DateTime.MinValue;
// download historical data
SendBackfillRequest(ibController);
return(false);
case QuotationStatus.DownloadingEod:
case QuotationStatus.DownloadingIntra:
if (histThrottling)
{
return(false);
}
// if previous request timed out
if (RequestTimeouts != 0)
{
SendBackfillRequest(ibController);
}
// download historical data
else if (CalcNextBackfillRequest())
{
SendBackfillRequest(ibController);
}
return(false);
// last CalcNextHistoricalDataRequest call for intraday bars should have set this state
case QuotationStatus.DownloadedIntra:
// if we need EOD data as well
if (FTDataSource.AllowMixedEODIntra)
{
if (histThrottling)
{
return(false);
}
// calc download properties for EOD
downloadPeriodicity = Periodicity.EndOfDay;
downloadStep = IBClientHelper.GetDownloadStep(Periodicity.EndOfDay);
downloadInterval = IBClientHelper.GetDownloadInterval(Periodicity.EndOfDay);
downloadStart = IBClientHelper.GetAdjustedStartDate(TickerData.RefreshStartDate, Periodicity.EndOfDay, GetEarliestDownloadDate(), true);
downloadEnd = downloadStart.AddMinutes(downloadInterval);
downloadContract = GetCurrentContract(downloadStart);
SendBackfillRequest(ibController);
TickerData.QuoteDataStatus = QuotationStatus.DownloadingEod;
}
else
{
TickerData.QuoteDataStatus = QuotationStatus.DownloadedEod;
}
return(false);
default:
LogAndMessage.LogAndQueue(TickerData, MessageType.Info, "Program error in backfill logic.");
IsFinished = true;
return(true);
}
}
}