private void HandleAllInstrumentsRequest()
{
_logger.Log(LogLevel.Info, "Instruments Server: received request for list of all instruments.");
var instruments = _instrumentManager.FindInstruments();
ReplyWithFoundInstruments(instruments);
}
private EarningsAnnouncementRequest GenerateRequests(EarningsUpdateJobSettings settings)
{
var startDate = DateTime.Now.Date.AddBusinessDays(-settings.BusinessDaysBack);
var endDate = DateTime.Now.Date.AddBusinessDays(settings.BusinessDaysAhead);
if (!settings.UseTag)
{
//grab all symbols
return(new EarningsAnnouncementRequest(
startDate,
endDate,
dataLocation: DataLocation.ExternalOnly,
dataSource: settings.DataSource));
}
else
{
//get the symbols using the tag filter
var symbols = _instrumentManager
.FindInstruments(x => x.Tags.Any(y => y.ID == settings.TagID))
.Result
.Select(x => x.Symbol)
.Distinct()
.ToList();
return(new EarningsAnnouncementRequest(
startDate,
endDate,
symbols.ToList(),
dataLocation: DataLocation.ExternalOnly,
dataSource: settings.DataSource));
}
}
private List <DividendRequest> GenerateRequests(DividendUpdateJobSettings settings)
{
var startDate = DateTime.Now.Date.AddBusinessDays(-settings.BusinessDaysBack);
var endDate = DateTime.Now.Date.AddBusinessDays(settings.BusinessDaysAhead);
var requests = new List <DividendRequest>();
if (!settings.UseTag)
{
//grab all symbols
requests.Add(new DividendRequest(
startDate,
endDate,
dataLocation: DataLocation.ExternalOnly,
dataSource: settings.DataSource,
symbol: string.Empty));
}
else
{
//get the symbols using the tag filter
var symbols = _instrumentManager
.FindInstruments(x => x.Tags.Any(y => y.ID == settings.TagID))
.Result
.Select(x => x.Symbol)
.Distinct()
.ToList();
foreach (var symbol in symbols)
{
requests.Add(new DividendRequest(
startDate,
endDate,
dataLocation: DataLocation.ExternalOnly,
dataSource: settings.DataSource,
symbol: symbol));
}
}
return(requests);
}
private void SocketReceiveReady(object sender, NetMQSocketEventArgs e)
{
var hasMore = false;
var request = string.Empty;
lock (_socketLock) {
var receiveResult = _socket?.TryReceiveFrameString(out request, out hasMore);
if (!receiveResult.HasValue || !receiveResult.Value || string.IsNullOrEmpty(request))
{
return;
}
}
var instruments = new List <Instrument>();
using (var ms = new MemoryStream()) {
// If the request is for a search, receive the instrument w/ the search parameters and pass it to the searcher
if (request == "SEARCH" && hasMore)
{
var buffer = _socket.ReceiveFrameBytes();
var searchInstrument = MyUtils.ProtoBufDeserialize <Instrument>(buffer, ms);
_logger.Info($"Instruments Server: Received search request: {searchInstrument}");
try {
instruments = _instrumentManager.FindInstruments(null, searchInstrument);
}
catch (Exception ex) {
_logger.Error($"Instruments Server: Instrument search error: {ex.Message}");
}
}
else if (request == "ALL") // If the request is for all the instruments, we don't need to receive anything else
{
_logger.Info("Instruments Server: received request for list of all instruments.");
try {
instruments = _instrumentManager.FindInstruments();
}
catch (Exception ex) {
_logger.Error($"Instruments Server: Instrument search error: {ex.Message}");
}
}
else if (request == "ADD" && hasMore) // Request to add instrument
{
var buffer = _socket.ReceiveFrameBytes();
var instrument = MyUtils.ProtoBufDeserialize <Instrument>(buffer, ms);
_logger.Info($"Instruments Server: Received instrument addition request. Instrument: {instrument}");
Instrument addedInstrument;
try {
addedInstrument = _instrumentManager.AddInstrument(instrument);
}
catch (Exception ex) {
addedInstrument = null;
_logger.Error($"Instruments Server: Instrument addition error: {ex.Message}");
}
_socket.SendMoreFrame(addedInstrument != null ? "SUCCESS" : "FAILURE");
_socket.SendFrame(MyUtils.ProtoBufSerialize(addedInstrument, ms));
return;
}
else // No request = loop again
{
return;
}
var uncompressed = MyUtils.ProtoBufSerialize(instruments, ms); // Serialize the list of instruments
ms.Read(uncompressed, 0, (int)ms.Length); // Get the uncompressed data
var result = LZ4Codec.Encode(uncompressed, 0, (int)ms.Length); // Compress it
// Before we send the result we must send the length of the uncompressed array, because it's needed for decompression
_socket.SendMoreFrame(BitConverter.GetBytes(uncompressed.Length));
// Then finally send the results
_socket.SendFrame(result);
}
}
/// <summary>
/// Taking a historical data request for a continuous futures instrument,
/// it returns a list of requests for the underlying contracts that are needed to fulfill it.
/// </summary>
private List <HistoricalDataRequest> GetRequiredRequests(HistoricalDataRequest request)
{
var requests = new List <HistoricalDataRequest>();
var cf = request.Instrument.ContinuousFuture;
var futures = _instrumentRepo.FindInstruments(x =>
x.UnderlyingSymbol == request.Instrument.ContinuousFuture.UnderlyingSymbol.Symbol &&
x.Type == InstrumentType.Future &&
x.DatasourceID == request.Instrument.DatasourceID).Result;
if (futures == null)
{
Log(LogLevel.Error, "CFB: Error in GetRequiredRequests, failed to return any contracts to historical request ID: " + request.AssignedID);
return(requests);
}
//remove any continuous futures
futures = futures.Where(x => !x.IsContinuousFuture).ToList();
//order them by ascending expiration date
futures = futures.OrderBy(x => x.Expiration).ToList();
//filter the futures months, we may not want all of them.
for (int i = 1; i <= 12; i++)
{
if (cf.MonthIsUsed(i))
{
continue;
}
int i1 = i;
futures = futures.Where(x => x.Expiration.HasValue && x.Expiration.Value.Month != i1).ToList();
}
//nothing found, return with empty hands
if (futures.Count == 0)
{
return(requests);
}
//the first contract we need is the first one expiring before the start of the request period
var expiringBeforeStart = futures
.Where(x => x.Expiration != null && x.Expiration.Value < request.StartingDate)
.Select(x => x.Expiration.Value).ToList();
DateTime firstExpiration =
expiringBeforeStart.Count > 0
? expiringBeforeStart.Max()
: futures.Select(x => x.Expiration.Value).Min();
futures = futures.Where(x => x.Expiration != null && x.Expiration.Value >= firstExpiration).ToList();
//I think the last contract we need is the one that is N months after the second contract that expires after the request period end
//where N is the number of months away from the front contract that the CF uses
var firstExpAfterEnd = futures.Where(x => x.Expiration > request.EndingDate).ElementAtOrDefault(1);
if (firstExpAfterEnd != null)
{
DateTime limitDate = firstExpAfterEnd.Expiration.Value.AddMonths(request.Instrument.ContinuousFuture.Month - 1);
futures = futures.Where(x => x.Expiration.Value.Year < limitDate.Year ||
(x.Expiration.Value.Year == limitDate.Year && x.Expiration.Value.Month <= limitDate.Month)).ToList();
}
//Make sure each month's contract is allowed only once, even if there are multiple copies in the db
//Sometimes you might get two versions of the same contract with 1 day difference in expiration date
//so this step is necessary to clean that up
futures = futures.Distinct(x => x.Expiration.Value.ToString("yyyyMM").GetHashCode()).ToList();
//save the number of requests we're gonna make
lock (_reqCountLock)
{
_requestCounts.Add(request.AssignedID, futures.Count);
}
//save the contracts used, we need them later
_contracts.Add(request.AssignedID, futures);
Log(LogLevel.Info, string.Format("CFB: fulfilling historical request ID {0}, requested data on contracts: {1}",
request.AssignedID,
string.Join(", ", futures.Select(x => x.Symbol))));
Instrument prevInst = null;
//request the data for all futures left
foreach (Instrument i in futures)
{
//the question of how much data, exactly, to ask for is complicated...
//I'm going with: difference of expiration dates (unless it's the first one, in which case 30 days)
//plus a month
//plus the CF selected month
int daysBack = 30;
if (prevInst == null)
{
daysBack += 30;
}
else
{
daysBack += (int)(i.Expiration.Value - prevInst.Expiration.Value).TotalDays;
}
daysBack += 30 * cf.Month;
var endDate = i.Expiration.Value > DateTime.Now.Date ? DateTime.Now.Date : i.Expiration.Value;
var req = new HistoricalDataRequest(
i,
request.Frequency,
endDate.AddDays(-daysBack),
endDate,
rthOnly: request.RTHOnly,
dataLocation: request.DataLocation == DataLocation.LocalOnly ? DataLocation.LocalOnly : DataLocation.Both);
requests.Add(req);
prevInst = i;
}
return(requests);
}
public InstrumentModule(IInstrumentSource instrumentRepo, IDataStorage dataStorage) : base("/instruments")
{
this.RequiresAuthentication();
Get["/", true] = async(_, token) => await instrumentRepo.FindInstruments().ConfigureAwait(false);
Get["/{Id:int}", true] = async(parameters, token) =>
{
//Instrument by ID
var id = (int)parameters.Id;
var instrument = (await instrumentRepo.FindInstruments(x => x.ID == id).ConfigureAwait(false)).FirstOrDefault();
if (instrument == null)
{
return(HttpStatusCode.NotFound);
}
return(instrument);
};
Get["/search", true] = async(_, token) =>
{
//Search using an instrument object
var inst = this.Bind <Instrument>();
if (inst == null)
{
return(HttpStatusCode.BadRequest);
}
return(await instrumentRepo.FindInstruments(inst).ConfigureAwait(false));
};
Get["/predsearch", true] = async(parameters, token) =>
{
//Search using a predicate
var predReq = this.Bind <PredicateSearchRequest>();
if (predReq == null)
{
return(HttpStatusCode.BadRequest);
}
//Deserialize LINQ expression and pass it to the instrument manager
Expression <Func <Instrument, bool> > expression;
try
{
expression = predReq.Filter;
}
catch (Exception ex)
{
return(Negotiate
.WithModel(new ValidationErrorResponse("Malformed predicate: " + ex.Message))
.WithStatusCode(HttpStatusCode.BadRequest));
}
var instruments = await instrumentRepo.FindInstruments(expression).ConfigureAwait(false);
return(instruments);
};
Post["/", true] = async(parameters, token) =>
{
Instrument instrument = this.BindAndValidate <Instrument>();
if (ModelValidationResult.IsValid == false)
{
return(this.ValidationFailure());
}
var addedInstrument = await instrumentRepo.AddInstrument(instrument);
return(addedInstrument);
};
Put["/", true] = async(parameters, token) =>
{
var instrument = this.BindAndValidate <Instrument>();
if (ModelValidationResult.IsValid == false)
{
return(this.ValidationFailure());
}
//find it
Instrument instrumentFromDB = (await instrumentRepo.FindInstruments(x => x.ID == instrument.ID).ConfigureAwait(false)).FirstOrDefault();
if (instrumentFromDB == null)
{
return(HttpStatusCode.NotFound);
}
//update it
await instrumentRepo.UpdateInstrument(instrumentFromDB, instrument).ConfigureAwait(false);
return(instrumentFromDB);
};
Delete["/{Id:int}", true] = async(parameters, token) =>
{
int id = parameters.Id;
var instrument = (await instrumentRepo.FindInstruments(x => x.ID == id).ConfigureAwait(false)).FirstOrDefault();
if (instrument == null)
{
return(HttpStatusCode.NotFound);
}
await instrumentRepo.RemoveInstrument(instrument, dataStorage).ConfigureAwait(false);
return(instrument);
};
}
/// <summary>
/// Called by the <see cref="T:Quartz.IScheduler"/> when a <see cref="T:Quartz.ITrigger"/>
/// fires that is associated with the <see cref="T:Quartz.IJob"/>.
/// </summary>
/// <remarks>
/// The implementation may wish to set a result object on the
/// JobExecutionContext before this method exits. The result itself
/// is meaningless to Quartz, but may be informative to
/// <see cref="T:Quartz.IJobListener"/>s or
/// <see cref="T:Quartz.ITriggerListener"/>s that are watching the job's
/// execution.
/// </remarks>
/// <param name="context">The execution context.</param>
public void Execute(IJobExecutionContext context)
{
_logger = LogManager.GetCurrentClassLogger();
if (_broker == null)
{
Log(LogLevel.Error, "Data Update Job failed: broker not set.");
return;
}
JobDataMap dataMap = context.JobDetail.JobDataMap;
DataUpdateJobSettings settings;
try
{
settings = JsonConvert.DeserializeObject <DataUpdateJobSettings>((string)dataMap["settings"]);
}
catch (Exception e)
{
_logger.Error(e, "Failed to deserialize data update job settings");
return;
}
Log(LogLevel.Info, string.Format("Data Update job {0} triggered.", settings.Name));
//Multiple jobs may be called simultaneously, so what we do is seed the Random based on the job name
byte[] bytes = new byte[settings.Name.Length * sizeof(char)];
Buffer.BlockCopy(settings.Name.ToCharArray(), 0, bytes, 0, bytes.Length);
Random r = new Random((int)DateTime.Now.TimeOfDay.TotalSeconds ^ BitConverter.ToInt32(bytes, 0));
_requesterID = "DataUpdateJob" + r.Next(); //we use this ID to identify this particular data update job
List <Instrument> instruments = settings.UseTag
? _instrumentManager.FindInstruments(x => x.Tags.Any(y => y.ID == settings.TagID)).Result
: _instrumentManager.FindInstruments(x => x.ID == settings.InstrumentID).Result;
if (instruments.Count == 0)
{
Log(LogLevel.Error, string.Format("Aborting data update job {0}: no instruments found.", settings.Name));
return;
}
_broker.HistoricalDataArrived += _broker_HistoricalDataArrived;
_broker.Error += _broker_Error;
int counter = 1;
//What we do here: we check what we have available locally..
//If there is something, we send a query to grab data between the last stored time and "now"
//Otherwise we send a query to grab everything since 1900
foreach (Instrument i in instruments)
{
if (!i.ID.HasValue)
{
continue;
}
//don't request data on expired securities unless the expiration was recent
if (i.Expiration.HasValue && (DateTime.Now - i.Expiration.Value).TotalDays > 15)
{
Log(LogLevel.Trace, string.Format("Data update job {0}: ignored instrument w/ ID {1} due to expiration date.", settings.Name, i.ID));
continue;
}
DateTime startingDT = new DateTime(1900, 1, 1);
var storageInfo = _localStorage.GetStorageInfo(i.ID.Value);
if (storageInfo.Any(x => x.Frequency == settings.Frequency))
{
var relevantStorageInfo = storageInfo.First(x => x.Frequency == settings.Frequency);
startingDT = relevantStorageInfo.LatestDate;
}
DateTime endDt = DateTime.Now;
//try to get the current time in the instrument's exchange timezone
string timeZone = i?.Exchange?.Timezone;
if (!string.IsNullOrEmpty(timeZone))
{
try
{
var tz = TimeZoneInfo.FindSystemTimeZoneById(timeZone);
endDt = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, tz);
}
catch (Exception e)
{
_logger.Error(e, "Could not find timezone " + timeZone);
}
}
var req = new HistoricalDataRequest(
i,
settings.Frequency,
startingDT,
endDt,
dataLocation: DataLocation.ExternalOnly,
saveToLocalStorage: true,
rthOnly: true,
requestID: counter)
{
RequesterIdentity = _requesterID
};
try
{
_broker.RequestHistoricalData(req);
lock (_reqIDLock)
{
_pendingRequests.Add(req);
}
}
catch (Exception ex)
{
_errors.Add(ex.Message);
}
counter++;
}
Stopwatch sw = new Stopwatch();
sw.Start();
//loop until time runs out or all requests are completed
while (_pendingRequests.Count > 0 &&
sw.ElapsedMilliseconds < _settings.Timeout * 1000)
{
Thread.Sleep(100);
}
JobComplete();
Log(LogLevel.Info, string.Format("Data Update job {0} completed.", settings.Name));
}
void _socket_ReceiveReady(object sender, NetMQSocketEventArgs e)
{
var ms = new MemoryStream();
List <Instrument> instruments;
bool hasMore;
string request = _socket.ReceiveString(SendReceiveOptions.DontWait, out hasMore);
if (request == null)
{
return;
}
//if the request is for a search, receive the instrument w/ the search parameters and pass it to the searcher
if (request == "SEARCH" && hasMore)
{
byte[] buffer = _socket.Receive();
var searchInstrument = MyUtils.ProtoBufDeserialize <Instrument>(buffer, ms);
Log(LogLevel.Info, string.Format("Instruments Server: Received search request: {0}",
searchInstrument));
try
{
instruments = _instrumentManager.FindInstruments(null, searchInstrument);
}
catch (Exception ex)
{
Log(LogLevel.Error, string.Format("Instruments Server: Instrument search error: {0}",
ex.Message));
instruments = new List <Instrument>();
}
}
else if (request == "ALL") //if the request is for all the instruments, we don't need to receive anything else
{
Log(LogLevel.Info, "Instruments Server: received request for list of all instruments.");
instruments = _instrumentManager.FindInstruments();
}
else if (request == "ADD" && hasMore) //request to add instrument
{
byte[] buffer = _socket.Receive();
var instrument = MyUtils.ProtoBufDeserialize <Instrument>(buffer, ms);
Log(LogLevel.Info, string.Format("Instruments Server: Received instrument addition request. Instrument: {0}",
instrument));
Instrument addedInstrument;
try
{
addedInstrument = _instrumentManager.AddInstrument(instrument);
}
catch (Exception ex)
{
addedInstrument = null;
Log(LogLevel.Error, string.Format("Instruments Server: Instrument addition error: {0}",
ex.Message));
}
_socket.SendMore(addedInstrument != null ? "SUCCESS" : "FAILURE");
_socket.Send(MyUtils.ProtoBufSerialize(addedInstrument, ms));
return;
}
else //no request = loop again
{
return;
}
byte[] uncompressed = MyUtils.ProtoBufSerialize(instruments, ms); //serialize the list of instruments
ms.Read(uncompressed, 0, (int)ms.Length); //get the uncompressed data
byte[] result = LZ4Codec.Encode(uncompressed, 0, (int)ms.Length); //compress it
//before we send the result we must send the length of the uncompressed array, because it's needed for decompression
_socket.SendMore(BitConverter.GetBytes(uncompressed.Length));
//then finally send the results
_socket.Send(result);
}
