private void HandleAllInstrumentsRequest() { _logger.Log(LogLevel.Info, "Instruments Server: received request for list of all instruments."); var instruments = _instrumentManager.FindInstruments(); ReplyWithFoundInstruments(instruments); }
private EarningsAnnouncementRequest GenerateRequests(EarningsUpdateJobSettings settings) { var startDate = DateTime.Now.Date.AddBusinessDays(-settings.BusinessDaysBack); var endDate = DateTime.Now.Date.AddBusinessDays(settings.BusinessDaysAhead); if (!settings.UseTag) { //grab all symbols return(new EarningsAnnouncementRequest( startDate, endDate, dataLocation: DataLocation.ExternalOnly, dataSource: settings.DataSource)); } else { //get the symbols using the tag filter var symbols = _instrumentManager .FindInstruments(x => x.Tags.Any(y => y.ID == settings.TagID)) .Result .Select(x => x.Symbol) .Distinct() .ToList(); return(new EarningsAnnouncementRequest( startDate, endDate, symbols.ToList(), dataLocation: DataLocation.ExternalOnly, dataSource: settings.DataSource)); } }
private List <DividendRequest> GenerateRequests(DividendUpdateJobSettings settings) { var startDate = DateTime.Now.Date.AddBusinessDays(-settings.BusinessDaysBack); var endDate = DateTime.Now.Date.AddBusinessDays(settings.BusinessDaysAhead); var requests = new List <DividendRequest>(); if (!settings.UseTag) { //grab all symbols requests.Add(new DividendRequest( startDate, endDate, dataLocation: DataLocation.ExternalOnly, dataSource: settings.DataSource, symbol: string.Empty)); } else { //get the symbols using the tag filter var symbols = _instrumentManager .FindInstruments(x => x.Tags.Any(y => y.ID == settings.TagID)) .Result .Select(x => x.Symbol) .Distinct() .ToList(); foreach (var symbol in symbols) { requests.Add(new DividendRequest( startDate, endDate, dataLocation: DataLocation.ExternalOnly, dataSource: settings.DataSource, symbol: symbol)); } } return(requests); }
private void SocketReceiveReady(object sender, NetMQSocketEventArgs e) { var hasMore = false; var request = string.Empty; lock (_socketLock) { var receiveResult = _socket?.TryReceiveFrameString(out request, out hasMore); if (!receiveResult.HasValue || !receiveResult.Value || string.IsNullOrEmpty(request)) { return; } } var instruments = new List <Instrument>(); using (var ms = new MemoryStream()) { // If the request is for a search, receive the instrument w/ the search parameters and pass it to the searcher if (request == "SEARCH" && hasMore) { var buffer = _socket.ReceiveFrameBytes(); var searchInstrument = MyUtils.ProtoBufDeserialize <Instrument>(buffer, ms); _logger.Info($"Instruments Server: Received search request: {searchInstrument}"); try { instruments = _instrumentManager.FindInstruments(null, searchInstrument); } catch (Exception ex) { _logger.Error($"Instruments Server: Instrument search error: {ex.Message}"); } } else if (request == "ALL") // If the request is for all the instruments, we don't need to receive anything else { _logger.Info("Instruments Server: received request for list of all instruments."); try { instruments = _instrumentManager.FindInstruments(); } catch (Exception ex) { _logger.Error($"Instruments Server: Instrument search error: {ex.Message}"); } } else if (request == "ADD" && hasMore) // Request to add instrument { var buffer = _socket.ReceiveFrameBytes(); var instrument = MyUtils.ProtoBufDeserialize <Instrument>(buffer, ms); _logger.Info($"Instruments Server: Received instrument addition request. Instrument: {instrument}"); Instrument addedInstrument; try { addedInstrument = _instrumentManager.AddInstrument(instrument); } catch (Exception ex) { addedInstrument = null; _logger.Error($"Instruments Server: Instrument addition error: {ex.Message}"); } _socket.SendMoreFrame(addedInstrument != null ? "SUCCESS" : "FAILURE"); _socket.SendFrame(MyUtils.ProtoBufSerialize(addedInstrument, ms)); return; } else // No request = loop again { return; } var uncompressed = MyUtils.ProtoBufSerialize(instruments, ms); // Serialize the list of instruments ms.Read(uncompressed, 0, (int)ms.Length); // Get the uncompressed data var result = LZ4Codec.Encode(uncompressed, 0, (int)ms.Length); // Compress it // Before we send the result we must send the length of the uncompressed array, because it's needed for decompression _socket.SendMoreFrame(BitConverter.GetBytes(uncompressed.Length)); // Then finally send the results _socket.SendFrame(result); } }
/// <summary> /// Taking a historical data request for a continuous futures instrument, /// it returns a list of requests for the underlying contracts that are needed to fulfill it. /// </summary> private List <HistoricalDataRequest> GetRequiredRequests(HistoricalDataRequest request) { var requests = new List <HistoricalDataRequest>(); var cf = request.Instrument.ContinuousFuture; var futures = _instrumentRepo.FindInstruments(x => x.UnderlyingSymbol == request.Instrument.ContinuousFuture.UnderlyingSymbol.Symbol && x.Type == InstrumentType.Future && x.DatasourceID == request.Instrument.DatasourceID).Result; if (futures == null) { Log(LogLevel.Error, "CFB: Error in GetRequiredRequests, failed to return any contracts to historical request ID: " + request.AssignedID); return(requests); } //remove any continuous futures futures = futures.Where(x => !x.IsContinuousFuture).ToList(); //order them by ascending expiration date futures = futures.OrderBy(x => x.Expiration).ToList(); //filter the futures months, we may not want all of them. for (int i = 1; i <= 12; i++) { if (cf.MonthIsUsed(i)) { continue; } int i1 = i; futures = futures.Where(x => x.Expiration.HasValue && x.Expiration.Value.Month != i1).ToList(); } //nothing found, return with empty hands if (futures.Count == 0) { return(requests); } //the first contract we need is the first one expiring before the start of the request period var expiringBeforeStart = futures .Where(x => x.Expiration != null && x.Expiration.Value < request.StartingDate) .Select(x => x.Expiration.Value).ToList(); DateTime firstExpiration = expiringBeforeStart.Count > 0 ? expiringBeforeStart.Max() : futures.Select(x => x.Expiration.Value).Min(); futures = futures.Where(x => x.Expiration != null && x.Expiration.Value >= firstExpiration).ToList(); //I think the last contract we need is the one that is N months after the second contract that expires after the request period end //where N is the number of months away from the front contract that the CF uses var firstExpAfterEnd = futures.Where(x => x.Expiration > request.EndingDate).ElementAtOrDefault(1); if (firstExpAfterEnd != null) { DateTime limitDate = firstExpAfterEnd.Expiration.Value.AddMonths(request.Instrument.ContinuousFuture.Month - 1); futures = futures.Where(x => x.Expiration.Value.Year < limitDate.Year || (x.Expiration.Value.Year == limitDate.Year && x.Expiration.Value.Month <= limitDate.Month)).ToList(); } //Make sure each month's contract is allowed only once, even if there are multiple copies in the db //Sometimes you might get two versions of the same contract with 1 day difference in expiration date //so this step is necessary to clean that up futures = futures.Distinct(x => x.Expiration.Value.ToString("yyyyMM").GetHashCode()).ToList(); //save the number of requests we're gonna make lock (_reqCountLock) { _requestCounts.Add(request.AssignedID, futures.Count); } //save the contracts used, we need them later _contracts.Add(request.AssignedID, futures); Log(LogLevel.Info, string.Format("CFB: fulfilling historical request ID {0}, requested data on contracts: {1}", request.AssignedID, string.Join(", ", futures.Select(x => x.Symbol)))); Instrument prevInst = null; //request the data for all futures left foreach (Instrument i in futures) { //the question of how much data, exactly, to ask for is complicated... //I'm going with: difference of expiration dates (unless it's the first one, in which case 30 days) //plus a month //plus the CF selected month int daysBack = 30; if (prevInst == null) { daysBack += 30; } else { daysBack += (int)(i.Expiration.Value - prevInst.Expiration.Value).TotalDays; } daysBack += 30 * cf.Month; var endDate = i.Expiration.Value > DateTime.Now.Date ? DateTime.Now.Date : i.Expiration.Value; var req = new HistoricalDataRequest( i, request.Frequency, endDate.AddDays(-daysBack), endDate, rthOnly: request.RTHOnly, dataLocation: request.DataLocation == DataLocation.LocalOnly ? DataLocation.LocalOnly : DataLocation.Both); requests.Add(req); prevInst = i; } return(requests); }
public InstrumentModule(IInstrumentSource instrumentRepo, IDataStorage dataStorage) : base("/instruments") { this.RequiresAuthentication(); Get["/", true] = async(_, token) => await instrumentRepo.FindInstruments().ConfigureAwait(false); Get["/{Id:int}", true] = async(parameters, token) => { //Instrument by ID var id = (int)parameters.Id; var instrument = (await instrumentRepo.FindInstruments(x => x.ID == id).ConfigureAwait(false)).FirstOrDefault(); if (instrument == null) { return(HttpStatusCode.NotFound); } return(instrument); }; Get["/search", true] = async(_, token) => { //Search using an instrument object var inst = this.Bind <Instrument>(); if (inst == null) { return(HttpStatusCode.BadRequest); } return(await instrumentRepo.FindInstruments(inst).ConfigureAwait(false)); }; Get["/predsearch", true] = async(parameters, token) => { //Search using a predicate var predReq = this.Bind <PredicateSearchRequest>(); if (predReq == null) { return(HttpStatusCode.BadRequest); } //Deserialize LINQ expression and pass it to the instrument manager Expression <Func <Instrument, bool> > expression; try { expression = predReq.Filter; } catch (Exception ex) { return(Negotiate .WithModel(new ValidationErrorResponse("Malformed predicate: " + ex.Message)) .WithStatusCode(HttpStatusCode.BadRequest)); } var instruments = await instrumentRepo.FindInstruments(expression).ConfigureAwait(false); return(instruments); }; Post["/", true] = async(parameters, token) => { Instrument instrument = this.BindAndValidate <Instrument>(); if (ModelValidationResult.IsValid == false) { return(this.ValidationFailure()); } var addedInstrument = await instrumentRepo.AddInstrument(instrument); return(addedInstrument); }; Put["/", true] = async(parameters, token) => { var instrument = this.BindAndValidate <Instrument>(); if (ModelValidationResult.IsValid == false) { return(this.ValidationFailure()); } //find it Instrument instrumentFromDB = (await instrumentRepo.FindInstruments(x => x.ID == instrument.ID).ConfigureAwait(false)).FirstOrDefault(); if (instrumentFromDB == null) { return(HttpStatusCode.NotFound); } //update it await instrumentRepo.UpdateInstrument(instrumentFromDB, instrument).ConfigureAwait(false); return(instrumentFromDB); }; Delete["/{Id:int}", true] = async(parameters, token) => { int id = parameters.Id; var instrument = (await instrumentRepo.FindInstruments(x => x.ID == id).ConfigureAwait(false)).FirstOrDefault(); if (instrument == null) { return(HttpStatusCode.NotFound); } await instrumentRepo.RemoveInstrument(instrument, dataStorage).ConfigureAwait(false); return(instrument); }; }
/// <summary> /// Called by the <see cref="T:Quartz.IScheduler"/> when a <see cref="T:Quartz.ITrigger"/> /// fires that is associated with the <see cref="T:Quartz.IJob"/>. /// </summary> /// <remarks> /// The implementation may wish to set a result object on the /// JobExecutionContext before this method exits. The result itself /// is meaningless to Quartz, but may be informative to /// <see cref="T:Quartz.IJobListener"/>s or /// <see cref="T:Quartz.ITriggerListener"/>s that are watching the job's /// execution. /// </remarks> /// <param name="context">The execution context.</param> public void Execute(IJobExecutionContext context) { _logger = LogManager.GetCurrentClassLogger(); if (_broker == null) { Log(LogLevel.Error, "Data Update Job failed: broker not set."); return; } JobDataMap dataMap = context.JobDetail.JobDataMap; DataUpdateJobSettings settings; try { settings = JsonConvert.DeserializeObject <DataUpdateJobSettings>((string)dataMap["settings"]); } catch (Exception e) { _logger.Error(e, "Failed to deserialize data update job settings"); return; } Log(LogLevel.Info, string.Format("Data Update job {0} triggered.", settings.Name)); //Multiple jobs may be called simultaneously, so what we do is seed the Random based on the job name byte[] bytes = new byte[settings.Name.Length * sizeof(char)]; Buffer.BlockCopy(settings.Name.ToCharArray(), 0, bytes, 0, bytes.Length); Random r = new Random((int)DateTime.Now.TimeOfDay.TotalSeconds ^ BitConverter.ToInt32(bytes, 0)); _requesterID = "DataUpdateJob" + r.Next(); //we use this ID to identify this particular data update job List <Instrument> instruments = settings.UseTag ? _instrumentManager.FindInstruments(x => x.Tags.Any(y => y.ID == settings.TagID)).Result : _instrumentManager.FindInstruments(x => x.ID == settings.InstrumentID).Result; if (instruments.Count == 0) { Log(LogLevel.Error, string.Format("Aborting data update job {0}: no instruments found.", settings.Name)); return; } _broker.HistoricalDataArrived += _broker_HistoricalDataArrived; _broker.Error += _broker_Error; int counter = 1; //What we do here: we check what we have available locally.. //If there is something, we send a query to grab data between the last stored time and "now" //Otherwise we send a query to grab everything since 1900 foreach (Instrument i in instruments) { if (!i.ID.HasValue) { continue; } //don't request data on expired securities unless the expiration was recent if (i.Expiration.HasValue && (DateTime.Now - i.Expiration.Value).TotalDays > 15) { Log(LogLevel.Trace, string.Format("Data update job {0}: ignored instrument w/ ID {1} due to expiration date.", settings.Name, i.ID)); continue; } DateTime startingDT = new DateTime(1900, 1, 1); var storageInfo = _localStorage.GetStorageInfo(i.ID.Value); if (storageInfo.Any(x => x.Frequency == settings.Frequency)) { var relevantStorageInfo = storageInfo.First(x => x.Frequency == settings.Frequency); startingDT = relevantStorageInfo.LatestDate; } DateTime endDt = DateTime.Now; //try to get the current time in the instrument's exchange timezone string timeZone = i?.Exchange?.Timezone; if (!string.IsNullOrEmpty(timeZone)) { try { var tz = TimeZoneInfo.FindSystemTimeZoneById(timeZone); endDt = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, tz); } catch (Exception e) { _logger.Error(e, "Could not find timezone " + timeZone); } } var req = new HistoricalDataRequest( i, settings.Frequency, startingDT, endDt, dataLocation: DataLocation.ExternalOnly, saveToLocalStorage: true, rthOnly: true, requestID: counter) { RequesterIdentity = _requesterID }; try { _broker.RequestHistoricalData(req); lock (_reqIDLock) { _pendingRequests.Add(req); } } catch (Exception ex) { _errors.Add(ex.Message); } counter++; } Stopwatch sw = new Stopwatch(); sw.Start(); //loop until time runs out or all requests are completed while (_pendingRequests.Count > 0 && sw.ElapsedMilliseconds < _settings.Timeout * 1000) { Thread.Sleep(100); } JobComplete(); Log(LogLevel.Info, string.Format("Data Update job {0} completed.", settings.Name)); }
void _socket_ReceiveReady(object sender, NetMQSocketEventArgs e) { var ms = new MemoryStream(); List <Instrument> instruments; bool hasMore; string request = _socket.ReceiveString(SendReceiveOptions.DontWait, out hasMore); if (request == null) { return; } //if the request is for a search, receive the instrument w/ the search parameters and pass it to the searcher if (request == "SEARCH" && hasMore) { byte[] buffer = _socket.Receive(); var searchInstrument = MyUtils.ProtoBufDeserialize <Instrument>(buffer, ms); Log(LogLevel.Info, string.Format("Instruments Server: Received search request: {0}", searchInstrument)); try { instruments = _instrumentManager.FindInstruments(null, searchInstrument); } catch (Exception ex) { Log(LogLevel.Error, string.Format("Instruments Server: Instrument search error: {0}", ex.Message)); instruments = new List <Instrument>(); } } else if (request == "ALL") //if the request is for all the instruments, we don't need to receive anything else { Log(LogLevel.Info, "Instruments Server: received request for list of all instruments."); instruments = _instrumentManager.FindInstruments(); } else if (request == "ADD" && hasMore) //request to add instrument { byte[] buffer = _socket.Receive(); var instrument = MyUtils.ProtoBufDeserialize <Instrument>(buffer, ms); Log(LogLevel.Info, string.Format("Instruments Server: Received instrument addition request. Instrument: {0}", instrument)); Instrument addedInstrument; try { addedInstrument = _instrumentManager.AddInstrument(instrument); } catch (Exception ex) { addedInstrument = null; Log(LogLevel.Error, string.Format("Instruments Server: Instrument addition error: {0}", ex.Message)); } _socket.SendMore(addedInstrument != null ? "SUCCESS" : "FAILURE"); _socket.Send(MyUtils.ProtoBufSerialize(addedInstrument, ms)); return; } else //no request = loop again { return; } byte[] uncompressed = MyUtils.ProtoBufSerialize(instruments, ms); //serialize the list of instruments ms.Read(uncompressed, 0, (int)ms.Length); //get the uncompressed data byte[] result = LZ4Codec.Encode(uncompressed, 0, (int)ms.Length); //compress it //before we send the result we must send the length of the uncompressed array, because it's needed for decompression _socket.SendMore(BitConverter.GetBytes(uncompressed.Length)); //then finally send the results _socket.Send(result); }