//check the latest date we have available in local storage, then request historical data from that date to the current time private void UpdateHistoricalDataBtn_ItemClick(object sender, RoutedEventArgs routedEventArgs) { var frequency = (BarSize)((MenuItem)sender).Tag; List <Instrument> selectedInstruments = InstrumentsGrid.SelectedItems.Cast <Instrument>().ToList(); int requestCount = 0; using (var localStorage = DataStorageFactory.Get()) { foreach (Instrument i in selectedInstruments) { if (!i.ID.HasValue) { continue; } //TODO add GetStorageInfo to client, then remove dependency on DataStorageFactory here var storageInfo = localStorage.GetStorageInfo(i.ID.Value); if (storageInfo.Any(x => x.Frequency == frequency)) { var relevantStorageInfo = storageInfo.First(x => x.Frequency == frequency); _client.RequestHistoricalData(new HistoricalDataRequest( i, frequency, relevantStorageInfo.LatestDate + frequency.ToTimeSpan(), DateTime.Now, dataLocation: DataLocation.ExternalOnly, saveToLocalStorage: true)); requestCount++; } } } if (_progressBar.Value >= _progressBar.Maximum) { _progressBar.Maximum = requestCount; _progressBar.Value = 0; } else { _progressBar.Maximum += requestCount; } }
private void LoadDataBtn_Click(object sender, RoutedEventArgs e) { Data.Clear(); //grab the data //todo remove dependency on DataStorageFactory using (var localStorage = DataStorageFactory.Get()) { var bars = localStorage.GetData(TheInstrument, StartTime, EndTime, (BarSize)ResolutionComboBox.SelectedItem); //find largest significant decimal by sampling the prices at the start and end of the series var decPlaces = new List <int>(); for (int i = 0; i < Math.Min(bars.Count, 20); i++) { decPlaces.Add(bars[i].Open.CountDecimalPlaces()); decPlaces.Add(bars[bars.Count - 1 - i].Close.CountDecimalPlaces()); } //set the column format to use that number so we don't get any useless trailing 0s SetPriceColumnFormat(decPlaces.Max()); foreach (OHLCBar b in bars) { //do any required time zone coversions if (TimezoneComboBox.Text == "UTC") { b.DT = TimeZoneInfo.ConvertTimeToUtc(b.DT, _tzInfo); } else if (TimezoneComboBox.Text == "Local") { b.DT = TimeZoneInfo.ConvertTime(b.DT, _tzInfo, TimeZoneInfo.Local); } Data.Add(b); } _loadedFrequency = (BarSize)ResolutionComboBox.SelectedItem; _loadedTimeZone = TimezoneComboBox.Text; } StatusLabel.Content = string.Format("Loaded {0} Bars", Data.Count); }
public MainWindow() { Common.Logging.LogManager.Adapter = new NLogLoggerFactoryAdapter(new Common.Logging.Configuration.NameValueCollection()); //make sure we can connect to the database CheckDBConnection(); //set the log directory SetLogDirectory(); //set the connection string DBUtils.SetConnectionString(); //set EF configuration, necessary for MySql to work DBUtils.SetDbConfiguration(); InitializeComponent(); DataContext = this; //load datagrid layout string layoutFile = AppDomain.CurrentDomain.BaseDirectory + "GridLayout.xml"; if (File.Exists(layoutFile)) { try { InstrumentsGrid.DeserializeLayout(File.ReadAllText(layoutFile)); } catch { } } LogMessages = new ConcurrentNotifierBlockingList <LogEventInfo>(); //target is where the log managers send their logs, here we grab the memory target which has a Subject to observe var target = LogManager.Configuration.AllTargets.Single(x => x.Name == "myTarget") as MemoryTarget; //Log unhandled exceptions AppDomain.CurrentDomain.UnhandledException += AppDomain_CurrentDomain_UnhandledException; //we subscribe to the messages and send them all to the LogMessages collection if (target != null) { target.Messages.Subscribe(msg => LogMessages.TryAdd(msg)); } //build the instruments grid context menu //we want a button for each BarSize enum value in the UpdateFreqSubMenu menu foreach (int value in Enum.GetValues(typeof(BarSize))) { var button = new MenuItem { Header = Regex.Replace(((BarSize)value).ToString(), "([A-Z])", " $1").Trim(), Tag = (BarSize)value }; button.Click += UpdateHistoricalDataBtn_ItemClick; ((MenuItem)Resources["UpdateFreqSubMenu"]).Items.Add(button); } //create metadata db if it doesn't exist var entityContext = new MyDBContext(); entityContext.Database.Initialize(false); //seed the datasources no matter what, because these are added frequently Seed.SeedDatasources(entityContext); //check for any exchanges, seed the db with initial values if nothing is found if (!entityContext.Exchanges.Any()) { Seed.DoSeed(); } //create data db if it doesn't exist var dataContext = new DataDBContext(); dataContext.Database.Initialize(false); dataContext.Dispose(); //create quartz db if it doesn't exist QuartzUtils.InitializeDatabase(Settings.Default.databaseType); //build the tags menu var allTags = entityContext.Tags.ToList(); BuildTagContextMenu(allTags); //build session templates menu BuildSetSessionTemplateMenu(); Instruments = new ObservableCollection <Instrument>(); var instrumentRepo = new InstrumentRepository(entityContext); var instrumentList = instrumentRepo.FindInstruments().Result; foreach (Instrument i in instrumentList) { Instruments.Add(i); } //create brokers var cfRealtimeBroker = new ContinuousFuturesBroker(new QDMSClient.QDMSClient( "RTDBCFClient", "127.0.0.1", Properties.Settings.Default.rtDBReqPort, Properties.Settings.Default.rtDBPubPort, Properties.Settings.Default.hDBPort, Properties.Settings.Default.httpPort, Properties.Settings.Default.apiKey, useSsl: Properties.Settings.Default.useSsl), connectImmediately: false); var cfHistoricalBroker = new ContinuousFuturesBroker(new QDMSClient.QDMSClient( "HDBCFClient", "127.0.0.1", Properties.Settings.Default.rtDBReqPort, Properties.Settings.Default.rtDBPubPort, Properties.Settings.Default.hDBPort, Properties.Settings.Default.httpPort, Properties.Settings.Default.apiKey, useSsl: Properties.Settings.Default.useSsl), connectImmediately: false); var localStorage = DataStorageFactory.Get(); RealTimeBroker = new RealTimeDataBroker(cfRealtimeBroker, localStorage, new IRealTimeDataSource[] { //new Xignite(Properties.Settings.Default.xigniteApiToken), //new Oanda(Properties.Settings.Default.oandaAccountId, Properties.Settings.Default.oandaAccessToken), new IB(Properties.Settings.Default.ibClientHost, Properties.Settings.Default.ibClientPort, Properties.Settings.Default.rtdClientIBID), //new ForexFeed(Properties.Settings.Default.forexFeedAccessKey, ForexFeed.PriceType.Mid) }); HistoricalBroker = new HistoricalDataBroker(cfHistoricalBroker, localStorage, new IHistoricalDataSource[] { new Yahoo(), new FRED(), //new Forexite(), new IB(Properties.Settings.Default.ibClientHost, Properties.Settings.Default.ibClientPort, Properties.Settings.Default.histClientIBID), new Quandl(Properties.Settings.Default.quandlAuthCode), new BarChart(Properties.Settings.Default.barChartApiKey) }); var countryCodeHelper = new CountryCodeHelper(entityContext.Countries.ToList()); EconomicReleaseBroker = new EconomicReleaseBroker("FXStreet", new[] { new fx.FXStreet(countryCodeHelper) }); //create the various servers _realTimeServer = new RealTimeDataServer(Properties.Settings.Default.rtDBPubPort, Properties.Settings.Default.rtDBReqPort, RealTimeBroker); _historicalDataServer = new HistoricalDataServer(Properties.Settings.Default.hDBPort, HistoricalBroker); //and start them _realTimeServer.StartServer(); _historicalDataServer.StartServer(); //we also need a client to make historical data requests with _client = new QDMSClient.QDMSClient( "SERVERCLIENT", "localhost", Properties.Settings.Default.rtDBReqPort, Properties.Settings.Default.rtDBPubPort, Properties.Settings.Default.hDBPort, Properties.Settings.Default.httpPort, Properties.Settings.Default.apiKey, useSsl: Properties.Settings.Default.useSsl); _client.Connect(); _client.HistoricalDataReceived += _client_HistoricalDataReceived; ActiveStreamGrid.ItemsSource = RealTimeBroker.ActiveStreams; //create the scheduler var quartzSettings = QuartzUtils.GetQuartzSettings(Settings.Default.databaseType); ISchedulerFactory schedulerFactory = new StdSchedulerFactory(quartzSettings); _scheduler = schedulerFactory.GetScheduler(); _scheduler.JobFactory = new JobFactory(HistoricalBroker, Properties.Settings.Default.updateJobEmailHost, Properties.Settings.Default.updateJobEmailPort, Properties.Settings.Default.updateJobEmailUsername, Properties.Settings.Default.updateJobEmailPassword, Properties.Settings.Default.updateJobEmailSender, Properties.Settings.Default.updateJobEmail, new UpdateJobSettings( noDataReceived: Properties.Settings.Default.updateJobReportNoData, errors: Properties.Settings.Default.updateJobReportErrors, outliers: Properties.Settings.Default.updateJobReportOutliers, requestTimeouts: Properties.Settings.Default.updateJobTimeouts, timeout: Properties.Settings.Default.updateJobTimeout, toEmail: Properties.Settings.Default.updateJobEmail, fromEmail: Properties.Settings.Default.updateJobEmailSender), localStorage, EconomicReleaseBroker); _scheduler.Start(); //Take jobs stored in the qmds db and move them to the quartz db - this can be removed in the next version MigrateJobs(entityContext, _scheduler); var bootstrapper = new CustomBootstrapper( DataStorageFactory.Get(), EconomicReleaseBroker, HistoricalBroker, RealTimeBroker, Properties.Settings.Default.apiKey); var uri = new Uri((Settings.Default.useSsl ? "https" : "http") + "://localhost:" + Properties.Settings.Default.httpPort); var host = new NancyHost(bootstrapper, uri); host.Start(); entityContext.Dispose(); ShowChangelog(); }
private void ImportBtn_Click(object sender, RoutedEventArgs e) { Stopwatch sw = new Stopwatch(); sw.Start(); //check that we've got the relevant data needed if (!Data.Columns.Contains("Date") && !Data.Columns.Contains("DateTime")) { MessageBox.Show("Must have a date column."); return; } if ((BarSize)FrequencyComboBox.SelectedItem < BarSize.OneDay && !Data.Columns.Contains("DateTime") && !Data.Columns.Contains("Time")) { MessageBox.Show("Must have time column at this frequency"); return; } if (!Data.Columns.Contains("Open") || !Data.Columns.Contains("High") || !Data.Columns.Contains("Low") || !Data.Columns.Contains("Close")) { MessageBox.Show("Must have all OHLC columns."); return; } //make sure the timezone is set, and get it if (string.IsNullOrEmpty(_instrument.Exchange.Timezone)) { MessageBox.Show("Instrument's exchange has no set timezone, can't import."); return; } var tzInfo = TimeZoneInfo.FindSystemTimeZoneById(_instrument.Exchange.Timezone); //get the multipliers decimal priceMultiplier; int volumeMultiplier; bool parseWorked = decimal.TryParse(PriceMultiplier.Text, out priceMultiplier); if (!parseWorked) { priceMultiplier = 1; } parseWorked = int.TryParse(VolumeMultiplier.Text, out volumeMultiplier); if (!parseWorked) { volumeMultiplier = 1; } //lines to skip int toSkip; parseWorked = int.TryParse(StartingLine.Text, out toSkip); if (!parseWorked) { toSkip = 1; } //get the frequency var frequency = (BarSize)FrequencyComboBox.SelectedItem; //separator char[] separator = DelimiterBox.Text.ToCharArray(); List <OHLCBar> bars = new List <OHLCBar>(); string[] columns = new string[Data.Columns.Count]; for (int i = 0; i < Data.Columns.Count; i++) { columns[i] = Data.Columns[i].ColumnName; } //determining time: if the freq is >= one day, then the time is simply the session end for this day Dictionary <int, TimeSpan> sessionEndTimes = new Dictionary <int, TimeSpan>(); //1 day and up: we can load it all in one go with no trouble, also may require adjustment bool periodicSaving = frequency < BarSize.OneDay; OHLCBar bar; var barsCount = 0; using (StreamReader sr = new StreamReader(FilePathTextBox.Text)) { string line; while ((line = sr.ReadLine()) != null) { barsCount++; if (barsCount < toSkip) { continue; } try { bar = ParseLine(line.Split(separator), columns, priceMultiplier, volumeMultiplier); } catch (Exception ex) { MessageBox.Show("Importing error: " + ex.Message); return; } //only add the bar if it falls within the specified date range if (bar.DT >= MinDT.Value && bar.DT <= MaxDT.Value) { bars.Add(bar); } //with 30 bars, we make a check to ensure that the user has entered the correct frequency if (bars.Count == 30) { //the reason we have to use a bunch of bars and look for the most frequent timespan between them //is that session breaks, daily breaks, weekends, etc. can have different timespans despite the //correct frequency being chosen List <int> secDiffs = new List <int>(); for (int i = 1; i < bars.Count; i++) { secDiffs.Add((int)Math.Round((bars[i].DT - bars[i - 1].DT).TotalSeconds)); } int mostFrequent = secDiffs.MostFrequent(); if ((int)Math.Round(frequency.ToTimeSpan().TotalSeconds) != mostFrequent) { MessageBox.Show("You appear to have selected the wrong frequency."); return; } } if (periodicSaving && bars.Count > 1000) { //convert to exchange timezone ConvertTimeZone(bars, tzInfo); //low frequencies, < 1 day. No adjustment required and inserting data at intervals instead of all at once using (var storage = DataStorageFactory.Get()) { try { storage.AddData(bars, _instrument, frequency, OverwriteCheckbox.IsChecked.HasValue && OverwriteCheckbox.IsChecked.Value, false); } catch (Exception ex) { MessageBox.Show("Error: " + ex.Message); } } bars.Clear(); } } } if (bars.Count == 0) { return; } //convert to exchange timezone ConvertTimeZone(bars, tzInfo); //if only the date column is set, we need to get the session info and generate the closing time ourselves if (frequency >= BarSize.OneDay && !Data.Columns.Contains("Time") && !Data.Columns.Contains("DateTime")) { //get the closing time for every day of the week var dotwValues = MyUtils.GetEnumValues <DayOfTheWeek>(); foreach (DayOfTheWeek d in dotwValues) { if (_instrument.Sessions.Any(x => x.ClosingDay == d && x.IsSessionEnd)) { var endTime = _instrument.Sessions.First(x => x.ClosingDay == d && x.IsSessionEnd).ClosingTime; sessionEndTimes.Add((int)d, endTime); } else { sessionEndTimes.Add((int)d, TimeSpan.FromSeconds(0)); } } for (int i = 0; i < bars.Count; i++) { int dayOfWeek = bars[i].DT.DayOfWeek.ToInt(); bars[i].DT = bars[i].DT.Date + sessionEndTimes[dayOfWeek]; } } //if there are no dividends/splits, but there IS an adjclose column, use that to adjust data right here //if there are divs/splits, adjustment will be done by the local storage if (frequency >= BarSize.OneDay && !Data.Columns.Contains("Dividends") && !Data.Columns.Contains("Splits") && Data.Columns.Contains("AdjClose")) { //if we have an adjusted close to work off of, we just use the ratio to get the OHL for (int i = 0; i < bars.Count; i++) { if (bars[i].AdjClose == null) { continue; } decimal ratio = bars[i].AdjClose.Value / bars[i].Close; bars[i].AdjOpen = bars[i].Open * ratio; bars[i].AdjHigh = bars[i].High * ratio; bars[i].AdjLow = bars[i].Low * ratio; } } //sort by date if (frequency >= BarSize.OneDay) { bars.Sort((x, y) => x.DT.CompareTo(y.DT)); } //try to import using (var storage = DataStorageFactory.Get()) { try { storage.AddData(bars, _instrument, frequency, OverwriteCheckbox.IsChecked.HasValue && OverwriteCheckbox.IsChecked.Value, frequency >= BarSize.OneDay); } catch (Exception ex) { MessageBox.Show("Error: " + ex.Message); } } sw.Stop(); MessageBox.Show(string.Format("Imported {0} bars in {1} ms.", barsCount, sw.ElapsedMilliseconds)); }