/// <summary>
/// send historical bar request
/// </summary>
public virtual void SendHistoricalBarRequest(BarRequest br)
{
if (SendReqHistBarEvent != null)
{
SendReqHistBarEvent(br);
}
else
{
SendDebug("SendreqHistBar is not supported in this application");
}
}
public void ReqHistoricalData(BarRequest br)
{
if (_client != null)
_client.RequestHistoricalData(br, true);
}
/// <summary>
/// Arguments are GOOG/IB, rundate, basketfile, datapath
/// </summary>
static void Main(string[] args)
{
#region parameters
string _client = "IB"; // IB or GOOG
IClient _iclient;
string _enddate = DateTime.Today.ToString("yyyyMMdd");
string _basketfile = @"c:\QuantTrading\Config\basket.xml";
string _datapath = @"c:\QuantTrading\HistData\";
Basket _basket;
if (args.Length > 0)
{
_client = args[0];
}
if (args.Length > 1)
{
_enddate = args[1];
}
if (args.Length > 2)
{
_basketfile = args[2];
}
if (args.Length > 3)
{
_datapath = args[3];
}
_datapath = _datapath + _client + @"\";
_basket = Basket.DeserializeFromXML(_basketfile);
#endregion
_sec2bars = new Dictionary<string, List<string>>(_basket.Count);
Dictionary<string,string> _outfiles = new Dictionary<string,string>(_basket.Count);
Dictionary<string, long> _totalBars = new Dictionary<string, long>(_basket.Count);
_processedBars = new Dictionary<string, long>(_basket.Count);
DateTime _startTime = DateTime.SpecifyKind(DateTime.ParseExact(_enddate, "yyyyMMdd", null), DateTimeKind.Local);
DateTime _endTime = _startTime + new TimeSpan(1, 0, 0, 0); // one day later
double sec = _endTime.Subtract(_startTime).TotalSeconds; // should be 24 hours or 86,400 secs
foreach(string s in _basket.Securities)
{
// set up bar counts
_totalBars.Add(s, (long)sec); // one bar is one sec; if one minute, divide it by 60
_processedBars.Add(s, 0); // initialize processed to 0
// set up out filenames
string filename = _datapath + s + "_" + _startTime.Date.ToString("yyyyMMdd") + ".csv";
_outfiles.Add(s, filename);
List<string> lines = new List<string>(90000); // set something greater than 86,4000
lines.Add("DateTime,Open,High,Low,Close,Volume");
_sec2bars.Add(s, lines);
}
if (_client == "IB")
{
_iclient = new IBClient(); // currently it uses default ip:port
_iclient.SendDebugEventDelegate += _iclient_SendDebugEventDelegate;
_iclient.GotHistoricalBarDelegate += _iclient_GotHistoricalBarDelegate;
_iclient.Connect();
long totalRequests = (long)sec / 1800; // one request is 30 min, totalRequests = 48
TimeSpan thirtyMin = new TimeSpan(0, 30, 0);
foreach (string sym in _basket.Securities)
{
DateTime s = _startTime;
DateTime t = _startTime + thirtyMin;
Console.WriteLine("Requesting historical bars for :" + sym);
for (int i = 0; i < totalRequests; i++)
{
Console.WriteLine("Request #: " + (i+1).ToString() + "/"+totalRequests);
// 1 = 1 second
BarRequest br = new BarRequest(sym, 1, Util.ToIntDate(s.Date), Util.ToIntTime(s.Hour, s.Minute, s.Second),
Util.ToIntDate(t.Date), Util.ToIntTime(t.Hour, t.Minute, t.Second), _client);
_iclient.RequestHistoricalData(br, true);
// Do not make more than 60 historical data requests in any ten-minute period.
// If I have 10 names, each can only make 6 requests in ten minute;
// I use 5 minute for a pause; Then 24 hours takes 120 min or 1.5hour
// Thread.Sleep(new TimeSpan(0, 5, 0));
// wait 10 secs
Thread.Sleep(10000);
s += thirtyMin;
t += thirtyMin;
}
}
}
else if (_client == "GOOG")
{
_iclient = new GoogleClient(1);
_iclient.SendDebugEventDelegate += _iclient_SendDebugEventDelegate;
_iclient.GotHistoricalBarDelegate += _iclient_GotHistoricalBarDelegate;
foreach (string sym in _basket.Securities)
{
Console.WriteLine("Requesting historical bars for :" + sym);
BarRequest br = new BarRequest(sym, 60, Util.ToIntDate(DateTime.Today), Util.ToIntTime(DateTime.Today),
Util.ToIntDate(DateTime.Today), Util.ToIntTime(DateTime.Today), _client);
_iclient.RequestHistoricalData(br);
}
}
// write to files
Console.WriteLine("Wait three minutes for bars being processed.....");
Thread.Sleep(new TimeSpan(0, 3, 0)); // wait three minutes for all hist bar to be processed.
foreach (string s in _basket.Securities)
{
List<string> noDups = _sec2bars[s].Distinct().ToList();
//_sec2bars[s].Insert(0, _processedBars[s].ToString());
File.WriteAllLines(_outfiles[s], noDups);
}
}
/// <summary>
/// http://www.marketcalls.in/database/google-realtime-intraday-backfill-data.html
/// http://www.codeproject.com/KB/IP/google_finance_downloader.aspx
/// </summary>
/// <param name="br"></param>
public void RequestHistoricalData(BarRequest br, bool useRTH=false)
{
// Google always returns the most recent data.
// i is interval in seconds, set to 60s, ignore interval
// ignore date; p: period
// q is the symbol (AAPL)
// x is the exchange (NASD)
// sessions is the session requested (ext_hours)
// p is the time period (5d = 5 days), set to 1d, ignore time span
// f is the requested fields (d,c,v,o,h,l)
// df = (cpct)
// auto = (1)
// ts is potentially a time stamp (1324323553 905) or time shift
try
{
using (WebClient client = new WebClient())
{
string google;
if (br.Interval != 86400)
{
google = @"https://www.google.com/finance/getprices?i="+br.Interval.ToString()+@"&p=1d&f=d,o,h,l,c,v&df=cpct&q=";
}
else // for oneday, today is empty
{
google = @"https://www.google.com/finance/getprices?i=86400&p=2d&f=d,o,h,l,c,v&df=cpct&q=";
}
string[] symbol = br.FullSymbol.Split(' ');
System.IO.Stream data = client.OpenRead(google + symbol[0]);
System.IO.StreamReader read = new System.IO.StreamReader(data);
string[] lines = new string[] { read.ReadToEnd() };
string[] lines2 = lines[0].Split('\n');
// get time zone adjustment
// In line 6, GOOG has time zone offset = -240 which is new york time;
// while SPX has time zone offset = -300, which is chicago time.
// The following find the additional offset relative to local time.
/*
int localtimezonediffinminutes = (int)Util.GetUtcOffset(DateTime.Today).TotalMinutes; // negative offset
string stime = lines[6];
int itime;
bool btime = Int32.TryParse(stime.Substring(stime.IndexOf('=') + 1), out itime); // negative offset
int additionaloffset = localtimezonediffinminutes - itime; // (-240) - (-300) = 60 mins
*/
IEnumerable<string> history = lines2.Skip(7); // skip the first 7 lines: header
int nlines = 0; // count of lines
string[] entries;
DateTime dstart = DateTime.Now; // just for initialization
DateTime dt = DateTime.Now;
foreach (string line in history)
{
if (!string.IsNullOrEmpty(line)) // skip empty lines, i.e., the last line
{
entries = line.Split(',');
if (nlines == 0)
{
// http://www.epochconverter.com/
dt = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
double secs = double.Parse(entries[0].Remove(0, 1)); // remove character 'a'
dt = dt.AddSeconds(secs);
// GMT to EST
// dstart = TimeZoneInfo.ConvertTimeFromUtc(dt1, TimeZoneInfo.FindSystemTimeZoneById("Eastern Standard Time"));
dt = TimeZoneInfo.ConvertTimeFromUtc(dt, TimeZoneInfo.Local);
dstart = dt;
}
else
{
dstart = dt.AddSeconds(Int32.Parse(entries[0])*br.Interval);
}
nlines++;
// write line to database
Bar bar = new Bar();
bar.Interval = 1; // 1 sec
bar.FullSymbol = br.FullSymbol;
bar.Open = decimal.Parse(entries[4]);
bar.Date = Util.ToIntDate(dstart);
bar.BarOrderInADay = bar.GetOrder(Util.ToIntTime(dstart));
bar.High = decimal.Parse(entries[2]);
bar.Low = decimal.Parse(entries[3]);
bar.Close = decimal.Parse(entries[1]);
bar.Volume = long.Parse(entries[5]);
if (GotHistoricalBarDelegate != null)
GotHistoricalBarDelegate(bar);
}
}
}
}
catch (Exception e)
{
Debug("Error in requesting historical data from Google client.");
Debug(e.Message);
}
}
/// <summary>
/// http://www.marketcalls.in/database/google-realtime-intraday-backfill-data.html
/// http://www.codeproject.com/KB/IP/google_finance_downloader.aspx
/// </summary>
/// <param name="br"></param>
public void RequestHistoricalData(BarRequest br, bool useRTH = false)
{
// Google always returns the most recent data.
// i is interval in seconds, set to 60s, ignore interval
// ignore date; p: period
// q is the symbol (AAPL)
// x is the exchange (NASD)
// sessions is the session requested (ext_hours)
// p is the time period (5d = 5 days), set to 1d, ignore time span
// f is the requested fields (d,c,v,o,h,l)
// df = (cpct)
// auto = (1)
// ts is potentially a time stamp (1324323553 905) or time shift
try
{
using (WebClient client = new WebClient())
{
string google;
if (br.Interval != 86400)
{
google = @"https://www.google.com/finance/getprices?i=" + br.Interval.ToString() + @"&p=1d&f=d,o,h,l,c,v&df=cpct&q=";
}
else // for oneday, today is empty
{
google = @"https://www.google.com/finance/getprices?i=86400&p=2d&f=d,o,h,l,c,v&df=cpct&q=";
}
string[] symbol = br.FullSymbol.Split(' ');
System.IO.Stream data = client.OpenRead(google + symbol[0]);
System.IO.StreamReader read = new System.IO.StreamReader(data);
string[] lines = new string[] { read.ReadToEnd() };
string[] lines2 = lines[0].Split('\n');
// get time zone adjustment
// In line 6, GOOG has time zone offset = -240 which is new york time;
// while SPX has time zone offset = -300, which is chicago time.
// The following find the additional offset relative to local time.
/*
* int localtimezonediffinminutes = (int)Util.GetUtcOffset(DateTime.Today).TotalMinutes; // negative offset
* string stime = lines[6];
* int itime;
* bool btime = Int32.TryParse(stime.Substring(stime.IndexOf('=') + 1), out itime); // negative offset
* int additionaloffset = localtimezonediffinminutes - itime; // (-240) - (-300) = 60 mins
*/
IEnumerable <string> history = lines2.Skip(7); // skip the first 7 lines: header
int nlines = 0; // count of lines
string[] entries;
DateTime dstart = DateTime.Now; // just for initialization
DateTime dt = DateTime.Now;
foreach (string line in history)
{
if (!string.IsNullOrEmpty(line)) // skip empty lines, i.e., the last line
{
entries = line.Split(',');
if (nlines == 0)
{
// http://www.epochconverter.com/
dt = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
double secs = double.Parse(entries[0].Remove(0, 1)); // remove character 'a'
dt = dt.AddSeconds(secs);
// GMT to EST
// dstart = TimeZoneInfo.ConvertTimeFromUtc(dt1, TimeZoneInfo.FindSystemTimeZoneById("Eastern Standard Time"));
dt = TimeZoneInfo.ConvertTimeFromUtc(dt, TimeZoneInfo.Local);
dstart = dt;
}
else
{
dstart = dt.AddSeconds(Int32.Parse(entries[0]) * br.Interval);
}
nlines++;
// write line to database
Bar bar = new Bar();
bar.Interval = 1; // 1 sec
bar.FullSymbol = br.FullSymbol;
bar.Open = decimal.Parse(entries[4]);
bar.Date = Util.ToIntDate(dstart);
bar.BarOrderInADay = bar.GetOrder(Util.ToIntTime(dstart));
bar.High = decimal.Parse(entries[2]);
bar.Low = decimal.Parse(entries[3]);
bar.Close = decimal.Parse(entries[1]);
bar.Volume = long.Parse(entries[5]);
if (GotHistoricalBarDelegate != null)
{
GotHistoricalBarDelegate(bar);
}
}
}
}
}
catch (Exception e)
{
Debug("Error in requesting historical data from Google client.");
Debug(e.Message);
}
}
void _strategy_SendReqHistBar(BarRequest br)
{
_eventAggregator.GetEvent<SendHistDataRequestEvent>().Publish(br);
}
/// <summary>
/// request historical bar
/// </summary>
/// <param name="barsize"> less than 1 day, in seconds </param>
public void RequestHistoricalData(BarRequest br, bool useRTH = false)
//Security sec, DateTime starttime, DateTime endtime, int barsize)
{
Contract contract = SecurityFullNameToContract(br.FullSymbol);
if (contract.SecType == "STK")
contract.IncludeExpired = false;
else
contract.IncludeExpired = true;
int useReguarTradingHour = useRTH ? 1 : 0;
string barSize;
switch (br.Interval)
{
case 1:
barSize = "1 secs"; // not 1 sec
break;
case 5:
barSize = "5 secs";
break;
case 15:
barSize = "15 secs";
break;
case 30:
barSize = "30 secs";
break;
case 60:
barSize = "1 min";
break;
case 120:
barSize = "2 mins";
break;
case 180:
barSize = "3 mins";
break;
case 300:
barSize = "5 mins";
break;
case 900:
barSize = "15 mins";
break;
case 1800:
barSize = "30 mins";
break;
case 3600:
barSize = "1 hour";
break;
case 86400:
barSize = "1 day";
break;
default:
throw new ArgumentOutOfRangeException("Invalid barsize/interval.");
}
DateTime startdatetime = br.StartDateTime;
DateTime enddatetime = br.EndDateTime;
//yyyymmdd hh:mm:ss tmz
String enddatetimestring = enddatetime.ToString("yyyyMMdd HH:mm:ss", CultureInfo.InvariantCulture) + " EST";
TimeSpan duration = enddatetime - startdatetime;
string durationstring;
// Request is less than one day --> request in seconds
if (startdatetime > enddatetime.AddDays(-1))
{
durationstring = duration.TotalSeconds.ToString() + " S";
}
// Request is greater than 1 day and less than 7 days -> Request in days
else if (startdatetime > enddatetime.AddDays(-7))
{
durationstring = duration.TotalDays.ToString() + " D";
}
// Request is greater than 7 days and less than 1 month -> Request in weeks
else if (startdatetime > enddatetime.AddMonths(-1))
{
int numberOfWeeksToRequest = (int)Math.Ceiling(duration.TotalDays / 7.0);
durationstring = numberOfWeeksToRequest.ToString() + " W";
}
else
{
throw new ArgumentOutOfRangeException("Period cannot be bigger than 52 weeks.");
}
_historicalBarRequests.Add(br);
_ibSocket.reqHistoricalData(_historicalBarRequests.Count - 1, contract, enddatetimestring, durationstring,
barSize, "TRADES", useReguarTradingHour, 1);
}
/// <summary>
/// send historical bar request
/// </summary>
public virtual void SendHistoricalBarRequest(BarRequest br)
{
if (SendReqHistBarEvent != null)
SendReqHistBarEvent(br);
else
SendDebug("SendreqHistBar is not supported in this application");
}
private void Start(bool toconnect)
{
// either not connected and ask for connection
if ( (!_isconnected) && (toconnect) )
{
_isconnected = toconnect;
// 0. Initalize sec basket
List<decimal> closeprices = new List<decimal>();
for (int i = 0; i < _basket.Count; i++)
{
closeprices.Add(0.0m);
}
_quoteupdateservice.InitTickerAndPreClose(_basket.Securities.ToArray(), closeprices.ToArray());
// 1. brokerservice and initialize globalidservice
_brokerservice.Start();
_globalidservice.SetInitialStrategyId(0);
// 3. QuoteDispatcherService
_quotedispatcherservice.Start();
// 4. Request yesterday's close in a separate thread, to avoid IB hist request limit
if (!_hasconnected)
{
_hasconnected = true;
Task.Factory.StartNew(() =>
{
string broker = _configmanagerservice.DefaultBroker;
for (int i = 0; i < _basket.Count; i++)
{
if (broker == "IB")
Thread.Sleep(10000); // wait ten sec
else
Thread.Sleep(1000); // wait one sec
BarRequest br = new BarRequest(_basket[i], 86400, Util.ToIntDate(_preday), 0, Util.ToIntDate(_today), 0, broker);
_brokerservice.ReqHistoricalData(br);
}
});
}
}
// or connected and ask for disconnection
else if ((_isconnected) && (!toconnect))
{
_isconnected = toconnect;
Stop();
}
else
{
_logger.Log("connection/disconnection order messed up", Category.Info, Priority.High);
}
}
/// <summary>
/// request historical bar
/// </summary>
/// <param name="barsize"> less than 1 day, in seconds </param>
public void RequestHistoricalData(BarRequest br, bool useRTH = false)
//Security sec, DateTime starttime, DateTime endtime, int barsize)
{
Contract contract = SecurityFullNameToContract(br.FullSymbol);
if (contract.SecType == "STK")
{
contract.IncludeExpired = false;
}
else
{
contract.IncludeExpired = true;
}
int useReguarTradingHour = useRTH ? 1 : 0;
string barSize;
switch (br.Interval)
{
case 1:
barSize = "1 secs"; // not 1 sec
break;
case 5:
barSize = "5 secs";
break;
case 15:
barSize = "15 secs";
break;
case 30:
barSize = "30 secs";
break;
case 60:
barSize = "1 min";
break;
case 120:
barSize = "2 mins";
break;
case 180:
barSize = "3 mins";
break;
case 300:
barSize = "5 mins";
break;
case 900:
barSize = "15 mins";
break;
case 1800:
barSize = "30 mins";
break;
case 3600:
barSize = "1 hour";
break;
case 86400:
barSize = "1 day";
break;
default:
throw new ArgumentOutOfRangeException("Invalid barsize/interval.");
}
DateTime startdatetime = br.StartDateTime;
DateTime enddatetime = br.EndDateTime;
//yyyymmdd hh:mm:ss tmz
String enddatetimestring = enddatetime.ToString("yyyyMMdd HH:mm:ss", CultureInfo.InvariantCulture) + " EST";
TimeSpan duration = enddatetime - startdatetime;
string durationstring;
// Request is less than one day --> request in seconds
if (startdatetime > enddatetime.AddDays(-1))
{
durationstring = duration.TotalSeconds.ToString() + " S";
}
// Request is greater than 1 day and less than 7 days -> Request in days
else if (startdatetime > enddatetime.AddDays(-7))
{
durationstring = duration.TotalDays.ToString() + " D";
}
// Request is greater than 7 days and less than 1 month -> Request in weeks
else if (startdatetime > enddatetime.AddMonths(-1))
{
int numberOfWeeksToRequest = (int)Math.Ceiling(duration.TotalDays / 7.0);
durationstring = numberOfWeeksToRequest.ToString() + " W";
}
else
{
throw new ArgumentOutOfRangeException("Period cannot be bigger than 52 weeks.");
}
_historicalBarRequests.Add(br);
_ibSocket.reqHistoricalData(_historicalBarRequests.Count - 1, contract, enddatetimestring, durationstring,
barSize, "TRADES", useReguarTradingHour, 1);
}
