void RequestBars(BarRequest br)
{
string command;
long id = _idt.AssignId;
if (br.Interval == (int)BarInterval.Day)
{
command = String.Format("HDT,{0},{1},,1,{2}\r\n", br.Symbol, br.StartDateTime.ToLongDateString(), br.EndDateTime.ToLongDateString(), id);
}
else
{
command = String.Format("HIT,{0},{1},{2} {3},{4} {5},,000000,235959,1,{6}\r\n", br.Symbol, br.Interval, br.StartDateTime.ToString("yyyyMMdd"), br.StartDateTime.ToString("HHmmss"), br.EndDateTime.ToString("yyyyMMdd"), br.EndDateTime.ToString("HHmmss"), id);
}
reqid2req.Add(id, br);
// we form a watch command in the form of wSYMBOL\r\n
byte[] watchCommand = new byte[command.Length];
watchCommand = Encoding.ASCII.GetBytes(command);
try
{
m_hist.Send(watchCommand, watchCommand.Length, SocketFlags.None);
debug("Requested historical bars for: " + br.Symbol);
}
catch (Exception ex)
{
debug("Exception sending barrequest: " + br.ToString());
debug(ex.Message + ex.StackTrace);
}
}
public void CallMethodBar()
{
MockRepository mocks = new MockRepository();
BarRequest barRequest = BarRequest.CreateBuilder().Build();
BarResponse barResponse = BarResponse.CreateBuilder().Build();
IRpcController controller = mocks.StrictMock <IRpcController>();
bool barCalled = false;
TestService service = new TestServiceImpl(null, (request, responseAction) => {
Assert.AreSame(barRequest, request);
barCalled = true;
responseAction(barResponse);
}, controller);
bool doneHandlerCalled = false;
Action <IMessage> doneHandler = (response => {
Assert.AreSame(barResponse, response);
doneHandlerCalled = true;
});
using (mocks.Record()) {
// No mock interactions to record
}
service.CallMethod(BarDescriptor, controller, barRequest, doneHandler);
Assert.IsTrue(doneHandlerCalled);
Assert.IsTrue(barCalled);
mocks.VerifyAll();
}
public void ReqHistoricalData(BarRequest br)
{
if (_client != null)
{
_client.RequestHistoricalData(br, true);
}
}
public void CallMethodBar()
{
BarRequest barRequest = BarRequest.CreateBuilder().Build();
BarResponse barResponse = BarResponse.CreateBuilder().Build();
IRpcController controller = new RpcTestController();
bool barCalled = false;
TestGenericService service = new TestServiceImpl(null, (request, responseAction) =>
{
Assert.AreSame(barRequest, request);
barCalled = true;
responseAction(barResponse);
}, controller);
bool doneHandlerCalled = false;
Action <IMessage> doneHandler = (response =>
{
Assert.AreSame(barResponse, response);
doneHandlerCalled = true;
});
service.CallMethod(BarDescriptor, controller, barRequest, doneHandler);
Assert.IsTrue(doneHandlerCalled);
Assert.IsTrue(barCalled);
}
public void BarTest()
{
// verify no bars
Assert.AreEqual(0, barcount, "bars received early");
// request bars
c.TLSend(MessageTypes.BARREQUEST, BarRequest.BuildBarRequest(SYM, BarInterval.FiveMin));
// verify we received it
Assert.Greater(barcount, 30, "no bar received");
}
public BarResponse Execute(BarRequest request)
{
var result = _taskRunner.Run();
return(new BarResponse()
{
Code = result ? "Ok" : "Error"
});
}
private void Start(bool toconnect)
{
// either not connected and ask for connection
if ((!_isconnected) && (toconnect))
{
_isconnected = toconnect;
// 0. Initalize sec basket
List <decimal> closeprices = new List <decimal>();
for (int i = 0; i < _basket.Count; i++)
{
closeprices.Add(0.0m);
}
_quoteupdateservice.InitTickerAndPreClose(_basket.Securities.ToArray(), closeprices.ToArray());
// 1. brokerservice and initialize globalidservice
_brokerservice.Start();
_globalidservice.SetInitialStrategyId(0);
// 3. QuoteDispatcherService
_quotedispatcherservice.Start();
// 4. Request yesterday's close in a separate thread, to avoid IB hist request limit
if (!_hasconnected)
{
_hasconnected = true;
Task.Factory.StartNew(() =>
{
string broker = _configmanagerservice.DefaultBroker;
for (int i = 0; i < _basket.Count; i++)
{
if (broker == "IB")
{
Thread.Sleep(10000); // wait ten sec
}
else
{
Thread.Sleep(1000); // wait one sec
}
BarRequest br = new BarRequest(_basket[i], 86400, Util.ToIntDate(_preday), 0, Util.ToIntDate(_today), 0, broker);
_brokerservice.ReqHistoricalData(br);
}
});
}
}
// or connected and ask for disconnection
else if ((_isconnected) && (!toconnect))
{
_isconnected = toconnect;
Stop();
}
else
{
_logger.Log("connection/disconnection order messed up", Category.Info, Priority.High);
}
}
public async Task <BarResponse> Delete(BarRequest request)
{
// do some await...
return(new BarResponse
{
StringValue = request.StringValue,
IntValue = request.IntValue
});
}
public override void GotTick(Tick k)
{
// if we don't have bar data, request historical data
if (_blt[k.symbol].Count == 0)
{
D(k.symbol + " no bars found, requesting...");
sendmessage(MessageTypes.BARREQUEST, BarRequest.BuildBarRequest(k.symbol, BarInterval.Hour));
}
D(k.symbol + " bar count: " + _blt[k.symbol].Count);
// update whatever data we have with ticks
_blt.newTick(k);
}
long IQFeedHelper_newUnknownRequest(MessageTypes t, string msg)
{
switch (t)
{
case MessageTypes.DAYHIGH:
{
// get index for request
int idx = _highs.getindex(msg);
// ignore if no index
if (idx == GenericTracker.UNKNOWN)
{
return(0);
}
decimal v = _highs[idx];
// ensure we have a high
if (v == decimal.MinValue)
{
return(0);
}
return(WMUtil.pack(v));
}
case MessageTypes.DAYLOW:
{
// get index for request
int idx = _highs.getindex(msg);
// ignore if no index
if (idx == GenericTracker.UNKNOWN)
{
return(0);
}
decimal v = _highs[idx];
// ensure we have a high
if (v == decimal.MaxValue)
{
return(0);
}
return(WMUtil.pack(v));
}
case MessageTypes.BARREQUEST:
{
BarRequest br = BarImpl.ParseBarRequest(msg);
RequestBars(br);
return((long)MessageTypes.OK);
}
}
return((long)MessageTypes.FEATURE_NOT_IMPLEMENTED);
}
void processhistory(int lHandle, BarRequest br)
{
int numbars = esig.get_GetNumBars(lHandle);
if (numbars == 0)
{
verb("no bars available for reqhandle: " + lHandle);
return;
}
numbars *= -1;
for (int i = numbars; i <= 0; i++)
{
try
{
BarData bd = esig.get_GetBar(lHandle, i);
if (VerboseDebugging)
{
verb(br.Symbol + " " + bd.dtTime.ToString() + " " + bd.dOpen + " " + bd.dHigh + " " + bd.dLow + " " + bd.dClose + " " + bd.dVolume);
}
Bar b = new BarImpl((decimal)bd.dOpen, (decimal)bd.dHigh, (decimal)bd.dLow, (decimal)bd.dClose, (long)bd.dVolume, Util.ToTLDate(bd.dtTime), Util.ToTLTime(bd.dtTime), br.Symbol, br.Interval);
string msg = BarImpl.Serialize(b);
if (!b.isValid && !AllowSendInvalidBars)
{
debug("Not sending invalid bar: " + b.ToString() + " raw: " + msg);
continue;
}
tl.TLSend(msg, MessageTypes.BARRESPONSE, br.Client);
}
catch (Exception ex)
{
verb("error obtaining bar: " + i + " from: " + lHandle);
verb(ex.Message + ex.StackTrace);
}
}
if (ReleaseBarHistoryAfteRequest)
{
try
{
esig.ReleaseHistory(lHandle);
_barhandle2barrequest.Remove(lHandle);
}
catch { }
}
}
long s_newUnknownRequest(MessageTypes t, string msg)
{
if (t == MessageTypes.BARREQUEST)
{
var br = BarRequest.Deserialize(msg);
var MAXBARPERREQUEST = br.BarsBack;
if (br.isValid)
{
for (int i = 0; i < MAXBARPERREQUEST; i++)
{
var bar = GetRandomBar(br.symbol, br.Interval);
s.TLSend(BarImpl.Serialize(bar), MessageTypes.BARRESPONSE, c.Name);
}
}
}
if (t == MessageTypes.LOOPBACKSERVER)
{
return(SPECIAL);
}
return(0);
}
long EsignalServer_newUnknownRequest(MessageTypes t, string msg)
{
switch (t)
{
case MessageTypes.BARREQUEST:
{
verb("got barrequest: " + msg);
try
{
BarRequest br = BarImpl.ParseBarRequest(msg);
_barrequests.Write(br);
}
catch (Exception ex)
{
debug("error parsing bar request: " + msg);
debug(ex.Message + ex.StackTrace);
}
return(0);
}
}
return((long)MessageTypes.FEATURE_NOT_IMPLEMENTED);
}
void client_gotUnknownMessage(MessageTypes type, long source, long dest, long msgid, string request, ref string response)
{
if (type == MessageTypes.BARRESPONSE)
{
lastbarraw = response;
var b = BarImpl.Deserialize(response);
var label = b.Symbol + b.Interval.ToString() + b.CustomInterval.ToString();
var symintidx = symintcount.getindex(label);
if (symintidx < 0)
{
symintidx = symintcount.addindex(label, new List <Bar>());
g.d("got new symbol/interval: " + b.Symbol + " " + b.Interval + b.CustomInterval + " bar:" + b.ToString());
}
symintcount[symintidx].Add(b);
if (!recvsyms.Contains(b.Symbol))
{
recvsyms.Add(b.Symbol);
}
recvbarcount++;
}
else if (type == MessageTypes.BARRESPONSE_FINAL)
{
var br = BarRequest.Deserialize(response);
reccomplete++;
g.d("completed: " + br.symbol + " " + br.Interval + " " + br.ID);
}
else
{
g.d("got unknown message: " + type + request + response);
}
var ok = mt.GotMessage(type, source, dest, msgid, request, ref response);
if (!ok)
{
g.d("error processing: " + type + " request: " + request + " response: " + response);
}
msgok &= ok;
}
public void BarsBackSerializeDeSerialize()
{
// build request
var br = new BarRequest(sym, 300, 300, 100, string.Empty);
// verify it
Assert.IsTrue(br.isValid, "original not valid");
Assert.AreEqual(300, br.Interval, "interval missing");
Assert.AreEqual(sym, br.symbol, "symbol missing");
Assert.AreEqual(100, br.BarsBack, "bars back missing");
// serialize request
var msg = br.Serialize();
// deserialize
var copy = BarRequest.Deserialize(msg);
// verify valid
Assert.IsTrue(copy.isValid, "copy not valid");
// verify same
Assert.AreEqual(br.symbol, copy.symbol, "symbol mismatch");
Assert.AreEqual(br.BarsBack, copy.BarsBack, "symbol mismatch");
Assert.AreEqual(br.Interval, copy.Interval, "symbol mismatch");
}
public override void Bar(IRpcController controller, BarRequest request, Action <BarResponse> done)
{
Assert.AreSame(expectedController, controller);
barHandler(request, done);
}
void bw_DoWork(object sender, DoWorkEventArgs e)
{
if (!_valid)
{
return;
}
while (_go)
{
try
{
if (qc > qr)
{
// get requested symbols
string[] syms = _tmpregister.Split(',');
// go through each one
foreach (string sym in syms)
{
// if we don't have subscription already
if (!contains(sym))
{
// add it to list
_mb.Add(sym);
// request subscription
esig.RequestSymbol(sym, 1);
}
}
if (ReleaseDeadSymbols)
{
// clone requested basket
Basket newbasket = new BasketImpl(syms);
// clone existing basket as deadbasket
Basket deadbasket = new BasketImpl(_mb);
// existing - new = deadsymbols
deadbasket.Remove(newbasket);
// release dead symbols
string symsreleased = string.Empty;
foreach (Security dead in deadbasket)
{
try
{
esig.ReleaseSymbol(dead.Symbol);
symsreleased += dead.Symbol + " ";
}
catch { }
}
if (symsreleased != string.Empty)
{
verb("released unused symbols: " + symsreleased);
}
}
qr = qc;
}
while (_barrequests.hasItems)
{
BarRequest br = new BarRequest();
try
{
br = _barrequests.Read();
BarInterval bi = (BarInterval)br.Interval;
string interval = string.Empty;
int barsback = DefaultBarsBack;
if (bi == BarInterval.CustomTicks)
{
interval = br.CustomInterval + "T";
}
else if (bi == BarInterval.CustomTime)
{
interval = br.CustomInterval + "S";
}
else if (bi == BarInterval.CustomVol)
{
interval = br.CustomInterval + "V";
}
else
{
if (br.Interval == (int)BarInterval.Day)
{
interval = "D";
}
else
{
interval = (br.Interval / 60).ToString();
}
barsback = BarImpl.BarsBackFromDate(bi, br.StartDateTime, br.EndDateTime);
}
int alldata = BarRequestsGetAllData ? -1 : 0;
int hnd = esig.get_RequestHistory(br.Symbol, interval, (bi == BarInterval.Day) ? barType.btDAYS : barType.btBARS, barsback, alldata, alldata);
verb("requested bar data for " + br.Symbol + " on: " + br.Interval.ToString() + " " + br.CustomInterval.ToString() + " reqhandle: " + hnd);
// cache request
if (!_barhandle2barrequest.ContainsKey(hnd))
{
_barhandle2barrequest.Add(hnd, br);
}
else
{
verb("already had bar request: " + hnd + " " + _barhandle2barrequest[hnd].ToString());
}
if (esig.get_IsHistoryReady(hnd) != 0)
{
processhistory(hnd, br);
}
}
catch (Exception ex)
{
debug("error on historical bar request: " + br.ToString());
debug(ex.Message + ex.StackTrace);
}
}
}
catch (Exception ex)
{
if (GotDebug != null)
{
GotDebug(DebugImpl.Create(ex.Message + ex.StackTrace, DebugLevel.Debug));
}
}
if (e.Cancel || !_go)
{
break;
}
System.Threading.Thread.Sleep(WaitBetweenEvents);
}
}
public override void Bar(IRpcController controller, BarRequest request, Action<BarResponse> done)
{
Assert.AreSame(expectedController, controller);
barHandler(request, done);
}
void _strategy_SendReqHistBar(BarRequest br)
{
_eventAggregator.GetEvent <SendHistDataRequestEvent>().Publish(br);
}
public void HistoricalBarRequest()
{
Stopwatch sw = new Stopwatch();
// verify we are ready
Assert.IsTrue(isconok, "connection not ready");
// prep test case data
msgok &= true;
recvbarcount = 0;
reccomplete = 0;
recvsyms.Clear();
symintcount.Clear();
lastbarraw = string.Empty;
var syms = new string[] { "IBM", "GOOG", "QQQ" };
//var syms = new string[] { "IBM"};
var ints = new int[] { (int)BarInterval.Day, (int)BarInterval.FiveMin, (int)BarInterval.Hour };
var barsback = new int[] { 100, 100, 20 };
// verify we have no data
foreach (var sym in syms)
{
foreach (var bi in ints)
{
Assert.AreEqual(0, barcount(sym, bi), "had bar data: " + sym + " " + bi);
}
}
// request a bunch of data
sw.Start();
foreach (var sym in syms)
{
for (int i = 0; i < ints.Length; i++)
{
var bint = ints[i];
var bintsize = bint;
var bb = barsback[i];
// build request
if ((bint < (int)BarInterval.Day) && (bint > 0))
{
bint = (int)BarInterval.CustomTime;
}
var br = BarRequest.BuildBarRequestBarsBack(sym, bb, bint, bintsize, client.Name);
var ok = client.TLSend(MessageTypes.BARREQUEST, br);
Assert.AreEqual((int)MessageTypes.OK, ok, "failed bar request: " + br);
}
}
// wait for it
g.d("waiting for bars...");
int att = 0;
bool stillwaiting = true;
var expectbars = Calc.Sum(barsback) * syms.Length;
var expectcomplete = (syms.Length * barsback.Length);
#if DEBUG
const int waitseconds = 60;
#else
const int waitseconds = 15;
#endif
while (stillwaiting && (att++ < (waitseconds * 200)))
{
Util.sleep(5);
System.Windows.Forms.Application.DoEvents();
stillwaiting = (recvbarcount < expectbars) && (reccomplete < expectcomplete);
}
if (!stillwaiting)
{
sw.Stop();
var elap = sw.Elapsed.TotalSeconds;
var responsepersec = elap == 0 ? 0 : expectcomplete / elap;
var barssec = recvbarcount == 0 ? 0 : recvbarcount / elap;
g.d("received expected # of bars: " + recvbarcount + "/" + expectbars + "\t" + "rate (req/sec): " + responsepersec.ToString("N2") + " bars/sec: " + barssec.ToString("N2"));
}
else
{
g.d("wait timeout, got " + recvbarcount + "/" + expectbars);
}
//Assert.GreaterOrEqual(recvbarcount, expect, "did not receive enough bars");
Assert.IsTrue(msgok, "errors processing messages");
Assert.AreEqual(syms.Length, recvsyms.Count, "unexpected bar symbols: " + Util.join(recvsyms) + Environment.NewLine + lastbar.ToString());
// verify we got it all
foreach (var sym in syms)
{
for (int i = 0; i < ints.Length; i++)
{
var bi = ints[i];
var bisize = bi;
if ((bi < (int)BarInterval.Day) && (bi > 0))
{
bi = (int)BarInterval.CustomTime;
}
var count = barsback[i];
var symcount = barcount(sym, bi);
var bars = symintcount[sym + bi + bisize];
if (g.ta(bars.Count == count, "error: " + sym + bi + " expected: " + count + " got: " + bars.Count + " raw bars: " + string.Join(Environment.NewLine, bars)))
{
g.d(sym + " tracking " + bi + " bars, count: " + bars.Count);
}
else
{
Assert.AreEqual(count, bars.Count, "wrong bar count: " + sym + bi);
}
}
}
}
/// <summary>
/// Arguments are GOOG/IB, rundate, basketfile, datapath
/// </summary>
static void Main(string[] args)
{
#region parameters
string _client = "IB"; // IB or GOOG
IClient _iclient;
string _enddate = DateTime.Today.ToString("yyyyMMdd");
string _basketfile = @"c:\QuantTrading\Config\basket.xml";
string _datapath = @"c:\QuantTrading\HistData\";
Basket _basket;
if (args.Length > 0)
{
_client = args[0];
}
if (args.Length > 1)
{
_enddate = args[1];
}
if (args.Length > 2)
{
_basketfile = args[2];
}
if (args.Length > 3)
{
_datapath = args[3];
}
_datapath = _datapath + _client + @"\";
_basket = Basket.DeserializeFromXML(_basketfile);
#endregion
_sec2bars = new Dictionary <string, List <string> >(_basket.Count);
Dictionary <string, string> _outfiles = new Dictionary <string, string>(_basket.Count);
Dictionary <string, long> _totalBars = new Dictionary <string, long>(_basket.Count);
_processedBars = new Dictionary <string, long>(_basket.Count);
DateTime _startTime = DateTime.SpecifyKind(DateTime.ParseExact(_enddate, "yyyyMMdd", null), DateTimeKind.Local);
DateTime _endTime = _startTime + new TimeSpan(1, 0, 0, 0); // one day later
double sec = _endTime.Subtract(_startTime).TotalSeconds; // should be 24 hours or 86,400 secs
foreach (string s in _basket.Securities)
{
// set up bar counts
_totalBars.Add(s, (long)sec); // one bar is one sec; if one minute, divide it by 60
_processedBars.Add(s, 0); // initialize processed to 0
// set up out filenames
string filename = _datapath + s + "_" + _startTime.Date.ToString("yyyyMMdd") + ".csv";
_outfiles.Add(s, filename);
List <string> lines = new List <string>(90000); // set something greater than 86,4000
lines.Add("DateTime,Open,High,Low,Close,Volume");
_sec2bars.Add(s, lines);
}
if (_client == "IB")
{
_iclient = new IBClient(); // currently it uses default ip:port
_iclient.SendDebugEventDelegate += _iclient_SendDebugEventDelegate;
_iclient.GotHistoricalBarDelegate += _iclient_GotHistoricalBarDelegate;
_iclient.Connect();
long totalRequests = (long)sec / 1800; // one request is 30 min, totalRequests = 48
TimeSpan thirtyMin = new TimeSpan(0, 30, 0);
foreach (string sym in _basket.Securities)
{
DateTime s = _startTime;
DateTime t = _startTime + thirtyMin;
Console.WriteLine("Requesting historical bars for :" + sym);
for (int i = 0; i < totalRequests; i++)
{
Console.WriteLine("Request #: " + (i + 1).ToString() + "/" + totalRequests);
// 1 = 1 second
BarRequest br = new BarRequest(sym, 1, Util.ToIntDate(s.Date), Util.ToIntTime(s.Hour, s.Minute, s.Second),
Util.ToIntDate(t.Date), Util.ToIntTime(t.Hour, t.Minute, t.Second), _client);
_iclient.RequestHistoricalData(br, true);
// Do not make more than 60 historical data requests in any ten-minute period.
// If I have 10 names, each can only make 6 requests in ten minute;
// I use 5 minute for a pause; Then 24 hours takes 120 min or 1.5hour
// Thread.Sleep(new TimeSpan(0, 5, 0));
// wait 10 secs
Thread.Sleep(10000);
s += thirtyMin;
t += thirtyMin;
}
}
}
else if (_client == "GOOG")
{
_iclient = new GoogleClient(1);
_iclient.SendDebugEventDelegate += _iclient_SendDebugEventDelegate;
_iclient.GotHistoricalBarDelegate += _iclient_GotHistoricalBarDelegate;
foreach (string sym in _basket.Securities)
{
Console.WriteLine("Requesting historical bars for :" + sym);
BarRequest br = new BarRequest(sym, 60, Util.ToIntDate(DateTime.Today), Util.ToIntTime(DateTime.Today),
Util.ToIntDate(DateTime.Today), Util.ToIntTime(DateTime.Today), _client);
_iclient.RequestHistoricalData(br);
}
}
// write to files
Console.WriteLine("Wait three minutes for bars being processed.....");
Thread.Sleep(new TimeSpan(0, 3, 0)); // wait three minutes for all hist bar to be processed.
foreach (string s in _basket.Securities)
{
List <string> noDups = _sec2bars[s].Distinct().ToList();
//_sec2bars[s].Insert(0, _processedBars[s].ToString());
File.WriteAllLines(_outfiles[s], noDups);
}
}
public BarResponse Bar(BarRequest request)
{
return(new BarResponse {
BarResponseValue = $"Bar{request.BarRequestValue}"
});
}
public override bool Equals(object obj)
{
BarRequest barRequest = obj as BarRequest;
return(barRequest != null && barRequest.BarType == this.BarType && barRequest.BarSize == this.BarSize);
}
public void Process(BarRequest request)
{
}
