public override void processTick(Symbol symbol, Tick tick)
{
if (!partialBars.ContainsKey(symbol))
{
partialBars[symbol] = new Bar(tick.price, tick.price, tick.price, tick.price, tick.time);
}
partialBars[symbol] = partialBars[symbol].update(tick);
var processThisTick = system.runOnNewTick() || !lastTickProcessed.ContainsKey(symbol) || lastTickProcessed[symbol] <= tick.time.AddMinutes(-1);
try {
if (processThisTick)
{
manager.newTick();
if (!isLive)
{
isLive = true;
manager.goLive();
system.goLiveDO_NOT_CALL_EXCEPT_FROM_BRIDGE();
monitor().goLive(system);
}
var partialBar = updateBarSpud(symbol);
if (Systematic.isLoggingTicks())
{
LogC.info("processing tick " + symbol + ": bar=" + partialBar + ", tick=" + tick);
}
system.processTickDO_NOT_CALL_EXCEPT_FROM_BRIDGE(symbol, partialBar, tick);
manager.recalculate();
lastTickProcessed[symbol] = tick.time;
}
monitor().tickProcessed(symbol, tick);
} catch (Exception e) {
LogC.info("exception caught in processTick: ", e);
throw;
}
}
void addRow(LiveSystem system)
{
var row = table.NewRow();
systems.Add(row, system);
insertUnknownRow(row, system);
plans[system] = new List <Plan>();
setHeartbeatStatus(row, system, SystemStatus.UNKNOWN);
setTickStatus(row, system, SystemStatus.UNKNOWN);
tickTimers[system] = statusFades(status => setTickStatus(row, system, status));
heartbeatTimers[system] = statusFades(status => setHeartbeatStatus(row, system, status));
lastTickProcessed[system] = DateTime.MinValue;
subscribeSystemHeartbeat(row, system);
var symbols = convert(list <JMarket>(system.markets()), m => new Symbol(m));
each(symbols, symbol => {
try {
symbol.subscribe(bar => onTickPublished(row, system, bar.time));
} catch (Exception ex) {
LogC.err("exception caught subscribing to tick data for " + symbol + ", " + system, ex);
gui.alertUser("exception caught susbcribing to data for " + symbol + ", " + system + ".\nSkipping... see log for details.");
}
});
}
public static void Main(string[] args)
{
ThreadPool.SetMaxThreads(50, 50);
STORunner stoRunner = null;
var arguments = Arguments.arguments(args, jStrings("numInstances", "id", "start", "end", "maxRunHours", "DO_NOT_USE_THIS_OPTION_UNLESS_YOU_ARE_JERIC"));
var numInstances = arguments.integer("numInstances");
var systemId = arguments.integer("id");
var start = arguments.get("start", 1);
var end = arguments.containsKey("end") ? arguments.integer("end") : (int?)null;
var maxRunMillis = numInstances == 0 ? 0 : (long)(arguments.numeric("maxRunHours") * 3600 * 1000);
var noKill = arguments.get("DO_NOT_USE_THIS_OPTION_UNLESS_YOU_ARE_JERIC", false);
try {
stoRunner = new STORunner(numInstances, systemId, start, end, noKill);
stoRunner.createDirectories();
var dbServer = stoRunner.startDbServer();
stoRunner.initialize(maxRunMillis);
stoRunner.enqueueRequests();
stoRunner.processResults();
dbServer.stopServer();
stoRunner.copyMetrics();
} catch (Exception e) {
if (stoRunner != null)
{
stoRunner.killRun(LogC.errMessage("caught exception in Main: ", e));
}
}
Environment.Exit(0);
}
public void logAndAlert(string s, Exception e)
{
var message = LogC.errMessage(s, e);
LogC.err(message);
alertUser(s + ": examine " + LogC.errFile() + "\n" + message);
}
public virtual void placeOrder(Order order)
{
var place = (forecastMode ? "forecast" : "place");
LogC.info(place + " order: " + order);
order.placedBy(this, bars[order.symbol][0].time);
var symbol = order.symbol;
Bomb.unless(isActive(), () => "cannot " + place + " an order in a deactivated symbol: " + order);
if (inClose)
{
order.placedOnClose();
}
if (forecastMode)
{
if (order.canFill(Bomb.missing(bars, symbol)[0].close, true))
{
// bug - if two exits are placed in onClose, they can both be "filled" for the
// forecast, which is not what would happen in the actual onClose
monitor().orderForecast(this, order);
}
}
else
{
Bomb.when(exists(orders(symbol), order.descriptionAndSymbolMatch), () =>
"symbol+description must be unique for outstanding orders. Tried placing: " +
order + "\nin:\n" + toShortString(orders(symbol)));
monitor().orderPlaced(order);
orders_.get(symbol).Add(order);
}
}
public void testCanFailToRunCompleteSimulation()
{
Db.reallyRollback(); // open transaction system details insert (on unrelated row, jeff hates sqlserver) from DbTestCase causes test failure.
var gui = new FakeResearchGUI();
gui.setMarkets(O.list("RE.TEST.TY.1C"));
var parameters = new Parameters {
{ "ATRLen", 10 },
{ "ATRlong", 100 },
{ "BreakDays", 30 },
{ "FixEquity", 1 },
{ "InitEquity", 6000000 },
{ "LeadBars", 50 },
{ "MaxPyramid", 1 },
{ "Risk", 0.02 },
{ "nATR", 2 },
{ "upATR", 2 },
{ "systemId", 39 } // ok?
};
gui.setParameters(parameters);
gui.setStartDate(date("2001/01/01"));
gui.setEndDate(date("2001/04/01"));
gui.runSystem();
O.wait(() => gui.runButtonEnabled == false);
O.wait(100, 100, gui.runComplete);
IsTrue(gui.runButtonEnabled);
gui.hasMessage("failed");
LogC.info("done");
}
public void testSymbolValueLive()
{
Log.setFile(@"C:\logs\jefftest");
SystemTimeSeriesTable.SYSTEM_TS.insert("testone", "ASCII", "ActiveMQ", "somedangtopic");
AsciiTable.SYSTEM_ASCII.insert("testone", @"\\nysrv37\share\Tools\RightEdge\TransitionTest\TY1C.full.csv", true, 1);
var manager = new SpudManager();
var barSpud = new BarSpud(manager);
var barSpud2 = new BarSpud(manager);
var symbol = new Symbol("testone");
var spud = symbol.doubles(barSpud);
var spud2 = new Symbol("RE.TEST.TY.1C").doubles(barSpud2);
manager.newBar();
barSpud.set(new Bar(1, 3, 1, 2, O.date("2008/08/06")));
barSpud2.set(new Bar(1, 3, 1, 2, O.date("2008/08/06")));
manager.newTick();
spud.doSubscribe();
barSpud.lastTickedAt(O.date("2008/08/06 11:22:34"));
spud2.doSubscribe();
IsTrue(spud2.isDirty());
symbol.javaSymbol().jmsLive().topic().send("value=97.1|timestamp=2008/08/06 11:22:33|MSTimestamp=2008/06/05 13:10:08");
Bombs(() => LogC.info("" + spud[0]), "stale");
symbol.javaSymbol().jmsLive().topic().send("value=97.2|timestamp=2008/08/06 11:22:34|MSTimestamp=2008/06/05 13:10:08");
O.wait(() => spud[0] == 97.2);
barSpud.lastTickedAt(O.date("2008/08/06 11:22:35"));
spud.allowStaleTicks();
AreEqual(97.2, spud[0]);
AreEqual(116.140625, spud2[0]);
}
public STORunner(int numInstances, int systemId, int start, int?end, bool noKill)
{
this.numInstances = numInstances;
this.systemId = systemId;
this.start = start;
this.noKill = noKill;
details = SystemDetailsTable.DETAILS.details(systemId);
this.end = end.HasValue ? end.Value : details.lastRunNumber();
writer = new STOMetricsWriter(details, false);
STO.populateSymbolsPortfolios(details, out symbols, out portfolios);
runner = new EC2Runner(systemId + "-" + Dates.yyyyMmDdHhMmSsNoSeparator(Dates.now()));
LogC.info("checking for completed runs");
existingRuns = completedRuns();
var allRunsComplete = true;
for (var i = this.start; i <= this.end; i++)
{
if (!existingRuns.Contains(i))
{
allRunsComplete = false;
break;
}
}
if (allRunsComplete)
{
this.numInstances = 0;
}
unhandledExceptionProcessor = ((sender, args) => killRun(LogC.errMessage("unhandled exception", (Exception)args.ExceptionObject)));
}
void debug(string message)
{
if (!toEnd && isAverage && toEnd)
{
LogC.debug(message);
}
}
public MainWindow(string[] unused) : base("NORAD")
{
var arguments = Arguments.arguments(unused, o.jStrings("live", "settings"));
System.Environment.SetEnvironmentVariable("RE_TEST_MODE", "TRUE");
LogC.useJavaLog = true;
LogC.info("starting MainWindow.");
var workbench = new WorkbenchPanel();
dockManager.Content = workbench;
var path = Systematic.mainDir().file("dotNET/Gui/Resources/target.ico").path();
Icon = BitmapFrame.Create(new FileStream(path, FileMode.Open, FileAccess.Read));
if (arguments.containsKey("settings"))
{
if (arguments.get("live", false))
{
workbench.liveButton.doClick();
}
else
{
workbench.runButton.doClick();
}
}
}
public static Simulator run(int systemId, IEnumerable <Symbol> symbols, IEnumerable <QPortfolio> portfolios, int runNumber, BarLoader data, bool dumpParamsOnly, bool shutdown)
{
var parameters = new Parameters {
{ "systemId", systemId },
{ "RunMode", (double)RunMode.STO },
{ "RunNumber", runNumber }
};
try {
parameters.runNumber();
LogC.err("" + parameters);
if (dumpParamsOnly)
{
return(null);
}
var simulator = new Simulator(new SystemArguments(symbols, portfolios, parameters), data, "QUEDGE");
simulator.processBars();
if (shutdown)
{
simulator.shutdown();
}
else
{
var metrics = simulator.metrics();
LogC.verbose(() => "skipping shutdown based on noShutdown parameter.");
LogC.verbose(() => toShortString(metrics["ALL"]));
}
return(simulator);
} catch (Exception e) {
throw Bomb.toss("\nFailed in run " + runNumber + "\nUsing:\n" + parameters + "\n", e);
}
}
public void load()
{
if (loaded)
{
return;
}
loaded = true;
LogC.verbose(() => "loading parameters");
var extras = new Dictionary <string, string>();
if (!has("RunMode") || isRE() || isCloudSTO())
{
return;
}
if (isSTO())
{
extras = stoParameters();
}
else if (isLive())
{
extras = liveParameters();
}
else
{
Bomb.toss("unexpected run mode " + get <int>("RunMode"));
}
extras["systemId"] = getDirect <string>("systemId");
data.Clear();
eachKey(extras, delegate(String k) { data[k] = extras[k]; });
LogC.verbose(() => "loaded parameters:\n" + this);
}
void process(Message message, Dictionary <int, bool> completed, int totalRuns)
{
var response = (STOResponse)message.@object();
runner.received(response);
var completedRun = response.runNumber();
var runTimeMillis = response.runTimeMillis();
var completionTime = response.completedAt();
var isLocal = response.instanceId().Equals("LOCAL");
message.delete();
// Letting message get GC'd, rather than sitting in the thread pool really helps with the Java heap space.
queueWorkItem(() => {
lock (completed) {
if (completed.ContainsKey(completedRun))
{
return;
}
completed[completedRun] = true;
if (!isLocal)
{
completionTimes.Add(date(completionTime));
updateRunsPerMinute(runTimeMillis);
}
}
LogC.info("completed run " + completedRun + paren(completed.Count + "/" + totalRuns));
new Topic(CloudMonitor.progressTopic(systemId)).send(new Dictionary <string, object> {
{ "RunsComplete", completed.Count },
{ "TotalRuns", totalRuns },
{ "RunsPerMinute", runsPerMinute },
});
writeResults(completedRun);
});
}
static void checkMetric(string rMetricName, Metrics.MetricCalculator metric, List values, Csv csv)
{
LogC.info("checking metric " + rMetricName);
checkOne(metric, double.Parse(csv.value("tol1", values)), CURVE_FILE, double.Parse(csv.value("value1", values)));
checkOne(metric, double.Parse(csv.value("tol2", values)), ANOTHER_FILE, double.Parse(csv.value("value2", values)));
checkOne(metric, double.Parse(csv.value("tol3", values)), INTRADAY_FILE, double.Parse(csv.value("value3", values)));
}
public void enqueueRequests()
{
LogC.info("enqueueing requests " + runner.requestQueueName());
zeroTo(end - start + 1, i => ThreadPool.QueueUserWorkItem(o => enqueueRun(start + i)));
wait(int.MaxValue, 1000, () => runsQueued == end - start + 1);
LogC.info("done enqueueing requests " + runner.requestQueueName());
}
public void logAndAlert(string s, Exception e)
{
var message = LogC.errMessage("failed during simulation run\n", e);
LogC.err(message);
alertUser("failed during simulation. examine " + LogC.errFile() + "\n" + message);
}
public static void Main(string[] args)
{
var arguments = Arguments.arguments(args, jStrings("system", "prefix"));
var system = arguments.get("system");
var prefix = arguments.get("prefix", PREFIX_DEFAULT);
LogC.setOut("Tomahawk", Systematic.logsDir().file("Tomahawk." + system + ".log").path(), true);
LogC.useJavaLog = true;
LogC.info("running system " + system + ", process " + processId());
var liveSystems = accept(list <LiveSystem>(MsivLiveHistory.LIVE.liveSystems()), ls => ls.siv().system().Equals(system));
each(liveSystems, liveSystem => {
Bomb.when(liveSystem.details().runInNativeCurrency(), () => "not allowed to run live systems in native currency");
var markets = list <Market>(liveSystem.markets());
var symbols = convert(markets, market => new Symbol(market.name(), market.bigPointValue()));
var systemId = liveSystem.id();
var parameters = new Parameters {
{ "systemId", systemId },
{ "RunMode", (double)RunMode.LIVE }
};
Bomb.when(isEmpty(symbols), () => "No markets for " + systemId);
var systemArguments = new SystemArguments(symbols, parameters);
var start = date(systemArguments.interval().isDaily() ? BloombergSecurity.BBG_START_HISTORICAL : BloombergSecurity.BBG_START_INTRADAY);
if (parameters.has("DaysBack"))
{
start = now().AddDays(-parameters.get <int>("DaysBack"));
}
var loader = new SystemDbBarLoader(liveSystem.details().interval(), symbols, start);
var simulator = new Simulator(systemArguments, loader, prefix);
simulator.processBars();
simulator.goLive();
});
sleep(Int32.MaxValue);
}
public MainWindow(string[] args) : base("Recon")
{
LogC.info("started MainWindow.");
LogC.useJavaLog = true;
var arguments = Arguments.arguments(args, O.jStrings("prefix", "pane"));
OrderTable.prefix = arguments.get("prefix", "TOMAHAWK");
var pane = arguments.get("pane", "all");
dockManager.Content = dockPane;
addPanels(dockPane, pane);
Title = "Recon " + OrderTable.prefix;
if (DEVELOPERS.Contains(O.username()))
{
return;
}
while (!UsersTable.USERS.isLoggedIn(O.username()))
{
var result = MessageBox.Show(
"You are not logged in to Bloomberg, or do not have permission to view data from the Server API. Press OK to try again, Cancel to close Recon.",
"Please Log In To Bloomberg",
MessageBoxButton.OKCancel,
MessageBoxImage.Error,
MessageBoxResult.OK
);
if (result.Equals(MessageBoxResult.Cancel))
{
O.systemExit(0);
}
}
}
static bool usePane(string userPane, string potential)
{
var result = "all".Equals(userPane) || userPane.Equals(potential);
LogC.info("usePane " + potential + " " + result);
return(result);
}
public static void Main(string[] args)
{
Directory.CreateDirectory(@"E:\logs");
LogC.setOut("STOServer", @"E:\logs\STOServer." + processId() + ".log", true);
LogC.setErr(@"E:\logs\STOServer." + processId() + ".error.log");
AppDomain.CurrentDomain.UnhandledException += handleUncaughtExceptions;
S3Cache.setDefaultSqsDbMode(true);
if (!(hostname().StartsWith("NY") || hostname().StartsWith("LN")))
{
QHttpClient.turnOffProxy();
}
var runner = EC2Runner.fromUserData();
S3Cache.setS3Cache(runner.s3Cache());
var lastRunTime = 120;
while (trueDat())
{
info("lastRunTime: " + lastRunTime);
STORequest request = null;
try {
lastRunTime = processNextMessage(runner, lastRunTime, out request);
} catch (Exception e) {
LogC.err("failed " + request, e);
LogC.info("failed " + request, e);
continue;
}
info("finished " + request);
}
}
public static void Main(string[] args)
{
Log.doNotDebugSqlForever();
var startTime = now();
var arguments = Arguments.arguments(args, jStrings("systemId", "runs", "start", "end", "markets", "skipExisting"));
var skipExisting = arguments.get("skipExisting", false);
var systemId = arguments.get("systemId", -1);
var details = SystemDetailsTable.DETAILS.details(systemId);
var runs = arguments.containsKey("runs")
? convert(split(",", arguments.get("runs")), run => int.Parse(run))
: seq(arguments.integer("start"), arguments.integer("end"));
var curvesDir = sto.STO.fromId(systemId).curvesDir();
var markets = arguments.containsKey("markets") ? split(",", arguments.@string("markets")) : allMarkets(details);
ThreadPool.SetMaxThreads(200, 200);
each(runs, run => each(markets, market => {
var directory = curvesDir.directory(new[] { details.siv().svimName("_", market, "") });
var runFile = directory.file("run_" + run + ".bin");
if (skipExisting && runFile.exists())
{
info("skipped " + runFile.path());
return;
}
var path_ = runFile.path();
queueWorkItem(() => { readFromS3(systemId, market, run, path_); LogC.info("wrote " + path_); });
}));
waitForAllWorkItems(6000, 100);
LogC.info("took " + now().Subtract(startTime).TotalSeconds + " seconds.");
}
public void logSystemCreation(SystemArguments arguments)
{
if (has("systemId"))
{
LogC.consoleOut("running id(" + get <int>("systemId") + "), system(" + arguments.siv().system() + ")");
}
else
{
LogC.consoleOut("running in research mode(" + systemClassName() + ")");
}
if (reDebug())
{
LogC.consoleOut("in DEBUG mode (slow)");
}
if (isLive())
{
LogC.consoleOut("PV = " + pvName());
}
else if (isSTO())
{
LogC.consoleOut("STO = " + paren(commaSep(stoDir(), stoId(), runNumber())));
}
else
{
logParameters();
}
}
public static bool RegisterHotKey(this HotKeyData keyData, Form parent)
{
if (!keyData.UseHotKey)
{
return(true);
}
// update HotKey
User32HotKey.KeyModifiers modifiers = User32HotKey.KeyModifiers.None;
if (keyData.KeyData.HasFlag(Keys.Control))
{
modifiers |= User32HotKey.KeyModifiers.Control;
}
if (keyData.KeyData.HasFlag(Keys.Shift))
{
modifiers |= User32HotKey.KeyModifiers.Shift;
}
if (keyData.KeyData.HasFlag(Keys.Alt))
{
modifiers |= User32HotKey.KeyModifiers.Alt;
}
if (keyData.KeyData.HasFlag(Keys.LWin) || keyData.KeyData.HasFlag(Keys.RWin))
{
modifiers |= User32HotKey.KeyModifiers.Windows;
}
Keys key = (Keys)((int)keyData.KeyData & 0x0000FFFF); //filter out modifiers
LogC.WriteLine("RegisterHotKey: " + keyData);
return(User32HotKey.RegisterHotKey(parent.Handle, keyData.AppId, modifiers, key));
} //end RegisterHotKey
public IEnumerator GetEnumerator()
{
LogC.info("GetEnumerator called"); foreach (var run in runs())
{
LogC.info("yielding next"); yield return(cache.get(run));
}
}
public CloudSTOPanel()
{
var panel = new QDockPanel();
Content = panel;
Loaded += initialize;
instanceGrid = new QDataTableGrid(loadInstanceRow, unloadInstanceRow);
var controls = new QDockPanel();
systemIdBox = new QComboBox("-1", resetInstanceId, true)
{
MinWidth = 80, IsTextSearchEnabled = true, IsSynchronizedWithCurrentItem = false
};
controls.add(systemIdBox, Dock.Left);
var grid = new QGrid();
grid.addColumns(4);
instancesLabel = label("");
grid.add(instancesLabel, 0);
redGreenLabel = label("");
grid.add(redGreenLabel, 1);
summaryLabel = label("");
grid.add(summaryLabel, 2);
completionTimeLabel = label("");
grid.add(completionTimeLabel, 3);
controls.add(grid, Dock.Left);
controls.add(new QButton("Refresh System Ids", refreshSystemIds), Dock.Left);
panel.add(controls, Dock.Top);
panel.add(instanceGrid, Dock.Top);
LogC.info("starting cloud STO tracker");
tracker = new CloudSTOTracker(this);
}
public bool StartWatching()
{
Directory.CreateDirectory(@"C:\logs");
LogC.setOut("LiveData", @"C:\logs\QLiveData" + Process.GetCurrentProcess().Id + ".log", true);
watching = true;
return(true);
}
public static void Main(string[] arguments)
{
var args = Arguments.arguments(arguments, jStrings("systemId", "run", "symbol"));
var results = readFromS3(args.integer("systemId"), args.integer("run"));
LogC.info(toShortString(results[args.@string("symbol")]));
}
void killIfNeeded(Process process)
{
LogC.info("killing " + O.the(O.accept(processes, entry => entry.Value.Equals(process))).Key);
if (!process.HasExited)
{
process.Kill();
}
}
static object loggedError(string message, Exception e)
{
LogC.setErr(@"c:\QExcel.log");
LogC.err(message, e);
message += ". check " + LogC.errFile() + " to see stack trace.";
LogC.setErr(null);
return(message);
}
void safeProcessUpdate(ObservationListener listener, Date date, double value)
{
try {
listener.onUpdate(date, value);
} catch (Exception e) {
LogC.err(symbol.name + " failed processing observation " + ymdHuman(date) + ": " + value, e);
}
}
protected override void Connection_Disconnected(object sender, LogC.RabbitAmqp.ConnectionEventArgs e)
{
this.HaltSubscription();
base.Connection_Disconnected(sender, e);
}