public IEnumerator GetEnumerator()
{
LogC.info("GetEnumerator called"); foreach (var run in runs())
{
LogC.info("yielding next"); yield return(cache.get(run));
}
}
public override void processTick(Symbol symbol, Tick tick)
{
if (!partialBars.ContainsKey(symbol))
{
partialBars[symbol] = new Bar(tick.price, tick.price, tick.price, tick.price, tick.time);
}
partialBars[symbol] = partialBars[symbol].update(tick);
var processThisTick = system.runOnNewTick() || !lastTickProcessed.ContainsKey(symbol) || lastTickProcessed[symbol] <= tick.time.AddMinutes(-1);
try {
if (processThisTick)
{
manager.newTick();
if (!isLive)
{
isLive = true;
manager.goLive();
system.goLiveDO_NOT_CALL_EXCEPT_FROM_BRIDGE();
monitor().goLive(system);
}
var partialBar = updateBarSpud(symbol);
if (Systematic.isLoggingTicks())
{
LogC.info("processing tick " + symbol + ": bar=" + partialBar + ", tick=" + tick);
}
system.processTickDO_NOT_CALL_EXCEPT_FROM_BRIDGE(symbol, partialBar, tick);
manager.recalculate();
lastTickProcessed[symbol] = tick.time;
}
monitor().tickProcessed(symbol, tick);
} catch (Exception e) {
LogC.info("exception caught in processTick: ", e);
throw;
}
}
public MainWindow(string[] unused) : base("NORAD")
{
var arguments = Arguments.arguments(unused, o.jStrings("live", "settings"));
System.Environment.SetEnvironmentVariable("RE_TEST_MODE", "TRUE");
LogC.useJavaLog = true;
LogC.info("starting MainWindow.");
var workbench = new WorkbenchPanel();
dockManager.Content = workbench;
var path = Systematic.mainDir().file("dotNET/Gui/Resources/target.ico").path();
Icon = BitmapFrame.Create(new FileStream(path, FileMode.Open, FileAccess.Read));
if (arguments.containsKey("settings"))
{
if (arguments.get("live", false))
{
workbench.liveButton.doClick();
}
else
{
workbench.runButton.doClick();
}
}
}
public static void Main(string[] arguments)
{
var args = Arguments.arguments(arguments, jStrings("systemId", "run", "symbol"));
var results = readFromS3(args.integer("systemId"), args.integer("run"));
LogC.info(toShortString(results[args.@string("symbol")]));
}
public virtual void placeOrder(Order order)
{
var place = (forecastMode ? "forecast" : "place");
LogC.info(place + " order: " + order);
order.placedBy(this, bars[order.symbol][0].time);
var symbol = order.symbol;
Bomb.unless(isActive(), () => "cannot " + place + " an order in a deactivated symbol: " + order);
if (inClose)
{
order.placedOnClose();
}
if (forecastMode)
{
if (order.canFill(Bomb.missing(bars, symbol)[0].close, true))
{
// bug - if two exits are placed in onClose, they can both be "filled" for the
// forecast, which is not what would happen in the actual onClose
monitor().orderForecast(this, order);
}
}
else
{
Bomb.when(exists(orders(symbol), order.descriptionAndSymbolMatch), () =>
"symbol+description must be unique for outstanding orders. Tried placing: " +
order + "\nin:\n" + toShortString(orders(symbol)));
monitor().orderPlaced(order);
orders_.get(symbol).Add(order);
}
}
public void testSymbolValueLive()
{
Log.setFile(@"C:\logs\jefftest");
SystemTimeSeriesTable.SYSTEM_TS.insert("testone", "ASCII", "ActiveMQ", "somedangtopic");
AsciiTable.SYSTEM_ASCII.insert("testone", @"\\nysrv37\share\Tools\RightEdge\TransitionTest\TY1C.full.csv", true, 1);
var manager = new SpudManager();
var barSpud = new BarSpud(manager);
var barSpud2 = new BarSpud(manager);
var symbol = new Symbol("testone");
var spud = symbol.doubles(barSpud);
var spud2 = new Symbol("RE.TEST.TY.1C").doubles(barSpud2);
manager.newBar();
barSpud.set(new Bar(1, 3, 1, 2, O.date("2008/08/06")));
barSpud2.set(new Bar(1, 3, 1, 2, O.date("2008/08/06")));
manager.newTick();
spud.doSubscribe();
barSpud.lastTickedAt(O.date("2008/08/06 11:22:34"));
spud2.doSubscribe();
IsTrue(spud2.isDirty());
symbol.javaSymbol().jmsLive().topic().send("value=97.1|timestamp=2008/08/06 11:22:33|MSTimestamp=2008/06/05 13:10:08");
Bombs(() => LogC.info("" + spud[0]), "stale");
symbol.javaSymbol().jmsLive().topic().send("value=97.2|timestamp=2008/08/06 11:22:34|MSTimestamp=2008/06/05 13:10:08");
O.wait(() => spud[0] == 97.2);
barSpud.lastTickedAt(O.date("2008/08/06 11:22:35"));
spud.allowStaleTicks();
AreEqual(97.2, spud[0]);
AreEqual(116.140625, spud2[0]);
}
public void testCanFailToRunCompleteSimulation()
{
Db.reallyRollback(); // open transaction system details insert (on unrelated row, jeff hates sqlserver) from DbTestCase causes test failure.
var gui = new FakeResearchGUI();
gui.setMarkets(O.list("RE.TEST.TY.1C"));
var parameters = new Parameters {
{ "ATRLen", 10 },
{ "ATRlong", 100 },
{ "BreakDays", 30 },
{ "FixEquity", 1 },
{ "InitEquity", 6000000 },
{ "LeadBars", 50 },
{ "MaxPyramid", 1 },
{ "Risk", 0.02 },
{ "nATR", 2 },
{ "upATR", 2 },
{ "systemId", 39 } // ok?
};
gui.setParameters(parameters);
gui.setStartDate(date("2001/01/01"));
gui.setEndDate(date("2001/04/01"));
gui.runSystem();
O.wait(() => gui.runButtonEnabled == false);
O.wait(100, 100, gui.runComplete);
IsTrue(gui.runButtonEnabled);
gui.hasMessage("failed");
LogC.info("done");
}
static void checkMetric(string rMetricName, Metrics.MetricCalculator metric, List values, Csv csv)
{
LogC.info("checking metric " + rMetricName);
checkOne(metric, double.Parse(csv.value("tol1", values)), CURVE_FILE, double.Parse(csv.value("value1", values)));
checkOne(metric, double.Parse(csv.value("tol2", values)), ANOTHER_FILE, double.Parse(csv.value("value2", values)));
checkOne(metric, double.Parse(csv.value("tol3", values)), INTRADAY_FILE, double.Parse(csv.value("value3", values)));
}
public MainWindow(string[] args) : base("Recon")
{
LogC.info("started MainWindow.");
LogC.useJavaLog = true;
var arguments = Arguments.arguments(args, O.jStrings("prefix", "pane"));
OrderTable.prefix = arguments.get("prefix", "TOMAHAWK");
var pane = arguments.get("pane", "all");
dockManager.Content = dockPane;
addPanels(dockPane, pane);
Title = "Recon " + OrderTable.prefix;
if (DEVELOPERS.Contains(O.username()))
{
return;
}
while (!UsersTable.USERS.isLoggedIn(O.username()))
{
var result = MessageBox.Show(
"You are not logged in to Bloomberg, or do not have permission to view data from the Server API. Press OK to try again, Cancel to close Recon.",
"Please Log In To Bloomberg",
MessageBoxButton.OKCancel,
MessageBoxImage.Error,
MessageBoxResult.OK
);
if (result.Equals(MessageBoxResult.Cancel))
{
O.systemExit(0);
}
}
}
public void enqueueRequests()
{
LogC.info("enqueueing requests " + runner.requestQueueName());
zeroTo(end - start + 1, i => ThreadPool.QueueUserWorkItem(o => enqueueRun(start + i)));
wait(int.MaxValue, 1000, () => runsQueued == end - start + 1);
LogC.info("done enqueueing requests " + runner.requestQueueName());
}
void process(Message message, Dictionary <int, bool> completed, int totalRuns)
{
var response = (STOResponse)message.@object();
runner.received(response);
var completedRun = response.runNumber();
var runTimeMillis = response.runTimeMillis();
var completionTime = response.completedAt();
var isLocal = response.instanceId().Equals("LOCAL");
message.delete();
// Letting message get GC'd, rather than sitting in the thread pool really helps with the Java heap space.
queueWorkItem(() => {
lock (completed) {
if (completed.ContainsKey(completedRun))
{
return;
}
completed[completedRun] = true;
if (!isLocal)
{
completionTimes.Add(date(completionTime));
updateRunsPerMinute(runTimeMillis);
}
}
LogC.info("completed run " + completedRun + paren(completed.Count + "/" + totalRuns));
new Topic(CloudMonitor.progressTopic(systemId)).send(new Dictionary <string, object> {
{ "RunsComplete", completed.Count },
{ "TotalRuns", totalRuns },
{ "RunsPerMinute", runsPerMinute },
});
writeResults(completedRun);
});
}
public static void Main(string[] args)
{
Log.doNotDebugSqlForever();
var startTime = now();
var arguments = Arguments.arguments(args, jStrings("systemId", "runs", "start", "end", "markets", "skipExisting"));
var skipExisting = arguments.get("skipExisting", false);
var systemId = arguments.get("systemId", -1);
var details = SystemDetailsTable.DETAILS.details(systemId);
var runs = arguments.containsKey("runs")
? convert(split(",", arguments.get("runs")), run => int.Parse(run))
: seq(arguments.integer("start"), arguments.integer("end"));
var curvesDir = sto.STO.fromId(systemId).curvesDir();
var markets = arguments.containsKey("markets") ? split(",", arguments.@string("markets")) : allMarkets(details);
ThreadPool.SetMaxThreads(200, 200);
each(runs, run => each(markets, market => {
var directory = curvesDir.directory(new[] { details.siv().svimName("_", market, "") });
var runFile = directory.file("run_" + run + ".bin");
if (skipExisting && runFile.exists())
{
info("skipped " + runFile.path());
return;
}
var path_ = runFile.path();
queueWorkItem(() => { readFromS3(systemId, market, run, path_); LogC.info("wrote " + path_); });
}));
waitForAllWorkItems(6000, 100);
LogC.info("took " + now().Subtract(startTime).TotalSeconds + " seconds.");
}
public STORunner(int numInstances, int systemId, int start, int?end, bool noKill)
{
this.numInstances = numInstances;
this.systemId = systemId;
this.start = start;
this.noKill = noKill;
details = SystemDetailsTable.DETAILS.details(systemId);
this.end = end.HasValue ? end.Value : details.lastRunNumber();
writer = new STOMetricsWriter(details, false);
STO.populateSymbolsPortfolios(details, out symbols, out portfolios);
runner = new EC2Runner(systemId + "-" + Dates.yyyyMmDdHhMmSsNoSeparator(Dates.now()));
LogC.info("checking for completed runs");
existingRuns = completedRuns();
var allRunsComplete = true;
for (var i = this.start; i <= this.end; i++)
{
if (!existingRuns.Contains(i))
{
allRunsComplete = false;
break;
}
}
if (allRunsComplete)
{
this.numInstances = 0;
}
unhandledExceptionProcessor = ((sender, args) => killRun(LogC.errMessage("unhandled exception", (Exception)args.ExceptionObject)));
}
public static void Main(string[] args)
{
var arguments = Arguments.arguments(args, jStrings("system", "prefix"));
var system = arguments.get("system");
var prefix = arguments.get("prefix", PREFIX_DEFAULT);
LogC.setOut("Tomahawk", Systematic.logsDir().file("Tomahawk." + system + ".log").path(), true);
LogC.useJavaLog = true;
LogC.info("running system " + system + ", process " + processId());
var liveSystems = accept(list <LiveSystem>(MsivLiveHistory.LIVE.liveSystems()), ls => ls.siv().system().Equals(system));
each(liveSystems, liveSystem => {
Bomb.when(liveSystem.details().runInNativeCurrency(), () => "not allowed to run live systems in native currency");
var markets = list <Market>(liveSystem.markets());
var symbols = convert(markets, market => new Symbol(market.name(), market.bigPointValue()));
var systemId = liveSystem.id();
var parameters = new Parameters {
{ "systemId", systemId },
{ "RunMode", (double)RunMode.LIVE }
};
Bomb.when(isEmpty(symbols), () => "No markets for " + systemId);
var systemArguments = new SystemArguments(symbols, parameters);
var start = date(systemArguments.interval().isDaily() ? BloombergSecurity.BBG_START_HISTORICAL : BloombergSecurity.BBG_START_INTRADAY);
if (parameters.has("DaysBack"))
{
start = now().AddDays(-parameters.get <int>("DaysBack"));
}
var loader = new SystemDbBarLoader(liveSystem.details().interval(), symbols, start);
var simulator = new Simulator(systemArguments, loader, prefix);
simulator.processBars();
simulator.goLive();
});
sleep(Int32.MaxValue);
}
public CloudSTOPanel()
{
var panel = new QDockPanel();
Content = panel;
Loaded += initialize;
instanceGrid = new QDataTableGrid(loadInstanceRow, unloadInstanceRow);
var controls = new QDockPanel();
systemIdBox = new QComboBox("-1", resetInstanceId, true)
{
MinWidth = 80, IsTextSearchEnabled = true, IsSynchronizedWithCurrentItem = false
};
controls.add(systemIdBox, Dock.Left);
var grid = new QGrid();
grid.addColumns(4);
instancesLabel = label("");
grid.add(instancesLabel, 0);
redGreenLabel = label("");
grid.add(redGreenLabel, 1);
summaryLabel = label("");
grid.add(summaryLabel, 2);
completionTimeLabel = label("");
grid.add(completionTimeLabel, 3);
controls.add(grid, Dock.Left);
controls.add(new QButton("Refresh System Ids", refreshSystemIds), Dock.Left);
panel.add(controls, Dock.Top);
panel.add(instanceGrid, Dock.Top);
LogC.info("starting cloud STO tracker");
tracker = new CloudSTOTracker(this);
}
static bool usePane(string userPane, string potential)
{
var result = "all".Equals(userPane) || userPane.Equals(potential);
LogC.info("usePane " + potential + " " + result);
return(result);
}
public static void Main(string[] args)
{
Directory.CreateDirectory(@"E:\logs");
LogC.setOut("STOServer", @"E:\logs\STOServer." + processId() + ".log", true);
LogC.setErr(@"E:\logs\STOServer." + processId() + ".error.log");
AppDomain.CurrentDomain.UnhandledException += handleUncaughtExceptions;
S3Cache.setDefaultSqsDbMode(true);
if (!(hostname().StartsWith("NY") || hostname().StartsWith("LN")))
{
QHttpClient.turnOffProxy();
}
var runner = EC2Runner.fromUserData();
S3Cache.setS3Cache(runner.s3Cache());
var lastRunTime = 120;
while (trueDat())
{
info("lastRunTime: " + lastRunTime);
STORequest request = null;
try {
lastRunTime = processNextMessage(runner, lastRunTime, out request);
} catch (Exception e) {
LogC.err("failed " + request, e);
LogC.info("failed " + request, e);
continue;
}
info("finished " + request);
}
}
void killIfNeeded(Process process)
{
LogC.info("killing " + O.the(O.accept(processes, entry => entry.Value.Equals(process))).Key);
if (!process.HasExited)
{
process.Kill();
}
}
public void add(PlotDefinition plot)
{
plot.requireManagerMatches(barSpud);
LogC.info("adding plot " + plot);
pane(plot).add(plot);
resetYAxis();
Invalidate();
}
public void testCanCreateFromDb()
{
LogC.info("full class name = " + typeof(MultiPairSystem <SimplePairSystem, SomeCombination>).FullName);
const string name = "Q.Trading.MultiPairSystem`2[[Q.Systems.Examples.SimplePairSystem],[Q.Systems.Examples.SomeCombination]]";
var type = Type.GetType(name, true, false);
AreEqual(typeof(MultiPairSystem <SimplePairSystem, SomeCombination>), type);
}
public MysteryObj()
{
LogC.info("constructing mystery obj");
lock (rng) number = rng.Next(1000);
lock (rng) text = "this is a string " + rng.Next(1000);
doNothing(number);
doNothing(text);
}
public void benchmarkMin()
{
indicator = new Min(values, 20);
var start = DateTime.Now;
var rand = new Random();
O.zeroTo(100000, i => addPoint(rand.Next()));
LogC.info("total time: " + DateTime.Now.Subtract(start).TotalMilliseconds + " millis");
}
public void cancel()
{
LogC.info("cancel " + this);
cancelled = true;
if (ferretSubmission != null)
{
cancelFerretOrder();
}
}
public void testSortedRunNumbersByParameter()
{
var minLengthDnRuns = O.list(O.convert(O.seq(2500), i => 1 + 20 * i));
var results = hamster();
var actual = results.runsByParameter("LengthDn").GetRange(0, 2500);
LogC.info(Objects.toShortString(actual));
AreEqual(minLengthDnRuns, O.sort(actual));
}
public void testSortedRunNumbersByDownsideDeviation()
{
var minDeviationRuns = O.list(9727, 19727, 29727, 39727, 49727);
var results = hamster();
var actual = results.runsByMetric("QDownsideDeviation").GetRange(125, 5);
LogC.info(Objects.toShortString(actual));
AreEqual(minDeviationRuns, O.sort(actual));
}
public MetricResults metrics()
{
var metrics = new MetricResults();
var start = DateTime.Now;
each(collectors, (portfolio, collector) => metrics[portfolio.name] = collector.metrics());
var seconds = (DateTime.Now - start).TotalSeconds;
LogC.info("metric calculation took " + seconds + " seconds (" + (collectors.Count / seconds) + " collectors per second)");
return(metrics);
}
void stopInstances()
{
if (noKill)
{
LogC.info("NOT STOPPING INSTANCES - IF YOU ARE NOT JERIC");
}
else
{
runner.stopInstances();
}
}
static void stopProcesses(string key)
{
O.each(O.copy(processes.get(key)), process => {
if (!process.HasExited)
{
LogC.info("killing pid " + process.Id);
process.Kill();
}
processes.get(key).Remove(process);
});
LogC.info("all processes killed for " + key);
}
public override void orderFilled(Position position, Trade trade)
{
try {
LogC.info("SIMFILL: " + trade + " position: " + position);
system.orderFilledDO_NOT_CALL_EXCEPT_FROM_BRIDGE(position, trade);
statistics().addOrder(position, trade);
manager.recalculate();
monitor().orderFilled(position, trade, system.tradeEmail);
} catch (Exception e) {
LogC.info("exception caught in orderFilled: ", e);
throw;
}
}
void writeResultsToFile(int run, IDictionary <string, Dictionary <string, double> > results)
{
try {
writer.writeResults(run, results);
lock (runsLeftLock) {
runsLeft--;
LogC.info("completed run " + run + ". " + runsLeft + " left.");
}
} catch (Exception e) {
LogC.info(LogC.errMessage("exception thrown in writeResults", e));
throw;
}
}