private IDataFeed RunDataFeed(IAlgorithm algorithm, out FuncDataQueueHandler dataQueueHandler, ITimeProvider timeProvider = null, Func <FuncDataQueueHandler, IEnumerable <BaseData> > getNextTicksFunction = null)
{
getNextTicksFunction = getNextTicksFunction ?? (fdqh => fdqh.Subscriptions.Select(symbol => new Tick(DateTime.Now, symbol, 1, 2)
{
Quantity = 1
}));
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler(); // new ResultHandlerStub();
dataQueueHandler = new FuncDataQueueHandler(getNextTicksFunction);
var feed = new TestableLiveTradingDataFeed(dataQueueHandler, timeProvider);
var mapFileProvider = new LocalDiskMapFileProvider();
var fileProvider = new DefaultDataFileProvider();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), fileProvider);
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
// wait for feed.Run to actually begin
feedThreadStarted.WaitOne();
return(feed);
}
public void EmitsData()
{
var algorithm = new AlgorithmStub(forex: new List <string> {
"EURUSD"
});
// job is used to send into DataQueueHandler
var job = new LiveNodePacket();
// result handler is used due to dependency in SubscriptionDataReader
var resultHandler = new BacktestingResultHandler();
var dataFileProvider = new DefaultDataFileProvider();
var lastTime = DateTime.MinValue;
var timeProvider = new RealTimeProvider();
var dataQueueHandler = new FuncDataQueueHandler(fdqh =>
{
var time = timeProvider.GetUtcNow().ConvertFromUtc(TimeZones.EasternStandard);
if (time == lastTime)
{
return(Enumerable.Empty <BaseData>());
}
lastTime = time;
return(Enumerable.Range(0, 9).Select(x => new Tick(time.AddMilliseconds(x * 100), Symbols.EURUSD, 1.3m, 1.2m, 1.3m)));
});
var feed = new TestableLiveTradingDataFeed(dataQueueHandler, timeProvider);
var mapFileProvider = new LocalDiskMapFileProvider();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, new LocalDiskFactorFileProvider(mapFileProvider), dataFileProvider);
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
// wait for feed.Run to actually begin
feedThreadStarted.WaitOne();
var emittedData = false;
ConsumeBridge(feed, TimeSpan.FromSeconds(10), true, ts =>
{
if (ts.Slice.Count != 0)
{
emittedData = true;
Console.WriteLine("HasData: " + ts.Slice.Bars[Symbols.EURUSD].EndTime);
Console.WriteLine();
}
});
Assert.IsTrue(emittedData);
}
public void ReadsFineFundamental(FineFundamentalTestParameters parameters)
{
var stopwatch = Stopwatch.StartNew();
var rows = new List <FineFundamental>();
var config = new SubscriptionDataConfig(typeof(FineFundamental), parameters.Symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
var security = new Security(SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork), config, new Cash(CashBook.AccountCurrency, 0, 1), SymbolProperties.GetDefault(CashBook.AccountCurrency));
var request = new SubscriptionRequest(false, null, security, config, parameters.StartDate, parameters.EndDate);
var fileProvider = new DefaultDataFileProvider();
var factory = new FineFundamentalSubscriptionEnumeratorFactory();
var enumerator = factory.CreateEnumerator(request, fileProvider);
while (enumerator.MoveNext())
{
var current = enumerator.Current as FineFundamental;
rows.Add(current);
}
stopwatch.Stop();
Console.WriteLine("Total rows: {0}, elapsed time: {1}", rows.Count, stopwatch.Elapsed);
Assert.AreEqual(parameters.RowCount, rows.Count);
if (parameters.RowCount != 1)
{
return;
}
var row = rows[0];
Assert.AreEqual(parameters.CompanyShortName, row.CompanyReference.ShortName);
Assert.AreEqual(parameters.Symbol, row.Symbol);
Assert.AreEqual(parameters.Symbol != Symbol.Empty ? parameters.Symbol.Value : null, row.CompanyReference.PrimarySymbol);
Assert.AreEqual(parameters.Symbol != Symbol.Empty ? parameters.Symbol.Value : null, row.SecurityReference.SecuritySymbol);
Assert.AreEqual(parameters.Ebitda3M, row.FinancialStatements.IncomeStatement.EBITDA.ThreeMonths);
Assert.AreEqual(parameters.Ebitda12M, row.FinancialStatements.IncomeStatement.EBITDA.TwelveMonths);
Assert.AreEqual(parameters.Ebitda12M, row.FinancialStatements.IncomeStatement.EBITDA);
Assert.AreEqual(parameters.CostOfRevenue3M, row.FinancialStatements.IncomeStatement.CostOfRevenue.ThreeMonths);
Assert.AreEqual(parameters.CostOfRevenue12M, row.FinancialStatements.IncomeStatement.CostOfRevenue.TwelveMonths);
Assert.AreEqual(parameters.CostOfRevenue12M, row.FinancialStatements.IncomeStatement.CostOfRevenue);
Assert.AreEqual(parameters.EquityPerShareGrowth1Y, row.EarningRatios.EquityPerShareGrowth.OneYear);
Assert.AreEqual(parameters.EquityPerShareGrowth1Y, row.EarningRatios.EquityPerShareGrowth);
Assert.AreEqual(parameters.PeRatio, row.ValuationRatios.PERatio);
}
public void TestsFileSystemDataFeedSpeed()
{
var job = new BacktestNodePacket();
var resultHandler = new BacktestingResultHandler();
var mapFileProvider = new LocalDiskMapFileProvider();
var factorFileProvider = new LocalDiskFactorFileProvider(mapFileProvider);
var dataFileProvider = new DefaultDataFileProvider();
var algorithm = new BenchmarkTest();
var feed = new FileSystemDataFeed();
feed.Initialize(algorithm, job, resultHandler, mapFileProvider, factorFileProvider, dataFileProvider);
algorithm.Initialize();
var feedThreadStarted = new ManualResetEvent(false);
Task.Factory.StartNew(() =>
{
feedThreadStarted.Set();
feed.Run();
});
feedThreadStarted.WaitOne();
var stopwatch = Stopwatch.StartNew();
var lastMonth = -1;
var count = 0;
foreach (var timeSlice in feed)
{
if (timeSlice.Time.Month != lastMonth)
{
Console.WriteLine(DateTime.Now + " - Time: " + timeSlice.Time);
lastMonth = timeSlice.Time.Month;
}
count++;
}
Console.WriteLine("Count: " + count);
stopwatch.Stop();
Console.WriteLine("Elapsed time: " + stopwatch.Elapsed);
}
public void ReadsZipEntryNames()
{
var time = new DateTime(2016, 03, 03, 12, 48, 15);
var source = Path.Combine("TestData", "20151224_quote_american.zip");
var config = new SubscriptionDataConfig(typeof(ZipEntryName), Symbol.Create("XLRE", SecurityType.Option, Market.USA), Resolution.Tick,
TimeZones.NewYork, TimeZones.NewYork, false, false, false);
var dataFileProvider = new DefaultDataFileProvider();
var factory = new ZipEntryNameSubscriptionDataSourceReader(dataFileProvider, config, time, false);
var expected = new[]
{
Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Call, 21m, new DateTime(2016, 08, 19)),
Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Call, 22m, new DateTime(2016, 08, 19)),
Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Put, 37m, new DateTime(2016, 08, 19)),
};
var actual = factory.Read(new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName)).ToList();
// we only really care about the symbols
CollectionAssert.AreEqual(expected, actual.Select(x => x.Symbol));
Assert.IsTrue(actual.All(x => x is ZipEntryName));
}
private void Main(ProgramOptions options)
{
// Validate rules folder
if (string.IsNullOrWhiteSpace(options.RulesFolder) || options.RulesFolder.IndexOfAny(new char[] { '"', '\'', '\\', '/', ':' }) >= 0)
{
Display.WriteText("Specified rule folder name does not seems to be valid: \n\n" + options.RulesFolder +
"\n\nEnsure the value is a single folder name (not a full path) without special characters: \", ', \\, / and :.");
return;
}
// Initialize data file provider
IDataFileProvider fileProvider;
try
{
// Use data folder path from options which will default the parent direcotry unless another path is specified
// Pass custom plugins.txt file path if provided via options
fileProvider = new DefaultDataFileProvider(options.DataFolder, options.PluginListFile);
}
catch (Exception ex)
{
// Program will exit on error
// Appropriate hint is displayed
Display.WriteText("Data folder path does not seems to be valid: " + ex.Message + "\n\n" + options.DataFolder +
"\n\nUse option -d or --data to specify correct path to the data folder or use option -h or --help for more help.");
return;
}
// Mod Ogranizer
if (!string.IsNullOrEmpty(options.ModOrganizerProfile))
{
try
{
// After verifying data folder with DefaultDataFolderProvider
// replace it with MO data file provider
fileProvider = new ModOrganizerDataFileProvider(options.DataFolder, options.PluginListFile, options.ModOrganizerProfile, options.ModOrganizerModsPath);
}
catch (Exception ex)
{
// Program will exit on error
// Appropriate hint is displayed
Display.WriteText("Incorrect Mod Organizer configuration: " + ex.Message +
"\n\nUse option -h or --help for more help.");
return;
}
}
// Determine output plugin file name, use filename from options if provided
string targetPluginFileName = options.OutputFilename ?? string.Format("Patcher-{0}.esp", options.RulesFolder);
// File log will be created in the data folder, named as the output plugin plus .log extension
string logFileName = Path.Combine(Program.ProgramFolder, Program.ProgramLogsFolder, targetPluginFileName + ".log");
using (var logger = new StreamLogger(fileProvider.GetDataFile(FileMode.Create, logFileName).Open()))
{
Log.AddLogger(logger);
// Put some handy info at the beginning of the log file
Log.Info(Program.GetProgramVersionInfo());
Log.Fine("Options: " + options);
try
{
// Create suitable data context according to the data folder content
using (DataContext context = DataContext.CreateContext(fileProvider))
{
Log.Fine("Initialized data context: " + context.GetType().FullName);
// Context tweaks
context.AsyncFormIndexing = true;
context.AsyncFormLoading = options.MaxLoadingThreads > 0;
context.AsyncFormLoadingWorkerThreshold = 100;
context.AsyncFromLoadingMaxWorkers = Math.Max(1, options.MaxLoadingThreads);
if (!options.Append)
{
// Inform context to ignore output plugin in case it is active
context.IgnorePlugins.Add(targetPluginFileName);
}
// Index all forms except hidden (such as cells and worlds)
context.Load();
if (options.Append && context.Plugins.Exists(targetPluginFileName) && context.Plugins.Count > 0)
{
// Make sure output plugin is loaded last when appending
var plugin = context.Plugins[(byte)(context.Plugins.Count - 1)];
if (!plugin.FileName.Equals(targetPluginFileName))
{
throw new ApplicationException("Output plugin must be loaded last when appending changes.");
}
}
using (RuleEngine engine = new RuleEngine(context))
{
engine.RulesFolder = options.RulesFolder;
// Apply debug scope to rule engine
if (!string.IsNullOrEmpty(options.DebugScope))
{
if (options.DebugScope == "*")
{
engine.DebugAll = true;
}
else
{
var parts = options.DebugScope.Split('/', '\\');
if (parts.Length > 0)
{
engine.DebugPluginFileName = parts[0];
}
if (parts.Length > 1)
{
engine.DebugRuleFileName = parts[1];
}
}
}
foreach (var param in options.Parameters)
{
var split = param.Split('=');
if (split.Length != 2 || !split[0].Contains(':'))
{
Log.Warning("Ignored malformatted parameter: '{0}' Expected format is 'plugin:param=value'", param);
}
engine.Params.Add(split[0], split[1]);
}
// Load rules
engine.Load();
#if DEBUG
// Load supported forms in debug mode
context.LoadForms(f => context.IsSupportedFormKind(f.FormKind));
#else
// Load all indexed forms in release mode
context.LoadForms();
#endif
if (options.Append && context.Plugins.Exists(targetPluginFileName))
{
// Use existing plugin as the target plugin
engine.ActivePlugin = context.Plugins[targetPluginFileName];
}
else
{
// Create and set up target plugin
engine.ActivePlugin = context.CreatePlugin(targetPluginFileName);
engine.ActivePlugin.Author = options.Author ?? Program.GetProgramVersionInfo();
engine.ActivePlugin.Description = options.Description ?? string.Format("Generated by {0}", Program.GetProgramVersionInfo());
}
// See if target plugin exists
var targetPluginFile = fileProvider.GetDataFile(FileMode.Open, targetPluginFileName);
if (targetPluginFile.Exists())
{
Log.Info("Target plugin {0} already exists and will be overwriten, however previously used FormIDs will be preserved if possible.", targetPluginFileName);
try
{
var inspector = new PluginInspector(context, targetPluginFile);
foreach (var record in inspector.NewRecords)
{
engine.ActivePlugin.ReserveFormId(record.FormId, record.EditorId);
}
}
catch (Exception ex)
{
Log.Warning("Previously used Form IDs cannot be preserved because target plugin {0} could not be read: {1}", targetPluginFileName, ex.ToString());
}
}
engine.Run();
if (!options.KeepDirtyEdits)
{
engine.ActivePlugin.PurgeDirtyEdits();
}
// Prepare list of master to be removed by force
IEnumerable <string> removeMasters = options.RemovedMasters != null?options.RemovedMasters.Split(',') : null;
// Save target plugin
engine.ActivePlugin.Save(removeMasters);
}
}
}
catch (UserAbortException ex)
{
Log.Error("Program aborted: " + ex.Message);
}
#if !DEBUG
catch (Exception ex)
{
Log.Error("Program error: " + ex.Message);
Log.Error(ex.ToString());
}
#endif
}
return;
}
