static StockNameTable ProcessListOfFiles(string listFile, DateTime startDate, DateTime endDate, string outputFileFolder)
{
if (string.IsNullOrEmpty(listFile) || string.IsNullOrEmpty(outputFileFolder))
{
throw new ArgumentNullException();
}
var table = new StockNameTable();
// Get all input files from list file
var files = File.ReadAllLines(listFile, Encoding.UTF8);
Parallel.ForEach(
files,
file =>
{
if (!String.IsNullOrWhiteSpace(file))
{
var stockName = ProcessOneFile(file.Trim(), startDate, endDate, outputFileFolder);
if (stockName != null)
{
lock (table)
{
table.AddStock(stockName);
}
}
}
Console.Write(".");
});
return(table);
}
private static StockNameTable FetchReports(
IReportFetcher fetcher,
StockNameTable stocks,
string outputFolder,
int intervalInSecond,
int randomRangeInSecond)
{
StockNameTable failedStocks = new StockNameTable();
string defaultSuffix = fetcher.GetDefaultSuffixOfOutputFile();
Random rand = new Random();
foreach (var stock in stocks.StockNames)
{
string errorMessage;
string outputFile = string.Format("{0}.{1}", stock.Code, defaultSuffix);
outputFile = Path.Combine(outputFolder, outputFile);
bool succeeded = fetcher.FetchReport(stock, outputFile, out errorMessage);
if (!succeeded)
{
Console.WriteLine("Fetch report for {0} failed. Error: {1}", stock.Code, errorMessage);
failedStocks.AddStock(stock);
}
Thread.Sleep((intervalInSecond + rand.Next(randomRangeInSecond)) * 1000);
}
return(failedStocks);
}
public static StockHistoryData Load(string file, StockNameTable nameTable)
{
StockHistoryData data;
data = _cache.GetOrAdd(file, (string f) => StockHistoryData.LoadFromFile(f, DateTime.MinValue, DateTime.MaxValue, nameTable));
return(data);
}
private void LoadDataSettingsFile(string fileName)
{
_stockDataSettings = ChinaStockDataSettings.LoadFromFile(fileName);
_stockNameTable = new StockNameTable(_stockDataSettings.StockNameTableFile);
// fill the codes and names to grid view
var stockProperties = _stockNameTable.StockNames
.Select(sn => new StockProperty()
{
Code = sn.Code,
Name = string.Join("|", sn.Names)
})
.OrderBy(sp => sp.Code)
.ToArray();
dataGridViewCodes.DataSource = new SortableBindingList <StockProperty>(stockProperties);
// reset data accessor (cache)
ChinaStockDataAccessor.Reset();
}
public ChinaStockDataProvider(StockNameTable nameTable, string[] dataFiles, DateTime start, DateTime end, int warmupDataSize)
{
if (nameTable == null)
{
throw new ArgumentNullException("nameTable");
}
if (dataFiles == null || dataFiles.Length == 0)
{
throw new ArgumentNullException("dataFiles");
}
if (start > end)
{
throw new ArgumentException("start time should not be greater than end time");
}
if (warmupDataSize < 0)
{
throw new ArgumentOutOfRangeException("warm up data size can't be negative");
}
// load data
var allTradingData = new List <StockHistoryData>(dataFiles.Length);
var allFirstNonWarmupDataPeriods = new Dictionary <string, DateTime>();
ChinaStockDataAccessor.Initialize();
// shuffle data files to avoid data conflict when multiple data provider are initialized simultaneously
// the algorithm here is a hacking way, but it works well
dataFiles = dataFiles.OrderBy(s => Guid.NewGuid()).ToArray();
Parallel.ForEach(
dataFiles,
file =>
{
if (!String.IsNullOrWhiteSpace(file) && File.Exists(file))
{
var data = ChinaStockDataAccessor.Load(file, nameTable);
if (data == null || data.DataOrderedByTime.Length == 0)
{
return;
}
int startIndex;
int endIndex;
Split(data.DataOrderedByTime, start, end, out startIndex, out endIndex);
if (startIndex > endIndex)
{
// no any trading data, ignore it.
return;
}
// now we have ensured endIndex >= startIndex;
DateTime firstNonWarmupDataPeriod = DateTime.MaxValue;
Bar[] tradingData;
if (warmupDataSize > 0)
{
if (startIndex >= warmupDataSize)
{
firstNonWarmupDataPeriod = data.DataOrderedByTime[startIndex].Time;
tradingData = new Bar[endIndex - startIndex + warmupDataSize + 1];
Array.Copy(data.DataOrderedByTime, startIndex - warmupDataSize, tradingData, 0, tradingData.Length);
}
else if (endIndex >= warmupDataSize)
{
firstNonWarmupDataPeriod = data.DataOrderedByTime[warmupDataSize].Time;
tradingData = new Bar[endIndex + 1];
Array.Copy(data.DataOrderedByTime, 0, tradingData, 0, tradingData.Length);
}
else
{
// all data are for warming up and there is no official data for evaluation or other
// usage, so we just skip the data.
return;
}
}
else
{
firstNonWarmupDataPeriod = data.DataOrderedByTime[startIndex].Time;
tradingData = new Bar[endIndex - startIndex + 1];
Array.Copy(data.DataOrderedByTime, startIndex, tradingData, 0, tradingData.Length);
}
// check if data is ok
//Bar lastBar = tradingData[0];
//for (int i = 1; i < tradingData.Length; ++i)
//{
// Bar bar = tradingData[i];
// if (bar.HighestPrice > lastBar.ClosePrice * 3
// || bar.LowestPrice < lastBar.ClosePrice * 0.5)
// {
// // invalid data
// //Console.WriteLine(
// // "Invalid data file {3}/{4}, High:{0:0.000} Low:{1:0.000} Prev.Close: {2:0.000}",
// // bar.HighestPrice,
// // bar.LowestPrice,
// // lastBar.ClosePrice,
// // data.Name.Code,
// // data.Name.Names.Last());
// return;
// }
// lastBar = bar;
//}
lock (allFirstNonWarmupDataPeriods)
{
allFirstNonWarmupDataPeriods.Add(data.Name.Code, firstNonWarmupDataPeriod);
}
lock (allTradingData)
{
allTradingData.Add(new StockHistoryData(data.Name, data.IntervalInSecond, tradingData));
}
}
});
Console.WriteLine("{0}/{1} data files are loaded", allTradingData.Count(), dataFiles.Count());
// get all periods.
_allPeriodsOrdered = allTradingData
.SelectMany(s => s.DataOrderedByTime.Select(b => b.Time))
.GroupBy(dt => dt)
.Select(g => g.Key)
.OrderBy(dt => dt)
.ToArray();
if (_allPeriodsOrdered.Length == 0)
{
throw new InvalidDataException("No any trading data are loaded, please adjust the time range");
}
for (var i = 0; i < _allPeriodsOrdered.Length; ++i)
{
_periodIndices.Add(_allPeriodsOrdered[i], i);
}
// build trading objects
var tempTradingData = allTradingData.OrderBy(t => t.Name.Code).ToArray();
_stocks = Enumerable.Range(0, tempTradingData.Length)
.Select(i => (ITradingObject) new ChinaStock(i, tempTradingData[i].Name))
.ToArray();
for (var i = 0; i < _stocks.Length; ++i)
{
_stockIndices.Add(_stocks[i].Code, i);
}
// prepare first non-warmup data periods
_firstNonWarmupDataPeriods = new DateTime[_stocks.Length];
for (var i = 0; i < _firstNonWarmupDataPeriods.Length; ++i)
{
_firstNonWarmupDataPeriods[i] = allFirstNonWarmupDataPeriods.ContainsKey(_stocks[i].Code)
? allFirstNonWarmupDataPeriods[_stocks[i].Code]
: DateTime.MaxValue;
}
// expand data to #period * #stock
_allTradingData = new Bar[_allPeriodsOrdered.Length][];
for (var i = 0; i < _allTradingData.Length; ++i)
{
_allTradingData[i] = new Bar[_stocks.Length];
}
foreach (var historyData in allTradingData)
{
var stockIndex = GetIndexOfTradingObject(historyData.Name.Code);
var data = historyData.DataOrderedByTime;
var dataIndex = 0;
for (var periodIndex = 0; periodIndex < _allPeriodsOrdered.Length; ++periodIndex)
{
if (dataIndex >= data.Length || _allPeriodsOrdered[periodIndex] < data[dataIndex].Time)
{
_allTradingData[periodIndex][stockIndex].Time = Bar.InvalidTime;
}
else if (_allPeriodsOrdered[periodIndex] == data[dataIndex].Time)
{
_allTradingData[periodIndex][stockIndex] = data[dataIndex];
++dataIndex;
}
else
{
// impossible!
throw new InvalidOperationException("Logic error");
}
}
}
}
static void Run(Options options)
{
options.FinanceReportServerAddress = ConfigurationManager.AppSettings[ServerKey];
options.Print(Console.Out);
// create stock name table
StockNameTable stockNameTable
= new StockNameTable
(
options.StockNameTable,
(string invalidLine) =>
{
Console.WriteLine("Invalid line: {0}", invalidLine);
return(true);
}
);
// create report fetcher
IReportFetcher fetcher = ReportFetcherFactory.Create(options.FinanceReportServerAddress);
// make sure output folder exists, otherwise create it.
string outputFolder = Path.GetFullPath(options.OutputFolder);
if (!Directory.Exists(outputFolder))
{
try
{
Directory.CreateDirectory(outputFolder);
}
catch (IOException ex)
{
Console.WriteLine("create folder {0} failed, error:\n{1}", outputFolder, ex.ToString());
}
}
StockNameTable lastRoundStocks = stockNameTable;
StockNameTable failedStocks = null;
while (true)
{
failedStocks = FetchReports(fetcher, lastRoundStocks, outputFolder, options.IntervalInSecond, options.RandomRange);
if (failedStocks == null || failedStocks.Count == 0 || failedStocks.Count == lastRoundStocks.Count)
{
// break when there is no progress.
break;
}
lastRoundStocks = failedStocks;
}
if (failedStocks != null && failedStocks.Count > 0)
{
Console.WriteLine("Following stocks' report are not been fetched:");
Console.WriteLine("==============================================");
foreach (var stock in failedStocks.StockNames)
{
Console.WriteLine("{0}", stock.Code);
}
Console.WriteLine("==============================================");
}
Console.WriteLine("Done.");
}
static void Run(Options options)
{
// check the validation of options
CheckOptions(options);
// generate example files if necessary
if (options.ShouldGenerateExampleFiles)
{
GenerateExampleFiles(options);
return;
}
// register handler for Ctrl+C/Ctrl+Break
Console.CancelKeyPress += ConsoleCancelKeyPress;
// load settings from files
var tradingSettings = TradingSettings.LoadFromFile(options.TradingSettingsFile);
var combinedStrategySettings = CombinedStrategySettings.LoadFromFile(options.CombinedStrategySettingsFile);
var stockDataSettings = ChinaStockDataSettings.LoadFromFile(options.StockDataSettingsFile);
// load codes and stock name table
var stockNameTable = new StockNameTable(stockDataSettings.StockNameTableFile);
var codes = LoadCodeOfStocks(options.CodeFile);
// load stock block relationship if necessary, and filter codes
StockBlockRelationshipManager stockBlockRelationshipManager = null;
if (!string.IsNullOrWhiteSpace(options.StockBlockRelationshipFile))
{
stockBlockRelationshipManager = LoadStockBlockRelationship(options.StockBlockRelationshipFile);
// filter stock block relationship for loaded codes only
stockBlockRelationshipManager = stockBlockRelationshipManager.CreateSubsetForStocks(codes);
// codes will be updated according to stock-block relationships
codes = stockBlockRelationshipManager.Stocks;
}
var allDataFiles = codes
.Select(stockDataSettings.BuildActualDataFilePathAndName)
.ToArray();
// dump data for temporary usage, will be commented out in real code
// create data provider
//var dumpDataProvider
// = new ChinaStockDataProvider(
// stockNameTable,
// allDataFiles,
// options.StartDate,
// options.EndDate,
// 0);
//DumpData(dumpDataProvider);
// generate evaluation time intervals
var intervals =
GenerateIntervals(
options.StartDate,
options.EndDate,
options.YearInterval)
.ToArray();
using (_contextManager = new EvaluationResultContextManager(options.EvaluationName))
{
// save evluation summary
var evaluationSummary = new EvaluationSummary
{
StrategySettings = combinedStrategySettings.GetActiveSettings(),
TradingSettings = tradingSettings,
DataSettings = stockDataSettings,
StartTime = options.StartDate,
EndTime = options.EndDate,
YearInterval = options.YearInterval,
ObjectNames = codes
.Select(c => stockNameTable.ContainsStock(c)
? c + '|' + stockNameTable[c].Names[0]
: c)
.ToArray()
};
_contextManager.SaveEvaluationSummary(evaluationSummary);
Action <Tuple <DateTime, DateTime> > evaluatingAction =
(Tuple <DateTime, DateTime> interval) =>
{
if (_toBeStopped)
{
return;
}
// initialize data provider
var dataProvider
= new ChinaStockDataProvider(
stockNameTable,
allDataFiles,
interval.Item1, // interval start date
interval.Item2, // interval end date
options.WarmupPeriods);
var finalCodes = dataProvider.GetAllTradingObjects().Select(to => to.Code);
var filteredStockBlockRelationshipManager = stockBlockRelationshipManager == null
? null
: stockBlockRelationshipManager.CreateSubsetForStocks(finalCodes);
// initialize combined strategy assembler
var combinedStrategyAssembler = new CombinedStrategyAssembler(combinedStrategySettings, true);
var strategyInstances
= new List <Tuple <CombinedStrategy, IDictionary <ParameterAttribute, object> > >();
IDictionary <ParameterAttribute, object> values;
while ((values = combinedStrategyAssembler.GetNextSetOfParameterValues()) != null)
{
var strategy = combinedStrategyAssembler.NewStrategy();
strategyInstances.Add(Tuple.Create(strategy, values));
}
if (strategyInstances.Any())
{
// initialize ResultSummary
ResultSummary.Initialize(strategyInstances.First().Item2, options.EnableERatioOutput);
}
SetTotalStrategyNumber(intervals.Count() * strategyInstances.Count());
try
{
Parallel.For(
0,
strategyInstances.Count,
// below line is for performance profiling only.
new ParallelOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount * 2
},
t =>
{
for (int i = 0; i < options.AccountNumber; ++i)
{
if (_toBeStopped)
{
return;
}
ICapitalManager capitalManager = options.ProportionOfCapitalForIncrementalPosition > 0.0
? (ICapitalManager) new AdvancedCapitalManager(options.InitialCapital, options.ProportionOfCapitalForIncrementalPosition)
: (ICapitalManager) new SimpleCapitalManager(options.InitialCapital)
;
EvaluateStrategy(
options.AccountNumber,
i,
_contextManager,
strategyInstances[t].Item1,
strategyInstances[t].Item2,
interval.Item1,
interval.Item2,
capitalManager,
dataProvider,
filteredStockBlockRelationshipManager,
options.ShouldDumpData,
tradingSettings);
}
IncreaseProgress();
// reset the strategy object to ensure the referred IEvaluationContext object being
// released, otherwise it will never be released until all strategies are evaluated.
// This is a bug hid for long time.
strategyInstances[t] = null;
});
}
catch
{
_toBeStopped = true;
}
finally
{
lock (_contextManager)
{
// save result summary
_contextManager.SaveResultSummaries();
}
}
};
if (options.ParallelExecution)
{
Parallel.ForEach(intervals, evaluatingAction);
}
else
{
foreach (var interval in intervals)
{
evaluatingAction(interval);
}
}
}
_contextManager = null;
Console.WriteLine();
Console.WriteLine("Done.");
}
static void Run(Options options)
{
// check the validation of options
CheckOptions(options);
// register handler for Ctrl+C/Ctrl+Break
Console.CancelKeyPress += ConsoleCancelKeyPress;
// load settings from files
var stockDataSettings = ChinaStockDataSettings.LoadFromFile(options.StockDataSettingsFile);
// load codes and stock name table
var stockNameTable = new StockNameTable(stockDataSettings.StockNameTableFile);
var codes = LoadCodeOfStocks(options.CodeFile);
var allDataFiles = codes
.Select(stockDataSettings.BuildActualDataFilePathAndName)
.ToArray();
// initialize data provider
var dataProvider
= new ChinaStockDataProvider(
stockNameTable,
allDataFiles,
options.StartDate,
options.EndDate,
0);
try
{
var tradingObjects = dataProvider.GetAllTradingObjects();
var barCounters = CreateBarCounters().ToArray();
if (barCounters != null && barCounters.Count() > 0)
{
Parallel.For(
0,
tradingObjects.Length,
// below line is for performance profiling only.
new ParallelOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount * 2
},
t =>
{
if (_toBeStopped)
{
return;
}
var tradingObject = tradingObjects[t];
var bars = dataProvider.GetAllBarsForTradingObject(tradingObject.Index);
foreach (var counter in barCounters)
{
counter.Count(bars, tradingObject);
}
});
foreach (var counter in barCounters)
{
string outputFile = counter.Name + "." + options.OutputFile;
counter.SaveResults(outputFile);
}
}
}
catch
{
_toBeStopped = true;
}
Console.WriteLine();
Console.WriteLine("Done.");
}
static int Run(Options options)
{
if (string.IsNullOrEmpty(options.OutputFileFolder))
{
Console.WriteLine("output file folder is empty");
return(-2);
}
var folder = Path.GetFullPath(options.OutputFileFolder);
// try to create output file folder if it does not exist
if (!Directory.Exists(folder))
{
try
{
Directory.CreateDirectory(folder);
}
catch (Exception ex)
{
Console.WriteLine("Create output file folder {0} failed. Exception: \n{1}", folder, ex);
return(-3);
}
}
StockNameTable table;
if (!string.IsNullOrEmpty(options.InputFile))
{
// single input file
StockName name = ProcessOneFile(options.InputFile, options.StartDate, options.EndDate, folder);
table = new StockNameTable();
if (name != null)
{
table.AddStock(name);
}
}
else
{
table = ProcessListOfFiles(options.InputFileList, options.StartDate, options.EndDate, folder);
}
if (!string.IsNullOrEmpty(options.NameFile))
{
Console.WriteLine();
Console.WriteLine("Output name file: {0}", options.NameFile);
File.WriteAllLines(
options.NameFile,
table.StockNames.Select(sn => sn.ToString()).ToArray(),
Encoding.UTF8);
}
if (!string.IsNullOrEmpty(options.CodeFile))
{
Console.WriteLine();
Console.WriteLine("Output code file: {0}", options.CodeFile);
File.WriteAllLines(
options.CodeFile,
table.StockNames.Select(sn => sn.Code).ToArray(),
Encoding.UTF8);
}
Console.WriteLine("Done.");
return(0);
}
static void Run(Options options)
{
// check the validation of options
CheckOptions(options);
// load settings from files
var combinedStrategySettings = CombinedStrategySettings.LoadFromFile(options.CombinedStrategySettingsFile);
var stockDataSettings = ChinaStockDataSettings.LoadFromFile(options.StockDataSettingsFile);
var positions = LoadPositions(options.PositionFile);
// load codes and stock name table
var stockNameTable = new StockNameTable(stockDataSettings.StockNameTableFile);
var codes = LoadCodeOfStocks(options.CodeFile);
// load stock block relationship if necessary, and filter codes
StockBlockRelationshipManager stockBlockRelationshipManager = null;
if (!string.IsNullOrWhiteSpace(options.StockBlockRelationshipFile))
{
stockBlockRelationshipManager = LoadStockBlockRelationship(options.StockBlockRelationshipFile);
// filter stock block relationship for loaded codes only
stockBlockRelationshipManager = stockBlockRelationshipManager.CreateSubsetForStocks(codes);
// codes will be updated according to stock-block relationships
codes = stockBlockRelationshipManager.Stocks;
}
var allDataFiles = codes
.Select(stockDataSettings.BuildActualDataFilePathAndName)
.ToArray();
// initialize data provider
var dataProvider
= new ChinaStockDataProvider(
stockNameTable,
allDataFiles,
options.StartDate,
options.EndDate,
options.WarmupPeriods);
var finalCodes = dataProvider.GetAllTradingObjects().Select(to => to.Code);
var filteredStockBlockRelationshipManager = stockBlockRelationshipManager == null
? null
: stockBlockRelationshipManager.CreateSubsetForStocks(finalCodes);
if (filteredStockBlockRelationshipManager != null)
{
var filteredCodes = filteredStockBlockRelationshipManager.Stocks;
var filteredDataFiles = filteredCodes
.Select(stockDataSettings.BuildActualDataFilePathAndName)
.ToArray();
// rebuild data provider according to filtered codes
dataProvider = new ChinaStockDataProvider(
stockNameTable,
filteredDataFiles,
options.StartDate,
options.EndDate,
options.WarmupPeriods);
}
// initialize combined strategy assembler
var combinedStrategyAssembler = new CombinedStrategyAssembler(combinedStrategySettings, false);
var strategyInstances
= new List <Tuple <CombinedStrategy, IDictionary <ParameterAttribute, object> > >();
IDictionary <ParameterAttribute, object> values;
while ((values = combinedStrategyAssembler.GetNextSetOfParameterValues()) != null)
{
var strategy = combinedStrategyAssembler.NewStrategy();
strategyInstances.Add(Tuple.Create(strategy, values));
}
if (strategyInstances.Count != 1)
{
throw new InvalidDataException("Strategy has more or less than one instance, please check strategy settings");
}
string predictionContextDirectory = options.PredicationName + "_" + DateTime.Now.ToString("yyyyMMddTHHmmss");
if (!Directory.Exists(predictionContextDirectory))
{
Directory.CreateDirectory(predictionContextDirectory);
}
using (PredicationContext context = new PredicationContext(predictionContextDirectory))
{
PredicateStrategy(
context,
strategyInstances.First().Item1,
strategyInstances.First().Item2,
options.StartDate,
options.InitialCapital,
options.CurrentCapital,
positions,
dataProvider,
filteredStockBlockRelationshipManager,
options.PositionFrozenDays);
}
Console.WriteLine();
Console.WriteLine("Done.");
}
