public void TestRegression()
{
FileInfo rawFile = TEMP_DIR.CreateFile("simple.csv");
FileInfo egaFile = TEMP_DIR.CreateFile("simple.ega");
FileInfo outputFile = TEMP_DIR.CreateFile("simple_output.csv");
FileUtil.CopyResource("Encog.Resources.simple.csv", rawFile);
FileUtil.CopyResource("Encog.Resources.simple-r.ega", egaFile);
EncogAnalyst analyst = new EncogAnalyst();
analyst.Load(egaFile);
analyst.ExecuteTask("task-full");
ReadCSV csv = new ReadCSV(outputFile.ToString(), true, CSVFormat.English);
while (csv.Next())
{
double diff = Math.Abs(csv.GetDouble(2) - csv.GetDouble(4));
Assert.IsTrue(diff < 1.5);
}
Assert.AreEqual(4, analyst.Script.Fields.Length);
Assert.AreEqual(3, analyst.Script.Fields[3].ClassMembers.Count);
csv.Close();
}
public void TestClassification()
{
FileInfo rawFile = TEMP_DIR.CreateFile("simple.csv");
FileInfo egaFile = TEMP_DIR.CreateFile("simple.ega");
FileInfo outputFile = TEMP_DIR.CreateFile("simple_output.csv");
FileUtil.CopyResource("Encog.Resources.simple.csv", rawFile);
FileUtil.CopyResource("Encog.Resources.simple-c.ega", egaFile);
EncogAnalyst analyst = new EncogAnalyst();
analyst.AddAnalystListener(new ConsoleAnalystListener());
analyst.Load(egaFile);
analyst.ExecuteTask("task-full");
ReadCSV csv = new ReadCSV(outputFile.ToString(), true, CSVFormat.English);
while (csv.Next())
{
Assert.AreEqual(csv.Get(3), csv.Get(4));
}
Assert.AreEqual(4, analyst.Script.Fields.Length);
Assert.AreEqual(3, analyst.Script.Fields[3].ClassMembers.Count);
csv.Close();
}
/// <summary>
/// Load the specified financial data.
/// </summary>
/// <param name="ticker">The ticker symbol to load.</param>
/// <param name="dataNeeded">The financial data needed.</param>
/// <param name="from">The beginning date to load data from.</param>
/// <param name="to">The ending date to load data to.</param>
/// <returns>A collection of LoadedMarketData objects that represent the data
/// loaded.</returns>
public ICollection<LoadedMarketData> Load(TickerSymbol ticker,
IList<MarketDataType> dataNeeded, DateTime from,
DateTime to)
{
ICollection<LoadedMarketData> result =
new List<LoadedMarketData>();
Uri url = BuildURL(ticker, from, to);
WebRequest http = WebRequest.Create(url);
var response = (HttpWebResponse) http.GetResponse();
using (Stream istream = response.GetResponseStream())
{
var csv = new ReadCSV(istream, true, CSVFormat.DecimalPoint);
while (csv.Next())
{
DateTime date = csv.GetDate("date");
double adjClose = csv.GetDouble("adj close");
double open = csv.GetDouble("open");
double close = csv.GetDouble("close");
double high = csv.GetDouble("high");
double low = csv.GetDouble("low");
double volume = csv.GetDouble("volume");
var data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.AdjustedClose, adjClose);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Close, close);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
istream.Close();
}
return result;
}
/// <summary>
/// Process and balance the data.
/// </summary>
/// <param name="outputFile">The output file to write data to.</param>
/// <param name="targetField"></param>
/// <param name="countPer">The desired count per class.</param>
public void Process(FileInfo outputFile, int targetField,
int countPer)
{
ValidateAnalyzed();
StreamWriter tw = PrepareOutputFile(outputFile);
_counts = new Dictionary <String, Int32>();
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
ResetStatus();
while (csv.Next() && !ShouldStop())
{
var row = new LoadedRow(csv);
UpdateStatus(false);
String key = row.Data[targetField];
int count;
if (!_counts.ContainsKey(key))
{
count = 0;
}
else
{
count = _counts[key];
}
if (count < countPer)
{
WriteRow(tw, row);
count++;
}
_counts[key] = count;
}
ReportDone(false);
csv.Close();
tw.Close();
}
/// <summary>
/// Load the specified financial data.
/// </summary>
/// <param name="ticker">The ticker symbol to load.</param>
/// <param name="dataNeeded">The financial data needed.</param>
/// <param name="from">The beginning date to load data from.</param>
/// <param name="to">The ending date to load data to.</param>
/// <returns>A collection of LoadedMarketData objects that represent the data
/// loaded.</returns>
public ICollection <LoadedMarketData> Load(TickerSymbol ticker, IList <MarketDataType> dataNeeded, DateTime from, DateTime to)
{
ICollection <LoadedMarketData> result =
new List <LoadedMarketData>();
Uri url = BuildURL(ticker, from, to);
WebRequest http = WebRequest.Create(url);
var response = (HttpWebResponse)http.GetResponse();
using (Stream istream = response.GetResponseStream())
{
var csv = new ReadCSV(istream, true, CSVFormat.DecimalPoint);
while (csv.Next())
{
DateTime date = csv.GetDate("date");
double adjClose = csv.GetDouble("adj close");
double open = csv.GetDouble("open");
double close = csv.GetDouble("close");
double high = csv.GetDouble("high");
double low = csv.GetDouble("low");
double volume = csv.GetDouble("volume");
var data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.AdjustedClose, adjClose);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Close, close);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
istream.Close();
}
return(result);
}
/// <summary>
/// Used to calibrate the training file.
/// </summary>
/// <param name="file">The file to consider.</param>
protected void CalibrateFile(string file)
{
var csv = new ReadCSV(file, true, CSVFormat.English);
while (csv.Next())
{
var a = new double[1];
double close = csv.GetDouble(1);
const int fastIndex = 2;
const int slowIndex = fastIndex + Config.InputWindow;
a[0] = close;
for (int i = 0; i < Config.InputWindow; i++)
{
double fast = csv.GetDouble(fastIndex + i);
double slow = csv.GetDouble(slowIndex + i);
if (!double.IsNaN(fast) && !double.IsNaN(slow))
{
double diff = (fast - slow) / Config.PipSize;
_minDifference = Math.Min(_minDifference, diff);
_maxDifference = Math.Max(_maxDifference, diff);
}
}
_window.Add(a);
if (_window.IsFull())
{
double max = (_window.CalculateMax(0, Config.InputWindow) - close) / Config.PipSize;
double min = (_window.CalculateMin(0, Config.InputWindow) - close) / Config.PipSize;
double o = Math.Abs(max) > Math.Abs(min) ? max : min;
_maxPiPs = Math.Max(_maxPiPs, (int)o);
_minPiPs = Math.Min(_minPiPs, (int)o);
}
}
}
/// <summary>
/// Construct the object.
/// </summary>
/// <param name="filename">The filename.</param>
/// <param name="headers">False if headers are not extended.</param>
/// <param name="format">The CSV format.</param>
public CSVHeaders(FileInfo filename, bool headers,
CSVFormat format)
{
_headerList = new List <String>();
_columnMapping = new Dictionary <String, Int32>();
ReadCSV csv = null;
try
{
csv = new ReadCSV(filename.ToString(), headers, format);
if (csv.Next())
{
if (headers)
{
foreach (String str in csv.ColumnNames)
{
_headerList.Add(str);
}
}
else
{
for (int i = 0; i < csv.ColumnCount; i++)
{
_headerList.Add("field:" + (i + 1));
}
}
}
Init();
}
finally
{
if (csv != null)
{
csv.Close();
}
}
}
/// <summary>
/// Process the input file.
/// </summary>
/// <param name="outputFile">The output file to write to.</param>
public void Process(FileInfo outputFile)
{
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
StreamWriter tw = PrepareOutputFile(outputFile);
_filteredCount = 0;
ResetStatus();
while (csv.Next() && !ShouldStop())
{
UpdateStatus(false);
var row = new LoadedRow(csv);
if (ShouldProcess(row))
{
WriteRow(tw, row);
_filteredCount++;
}
}
ReportDone(false);
tw.Close();
csv.Close();
}
/// <summary>
/// Perform a basic analyze of the file. This method is used mostly
/// internally.
/// </summary>
///
public void PerformBasicCounts()
{
if (_outputFormat == null)
{
_outputFormat = _inputFormat;
}
ResetStatus();
int rc = 0;
var csv = new ReadCSV(_inputFilename.ToString(),
_expectInputHeaders, _inputFormat);
while (csv.Next() && !_cancel)
{
UpdateStatus(true);
rc++;
}
_recordCount = rc;
_columnCount = csv.ColumnCount;
ReadHeaders(csv);
csv.Close();
ReportDone(true);
}
/// <summary>
/// Normalize the input file. Write to the specified file.
/// </summary>
///
/// <param name="file">The file to write to.</param>
public void Normalize(FileInfo file)
{
if (_analyst == null)
{
throw new EncogError(
"Can't normalize yet, file has not been analyzed.");
}
ReadCSV csv = null;
StreamWriter tw = null;
try
{
csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, InputFormat);
file.Delete();
tw = new StreamWriter(file.OpenWrite());
// write headers, if needed
if (ProduceOutputHeaders)
{
WriteHeaders(tw);
}
ResetStatus();
int outputLength = _analyst.DetermineTotalColumns();
// write file contents
while (csv.Next() && !ShouldStop())
{
UpdateStatus(false);
double[] output = ExtractFields(
_analyst, _analystHeaders, csv, outputLength,
false);
if (_series.TotalDepth > 1)
{
output = _series.Process(output);
}
if (output != null)
{
var line = new StringBuilder();
NumberList.ToList(OutputFormat, line, output);
tw.WriteLine(line);
}
}
}
catch (IOException e)
{
throw new QuantError(e);
}
finally
{
ReportDone(false);
if (csv != null)
{
try
{
csv.Close();
}
catch (Exception ex)
{
EncogLogging.Log(ex);
}
}
if (tw != null)
{
try
{
tw.Close();
}
catch (Exception ex)
{
EncogLogging.Log(ex);
}
}
}
}
/// <summary>
/// Load the specified financial data.
/// </summary>
/// <param name="ticker">The ticker symbol to load.</param>
/// <param name="dataNeeded">The financial data needed.</param>
/// <param name="from">The beginning date to load data from.</param>
/// <param name="to">The ending date to load data to.</param>
/// <returns>A collection of LoadedMarketData objects that represent the data
/// loaded.</returns>
public ICollection<LoadedMarketData> Load(TickerSymbol ticker,
IList<MarketDataType> dataNeeded, DateTime from,
DateTime to)
{
ICollection<LoadedMarketData> result =
new List<LoadedMarketData>();
Uri url = buildURL(ticker, from, to);
WebRequest http = HttpWebRequest.Create(url);
HttpWebResponse response = (HttpWebResponse)http.GetResponse();
using (Stream istream = response.GetResponseStream())
{
ReadCSV csv = new ReadCSV(istream, true, CSVFormat.DECIMAL_POINT);
while (csv.Next())
{
DateTime date = csv.GetDate("date");
double adjClose = csv.GetDouble("adj close");
double open = csv.GetDouble("open");
double close = csv.GetDouble("close");
double high = csv.GetDouble("high");
double low = csv.GetDouble("low");
double volume = csv.GetDouble("volume");
LoadedMarketData data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.ADJUSTED_CLOSE, adjClose);
data.SetData(MarketDataType.OPEN, open);
data.SetData(MarketDataType.CLOSE, close);
data.SetData(MarketDataType.HIGH, high);
data.SetData(MarketDataType.LOW, low);
data.SetData(MarketDataType.OPEN, open);
data.SetData(MarketDataType.VOLUME, volume);
result.Add(data);
}
csv.Close();
istream.Close();
}
return result;
}
/// <summary>
/// Process the file.
/// </summary>
///
/// <param name="outputFile">The output file.</param>
/// <param name="method">THe method to use.</param>
public void Process(FileInfo outputFile, IMLMethod method)
{
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
IMLData output;
foreach (AnalystField field in _analyst.Script.Normalize.NormalizedFields)
{
field.Init();
}
int outputLength = _analyst.DetermineTotalInputFieldCount();
StreamWriter tw = PrepareOutputFile(outputFile);
ResetStatus();
while (csv.Next())
{
UpdateStatus(false);
var row = new LoadedRow(csv, _outputColumns);
double[] inputArray = AnalystNormalizeCSV.ExtractFields(_analyst,
_analystHeaders, csv, outputLength, true);
if (_series.TotalDepth > 1)
{
inputArray = _series.Process(inputArray);
}
if (inputArray != null)
{
IMLData input = new BasicMLData(inputArray);
// evaluation data
if ((method is IMLClassification) &&
!(method is IMLRegression))
{
// classification only?
var tmp = new BasicMLData(1);
tmp[0] = ((IMLClassification)method).Classify(input);
output = tmp;
}
else
{
// regression
output = ((IMLRegression)method).Compute(input);
}
// skip file data
int index = _fileColumns;
int outputIndex = 0;
// display output
foreach (AnalystField field in _analyst.Script.Normalize.NormalizedFields)
{
if (_analystHeaders.Find(field.Name) != -1)
{
if (field.Output)
{
if (field.Classify)
{
// classification
ClassItem cls = field.DetermineClass(
outputIndex, output);
outputIndex += field.ColumnsNeeded;
if (cls == null)
{
row.Data[index++] = "?Unknown?";
}
else
{
row.Data[index++] = cls.Name;
}
}
else
{
// regression
double n = output[outputIndex++];
n = field.DeNormalize(n);
row.Data[index++] = Format
.Format(n, Precision);
}
}
}
}
}
WriteRow(tw, row);
}
ReportDone(false);
tw.Close();
csv.Close();
}
/// <summary>
/// Reads the CSV and call loader.
/// Used internally to load the csv and place data in the marketdataset.
/// </summary>
/// <param name="symbol">The symbol.</param>
/// <param name="neededTypes">The needed types.</param>
/// <param name="from">From.</param>
/// <param name="to">To.</param>
/// <param name="File">The file.</param>
/// <returns></returns>
private ICollection <LoadedMarketData> ReadAndCallLoader(
TickerSymbol symbol,
IEnumerable <MarketDataType> neededTypes,
DateTime from,
DateTime to,
string File)
{
//We got a file, lets load it.
ICollection <LoadedMarketData> result = new List <LoadedMarketData>();
ReadCSV csv = new ReadCSV(File, true, CSVFormat.English);
//In case we want to use a different date format...and have used the SetDateFormat method, our DateFormat must then not be null..
//We will use the ?? operator to check for nullables.
csv.DateFormat = DateFormat ?? "yyyy-MM-dd HH:mm:ss";
csv.TimeFormat = "HH:mm:ss";
DateTime ParsedDate = from;
bool writeonce = true;
while (csv.Next())
{
DateTime date = csv.GetDate(0);
ParsedDate = date;
if (writeonce)
{
Console.WriteLine(@"First parsed date in csv:" + ParsedDate.ToShortDateString());
Console.WriteLine(@"Stopping at date:" + to.ToShortDateString());
Console.WriteLine(@"Current DateTime:" + ParsedDate.ToShortDateString() + @" Time:" +
ParsedDate.ToShortTimeString() + @" Asked Start date was " +
from.ToShortDateString());
writeonce = false;
}
if (ParsedDate >= from && ParsedDate <= to)
{
DateTime datex = csv.GetDate(0);
double open = csv.GetDouble(1);
double close = csv.GetDouble(2);
double high = csv.GetDouble(3);
double low = csv.GetDouble(4);
double volume = csv.GetDouble(5);
double range = Math.Abs(open - close);
double HighLowRange = Math.Abs(high - low);
double DirectionalRange = close - open;
LoadedMarketData data = new LoadedMarketData(datex, symbol);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Close, close);
data.SetData(MarketDataType.Volume, volume);
data.SetData(MarketDataType.RangeHighLow, Math.Round(HighLowRange, 6));
data.SetData(MarketDataType.RangeOpenClose, Math.Round(range, 6));
data.SetData(MarketDataType.RangeOpenCloseNonAbsolute, Math.Round(DirectionalRange, 6));
result.Add(data);
}
}
csv.Close();
return(result);
}
public void EvaluateNetwork()
{
BasicNetwork network = LoadNetwork();
DataNormalization norm = LoadNormalization();
var csv = new ReadCSV(_config.EvaluateFile.ToString(), false, ',');
var input = new double[norm.InputFields.Count];
var eqField = (OutputEquilateral) norm.FindOutputField(
typeof (OutputEquilateral), 0);
int correct = 0;
int total = 0;
while (csv.Next())
{
total++;
for (int i = 0; i < input.Length; i++)
{
input[i] = csv.GetDouble(i);
}
IMLData inputData = norm.BuildForNetworkInput(input);
IMLData output = network.Compute(inputData);
int coverTypeActual = DetermineTreeType(eqField, output);
int coverTypeIdeal = (int) csv.GetDouble(54) - 1;
KeepScore(coverTypeActual, coverTypeIdeal);
if (coverTypeActual == coverTypeIdeal)
{
correct++;
}
}
Console.WriteLine(@"Total cases:" + total);
Console.WriteLine(@"Correct cases:" + correct);
double percent = correct/(double) total;
Console.WriteLine(@"Correct percent:"
+ Format.FormatPercentWhole(percent));
for (int i = 0; i < 7; i++)
{
double p = (_treeCorrect[i]/(double) _treeCount[i]);
Console.WriteLine(@"Tree Type #" + i + @" - Correct/total: "
+ _treeCorrect[i] + @"/" + _treeCount[i] + @"("
+ Format.FormatPercentWhole(p) + @")");
}
}
public ICollection<LoadedMarketData> Load(
TickerSymbol ticker,
IList<MarketDataType> dataNeeded,
DateTime from,
DateTime to)
{
// TODO: nyyyyyyyaaagh!
ICollection<LoadedMarketData> result =
new List<LoadedMarketData>();
Uri url = BuildURL(ticker, from, to);
WebRequest http = HttpWebRequest.Create(url);
HttpWebResponse response = http.GetResponse() as HttpWebResponse;
using (Stream istream = response.GetResponseStream())
{
ReadCSV csv = new ReadCSV(
istream,
true,
CSVFormat.DECIMAL_POINT
);
while (csv.Next())
{
// todo: edit headers to match
DateTime date = csv.GetDate("DATE");
date =
date.Add(
new TimeSpan(
csv.GetDate("TIME").Hour,
csv.GetDate("TIME").Minute,
csv.GetDate("TIME").Second
)
);
double open = csv.GetDouble("OPEN");
double high = csv.GetDouble("MIN");
double low = csv.GetDouble("MAX");
double close = csv.GetDouble("CLOSE");
double volume = csv.GetDouble("VOLUME");
LoadedMarketData data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.OPEN, open);
data.SetData(MarketDataType.HIGH, high);
data.SetData(MarketDataType.LOW, low);
data.SetData(MarketDataType.CLOSE, close);
data.SetData(MarketDataType.VOLUME, volume);
result.Add(data);
}
csv.Close();
istream.Close();
}
return result;
}
public ICollection <LoadedMarketData> ReadAndCallLoader(TickerSymbol symbol, IList <MarketDataType> neededTypes, DateTime from, DateTime to, string File)
{
try
{
//We got a file, lets load it.
ICollection <LoadedMarketData> result = new List <LoadedMarketData>();
ReadCSV csv = new ReadCSV(File, true, CSVFormat.DecimalComma);
csv.DateFormat = "yyyy.MM.dd HH:mm";
DateTime ParsedDate = from;
// Time,Open,High,Low,Close,Volume
while (csv.Next() /*&& ParsedDate >= from && ParsedDate <= to*/)
{
DateTime date = csv.GetDate("Date");
if (date > to)
{
break;
}
if (date < from)
{
continue;
}
double close = csv.GetDouble("Close");
//double Bid= csv.GetDouble("Bid");
//double Ask = csv.GetDouble("Ask");
//double AskVolume = csv.GetDouble("AskVolume");
//double BidVolume= csv.GetDouble("BidVolume");
//double _trade = ( Bid + Ask ) /2;
//double _tradeSize = (AskVolume + BidVolume) / 2;
LoadedMarketData data = new LoadedMarketData(date, symbol);
data.SetData(MarketDataType.Close, close);
//data.SetData(MarketDataType.Trade, _trade);
//data.SetData(MarketDataType.Volume, _tradeSize);
result.Add(data);
Console.WriteLine("Current DateTime:" + date.ToShortDateString() + " Time:" + date.ToShortTimeString() + " Start date was " + from.ToShortDateString());
Console.WriteLine("Stopping at date:" + to.ToShortDateString());
//double open = csv.GetDouble("Open");
//double close = csv.GetDouble("High");
//double high = csv.GetDouble("Low");
//double low = csv.GetDouble("Close");
//double volume = csv.GetDouble("Volume");
//LoadedMarketData data = new LoadedMarketData(date, symbol);
//data.SetData(MarketDataType.Open, open);
//data.SetData(MarketDataType.High, high);
//data.SetData(MarketDataType.Low, low);
//data.SetData(MarketDataType.Close, close);
//data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
return(result);
}
catch (Exception ex)
{
Console.WriteLine("Something went wrong reading the csv");
Console.WriteLine("Something went wrong reading the csv:" + ex.Message);
}
Console.WriteLine("Something went wrong reading the csv");
return(null);
}
/// <summary>
/// Reads and parses CSV data from file
/// </summary>
/// <param name="ticker">Ticker associated with CSV file</param>
/// <param name="neededTypes">Columns to parse (headers)</param>
/// <param name="from">DateTime from</param>
/// <param name="to">DateTime to</param>
/// <param name="File">Filepath to CSV</param>
/// <returns>Marketdata</returns>
public ICollection <LoadedMarketData> ReadAndCallLoader(TickerSymbol ticker, IList <MarketDataType> neededTypes, DateTime from, DateTime to, string File)
{
try
{
LoadedFile = File;
Console.WriteLine("Loading instrument: " + ticker.Symbol + " from: " + File);
//We got a file, lets load it.
ICollection <LoadedMarketData> result = new List <LoadedMarketData>();
ReadCSV csv = new ReadCSV(File, true, LoadedFormat);
if (DateTimeDualColumn)
{
csv.DateFormat = DateFormat;
csv.TimeFormat = TimeFormat;
}
else
{
csv.DateFormat = DateFormat;
}
//"Date","Time","Open","High","Low","Close","Volume"
while (csv.Next())
{
string datetime = "";
if (DateTimeDualColumn)
{
datetime = csv.GetDate("Date").ToShortDateString() + " " +
csv.GetTime("Time").ToShortTimeString();
}
else
{
datetime = csv.GetDate("Date").ToShortDateString();
}
DateTime date = DateTime.Parse(datetime);
if (date > from && date < to)
{
// CSV columns
double open = csv.GetDouble("Open");
double high = csv.GetDouble("High");
double low = csv.GetDouble("Low");
double close = csv.GetDouble("Close");
double volume = csv.GetDouble("Volume");
LoadedMarketData data = new LoadedMarketData(date, ticker);
foreach (MarketDataType marketDataType in neededTypes)
{
switch (marketDataType.ToString())
{
case "Open":
data.SetData(MarketDataType.Open, open);
break;
case "High":
data.SetData(MarketDataType.High, high);
break;
case "Low":
data.SetData(MarketDataType.Low, low);
break;
case "Close":
data.SetData(MarketDataType.Close, close);
break;
case "Volume":
data.SetData(MarketDataType.Volume, volume);
break;
case "RangeHighLow":
data.SetData(MarketDataType.RangeHighLow, Math.Round(Math.Abs(high - low), 6));
break;
case "RangeOpenClose":
data.SetData(MarketDataType.RangeOpenClose, Math.Round(Math.Abs(close - open), 6));
break;
case "RangeOpenCloseNonAbsolute":
data.SetData(MarketDataType.RangeOpenCloseNonAbsolute, Math.Round(close - open, 6));
break;
case "Weighted":
data.SetData(MarketDataType.Weighted, Math.Round((high + low + 2 * close) / 4, 6));
break;
}
}
result.Add(data);
}
}
csv.Close();
return(result);
}
catch (Exception ex)
{
Console.WriteLine("Something went wrong reading the csv: " + ex.Message);
}
return(null);
}
public ICollection<LoadedMarketData> Load(TickerSymbol ticker, IList<MarketDataType> dataNeeded, DateTime from, DateTime to)
{
ICollection<LoadedMarketData> is2 = new List<LoadedMarketData>();
HttpWebResponse response = (HttpWebResponse) WebRequest.Create(x38c212309d8d5dd3(ticker, from, to)).GetResponse();
using (Stream stream = response.GetResponseStream())
{
DateTime time;
double num;
double num2;
double num3;
double num4;
double num5;
double num6;
LoadedMarketData data;
ReadCSV dcsv = new ReadCSV(stream, true, CSVFormat.DecimalPoint);
goto Label_005B;
Label_003F:
data.SetData(MarketDataType.Open, num2);
Label_0049:
data.SetData(MarketDataType.Volume, num6);
is2.Add(data);
Label_005B:
if (dcsv.Next())
{
goto Label_01AF;
}
dcsv.Close();
stream.Close();
if ((((uint) num) - ((uint) num5)) <= uint.MaxValue)
{
goto Label_0125;
}
if ((((uint) num) & 0) == 0)
{
goto Label_00B0;
}
Label_00A4:
data.SetData(MarketDataType.Low, num5);
goto Label_003F;
Label_00B0:
data.SetData(MarketDataType.AdjustedClose, num);
do
{
data.SetData(MarketDataType.Open, num2);
if ((((uint) num) - ((uint) num6)) > uint.MaxValue)
{
goto Label_0049;
}
data.SetData(MarketDataType.Close, num3);
data.SetData(MarketDataType.High, num4);
}
while ((((uint) num5) - ((uint) num3)) > uint.MaxValue);
if (((uint) num2) >= 0)
{
goto Label_00A4;
}
goto Label_003F;
Label_0125:
if (((uint) num2) >= 0)
{
return is2;
}
goto Label_005B;
Label_013C:
num6 = dcsv.GetDouble("volume");
data = new LoadedMarketData(time, ticker);
if ((((uint) num2) | 2) == 0)
{
goto Label_017C;
}
goto Label_00B0;
Label_016E:
num2 = dcsv.GetDouble("open");
Label_017C:
num3 = dcsv.GetDouble("close");
num4 = dcsv.GetDouble("high");
num5 = dcsv.GetDouble("low");
goto Label_013C;
Label_01AF:
time = dcsv.GetDate("date");
num = dcsv.GetDouble("adj close");
if ((((uint) num3) + ((uint) num6)) <= uint.MaxValue)
{
goto Label_016E;
}
goto Label_00B0;
}
}
/// <summary>
/// Analyze the file.
/// </summary>
private void AnalyzeFile()
{
ScriptProperties prop = _analyst.Script.Properties;
// get filenames, headers & format
String sourceID = prop.GetPropertyString(
ScriptProperties.HeaderDatasourceRawFile);
FileInfo sourceFile = _analyst.Script.ResolveFilename(sourceID);
CSVFormat format = _analyst.Script.DetermineFormat();
bool headers = _analyst.Script.ExpectInputHeaders(sourceID);
// read the file
_rowCount = 0;
_missingCount = 0;
var csv = new ReadCSV(sourceFile.ToString(), headers, format);
while (csv.Next())
{
_rowCount++;
if (csv.HasMissing())
{
_missingCount++;
}
}
csv.Close();
}
/// <summary>
/// Load financial data.
/// </summary>
/// <param name="ticker">The ticker symbol.</param>
/// <param name="output">The output file.</param>
/// <param name="outputFormat">The output format.</param>
/// <param name="from">Starting date.</param>
/// <param name="to">Ending date.</param>
public void LoadAllData(String ticker, String output, CSVFormat outputFormat, DateTime from,
DateTime to)
{
try
{
Uri urlData = BuildURL(ticker, from, to);
WebRequest httpData = WebRequest.Create(urlData);
var responseData = (HttpWebResponse) httpData.GetResponse();
if (responseData != null)
{
Stream istreamData = responseData.GetResponseStream();
var csvData = new ReadCSV(istreamData, true, CSVFormat.English);
TextWriter tw = new StreamWriter(output);
tw.WriteLine("date,time,open price,high price,low price,close price,volume,adjusted price");
while (csvData.Next())
{
DateTime date = csvData.GetDate("date");
double adjustedClose = csvData.GetDouble("adj close");
double open = csvData.GetDouble("open");
double close = csvData.GetDouble("close");
double high = csvData.GetDouble("high");
double low = csvData.GetDouble("low");
var volume = (long) csvData.GetDouble("volume");
var line = new StringBuilder();
line.Append(NumericDateUtil.DateTime2Long(date));
line.Append(outputFormat.Separator);
line.Append(NumericDateUtil.Time2Int(date));
line.Append(outputFormat.Separator);
line.Append(outputFormat.Format(open, Precision));
line.Append(outputFormat.Separator);
line.Append(outputFormat.Format(high, Precision));
line.Append(outputFormat.Separator);
line.Append(outputFormat.Format(low, Precision));
line.Append(outputFormat.Separator);
line.Append(outputFormat.Format(close, Precision));
line.Append(outputFormat.Separator);
line.Append(volume);
line.Append(outputFormat.Separator);
line.Append(outputFormat.Format(adjustedClose, Precision));
tw.WriteLine(line.ToString());
}
tw.Close();
}
}
catch (WebException ex)
{
throw new QuantError(ex);
}
}
/// <summary>
/// Perform the analysis.
/// </summary>
/// <param name="target">The Encog analyst object to analyze.</param>
public void Process(EncogAnalyst target)
{
int count = 0;
CSVFormat csvFormat = ConvertStringConst
.ConvertToCSVFormat(_format);
var csv = new ReadCSV(_filename, _headers, csvFormat);
// pass one, calculate the min/max
while (csv.Next())
{
if (_fields == null)
{
GenerateFields(csv);
}
for (int i = 0; i < csv.ColumnCount; i++)
{
if (_fields != null)
{
_fields[i].Analyze1(csv.Get(i));
}
}
count++;
}
if (count == 0)
{
throw new AnalystError("Can't analyze file, it is empty.");
}
if (_fields != null)
{
foreach (AnalyzedField field in _fields)
{
field.CompletePass1();
}
}
csv.Close();
// pass two, standard deviation
csv = new ReadCSV(_filename, _headers, csvFormat);
while (csv.Next())
{
for (int i = 0; i < csv.ColumnCount; i++)
{
if (_fields != null)
{
_fields[i].Analyze2(csv.Get(i));
}
}
}
if (_fields != null)
{
foreach (AnalyzedField field in _fields)
{
field.CompletePass2();
}
}
csv.Close();
String str = _script.Properties.GetPropertyString(
ScriptProperties.SetupConfigAllowedClasses) ?? "";
bool allowInt = str.Contains("int");
bool allowReal = str.Contains("real") ||
str.Contains("double");
bool allowString = str.Contains("string");
// remove any classes that did not qualify
foreach (AnalyzedField field in _fields)
{
if (field.Class)
{
if (!allowInt && field.Integer)
{
field.Class = false;
}
if (!allowString && (!field.Integer && !field.Real))
{
field.Class = false;
}
if (!allowReal && field.Real && !field.Integer)
{
field.Class = false;
}
}
}
// merge with existing
if ((target.Script.Fields != null) &&
(_fields.Length == target.Script.Fields.Length))
{
for (int i = 0; i < _fields.Length; i++)
{
// copy the old field name
_fields[i].Name = target.Script.Fields[i].Name;
if (_fields[i].Class)
{
IList <AnalystClassItem> t = _fields[i].AnalyzedClassMembers;
IList <AnalystClassItem> s = target.Script.Fields[i].ClassMembers;
if (s.Count == t.Count)
{
for (int j = 0; j < s.Count; j++)
{
if (t[j].Code.Equals(s[j].Code))
{
t[j].Name = s[j].Name;
}
}
}
}
}
}
// now copy the fields
var df = new DataField[_fields.Length];
for (int i_4 = 0; i_4 < df.Length; i_4++)
{
df[i_4] = _fields[i_4].FinalizeField();
}
target.Script.Fields = df;
}
/// <summary>
/// Program entry point.
/// </summary>
/// <param name="app">Holds arguments and other info.</param>
public void Execute(IExampleInterface app)
{
ErrorCalculation.Mode = ErrorCalculationMode.RMS;
// Download the data that we will attempt to model.
string filename = DownloadData(app.Args);
// Define the format of the data file.
// This area will change, depending on the columns and
// format of the file that you are trying to model.
var format = new CSVFormat('.', ' '); // decimal point and
// space separated
IVersatileDataSource source = new CSVDataSource(filename, true,
format);
var data = new VersatileMLDataSet(source);
data.NormHelper.Format = format;
ColumnDefinition columnSSN = data.DefineSourceColumn("SSN",
ColumnType.Continuous);
ColumnDefinition columnDEV = data.DefineSourceColumn("DEV",
ColumnType.Continuous);
// Analyze the data, determine the min/max/mean/sd of every column.
data.Analyze();
// Use SSN & DEV to predict SSN. For time-series it is okay to have
// SSN both as
// an input and an output.
data.DefineInput(columnSSN);
data.DefineInput(columnDEV);
data.DefineOutput(columnSSN);
// Create feedforward neural network as the model type.
// MLMethodFactory.TYPE_FEEDFORWARD.
// You could also other model types, such as:
// MLMethodFactory.SVM: Support Vector Machine (SVM)
// MLMethodFactory.TYPE_RBFNETWORK: RBF Neural Network
// MLMethodFactor.TYPE_NEAT: NEAT Neural Network
// MLMethodFactor.TYPE_PNN: Probabilistic Neural Network
var model = new EncogModel(data);
model.SelectMethod(data, MLMethodFactory.TypeFeedforward);
// Send any output to the console.
model.Report = new ConsoleStatusReportable();
// Now normalize the data. Encog will automatically determine the
// correct normalization
// type based on the model you chose in the last step.
data.Normalize();
// Set time series.
data.LeadWindowSize = 1;
data.LagWindowSize = WindowSize;
// Hold back some data for a final validation.
// Do not shuffle the data into a random ordering. (never shuffle
// time series)
// Use a seed of 1001 so that we always use the same holdback and
// will get more consistent results.
model.HoldBackValidation(0.3, false, 1001);
// Choose whatever is the default training type for this model.
model.SelectTrainingType(data);
// Use a 5-fold cross-validated train. Return the best method found.
// (never shuffle time series)
var bestMethod = (IMLRegression)model.Crossvalidate(5,
false);
// Display the training and validation errors.
Console.WriteLine(@"Training error: "
+ model.CalculateError(bestMethod,
model.TrainingDataset));
Console.WriteLine(@"Validation error: "
+ model.CalculateError(bestMethod,
model.ValidationDataset));
// Display our normalization parameters.
NormalizationHelper helper = data.NormHelper;
Console.WriteLine(helper.ToString());
// Display the final model.
Console.WriteLine(@"Final model: " + bestMethod);
// Loop over the entire, original, dataset and feed it through the
// model. This also shows how you would process new data, that was
// not part of your training set. You do not need to retrain, simply
// use the NormalizationHelper class. After you train, you can save
// the NormalizationHelper to later normalize and denormalize your
// data.
source.Close();
var csv = new ReadCSV(filename, true, format);
var line = new String[2];
// Create a vector to hold each time-slice, as we build them.
// These will be grouped together into windows.
var slice = new double[2];
var window = new VectorWindow(WindowSize + 1);
IMLData input = helper.AllocateInputVector(WindowSize + 1);
// Only display the first 100
int stopAfter = 100;
while (csv.Next() && stopAfter > 0)
{
var result = new StringBuilder();
line[0] = csv.Get(2); // ssn
line[1] = csv.Get(3); // dev
helper.NormalizeInputVector(line, slice, false);
// enough data to build a full window?
if (window.IsReady())
{
window.CopyWindow(((BasicMLData)input).Data, 0);
String correct = csv.Get(2); // trying to predict SSN.
IMLData output = bestMethod.Compute(input);
String predicted = helper
.DenormalizeOutputVectorToString(output)[0];
result.Append(line);
result.Append(" -> predicted: ");
result.Append(predicted);
result.Append("(correct: ");
result.Append(correct);
result.Append(")");
Console.WriteLine(result.ToString());
}
// Add the normalized slice to the window. We do this just after
// the after checking to see if the window is ready so that the
// window is always one behind the current row. This is because
// we are trying to predict next row.
window.Add(slice);
stopAfter--;
}
csv.Close();
// Delete data file and shut down.
File.Delete(filename);
EncogFramework.Instance.Shutdown();
}
private void x076efb43809972d8()
{
string str;
FileInfo info;
CSVFormat format;
bool flag;
ScriptProperties properties = this._x554f16462d8d4675.Script.Properties;
Label_00AD:
str = properties.GetPropertyString("HEADER:DATASOURCE_rawFile");
do
{
info = this._x554f16462d8d4675.Script.ResolveFilename(str);
if (((uint) flag) < 0)
{
goto Label_00AD;
}
format = this._x554f16462d8d4675.Script.DetermineInputFormat(str);
flag = this._x554f16462d8d4675.Script.ExpectInputHeaders(str);
}
while (0 != 0);
this._x0fe0496cde3d05e3 = 0;
if (0 == 0)
{
this._xed3494f8db69efb7 = 0;
ReadCSV dcsv = new ReadCSV(info.ToString(), flag, format);
while (dcsv.Next())
{
this._x0fe0496cde3d05e3++;
if (dcsv.HasMissing())
{
this._xed3494f8db69efb7++;
}
}
dcsv.Close();
if (((uint) flag) > uint.MaxValue)
{
goto Label_00AD;
}
}
}
/// <summary>
/// Process the input file and segregate into the output files.
/// </summary>
///
public void Process()
{
Validate();
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, InputFormat);
ResetStatus();
foreach (SegregateTargetPercent target in _targets)
{
StreamWriter tw = PrepareOutputFile(target.Filename);
while ((target.NumberRemaining > 0) && csv.Next()
&& !ShouldStop())
{
UpdateStatus(false);
var row = new LoadedRow(csv);
WriteRow(tw, row);
target.NumberRemaining = target.NumberRemaining - 1;
}
tw.Close();
}
ReportDone(false);
csv.Close();
}
/// <summary>
/// Private constructor.
/// </summary>
private PropertyConstraints()
{
_data = new Dictionary <String, List <PropertyEntry> >();
try
{
Stream mask0 = ResourceLoader.CreateStream("Encog.Resources.analyst.csv");
var csv = new ReadCSV(mask0, false, CSVFormat.EgFormat);
while (csv.Next())
{
String sectionStr = csv.Get(0);
String nameStr = csv.Get(1);
String typeStr = csv.Get(2);
// determine type
PropertyType t;
if ("boolean".Equals(typeStr, StringComparison.InvariantCultureIgnoreCase))
{
t = PropertyType.TypeBoolean;
}
else if ("real".Equals(typeStr, StringComparison.InvariantCultureIgnoreCase))
{
t = PropertyType.TypeDouble;
}
else if ("format".Equals(typeStr, StringComparison.InvariantCultureIgnoreCase))
{
t = PropertyType.TypeFormat;
}
else if ("int".Equals(typeStr, StringComparison.InvariantCultureIgnoreCase))
{
t = PropertyType.TypeInteger;
}
else if ("list-string".Equals(typeStr, StringComparison.InvariantCultureIgnoreCase))
{
t = PropertyType.TypeListString;
}
else if ("string".Equals(typeStr, StringComparison.InvariantCultureIgnoreCase))
{
t = PropertyType.TypeString;
}
else
{
throw new AnalystError("Unknown type constraint: "
+ typeStr);
}
var entry = new PropertyEntry(t, nameStr,
sectionStr);
List <PropertyEntry> list;
if (_data.ContainsKey(sectionStr))
{
list = _data[sectionStr];
}
else
{
list = new List <PropertyEntry>();
_data[sectionStr] = list;
}
list.Add(entry);
}
csv.Close();
mask0.Close();
}
catch (IOException e)
{
throw new EncogError(e);
}
}
/// <summary>
/// Program entry point.
/// </summary>
/// <param name="app">Holds arguments and other info.</param>
public void Execute(IExampleInterface app)
{
// Download the data that we will attempt to model.
string irisFile = DownloadData(app.Args);
// Define the format of the data file.
// This area will change, depending on the columns and
// format of the file that you are trying to model.
IVersatileDataSource source = new CSVDataSource(irisFile, false,
CSVFormat.DecimalPoint);
var data = new VersatileMLDataSet(source);
data.DefineSourceColumn("sepal-length", 0, ColumnType.Continuous);
data.DefineSourceColumn("sepal-width", 1, ColumnType.Continuous);
data.DefineSourceColumn("petal-length", 2, ColumnType.Continuous);
data.DefineSourceColumn("petal-width", 3, ColumnType.Continuous);
// Define the column that we are trying to predict.
ColumnDefinition outputColumn = data.DefineSourceColumn("species", 4,
ColumnType.Nominal);
// Analyze the data, determine the min/max/mean/sd of every column.
data.Analyze();
// Map the prediction column to the output of the model, and all
// other columns to the input.
data.DefineSingleOutputOthersInput(outputColumn);
// Create feedforward neural network as the model type. MLMethodFactory.TYPE_FEEDFORWARD.
// You could also other model types, such as:
// MLMethodFactory.SVM: Support Vector Machine (SVM)
// MLMethodFactory.TYPE_RBFNETWORK: RBF Neural Network
// MLMethodFactor.TYPE_NEAT: NEAT Neural Network
// MLMethodFactor.TYPE_PNN: Probabilistic Neural Network
var model = new EncogModel(data);
model.SelectMethod(data, MLMethodFactory.TypeFeedforward);
// Send any output to the console.
model.Report = new ConsoleStatusReportable();
// Now normalize the data. Encog will automatically determine the correct normalization
// type based on the model you chose in the last step.
data.Normalize();
// Hold back some data for a final validation.
// Shuffle the data into a random ordering.
// Use a seed of 1001 so that we always use the same holdback and will get more consistent results.
model.HoldBackValidation(0.3, true, 1001);
// Choose whatever is the default training type for this model.
model.SelectTrainingType(data);
// Use a 5-fold cross-validated train. Return the best method found.
var bestMethod = (IMLRegression)model.Crossvalidate(5, true);
// Display the training and validation errors.
Console.WriteLine(@"Training error: " + model.CalculateError(bestMethod, model.TrainingDataset));
Console.WriteLine(@"Validation error: " + model.CalculateError(bestMethod, model.ValidationDataset));
// Display our normalization parameters.
NormalizationHelper helper = data.NormHelper;
Console.WriteLine(helper.ToString());
// Display the final model.
Console.WriteLine(@"Final model: " + bestMethod);
// Loop over the entire, original, dataset and feed it through the model.
// This also shows how you would process new data, that was not part of your
// training set. You do not need to retrain, simply use the NormalizationHelper
// class. After you train, you can save the NormalizationHelper to later
// normalize and denormalize your data.
source.Close();
var csv = new ReadCSV(irisFile, false, CSVFormat.DecimalPoint);
var line = new String[4];
IMLData input = helper.AllocateInputVector();
while (csv.Next())
{
var result = new StringBuilder();
line[0] = csv.Get(0);
line[1] = csv.Get(1);
line[2] = csv.Get(2);
line[3] = csv.Get(3);
String correct = csv.Get(4);
helper.NormalizeInputVector(line, ((BasicMLData)input).Data, false);
IMLData output = bestMethod.Compute(input);
String irisChosen = helper.DenormalizeOutputVectorToString(output)[0];
result.Append(line);
result.Append(" -> predicted: ");
result.Append(irisChosen);
result.Append("(correct: ");
result.Append(correct);
result.Append(")");
Console.WriteLine(result.ToString());
}
csv.Close();
// Delete data file ande shut down.
File.Delete(irisFile);
EncogFramework.Instance.Shutdown();
}
/// <summary>
/// Analyze the input file.
/// </summary>
/// <param name="input">The input file.</param>
/// <param name="headers">True, if there are headers.</param>
/// <param name="format">The format of the CSV data.</param>
public virtual void Analyze(FileInfo input, bool headers,
CSVFormat format)
{
ResetStatus();
InputFilename = input;
ExpectInputHeaders = headers;
Format = format;
_columnMapping.Clear();
_columns.Clear();
// first count the rows
TextReader reader = null;
try
{
int recordCount = 0;
reader = new StreamReader(InputFilename.OpenRead());
while (reader.ReadLine() != null)
{
UpdateStatus(true);
recordCount++;
}
if (headers)
{
recordCount--;
}
RecordCount = recordCount;
}
catch (IOException ex)
{
throw new QuantError(ex);
}
finally
{
ReportDone(true);
if (reader != null)
{
try
{
reader.Close();
}
catch (IOException e)
{
throw new QuantError(e);
}
}
InputFilename = input;
ExpectInputHeaders = headers;
Format = format;
}
// now analyze columns
ReadCSV csv = null;
try
{
csv = new ReadCSV(input.ToString(), headers, format);
if (!csv.Next())
{
throw new QuantError("File is empty");
}
for (int i = 0; i < csv.ColumnCount; i++)
{
String name;
if (headers)
{
name = AttemptResolveName(csv.ColumnNames[i]);
}
else
{
name = "Column-" + (i + 1);
}
// determine if it should be an input/output field
String str = csv.Get(i);
bool io = false;
try
{
Format.Parse(str);
io = true;
}
catch (FormatException ex)
{
EncogLogging.Log(ex);
}
AddColumn(new FileData(name, i, io, io));
}
}
finally
{
if (csv != null)
{
csv.Close();
}
Analyzed = true;
}
}