/// <summary>
/// Read the input file.
/// </summary>
private void ReadInputFile()
{
ResetStatus();
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
while (csv.Next() && !ShouldStop())
{
UpdateStatus("Reading input file");
var row = new LoadedRow(csv);
_data.Add(row);
}
Count = csv.ColumnCount;
if (ExpectInputHeaders)
{
InputHeadings = new String[csv.ColumnCount];
for (int i = 0; i < csv.ColumnCount; i++)
{
InputHeadings[i] = csv.ColumnNames[i];
}
}
csv.Close();
}
/// <summary>
/// Analyze the file.
/// </summary>
private void AnalyzeFile()
{
ScriptProperties prop = _analyst.Script.Properties;
// get filenames, headers & format
String sourceID = prop.GetPropertyString(
ScriptProperties.HeaderDatasourceRawFile);
FileInfo sourceFile = _analyst.Script.ResolveFilename(sourceID);
CSVFormat format = _analyst.Script.DetermineFormat();
bool headers = _analyst.Script.ExpectInputHeaders(sourceID);
// read the file
_rowCount = 0;
_missingCount = 0;
var csv = new ReadCSV(sourceFile.ToString(), headers, format);
while (csv.Next())
{
_rowCount++;
if (csv.HasMissing())
{
_missingCount++;
}
}
csv.Close();
}
public void LoadCSVFileTestClass()
{
var path = parameters.TestFile;
var csvRead = new ReadCSV(new FileStream(path, FileMode.Open), true, CSVFormat.DecimalPoint);
var values = new List <double[]>();
while (csvRead.Next())
{
values.Add(new double[2] {
csvRead.GetDouble(0), csvRead.GetDouble(1)
});
}
csvRead.Close();
for (int i = 0; i < values.Count; i++)
{
for (int argc = 0; argc < values[i].Length; argc++)
{
values[i][argc] = ArgNormalize[argc].Normalize(values[i][argc]);
}
}
//Normalization(values);
testSet = values.ToArray();
}
/// <summary>
/// Convert a CSV file to binary.
/// </summary>
/// <param name="csvFile">The CSV file to convert.</param>
/// <param name="format">The format.</param>
/// <param name="binFile">The binary file.</param>
/// <param name="input">The input.</param>
/// <param name="ideal">The ideal.</param>
/// <param name="headers">True, if headers are present.</param>
public static void ConvertCSV2Binary(FileInfo csvFile, CSVFormat format,
FileInfo binFile, int[] input, int[] ideal, bool headers)
{
binFile.Delete();
var csv = new ReadCSV(csvFile.ToString(), headers, format);
var buffer = new BufferedMLDataSet(binFile.ToString());
buffer.BeginLoad(input.Length, ideal.Length);
while (csv.Next())
{
var inputData = new BasicMLData(input.Length);
var idealData = new BasicMLData(ideal.Length);
// handle input data
for (int i = 0; i < input.Length; i++)
{
inputData[i] = csv.GetDouble(input[i]);
}
// handle input data
for (int i = 0; i < ideal.Length; i++)
{
idealData[i] = csv.GetDouble(ideal[i]);
}
// add to dataset
buffer.Add(inputData, idealData);
}
buffer.EndLoad();
buffer.Close();
csv.Close();
}
public void TestRegression()
{
FileInfo rawFile = TEMP_DIR.CreateFile("simple.csv");
FileInfo egaFile = TEMP_DIR.CreateFile("simple.ega");
FileInfo outputFile = TEMP_DIR.CreateFile("simple_output.csv");
FileUtil.CopyResource("Encog.Resources.simple.csv", rawFile);
FileUtil.CopyResource("Encog.Resources.simple-r.ega", egaFile);
EncogAnalyst analyst = new EncogAnalyst();
analyst.Load(egaFile);
analyst.ExecuteTask("task-full");
ReadCSV csv = new ReadCSV(outputFile.ToString(), true, CSVFormat.English);
while (csv.Next())
{
double diff = Math.Abs(csv.GetDouble(2) - csv.GetDouble(4));
Assert.IsTrue(diff < 1.5);
}
Assert.AreEqual(4, analyst.Script.Fields.Length);
Assert.AreEqual(3, analyst.Script.Fields[3].ClassMembers.Count);
csv.Close();
}
/// <summary>
/// Process the input file and segregate into the output files.
/// </summary>
public void Process()
{
Validate();
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
ResetStatus();
foreach (SegregateTargetPercent target in _targets)
{
StreamWriter tw = PrepareOutputFile(target.Filename);
while ((target.NumberRemaining > 0) && csv.Next() &&
!ShouldStop())
{
UpdateStatus(false);
var row = new LoadedRow(csv);
WriteRow(tw, row);
target.NumberRemaining = target.NumberRemaining - 1;
}
tw.Close();
}
ReportDone(false);
csv.Close();
}
/// <summary>
/// Process the file and output to the target file.
/// </summary>
/// <param name="target">The target file to write to.</param>
public void Process(string target)
{
var csv = new ReadCSV(InputFilename.ToString(), ExpectInputHeaders, Format);
TextWriter tw = new StreamWriter(target);
ResetStatus();
while (csv.Next())
{
var line = new StringBuilder();
UpdateStatus(false);
line.Append(GetColumnData(FileData.Date, csv));
line.Append(" ");
line.Append(GetColumnData(FileData.Time, csv));
line.Append(";");
line.Append(Format.Format(double.Parse(GetColumnData(FileData.Open, csv)), Precision));
line.Append(";");
line.Append(Format.Format(double.Parse(GetColumnData(FileData.High, csv)), Precision));
line.Append(";");
line.Append(Format.Format(double.Parse(GetColumnData(FileData.Low, csv)), Precision));
line.Append(";");
line.Append(Format.Format(double.Parse(GetColumnData(FileData.Close, csv)), Precision));
line.Append(";");
line.Append(Format.Format(double.Parse(GetColumnData(FileData.Volume, csv)), Precision));
tw.WriteLine(line.ToString());
}
ReportDone(false);
csv.Close();
tw.Close();
}
public void TestClassification()
{
FileInfo rawFile = TEMP_DIR.CreateFile("simple.csv");
FileInfo egaFile = TEMP_DIR.CreateFile("simple.ega");
FileInfo outputFile = TEMP_DIR.CreateFile("simple_output.csv");
FileUtil.CopyResource("Encog.Resources.simple.csv", rawFile);
FileUtil.CopyResource("Encog.Resources.simple-c.ega", egaFile);
EncogAnalyst analyst = new EncogAnalyst();
analyst.AddAnalystListener(new ConsoleAnalystListener());
analyst.Load(egaFile);
analyst.ExecuteTask("task-full");
ReadCSV csv = new ReadCSV(outputFile.ToString(), true, CSVFormat.English);
while (csv.Next())
{
Assert.AreEqual(csv.Get(3), csv.Get(4));
}
Assert.AreEqual(4, analyst.Script.Fields.Length);
Assert.AreEqual(3, analyst.Script.Fields[3].ClassMembers.Count);
csv.Close();
}
public void TestRegression()
{
FileInfo rawFile = TEMP_DIR.CreateFile("EncogCSharp/Resources/simple.csv");
FileInfo egaFile = TEMP_DIR.CreateFile("EncogCSharp/Resources/simple.ega");
FileInfo outputFile = TEMP_DIR.CreateFile("EncogCSharp/Resources/simple_output.csv");
FileUtil.CopyResource("EncogCSharp/Resources/simple.csv", rawFile);
FileUtil.CopyResource("EncogCSharp/Resources/simple-r.ega", egaFile);
EncogAnalyst analyst = new EncogAnalyst();
analyst.Load(egaFile);
analyst.ExecuteTask("task-full");
ReadCSV csv = new ReadCSV(outputFile.ToString(), true, CSVFormat.English);
while (csv.Next())
{
double diff = Math.Abs(csv.GetDouble(2) - csv.GetDouble(4));
Assert.IsTrue(diff < 1.5);
}
Assert.AreEqual(4, analyst.Script.Fields.Length);
Assert.AreEqual(3, analyst.Script.Fields[3].ClassMembers.Count);
csv.Close();
}
public void TestClassification()
{
FileInfo rawFile = TEMP_DIR.CreateFile("simple.csv");
FileInfo egaFile = TEMP_DIR.CreateFile("simple.ega");
FileInfo outputFile = TEMP_DIR.CreateFile("simple_output.csv");
FileUtil.CopyResource("Encog.Resources.simple.csv", rawFile);
FileUtil.CopyResource("Encog.Resources.simple-c.ega", egaFile);
EncogAnalyst analyst = new EncogAnalyst();
analyst.AddAnalystListener(new ConsoleAnalystListener());
analyst.Load(egaFile);
analyst.ExecuteTask("task-full");
ReadCSV csv = new ReadCSV(outputFile.ToString(), true, CSVFormat.English);
while (csv.Next())
{
Assert.AreEqual(csv.Get(3), csv.Get(4));
}
Assert.AreEqual(4, analyst.Script.Fields.Length);
Assert.AreEqual(3, analyst.Script.Fields[3].ClassMembers.Count);
csv.Close();
}
public void LoadCSVFileTrainingClass()
{
var path = parameters.TrainFile;
var csvRead = new ReadCSV(new FileStream(path, FileMode.Open), true, CSVFormat.DecimalPoint);
var values = new List <double[]>();
var classes = new List <double[]>();
while (csvRead.Next())
{
values.Add(new double[2] {
csvRead.GetDouble(0), csvRead.GetDouble(1)
});
classes.Add(new double[1] {
csvRead.GetDouble(2)
});
}
csvRead.Close();
var min = parameters.FunctionType == FunctionTypeEnum.Unipolar ? 0d : -1d;
ArgNormalize = new NormalizedField[2] {
new NormalizedField(NormalizationAction.Normalize, "X", values.Max(v => v[0]), values.Min(v => v[0]), 1.0, min),
new NormalizedField(NormalizationAction.Normalize, "Y", values.Max(v => v[1]), values.Min(v => v[1]), 1.0, min)
};
for (int i = 0; i < values.Count; i++)
{
for (int argc = 0; argc < values[i].Length; argc++)
{
values[i][argc] = ArgNormalize[argc].Normalize(values[i][argc]);
}
}
//Normalization(values);
classCount = classes.Select(c => c[0]).Distinct().Count();
var normalizeClasses = new List <double[]>();
for (int i = 0; i < classes.Count; ++i)
{
var newClasses = new double[classCount];
for (int j = 0; j < newClasses.Length; j++)
{
newClasses[j] = min;
}
newClasses[(int)classes[i][0] - 1] = 1;// dodoac normalizacje na -1
normalizeClasses.Add(newClasses);
}
var trainSetCount = (int)((double)values.Count * ((100.0 - 15) / 100));
values.Shuffle();
normalizeClasses.Shuffle();
MyExtensions.ResetStableShuffle();
TrainSet = new BasicNeuralDataSet(values.Take(trainSetCount).ToArray(),
normalizeClasses.Take(trainSetCount).ToArray());
ValidationSet = new BasicNeuralDataSet(values.Skip(trainSetCount).ToArray(),
normalizeClasses.Skip(trainSetCount).ToArray());
}
public ICollection <LoadedMarketData> ReadAndCallLoader(TickerSymbol symbol, IList <MarketDataType> neededTypes, DateTime from, DateTime to, string File)
{
try
{
//We got a file, lets load it.
ICollection <LoadedMarketData> result = new List <LoadedMarketData>();
ReadCSV csv = new ReadCSV(File, true, CSVFormat.English);
csv.DateFormat = "yyyy.MM.dd HH:mm:ss";
DateTime ParsedDate = from;
// Time,Open,High,Low,Close,Volume
while (csv.Next() && ParsedDate >= from && ParsedDate <= to)
{
DateTime date = csv.GetDate("Time");
double Bid = csv.GetDouble("Bid");
double Ask = csv.GetDouble("Ask");
double AskVolume = csv.GetDouble("AskVolume");
double BidVolume = csv.GetDouble("BidVolume");
double _trade = (Bid + Ask) / 2;
double _tradeSize = (AskVolume + BidVolume) / 2;
LoadedMarketData data = new LoadedMarketData(date, symbol);
data.SetData(MarketDataType.Trade, _trade);
data.SetData(MarketDataType.Volume, _tradeSize);
result.Add(data);
Console.WriteLine("Current DateTime:" + ParsedDate.ToShortDateString() + " Time:" + ParsedDate.ToShortTimeString() + " Start date was " + from.ToShortDateString());
Console.WriteLine("Stopping at date:" + to.ToShortDateString());
ParsedDate = date;
//double open = csv.GetDouble("Open");
//double close = csv.GetDouble("High");
//double high = csv.GetDouble("Low");
//double low = csv.GetDouble("Close");
//double volume = csv.GetDouble("Volume");
//LoadedMarketData data = new LoadedMarketData(date, symbol);
//data.SetData(MarketDataType.Open, open);
//data.SetData(MarketDataType.High, high);
//data.SetData(MarketDataType.Low, low);
//data.SetData(MarketDataType.Close, close);
//data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
return(result);
}
catch (Exception ex)
{
Console.WriteLine("Something went wrong reading the csv");
Console.WriteLine("Something went wrong reading the csv:" + ex.Message);
}
Console.WriteLine("Something went wrong reading the csv");
return(null);
}
/// <summary>
/// Reads the CSV and call loader.
/// Used internally to load the csv and place data in the marketdataset.
/// </summary>
/// <param name="symbol">The symbol.</param>
/// <param name="neededTypes">The needed types.</param>
/// <param name="from">From.</param>
/// <param name="to">To.</param>
/// <param name="File">The file.</param>
/// <returns></returns>
ICollection <LoadedMarketData> ReadAndCallLoader(TickerSymbol symbol, IEnumerable <MarketDataType> neededTypes, DateTime from, DateTime to, string File)
{
//We got a file, lets load it.
ICollection <LoadedMarketData> result = new List <LoadedMarketData>();
ReadCSV csv = new ReadCSV(File, true, CSVFormat.English);
//In case we want to use a different date format...and have used the SetDateFormat method, our DateFormat must then not be null..
//We will use the ?? operator to check for nullables.
csv.DateFormat = DateFormat ?? "yyyy-MM-dd HH:mm:ss";
csv.TimeFormat = "HH:mm:ss";
DateTime ParsedDate = from;
bool writeonce = true;
while (csv.Next())
{
DateTime date = csv.GetDate(0);
ParsedDate = date;
if (writeonce)
{
Console.WriteLine(@"First parsed date in csv:" + ParsedDate.ToShortDateString());
Console.WriteLine(@"Stopping at date:" + to.ToShortDateString());
Console.WriteLine(@"Current DateTime:" + ParsedDate.ToShortDateString() + @" Time:" +
ParsedDate.ToShortTimeString() + @" Asked Start date was " +
from.ToShortDateString());
writeonce = false;
}
if (ParsedDate >= from && ParsedDate <= to)
{
DateTime datex = csv.GetDate(0);
double open = csv.GetDouble(1);
double close = csv.GetDouble(2);
double high = csv.GetDouble(3);
double low = csv.GetDouble(4);
double volume = csv.GetDouble(5);
double range = Math.Abs(open - close);
double HighLowRange = Math.Abs(high - low);
double DirectionalRange = close - open;
LoadedMarketData data = new LoadedMarketData(datex, symbol);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Close, close);
data.SetData(MarketDataType.Volume, volume);
data.SetData(MarketDataType.RangeHighLow, Math.Round(HighLowRange, 6));
data.SetData(MarketDataType.RangeOpenClose, Math.Round(range, 6));
data.SetData(MarketDataType.RangeOpenCloseNonAbsolute, Math.Round(DirectionalRange, 6));
result.Add(data);
}
}
csv.Close();
return(result);
}
public ICollection <LoadedMarketData> Load(
TickerSymbol ticker,
IList <MarketDataType> dataNeeded,
DateTime from,
DateTime to)
{
// TODO: nyyyyyyyaaagh!
ICollection <LoadedMarketData> result =
new List <LoadedMarketData>();
Uri url = BuildURL(ticker, from, to);
WebRequest http = HttpWebRequest.Create(url);
HttpWebResponse response = http.GetResponse() as HttpWebResponse;
using (Stream istream = response.GetResponseStream())
{
ReadCSV csv = new ReadCSV(
istream,
true,
CSVFormat.DECIMAL_POINT
);
while (csv.Next())
{
// todo: edit headers to match
DateTime date = csv.GetDate("DATE");
date =
date.Add(
new TimeSpan(
csv.GetDate("TIME").Hour,
csv.GetDate("TIME").Minute,
csv.GetDate("TIME").Second
)
);
double open = csv.GetDouble("OPEN");
double high = csv.GetDouble("MIN");
double low = csv.GetDouble("MAX");
double close = csv.GetDouble("CLOSE");
double volume = csv.GetDouble("VOLUME");
LoadedMarketData data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.OPEN, open);
data.SetData(MarketDataType.HIGH, high);
data.SetData(MarketDataType.LOW, low);
data.SetData(MarketDataType.CLOSE, close);
data.SetData(MarketDataType.VOLUME, volume);
result.Add(data);
}
csv.Close();
istream.Close();
}
return(result);
}
/// <inheritdoc />
public string[] ReadLine()
{
if (_reader == null)
{
throw new EncogError("Please call rewind before reading the file.");
}
if (_reader.Next())
{
int len = _reader.ColumnCount;
var result = new string[len];
for (int i = 0; i < result.Length; i++)
{
result[i] = _reader.Get(i);
}
return(result);
}
_reader.Close();
return(null);
}
/// <summary>
/// Process the file.
/// </summary>
/// <param name="outputFile">The output file.</param>
/// <param name="method">The method to use.</param>
public void Process(FileInfo outputFile, IMLRegression method)
{
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
if (method.InputCount != _inputCount)
{
throw new AnalystError("This machine learning method has "
+ method.InputCount
+ " inputs, however, the data has " + _inputCount
+ " inputs.");
}
var input = new BasicMLData(method.InputCount);
StreamWriter tw = AnalystPrepareOutputFile(outputFile);
ResetStatus();
while (csv.Next())
{
UpdateStatus(false);
var row = new LoadedRow(csv, _idealCount);
int dataIndex = 0;
// load the input data
for (int i = 0; i < _inputCount; i++)
{
String str = row.Data[i];
double d = Format.Parse(str);
input[i] = d;
dataIndex++;
}
// do we need to skip the ideal values?
dataIndex += _idealCount;
// compute the result
IMLData output = method.Compute(input);
// display the computed result
for (int i = 0; i < _outputCount; i++)
{
double d = output[i];
row.Data[dataIndex++] = Format.Format(d, Precision);
}
WriteRow(tw, row);
}
ReportDone(false);
tw.Close();
csv.Close();
}
/// <inheritDoc/>
public void Close()
{
if (_readCSV != null)
{
_readCSV.Close();
_readCSV = null;
}
if (_output != null)
{
_output.Close();
_output = null;
}
}
public void loadPrime(String primeFilename)
{
var csv = new ReadCSV(primeFilename, true, CSVFormat.English);
while (csv.Next())
{
var date = csv.GetDate("date");
double rate = csv.GetDouble("prime");
var ir = new InterestRate(date, rate);
rates.Add(ir);
}
csv.Close();
rates.Sort();
}
public void loadPrime(String primeFilename)
{
ReadCSV csv = new ReadCSV(primeFilename);
while (csv.Next())
{
DateTime date = csv.GetDate("date");
double rate = csv.GetDouble("prime");
InterestRate ir = new InterestRate(date, rate);
this.rates.Add(ir);
}
csv.Close();
this.rates.Sort();
}
public void loadSP500(String sp500Filename)
{
ReadCSV csv = new ReadCSV(sp500Filename);
while (csv.Next())
{
DateTime date = csv.GetDate("date");
double amount = csv.GetDouble("adj close");
FinancialSample sample = new FinancialSample();
sample.setAmount(amount);
sample.setDate(date);
this.samples.Add(sample);
}
csv.Close();
this.samples.Sort();
}
public void loadSP500(String sp500Filename)
{
var csv = new ReadCSV(sp500Filename, true, CSVFormat.English);
while (csv.Next())
{
var date = csv.GetDate("date");
double amount = csv.GetDouble("adj close");
var sample = new FinancialSample();
sample.setAmount(amount);
sample.setDate(date);
samples.Add(sample);
}
csv.Close();
samples.Sort();
}
/// <summary>
/// Analyze the data. This counts the records and prepares the data to be
/// processed.
/// </summary>
///
/// <param name="theAnalyst">The analyst to use.</param>
/// <param name="inputFile">The input file to analyze.</param>
/// <param name="headers">True, if the input file has headers.</param>
/// <param name="format">The format of the input file.</param>
public void Analyze(EncogAnalyst theAnalyst,
FileInfo inputFile, bool headers, CSVFormat format)
{
InputFilename = inputFile;
ExpectInputHeaders = headers;
InputFormat = format;
Analyzed = true;
_analyst = theAnalyst;
if (OutputFormat == null)
{
OutputFormat = InputFormat;
}
_data = new BasicMLDataSet();
ResetStatus();
int recordCount = 0;
int outputLength = _analyst.DetermineTotalColumns();
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, InputFormat);
ReadHeaders(csv);
_analystHeaders = new CSVHeaders(InputHeadings);
while (csv.Next() && !ShouldStop())
{
UpdateStatus(true);
var row = new LoadedRow(csv, 1);
double[] inputArray = AnalystNormalizeCSV.ExtractFields(
_analyst, _analystHeaders, csv, outputLength, true);
var input = new ClusterRow(inputArray, row);
_data.Add(input);
recordCount++;
}
RecordCount = recordCount;
Count = csv.ColumnCount;
ReadHeaders(csv);
csv.Close();
ReportDone(true);
}
/// <summary>
/// Loads the specified financial data.
/// </summary>
/// <param name="ticker">The currency pair to load.</param>
/// <param name="dataNeeded">The financial data needed.</param>
/// <param name="from">The beginning date to load data from.</param>
/// <param name="to">The ending date to load data to.</param>
/// <returns>A collection of LoadedMarketData objects that represent
/// the data loaded.</returns>
public ICollection <LoadedMarketData> Load(
TickerSymbol ticker,
IList <MarketDataType> dataNeeded,
DateTime from,
DateTime to
)
{
ICollection <LoadedMarketData> result =
new List <LoadedMarketData>();
Uri url = buildURL(ticker, from, to);
WebRequest http = HttpWebRequest.Create(url);
HttpWebResponse response = http.GetResponse() as HttpWebResponse;
using (Stream istream = response.GetResponseStream())
{
ReadCSV csv =
new ReadCSV(istream, true, CSVFormat.DECIMAL_POINT);
while (csv.Next())
{
// TODO: check these values if possible
DateTime date = csv.GetDate("date");
double adjClose = csv.GetDouble("adj close"); // TODO: deprecate?
double open = csv.GetDouble("open");
double close = csv.GetDouble("close");
double high = csv.GetDouble("high");
double low = csv.GetDouble("low");
double volume = csv.GetDouble("volume"); // TODO: deprecate?
LoadedMarketData data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.ADJUSTED_CLOSE, adjClose);
data.SetData(MarketDataType.OPEN, open);
data.SetData(MarketDataType.CLOSE, close);
data.SetData(MarketDataType.HIGH, high);
data.SetData(MarketDataType.LOW, low);
data.SetData(MarketDataType.OPEN, open);
data.SetData(MarketDataType.VOLUME, volume);
result.Add(data);
}
csv.Close();
istream.Close();
}
return(result);
}
/// <summary>
/// Load financial data from Google.
/// </summary>
/// <param name="ticker">The ticker to load from.</param>
/// <param name="dataNeeded">The data needed.</param>
/// <param name="from">The starting time.</param>
/// <param name="to">The ending time.</param>
/// <returns>The loaded data.</returns>
public ICollection <LoadedMarketData> Load(TickerSymbol ticker, IList <MarketDataType> dataNeeded, DateTime from,
DateTime to)
{
ICollection <LoadedMarketData> result = new List <LoadedMarketData>();
Uri url = BuildUrl(ticker, from, to);
WebRequest http = WebRequest.Create(url);
var response = (HttpWebResponse)http.GetResponse();
if (response != null)
{
using (Stream istream = response.GetResponseStream())
{
var csv = new ReadCSV(istream, true, CSVFormat.DecimalPoint);
while (csv.Next())
{
DateTime date = csv.GetDate("date");
double open = csv.GetDouble("open");
double close = csv.GetDouble("close");
double high = csv.GetDouble("high");
double low = csv.GetDouble("low");
double volume = csv.GetDouble("volume");
var data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Close, close);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
if (istream != null)
{
istream.Close();
}
}
}
return(result);
}
public ICollection <LoadedMarketData> ReadAndCallLoader(TickerSymbol symbol, IList <MarketDataType> neededTypes, DateTime from, DateTime to, string File)
{
try
{
//We got a file, lets load it.
ICollection <LoadedMarketData> result = new List <LoadedMarketData>();
ReadCSV csv = new ReadCSV(File, true, LoadedFormat);
csv.DateFormat = DateTimeFormat.Normalize();
// Time,Open,High,Low,Close,Volume
while (csv.Next())
{
DateTime date = csv.GetDate("Time");
double open = csv.GetDouble("Open");
double close = csv.GetDouble("High");
double high = csv.GetDouble("Low");
double low = csv.GetDouble("Close");
double volume = csv.GetDouble("Volume");
LoadedMarketData data = new LoadedMarketData(date, symbol);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Close, close);
data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
return(result);
}
catch (Exception ex)
{
Console.WriteLine("Something went wrong reading the csv");
Console.WriteLine("Something went wrong reading the csv:" + ex.Message);
}
Console.WriteLine("Something went wrong reading the csv");
return(null);
}
public void LoadCSVFileTrainingRegg()
{
var path = parameters.TrainFile;
var csvRead = new ReadCSV(new FileStream(path, FileMode.Open), true, CSVFormat.DecimalPoint);
var valuesX = new List <double[]>();
var valuesY = new List <double[]>();
while (csvRead.Next())
{
valuesX.Add(new double[1] {
csvRead.GetDouble(0)
});
valuesY.Add(new double[1] {
csvRead.GetDouble(1)
});
}
csvRead.Close();
var min = parameters.FunctionType == FunctionTypeEnum.Unipolar ? 0d : -1d;
ArgNormalize = new NormalizedField[] { new NormalizedField(NormalizationAction.Normalize, "X", valuesX.Max(v => v[0]), valuesX.Min(v => v[0]), 1.0, min) };
YNormalize = new NormalizedField(NormalizationAction.Normalize, "Y", valuesY.Max(v => v[0]), valuesY.Min(v => v[0]), 1.0, min);
for (int i = 0; i < valuesX.Count; i++)
{
valuesX[i][0] = ArgNormalize[0].Normalize(valuesX[i][0]);
valuesY[i][0] = YNormalize.Normalize(valuesY[i][0]);
}
//if(parameters.FunctionType==FunctionTypeEnum.Bipolar)
// ansMeans = means[0];
//ansfactor = factor[0];
//ansMIn = Min[0];
var trainSetCount = (int)((double)valuesX.Count * ((100.0 - 15) / 100));
valuesX.Shuffle();
valuesY.Shuffle();
MyExtensions.ResetStableShuffle();
TrainSet = new BasicNeuralDataSet(valuesX.Take(trainSetCount).ToArray(),
valuesY.Take(trainSetCount).ToArray());
ValidationSet = new BasicNeuralDataSet(valuesX.Skip(trainSetCount).ToArray(),
valuesY.Skip(trainSetCount).ToArray());
}
/// <summary>
/// Perform a basic analyze of the file. This method is used mostly
/// internally.
/// </summary>
public void PerformBasicCounts()
{
ResetStatus();
int rc = 0;
var csv = new ReadCSV(_inputFilename.ToString(),
_expectInputHeaders, _format);
while (csv.Next() && !_cancel)
{
UpdateStatus(true);
rc++;
}
_recordCount = rc;
_columnCount = csv.ColumnCount;
ReadHeaders(csv);
csv.Close();
ReportDone(true);
}
/// <summary>
/// Process, and generate the output file.
/// </summary>
/// <param name="outputFile">The output file.</param>
public void Process(FileInfo outputFile)
{
ValidateAnalyzed();
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
LoadedRow row;
StreamWriter tw = PrepareOutputFile(outputFile);
ResetStatus();
while ((row = GetNextRow(csv)) != null)
{
WriteRow(tw, row);
UpdateStatus(false);
}
ReportDone(false);
tw.Close();
csv.Close();
}
/// <summary>
/// Load the specified financial data.
/// </summary>
/// <param name="ticker">The ticker symbol to load.</param>
/// <param name="dataNeeded">The financial data needed.</param>
/// <param name="from">The beginning date to load data from.</param>
/// <param name="to">The ending date to load data to.</param>
/// <returns>A collection of LoadedMarketData objects that represent the data
/// loaded.</returns>
public ICollection<LoadedMarketData> Load(TickerSymbol ticker,
IList<MarketDataType> dataNeeded, DateTime from,
DateTime to)
{
ICollection<LoadedMarketData> result =
new List<LoadedMarketData>();
Uri url = BuildURL(ticker, from, to);
WebRequest http = WebRequest.Create(url);
var response = (HttpWebResponse) http.GetResponse();
using (Stream istream = response.GetResponseStream())
{
var csv = new ReadCSV(istream, true, CSVFormat.DecimalPoint);
while (csv.Next())
{
DateTime date = csv.GetDate("date");
double adjClose = csv.GetDouble("adj close");
double open = csv.GetDouble("open");
double close = csv.GetDouble("close");
double high = csv.GetDouble("high");
double low = csv.GetDouble("low");
double volume = csv.GetDouble("volume");
var data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.AdjustedClose, adjClose);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Close, close);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
istream.Close();
}
return result;
}
/// <summary>
/// Load financial data from a CSV file.
/// </summary>
/// <param name="ticker">The ticker being loaded, ignored for a CSV load.</param>
/// <param name="dataNeeded">The data needed.</param>
/// <param name="from">The starting date.</param>
/// <param name="to">The ending date.</param>
/// <returns></returns>
public ICollection <LoadedMarketData> Load(TickerSymbol ticker, IList <MarketDataType> dataNeeded, DateTime from,
DateTime to)
{
try
{
if (File.Exists(TheFile))
{
//We got a file, lets load it.
TheFile = TheFile;
ICollection <LoadedMarketData> result = new List <LoadedMarketData>();
var csv = new ReadCSV(TheFile, true, CSVFormat.English);
// Time,Open,High,Low,Close,Volume
while (csv.Next())
{
DateTime date = csv.GetDate("Time");
double open = csv.GetDouble("Open");
double close = csv.GetDouble("High");
double high = csv.GetDouble("Low");
double low = csv.GetDouble("Close");
double volume = csv.GetDouble("Volume");
var data = new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.Open, open);
data.SetData(MarketDataType.Volume, close);
data.SetData(MarketDataType.High, high);
data.SetData(MarketDataType.Low, low);
data.SetData(MarketDataType.Volume, volume);
result.Add(data);
}
csv.Close();
return(result);
}
}
catch (Exception ex)
{
throw new LoaderError(ex);
}
throw new LoaderError(@"Something went wrong reading the csv");
}
/// <summary>
/// Read the CSV file.
/// </summary>
private void ReadFile()
{
ReadCSV csv = null;
try
{
csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
ResetStatus();
int row = 0;
while (csv.Next() && !ShouldStop())
{
UpdateStatus("Reading data");
foreach (BaseCachedColumn column in Columns)
{
if (column is FileData)
{
if (column.Input)
{
var fd = (FileData)column;
String str = csv.Get(fd.Index);
double d = Format.Parse(str);
fd.Data[row] = d;
}
}
}
row++;
}
}
finally
{
ReportDone("Reading data");
if (csv != null)
{
csv.Close();
}
}
}
/// <summary>
/// Process and balance the data.
/// </summary>
/// <param name="outputFile">The output file to write data to.</param>
/// <param name="targetField"></param>
/// <param name="countPer">The desired count per class.</param>
public void Process(FileInfo outputFile, int targetField,
int countPer)
{
ValidateAnalyzed();
StreamWriter tw = PrepareOutputFile(outputFile);
_counts = new Dictionary <String, Int32>();
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, Format);
ResetStatus();
while (csv.Next() && !ShouldStop())
{
var row = new LoadedRow(csv);
UpdateStatus(false);
String key = row.Data[targetField];
int count;
if (!_counts.ContainsKey(key))
{
count = 0;
}
else
{
count = _counts[key];
}
if (count < countPer)
{
WriteRow(tw, row);
count++;
}
_counts[key] = count;
}
ReportDone(false);
csv.Close();
tw.Close();
}
/// <summary>
/// Construct the object.
/// </summary>
/// <param name="filename">The filename.</param>
/// <param name="headers">False if headers are not extended.</param>
/// <param name="format">The CSV format.</param>
public CSVHeaders(FileInfo filename, bool headers,
CSVFormat format)
{
_headerList = new List <String>();
_columnMapping = new Dictionary <String, Int32>();
ReadCSV csv = null;
try
{
csv = new ReadCSV(filename.ToString(), headers, format);
if (csv.Next())
{
if (headers)
{
foreach (String str in csv.ColumnNames)
{
_headerList.Add(str);
}
}
else
{
for (int i = 0; i < csv.ColumnCount; i++)
{
_headerList.Add("field:" + (i + 1));
}
}
}
Init();
}
finally
{
if (csv != null)
{
csv.Close();
}
}
}
/// <summary>
/// Load the specified financial data.
/// </summary>
/// <param name="ticker">The ticker symbol to load.</param>
/// <param name="dataNeeded">The financial data needed.</param>
/// <param name="from">The beginning date to load data from.</param>
/// <param name="to">The ending date to load data to.</param>
/// <returns>A collection of LoadedMarketData objects that represent the data
/// loaded.</returns>
public ICollection<LoadedMarketData> Load(TickerSymbol ticker,
IList<MarketDataType> dataNeeded, DateTime from,
DateTime to)
{
ICollection<LoadedMarketData> result =
new List<LoadedMarketData>();
Uri url = buildURL(ticker, from, to);
WebRequest http = HttpWebRequest.Create(url);
HttpWebResponse response = (HttpWebResponse)http.GetResponse();
using (Stream istream = response.GetResponseStream())
{
ReadCSV csv = new ReadCSV(istream, true, CSVFormat.DECIMAL_POINT);
while (csv.Next())
{
DateTime date = csv.GetDate("date");
double adjClose = csv.GetDouble("adj close");
double open = csv.GetDouble("open");
double close = csv.GetDouble("close");
double high = csv.GetDouble("high");
double low = csv.GetDouble("low");
double volume = csv.GetDouble("volume");
LoadedMarketData data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.ADJUSTED_CLOSE, adjClose);
data.SetData(MarketDataType.OPEN, open);
data.SetData(MarketDataType.CLOSE, close);
data.SetData(MarketDataType.HIGH, high);
data.SetData(MarketDataType.LOW, low);
data.SetData(MarketDataType.OPEN, open);
data.SetData(MarketDataType.VOLUME, volume);
result.Add(data);
}
csv.Close();
istream.Close();
}
return result;
}
/// <summary>
/// Analyze the file.
/// </summary>
private void AnalyzeFile()
{
ScriptProperties prop = _analyst.Script.Properties;
// get filenames, headers & format
String sourceID = prop.GetPropertyString(
ScriptProperties.HeaderDatasourceRawFile);
FileInfo sourceFile = _analyst.Script.ResolveFilename(sourceID);
CSVFormat format = _analyst.Script.DetermineFormat();
bool headers = _analyst.Script.ExpectInputHeaders(sourceID);
// read the file
_rowCount = 0;
_missingCount = 0;
var csv = new ReadCSV(sourceFile.ToString(), headers, format);
while (csv.Next())
{
_rowCount++;
if (csv.HasMissing())
{
_missingCount++;
}
}
csv.Close();
}
/// <summary>
/// Process the input file and segregate into the output files.
/// </summary>
///
public void Process()
{
Validate();
var csv = new ReadCSV(InputFilename.ToString(),
ExpectInputHeaders, InputFormat);
ResetStatus();
foreach (SegregateTargetPercent target in _targets)
{
StreamWriter tw = PrepareOutputFile(target.Filename);
while ((target.NumberRemaining > 0) && csv.Next()
&& !ShouldStop())
{
UpdateStatus(false);
var row = new LoadedRow(csv);
WriteRow(tw, row);
target.NumberRemaining = target.NumberRemaining - 1;
}
tw.Close();
}
ReportDone(false);
csv.Close();
}
public ICollection<LoadedMarketData> Load(
TickerSymbol ticker,
IList<MarketDataType> dataNeeded,
DateTime from,
DateTime to)
{
// TODO: nyyyyyyyaaagh!
ICollection<LoadedMarketData> result =
new List<LoadedMarketData>();
Uri url = BuildURL(ticker, from, to);
WebRequest http = HttpWebRequest.Create(url);
HttpWebResponse response = http.GetResponse() as HttpWebResponse;
using (Stream istream = response.GetResponseStream())
{
ReadCSV csv = new ReadCSV(
istream,
true,
CSVFormat.DECIMAL_POINT
);
while (csv.Next())
{
// todo: edit headers to match
DateTime date = csv.GetDate("DATE");
date =
date.Add(
new TimeSpan(
csv.GetDate("TIME").Hour,
csv.GetDate("TIME").Minute,
csv.GetDate("TIME").Second
)
);
double open = csv.GetDouble("OPEN");
double high = csv.GetDouble("MIN");
double low = csv.GetDouble("MAX");
double close = csv.GetDouble("CLOSE");
double volume = csv.GetDouble("VOLUME");
LoadedMarketData data =
new LoadedMarketData(date, ticker);
data.SetData(MarketDataType.OPEN, open);
data.SetData(MarketDataType.HIGH, high);
data.SetData(MarketDataType.LOW, low);
data.SetData(MarketDataType.CLOSE, close);
data.SetData(MarketDataType.VOLUME, volume);
result.Add(data);
}
csv.Close();
istream.Close();
}
return result;
}
private void x076efb43809972d8()
{
string str;
FileInfo info;
CSVFormat format;
bool flag;
ScriptProperties properties = this._x554f16462d8d4675.Script.Properties;
Label_00AD:
str = properties.GetPropertyString("HEADER:DATASOURCE_rawFile");
do
{
info = this._x554f16462d8d4675.Script.ResolveFilename(str);
if (((uint) flag) < 0)
{
goto Label_00AD;
}
format = this._x554f16462d8d4675.Script.DetermineInputFormat(str);
flag = this._x554f16462d8d4675.Script.ExpectInputHeaders(str);
}
while (0 != 0);
this._x0fe0496cde3d05e3 = 0;
if (0 == 0)
{
this._xed3494f8db69efb7 = 0;
ReadCSV dcsv = new ReadCSV(info.ToString(), flag, format);
while (dcsv.Next())
{
this._x0fe0496cde3d05e3++;
if (dcsv.HasMissing())
{
this._xed3494f8db69efb7++;
}
}
dcsv.Close();
if (((uint) flag) > uint.MaxValue)
{
goto Label_00AD;
}
}
}
public ICollection<LoadedMarketData> Load(TickerSymbol ticker, IList<MarketDataType> dataNeeded, DateTime from, DateTime to)
{
ICollection<LoadedMarketData> is2 = new List<LoadedMarketData>();
HttpWebResponse response = (HttpWebResponse) WebRequest.Create(x38c212309d8d5dd3(ticker, from, to)).GetResponse();
using (Stream stream = response.GetResponseStream())
{
DateTime time;
double num;
double num2;
double num3;
double num4;
double num5;
double num6;
LoadedMarketData data;
ReadCSV dcsv = new ReadCSV(stream, true, CSVFormat.DecimalPoint);
goto Label_005B;
Label_003F:
data.SetData(MarketDataType.Open, num2);
Label_0049:
data.SetData(MarketDataType.Volume, num6);
is2.Add(data);
Label_005B:
if (dcsv.Next())
{
goto Label_01AF;
}
dcsv.Close();
stream.Close();
if ((((uint) num) - ((uint) num5)) <= uint.MaxValue)
{
goto Label_0125;
}
if ((((uint) num) & 0) == 0)
{
goto Label_00B0;
}
Label_00A4:
data.SetData(MarketDataType.Low, num5);
goto Label_003F;
Label_00B0:
data.SetData(MarketDataType.AdjustedClose, num);
do
{
data.SetData(MarketDataType.Open, num2);
if ((((uint) num) - ((uint) num6)) > uint.MaxValue)
{
goto Label_0049;
}
data.SetData(MarketDataType.Close, num3);
data.SetData(MarketDataType.High, num4);
}
while ((((uint) num5) - ((uint) num3)) > uint.MaxValue);
if (((uint) num2) >= 0)
{
goto Label_00A4;
}
goto Label_003F;
Label_0125:
if (((uint) num2) >= 0)
{
return is2;
}
goto Label_005B;
Label_013C:
num6 = dcsv.GetDouble("volume");
data = new LoadedMarketData(time, ticker);
if ((((uint) num2) | 2) == 0)
{
goto Label_017C;
}
goto Label_00B0;
Label_016E:
num2 = dcsv.GetDouble("open");
Label_017C:
num3 = dcsv.GetDouble("close");
num4 = dcsv.GetDouble("high");
num5 = dcsv.GetDouble("low");
goto Label_013C;
Label_01AF:
time = dcsv.GetDate("date");
num = dcsv.GetDouble("adj close");
if ((((uint) num3) + ((uint) num6)) <= uint.MaxValue)
{
goto Label_016E;
}
goto Label_00B0;
}
}