public void TestCSVLoader()
{
var loader = new CSVFinal();
loader.DateFormat = "yyyy.MM.dd hh:mm:ss";
var tickerAAPL = new TickerSymbol("AAPL", "NY");
var desc = new MarketDataDescription(tickerAAPL, MarketDataType.Close, true, true);
MarketMLDataSet marketData = new MarketMLDataSet(loader, 5, 1);
marketData.AddDescription(desc);
marketData.SequenceGrandularity = Util.Time.TimeUnit.Hours;
var begin = new DateTime(2006, 1, 1);
var end = new DateTime(2007, 7, 31);
loader.GetFile((AssemblyDirectory + "\\smallCSV.csv"));
marketData.Load(begin, end);
marketData.Generate();
// first test the points
IEnumerator <TemporalPoint> itr = marketData.Points.GetEnumerator();
itr.MoveNext();
TemporalPoint point = itr.Current;
Assert.AreEqual(0, point.Sequence);
Assert.AreEqual(1, point.Data.Length);
Assert.AreEqual(1.12884, point[0]);
Assert.AreEqual(5, marketData.Points.Count);
}
/// <summary>
/// Load one ticker symbol.
/// </summary>
/// <param name="ticker">The ticker symbol to load.</param>
/// <param name="from">Load data from this date.</param>
/// <param name="to">Load data to this date.</param>
private void LoadSymbol(TickerSymbol ticker, DateTime from,
DateTime to)
{
ICollection <LoadedMarketData> data = this.Loader.Load(ticker,
null, from, to);
foreach (LoadedMarketData item in data)
{
TemporalPoint point = this.CreatePoint(item.When);
LoadPointFromMarketData(ticker, point, item);
}
}
/// <summary>
/// Load one point of market data.
/// </summary>
/// <param name="ticker">The ticker symbol to load.</param>
/// <param name="point">The point to load at.</param>
/// <param name="item">The item being loaded.</param>
private void LoadPointFromMarketData(TickerSymbol ticker,
TemporalPoint point, LoadedMarketData item)
{
foreach (TemporalDataDescription desc in Descriptions)
{
var mdesc = (MarketDataDescription)desc;
if (mdesc.Ticker.Equals(ticker))
{
point.Data[mdesc.Index] = item.Data[mdesc.DataType];
}
}
}
/// <summary>
/// Generates a temporal data set with a given double serie.
/// uses Type percent change.
/// </summary>
/// <param name="inputserie">The inputserie.</param>
/// <param name="windowsize">The windowsize.</param>
/// <param name="predictsize">The predictsize.</param>
/// <returns></returns>
public static TemporalMLDataSet GenerateTrainingWithPercentChangeOnSerie(double[] inputserie, int windowsize, int predictsize)
{
TemporalMLDataSet result = new TemporalMLDataSet(windowsize, predictsize);
TemporalDataDescription desc = new TemporalDataDescription(TemporalDataDescription.Type.PercentChange, true, true);
result.AddDescription(desc);
for (int index = 0; index < inputserie.Length - 1; index++)
{
TemporalPoint point = new TemporalPoint(1);
point.Sequence = index;
point.Data[0] = inputserie[index];
result.Points.Add(point);
}
result.Generate();
return(result);
}
/// <summary>
/// Create a datapoint at the specified date.
/// </summary>
/// <param name="when">The date to create the point at.</param>
/// <returns>Returns the TemporalPoint created for the specified date.</returns>
public override TemporalPoint CreatePoint(DateTime when)
{
int sequence = GetSequenceFromDate(when);
TemporalPoint result = null;
if (pointIndex.ContainsKey(sequence))
{
result = this.pointIndex[sequence];
}
else
{
result = base.CreatePoint(when);
this.pointIndex[result.Sequence] = result;
}
return(result);
}
public IMLDataSet GenerateTraining()
{
var result = new TemporalMLDataSet(WindowSize, 1);
var desc = new TemporalDataDescription(TemporalDataDescription.Type.Raw, true, true);
result.AddDescription(desc);
for (int i = WindowSize; i < _normalizedTrainingData.Length; i++)
{
var point = new TemporalPoint(1)
{
Sequence = i
};
point.Data[0] = _normalizedTrainingData[i];
result.Points.Add(point);
}
result.Generate();
return(result);
}
public static IMLDataSet GenerateTraining()
{
TemporalMLDataSet result = new TemporalMLDataSet(WindowSize, 1);
TemporalDataDescription desc =
new TemporalDataDescription(TemporalDataDescription.Type.Raw, true, true);
result.AddDescription(desc);
for (int year = TrainStart; year < TrainEnd; year++)
{
TemporalPoint point = new TemporalPoint(1);
point.Sequence = year;
point.Data[0] = _normalizedSunspots[year];
result.Points.Add(point);
}
result.Generate();
return(result);
}
public IMLDataSet GenerateTraining()
{
var result = new TemporalMLDataSet(WindowSize, 1);
var desc = new TemporalDataDescription(TemporalDataDescription.Type.Raw, true, true);
result.AddDescription(desc);
for (var year = TrainStart; year < TrainEnd; year++)
{
var point = new TemporalPoint(1)
{
Sequence = year
};
point.Data[0] = _normalizedForexPair[year];
result.Points.Add(point);
}
result.Generate();
return(result);
}
/// <summary>
/// Load one ticker symbol.
/// </summary>
/// <param name="ticker">The ticker symbol to load.</param>
/// <param name="from">Load data from this date.</param>
/// <param name="to">Load data to this date.</param>
private void LoadSymbol(TickerSymbol ticker, DateTime from,
DateTime to)
{
IList <MarketDataType> types = new List <MarketDataType>();
foreach (MarketDataDescription desc in Descriptions)
{
if (desc.Ticker.Equals(ticker))
{
types.Add(desc.DataType);
}
}
ICollection <LoadedMarketData> data = Loader.Load(ticker, types, from, to);
foreach (LoadedMarketData item in data)
{
TemporalPoint point = CreatePoint(item.When);
LoadPointFromMarketData(ticker, point, item);
}
}
/// <summary>
/// Generates a temporal data set with a given double serie or a any number of double series , making your inputs.
/// uses Type percent change.
/// </summary>
/// <param name="windowsize">The windowsize.</param>
/// <param name="predictsize">The predictsize.</param>
/// <param name="inputserie">The inputserie.</param>
/// <returns></returns>
public static TemporalMLDataSet GenerateTrainingWithPercentChangeOnSerie(int windowsize, int predictsize, params double[][] inputserie)
{
TemporalMLDataSet result = new TemporalMLDataSet(windowsize, predictsize);
TemporalDataDescription desc = new TemporalDataDescription(TemporalDataDescription.Type.PercentChange, true,
true);
result.AddDescription(desc);
foreach (double[] t in inputserie)
{
for (int j = 0; j < t.Length; j++)
{
TemporalPoint point = new TemporalPoint(1);
point.Sequence = j;
point.Data[0] = t[j];
result.Points.Add(point);
}
result.Generate();
return(result);
}
return(null);
}
public IMLDataSet GenerateTraining()
{
var result = new TemporalMLDataSet(WindowSize, 1);
var desc = new TemporalDataDescription(
TemporalDataDescription.Type.Raw, true, true);
result.AddDescription(desc);
for (int day = TrainStart; day < TrainEnd; day++)
{
var point = new TemporalPoint(1)
{
Sequence = day,
Data = { [0] = _normalizedArray[day] }
};
result.Points.Add(point);
}
result.Generate();
return(result);
}
public IMLDataSet GenerateTraining(double[] normalizedData)
{
var result = new TemporalMLDataSet(WindowSize, 1);
TemporalDataDescription desc = new TemporalDataDescription(TemporalDataDescription.Type.Raw, true, true);
result.AddDescription(desc);
int TrainStart = 0;
int TrainEnd = normalizedData.Length;
for (int index = TrainStart; index < TrainEnd; index++)
{
TemporalPoint point = new TemporalPoint(1)
{
Sequence = index
};
point.Data[0] = normalizedData[index];
result.Points.Add(point);
}
result.Generate();
return(result);
}
/// <summary>
/// Generates a temporal data set with a given double serie.
/// uses Type percent change.
/// </summary>
/// <param name="inputserie">The inputserie.</param>
/// <param name="windowsize">The windowsize.</param>
/// <param name="predictsize">The predictsize.</param>
/// <returns></returns>
public static TemporalMLDataSet GenerateTrainingWithPercentChangeOnSerie(double[] inputserie, int windowsize,
int predictsize)
{
var result = new TemporalMLDataSet(windowsize, predictsize);
var desc = new TemporalDataDescription(TemporalDataDescription.Type.PercentChange, true, true);
result.AddDescription(desc);
for (int index = 0; index < inputserie.Length - 1; index++)
{
var point = new TemporalPoint(1) {Sequence = index};
point.Data[0] = inputserie[index];
result.Points.Add(point);
}
result.Generate();
return result;
}
/// <summary>
/// Load one point of market data.
/// </summary>
/// <param name="ticker">The ticker symbol to load.</param>
/// <param name="point">The point to load at.</param>
/// <param name="item">The item being loaded.</param>
private void LoadPointFromMarketData(TickerSymbol ticker,
TemporalPoint point, LoadedMarketData item)
{
foreach (TemporalDataDescription desc in Descriptions)
{
var mdesc = (MarketDataDescription) desc;
if (mdesc.Ticker.Equals(ticker))
{
point.Data[mdesc.Index] = item.Data[mdesc.DataType];
}
}
}
public IMLDataSet GenerateTraining()
{
var result = new TemporalMLDataSet(WindowSize, 1);
var desc = new TemporalDataDescription(TemporalDataDescription.Type.Raw, true, true);
result.AddDescription(desc);
for (int year = TrainStart; year < TrainEnd; year++)
{
var point = new TemporalPoint(1) { Sequence = year };
point.Data[0] = _normalizedSunspots[year];
result.Points.Add(point);
}
result.Generate();
return result;
}
/// <summary>
/// Is the specified point within the range. If a point is in the selection
/// range, then the point will be used to generate the training sets.
/// </summary>
/// <param name="point">The point to consider.</param>
/// <returns>True if the point is within the range.</returns>
public virtual bool IsPointInRange(TemporalPoint point)
{
return ((point.Sequence >= LowSequence) && (point.Sequence <= HighSequence));
}
/// <summary>
/// Create a temporal point from a time. Using the grandularity each date is
/// given a unique sequence number. No two dates that fall in the same
/// grandularity should be specified.
/// </summary>
/// <param name="when">The time that this point should be created at.</param>
/// <returns>The point TemporalPoint created.</returns>
public virtual TemporalPoint CreatePoint(DateTime when)
{
int sequence = GetSequenceFromDate(when);
var point = new TemporalPoint(_descriptions.Count) {Sequence = sequence};
_points.Add(point);
return point;
}
/// <summary>
/// Create a temporal data point using a sequence number. They can also be
/// created using time. No two points should have the same sequence number.
/// </summary>
/// <param name="sequence">The sequence number.</param>
/// <returns>A new TemporalPoint object.</returns>
public virtual TemporalPoint CreatePoint(int sequence)
{
var point = new TemporalPoint(_descriptions.Count) {Sequence = sequence};
_points.Add(point);
return point;
}
/// <summary>
/// Create a temporal data point using a sequence number. They can also be
/// created using time. No two points should have the same sequence number.
/// </summary>
/// <param name="sequence">The sequence number.</param>
/// <returns>A new TemporalPoint object.</returns>
public virtual TemporalPoint CreatePoint(int sequence)
{
TemporalPoint point = new TemporalPoint(this.descriptions.Count);
point.Sequence = sequence;
this.points.Add(point);
return point;
}
public virtual TemporalPoint CreatePoint(DateTime when)
{
int sequenceFromDate = this.GetSequenceFromDate(when);
TemporalPoint item = new TemporalPoint(this._x6849a3dfb0329317.Count) {
Sequence = sequenceFromDate
};
this._x6fa2570084b2ad39.Add(item);
return item;
}
/// <summary>
/// Generates a temporal data set with a given double serie or a any number of double series , making your inputs.
/// uses Type percent change.
/// </summary>
/// <param name="windowsize">The windowsize.</param>
/// <param name="predictsize">The predictsize.</param>
/// <param name="inputserie">The inputserie.</param>
/// <returns></returns>
public static TemporalMLDataSet GenerateTrainingWithPercentChangeOnSerie(int windowsize, int predictsize, params double[][] inputserie)
{
TemporalMLDataSet result = new TemporalMLDataSet(windowsize, predictsize);
TemporalDataDescription desc = new TemporalDataDescription(TemporalDataDescription.Type.PercentChange, true,
true);
result.AddDescription(desc);
foreach (double[] t in inputserie)
{
for (int j = 0; j < t.Length; j++)
{
TemporalPoint point = new TemporalPoint(1);
point.Sequence = j;
point.Data[0] = t[j];
result.Points.Add(point);
}
result.Generate();
return result;
}
return null;
}
public IMLDataSet GenerateTraining()
{
var result = new TemporalMLDataSet(WindowSize, 1);
var desc = new TemporalDataDescription(TemporalDataDescription.Type.Raw, true, true);
result.AddDescription(desc);
for (int i = WindowSize; i < _normalizedTrainingData.Length; i++)
{
var point = new TemporalPoint(1) {Sequence = i};
point.Data[0] = _normalizedTrainingData[i];
result.Points.Add(point);
}
result.Generate();
return result;
}
public virtual TemporalPoint CreatePoint(int sequence)
{
TemporalPoint item = new TemporalPoint(this._x6849a3dfb0329317.Count) {
Sequence = sequence
};
this._x6fa2570084b2ad39.Add(item);
return item;
}
public void MarketData()
{
IMarketLoader loader = new YahooFinanceLoader();
var tickerAAPL = new TickerSymbol("AAPL", null);
var tickerMSFT = new TickerSymbol("MSFT", null);
var marketData = new MarketMLDataSet(loader, 5, 1);
marketData.AddDescription(new MarketDataDescription(tickerAAPL, MarketDataType.Close, true, true));
marketData.AddDescription(new MarketDataDescription(tickerMSFT, MarketDataType.Close, true, false));
marketData.AddDescription(new MarketDataDescription(tickerAAPL, MarketDataType.Volume, true, false));
marketData.AddDescription(new MarketDataDescription(tickerMSFT, MarketDataType.Volume, true, false));
var begin = new DateTime(2008, 7, 1);
var end = new DateTime(2008, 7, 31);
marketData.Load(begin, end);
marketData.Generate();
Assert.AreEqual(22, marketData.Points.Count);
// first test the points
IEnumerator <TemporalPoint> itr = marketData.Points.GetEnumerator();
itr.MoveNext();
TemporalPoint point = itr.Current;
Assert.AreEqual(0, point.Sequence);
Assert.AreEqual(4, point.Data.Length);
Assert.AreEqual(174.68, point[0]);
Assert.AreEqual(26.87, point[1]);
Assert.AreEqual(39, (int)(point[2] / 1000000));
Assert.AreEqual(100, (int)(point[3] / 1000000));
itr.MoveNext();
point = itr.Current;
Assert.AreEqual(1, point.Sequence);
Assert.AreEqual(4, point.Data.Length);
Assert.AreEqual(168.18, point[0]);
Assert.AreEqual(25.88, point[1]);
Assert.AreEqual(29, (int)(point[2] / 1000000));
Assert.AreEqual(84, (int)(point[3] / 1000000));
itr.MoveNext();
point = itr.Current;
Assert.AreEqual(2, point.Sequence);
Assert.AreEqual(4, point.Data.Length);
Assert.AreEqual(170.12, point[0]);
Assert.AreEqual(25.98, point[1]);
Assert.AreEqual(18, (int)(point[2] / 1000000));
Assert.AreEqual(37, (int)(point[3] / 1000000));
// now check the actual data
Assert.AreEqual(16, marketData.Data.Count);
Assert.AreEqual(20, marketData.InputNeuronCount);
Assert.AreEqual(1, marketData.OutputNeuronCount);
IEnumerator <IMLDataPair> itr2 = marketData.Data.GetEnumerator();
itr2.MoveNext();
IMLDataPair pair = itr2.Current;
Assert.AreEqual(20, pair.Input.Count);
Assert.AreEqual(1, pair.Ideal.Count);
Assert.AreEqual(-0.037, Math.Round(pair.Input[0] * 1000.0) / 1000.0);
Assert.AreEqual(-0.037, Math.Round(pair.Input[1] * 1000.0) / 1000.0);
Assert.AreEqual(-0.246, Math.Round(pair.Input[2] * 1000.0) / 1000.0);
Assert.AreEqual(-0.156, Math.Round(pair.Input[3] * 1000.0) / 1000.0);
Assert.AreEqual(0.012, Math.Round(pair.Input[4] * 1000.0) / 1000.0);
Assert.AreEqual(0.0040, Math.Round(pair.Input[5] * 1000.0) / 1000.0);
Assert.AreEqual(-0.375, Math.Round(pair.Input[6] * 1000.0) / 1000.0);
Assert.AreEqual(-0.562, Math.Round(pair.Input[7] * 1000.0) / 1000.0);
Assert.AreEqual(0.03, Math.Round(pair.Input[8] * 1000.0) / 1000.0);
Assert.AreEqual(0.0020, Math.Round(pair.Input[9] * 1000.0) / 1000.0);
Assert.AreEqual(0.57, Math.Round(pair.Input[10] * 100.0) / 100.0);
Assert.AreEqual(0.929, Math.Round(pair.Input[11] * 1000.0) / 1000.0);
Assert.AreEqual(0.025, Math.Round(pair.Input[12] * 1000.0) / 1000.0);
Assert.AreEqual(-0.0070, Math.Round(pair.Input[13] * 1000.0) / 1000.0);
// for some reason, Yahoo likes to vary the volume numbers slightly, sometimes!
Assert.AreEqual(0.1, Math.Round(pair.Input[14] * 10.0) / 10.0);
Assert.AreEqual(-0.084, Math.Round(pair.Input[15] * 1000.0) / 1000.0);
Assert.AreEqual(-0.03, Math.Round(pair.Input[16] * 1000.0) / 1000.0);
Assert.AreEqual(-0.024, Math.Round(pair.Input[17] * 1000.0) / 1000.0);
Assert.AreEqual(0.008, Math.Round(pair.Input[18] * 1000.0) / 1000.0);
Assert.AreEqual(-0.172, Math.Round(pair.Input[19] * 1000.0) / 1000.0);
itr2.MoveNext();
pair = itr2.Current;
Assert.AreEqual(20, pair.Input.Count);
Assert.AreEqual(1, pair.Ideal.Count);
Assert.AreEqual(0.012, Math.Round(pair.Input[0] * 1000.0) / 1000.0);
Assert.AreEqual(0.0040, Math.Round(pair.Input[1] * 1000.0) / 1000.0);
Assert.AreEqual(-0.375, Math.Round(pair.Input[2] * 1000.0) / 1000.0);
Assert.AreEqual(-0.562, Math.Round(pair.Input[3] * 1000.0) / 1000.0);
Assert.AreEqual(0.03, Math.Round(pair.Input[4] * 1000.0) / 1000.0);
Assert.AreEqual(0.0020, Math.Round(pair.Input[5] * 1000.0) / 1000.0);
Assert.AreEqual(0.6, Math.Round(pair.Input[6] * 10.0) / 10.0);
Assert.AreEqual(0.929, Math.Round(pair.Input[7] * 1000.0) / 1000.0);
Assert.AreEqual(0.025, Math.Round(pair.Input[8] * 1000.0) / 1000.0);
Assert.AreEqual(-0.0070, Math.Round(pair.Input[9] * 1000.0) / 1000.0);
Assert.AreEqual(0.1, Math.Round(pair.Input[10] * 10.0) / 10.0);
Assert.AreEqual(-0.084, Math.Round(pair.Input[11] * 1000.0) / 1000.0);
Assert.AreEqual(-0.03, Math.Round(pair.Input[12] * 1000.0) / 1000.0);
Assert.AreEqual(-0.024, Math.Round(pair.Input[13] * 1000.0) / 1000.0);
Assert.AreEqual(0.0080, Math.Round(pair.Input[14] * 1000.0) / 1000.0);
Assert.AreEqual(-0.172, Math.Round(pair.Input[15] * 1000.0) / 1000.0);
Assert.AreEqual(0.014, Math.Round(pair.Input[16] * 1000.0) / 1000.0);
Assert.AreEqual(0.0090, Math.Round(pair.Input[17] * 1000.0) / 1000.0);
Assert.AreEqual(-0.1, Math.Round(pair.Input[18] * 10.0) / 10.0);
Assert.AreEqual(0.066, Math.Round(pair.Input[19] * 1000.0) / 1000.0);
}
private void xd619c0bf81b12658(TickerSymbol x96e4701dec47675e, TemporalPoint x2f7096dac971d6ec, LoadedMarketData xccb63ca5f63dc470)
{
foreach (TemporalDataDescription description in this.Descriptions)
{
MarketDataDescription description2 = (MarketDataDescription) description;
if (description2.Ticker.Equals(x96e4701dec47675e))
{
x2f7096dac971d6ec.Data[description2.Index] = xccb63ca5f63dc470.Data[description2.DataType];
}
}
}
/// <summary>
/// Create a temporal point from a time. Using the grandularity each date is
/// given a unique sequence number. No two dates that fall in the same
/// grandularity should be specified.
/// </summary>
/// <param name="when">The time that this point should be created at.</param>
/// <returns>The point TemporalPoint created.</returns>
public virtual TemporalPoint CreatePoint(DateTime when)
{
int sequence = GetSequenceFromDate(when);
TemporalPoint point = new TemporalPoint(this.descriptions.Count);
point.Sequence = sequence;
this.points.Add(point);
return point;
}