private Frame <int, string> CreateNonNormalizedFrame(TimeSeriesResponse response)
{
Frame <int, string> TotalFrame = Frame.CreateEmpty <int, string>();
int totalIndex = 0;
foreach (var timeseries in response.timeseriesData)
{
var ric = timeseries.ric;
var errorCode = timeseries.statusCode;
if (errorCode.ToString().ToLower() == "error")
{
_logger.LogError("Error: {0} {1}", timeseries.ric, timeseries.errorMessage);
continue;
}
Frame <int, string> f = Frame.CreateEmpty <int, string>();
var symbolColumn = Enumerable.Repeat(ric, timeseries.dataPoints.Count()).ToList <string>();
f.AddColumn("Security", CreateSeriesString(symbolColumn, totalIndex));
for (int i = 0; i < timeseries.fields.Count(); i++)
{
var field = timeseries.fields[i];
var list = timeseries.dataPoints.Select(x => x[i]).ToList();
//f.AddColumn(field.name, CreateSeriesObject(list, totalIndex));
// Console.WriteLine(field.name);
if (field.type.ToLower() == "datetime")
{
f.AddColumn(field.name, CreateSeries <DateTime>(list, totalIndex));
}
else if (field.type.ToLower() == "double")
{
f.AddColumn(field.name, CreateSeriesDouble(list, totalIndex));
}
else if (field.type.ToLower() == "integer" || field.type.ToLower() == "int")
{
f.AddColumn(field.name, CreateSeries <int>(list, totalIndex));
}
else
{
f.AddColumn(field.name, CreateSeries <string>(list, totalIndex));
}
}
TotalFrame = TotalFrame.Merge(f);
totalIndex = totalIndex + timeseries.dataPoints.Count();
}
return(TotalFrame);
}
public Frame <int, string> TransformMailSubjectsToWords()
{
mailSubjectWords = CreateWordFrame(transformedMails.GetColumn <string>("subject"));
mailSubjectWords.AddColumn("is_ham", transformedMails.GetColumn <int>("is_ham"));
return(mailSubjectWords);
}
public void MergeLabelsToEmailData(Frame <int, string> emailDataFrame)
{
var labelDataFrame = Frame.ReadCsv(_appSettings.LabelFilePath,
hasHeaders: false,
separators: " ",
schema: "int,string");
emailDataFrame.AddColumn(DataConstants.Ham, labelDataFrame.GetColumnAt <string>(0));
}
private Frame <int, string> CreateNormalizedFrame(TimeSeriesResponse response)
{
Frame <int, string> TotalFrame = Frame.CreateEmpty <int, string>();
int totalIndex = 0;
foreach (var timeseries in response.timeseriesData)
{
var ric = timeseries.ric;
var errorCode = timeseries.statusCode;
if (errorCode.ToString().ToLower() == "error")
{
_logger.LogError("Error: {0} {1}", timeseries.ric, timeseries.errorMessage);
continue;
}
var fields = timeseries.fields.Select(x => x.name).ToList();
var timeStampIndex = fields.IndexOf("TIMESTAMP");
fields.Remove("TIMESTAMP");
var timestamps = timeseries.dataPoints.Select(x => x[timeStampIndex]).ToList();
timeseries.dataPoints.ForEach(x => x.RemoveAt(timeStampIndex));
var fieldCount = fields.Count();
var columnSize = timeseries.dataPoints.Count();
var symbolColumn = Enumerable.Repeat(ric, fieldCount * columnSize).ToList <string>();
var fieldsColumn = Enumerable.Repeat(fields, columnSize).ToList().SelectMany(i => i).ToList();
var valueColumn = timeseries.dataPoints.SelectMany(i => i).ToList();
var timeStampColumn = timestamps.Select(x => Enumerable.Repeat(x, fieldCount).ToList()).ToList().SelectMany(i => i).ToList();
Frame <int, string> f = Frame.CreateEmpty <int, string>();
f.AddColumn("Date", CreateSeries <DateTime>(timeStampColumn, totalIndex));
f.AddColumn("Field", CreateSeriesString(fieldsColumn, totalIndex));
f.AddColumn("Security", CreateSeriesString(symbolColumn, totalIndex));
f.AddColumn("Value", CreateSeriesDouble(valueColumn, totalIndex));
TotalFrame = TotalFrame.Merge(f);
totalIndex = totalIndex + (fieldCount * columnSize);
}
return(TotalFrame);
}
private static void EncodeDigit(Frame <int, string> data)
{
for (int i = 0; i < 10; i++)
{
data.AddColumn($"Label{i}",
data["Column1"]
.Values
.Select(v => (int)v == i ? 1.0 : 0.0));
}
}
private Frame <int, string> CreateFrame(NewsHeadlinesResponse response)
{
if (response == null)
{
return(Frame.CreateEmpty <int, string>());
}
Frame <int, string> headlinesFrame = Frame.CreateEmpty <int, string>();
headlinesFrame.AddColumn("FirstCreated", CreateSeriesDateTime(response.headlines.Select(h => h.firstCreated).ToList(), 0));
headlinesFrame.AddColumn("VersionCreated", CreateSeriesDateTime(response.headlines.Select(h => h.versionCreated).ToList(), 0));
headlinesFrame.AddColumn("Text", CreateSeriesString(response.headlines.Select(h => h.text).ToList(), 0));
headlinesFrame.AddColumn("StoryId", CreateSeriesString(response.headlines.Select(h => h.storyId).ToList(), 0));
headlinesFrame.AddColumn("SourceCode", CreateSeriesString(response.headlines.Select(h => h.sourceCode).ToList(), 0));
return(headlinesFrame);
}
//------------------------------------------------------------------------------------------------
public static Frame<string, string> sortFrameByNaturalOrder(Frame<string, string> input, List<string>contractSortOrder )
//------------------------------------------------------------------------------------------------
{
var series = input.GetColumn<string>("ticker").SelectValues(s => futuresSortValue(s, contractSortOrder));
input.AddColumn("SortValue",series);
input = input.SortRows("SortValue");
return input;
}