/// <summary>
/// takes all or part of quandl object and converts it into GT
/// </summary>
/// <param name="ro"></param>
/// <param name="datecol"></param>
/// <param name="valcol"></param>
/// <param name="startrow"></param>
/// <param name="endrow"></param>
/// <returns></returns>
public static GenericTracker<decimal> Qdl2GT(RootObject ro, string datecol, string valcol, int startrow, int endrow, DebugDelegate d)
{
// get columns
var datecolidx = GetQdlColidx(ro,datecol);
var valcolidx = GetQdlColidx(ro,valcol);
// slice out the data
var subset = GetAllRows(ro, startrow, endrow, d);
// get date column
var dates = QdlCol2Dates( GetColumn(subset, datecolidx));
var vals =QdlCol2Vals( GetColumn(subset, valcolidx));
// populate GT
GenericTracker<decimal> gt = new GenericTracker<decimal>(dates.Count);
for (int i = 0; i < dates.Count; i++)
{
var val = vals[i];
var dt = dates[i];
if (val == qh.ERROR_VALUE)
continue;
if (dt == qh.ERROR_DATE)
continue;
var tldate = Util.ToTLDate(dt);
gt.addindex(tldate.ToString("F0"), val);
}
return gt;
}
/// <summary>
/// converts a quandl object into a gt (w/default date column)
/// </summary>
/// <param name="ro"></param>
/// <param name="valcol"></param>
/// <param name="d"></param>
/// <returns></returns>
public static GenericTracker <decimal> Qdl2GT(RootObject ro, string valcol, DebugDelegate d)
{
return(Qdl2GT(ro, DefaultDateColumnName, valcol, d));
}
public static RootObject GetRandomData(Datacode dc, int numrecords, decimal basevalue, decimal mult, DateTime startdate)
{
RootObject ro = new RootObject();
if (dc!=null)
ro.source_code = dc.ToDatacodeString('.');
System.Random rnd = new Random();
ro.id = rnd.Next(1, 1000000);
ro.isCacheable = false;
ro.column_names = new List<string>();
ro.column_names.Add(qh.DefaultDateColumnName);
ro.column_names.Add(qh.DefaultValueColumnName);
ro.data = new List<List<object>>();
for (int i = 0; i < numrecords; i++)
{
var val = ((decimal)rnd.NextDouble() * mult) + basevalue;
var date = new DateTime(startdate.Ticks);
ro.data.Add(new List<object>(new object[] { date,val }));
// next date
startdate = startdate.AddDays(1);
}
return ro;
}
/// <summary>
/// pull all data with valid dates
/// </summary>
/// <param name="ro"></param>
/// <param name="colname"></param>
/// <param name="d"></param>
/// <returns></returns>
public static List<List<object>> GetAllDataWithDates(RootObject ro, string colname, DebugDelegate d)
{
var cidx = GetQdlColidx(ro,colname);
if (cidx<0)
{
if (d!=null)
d("Invalid column name: "+colname+" in: "+ro.id);
return new List<List<object>>();
}
return GetAllDataWithDates(ro, cidx, d);
}
/// <summary>
/// gets only column from data set
/// </summary>
/// <param name="ro"></param>
/// <param name="colidx"></param>
/// <returns></returns>
public static List<object> GetColumn(RootObject ro , int colidx)
{
return GetColumn(ro.data, colidx);
}
/// <summary>
/// pull all date with valid dates
/// </summary>
/// <param name="ro"></param>
/// <param name="colname"></param>
/// <returns></returns>
public static List <List <object> > GetAllDataWithDates(RootObject ro, string colname)
{
return(GetAllDataWithDates(ro, colname, null));
}
/// <summary>
/// gets columns from end of dataset
/// </summary>
/// <param name="ro"></param>
/// <param name="colname"></param>
/// <param name="start"></param>
/// <returns></returns>
public static List <object> GetColumn_After(RootObject ro, string colname, int start)
{
return(GetColumn(ro, colname, start, ro.LastRowIndex));
}
/// <summary>
/// returns whether a given row index is valid or not
/// </summary>
/// <param name="rowidx"></param>
/// <param name="ro"></param>
/// <returns></returns>
public static bool isValidQdlRowIdx(int rowidx, RootObject ro)
{
return(isValidQdlRowIdx(rowidx, ro.data));
}
/// <summary>
/// gets a quandl date given row and column information
/// </summary>
/// <param name="ro"></param>
/// <param name="datacolname"></param>
/// <param name="dataidx"></param>
/// <param name="d"></param>
/// <returns></returns>
public static DateTime GetQdlDate(RootObject ro, string datacolname, int dataidx, DebugDelegate d)
{
var tmp = GetQdlData(ro, datacolname, dataidx, d);
if (tmp == null)
return ERROR_DATE;
var dt = ERROR_DATE;
if (DateTime.TryParse(tmp.ToString(), out dt))
return dt;
return ERROR_DATE;
}
/// <summary>
/// gets quandl date from a given row (using default column date name)
/// </summary>
/// <param name="ro"></param>
/// <param name="dataidx"></param>
/// <param name="d"></param>
/// <returns></returns>
public static DateTime GetQdlDate(RootObject ro, int dataidx, DebugDelegate d)
{
return GetQdlDate(ro, DefaultDateColumnName, dataidx, d);
}
/// <summary>
/// gets index immediately preceeding certain date
/// </summary>
/// <param name="ro"></param>
/// <param name="dt"></param>
/// <param name="colidx"></param>
/// <param name="d"></param>
/// <returns></returns>
public static int GetQdlDataIndex_Before(RootObject ro, DateTime dt, int colidx, DebugDelegate d)
{
if (_d == null)
_d = d;
var date = Util.ToTLDate(dt);
if ((colidx < 0) || (colidx >= ro.column_names.Count))
{
debug(ro.id + " can't get column data index for date: " + date + " because column id is invalid: " + colidx);
return -1;
}
for (int i = ro.LastRowIndex; i >= 0; i--)
{
var thisdate = DateTime.MinValue;
var os = ro.data[i][colidx].ToString();
if (DateTime.TryParse(os, out thisdate))
{
var tldate = Util.ToTLDate(thisdate);
if (tldate < date)
{
return i;
}
}
}
return -1;
}
/// <summary>
/// gets first index before a certain date
/// </summary>
/// <param name="ro"></param>
/// <param name="dt"></param>
/// <param name="datecol"></param>
/// <param name="d"></param>
/// <returns></returns>
public static int GetQdlDataIndex_Before(RootObject ro, DateTime dt, string datecol, DebugDelegate d)
{
var colidx = GetQdlColidx(ro, datecol);
return GetQdlDataIndex_Before(ro, dt, colidx, d);
}
/// <summary>
/// gets first index before a certain date
/// </summary>
/// <param name="ro"></param>
/// <param name="dt"></param>
/// <param name="d"></param>
/// <returns></returns>
public static int GetQdlDataIndex_Before(RootObject ro, DateTime dt, DebugDelegate d)
{
return GetQdlDataIndex_Before(ro, dt, DefaultDateColumnName, d);
}
/// <summary>
/// converts a quandl object into a gt (w/default date column)
/// </summary>
/// <param name="ro"></param>
/// <param name="valcol"></param>
/// <param name="d"></param>
/// <returns></returns>
public static GenericTracker<decimal> Qdl2GT(RootObject ro, string valcol, DebugDelegate d)
{
return Qdl2GT(ro, DefaultDateColumnName, valcol, d);
}
static List<string> colnames_insensitive(RootObject ro)
{
List<string> cnl = new List<string>();
for (int i = 0; i < ro.column_names.Count; i++)
cnl.Add(ro.column_names[i].ToLower());
return cnl;
}
/// <summary>
/// gets first index before a certain date
/// </summary>
/// <param name="ro"></param>
/// <param name="dt"></param>
/// <param name="datecol"></param>
/// <param name="d"></param>
/// <returns></returns>
public static int GetQdlDataIndex_Before(RootObject ro, DateTime dt, string datecol, DebugDelegate d)
{
var colidx = GetQdlColidx(ro, datecol);
return(GetQdlDataIndex_Before(ro, dt, colidx, d));
}
/// <summary>
/// gets quandl date from a given row (using default column date name)
/// </summary>
/// <param name="ro"></param>
/// <param name="dataidx"></param>
/// <param name="d"></param>
/// <returns></returns>
public static DateTime GetQdlDate(RootObject ro, int dataidx, DebugDelegate d)
{
return(GetQdlDate(ro, DefaultDateColumnName, dataidx, d));
}
/// <summary>
/// convert quandl object to gt
/// </summary>
/// <param name="ro"></param>
/// <param name="datecol"></param>
/// <param name="valcol"></param>
/// <param name="d"></param>
/// <returns></returns>
public static GenericTracker<decimal> Qdl2GT(RootObject ro, string datecol, string valcol, DebugDelegate d)
{
return Qdl2GT(ro, datecol, valcol, 0, ro.LastRowIndex, d);
}
/// <summary>
/// gets quandl value for a given date
/// </summary>
/// <param name="ro"></param>
/// <param name="dataidx"></param>
/// <param name="d"></param>
/// <returns></returns>
public static decimal GetQdlValue(RootObject ro, DateTime dt, string datecol, string valcol, DebugDelegate d)
{
var dataidx = GetQdlDataIndex(ro, dt, datecol, d);
return(GetQdlValue(ro, valcol, dataidx, d));
}
/// <summary>
/// returns whether a given column index is valid or not
/// </summary>
/// <param name="colidx"></param>
/// <param name="ro"></param>
/// <returns></returns>
public static bool isValidQdlColIdx(int colidx, RootObject ro)
{
return isValidQdlColIdx(colidx, ro.data);
}
/// <summary>
/// gets only a column from data set
/// </summary>
/// <param name="ro"></param>
/// <param name="colname"></param>
/// <returns></returns>
public static List <object> GetColumn(RootObject ro, string colname)
{
var colidx = GetQdlColidx(ro, colname);
return(GetColumn(ro.data, colidx));
}
/// <summary>
/// returns whether a given row index is valid or not
/// </summary>
/// <param name="rowidx"></param>
/// <param name="ro"></param>
/// <returns></returns>
public static bool isValidQdlRowIdx(int rowidx, RootObject ro)
{
return isValidQdlRowIdx(rowidx, ro.data);
}
/// <summary>
/// gets a column index from name (optional case sensitive)
/// </summary>
/// <param name="ro"></param>
/// <param name="colname"></param>
/// <returns></returns>
public static int GetQdlColidx(RootObject ro, string colname, bool casesensitive)
{
var colnames = casesensitive ? ro.column_names : colnames_insensitive(ro);
var compare = casesensitive ? colname : colname.ToLower();
return colnames.IndexOf(compare);
}
/// <summary>
/// gets quandl value for given date (default date column)
/// </summary>
/// <param name="ro"></param>
/// <param name="dt"></param>
/// <param name="valcol"></param>
/// <param name="d"></param>
/// <returns></returns>
public static decimal GetQdlValue(RootObject ro, DateTime dt, string valcol, DebugDelegate d)
{
return GetQdlValue(ro, dt, DefaultDateColumnName, valcol, d);
}
/// <summary>
/// pull all date with valid dates
/// </summary>
/// <param name="ro"></param>
/// <param name="colname"></param>
/// <returns></returns>
public static List<List<object>> GetAllDataWithDates(RootObject ro, string colname) { return GetAllDataWithDates(ro, colname, null); }
/// <summary>
/// gets quandl value for a given date
/// </summary>
/// <param name="ro"></param>
/// <param name="dataidx"></param>
/// <param name="d"></param>
/// <returns></returns>
public static decimal GetQdlValue(RootObject ro, DateTime dt, string datecol, string valcol, DebugDelegate d)
{
var dataidx = GetQdlDataIndex(ro, dt, datecol, d);
return GetQdlValue(ro, valcol, dataidx, d);
}
/// <summary>
/// pull all data with valid dates
/// </summary>
/// <param name="ro"></param>
/// <param name="colidx"></param>
/// <param name="d"></param>
/// <returns></returns>
public static List<List<object>> GetAllDataWithDates(RootObject ro, int colidx, DebugDelegate d)
{
List<List<object>> dates = new List<List<object>>();
for (int i = 0; i<ro.data.Count; i++)
{
var data = ro.data[i];
var tmpdate = data[colidx].ToString();
DateTime cur;
if (DateTime.TryParse(tmpdate, out cur))
{
dates.Add(data);
#if DEBUG
if (d != null)
d("got date: " + cur.ToShortDateString());
#endif
}
}
return dates;
}
/// <summary>
/// gets quandl value for a given row (using default column value name)
/// </summary>
/// <param name="ro"></param>
/// <param name="dataidx"></param>
/// <param name="d"></param>
/// <returns></returns>
public static decimal GetQdlValue(RootObject ro, int dataidx, DebugDelegate d)
{
return GetQdlValue(ro, DefaultValueColumnName, dataidx, d);
}
/// <summary>
/// gets a column index from name (case insensitive)
/// </summary>
/// <param name="ro"></param>
/// <param name="colname"></param>
/// <returns></returns>
public static int GetQdlColidx(RootObject ro, string colname)
{
return(GetQdlColidx(ro, colname, false));
}
/// <summary>
/// gets decimal qdl value
/// </summary>
/// <param name="ro"></param>
/// <param name="colname"></param>
/// <param name="dataidx"></param>
/// <param name="d"></param>
/// <returns></returns>
public static decimal GetQdlValue(RootObject ro, string colname, int dataidx, DebugDelegate d)
{
if (_d == null)
_d = d;
var obj = GetQdlData(ro, colname, dataidx, d);
decimal v = ERROR_VALUE;
if ((obj != null) && decimal.TryParse(obj.ToString(), out v))
return v;
return ERROR_VALUE;
}
/// <summary>
/// gets first index before a certain date
/// </summary>
/// <param name="ro"></param>
/// <param name="dt"></param>
/// <param name="d"></param>
/// <returns></returns>
public static int GetQdlDataIndex_Before(RootObject ro, DateTime dt, DebugDelegate d)
{
return(GetQdlDataIndex_Before(ro, dt, DefaultDateColumnName, d));
}
/// <summary>
/// get quandl data given row and column information
/// </summary>
/// <param name="ro"></param>
/// <param name="datacolname"></param>
/// <param name="dataidx"></param>
/// <param name="d"></param>
/// <returns></returns>
public static object GetQdlData(RootObject ro, string datacolname, int dataidx, DebugDelegate d)
{
_d = d;
// check column name
var colidx = ro.column_names.IndexOf(datacolname);
if (colidx < 0)
{
debug(ro.id + " can't get data because non existant colname: " + datacolname+" on set: "+ro.code);
return null;
}
return GetQdlData(ro, colidx, dataidx, d);
}
/// <summary>
/// convert quandl object to gt
/// </summary>
/// <param name="ro"></param>
/// <param name="datecol"></param>
/// <param name="valcol"></param>
/// <param name="d"></param>
/// <returns></returns>
public static GenericTracker <decimal> Qdl2GT(RootObject ro, string datecol, string valcol, DebugDelegate d)
{
return(Qdl2GT(ro, datecol, valcol, 0, ro.LastRowIndex, d));
}
/// <summary>
/// gets quandl data given row and column information
/// </summary>
/// <param name="ro"></param>
/// <param name="datacolname"></param>
/// <param name="dataidx"></param>
/// <param name="d"></param>
/// <returns></returns>
public static object GetQdlData(RootObject ro, int colidx, int dataidx, DebugDelegate d)
{
if (_d == null)
_d = d;
// check data index
if ((dataidx < 0) || (dataidx >= ro.data.Count))
{
debug(ro.id + " can't get data because invalid data index: " + dataidx+" on set: "+ro.code);
return null;
}
// get object
return ro.data[dataidx][colidx];
}
/// <summary>
/// returns whether a given column index is valid or not
/// </summary>
/// <param name="colidx"></param>
/// <param name="ro"></param>
/// <returns></returns>
public static bool isValidQdlColIdx(int colidx, RootObject ro)
{
return(isValidQdlColIdx(colidx, ro.data));
}
/// <summary>
/// gets all data between two rows (inclusive)
/// </summary>
/// <param name="ro"></param>
/// <param name="startidx"></param>
/// <param name="endidx"></param>
/// <param name="d"></param>
/// <returns></returns>
public static List<List<object>> GetAllRows(RootObject ro, int startidx, int endidx, DebugDelegate d)
{
if (_d == null)
_d = d;
List<List<object>> rows = new List<List<object>>();
for (int i = startidx; i <= endidx; i++)
{
if (i < 0)
i = 0;
if (i >= ro.data.Count)
break;
rows.Add(ro.data[i]);
}
return rows;
}
/// <summary>
/// gets quandl value for given date (default date column)
/// </summary>
/// <param name="ro"></param>
/// <param name="dt"></param>
/// <param name="valcol"></param>
/// <param name="d"></param>
/// <returns></returns>
public static decimal GetQdlValue(RootObject ro, DateTime dt, string valcol, DebugDelegate d)
{
return(GetQdlValue(ro, dt, DefaultDateColumnName, valcol, d));
}
/// <summary>
/// gets columns from end of dataset
/// </summary>
/// <param name="ro"></param>
/// <param name="colname"></param>
/// <param name="start"></param>
/// <returns></returns>
public static List<object> GetColumn_After(RootObject ro, string colname, int start)
{
return GetColumn(ro, colname, start, ro.LastRowIndex);
}
/// <summary>
/// gets quandl value for a given row (using default column value name)
/// </summary>
/// <param name="ro"></param>
/// <param name="dataidx"></param>
/// <param name="d"></param>
/// <returns></returns>
public static decimal GetQdlValue(RootObject ro, int dataidx, DebugDelegate d)
{
return(GetQdlValue(ro, DefaultValueColumnName, dataidx, d));
}
/// <summary>
/// gets column from start of data until a certain row
/// </summary>
/// <param name="ro"></param>
/// <param name="colname"></param>
/// <param name="end"></param>
/// <returns></returns>
public static List<object> GetColumn_Until(RootObject ro, string colname, int end)
{
return GetColumn(ro, colname, 0, end);
}
/// <summary>
/// gets column from start of data until a certain row
/// </summary>
/// <param name="ro"></param>
/// <param name="colname"></param>
/// <param name="end"></param>
/// <returns></returns>
public static List <object> GetColumn_Until(RootObject ro, string colname, int end)
{
return(GetColumn(ro, colname, 0, end));
}
/// <summary>
/// gets column between two rows
/// </summary>
/// <param name="ro"></param>
/// <param name="colname"></param>
/// <param name="start"></param>
/// <param name="end"></param>
/// <returns></returns>
public static List<object> GetColumn(RootObject ro, string colname, int start, int end)
{
var colidx = GetQdlColidx(ro, colname);
var rows = GetAllRows(ro, start, end, null);
return GetColumn(rows, colidx);
}
/// <summary>
/// gets only column from data set
/// </summary>
/// <param name="ro"></param>
/// <param name="colidx"></param>
/// <returns></returns>
public static List <object> GetColumn(RootObject ro, int colidx)
{
return(GetColumn(ro.data, colidx));
}
/// <summary>
/// gets only a column from data set
/// </summary>
/// <param name="ro"></param>
/// <param name="colname"></param>
/// <returns></returns>
public static List<object> GetColumn(RootObject ro, string colname)
{
var colidx = GetQdlColidx(ro, colname);
return GetColumn(ro.data, colidx);
}
/// <summary>
/// get a quandl data set for particular symbol/column/frequency
/// </summary>
/// <param name="ds"></param>
/// <param name="symbol"></param>
/// <param name="trycache"></param>
/// <param name="d"></param>
/// <returns></returns>
public static RootObject Get(qdl_ds ds, string symbol, string additionaldataset, qdl_freq freq, bool trycache, DebugDelegate d)
{
if (_d == null)
{
_d = d;
}
Datacode dc = null;
string raw = string.Empty;
RootObject ro = null;
try
{
if (string.IsNullOrWhiteSpace(additionaldataset))
{
dc = BuildQdlCode(ds, freq, symbol);
}
else
{
dc = BuildQdlCode(ds, freq, symbol, additionaldataset);
}
}
catch (Exception ex)
{
debug("Unable to get data, error building dataset: " + qh.dc2str(dc) + " " + symbol + " err: " + ex.Message + ex.StackTrace);
return(default(RootObject));
}
try
{
raw = GetData(symbol, dc, trycache, d);
// verify data retrieved before continuing
if (string.IsNullOrWhiteSpace(raw))
{
if (isUseFakeDataOnError)
{
ro = GetRandomData(dc, freq);
debug("FakeDataOnError enabled, using: " + ro.data.Count + " randomized data set.");
return(ro);
}
else
{
debug(symbol + " " + qh.dc2str(dc) + " no data was retrieved. Retry or contact [email protected]");
return(default(RootObject));
}
}
}
catch (Exception ex)
{
debug("Unable to get data, error retrieving set: " + qh.dc2str(dc) + " " + symbol + " err: " + ex.Message + ex.StackTrace);
return(default(RootObject));
}
try
{
ro = json.Deserialize2Root(raw, false, d);
v(symbol + " " + qh.dc2str(dc) + " got data model containing " + ro.column_names.Count.ToString("N0") + " columns and " + ro.data.Count.ToString("N0") + " records.");
}
catch (Exception ex)
{
if (isVerboseDebugging)
{
debug("Unable to get data, error parsing set: " + ds + " " + symbol + " err: " + ex.Message + ex.StackTrace);
debug(symbol + " " + qh.dc2str(dc) + " errored data set: " + raw);
}
else
{
debug("Unable to get data, error parsing set: " + qh.dc2str(dc) + " " + symbol + " err: " + ex.Message + ex.StackTrace);
}
return(default(RootObject));
}
return(ro);
}
/// <summary>
/// gets a column index from name (case insensitive)
/// </summary>
/// <param name="ro"></param>
/// <param name="colname"></param>
/// <returns></returns>
public static int GetQdlColidx(RootObject ro, string colname)
{
return GetQdlColidx(ro, colname, false);
}
