private static ResultsData RetrieveData(List <string> quandlCodeColumns,
List <DateTime?> dates, string collapse, string transformation, int?limit,
bool hideColumns = false)
{
var datasets = new Dictionary <string, DatasetParams>();
var datasetsWithoutColumns = new List <string>();
var uniqueQuandlCodes = GetDatasetQuandlCodes(quandlCodeColumns);
GetDatasetMetadata(uniqueQuandlCodes);
quandlCodeColumns = SanitizeColumnNames(quandlCodeColumns);
foreach (var quandlCodeColumn in quandlCodeColumns)
{
var splitString = SplitQuandlCode(quandlCodeColumn);
// Quandl code and column (ex: NSE/OIL/HIGH)
if (splitString.Length >= 3)
{
var quandlCode = string.Join("/", splitString[0], splitString[1]);
if (!datasets.ContainsKey(quandlCode))
{
datasets[quandlCode] = new DatasetParams(quandlCode, dates, collapse, transformation, limit);
}
// concatenate column name if it has a forward slash and was broken up in string split
string columnName = (splitString.Length > 3)
? string.Join("/", splitString.Skip(2).ToArray())
: splitString[2];
datasets[quandlCode].Columns.Add(columnName);
}
// Quandl code only (ex: NSE/OIL)
else if (splitString.Length == 2)
{
var quandlCode = string.Join("/", splitString[0], splitString[1]);
if (!datasets.ContainsKey(quandlCode))
{
datasets[quandlCode] = new DatasetParams(quandlCode, dates, collapse, transformation, limit);
}
datasetsWithoutColumns.Add(quandlCode);
}
// Invalid format
else
{
throw new DatasetParamError($"Invalid Quandl code: {quandlCodeColumn}");
}
}
// If any datasets without columns have been specified remove any customized columns that users specified to ensure all columns are pulled.
datasetsWithoutColumns.ForEach(qc => datasets[qc].Columns.Clear());
// Fetch data based on batch settings or in once based on user roles
Dataset[] fetchTaskCollection = new Dataset[] { };
var tasks = datasets.Select(dsp => new Web().GetDatasetData(dsp.Value.Code, dsp.Value.QueryParams));
int numberOfTasksForEachBatch = QuandlConfig.Instance.IsOnlyUser() ? 1 : 8;
foreach (var batchTask in tasks.Batch(numberOfTasksForEachBatch))
{
var fetchTask = Task.WhenAll(batchTask);
fetchTask.Wait();
var result = fetchTask.Result;
fetchTaskCollection = fetchTaskCollection.Concat(result).ToArray();
}
// Create a bunch of results which we can combine to one giant table
var combinedResults = new ResultsData(new List <List <object> >(), new List <string>());
foreach (var qcc in fetchTaskCollection.Select((x, i) => new { Value = x, Index = i }))
{
var dataset = qcc.Value;
var columns = dataset.Columns.Select(c => c.Code.ToUpper() == dataset.Columns[0].Code
? c.Code
: $"{dataset.Code}/{c.Code}".ToUpper()).ToList();
var newResults = new ResultsData(dataset.Data.DataPoints, columns);
combinedResults = combinedResults.Combine(newResults);
}
return(combinedResults);
}
