private static void AddCompleteDataset(QuandlDatasetData datasetData)
{
// If group doesnt exists, create it
if (!datasetsDataGroups.Exists(d => d.DatabaseCode == datasetData.DatabaseCode))
{
datasetsDataGroups.Add(new QuandlDatasetDataGroup()
{
DatabaseCode = datasetData.DatabaseCode, Datasets = new List <QuandlDatasetData>()
});
}
// Add it
datasetsDataGroups.Find(d => d.DatabaseCode == datasetData.DatabaseCode).Datasets.Add(datasetData);
}
private static void ReplaceCompleteDataset(QuandlDatasetData datasetData)
{
// Reference current groups list
var currentDatasetGroup = datasetsGroups
.Find(dg => dg.DatabaseCode == datasetData.DatabaseCode)
.Datasets;
// Index of the dataset to update
var index = currentDatasetGroup.IndexOf(currentDatasetGroup.First(ds => ds.Id == datasetData.Id));
// Replace
if (index != -1)
{
currentDatasetGroup[index] = datasetData;
}
}
private static async Task DownloadDatasetDataAsync(QuandlDataset dataset, int to)
{
using (PatientWebClient client = new PatientWebClient())
{
try
{
string data = await client.DownloadStringTaskAsync(new Uri("https://www.quandl.com/api/v3/datasets/" + dataset.DatabaseCode +
"/" + dataset.DatasetCode + "/data.json?api_key=" + Utils.Constants.API_KEY +
"&start_date=" + dataset.LastFetch.GetValueOrDefault(DateTime.MinValue).AddDays(1).ToString("yyyy-MM-dd"))); // Add one day because I dont want to include the current newest in the json
DataResponse response =
JsonConvert.DeserializeObject <DataResponse>(data, new JsonSerializerSettings {
ContractResolver = Utils.Converters.MakeUnderscoreContract()
});
QuandlDatasetData datasetData = response.DatasetData;
datasetData.SetBaseDataset(dataset);
datasetsFetched++;
Utils.ConsoleInformer.PrintProgress("1C", "Fetching dataset [" + dataset.DatasetCode + "]: ", Utils.Helpers.GetPercent(datasetsFetched, to).ToString() + "%");
// Replace old uncomplete dataset with new one
//ReplaceCompleteDataset(datasetData);
//AddCompleteDataset(datasetData);
// Insert
QuandlDatasetDataGroup datasetGroup = new QuandlDatasetDataGroup()
{
DatabaseCode = datasetData.DatabaseCode, Datasets = new List <QuandlDatasetData>()
};
datasetGroup.Datasets.Add(datasetData);
PostgresHelpers.QuandlDatasetActions.InsertQuandlDatasetsDataGroup(datasetGroup);
}
catch (Exception e)
{
// Add to fetch later (Create datasetgroup if doesnt exists in failed list)
if (!failedToFetch.Exists(d => d.DatabaseCode == dataset.DatabaseCode))
{
failedToFetch.Add(new QuandlDatasetGroup()
{
DatabaseCode = dataset.DatabaseCode, Datasets = new List <QuandlDataset>()
});
}
failedToFetch.Find(d => d.DatabaseCode == dataset.DatabaseCode).Datasets.Add(dataset);
if (e.Message.Contains("(429)"))
{
// Print only once
if (!blocked)
{
Utils.ConsoleInformer.Inform("Looks like quandl just blocked you");
}
blocked = true;
}
// Log
Utils.Helpers.Log("Failed to fetch data: from dataset: [" + dataset.DatabaseCode + "/" + dataset.DatasetCode + "] Will try to recover", "Ex: " + e.Message);
//errors.Add(new Tuple<string, string>("Failed to fetch data: from dataset: [" + dataset.DatabaseCode + "/" + dataset.DatasetCode + "]", "Ex: " + e.Message));
}
}
}