public static List <QuandlDatabase> GetImportedDatabases()
{
// Query
string query = @"SELECT id, name, databasecode, description, datasetscount, downloads, premium, image, favorite, import
FROM quandl.databases
WHERE import = true";
List <QuandlDatabase> databases = new List <QuandlDatabase>();
using (var conn = new NpgsqlConnection(Constants.CONNECTION_STRING))
{
using (var cmd = new NpgsqlCommand(query))
{
// Open connection
// ===============================================================
conn.Open();
cmd.Connection = conn;
try
{
// Execute the query and obtain a result set
NpgsqlDataReader dr = cmd.ExecuteReader();
// Each row
while (dr.Read())
{
databases.Add(QuandlDatabase.MakeQuandlDatabase(dr));
}
}
catch (Exception ex)
{
conn.Close();
Helpers.ExitWithError(ex.Message);
}
ConsoleInformer.PrintProgress("0B", "Querying imported databases: ", "100%");
// Close connection
// ===============================================================
conn.Close();
}
}
return(databases);
}
private static DatasetsResponse DownloadDataset(int page, QuandlDatabase database)
{
using (PatientWebClient client = new PatientWebClient())
{
try
{
var json = client.DownloadString("https://www.quandl.com/api/v3/datasets.json?database_code=" + database.DatabaseCode + "&sort_by=id&page=" + page + "&api_key=" + Utils.Constants.API_KEY);
DatasetsResponse response =
JsonConvert.DeserializeObject <DatasetsResponse>(json, new JsonSerializerSettings {
ContractResolver = Utils.Converters.MakeUnderscoreContract()
});
pagesSum++;
Utils.ConsoleInformer.PrintProgress("1B", "Fetching datasets [" + database.DatabaseCode + "]: ", Utils.Helpers.GetPercent(pagesSum, response.Meta.TotalPages).ToString() + "%");
// Add it to its own group
datasetsGroups.Find(d => d.DatabaseCode == database.DatabaseCode).Datasets.AddRange(response.Datasets);
// Insert datasets page directly
QuandlDatasetGroup datasetGroup = new QuandlDatasetGroup()
{
DatabaseCode = database.DatabaseCode, Datasets = response.Datasets
};
PostgresHelpers.QuandlDatasetActions.InsertQuandlDatasetGroup(datasetGroup);
return(response);
}
catch (Exception e)
{
if (e.Message.Contains("(429)") && !blocked)
{
Utils.ConsoleInformer.Inform("Looks like quandl just blocked you");
blocked = true;
}
// Log
Utils.Helpers.Log("Failed to fetch page: " + page + " from Database: [" + database.DatabaseCode + "]", "Ex: " + e.Message);
// Add error to inform and log later
errors.Add(new Tuple <string, string> ("Failed to fetch page: " + page + " from Database: [" + database.DatabaseCode + "]", "Ex: " + e.Message));
return(new DatasetsResponse());
}
}
}
private static async Task DownloadDatasetsAsync(int fromPage, int toPage, QuandlDatabase database)
{
var pages = Enumerable.Range(fromPage, toPage - 1);
await Task.WhenAll(pages.Select(i => DownloadDatasetsAsync(i, database)));
}
