public static List <QuandlDatabase> GetImportedDatabases() { // Query string query = @"SELECT id, name, databasecode, description, datasetscount, downloads, premium, image, favorite, import FROM quandl.databases WHERE import = true"; List <QuandlDatabase> databases = new List <QuandlDatabase>(); using (var conn = new NpgsqlConnection(Constants.CONNECTION_STRING)) { using (var cmd = new NpgsqlCommand(query)) { // Open connection // =============================================================== conn.Open(); cmd.Connection = conn; try { // Execute the query and obtain a result set NpgsqlDataReader dr = cmd.ExecuteReader(); // Each row while (dr.Read()) { databases.Add(QuandlDatabase.MakeQuandlDatabase(dr)); } } catch (Exception ex) { conn.Close(); Helpers.ExitWithError(ex.Message); } ConsoleInformer.PrintProgress("0B", "Querying imported databases: ", "100%"); // Close connection // =============================================================== conn.Close(); } } return(databases); }
private static DatasetsResponse DownloadDataset(int page, QuandlDatabase database) { using (PatientWebClient client = new PatientWebClient()) { try { var json = client.DownloadString("https://www.quandl.com/api/v3/datasets.json?database_code=" + database.DatabaseCode + "&sort_by=id&page=" + page + "&api_key=" + Utils.Constants.API_KEY); DatasetsResponse response = JsonConvert.DeserializeObject <DatasetsResponse>(json, new JsonSerializerSettings { ContractResolver = Utils.Converters.MakeUnderscoreContract() }); pagesSum++; Utils.ConsoleInformer.PrintProgress("1B", "Fetching datasets [" + database.DatabaseCode + "]: ", Utils.Helpers.GetPercent(pagesSum, response.Meta.TotalPages).ToString() + "%"); // Add it to its own group datasetsGroups.Find(d => d.DatabaseCode == database.DatabaseCode).Datasets.AddRange(response.Datasets); // Insert datasets page directly QuandlDatasetGroup datasetGroup = new QuandlDatasetGroup() { DatabaseCode = database.DatabaseCode, Datasets = response.Datasets }; PostgresHelpers.QuandlDatasetActions.InsertQuandlDatasetGroup(datasetGroup); return(response); } catch (Exception e) { if (e.Message.Contains("(429)") && !blocked) { Utils.ConsoleInformer.Inform("Looks like quandl just blocked you"); blocked = true; } // Log Utils.Helpers.Log("Failed to fetch page: " + page + " from Database: [" + database.DatabaseCode + "]", "Ex: " + e.Message); // Add error to inform and log later errors.Add(new Tuple <string, string> ("Failed to fetch page: " + page + " from Database: [" + database.DatabaseCode + "]", "Ex: " + e.Message)); return(new DatasetsResponse()); } } }
private static async Task DownloadDatasetsAsync(int fromPage, int toPage, QuandlDatabase database) { var pages = Enumerable.Range(fromPage, toPage - 1); await Task.WhenAll(pages.Select(i => DownloadDatasetsAsync(i, database))); }