public void Setup(DEMDataSet dataSet, string dataSetLocalDir) { try { if (dataSet == null) { throw new ArgumentNullException(nameof(dataSet), "Dataset is null."); } NasaGranuleDataSource dataSource = dataSet.DataSource as NasaGranuleDataSource; if (dataSource is null) { throw new ArgumentException(nameof(dataSet), $"Dataset source is {dataSet.DataSource.DataSourceType}, only {nameof(DEMDataSourceType.NASA)} sources are supported by {nameof(NasaGranuleFileService)} null."); } this.logger?.LogInformation($"Setup for {dataSet.Name} dataset."); if (!Directory.Exists(dataSetLocalDir)) { Directory.CreateDirectory(dataSetLocalDir); } string indexFileName = Path.Combine(dataSetLocalDir, UrlHelper.GetFileNameFromUrl(dataSource.IndexFilePath)); bool download = !File.Exists(indexFileName); List <NasaDemFile> links = null; if (download) { // Fetch Earth data collection page by page until we have no result left // this.logger.LogInformation($"Fetching granules from collection {dataSource.CollectionId} to disk... This will be done once."); bool hasData = true; int pageIndex = 0; int PAGE_SIZE = 2000; links = new List <NasaDemFile>(30000); do { pageIndex++; logger.LogInformation($"Getting entries on page {pageIndex} with page size of {PAGE_SIZE} ({(pageIndex - 1) * PAGE_SIZE} entries so far)..."); var url = dataSource.GetUrl(PAGE_SIZE, pageIndex); var json = _httpClient.GetStringAsync(url).GetAwaiter().GetResult(); hasData = !string.IsNullOrWhiteSpace(json); if (hasData) { var result = NasaCmrGranuleResult.FromJson(json); hasData = result.Feed.Entry.Any(); if (hasData) { // Only retrieve bbox and dem file link (zip file) links.AddRange(result.Feed.Entry.Select(GetNasaDemFile)); } } }while (hasData); var jsonResult = JsonConvert.SerializeObject(links); File.WriteAllText(indexFileName, JsonConvert.SerializeObject(links)); logger.LogInformation($"{links.Count} entries written to index file {indexFileName}"); } // Cache if (_cacheByDemName == null) { _cacheByDemName = new ConcurrentDictionary <string, List <DEMFileSource> >(); } if (_cacheByDemName.ContainsKey(indexFileName) == false) { _cacheByDemName[dataSet.Name] = this.GetSources(dataSet, indexFileName); } } catch (Exception ex) { this.logger?.LogError("Unhandled exception: " + ex.Message); this.logger?.LogInformation(ex.ToString()); throw; } }
public static string ToJson(this NasaCmrGranuleResult self) => JsonConvert.SerializeObject(self, DEM.Net.Core.EarthData.Converter.Settings);