public void Setup(DEMDataSet dataSet, string dataSetLocalDir)
{
try
{
if (dataSet == null)
{
throw new ArgumentNullException(nameof(dataSet), "Dataset is null.");
}
NasaGranuleDataSource dataSource = dataSet.DataSource as NasaGranuleDataSource;
if (dataSource is null)
{
throw new ArgumentException(nameof(dataSet), $"Dataset source is {dataSet.DataSource.DataSourceType}, only {nameof(DEMDataSourceType.NASA)} sources are supported by {nameof(NasaGranuleFileService)} null.");
}
this.logger?.LogInformation($"Setup for {dataSet.Name} dataset.");
if (!Directory.Exists(dataSetLocalDir))
{
Directory.CreateDirectory(dataSetLocalDir);
}
string indexFileName = Path.Combine(dataSetLocalDir, UrlHelper.GetFileNameFromUrl(dataSource.IndexFilePath));
bool download = !File.Exists(indexFileName);
List <NasaDemFile> links = null;
if (download)
{
// Fetch Earth data collection page by page until we have no result left
//
this.logger.LogInformation($"Fetching granules from collection {dataSource.CollectionId} to disk... This will be done once.");
bool hasData = true;
int pageIndex = 0;
int PAGE_SIZE = 2000;
links = new List <NasaDemFile>(30000);
do
{
pageIndex++;
logger.LogInformation($"Getting entries on page {pageIndex} with page size of {PAGE_SIZE} ({(pageIndex - 1) * PAGE_SIZE} entries so far)...");
var url = dataSource.GetUrl(PAGE_SIZE, pageIndex);
var json = _httpClient.GetStringAsync(url).GetAwaiter().GetResult();
hasData = !string.IsNullOrWhiteSpace(json);
if (hasData)
{
var result = NasaCmrGranuleResult.FromJson(json);
hasData = result.Feed.Entry.Any();
if (hasData)
{
// Only retrieve bbox and dem file link (zip file)
links.AddRange(result.Feed.Entry.Select(GetNasaDemFile));
}
}
}while (hasData);
var jsonResult = JsonConvert.SerializeObject(links);
File.WriteAllText(indexFileName, JsonConvert.SerializeObject(links));
logger.LogInformation($"{links.Count} entries written to index file {indexFileName}");
}
// Cache
if (_cacheByDemName == null)
{
_cacheByDemName = new ConcurrentDictionary <string, List <DEMFileSource> >();
}
if (_cacheByDemName.ContainsKey(indexFileName) == false)
{
_cacheByDemName[dataSet.Name] = this.GetSources(dataSet, indexFileName);
}
}
catch (Exception ex)
{
this.logger?.LogError("Unhandled exception: " + ex.Message);
this.logger?.LogInformation(ex.ToString());
throw;
}
}
public static string ToJson(this NasaCmrGranuleResult self) => JsonConvert.SerializeObject(self, DEM.Net.Core.EarthData.Converter.Settings);
