private async Task DownloadData(FintoDownloadSettingsItem download, FintoSettings settings)
{
var version = settings.Endpoints.TryGet(Version);
if (version == null)
{
throw new Exception("Version is not defined in FINTO configuration file.");
}
var versionNumber = download.Version;
if (versionNumber == null)
{
var defaultVersionNumber = settings.Endpoints.TryGet(DefaultVersion);
if (defaultVersionNumber == null)
{
throw new Exception("Default version number is not defined in FINTO configuration file.");
}
versionNumber = int.Parse(defaultVersionNumber);
}
version = string.Format(version, versionNumber);
foreach (var link in download.Links)
{
string action = settings.Endpoints.TryGet(link.Action);
if (string.IsNullOrEmpty(action))
{
Console.Error.WriteLine($"Action {action} not found. Check settings for {download.Name}.");
continue;
}
link.Content = await GetFinto(settings, version, action, link.Param, link.File, download.DownloadFolder, download.SaveToFile);
}
}
internal void Merge(FintoDownloadSettingsItem downloadItem)
{
if (downloadItem.Links == null || downloadItem.Links.Count == 0)
{
return;
}
if (string.IsNullOrEmpty(downloadItem.MergeTo) && downloadItem.SaveToFile)
{
return;
}
if (downloadItem.Links.Count == 1)
{
if (!downloadItem.SaveToFile && downloadItem.MergeTo == null)
{
downloadItem.MergeTo = downloadItem.Links[0].Content;
}
return;
}
Dictionary <string, VmServiceViewsJsonItem> filtered = new Dictionary <string, VmServiceViewsJsonItem>();
List <VmServiceViewsJsonItem> duplicated = new List <VmServiceViewsJsonItem>();
List <VmServiceViewsJsonItem> moreParents = new List <VmServiceViewsJsonItem>();
foreach (var link in downloadItem.Links)
{
try
{
var path = string.IsNullOrEmpty(link.File) ? link.Param : link.File;
var file = Path.Combine(downloadItem.DownloadFolder ?? OutputFolder, $"{path}.json");
var json = downloadItem.SaveToFile ? File.ReadAllText(file) : link.Content;
Console.WriteLine($"loading file {file}.");
var data = new List <VmServiceViewsJsonItem>();
switch (link.Action)
{
case "dataUri":
case "broaderUri":
case "narrowerUri":
data.Add(JsonConvert.DeserializeObject <VmServiceViewsJsonItem>(json));
break;
default:
data = JsonConvert.DeserializeObject <List <VmServiceViewsJsonItem> >(json);
break;
}
foreach (var item in data)
{
if (filtered.ContainsKey(item.Id))
{
var existing = filtered[item.Id];
duplicated.Add(item);
Console.WriteLine($"{path} id {item.Id} already exists.");
if (!item.ExactMatchURIs.All(x => existing.ExactMatchURIs.Contains(x)))
{
throw new Exception($"{item.Id} from {link.Param} has different exact matches");
}
if (!item.NarrowerURIs.All(x => existing.NarrowerURIs.Contains(x)))
{
throw new Exception($"{item.Id} from {link.Param} has different exact matches");
}
if (!item.BroaderURIs.All(x => existing.BroaderURIs.Contains(x)))
{
throw new Exception($"{item.Id} from {link.Param} has different exact matches");
}
continue;
}
if (item.BroaderURIs.Count > 1)
{
moreParents.Add(item);
}
filtered.Add(item.Id, item);
//item.Children = await GetNarrowers(item);
}
}
catch (Exception e)
{
Console.Error.WriteLine($"Couln't parse json for {link.Action}. {e.Message}");
}
}
if (downloadItem.FixHierarchy)
{
Console.WriteLine("Merge hierarchy.");
foreach (var item in filtered.Values.Concat(duplicated))
{
item.NarrowerURIs.Select(x => filtered.TryGet(x)).Where(x => x != null && !x.BroaderURIs.Contains(item.Id)).ForEach(x => x.BroaderURIs.Add(item.Id));
item.BroaderURIs.Select(x => filtered.TryGet(x)).Where(x => x != null && !x.NarrowerURIs.Contains(item.Id)).ForEach(x => x.NarrowerURIs.Add(item.Id));
}
}
string result = JsonConvert.SerializeObject(filtered.Values, Formatting.Indented);
var outputFile = new FileInfo(Path.Combine(downloadItem.MergeFolder ?? OutputFolder, $"{downloadItem.MergeTo}.json"));
if (!outputFile.Directory.Exists)
{
outputFile.Directory.Create();
}
Console.WriteLine($"Merged to {outputFile.FullName}.");
Console.WriteLine($"Duplicated ids {duplicated.Count}.");
Console.WriteLine($"Items with more parents {moreParents.Count}.");
if (downloadItem.SaveToFile)
{
File.WriteAllText(outputFile.FullName, result);
}
else
{
downloadItem.MergeTo = result;
}
// File.WriteAllText(Path.Combine(OutputFolder, "marged.duplicated.json"), JsonConvert.SerializeObject(duplicated));
// File.WriteAllText(Path.Combine(OutputFolder, "broaders.json"), JsonConvert.SerializeObject(moreParents));
}
