/// <summary>
/// Downloads and extracts any files from the provided Live Data Sources
/// </summary>
/// <param name="sourceConfigurations">The list of sources</param>
/// <param name="force">Forces installing any download versions regardless of what version exists in the cache</param>
/// <returns></returns>
public void DownloadSourcesAsync(List <DnaConfigurationLiveDataSource> sourceConfigurations, bool force = false)
{
// Flag if we end up downloading anything
var somethingDownloaded = false;
// Log it
CoreLogger.LogInformation("Updating Live Data Sources...");
if (sourceConfigurations != null)
{
// Keep track of sources we add in this loop
var addedConfigurations = new List <LiveDataSource>();
// Loop each source provided...
foreach (var sourceConfiguration in sourceConfigurations)
{
CoreLogger.Log($"LiveData: Processing source {sourceConfiguration.ConfigurationFileSource}...");
var liveDataSource = new LiveDataSource();
#region Get Source Configuration
// Is source a web link?
var isWeb = sourceConfiguration.ConfigurationFileSource.ToLower().StartsWith("http");
// Is source a local configuration file
var isLocal = sourceConfiguration.ConfigurationFileSource.ToLower().EndsWith(DnaSettings.LiveDataConfigurationFileName.ToLower());
// Is source folder? (used directly, not downloaded to cache folder)
// Detected by being a folder link that inside that folder exists a dna.live.config file
var isDirectSource = !isWeb && !isLocal && File.Exists(Path.Combine(sourceConfiguration.ConfigurationFileSource, DnaSettings.LiveDataConfigurationFileName));
// If this is a web source...
if (isWeb)
{
#region Download Configuration File
// Download its information
var informationString = WebHelpers.DownloadString(sourceConfiguration.ConfigurationFileSource);
// If it is null, it failed
if (string.IsNullOrEmpty(informationString))
{
// Log it
CoreLogger.Log($"LiveData: Failed to download configuration {sourceConfiguration.ConfigurationFileSource}", type: LogType.Warning);
// Stop
continue;
}
#endregion
#region Deserialize
// Try to deserialize the Json
try
{
liveDataSource = JsonConvert.DeserializeObject <LiveDataSource>(informationString);
}
catch (Exception ex)
{
// If we failed, log it
CoreLogger.Log($"LiveData: Failed to deserialize configuration {sourceConfiguration.ConfigurationFileSource}. {ex.Message}", type: LogType.Warning);
// Stop
continue;
}
#endregion
}
// If it ends with dna.live.config and the local file exists
else if (isLocal)
{
if (!File.Exists(sourceConfiguration.ConfigurationFileSource))
{
// Log it
CoreLogger.Log($"LiveData: Local configuration file not found {sourceConfiguration.ConfigurationFileSource}", type: LogType.Warning);
// Stop
continue;
}
#region Read Configuration File
// Read its information
var informationString = File.ReadAllText(sourceConfiguration.ConfigurationFileSource);
// If it is null, it failed
if (string.IsNullOrEmpty(informationString))
{
// Log it
CoreLogger.Log($"LiveData: Failed to read local configuration {sourceConfiguration.ConfigurationFileSource}", type: LogType.Warning);
// Stop
continue;
}
#endregion
#region Deserialize
// Try to deserialize the Json
try
{
liveDataSource = JsonConvert.DeserializeObject <LiveDataSource>(informationString);
}
catch (Exception ex)
{
// If we failed, log it
CoreLogger.Log($"LiveData: Failed to deserialize configuration {sourceConfiguration.ConfigurationFileSource}. {ex.Message}", type: LogType.Warning);
// Stop
continue;
}
#endregion
}
// Otherwise...
else
{
// If it is a folder that exists and contains the dna.live.config file
// specifying it this way means it should be treated as a direct access
// local file (not copied to the cache folder)
//
// So, ignore it for this step either way but if it doesn't contain
// a configuration file, warn it is an unknown source
if (!isDirectSource)
{
// Log it
CoreLogger.Log($"LiveData: Unknown source type {sourceConfiguration.ConfigurationFileSource}", type: LogType.Warning);
}
else
{
// Log it
CoreLogger.Log($"LiveData: Skipping local source folder (will be used directly not copied to cache) {sourceConfiguration.ConfigurationFileSource}");
}
// Stop either way
continue;
}
#endregion
#region Newer Version Check
// If we are forcing an update ignore this step
if (!force)
{
// Check if we have a newer version...
var newerVersion = Sources.FirstOrDefault(localSource =>
// Has the same name...
localSource.Name.EqualsIgnoreCase(liveDataSource.Name) &&
// And a higher version
localSource.Version >= liveDataSource.Version);
if (newerVersion != null)
{
// Log it
CoreLogger.Log($"LiveData: Skipping download as same or newer version exists {newerVersion.Name} ({newerVersion.CachedFilePath})");
// Stop
continue;
}
}
#endregion
#region Delete Old Versions
// Find any older version and delete it
var olderVersions = Sources.Where(localSource =>
// Has the same name...
localSource.Name.EqualsIgnoreCase(liveDataSource.Name) &&
// And a lower version (or we are forcing an update)
(force || localSource.Version < liveDataSource.Version)).ToList();
// If we got any lower versions...
if (olderVersions?.Count > 0)
{
// Loop each older version...
foreach (var olderVersion in olderVersions)
{
try
{
// Try and delete the folder
Directory.Delete(olderVersion.CachedFilePath, true);
}
catch (Exception ex)
{
// Log it
CoreLogger.Log($"LiveData: Failed to delete older version {olderVersion.CachedFilePath}. {ex.Message}", type: LogType.Warning);
// Stop
continue;
}
}
}
#endregion
#region Download Source
var zipFile = isWeb ?
// If Web: New unique filename to download to
FileHelpers.GetUnusedPath(Path.Combine(CachePath, $"{liveDataSource.Name}.zip")) :
// Otherwise source should point to zip file relative to current path
DnaConfiguration.ResolveFullPath(Path.GetDirectoryName(sourceConfiguration.ConfigurationFileSource), liveDataSource.Source, true, out bool wasRelative);
if (isWeb)
{
// If Url is relative...
if (!liveDataSource.Source.Contains("://"))
{
// Get URL folder
var urlFolder = sourceConfiguration.ConfigurationFileSource.Substring(0, sourceConfiguration.ConfigurationFileSource.LastIndexOf('/'));
// Prepend the current sources path
liveDataSource.Source = $"{urlFolder}/{liveDataSource.Source}";
}
// Now attempt to download the source zip file
CoreLogger.Log($"LiveData: Downloading source contents... {liveDataSource.Source}");
// Download to folder
var downloaded = WebHelpers.DownloadFile(liveDataSource.Source, zipFile);
// If it failed to download...
if (!downloaded)
{
// Log it
CoreLogger.Log($"LiveData: Failed to download source file {liveDataSource.Source}", type: LogType.Warning);
// Stop
continue;
}
}
else
{
// Make sure zip exists
if (!File.Exists(zipFile))
{
// Log it
CoreLogger.Log($"LiveData: Local source zip file does not exist {zipFile}", type: LogType.Warning);
// Stop
continue;
}
}
// Get unused folder to extract to
var saveFolder = FileHelpers.GetUnusedPath(Path.Combine(CachePath, liveDataSource.Name));
#endregion
// Flag if we succeeded so the local sources get refreshed after we are done
somethingDownloaded = true;
// Whatever happens now, fail or succeed, we should clean up the downloaded zip
try
{
#region Extract Source
// Try and extract the zip
var unzipSuccessful = ZipHelpers.Unzip(zipFile, saveFolder);
if (!unzipSuccessful)
{
// Log it
CoreLogger.Log($"LiveData: Failed to unzip downloaded file {zipFile}", type: LogType.Warning);
// Clean up folder
try
{
// If save folder exists...
if (Directory.Exists(saveFolder))
{
// Delete it
Directory.Delete(saveFolder, true);
}
}
catch (Exception ex)
{
// Log it
CoreLogger.Log($"LiveData: Failed to delete failed extraction folder {saveFolder}. {ex.Message}", type: LogType.Warning);
}
// Stop
continue;
}
#endregion
#region Verify Valid Configuration
// Verify the zip has valid dna.live.config file in and it successfully parses
// Get expected configuration path
var configFilePath = Path.Combine(saveFolder, DnaSettings.LiveDataConfigurationFileName);
// Flag if it is a valid source
var validSource = true;
// If the file does not exist or it fails to parse
if (!File.Exists(configFilePath))
{
// Log it
CoreLogger.Log($"LiveData: Live Data configuration file missing {configFilePath}.", type: LogType.Warning);
// Flag it
validSource = false;
}
else
{
// Try and parse the file
try
{
// Try and parse
var result = JsonConvert.DeserializeObject <LiveDataSource>(File.ReadAllText(configFilePath));
#region Already Added Check
// Make sure we don't already have this name
if (addedConfigurations.Any(source => source.Name.EqualsIgnoreCase(result.Name)))
{
// Log it
CoreLogger.Log($"LiveData: Ignoring source as another exists with same name {result.Name}. {result.CachedFilePath}", type: LogType.Warning);
// Flag it
validSource = false;
}
#endregion
// If it is a valid source...
if (validSource)
{
// Add to already added list
addedConfigurations.Add(result);
// Log successful install
CoreLogger.Log($"Installed new Live Data Source {result.Name} v{result.Version}, from {sourceConfiguration.ConfigurationFileSource}", type: LogType.Success);
}
}
catch (Exception ex)
{
// Log it
CoreLogger.Log($"LiveData: Failed to parse Live Data configuration file {configFilePath}. {ex.Message}", type: LogType.Error);
// Flag it
validSource = false;
}
}
// If it is not a valid file...
if (!validSource)
{
// Log it
CoreLogger.Log($"LiveData: Cleaning invalid source folder {saveFolder}.", type: LogType.Warning);
// Delete source folder
DeleteSource(saveFolder);
}
#endregion
}
finally
{
// If it was a downloaded file...
if (isWeb)
{
// Log it
CoreLogger.Log($"LiveData: Cleaning up downloaded file {zipFile}");
try
{
// Try and delete it
File.Delete(zipFile);
}
catch (Exception ex)
{
// Log it
CoreLogger.Log($"LiveData: Failed to delete downloaded file {zipFile}. {ex.Message}", type: LogType.Error);
}
}
}
}
}
// Rescan if we downloaded anything
if (somethingDownloaded)
{
// Refresh local sources
RefreshLocalSources(sourceConfigurations);
}
CoreLogger.Log($"LiveData: Finished downloading sources");
}
