public void Validate_NullVariant_ThrowException()
{
SingleConfig config = GetConfig();
config.variant = null;
Assert.Throws <UserErrorException>(() => config.Validate());
}
private AnnotationResources GetAndCacheAnnotationResources(SingleConfig input, CacheConfiguration cacheConfiguration)
{
if (_cacheConfigurationToAnnotationResources.TryGetValue(cacheConfiguration, out AnnotationResources annotationResources))
{
if (!_recentCacheConfigurations.Last.Value.Equals(cacheConfiguration))
{
_recentCacheConfigurations.Remove(cacheConfiguration);
_recentCacheConfigurations.AddLast(cacheConfiguration);
Logger.WriteLine($"Cached configurations: {string.Join("; ", _recentCacheConfigurations)}");
}
return(annotationResources);
}
if (_recentCacheConfigurations.Count == MaxNumCacheConfigurations)
{
CacheConfiguration configurationToRemove = _recentCacheConfigurations.First.Value;
_recentCacheConfigurations.RemoveFirst();
_cacheConfigurationToAnnotationResources.Remove(configurationToRemove);
GC.Collect();
GC.WaitForPendingFinalizers();
}
Logger.WriteLine($"Creating annotation resources for {cacheConfiguration}");
annotationResources = GetAnnotationResources(input);
_cacheConfigurationToAnnotationResources[cacheConfiguration] = annotationResources;
_recentCacheConfigurations.AddLast(cacheConfiguration);
Logger.WriteLine($"Cached configurations: {string.Join("; ", _recentCacheConfigurations)}");
return(annotationResources);
}
public void Validate_Success()
{
SingleConfig config = GetConfig();
Exception ex = Record.Exception(() => { config.Validate(); });
Assert.Null(ex);
}
public void Validate_NullGenomeAssembly_ThrowException()
{
SingleConfig config = GetConfig();
config.genomeAssembly = null;
Assert.Throws <UserErrorException>(() => config.Validate());
}
private void LoadOrCreate()
{
Version = new SingleConfig(Path.Combine(env.WebRootPath, "SystemFiles", ".build.info"), "Active", "1")["Version"];
using (MySqlConnection connection = new MySqlConnection(Startup.Settings.SqlConnection))
using (MySqlCommand command = new MySqlCommand())
{
try
{
connection.Open();
command.Connection = connection;
// create data table
command.CommandText = CREATE_DATA_TABLE;
command.ExecuteNonQuery();
// load data
command.CommandText = LOAD_DATA;
ReadAll(command.ExecuteReader());
// purge old data
command.CommandText = PURGE_RECORDS;
command.ExecuteNonQuery();
}
catch (MySqlException)
{
Startup.Logger?.LogAndThrow(CASCEdit.Logging.LogType.Critical, "Unable to connect to the database.");
}
}
}
private static void ValidateVepVersion(this SingleConfig config)
{
if (config.vepVersion == 0)
{
config.vepVersion = CacheUtilities.DefaultVepVersion;
}
if (CacheUtilities.IsVepVersionSupported(config.vepVersion))
{
return;
}
throw new UserErrorException($"An invalid VEP version ({config.vepVersion}) was specified. Please choose one of the following versions: {CacheUtilities.GetSupportedVersions()}");
}
private static void ValidateSupplementaryAnnotations(this SingleConfig config)
{
if (string.IsNullOrEmpty(config.supplementaryAnnotations))
{
return;
}
if (SupplementaryAnnotationUtilities.IsValueSupported(config.supplementaryAnnotations))
{
return;
}
throw new UserErrorException($"An invalid supplementary annotation value ({config.supplementaryAnnotations}) was specified. Please choose one of the following values: {SupplementaryAnnotationUtilities.GetSupportedValues()}");
}
public static void Open(CASSettings settings)
{
Settings = settings;
Settings.Format();
Logger.LogInformation("Loading Configs...");
Settings.Cache?.Load();
// load previous build / blizzard build
if (File.Exists(Path.Combine(Settings.OutputPath, Helper.GetCDNPath(".build.info"))))
{
LoadBuildInfo(Settings.OutputPath);
if (BuildInfo["Version"] != Settings.Cache?.Version)
{
Settings.Cache?.Clean();
LoadBuildInfo(Settings.SystemFilesPath);
}
else
{
Settings.SystemFilesPath = Settings.OutputPath; //Update system path to latest output build
}
}
else if (File.Exists(Path.Combine(settings.SystemFilesPath, ".build.info")))
{
LoadBuildInfo(Settings.SystemFilesPath);
}
else
{
LoadBuildInfo(BasePath);
}
// load hosted configs if not basic
if (!Settings.Basic)
{
Versions = new SingleConfig(Settings.PatchUrl + "/versions", "Region", BuildInfo["Branch"]);
CDNs = new SingleConfig(Settings.PatchUrl + "/cdns", "Name", BuildInfo["Branch"]);
// download urls
var cdns = CDNs["Hosts"].Split(' ').Select(x => $"http://{x}/{CDNs["Path"]}").ToList();
Settings.DownloadLocations = new HashSet <string>(cdns);
settings.DownloadLocations.Add($"https://bnet.marlam.in/{CDNs["Path"]}"); //Thanks to Marlamin for this access
// cdns urls
foreach (var c in CDNs["Hosts"].Split(' '))
{
settings.CDNs.Add(c);
}
}
LoadBuildConfig();
LoadCDNConfig();
}
private static void LoadBuildInfo(string path)
{
string buildInfoPath = Path.Combine(path, ".build.info");
if (!File.Exists(buildInfoPath))
{
Logger.LogCritical("Missing Build Info", buildInfoPath);
return;
}
BuildInfo = new SingleConfig(buildInfoPath, "Active", "1");
}
private static void LoadBuildInfo(string path)
{
string buildInfoPath = Path.Combine(path, ".build.info");
if (!File.Exists(buildInfoPath))
{
Logger.LogCritical("Missing Build Info", buildInfoPath);
return;
}
Logger.LogInformation($"Loading build info for {Settings.Product}");
BuildInfo = new SingleConfig(buildInfoPath, "Active", "1", Settings.Product);
}
// ReSharper disable once UnusedMember.Global
public Stream Run(SingleConfig config, ILambdaContext context)
{
string snsTopicArn = null;
Stream response;
try
{
LogUtilities.UpdateLogger(context.Logger, null);
LogUtilities.LogLambdaInfo(context, CommandLineUtilities.InformationalVersion);
LogUtilities.LogObject("Config", config);
LogUtilities.Log(new[] { LambdaUrlHelper.UrlBaseEnvironmentVariableName, LambdaUtilities.SnsTopicKey });
LambdaUtilities.GarbageCollect();
snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey);
config.Validate();
GenomeAssembly genomeAssembly = GenomeAssemblyHelper.Convert(config.genomeAssembly);
var cacheConfiguration = new CacheConfiguration(genomeAssembly, config.supplementaryAnnotations, config.vepVersion);
bool preloadRequired = !string.IsNullOrEmpty(config.supplementaryAnnotations);
AnnotationResources annotationResources = GetAndCacheAnnotationResources(config, cacheConfiguration);
if (genomeAssembly != GenomeAssembly.hg19)
{
annotationResources.Annotator.EnableMitochondrialAnnotation();
}
(IPosition position, string[] sampleNames) = config.GetPositionAndSampleNames(annotationResources.SequenceProvider, annotationResources.RefMinorProvider);
if (position.Chromosome.IsEmpty())
{
throw new UserErrorException($"An unknown chromosome was specified ({config.variant.chromosome})");
}
string annotationResult = GetPositionAnnotation(position, annotationResources, sampleNames, preloadRequired);
response = SingleResult.Create(config.id, LambdaUrlHelper.SuccessMessage, annotationResult);
}
catch (Exception exception)
{
response = ExceptionHandler.GetStream(config.id, snsTopicArn, exception);
}
return(response);
}
public static void Validate(this SingleConfig config)
{
if (string.IsNullOrEmpty(config.id))
{
throw new UserErrorException("Please specify the id.");
}
if (string.IsNullOrEmpty(config.genomeAssembly))
{
throw new UserErrorException("Please specify the genome assembly.");
}
if (config.variant == null)
{
throw new UserErrorException("Please specify the variant (chromosome, position, reference allele, and alt alleles).");
}
config.ValidateSupplementaryAnnotations();
config.ValidateVepVersion();
config.variant?.Validate();
}
private static AnnotationResources GetAnnotationResources(SingleConfig lambdaConfig)
{
GenomeAssembly genomeAssembly = GenomeAssemblyHelper.Convert(lambdaConfig.genomeAssembly);
string cachePathPrefix = CacheUtilities.GetCachePathPrefix(lambdaConfig.vepVersion, genomeAssembly);
string nirvanaS3Ref = LambdaUrlHelper.GetRefUrl(genomeAssembly);
string annotatorVersion = "Nirvana " + CommandLineUtilities.GetVersion(Assembly.GetAssembly(typeof(SingleAnnotationLambda)));
var metrics = new PerformanceMetrics();
Logger.WriteLine($"Cache prefix: {cachePathPrefix}");
//todo: get customStrTsv from lambdaConfig
var annotationResources = new AnnotationResources(nirvanaS3Ref, cachePathPrefix,
null, lambdaConfig.customAnnotations, null, false, false, false, metrics)
{
AnnotatorVersionTag = annotatorVersion
};
return(annotationResources);
}
private bool IsRebuildRequired()
{
Startup.Logger.LogInformation("Offline file change check.");
string BuildInfoVersion = new SingleConfig(Path.Combine(buildInfoPath, ".build.info"), "Active", "1", Startup.Settings.Product)["Version"];
if (BuildInfoVersion != Startup.Settings.GameVersion)
{
Startup.Logger.LogInformation($"New version detected: {BuildInfoVersion}. Rebuilding...");
return(true);
}
else
{
Startup.Logger.LogInformation($"Same version, nothing to do here.");
}
//No data files
if (!Directory.EnumerateFiles(dataPath, "*.*", SearchOption.AllDirectories).Any())
{
return(false);
}
string[] hashes = new[]
{
GetDirectoryHash(dataPath),
GetDirectoryHash(outputPath)
};
//Check for offline changes
if (!hashes.SequenceEqual(Startup.Settings.DirectoryHash) /*|| Startup.Settings.RebuildOnLoad*/)
{
return(true);
}
return(false);
}
private void LoadOrCreate()
{
Version = new SingleConfig(Path.Combine(env.WebRootPath, "SystemFiles", ".build.info"), "Active", "1", Startup.Settings.Product)["Version"];
if (!File.Exists(dbFileName))
{
SQLiteConnection.CreateFile(dbFileName);
}
using (SQLiteConnection connection = new SQLiteConnection("Data Source=" + dbFileName + ";Version=3;"))
using (SQLiteCommand command = new SQLiteCommand())
{
try
{
connection.Open();
command.Connection = connection;
// create data table
command.CommandText = CREATE_DATA_TABLE;
command.ExecuteNonQuery();
// load data
command.CommandText = LOAD_DATA;
ReadAll(command.ExecuteReader());
// purge old data
command.CommandText = PURGE_RECORDS;
command.ExecuteNonQuery();
}
catch (SQLiteException ex)
{
Startup.Logger?.LogFile(ex.Message);
Startup.Logger?.LogAndThrow(CASCEdit.Logging.LogType.Critical, "Something wrong with the database.");
}
}
}
private void UpdateCASCDirectory(object obj)
{
if (RebuildInProgress) //Saving already wait for build to finish
{
timer.Change(30 * 1000, Timeout.Infinite); //30 second delay
return;
}
RebuildInProgress = true;
timer.Change(Timeout.Infinite, Timeout.Infinite);
Startup.Logger.LogWarning($"CASC rebuild started [{DateTime.Now}] - {changes.Count} files to be amended.");
Stopwatch sw = Stopwatch.StartNew();
//Open the CASC Container
CASContainer.Open(settings);
CASContainer.OpenCdnIndices(false);
CASContainer.OpenEncoding();
CASContainer.OpenRoot(settings.Locale, Startup.Settings.MinimumFileDataId, Startup.Settings.OnlineListFile);
if (Startup.Settings.BNetAppSupport) // these are only needed by the bnet app launcher
{
CASContainer.OpenDownload();
CASContainer.OpenInstall();
CASContainer.DownloadInstallAssets();
}
//Remove Purged files
foreach (var purge in Startup.Cache.ToPurge)
{
CASContainer.RootHandler.RemoveFile(purge);
}
//Apply file changes
while (changes.Count > 0)
{
var key = changes.Keys.First();
if (changes.TryRemove(key, out FileSystemEventArgs change))
{
string fullpath = change.FullPath;
string cascpath = GetCASCPath(fullpath);
string oldpath = GetCASCPath((change as RenamedEventArgs)?.OldFullPath + "");
switch (change.ChangeType)
{
case WatcherChangeTypes.Renamed:
if (CASContainer.RootHandler.GetEntry(oldpath) == null)
{
CASContainer.RootHandler.AddFile(fullpath, cascpath);
}
else
{
CASContainer.RootHandler.RenameFile(oldpath, cascpath);
}
break;
case WatcherChangeTypes.Deleted:
CASContainer.RootHandler.RemoveFile(cascpath);
break;
default:
CASContainer.RootHandler.AddFile(fullpath, cascpath);
break;
}
}
}
//Save and Clean
CASContainer.Save();
//Update directory hashes
Startup.Settings.DirectoryHash = new[]
{
GetDirectoryHash(dataPath),
GetDirectoryHash(outputPath)
};
string GameVersion = new SingleConfig(Path.Combine(buildInfoPath, ".build.info"), "Active", "1", Startup.Settings.Product)["Version"];
Startup.Settings.GameVersion = GameVersion;
Startup.Settings.Save(_env);
sw.Stop();
Startup.Logger.LogWarning($"CASC rebuild finished [{DateTime.Now}] - {Math.Round(sw.Elapsed.TotalSeconds, 3)}s");
if (settings.StaticMode)
{
Environment.Exit(0);
}
RebuildInProgress = false;
}
public static (IPosition, string[]) GetPositionAndSampleNames(this SingleConfig config, ISequenceProvider sequenceProvider,
IRefMinorProvider refMinorProvider) => (ToPosition(config.variant.GetVcfFields(), sequenceProvider, refMinorProvider), config.variant.sampleNames);
