public static void UpdatePhysicsParameters(WrfConfiguration config,
PhysicsConfigurationProcessed physicsConfig, IFileSystem iFileSystem)
{
string wrfNamelistPath = config.WRFNamelist;
string wrfNamelistContent = iFileSystem.ReadFileContent(wrfNamelistPath);
Namelist nameList = NamelistParser.ParseFromString(wrfNamelistContent);
PropertyInfo[] physicsProperties = typeof(PhysicsConfigurationProcessed).GetProperties();
foreach (PropertyInfo prop in physicsProperties)
{
ConfigurationPropertyAttribute configPropertyAttribute =
prop.GetCustomAttribute <ConfigurationPropertyAttribute>();
if (configPropertyAttribute != null)
{
string propertyName = configPropertyAttribute.Name;
if (propertyName.ToLower() != "name")
{
List <object> values = new List <object>();
int value = (int)(float)prop.GetValue(physicsConfig);
values.Add(value);
nameList["physics"][propertyName].Values = values;
}
}
}
string newFileContent = NamelistParser.ParseToString(nameList);
iFileSystem.WriteFileContent(wrfNamelistPath, newFileContent);
}
static void RecurseAdd(List <PhysicsConfigurationProcessed> list, PhysicsConfiguration sourceConfig,
List <PropertyInfo> multipleProps, int level, Type physicsConfigurationType)
{
if (level < multipleProps.Count)
{
PropertyInfo readProp = multipleProps[level];
PropertyInfo writeProp = physicsConfigurationType.GetProperty(readProp.Name);
string[] vals = (readProp.GetValue(sourceConfig) as string).Split(',').Select(n => n.Trim()).ToArray();
int numberSame = list.Count / vals.Length;
for (int valueIndex = 0; valueIndex < vals.Length; valueIndex++)
{
int startIndex = valueIndex * numberSame;
int value = int.Parse(vals[valueIndex]);
List <PhysicsConfigurationProcessed> toSendDown =
new List <PhysicsConfigurationProcessed>(numberSame);
for (int i = startIndex; i < numberSame + startIndex; i++)
{
PhysicsConfigurationProcessed settingNow = list[i];
writeProp.SetValue(settingNow, value);
toSendDown.Add(settingNow);
}
RecurseAdd(toSendDown, sourceConfig, multipleProps,
level + 1, physicsConfigurationType);
}
}
}
static void ComputeStage(IFileSystem iFileSystem, ILogger iLogger,
IProcessLauncher iProcess, IEnvironment iEnvironment, IDatabase iDatabase,
WrfConfiguration config, PhysicsConfigurationProcessed physicsConfig)
{
DateTime runStartTime = DateTime.Now;
string runId = Guid.NewGuid().ToString().Replace("-", "");
iLogger.LogLine("Updating physics parameters...");
NamelistHelper.UpdatePhysicsParameters(config, physicsConfig, iFileSystem);
iLogger.LogLine("...done");
iLogger.LogLine("Changing directory to real directory...");
FileSystemHelper.SetCurrentDirectoryToWRFDirectory(config, iFileSystem);
iLogger.LogLine("...done");
iLogger.LogLine("Launching real.exe...");
ProcessHelper.MpiRunRealExecutable(config, iProcess);
iLogger.LogLine("...done");
iLogger.LogLine($"Launching wrf.exe against mp_physics={physicsConfig.MpPhysics}");
ProcessHelper.MpiRunWrfExecutable(config, iProcess);
iLogger.LogLine("...done");
iLogger.LogLine("Locating the WrfOut file...");
string wrfOutFile =
FileSystemHelper.RetrievePathToWrfOutFile(config, iFileSystem);
iLogger.LogLine($"...found at {wrfOutFile}.");
iLogger.LogLine("Retrieving scripts to run...");
string[] scripts = FileSystemHelper.RetrieveNclScriptsToRun(config, iFileSystem);
iLogger.LogLine($"...found {scripts.Length} scripts: {string.Join(",", scripts)}");
//string wrfOutFile = @"C:\Users\Ben\Desktop\wrfout";
DateTime runEndTime = DateTime.Now;
INetCDFReader netCdfReader = new NetCDFReader(wrfOutFile);
DatabaseHelper.CreateRunRecord(netCdfReader, iDatabase,
runStartTime, runEndTime, runId);
LatLongRect latLongRect = ConfigurationHelper.ParseLatLongRect(config);
DatabaseHelper.RecordVariables(netCdfReader, iDatabase,
runId, latLongRect.UpperLeftLong, latLongRect.LowerRightLong,
latLongRect.LowerRightLat, latLongRect.UpperLeftLat);
foreach (string script in scripts)
{
iLogger.LogLine($"Launching NCL against {script}...");
ProcessHelper.NclRunScript(config, iProcess, script, wrfOutFile);
iLogger.LogLine("...done");
ProcessHelper.MakeVideoWithFFMPEG(config, iProcess, script, runId);
}
}
public static void CreateRunRecord(INetCDFReader iNetCdf, IDatabase iDatabase,
DateTime startDate, DateTime endDate, string runId)
{
DateTime simulationStartDate = NetCDFReaderHelper.GetSimulationDate(iNetCdf);
float westEast = 0, southNorth = 0, bottomTop = 0;
NetCDFReaderHelper.ReadGridDimensions(iNetCdf, out westEast, out southNorth, out bottomTop);
PhysicsConfigurationProcessed physics =
NetCDFReaderHelper.ReadPhysicsSettings(iNetCdf);
iDatabase.SaveRun(startDate, endDate, simulationStartDate,
westEast, southNorth, bottomTop,
physics, runId);
}
public void SaveRun(DateTime startDate, DateTime endDate, DateTime simulationStartDate, float westEastDimension,
float southNorthDimension, float bottomTopDimension,
PhysicsConfigurationProcessed physics, string runId)
{
using (MySqlConnection conn = new MySqlConnection())
{
conn.ConnectionString = _connectionString;
conn.Open();
using (MySqlCommand cmd = conn.CreateCommand())
{
cmd.CommandText = "insert into Runs (RunStartDate, RunEndDate, SimulationStartDate, WestEastDimension, SouthNorthDimension, " +
"BottomTopDimension, Mp_physics, Ra_lw_physics, Ra_sw_physics, Sf_sfclay_physics, Sf_surface_physics, Bl_pbl_physics, Bldt, Cu_physics, " +
"Cudt, Isfflx, Ifsnow, Icloud, Surface_input_source, Num_soil_layers, Sf_urban_physics, RunId) values " +
"(@RunStartDate, @RunEndDate, @SimulationStartDate, @WestEastDimension, @SouthNorthDimension, " +
"@BottomTopDimension, @Mp_physics, @Ra_lw_physics, @Ra_sw_physics, @Sf_sfclay_physics, @Sf_surface_physics, @Bl_pbl_physics, @Bldt, @Cu_physics, " +
"@Cudt, @Isfflx, @Ifsnow, @Icloud, @Surface_input_source, @Num_soil_layers, @Sf_urban_physics, @RunId)";
cmd.CommandTimeout = 60;
cmd.Parameters.AddWithValue("RunStartDate", startDate);
cmd.Parameters.AddWithValue("RunEndDate", endDate);
cmd.Parameters.AddWithValue("SimulationStartDate", simulationStartDate);
cmd.Parameters.AddWithValue("WestEastDimension", westEastDimension);
cmd.Parameters.AddWithValue("SouthNorthDimension", southNorthDimension);
cmd.Parameters.AddWithValue("BottomTopDimension", bottomTopDimension);
cmd.Parameters.AddWithValue("Mp_physics", physics.MpPhysics);
cmd.Parameters.AddWithValue("Ra_lw_physics", physics.RaLwPhysics);
cmd.Parameters.AddWithValue("Ra_sw_physics", physics.RaSwPhysics);
cmd.Parameters.AddWithValue("Sf_sfclay_physics", physics.SfSfClayPhysics);
cmd.Parameters.AddWithValue("Sf_surface_physics", physics.SfSurfacePhysics);
cmd.Parameters.AddWithValue("Bl_pbl_physics", physics.BlPblPhysics);
cmd.Parameters.AddWithValue("Bldt", physics.Bldt);
cmd.Parameters.AddWithValue("Cu_physics", physics.CuPhysics);
cmd.Parameters.AddWithValue("Cudt", physics.Cudt);
cmd.Parameters.AddWithValue("Isfflx", physics.IsFflx);
cmd.Parameters.AddWithValue("Ifsnow", physics.IfSnow);
cmd.Parameters.AddWithValue("Icloud", physics.ICloud);
cmd.Parameters.AddWithValue("Surface_input_source", physics.SurfaceInputSource);
cmd.Parameters.AddWithValue("Num_soil_layers", physics.NumSoilLayers);
cmd.Parameters.AddWithValue("Sf_urban_physics", physics.SfUrbanPhysics);
cmd.Parameters.AddWithValue("RunId", runId);
cmd.ExecuteNonQuery();
}
}
}
public static PhysicsConfigurationProcessed ReadPhysicsSettings(INetCDFReader reader)
{
PhysicsConfigurationProcessed ret = new PhysicsConfigurationProcessed();
ret.Bldt = reader.ReadFloatAttribute("BLDT");
ret.BlPblPhysics = reader.ReadFloatAttribute("BL_PBL_PHYSICS");
ret.Cudt = reader.ReadFloatAttribute("CUDT");
ret.CuPhysics = reader.ReadFloatAttribute("CU_PHYSICS");
ret.ICloud = 0; // ?
ret.IfSnow = 0; // ?
ret.IsFflx = reader.ReadFloatAttribute("ISFFLX");
ret.MpPhysics = reader.ReadFloatAttribute("MP_PHYSICS");
ret.NumSoilLayers = 0; // ?
ret.Radt = reader.ReadFloatAttribute("RADT");
ret.RaLwPhysics = reader.ReadFloatAttribute("RA_LW_PHYSICS");
ret.RaSwPhysics = reader.ReadFloatAttribute("RA_SW_PHYSICS");
ret.SfSfClayPhysics = reader.ReadFloatAttribute("SF_SFCLAY_PHYSICS");
ret.SfSurfacePhysics = reader.ReadFloatAttribute("SF_SURFACE_PHYSICS");
ret.SfUrbanPhysics = reader.ReadFloatAttribute("SF_URBAN_PHYSICS");
ret.SurfaceInputSource = reader.ReadFloatAttribute("SURFACE_INPUT_SOURCE");
return(ret);
}
static List <PhysicsConfigurationProcessed> LoadPhysicsConfigurationsFromConfiguration()
{
Type physicsConfigType = typeof(PhysicsConfiguration);
Type physicsConfigProcessedType = typeof(PhysicsConfigurationProcessed);
Configuration obj = ConfigurationManager.OpenExeConfiguration(ConfigurationUserLevel.None);
PhysicsConfigurationSection section = (PhysicsConfigurationSection)obj.GetSection("physics");
List <PhysicsConfigurationProcessed> ret = new List <PhysicsConfigurationProcessed>();
foreach (PhysicsConfiguration configuration in section.PhysicsConfigurations)
{
int total = 1;
List <PropertyInfo> multipleProps = new List <PropertyInfo>();
List <PropertyInfo> singleProps = new List <PropertyInfo>();
foreach (PropertyInfo prop in physicsConfigType.GetProperties())
{
string val = prop.GetValue(configuration) as string;
if (val != null)
{
int numberOfCommas = (val as string).Count(n => n == ',');
if (numberOfCommas > 0)
{
multipleProps.Add(prop);
}
else
{
singleProps.Add(prop);
}
total *= (numberOfCommas + 1);
}
}
Type physicsConfigurationProcessedType = typeof(PhysicsConfigurationProcessed);
List <PhysicsConfigurationProcessed> toAdd = new List <PhysicsConfigurationProcessed>(total);
for (int c = 0; c < total; c++)
{
PhysicsConfigurationProcessed config = new PhysicsConfigurationProcessed();
foreach (PropertyInfo singleProp in singleProps)
{
if (singleProp.Name.ToLower() != "name")
{
string value = singleProp.GetValue(configuration) as string;
if (value != null)
{
physicsConfigProcessedType.GetProperty(singleProp.Name).SetValue(config, int.Parse(value));
}
}
}
toAdd.Add(config);
}
RecurseAdd(toAdd, configuration, multipleProps, 0, physicsConfigurationProcessedType);
ret.AddRange(toAdd);
}
return(ret);
}
static void Main(string[] args)
{
try
{
IFileSystem iFileSystem = new FileSystem();
INetwork iDownloader = new Downloader();
ILogger iLogger = new Logger("/home/brush/wrf/log.txt");
IProcessLauncher iProcess = new ProcessLauncher();
IEnvironment iEnvironment = new WrfSharp.Runner.Implementations.Environment();
ProcessLock lockFile = ProcessLock.TryLock();
if (lockFile != null)
{
using (lockFile)
{
string connectionString =
ConfigurationManager.ConnectionStrings["Default"].ConnectionString;
IDatabase iDatabase = MySQL.OpenConnection(connectionString);
iLogger.Log("Testing DB connectivity...");
if (!iDatabase.TestConnection())
{
iLogger.LogLine("....Connection failed. Check connection string.");
}
else
{
iLogger.LogLine("....Connection succeeded.");
}
List <PhysicsConfigurationProcessed> physicsConfigs = LoadPhysicsConfigurationsFromConfiguration();
iLogger.LogLine($"Loading configuration...");
WrfConfiguration config = LoadConfigurationFromAppSettings(iLogger);
iLogger.LogLine("...done");
string stateText = "Prepping";
_timer = new Timer(delegate(object state)
{
iDatabase.Checkin(stateText, DateTime.Now);
}, null, 0, 1000 * 60);
if (args.Length > 0 && args[0].ToLower() == "nodownload")
{
iLogger.LogLine("Downloading of new data skipped...");
}
else
{
PrepStage(iFileSystem, iDownloader, iLogger, iProcess, config);
}
Random rand = new Random();
physicsConfigs = physicsConfigs.OrderBy(m => rand.Next()).ToList();
// infinity if -1
if (config.MaximumNumberOfRuns == -1)
{
config.MaximumNumberOfRuns = int.MaxValue;
}
stateText = "Computing";
for (int c = 0; c < config.MaximumNumberOfRuns && c < physicsConfigs.Count; c++)
{
PhysicsConfigurationProcessed physicsConfig = physicsConfigs[c];
ComputeStage(iFileSystem, iLogger, iProcess,
iEnvironment, iDatabase, config, physicsConfig);
}
iDatabase.Checkin("Done", DateTime.Now);
}
}
else
{
//iLogger.LogLine("WrfSharp appears to be running already, or port 666 is in use.");
}
}
catch (Exception ex)
{
File.AppendAllText("crash.txt", ex.ToString());
}
}
