public static void UseUngribToUnpackageGRIBFiles(WrfConfiguration config, IProcessLauncher processLauncher) { string ungribPath = config.UngribFilePath; processLauncher.LaunchProcess(ungribPath, "", false); }
public static List <string> RemoveTempFilesInWRFDirectory(WrfConfiguration config, IFileSystem fileSystem, ILogger iLogger) { List <string> ret = new List <string>(); string[] files = fileSystem.GetFilesInDirectory(config.WRFDirectory); foreach (string file in files) { foreach (string prefix in PrefixesOfFilesToDeleteFromWRFDirectory) { if (PlatformIndependentGetFilename(file).StartsWith(prefix)) { ret.Add(file); fileSystem.DeleteFile(file); //iLogger.LogLine($"\t...deleted file {file}"); } } foreach (string suffix in SufixesOfFilesToDeleteFromWRFDirectory) { if (PlatformIndependentGetFilename(file).EndsWith(suffix)) { ret.Add(file); fileSystem.DeleteFile(file); //iLogger.LogLine($"\t...deleted file {file}"); } } } return(ret); }
public static void UpdatePhysicsParameters(WrfConfiguration config, PhysicsConfigurationProcessed physicsConfig, IFileSystem iFileSystem) { string wrfNamelistPath = config.WRFNamelist; string wrfNamelistContent = iFileSystem.ReadFileContent(wrfNamelistPath); Namelist nameList = NamelistParser.ParseFromString(wrfNamelistContent); PropertyInfo[] physicsProperties = typeof(PhysicsConfigurationProcessed).GetProperties(); foreach (PropertyInfo prop in physicsProperties) { ConfigurationPropertyAttribute configPropertyAttribute = prop.GetCustomAttribute <ConfigurationPropertyAttribute>(); if (configPropertyAttribute != null) { string propertyName = configPropertyAttribute.Name; if (propertyName.ToLower() != "name") { List <object> values = new List <object>(); int value = (int)(float)prop.GetValue(physicsConfig); values.Add(value); nameList["physics"][propertyName].Values = values; } } } string newFileContent = NamelistParser.ParseToString(nameList); iFileSystem.WriteFileContent(wrfNamelistPath, newFileContent); }
public static void UseMetgridToHorizontallyInterpolateData(WrfConfiguration config, IProcessLauncher processLauncher) { string metgridPath = config.MetgridFilePath; processLauncher.LaunchProcess(metgridPath, "", false); }
public static void UseGeogridToProcessTerrestrialData(WrfConfiguration config, IProcessLauncher processLauncher) { string geogrid = config.GeogridFilePath; processLauncher.LaunchProcess(geogrid, "", false); }
public static void MpiRunWrfExecutable(WrfConfiguration config, IProcessLauncher processLauncher) { string mpiRunPath = config.MpiRunPath; string wrfExecutablePath = config.WrfExecutablePath; processLauncher.LaunchProcess(mpiRunPath, $"--allow-run-as-root -np 6 {wrfExecutablePath}", false); }
public static void UpdateDatesInWRFNamelist(WrfConfiguration config, DateTime startDate, DateTime endDate, IFileSystem fileSystem) { string wrfNamelistPath = config.WRFNamelist; string wrfNamelistContent = fileSystem.ReadFileContent(wrfNamelistPath); Namelist nameList = NamelistParser.ParseFromString(wrfNamelistContent); nameList["time_control"]["start_year"].Values.Clear(); nameList["time_control"]["start_month"].Values.Clear(); nameList["time_control"]["start_day"].Values.Clear(); nameList["time_control"]["start_hour"].Values.Clear(); nameList["time_control"]["end_year"].Values.Clear(); nameList["time_control"]["end_month"].Values.Clear(); nameList["time_control"]["end_day"].Values.Clear(); nameList["time_control"]["end_hour"].Values.Clear(); nameList["time_control"]["start_year"].Values.Add(startDate.Year); nameList["time_control"]["start_month"].Values.Add(startDate.Month); nameList["time_control"]["start_day"].Values.Add(startDate.Day); nameList["time_control"]["start_hour"].Values.Add(startDate.Hour); nameList["time_control"]["end_year"].Values.Add(endDate.Year); nameList["time_control"]["end_month"].Values.Add(endDate.Month); nameList["time_control"]["end_day"].Values.Add(endDate.Day); nameList["time_control"]["end_hour"].Values.Add(endDate.Hour); string updatedContent = NamelistParser.ParseToString(nameList); fileSystem.WriteFileContent(wrfNamelistPath, updatedContent); }
static WrfConfiguration LoadConfigurationFromAppSettings(ILogger logger) { WrfConfiguration config = new WrfConfiguration(); Type configType = typeof(WrfConfiguration); string[] keys = ConfigurationManager.AppSettings.AllKeys; foreach (PropertyInfo prop in configType.GetProperties()) { if (keys.Any(n => n == prop.Name)) { string key = prop.Name; string value = ConfigurationManager.AppSettings[key]; logger.LogLine("\t" + key + " : " + value); if (prop.PropertyType == typeof(int)) { prop.SetValue(config, int.Parse(value)); } else if (prop.PropertyType == typeof(bool)) { prop.SetValue(config, bool.Parse(value)); } else { prop.SetValue(config, value); } } } return(config); }
public static void UseWgrib2ToFindStartAndEndDatesOnWGribFiles( WrfConfiguration config, out DateTime startDate, out DateTime endDate, IProcessLauncher processLauncher, IFileSystem fileSystem) { string dataDirectory = config.DataDirectory; string[] files = fileSystem.GetFilesInDirectory(dataDirectory); // go past the period and the 'f'. ex: .f003. string[] orderedFiles = files.OrderByDescending(n => int.Parse(n.Substring(n.LastIndexOf('.') + 2))).ToArray(); string lastFile = orderedFiles[0]; string firstFile = orderedFiles[orderedFiles.Length - 1]; lastFile = Path.Combine(dataDirectory, lastFile); firstFile = Path.Combine(dataDirectory, firstFile); string wgrib2Path = config.WGRIB2FilePath; //352:18979996:start_ft=2016071706 string stdOut = processLauncher.LaunchProcessAndCaptureSTDOUT(wgrib2Path, $"-start_ft {firstFile}"); startDate = GetDateTimeForStdOut(stdOut); stdOut = processLauncher.LaunchProcessAndCaptureSTDOUT(wgrib2Path, $"-start_ft {lastFile}"); endDate = GetDateTimeForStdOut(stdOut); }
public static void CleanDataDirectory(WrfConfiguration config, IFileSystem fileSystem) { if (fileSystem.DirectoryExists(config.DataDirectory)) { fileSystem.DeleteDirectory(config.DataDirectory); } fileSystem.CreateDirectory(config.DataDirectory); }
public static void UseLinkGribToCreateSymbolicLinks(WrfConfiguration config, IProcessLauncher processLauncher) { string csh = config.CSHFilePath; string linkGrib = config.LinkGribCsh; string dataDirectory = config.DataDirectory; processLauncher.LaunchProcess(csh, $"-c \"{linkGrib} {dataDirectory}\"", true); }
static void ComputeStage(IFileSystem iFileSystem, ILogger iLogger, IProcessLauncher iProcess, IEnvironment iEnvironment, IDatabase iDatabase, WrfConfiguration config, PhysicsConfigurationProcessed physicsConfig) { DateTime runStartTime = DateTime.Now; string runId = Guid.NewGuid().ToString().Replace("-", ""); iLogger.LogLine("Updating physics parameters..."); NamelistHelper.UpdatePhysicsParameters(config, physicsConfig, iFileSystem); iLogger.LogLine("...done"); iLogger.LogLine("Changing directory to real directory..."); FileSystemHelper.SetCurrentDirectoryToWRFDirectory(config, iFileSystem); iLogger.LogLine("...done"); iLogger.LogLine("Launching real.exe..."); ProcessHelper.MpiRunRealExecutable(config, iProcess); iLogger.LogLine("...done"); iLogger.LogLine($"Launching wrf.exe against mp_physics={physicsConfig.MpPhysics}"); ProcessHelper.MpiRunWrfExecutable(config, iProcess); iLogger.LogLine("...done"); iLogger.LogLine("Locating the WrfOut file..."); string wrfOutFile = FileSystemHelper.RetrievePathToWrfOutFile(config, iFileSystem); iLogger.LogLine($"...found at {wrfOutFile}."); iLogger.LogLine("Retrieving scripts to run..."); string[] scripts = FileSystemHelper.RetrieveNclScriptsToRun(config, iFileSystem); iLogger.LogLine($"...found {scripts.Length} scripts: {string.Join(",", scripts)}"); //string wrfOutFile = @"C:\Users\Ben\Desktop\wrfout"; DateTime runEndTime = DateTime.Now; INetCDFReader netCdfReader = new NetCDFReader(wrfOutFile); DatabaseHelper.CreateRunRecord(netCdfReader, iDatabase, runStartTime, runEndTime, runId); LatLongRect latLongRect = ConfigurationHelper.ParseLatLongRect(config); DatabaseHelper.RecordVariables(netCdfReader, iDatabase, runId, latLongRect.UpperLeftLong, latLongRect.LowerRightLong, latLongRect.LowerRightLat, latLongRect.UpperLeftLat); foreach (string script in scripts) { iLogger.LogLine($"Launching NCL against {script}..."); ProcessHelper.NclRunScript(config, iProcess, script, wrfOutFile); iLogger.LogLine("...done"); ProcessHelper.MakeVideoWithFFMPEG(config, iProcess, script, runId); } }
public static void NclRunScript(WrfConfiguration config, IProcessLauncher processLauncher, string scriptName, string pathToWrfOutFile) { string scriptPath = Path.Combine(config.ScriptsDirectory, scriptName); string nclExecutablePath = config.NCLPath; string arguments = $"{scriptPath} netcdfFile=\\\"{pathToWrfOutFile}\\\""; Console.WriteLine("arguments = " + arguments); processLauncher.LaunchProcess(nclExecutablePath, arguments, false); }
public static LatLongRect ParseLatLongRect(WrfConfiguration config) { string[] upperLeftLatLon = config.UpperLeftLatLon.Split(','); string[] lowerRightLatLon = config.LowerRightLatLon.Split(','); return(new LatLongRect { UpperLeftLat = float.Parse(upperLeftLatLon[0]), LowerRightLat = float.Parse(lowerRightLatLon[0]), LowerRightLong = float.Parse(lowerRightLatLon[1]), UpperLeftLong = float.Parse(upperLeftLatLon[1]), }); }
public void TestSavingContentInWPSNamelist() { IFileSystem fileSystem = new MockWPSNamelistFileSystem(); WrfConfiguration config = new WrfConfiguration(); config.WRFNamelist = ""; NamelistHelper.UpdateDatesInWPSNamelist(config, new DateTime(1980, 5, 26), new DateTime(1981, 5, 26), fileSystem); string result = fileSystem.ReadFileContent("getresult"); Assert.IsTrue(result.Contains("start_date = '1980-05-26_00:00:00'")); }
public static void CreateMetEmSymlinksInRealDirectory(WrfConfiguration config, IFileSystem fileSystem) { IEnumerable <string> files = fileSystem.GetFilesInDirectory( config.WPSDirectory).Where(m => PlatformIndependentGetFilename(m).StartsWith("met_em")); foreach (string file in files) { // replace windows slashes with linux slashes string link = Path.Combine(config.WRFDirectory, PlatformIndependentGetFilename(file)).Replace("\\", "/"); //Console.WriteLine($"Creating symbolic link from {file} to {link}"); fileSystem.CreateSymLink(file, link); } }
public static string[] RetrieveNclScriptsToRun(WrfConfiguration config, IFileSystem iFileSystem) { string nclScriptsDirectory = config.ScriptsDirectory; List <string> ret = new List <string>(); foreach (string file in iFileSystem.GetFilesInDirectory(nclScriptsDirectory)) { ret.Add(file); } return(ret.ToArray()); }
public void ParseLatLonRect() { WrfConfiguration config = new WrfConfiguration(); config.UpperLeftLatLon = "41.044190,-96.911967"; config.LowerRightLatLon = "40.525328,-96.467021"; LatLongRect rect = ConfigurationHelper.ParseLatLongRect(config); rect.LowerRightLat = 40.525328f; rect.LowerRightLong = -96.467021f; rect.UpperLeftLat = 41.044190f; rect.UpperLeftLong = -96.911967f; }
public void Wgrib2StartEndDates() { WrfConfiguration config = new WrfConfiguration(); config.DataDirectory = "/home/brush/Downloads/wrf/data"; DateTime startDate, endDate; ProcessHelper.UseWgrib2ToFindStartAndEndDatesOnWGribFiles( config, out startDate, out endDate, new MockProcessLauncherForWgrib2(), new MockGFSListingFileSystem()); Assert.AreEqual("7/17/2016 6:00:00 AM", startDate.ToString()); Assert.AreEqual("8/1/2016 6:00:00 PM", endDate.ToString()); }
public static void UpdateDatesInWPSNamelist(WrfConfiguration config, DateTime startDate, DateTime endDate, IFileSystem fileSystem) { string wpsNamelistPath = config.WPSNamelist; string wpsNamelistContent = fileSystem.ReadFileContent(wpsNamelistPath); Namelist nameList = NamelistParser.ParseFromString(wpsNamelistContent); nameList["share"]["start_date"].Values = new List <object>( new string[] { startDate.ToString("yyyy-MM-dd_HH:mm:ss") }); nameList["share"]["end_date"].Values = new List <object>( new string[] { endDate.ToString("yyyy-MM-dd_HH:mm:ss") }); string updatedContent = NamelistParser.ParseToString(nameList); fileSystem.WriteFileContent(wpsNamelistPath, updatedContent); }
public static string RetrievePathToWrfOutFile(WrfConfiguration config, IFileSystem fileSystem) { string wrfDirectory = config.WRFDirectory; Console.WriteLine("Looking at " + wrfDirectory); string supposedWrfOutFile = fileSystem.GetFilesInDirectory(wrfDirectory).Where( m => PlatformIndependentGetFilename(m.ToLower()).StartsWith("wrfout_")).FirstOrDefault(); if (supposedWrfOutFile != null) { return(supposedWrfOutFile); } else { throw new MissingWrfOutFileException(); } }
public void CreateEmSymlinks() { WrfConfiguration config = new WrfConfiguration(); config.WPSDirectory = "/home/brush/Downloads/wrf/WPS"; config.WRFDirectory = "/home/brush/Downloads/wrf/WRFV3/test/em_real"; MockWPSFileSystemWithMetEmFiles fs = new MockWPSFileSystemWithMetEmFiles(); FileSystemHelper.CreateMetEmSymlinksInRealDirectory(config, fs); Assert.AreEqual(129, fs.SymLinksCreated.Count); foreach (string symlink in fs.SymLinksCreated) { Assert.IsTrue(symlink.Contains("met_em")); } return; }
public void DownloadGFSProductsToDirectory() { WrfConfiguration config = new WrfConfiguration { DataDirectory = "/home/brush/Downloads/wrf/data" }; List <string> products = new List <string>(); products.Add("prod1"); products.Add("prod2"); products.Add("prod3"); List <string> localFiles = DownloadHelper.DownloadGFSProductsToDataDirectory( "http://www.ftp.ncep.noaa.gov/data/nccf/com/gfs/prod/gfs.2016072312/", products, config, new MockDownloader(), new MockLogger()); Assert.AreEqual("/home/brush/Downloads/wrf/data/prod1", localFiles[0]); Assert.AreEqual("/home/brush/Downloads/wrf/data/prod2", localFiles[1]); Assert.AreEqual("/home/brush/Downloads/wrf/data/prod3", localFiles[2]); }
public static void MakeVideoWithFFMPEG(WrfConfiguration config, IProcessLauncher iProcess, string script, string physicsModelName) { string scriptFileName = script.Substring(script.LastIndexOf('/') + 1); scriptFileName = scriptFileName.Substring(0, scriptFileName.IndexOf('.')); scriptFileName = scriptFileName.Replace("wrf_", "plt_"); string wrfDirectory = config.WRFDirectory; string mp4Directory = config.MP4OutputDirectory; if (!mp4Directory.EndsWith("/")) { mp4Directory += "/"; } string ffmpegPath = config.FFMPEGPath; iProcess.LaunchProcess(ffmpegPath, $"-y -r 4 -i {scriptFileName}.000%03d.png -c:v libx264 -pix_fmt yuv420p {mp4Directory}{physicsModelName}_{scriptFileName}.mp4", false); }
public void TestSavingContentInWrfNamelist() { IFileSystem fileSystem = new MockWRFNamelistFileSystem(); WrfConfiguration config = new WrfConfiguration(); config.WPSNamelist = ""; NamelistHelper.UpdateDatesInWRFNamelist(config, new DateTime(1980, 5, 26, 1, 0, 0, 0), new DateTime(1981, 6, 27, 2, 0, 0, 0), fileSystem); string result = fileSystem.ReadFileContent("getresult"); Assert.IsTrue(result.Contains("start_year = 1980")); Assert.IsTrue(result.Contains("start_month = 5")); Assert.IsTrue(result.Contains("start_day = 26")); Assert.IsTrue(result.Contains("start_hour = 1")); Assert.IsTrue(result.Contains("end_year = 1981")); Assert.IsTrue(result.Contains("end_month = 6")); Assert.IsTrue(result.Contains("end_day = 27")); Assert.IsTrue(result.Contains("end_hour = 2")); }
public static List <string> DownloadGFSProductsToDataDirectory(string productDirectoryUrl, List <string> products, WrfConfiguration config, INetwork iDownloader, ILogger logger) { List <string> localPaths = new List <string>(); foreach (string product in products) { string productUrl = UrlHelper.Join(productDirectoryUrl, product); string localPath = DownloadHelper.DownloadFile(productUrl, config.DataDirectory, iDownloader); if (logger != null) { logger.LogLine($"\t...downloaded {localPath}"); } localPaths.Add(localPath); } return(localPaths); }
public static void SetCurrentDirectoryToWPSDirectory(WrfConfiguration config, IFileSystem fileSystem) { fileSystem.ChangeCurrentDirectory(config.WPSDirectory); }
static void PrepStage(IFileSystem iFileSystem, INetwork iDownloader, ILogger iLogger, IProcessLauncher iProcess, WrfConfiguration config) { iLogger.Log("Cleaning data directory..."); FileSystemHelper.CleanDataDirectory(config, iFileSystem); iLogger.LogLine("...done"); iLogger.LogLine("Downloading GFS product page..."); string productPageContent = DownloadHelper.DownloadString( config.GFSProductUrl, iDownloader); iLogger.LogLine("...done"); iLogger.Log("Finding GFS products to use..."); string gfsProductDirectory = PageParsingHelper.FindDirectoryNameForLatestGFSEntry( productPageContent); string gfsProductUrl = UrlHelper.Join(config.GFSProductUrl, gfsProductDirectory); string pageContent = DownloadHelper.DownloadString( gfsProductUrl, iDownloader); List <string> productsToDownload = new List <string>(); for (;;) { productsToDownload = PageParsingHelper.FindAllGFSOneDegreePGRB2Files(pageContent); if (productsToDownload.Count != 93 && config.ForceLatestGFSData) { iLogger.Log($"...forcing latest GFS data, not enough yet ({productsToDownload.Count})..."); Thread.Sleep(1000 * 60); pageContent = DownloadHelper.DownloadString( gfsProductUrl, iDownloader); } else if (productsToDownload.Count != 93 && !config.ForceLatestGFSData) { iLogger.Log($"...falling back to previous run, incorrect asset count of...{productsToDownload.Count}"); gfsProductDirectory = PageParsingHelper.FindDirectoryNameForSecondToLastGFSEntry( productPageContent); gfsProductUrl = UrlHelper.Join(config.GFSProductUrl, gfsProductDirectory); pageContent = DownloadHelper.DownloadString( gfsProductUrl, iDownloader); productsToDownload = PageParsingHelper.FindAllGFSOneDegreePGRB2Files(pageContent); break; } else { iLogger.Log($"...the latest is ready!..."); break; } } iLogger.LogLine($"...done. Found {productsToDownload.Count} items at {gfsProductUrl}."); iLogger.LogLine("Downloading the products..."); DownloadHelper.DownloadGFSProductsToDataDirectory(gfsProductUrl, productsToDownload, config, iDownloader, iLogger); iLogger.LogLine("...done"); iLogger.LogLine("Cleaning intermediary files before run..."); FileSystemHelper.RemoveTempFilesInWPSDirectory(config, iFileSystem, iLogger); FileSystemHelper.RemoveTempFilesInWRFDirectory(config, iFileSystem, iLogger); iLogger.LogLine("...done"); iLogger.LogLine("Finding first and last GFS files that were downloaded..."); DateTime startDate, endDate; ProcessHelper.UseWgrib2ToFindStartAndEndDatesOnWGribFiles(config, out startDate, out endDate, iProcess, iFileSystem); iLogger.LogLine($"...done. First grib file is {startDate}, and last is {endDate}"); iLogger.LogLine($"Updating the start/end dates in the {config.WPSNamelist} file."); NamelistHelper.UpdateDatesInWPSNamelist(config, startDate, endDate, iFileSystem); iLogger.LogLine("...done"); iLogger.LogLine("Updating the start/end dates in the WRF namelist.config file."); NamelistHelper.UpdateDatesInWRFNamelist(config, startDate, endDate, iFileSystem); iLogger.LogLine("...done"); iLogger.LogLine("Setting current working directory to WPS directory..."); FileSystemHelper.SetCurrentDirectoryToWPSDirectory(config, iFileSystem); iLogger.LogLine("...done"); iLogger.LogLine("Launching geogrid.exe"); ProcessHelper.UseGeogridToProcessTerrestrialData(config, iProcess); iLogger.LogLine("...done"); iLogger.LogLine("Setting up symlinks through CSH script..."); ProcessHelper.UseLinkGribToCreateSymbolicLinks(config, iProcess); iLogger.LogLine("...done"); iLogger.LogLine("Using ungrib to unpackage GRIB files...."); ProcessHelper.UseUngribToUnpackageGRIBFiles(config, iProcess); iLogger.LogLine("...done"); iLogger.LogLine("Use metrgrid to horizontally interpolate data..."); ProcessHelper.UseMetgridToHorizontallyInterpolateData(config, iProcess); iLogger.LogLine("...done"); iLogger.LogLine("Creating symlinks in Real directory..."); FileSystemHelper.CreateMetEmSymlinksInRealDirectory(config, iFileSystem); iLogger.LogLine("...done"); }
static void Main(string[] args) { try { IFileSystem iFileSystem = new FileSystem(); INetwork iDownloader = new Downloader(); ILogger iLogger = new Logger("/home/brush/wrf/log.txt"); IProcessLauncher iProcess = new ProcessLauncher(); IEnvironment iEnvironment = new WrfSharp.Runner.Implementations.Environment(); ProcessLock lockFile = ProcessLock.TryLock(); if (lockFile != null) { using (lockFile) { string connectionString = ConfigurationManager.ConnectionStrings["Default"].ConnectionString; IDatabase iDatabase = MySQL.OpenConnection(connectionString); iLogger.Log("Testing DB connectivity..."); if (!iDatabase.TestConnection()) { iLogger.LogLine("....Connection failed. Check connection string."); } else { iLogger.LogLine("....Connection succeeded."); } List <PhysicsConfigurationProcessed> physicsConfigs = LoadPhysicsConfigurationsFromConfiguration(); iLogger.LogLine($"Loading configuration..."); WrfConfiguration config = LoadConfigurationFromAppSettings(iLogger); iLogger.LogLine("...done"); string stateText = "Prepping"; _timer = new Timer(delegate(object state) { iDatabase.Checkin(stateText, DateTime.Now); }, null, 0, 1000 * 60); if (args.Length > 0 && args[0].ToLower() == "nodownload") { iLogger.LogLine("Downloading of new data skipped..."); } else { PrepStage(iFileSystem, iDownloader, iLogger, iProcess, config); } Random rand = new Random(); physicsConfigs = physicsConfigs.OrderBy(m => rand.Next()).ToList(); // infinity if -1 if (config.MaximumNumberOfRuns == -1) { config.MaximumNumberOfRuns = int.MaxValue; } stateText = "Computing"; for (int c = 0; c < config.MaximumNumberOfRuns && c < physicsConfigs.Count; c++) { PhysicsConfigurationProcessed physicsConfig = physicsConfigs[c]; ComputeStage(iFileSystem, iLogger, iProcess, iEnvironment, iDatabase, config, physicsConfig); } iDatabase.Checkin("Done", DateTime.Now); } } else { //iLogger.LogLine("WrfSharp appears to be running already, or port 666 is in use."); } } catch (Exception ex) { File.AppendAllText("crash.txt", ex.ToString()); } }