public static TempStorageManifest SaveTempStorageManifest(string RootDir, string FinalFilename, List <string> Files) { var Saver = new TempStorageManifest(); Saver.Create(Files, RootDir); if (Saver.GetFileCount() != Files.Count) { throw new AutomationException("Saver manifest differs has wrong number of files {0} != {1}", Saver.GetFileCount(), Files.Count); } var TempFilename = FinalFilename + ".temp"; if (FileExists_NoExceptions(true, TempFilename)) { throw new AutomationException("Temp manifest file already exists {0}", TempFilename); } CreateDirectory(true, Path.GetDirectoryName(FinalFilename)); Saver.Save(TempFilename); var Tester = new TempStorageManifest(); Tester.Load(TempFilename, true); if (!Saver.Compare(Tester)) { throw new AutomationException("Temp manifest differs {0}", TempFilename); } RenameFile(TempFilename, FinalFilename, true); if (FileExists_NoExceptions(true, TempFilename)) { throw new AutomationException("Temp manifest didn't go away {0}", TempFilename); } var FinalTester = new TempStorageManifest(); FinalTester.Load(FinalFilename, true); if (!Saver.Compare(FinalTester)) { throw new AutomationException("Final manifest differs {0}", TempFilename); } Log("Saved {0} with {1} files and total size {2}", FinalFilename, Saver.GetFileCount(), Saver.GetTotalSize()); return(Saver); }
public static List<string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, out bool WasLocal, string GameFolder = "", string BaseFolder = "") { if (String.IsNullOrEmpty(BaseFolder)) { BaseFolder = Env.LocalRoot; } BaseFolder = CombinePaths(BaseFolder, "/"); if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\")) { throw new AutomationException("base folder {0} should end with a separator", BaseFolder); } var Files = new List<string>(); var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName); if (FileExists_NoExceptions(LocalManifest)) { Log("Found local manifest {0}", LocalManifest); var Local = new TempStorageManifest(); Local.Load(LocalManifest); Files = Local.GetFiles(BaseFolder); var LocalTest = new TempStorageManifest(); LocalTest.Create(Files, BaseFolder); if (!Local.Compare(LocalTest)) { throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest); } WasLocal = true; return Files; } WasLocal = false; var StartTime = DateTime.UtcNow; var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), "/"); if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\")) { throw new AutomationException("base folder {0} should end with a separator", BlockPath); } Log("Attempting to retrieve from {0}", BlockPath); if (!DirectoryExists_NoExceptions(BlockPath)) { throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath); } var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder); Robust_FileExists_NoExceptions(SharedManifest, "Storage Block Manifest Does Not Exists! {0}"); var Shared = new TempStorageManifest(); Shared.Load(SharedManifest); var SharedFiles = Shared.GetFiles(BlockPath); var DestFiles = new List<string>(); if (ThreadsToCopyWith() < 2) { foreach (string InFilename in SharedFiles) { var Filename = CombinePaths(InFilename); Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist"); if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath); } var RelativeFile = Filename.Substring(BlockPath.Length); var DestFile = CombinePaths(BaseFolder, RelativeFile); if (FileExists_NoExceptions(true, DestFile)) { Log("Dest file {0} already exists, deleting and overwriting", DestFile); DeleteFile(DestFile); } CopyFile(Filename, DestFile, true); Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}"); if (UnrealBuildTool.Utils.IsRunningOnMono) { FixUnixFilePermissions(DestFile); } FileInfo Info = new FileInfo(DestFile); DestFiles.Add(Info.FullName); } } else { var SrcFiles = new List<string>(); foreach (string InFilename in SharedFiles) { var Filename = CombinePaths(InFilename); //Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist"); if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath); } var RelativeFile = Filename.Substring(BlockPath.Length); var DestFile = CombinePaths(BaseFolder, RelativeFile); if (FileExists_NoExceptions(true, DestFile)) { Log("Dest file {0} already exists, deleting and overwriting", DestFile); DeleteFile(DestFile); } SrcFiles.Add(Filename); DestFiles.Add(DestFile); } ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith()); var NewDestFiles = new List<string>(); foreach (string DestFile in DestFiles) { Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}"); if (UnrealBuildTool.Utils.IsRunningOnMono) { FixUnixFilePermissions(DestFile); } FileInfo Info = new FileInfo(DestFile); NewDestFiles.Add(Info.FullName); } DestFiles = NewDestFiles; } var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles); if (!NewLocal.Compare(Shared)) { // we will rename this so it can't be used, but leave it around for inspection RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken"); throw new AutomationException("Shared and Local manifest mismatch."); } float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds); if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0) { var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration; Log("Read from shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration); } return DestFiles; }
public static TempStorageManifest SaveTempStorageManifest(string RootDir, string FinalFilename, List<string> Files) { var Saver = new TempStorageManifest(); Saver.Create(Files, RootDir); if (Saver.GetFileCount() != Files.Count) { throw new AutomationException("Saver manifest differs has wrong number of files {0} != {1}", Saver.GetFileCount(), Files.Count); } var TempFilename = FinalFilename + ".temp"; if (FileExists_NoExceptions(true, TempFilename)) { throw new AutomationException("Temp manifest file already exists {0}", TempFilename); } CreateDirectory(true, Path.GetDirectoryName(FinalFilename)); Saver.Save(TempFilename); var Tester = new TempStorageManifest(); Tester.Load(TempFilename, true); if (!Saver.Compare(Tester)) { throw new AutomationException("Temp manifest differs {0}", TempFilename); } RenameFile(TempFilename, FinalFilename, true); if (FileExists_NoExceptions(true, TempFilename)) { throw new AutomationException("Temp manifest didn't go away {0}", TempFilename); } var FinalTester = new TempStorageManifest(); FinalTester.Load(FinalFilename, true); if (!Saver.Compare(FinalTester)) { throw new AutomationException("Final manifest differs {0}", TempFilename); } Log("Saved {0} with {1} files and total size {2}", FinalFilename, Saver.GetFileCount(), Saver.GetTotalSize()); return Saver; }
public static List <string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, out bool WasLocal, string GameFolder = "", string BaseFolder = "") { if (String.IsNullOrEmpty(BaseFolder)) { BaseFolder = Env.LocalRoot; } BaseFolder = CombinePaths(BaseFolder, "/"); if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\")) { throw new AutomationException("base folder {0} should end with a separator", BaseFolder); } var Files = new List <string>(); var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName); if (FileExists_NoExceptions(LocalManifest)) { Log("Found local manifest {0}", LocalManifest); var Local = new TempStorageManifest(); Local.Load(LocalManifest); Files = Local.GetFiles(BaseFolder); var LocalTest = new TempStorageManifest(); LocalTest.Create(Files, BaseFolder); if (!Local.Compare(LocalTest)) { throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest); } WasLocal = true; return(Files); } WasLocal = false; var StartTime = DateTime.UtcNow; var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), "/"); if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\")) { throw new AutomationException("base folder {0} should end with a separator", BlockPath); } Log("Attempting to retrieve from {0}", BlockPath); if (!DirectoryExists_NoExceptions(BlockPath)) { throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath); } var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder); Robust_FileExists_NoExceptions(SharedManifest, "Storage Block Manifest Does Not Exists! {0}"); var Shared = new TempStorageManifest(); Shared.Load(SharedManifest); var SharedFiles = Shared.GetFiles(BlockPath); var DestFiles = new List <string>(); if (ThreadsToCopyWith() < 2) { foreach (string InFilename in SharedFiles) { var Filename = CombinePaths(InFilename); Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist"); if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath); } var RelativeFile = Filename.Substring(BlockPath.Length); var DestFile = CombinePaths(BaseFolder, RelativeFile); if (FileExists_NoExceptions(true, DestFile)) { Log("Dest file {0} already exists, deleting and overwriting", DestFile); DeleteFile(DestFile); } CopyFile(Filename, DestFile, true); Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}"); if (UnrealBuildTool.Utils.IsRunningOnMono) { FixUnixFilePermissions(DestFile); } FileInfo Info = new FileInfo(DestFile); DestFiles.Add(Info.FullName); } } else { var SrcFiles = new List <string>(); foreach (string InFilename in SharedFiles) { var Filename = CombinePaths(InFilename); //Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist"); if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath); } var RelativeFile = Filename.Substring(BlockPath.Length); var DestFile = CombinePaths(BaseFolder, RelativeFile); if (FileExists_NoExceptions(true, DestFile)) { Log("Dest file {0} already exists, deleting and overwriting", DestFile); DeleteFile(DestFile); } SrcFiles.Add(Filename); DestFiles.Add(DestFile); } ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith()); var NewDestFiles = new List <string>(); foreach (string DestFile in DestFiles) { Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}"); if (UnrealBuildTool.Utils.IsRunningOnMono) { FixUnixFilePermissions(DestFile); } FileInfo Info = new FileInfo(DestFile); NewDestFiles.Add(Info.FullName); } DestFiles = NewDestFiles; } var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles); if (!NewLocal.Compare(Shared)) { // we will rename this so it can't be used, but leave it around for inspection RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken"); throw new AutomationException("Shared and Local manifest mismatch."); } float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds); if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0) { var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration; Log("Read from shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration); } return(DestFiles); }
public static List<string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, string GameFolder = "", string BaseFolder = "") { if (String.IsNullOrEmpty(BaseFolder)) { BaseFolder = Env.LocalRoot; } BaseFolder = CombinePaths(BaseFolder, "/"); if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\")) { throw new AutomationException("base folder {0} should end with a separator", BaseFolder); } var Files = new List<string>(); var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName); if (FileExists_NoExceptions(LocalManifest)) { Log("Found local manifest {0}", LocalManifest); var Local = new TempStorageManifest(); Local.Load(LocalManifest); Files = Local.GetFiles(BaseFolder); var LocalTest = new TempStorageManifest(); LocalTest.Create(Files, BaseFolder); if (!Local.Compare(LocalTest)) { throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest); } return Files; } var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder, false), "/"); if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\")) { throw new AutomationException("base folder {0} should end with a separator", BlockPath); } Log("Attempting to retrieve from {0}", BlockPath); if (!DirectoryExists_NoExceptions(BlockPath)) { throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath); } var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder); if (!FileExists_NoExceptions(SharedManifest)) { throw new AutomationException("Storage Block Manifest Does Not Exists! {0}", SharedManifest); } var Shared = new TempStorageManifest(); Shared.Load(SharedManifest); var SharedFiles = Shared.GetFiles(BlockPath); var DestFiles = new List<string>(); foreach (string InFilename in SharedFiles) { var Filename = CombinePaths(InFilename); if (!FileExists_NoExceptions(true, Filename)) { throw new AutomationException("Could not add {0} to manifest because it does not exist", Filename); } if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase)) { throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath); } var RelativeFile = Filename.Substring(BlockPath.Length); var DestFile = CombinePaths(BaseFolder, RelativeFile); if (FileExists_NoExceptions(true, DestFile)) { Log("Dest file {0} already exists, deleting and overwriting", DestFile); DeleteFile(DestFile); } CopyFile(Filename, DestFile, true); if (!FileExists_NoExceptions(true, DestFile)) { throw new AutomationException("Could not copy {0} to {1}", Filename, DestFile); } FileInfo Info = new FileInfo(DestFile); DestFiles.Add(Info.FullName); } var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles); if (!NewLocal.Compare(Shared)) { // we will rename this so it can't be used, but leave it around for inspection RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken"); throw new AutomationException("Shared and Local manifest mismatch."); } return DestFiles; }