public static TempStorageManifest SaveTempStorageManifest(string RootDir, string FinalFilename, List <string> Files)
{
var Saver = new TempStorageManifest();
Saver.Create(Files, RootDir);
if (Saver.GetFileCount() != Files.Count)
{
throw new AutomationException("Saver manifest differs has wrong number of files {0} != {1}", Saver.GetFileCount(), Files.Count);
}
var TempFilename = FinalFilename + ".temp";
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest file already exists {0}", TempFilename);
}
CreateDirectory(true, Path.GetDirectoryName(FinalFilename));
Saver.Save(TempFilename);
var Tester = new TempStorageManifest();
Tester.Load(TempFilename, true);
if (!Saver.Compare(Tester))
{
throw new AutomationException("Temp manifest differs {0}", TempFilename);
}
RenameFile(TempFilename, FinalFilename, true);
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest didn't go away {0}", TempFilename);
}
var FinalTester = new TempStorageManifest();
FinalTester.Load(FinalFilename, true);
if (!Saver.Compare(FinalTester))
{
throw new AutomationException("Final manifest differs {0}", TempFilename);
}
Log("Saved {0} with {1} files and total size {2}", FinalFilename, Saver.GetFileCount(), Saver.GetTotalSize());
return(Saver);
}
public static List<string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, out bool WasLocal, string GameFolder = "", string BaseFolder = "")
{
if (String.IsNullOrEmpty(BaseFolder))
{
BaseFolder = Env.LocalRoot;
}
BaseFolder = CombinePaths(BaseFolder, "/");
if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BaseFolder);
}
var Files = new List<string>();
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
if (FileExists_NoExceptions(LocalManifest))
{
Log("Found local manifest {0}", LocalManifest);
var Local = new TempStorageManifest();
Local.Load(LocalManifest);
Files = Local.GetFiles(BaseFolder);
var LocalTest = new TempStorageManifest();
LocalTest.Create(Files, BaseFolder);
if (!Local.Compare(LocalTest))
{
throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest);
}
WasLocal = true;
return Files;
}
WasLocal = false;
var StartTime = DateTime.UtcNow;
var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), "/");
if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BlockPath);
}
Log("Attempting to retrieve from {0}", BlockPath);
if (!DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath);
}
var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
Robust_FileExists_NoExceptions(SharedManifest, "Storage Block Manifest Does Not Exists! {0}");
var Shared = new TempStorageManifest();
Shared.Load(SharedManifest);
var SharedFiles = Shared.GetFiles(BlockPath);
var DestFiles = new List<string>();
if (ThreadsToCopyWith() < 2)
{
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
CopyFile(Filename, DestFile, true);
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
DestFiles.Add(Info.FullName);
}
}
else
{
var SrcFiles = new List<string>();
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
//Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
SrcFiles.Add(Filename);
DestFiles.Add(DestFile);
}
ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith());
var NewDestFiles = new List<string>();
foreach (string DestFile in DestFiles)
{
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
NewDestFiles.Add(Info.FullName);
}
DestFiles = NewDestFiles;
}
var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles);
if (!NewLocal.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds);
if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0)
{
var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration;
Log("Read from shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration);
}
return DestFiles;
}
public static TempStorageManifest SaveTempStorageManifest(string RootDir, string FinalFilename, List<string> Files)
{
var Saver = new TempStorageManifest();
Saver.Create(Files, RootDir);
if (Saver.GetFileCount() != Files.Count)
{
throw new AutomationException("Saver manifest differs has wrong number of files {0} != {1}", Saver.GetFileCount(), Files.Count);
}
var TempFilename = FinalFilename + ".temp";
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest file already exists {0}", TempFilename);
}
CreateDirectory(true, Path.GetDirectoryName(FinalFilename));
Saver.Save(TempFilename);
var Tester = new TempStorageManifest();
Tester.Load(TempFilename, true);
if (!Saver.Compare(Tester))
{
throw new AutomationException("Temp manifest differs {0}", TempFilename);
}
RenameFile(TempFilename, FinalFilename, true);
if (FileExists_NoExceptions(true, TempFilename))
{
throw new AutomationException("Temp manifest didn't go away {0}", TempFilename);
}
var FinalTester = new TempStorageManifest();
FinalTester.Load(FinalFilename, true);
if (!Saver.Compare(FinalTester))
{
throw new AutomationException("Final manifest differs {0}", TempFilename);
}
Log("Saved {0} with {1} files and total size {2}", FinalFilename, Saver.GetFileCount(), Saver.GetTotalSize());
return Saver;
}
public static List <string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, out bool WasLocal, string GameFolder = "", string BaseFolder = "")
{
if (String.IsNullOrEmpty(BaseFolder))
{
BaseFolder = Env.LocalRoot;
}
BaseFolder = CombinePaths(BaseFolder, "/");
if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BaseFolder);
}
var Files = new List <string>();
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
if (FileExists_NoExceptions(LocalManifest))
{
Log("Found local manifest {0}", LocalManifest);
var Local = new TempStorageManifest();
Local.Load(LocalManifest);
Files = Local.GetFiles(BaseFolder);
var LocalTest = new TempStorageManifest();
LocalTest.Create(Files, BaseFolder);
if (!Local.Compare(LocalTest))
{
throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest);
}
WasLocal = true;
return(Files);
}
WasLocal = false;
var StartTime = DateTime.UtcNow;
var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder), "/");
if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BlockPath);
}
Log("Attempting to retrieve from {0}", BlockPath);
if (!DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath);
}
var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
Robust_FileExists_NoExceptions(SharedManifest, "Storage Block Manifest Does Not Exists! {0}");
var Shared = new TempStorageManifest();
Shared.Load(SharedManifest);
var SharedFiles = Shared.GetFiles(BlockPath);
var DestFiles = new List <string>();
if (ThreadsToCopyWith() < 2)
{
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
CopyFile(Filename, DestFile, true);
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
DestFiles.Add(Info.FullName);
}
}
else
{
var SrcFiles = new List <string>();
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
//Robust_FileExists_NoExceptions(true, Filename, "Could not add {0} to manifest because it does not exist");
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
SrcFiles.Add(Filename);
DestFiles.Add(DestFile);
}
ThreadedCopyFiles(SrcFiles.ToArray(), DestFiles.ToArray(), ThreadsToCopyWith());
var NewDestFiles = new List <string>();
foreach (string DestFile in DestFiles)
{
Robust_FileExists_NoExceptions(true, DestFile, "Could not copy to {0}");
if (UnrealBuildTool.Utils.IsRunningOnMono)
{
FixUnixFilePermissions(DestFile);
}
FileInfo Info = new FileInfo(DestFile);
NewDestFiles.Add(Info.FullName);
}
DestFiles = NewDestFiles;
}
var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles);
if (!NewLocal.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
float BuildDuration = (float)((DateTime.UtcNow - StartTime).TotalSeconds);
if (BuildDuration > 60.0f && Shared.GetTotalSize() > 0)
{
var MBSec = (((float)(Shared.GetTotalSize())) / (1024.0f * 1024.0f)) / BuildDuration;
Log("Read from shared temp storage at {0} MB/s {1}B {2}s", MBSec, Shared.GetTotalSize(), BuildDuration);
}
return(DestFiles);
}
public static List<string> RetrieveFromTempStorage(CommandEnvironment Env, string StorageBlockName, string GameFolder = "", string BaseFolder = "")
{
if (String.IsNullOrEmpty(BaseFolder))
{
BaseFolder = Env.LocalRoot;
}
BaseFolder = CombinePaths(BaseFolder, "/");
if (!BaseFolder.EndsWith("/") && !BaseFolder.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BaseFolder);
}
var Files = new List<string>();
var LocalManifest = LocalTempStorageManifestFilename(Env, StorageBlockName);
if (FileExists_NoExceptions(LocalManifest))
{
Log("Found local manifest {0}", LocalManifest);
var Local = new TempStorageManifest();
Local.Load(LocalManifest);
Files = Local.GetFiles(BaseFolder);
var LocalTest = new TempStorageManifest();
LocalTest.Create(Files, BaseFolder);
if (!Local.Compare(LocalTest))
{
throw new AutomationException("Local files in manifest {0} were tampered with.", LocalManifest);
}
return Files;
}
var BlockPath = CombinePaths(SharedTempStorageDirectory(StorageBlockName, GameFolder, false), "/");
if (!BlockPath.EndsWith("/") && !BlockPath.EndsWith("\\"))
{
throw new AutomationException("base folder {0} should end with a separator", BlockPath);
}
Log("Attempting to retrieve from {0}", BlockPath);
if (!DirectoryExists_NoExceptions(BlockPath))
{
throw new AutomationException("Storage Block Does Not Exists! {0}", BlockPath);
}
var SharedManifest = SharedTempStorageManifestFilename(Env, StorageBlockName, GameFolder);
if (!FileExists_NoExceptions(SharedManifest))
{
throw new AutomationException("Storage Block Manifest Does Not Exists! {0}", SharedManifest);
}
var Shared = new TempStorageManifest();
Shared.Load(SharedManifest);
var SharedFiles = Shared.GetFiles(BlockPath);
var DestFiles = new List<string>();
foreach (string InFilename in SharedFiles)
{
var Filename = CombinePaths(InFilename);
if (!FileExists_NoExceptions(true, Filename))
{
throw new AutomationException("Could not add {0} to manifest because it does not exist", Filename);
}
if (!Filename.StartsWith(BlockPath, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("Could not add {0} to manifest because it does not start with the base folder {1}", Filename, BlockPath);
}
var RelativeFile = Filename.Substring(BlockPath.Length);
var DestFile = CombinePaths(BaseFolder, RelativeFile);
if (FileExists_NoExceptions(true, DestFile))
{
Log("Dest file {0} already exists, deleting and overwriting", DestFile);
DeleteFile(DestFile);
}
CopyFile(Filename, DestFile, true);
if (!FileExists_NoExceptions(true, DestFile))
{
throw new AutomationException("Could not copy {0} to {1}", Filename, DestFile);
}
FileInfo Info = new FileInfo(DestFile);
DestFiles.Add(Info.FullName);
}
var NewLocal = SaveLocalTempStorageManifest(Env, BaseFolder, StorageBlockName, DestFiles);
if (!NewLocal.Compare(Shared))
{
// we will rename this so it can't be used, but leave it around for inspection
RenameFile_NoExceptions(LocalManifest, LocalManifest + ".broken");
throw new AutomationException("Shared and Local manifest mismatch.");
}
return DestFiles;
}
