/// <summary>
/// Completes an interrupted transaction to write the state file
/// </summary>
/// <param name="StateFile">Path to the state file</param>
static void CompleteStateTransaction(FileReference StateFile)
{
if (!FileReference.Exists(StateFile))
{
FileReference StateTransactionFile = GetStateTransactionFile(StateFile);
if (FileReference.Exists(StateTransactionFile))
{
Log.TraceInformation("Completing partial transaction through {0}", StateTransactionFile);
FileReference.Move(StateTransactionFile, StateFile);
}
}
}
/// <summary>
/// Writes the state to disk in a way that can be recovered if the operation is interrupted
/// </summary>
/// <param name="StateFile">The file to write to</param>
/// <param name="State">The state object</param>
static void WriteState(FileReference StateFile, BuildHealthState State)
{
Stopwatch Timer = Stopwatch.StartNew();
// Write out the state to the transaction file
FileReference StateTransactionFile = GetStateTransactionFile(StateFile);
SerializeJson(StateTransactionFile, State);
// Remove the original file, then move the transaction file into place
FileReference.Delete(StateFile);
FileReference.Move(StateTransactionFile, StateFile);
Log.TraceInformation("Took {0}s to write state", Timer.Elapsed.TotalSeconds);
}
public void Dispose()
{
// Try to write the output file in a transactional way; write it to a temporary file and rename it.
DirectoryReference.CreateDirectory(OutputFile.Directory);
InputData Data = new InputData();
Data.Jobs.Add(Job);
FileReference TempOutputFile = new FileReference(OutputFile.FullName + ".incoming");
using (MemoryStream Stream = new MemoryStream())
{
DataContractJsonSerializer InputFileDataSerializer = new DataContractJsonSerializer(typeof(InputData));
InputFileDataSerializer.WriteObject(Stream, Data);
FileReference.WriteAllBytes(TempOutputFile, Stream.ToArray());
}
FileReference.Delete(OutputFile);
FileReference.Move(TempOutputFile, OutputFile);
}
static void BackupLogFile(FileReference LogFile, TimeSpan RetentionSpan)
{
string LogPrefix = LogFile.GetFileNameWithoutExtension() + "-backup-";
string LogSuffix = LogFile.GetExtension();
if (FileReference.Exists(LogFile))
{
string Timestamp = FileReference.GetLastWriteTime(LogFile).ToString("yyyy.MM.dd-HH.mm.ss");
string UniqueSuffix = "";
for (int NextSuffixIdx = 2; ; NextSuffixIdx++)
{
FileReference BackupLogFile = FileReference.Combine(LogFile.Directory, String.Format("{0}{1}{2}{3}", LogPrefix, Timestamp, UniqueSuffix, LogSuffix));
if (!FileReference.Exists(BackupLogFile))
{
try
{
FileReference.Move(LogFile, BackupLogFile);
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to backup {0} to {1} ({2})", LogFile, BackupLogFile, Ex.Message);
Log.TraceLog(ExceptionUtils.FormatException(Ex));
}
break;
}
UniqueSuffix = String.Format("_{0}", NextSuffixIdx);
}
}
DateTime DeleteTime = DateTime.UtcNow - RetentionSpan;
foreach (FileInfo File in new DirectoryInfo(LogFile.Directory.FullName).EnumerateFiles(LogPrefix + "*" + LogSuffix))
{
if (File.LastWriteTimeUtc < DeleteTime)
{
File.Delete();
}
}
}
void StageBootstrapExecutable(DeploymentContext SC, string ExeName, string TargetFile, string StagedRelativeTargetPath, string StagedArguments)
{
DirectoryReference InputApp = DirectoryReference.Combine(SC.LocalRoot, "Engine", "Binaries", SC.PlatformDir, "BootstrapPackagedGame.app");
if (InternalUtils.SafeDirectoryExists(InputApp.FullName))
{
// Create the new bootstrap program
DirectoryReference IntermediateDir = DirectoryReference.Combine(SC.ProjectRoot, "Intermediate", "Staging");
InternalUtils.SafeCreateDirectory(IntermediateDir.FullName);
DirectoryReference IntermediateApp = DirectoryReference.Combine(IntermediateDir, ExeName);
if (DirectoryReference.Exists(IntermediateApp))
{
DirectoryReference.Delete(IntermediateApp, true);
}
CloneDirectory(InputApp.FullName, IntermediateApp.FullName);
// Rename the executable
string GameName = Path.GetFileNameWithoutExtension(ExeName);
FileReference.Move(FileReference.Combine(IntermediateApp, "Contents", "MacOS", "BootstrapPackagedGame"), FileReference.Combine(IntermediateApp, "Contents", "MacOS", GameName));
// Copy the icon
string SrcInfoPlistPath = CombinePaths(TargetFile, "Contents", "Info.plist");
string SrcInfoPlist = File.ReadAllText(SrcInfoPlistPath);
string IconName = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleIconFile");
if (!string.IsNullOrEmpty(IconName))
{
string IconPath = CombinePaths(TargetFile, "Contents", "Resources", IconName + ".icns");
InternalUtils.SafeCreateDirectory(CombinePaths(IntermediateApp.FullName, "Contents", "Resources"));
File.Copy(IconPath, CombinePaths(IntermediateApp.FullName, "Contents", "Resources", IconName + ".icns"));
}
// Update Info.plist contents
string DestInfoPlistPath = CombinePaths(IntermediateApp.FullName, "Contents", "Info.plist");
string DestInfoPlist = File.ReadAllText(DestInfoPlistPath);
string AppIdentifier = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleIdentifier");
if (AppIdentifier == "com.epicgames.UE4Game")
{
AppIdentifier = "";
}
string Copyright = GetValueFromInfoPlist(SrcInfoPlist, "NSHumanReadableCopyright");
string BundleVersion = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleVersion", "1");
string ShortVersion = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleShortVersionString", "1.0");
DestInfoPlist = DestInfoPlist.Replace("com.epicgames.BootstrapPackagedGame", string.IsNullOrEmpty(AppIdentifier) ? "com.epicgames." + GameName + "_bootstrap" : AppIdentifier + "_bootstrap");
DestInfoPlist = DestInfoPlist.Replace("BootstrapPackagedGame", GameName);
DestInfoPlist = DestInfoPlist.Replace("__UE4_ICON_FILE__", IconName);
DestInfoPlist = DestInfoPlist.Replace("__UE4_APP_TO_LAUNCH__", StagedRelativeTargetPath);
DestInfoPlist = DestInfoPlist.Replace("__UE4_COMMANDLINE__", StagedArguments);
DestInfoPlist = DestInfoPlist.Replace("__UE4_COPYRIGHT__", Copyright);
DestInfoPlist = DestInfoPlist.Replace("__UE4_BUNDLE_VERSION__", BundleVersion);
DestInfoPlist = DestInfoPlist.Replace("__UE4_SHORT_VERSION__", ShortVersion);
File.WriteAllText(DestInfoPlistPath, DestInfoPlist);
StageAppBundle(SC, IntermediateApp, new StagedDirectoryReference(ExeName));
}
}
public override void ExecuteBuild()
{
var Params = new ProjectParams
(
Command: this,
// Shared
RawProjectPath: ProjectPath
);
Log("********** CRYPTOKEYS COMMAND STARTED **********");
string UE4EditorExe = HostPlatform.Current.GetUE4ExePath(Params.UE4Exe);
if (!FileExists(UE4EditorExe))
{
throw new AutomationException("Missing " + UE4EditorExe + " executable. Needs to be built first.");
}
bool bCycleAllKeys = ParseParam("updateallkeys");
bool bCycleEncryptionKey = bCycleAllKeys || ParseParam("updateencryptionkey");
bool bCycleSigningKey = bCycleAllKeys || ParseParam("updatesigningkey");
if (!bCycleAllKeys && !bCycleEncryptionKey && !bCycleSigningKey)
{
throw new Exception("A target for key cycling must be specified when using the cryptokeys automation script\n\t-updateallkeys: Update all keys\n\t-updateencryptionkey: Update encryption key\n\t-updatesigningkey: Update signing key");
}
FileReference OutputFile = FileReference.Combine(ProjectPath.Directory, "Config", "DefaultCrypto.ini");
FileReference NoRedistOutputFile = FileReference.Combine(ProjectPath.Directory, "Config", "NoRedist", "DefaultCrypto.ini");
FileReference DestinationFile = OutputFile;
// If the project has a DefaultCrypto.ini in a NoRedist folder, we want to copy the newly generated file into that location
if (FileReference.Exists(NoRedistOutputFile))
{
DestinationFile = NoRedistOutputFile;
}
string ChangeDescription = "Automated update of ";
if (bCycleEncryptionKey)
{
ChangeDescription += "encryption";
}
if (bCycleSigningKey)
{
if (bCycleEncryptionKey)
{
ChangeDescription += " and ";
}
ChangeDescription += "signing";
}
ChangeDescription += " key";
if (bCycleEncryptionKey && bCycleSigningKey)
{
ChangeDescription += "s";
}
ChangeDescription += " for project " + Params.ShortProjectName;
P4Connection SubmitP4 = null;
int NewCL = 0;
if (CommandUtils.P4Enabled)
{
SubmitP4 = CommandUtils.P4;
NewCL = SubmitP4.CreateChange(Description: ChangeDescription);
SubmitP4.Revert(String.Format("-k \"{0}\"", DestinationFile.FullName));
SubmitP4.Sync(String.Format("-k \"{0}\"", DestinationFile.FullName), AllowSpew: false);
SubmitP4.Add(NewCL, String.Format("\"{0}\"", DestinationFile.FullName));
SubmitP4.Edit(NewCL, String.Format("\"{0}\"", DestinationFile.FullName));
}
else
{
Log(ChangeDescription);
FileReference.MakeWriteable(OutputFile);
}
string CommandletParams = "";
if (bCycleAllKeys)
{
CommandletParams = "-updateallkeys";
}
else if (bCycleEncryptionKey)
{
CommandletParams = "-updateencryptionkey";
}
else if (bCycleSigningKey)
{
CommandletParams = "-updatesigningkey";
}
RunCommandlet(ProjectPath, UE4EditorExe, "CryptoKeys", CommandletParams);
if (DestinationFile != OutputFile)
{
File.Delete(DestinationFile.FullName);
FileReference.Move(OutputFile, DestinationFile);
}
if (SubmitP4 != null)
{
int ActualCL;
SubmitP4.Submit(NewCL, out ActualCL);
}
}
public override bool PublishSymbols(DirectoryReference SymbolStoreDirectory, List <FileReference> Files, string Product, string BuildVersion = null)
{
// Get the SYMSTORE.EXE path, using the latest SDK version we can find.
FileReference SymStoreExe = GetSymStoreExe();
List <FileReference> FilesToAdd = Files.Where(x => x.HasExtension(".pdb") || x.HasExtension(".exe") || x.HasExtension(".dll")).ToList();
if (FilesToAdd.Count > 0)
{
DateTime Start = DateTime.Now;
DirectoryReference TempSymStoreDir = DirectoryReference.Combine(RootDirectory, "Saved", "SymStore");
DirectoryReference.CreateDirectory(TempSymStoreDir);
DeleteDirectoryContents(TempSymStoreDir);
string TempFileName = Path.GetTempFileName();
try
{
File.WriteAllLines(TempFileName, FilesToAdd.Select(x => x.FullName), Encoding.ASCII);
// Copy everything to the temp symstore
ProcessStartInfo StartInfo = new ProcessStartInfo();
StartInfo.FileName = SymStoreExe.FullName;
StartInfo.Arguments = string.Format("add /f \"@{0}\" /s \"{1}\" /t \"{2}\" /compress", TempFileName, TempSymStoreDir, Product);
StartInfo.UseShellExecute = false;
StartInfo.CreateNoWindow = true;
if (Utils.RunLocalProcessAndLogOutput(StartInfo) != 0)
{
return(false);
}
}
finally
{
File.Delete(TempFileName);
}
DateTime CompressDone = DateTime.Now;
LogInformation("Took {0}s to compress the symbol files", (CompressDone - Start).TotalSeconds);
// Take each new compressed file made and try and copy it to the real symstore. Exclude any symstore admin files
foreach (FileReference File in DirectoryReference.EnumerateFiles(TempSymStoreDir, "*.*", SearchOption.AllDirectories).Where(File => File.HasExtension(".dl_") || File.HasExtension(".ex_") || File.HasExtension(".pd_")))
{
string RelativePath = File.MakeRelativeTo(DirectoryReference.Combine(TempSymStoreDir));
FileReference ActualDestinationFile = FileReference.Combine(SymbolStoreDirectory, RelativePath);
// Try and add a version file. Do this before checking to see if the symbol is there already in the case of exact matches (multiple builds could use the same pdb, for example)
if (!string.IsNullOrWhiteSpace(BuildVersion))
{
FileReference BuildVersionFile = FileReference.Combine(ActualDestinationFile.Directory, string.Format("{0}.version", BuildVersion));
// Attempt to create the file. Just continue if it fails.
try
{
DirectoryReference.CreateDirectory(BuildVersionFile.Directory);
FileReference.WriteAllText(BuildVersionFile, string.Empty);
}
catch (Exception Ex)
{
LogWarning("Failed to write the version file, reason {0}", Ex.ToString());
}
}
// Don't bother copying the temp file if the destination file is there already.
if (FileReference.Exists(ActualDestinationFile))
{
LogInformation("Destination file {0} already exists, skipping", ActualDestinationFile.FullName);
continue;
}
FileReference TempDestinationFile = new FileReference(ActualDestinationFile.FullName + Guid.NewGuid().ToString());
try
{
CommandUtils.CopyFile(File.FullName, TempDestinationFile.FullName);
}
catch (Exception Ex)
{
throw new AutomationException("Couldn't copy the symbol file to the temp store! Reason: {0}", Ex.ToString());
}
// Move the file in the temp store over.
try
{
FileReference.Move(TempDestinationFile, ActualDestinationFile);
}
catch (Exception Ex)
{
// If the file is there already, it was likely either copied elsewhere (and this is an ioexception) or it had a file handle open already.
// Either way, it's fine to just continue on.
if (FileReference.Exists(ActualDestinationFile))
{
LogInformation("Destination file {0} already exists or was in use, skipping.", ActualDestinationFile.FullName);
continue;
}
// If it doesn't exist, we actually failed to copy it entirely.
else
{
LogWarning("Couldn't move temp file {0} to the symbol store at location {1}! Reason: {2}", TempDestinationFile.FullName, ActualDestinationFile.FullName, Ex.ToString());
}
}
// Delete the temp one no matter what, don't want them hanging around in the symstore
finally
{
FileReference.Delete(TempDestinationFile);
}
}
LogInformation("Took {0}s to copy the symbol files to the store", (DateTime.Now - CompressDone).TotalSeconds);
}
return(true);
}
public override void ExecuteBuild()
{
string BucketName = ParseRequiredStringParam("Bucket");
FileReference CredentialsFile = ParseRequiredFileReferenceParam("CredentialsFile");
string CredentialsKey = ParseRequiredStringParam("CredentialsKey");
DirectoryReference CacheDir = ParseRequiredDirectoryReferenceParam("CacheDir");
DirectoryReference FilterDir = ParseRequiredDirectoryReferenceParam("FilterDir");
int Days = ParseParamInt("Days", 7);
int MaxFileSize = ParseParamInt("MaxFileSize", 0);
string RootManifestPath = ParseRequiredStringParam("Manifest");
string KeyPrefix = ParseParamValue("KeyPrefix", "");
bool bReset = ParseParam("Reset");
// The credentials to upload with
AWSCredentials Credentials;
// Try to get the credentials by the key passed in from the script
CredentialProfileStoreChain CredentialsChain = new CredentialProfileStoreChain(CredentialsFile.FullName);
if (!CredentialsChain.TryGetAWSCredentials(CredentialsKey, out Credentials))
{
throw new AutomationException("Unknown credentials key: {0}", CredentialsKey);
}
// Create the new client
using (AmazonS3Client Client = new AmazonS3Client(Credentials, Region))
{
using (SemaphoreSlim RequestSemaphore = new SemaphoreSlim(4))
{
// Read the filters
HashSet <string> Paths = new HashSet <string>();
foreach (FileInfo FilterFile in FilterDir.ToDirectoryInfo().EnumerateFiles("*.txt"))
{
TimeSpan Age = DateTime.UtcNow - FilterFile.LastWriteTimeUtc;
if (Age < TimeSpan.FromDays(3))
{
Log.TraceInformation("Reading {0}", FilterFile.FullName);
string[] Lines = File.ReadAllLines(FilterFile.FullName);
foreach (string Line in Lines)
{
string TrimLine = Line.Trim().Replace('\\', '/');
if (TrimLine.Length > 0)
{
Paths.Add(TrimLine);
}
}
}
else if (Age > TimeSpan.FromDays(5))
{
try
{
Log.TraceInformation("Deleting {0}", FilterFile.FullName);
FilterFile.Delete();
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to delete: {0}", Ex.Message);
Log.TraceLog(ExceptionUtils.FormatExceptionDetails(Ex));
}
}
}
Log.TraceInformation("Found {0:n0} files", Paths.Count);
// Enumerate all the files that are in the network DDC
Log.TraceInformation("");
Log.TraceInformation("Filtering files in {0}...", CacheDir);
List <DerivedDataFile> Files = ParallelExecute <string, DerivedDataFile>(Paths, (Path, FilesBag) => ReadFileInfo(CacheDir, Path, FilesBag));
// Filter to the maximum size
if (MaxFileSize != 0)
{
int NumRemovedMaxSize = Files.RemoveAll(x => x.Info.Length > MaxFileSize);
Log.TraceInformation("");
Log.TraceInformation("Removed {0} files above size limit ({1:n0} bytes)", NumRemovedMaxSize, MaxFileSize);
}
// Create the working directory
DirectoryReference WorkingDir = DirectoryReference.Combine(EngineDirectory, "Saved", "UploadDDC");
DirectoryReference.CreateDirectory(WorkingDir);
// Get the path to the manifest
FileReference RootManifestFile = FileReference.Combine(CommandUtils.RootDirectory, RootManifestPath);
// Read the old root manifest
RootManifest OldRootManifest = new RootManifest();
if (FileReference.Exists(RootManifestFile))
{
OldRootManifest.Read(JsonObject.Read(RootManifestFile));
}
// Read the old bundle manifest
BundleManifest OldBundleManifest = new BundleManifest();
if (OldRootManifest.Entries.Count > 0)
{
FileReference LocalBundleManifest = FileReference.Combine(WorkingDir, "OldBundleManifest.json");
if (TryDownloadFile(Client, BucketName, OldRootManifest.Entries.Last().Key, LocalBundleManifest))
{
OldBundleManifest.Read(JsonObject.Read(LocalBundleManifest));
}
}
// Create the new manifest
BundleManifest NewBundleManifest = new BundleManifest();
// Try to download the old manifest, and add all the bundles we want to keep to the new manifest
if (!bReset)
{
foreach (BundleManifest.Entry Bundle in OldBundleManifest.Entries)
{
FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
if (!FileReference.Exists(BundleFile))
{
Log.TraceInformation("Downloading {0}", BundleFile);
FileReference TempCompressedFile = new FileReference(BundleFile.FullName + ".incoming.gz");
if (!TryDownloadFile(Client, BucketName, Bundle.ObjectKey, TempCompressedFile))
{
Log.TraceWarning("Unable to download {0}", Bundle.ObjectKey);
continue;
}
FileReference TempUncompressedFile = new FileReference(BundleFile.FullName + ".incoming");
try
{
DecompressFile(TempCompressedFile, TempUncompressedFile);
}
catch (Exception Ex)
{
Log.TraceWarning("Unable to uncompress {0}: {1}", Bundle.ObjectKey, Ex.ToString());
continue;
}
FileReference.Move(TempUncompressedFile, BundleFile);
}
NewBundleManifest.Entries.Add(Bundle);
}
}
// Figure out all the item digests that we already have
Dictionary <BundleManifest.Entry, HashSet <ContentHash> > BundleToKeyHashes = new Dictionary <BundleManifest.Entry, HashSet <ContentHash> >();
foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
{
HashSet <ContentHash> KeyHashes = new HashSet <ContentHash>();
FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name);
using (FileStream Stream = FileReference.Open(BundleFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
BinaryReader Reader = new BinaryReader(Stream);
uint Signature = Reader.ReadUInt32();
if (Signature != BundleSignatureV1)
{
throw new Exception(String.Format("Invalid signature for {0}", BundleFile));
}
int NumEntries = Reader.ReadInt32();
for (int EntryIdx = 0; EntryIdx < NumEntries; EntryIdx++)
{
byte[] Digest = new byte[ContentHash.LengthSHA1];
if (Reader.Read(Digest, 0, ContentHash.LengthSHA1) != ContentHash.LengthSHA1)
{
throw new Exception("Unexpected EOF");
}
KeyHashes.Add(new ContentHash(Digest));
Stream.Seek(4, SeekOrigin.Current);
}
}
BundleToKeyHashes[Bundle] = KeyHashes;
}
// Calculate the download size of the manifest
long DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);
// Remove any bundles which have less than the minimum required size in valid data. We don't mark the manifest as dirty yet; these
// files will only be rewritten if new content is added, to prevent the last bundle being rewritten multiple times.
foreach (KeyValuePair <BundleManifest.Entry, HashSet <ContentHash> > Pair in BundleToKeyHashes)
{
long ValidBundleSize = Files.Where(x => Pair.Value.Contains(x.KeyHash)).Sum(x => (long)x.Info.Length);
if (ValidBundleSize < MinBundleSize)
{
NewBundleManifest.Entries.Remove(Pair.Key);
}
}
// Find all the valid digests
HashSet <ContentHash> ReusedKeyHashes = new HashSet <ContentHash>();
foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries)
{
ReusedKeyHashes.UnionWith(BundleToKeyHashes[Bundle]);
}
// Remove all the files which already exist
int NumRemovedExist = Files.RemoveAll(x => ReusedKeyHashes.Contains(x.KeyHash));
if (NumRemovedExist > 0)
{
Log.TraceInformation("");
Log.TraceInformation("Removed {0:n0} files which already exist", NumRemovedExist);
}
// Read all the files we want to include
List <Tuple <DerivedDataFile, byte[]> > FilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
if (Files.Count > 0)
{
Log.TraceInformation("");
Log.TraceInformation("Reading remaining {0:n0} files into memory ({1:n1}mb)...", Files.Count, (float)Files.Sum(x => (long)x.Info.Length) / (1024 * 1024));
FilesToInclude.AddRange(ParallelExecute <DerivedDataFile, Tuple <DerivedDataFile, byte[]> >(Files, (x, y) => ReadFileData(x, y)));
}
// Generate new data
using (RNGCryptoServiceProvider Crypto = new RNGCryptoServiceProvider())
{
// Flag for whether to update the manifest
bool bUpdateManifest = false;
// Upload the new bundle
Log.TraceInformation("");
if (FilesToInclude.Count == 0)
{
Log.TraceInformation("No new files to add.");
}
else
{
// Sort the files to include by creation time. This will bias towards grouping older, more "permanent", items together.
Log.TraceInformation("Sorting input files");
List <Tuple <DerivedDataFile, byte[]> > SortedFilesToInclude = FilesToInclude.OrderBy(x => x.Item1.Info.CreationTimeUtc).ToList();
// Get the target bundle size
long TotalSize = SortedFilesToInclude.Sum(x => (long)x.Item2.Length);
int NumBundles = (int)((TotalSize + (MaxBundleSize - 1)) / MaxBundleSize);
long TargetBundleSize = TotalSize / NumBundles;
// Split the input data into bundles
List <List <Tuple <DerivedDataFile, byte[]> > > BundleFilesToIncludeList = new List <List <Tuple <DerivedDataFile, byte[]> > >();
long BundleSize = 0;
for (int FileIdx = 0; FileIdx < SortedFilesToInclude.Count; BundleSize = BundleSize % TargetBundleSize)
{
List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >();
for (; BundleSize < TargetBundleSize && FileIdx < SortedFilesToInclude.Count; FileIdx++)
{
BundleFilesToInclude.Add(SortedFilesToInclude[FileIdx]);
BundleSize += SortedFilesToInclude[FileIdx].Item2.Length;
}
BundleFilesToIncludeList.Add(BundleFilesToInclude);
}
// Upload each bundle
DateTime NewBundleTime = DateTime.UtcNow;
for (int BundleIdx = 0; BundleIdx < BundleFilesToIncludeList.Count; BundleIdx++)
{
List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = BundleFilesToIncludeList[BundleIdx];
// Get the new bundle info
string NewBundleSuffix = (BundleFilesToIncludeList.Count > 1) ? String.Format("-{0}_of_{1}", BundleIdx + 1, BundleFilesToIncludeList.Count) : "";
string NewBundleName = String.Format("Bundle-{0:yyyy.MM.dd-HH.mm}{1}.ddb", NewBundleTime.ToLocalTime(), NewBundleSuffix);
// Create a random number for the object key
string NewBundleObjectKey = KeyPrefix + "bulk/" + CreateObjectName(Crypto);
// Create the bundle header
byte[] Header;
using (MemoryStream HeaderStream = new MemoryStream())
{
BinaryWriter Writer = new BinaryWriter(HeaderStream);
Writer.Write(BundleSignatureV1);
Writer.Write(BundleFilesToInclude.Count);
foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
{
Writer.Write(FileToInclude.Item1.KeyHash.Bytes, 0, ContentHash.LengthSHA1);
Writer.Write((int)FileToInclude.Item2.Length);
}
Header = HeaderStream.ToArray();
}
// Create the output file
FileReference NewBundleFile = FileReference.Combine(WorkingDir, NewBundleName + ".gz");
Log.TraceInformation("Writing {0}", NewBundleFile);
using (FileStream BundleStream = FileReference.Open(NewBundleFile, FileMode.Create, FileAccess.Write, FileShare.Read))
{
using (GZipStream ZipStream = new GZipStream(BundleStream, CompressionLevel.Optimal, true))
{
ZipStream.Write(Header, 0, Header.Length);
foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude)
{
ZipStream.Write(FileToInclude.Item2, 0, FileToInclude.Item2.Length);
}
}
}
// Upload the file
long NewBundleCompressedLength = NewBundleFile.ToFileInfo().Length;
long NewBundleUncompressedLength = Header.Length + BundleFilesToInclude.Sum(x => (long)x.Item2.Length);
Log.TraceInformation("Uploading bundle to {0} ({1:n1}mb)", NewBundleObjectKey, NewBundleCompressedLength / (1024.0f * 1024.0f));
UploadFile(Client, BucketName, NewBundleFile, 0, NewBundleObjectKey, RequestSemaphore, null);
// Add the bundle to the new manifest
BundleManifest.Entry Bundle = new BundleManifest.Entry();
Bundle.Name = NewBundleName;
Bundle.ObjectKey = NewBundleObjectKey;
Bundle.Time = NewBundleTime;
Bundle.CompressedLength = (int)NewBundleCompressedLength;
Bundle.UncompressedLength = (int)NewBundleUncompressedLength;
NewBundleManifest.Entries.Add(Bundle);
// Mark the manifest as requiring an update
bUpdateManifest = true;
}
}
// Update the manifest
if (bUpdateManifest)
{
DateTime UtcNow = DateTime.UtcNow;
DateTime RemoveBundleManifestsBefore = UtcNow - TimeSpan.FromDays(3.0);
// Update the root manifest
RootManifest NewRootManifest = new RootManifest();
NewRootManifest.AccessKey = OldRootManifest.AccessKey;
NewRootManifest.SecretKey = OldRootManifest.SecretKey;
foreach (RootManifest.Entry Entry in OldRootManifest.Entries)
{
if (Entry.CreateTime >= RemoveBundleManifestsBefore)
{
NewRootManifest.Entries.Add(Entry);
}
}
// Make sure there's an entry for the last 24h
DateTime RequireBundleManifestAfter = UtcNow - TimeSpan.FromDays(1.0);
if (!NewRootManifest.Entries.Any(x => x.CreateTime > RequireBundleManifestAfter))
{
RootManifest.Entry NewEntry = new RootManifest.Entry();
NewEntry.CreateTime = UtcNow;
NewEntry.Key = KeyPrefix + CreateObjectName(Crypto);
NewRootManifest.Entries.Add(NewEntry);
}
// Save out the new bundle manifest
FileReference NewBundleManifestFile = FileReference.Combine(WorkingDir, "NewBundleManifest.json");
NewBundleManifest.Save(NewBundleManifestFile);
// Update all the bundle manifests still valid
foreach (RootManifest.Entry Entry in NewRootManifest.Entries)
{
Log.TraceInformation("Uploading bundle manifest to {0}", Entry.Key);
UploadFile(Client, BucketName, NewBundleManifestFile, 0, Entry.Key, RequestSemaphore, null);
}
// Overwrite all the existing manifests
if (AllowSubmit)
{
List <string> ExistingFiles = P4.Files(CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
// Create a changelist containing the new manifest
int ChangeNumber = P4.CreateChange(Description: "Updating DDC bundle manifest");
if (ExistingFiles.Count > 0)
{
P4.Edit(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
NewRootManifest.Save(RootManifestFile);
}
else
{
NewRootManifest.Save(RootManifestFile);
P4.Add(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName));
}
// Submit it
int SubmittedChangeNumber;
P4.Submit(ChangeNumber, out SubmittedChangeNumber, true);
if (SubmittedChangeNumber <= 0)
{
throw new AutomationException("Failed to submit change");
}
// Delete any bundles that are no longer referenced
HashSet <string> KeepObjectKeys = new HashSet <string>(NewBundleManifest.Entries.Select(x => x.ObjectKey));
foreach (BundleManifest.Entry OldEntry in OldBundleManifest.Entries)
{
if (!KeepObjectKeys.Contains(OldEntry.ObjectKey))
{
Log.TraceInformation("Deleting unreferenced bundle {0}", OldEntry.ObjectKey);
DeleteFile(Client, BucketName, OldEntry.ObjectKey, RequestSemaphore, null);
}
}
// Delete any bundle manifests which are no longer referenced
HashSet <string> KeepManifestKeys = new HashSet <string>(NewRootManifest.Entries.Select(x => x.Key));
foreach (RootManifest.Entry OldEntry in OldRootManifest.Entries)
{
if (!KeepManifestKeys.Contains(OldEntry.Key))
{
Log.TraceInformation("Deleting unreferenced manifest {0}", OldEntry.Key);
DeleteFile(Client, BucketName, OldEntry.Key, RequestSemaphore, null);
}
}
}
else
{
// Skip submitting
Log.TraceWarning("Skipping manifest submit due to missing -Submit argument.");
}
// Update the new download size
DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength);
}
}
// Print some stats about the final manifest
Log.TraceInformation("");
Log.TraceInformation("Total download size {0:n1}mb", DownloadSize / (1024.0 * 1024.0));
Log.TraceInformation("");
}
}
}