/// <summary> /// Completes an interrupted transaction to write the state file /// </summary> /// <param name="StateFile">Path to the state file</param> static void CompleteStateTransaction(FileReference StateFile) { if (!FileReference.Exists(StateFile)) { FileReference StateTransactionFile = GetStateTransactionFile(StateFile); if (FileReference.Exists(StateTransactionFile)) { Log.TraceInformation("Completing partial transaction through {0}", StateTransactionFile); FileReference.Move(StateTransactionFile, StateFile); } } }
/// <summary> /// Writes the state to disk in a way that can be recovered if the operation is interrupted /// </summary> /// <param name="StateFile">The file to write to</param> /// <param name="State">The state object</param> static void WriteState(FileReference StateFile, BuildHealthState State) { Stopwatch Timer = Stopwatch.StartNew(); // Write out the state to the transaction file FileReference StateTransactionFile = GetStateTransactionFile(StateFile); SerializeJson(StateTransactionFile, State); // Remove the original file, then move the transaction file into place FileReference.Delete(StateFile); FileReference.Move(StateTransactionFile, StateFile); Log.TraceInformation("Took {0}s to write state", Timer.Elapsed.TotalSeconds); }
public void Dispose() { // Try to write the output file in a transactional way; write it to a temporary file and rename it. DirectoryReference.CreateDirectory(OutputFile.Directory); InputData Data = new InputData(); Data.Jobs.Add(Job); FileReference TempOutputFile = new FileReference(OutputFile.FullName + ".incoming"); using (MemoryStream Stream = new MemoryStream()) { DataContractJsonSerializer InputFileDataSerializer = new DataContractJsonSerializer(typeof(InputData)); InputFileDataSerializer.WriteObject(Stream, Data); FileReference.WriteAllBytes(TempOutputFile, Stream.ToArray()); } FileReference.Delete(OutputFile); FileReference.Move(TempOutputFile, OutputFile); }
static void BackupLogFile(FileReference LogFile, TimeSpan RetentionSpan) { string LogPrefix = LogFile.GetFileNameWithoutExtension() + "-backup-"; string LogSuffix = LogFile.GetExtension(); if (FileReference.Exists(LogFile)) { string Timestamp = FileReference.GetLastWriteTime(LogFile).ToString("yyyy.MM.dd-HH.mm.ss"); string UniqueSuffix = ""; for (int NextSuffixIdx = 2; ; NextSuffixIdx++) { FileReference BackupLogFile = FileReference.Combine(LogFile.Directory, String.Format("{0}{1}{2}{3}", LogPrefix, Timestamp, UniqueSuffix, LogSuffix)); if (!FileReference.Exists(BackupLogFile)) { try { FileReference.Move(LogFile, BackupLogFile); } catch (Exception Ex) { Log.TraceWarning("Unable to backup {0} to {1} ({2})", LogFile, BackupLogFile, Ex.Message); Log.TraceLog(ExceptionUtils.FormatException(Ex)); } break; } UniqueSuffix = String.Format("_{0}", NextSuffixIdx); } } DateTime DeleteTime = DateTime.UtcNow - RetentionSpan; foreach (FileInfo File in new DirectoryInfo(LogFile.Directory.FullName).EnumerateFiles(LogPrefix + "*" + LogSuffix)) { if (File.LastWriteTimeUtc < DeleteTime) { File.Delete(); } } }
void StageBootstrapExecutable(DeploymentContext SC, string ExeName, string TargetFile, string StagedRelativeTargetPath, string StagedArguments) { DirectoryReference InputApp = DirectoryReference.Combine(SC.LocalRoot, "Engine", "Binaries", SC.PlatformDir, "BootstrapPackagedGame.app"); if (InternalUtils.SafeDirectoryExists(InputApp.FullName)) { // Create the new bootstrap program DirectoryReference IntermediateDir = DirectoryReference.Combine(SC.ProjectRoot, "Intermediate", "Staging"); InternalUtils.SafeCreateDirectory(IntermediateDir.FullName); DirectoryReference IntermediateApp = DirectoryReference.Combine(IntermediateDir, ExeName); if (DirectoryReference.Exists(IntermediateApp)) { DirectoryReference.Delete(IntermediateApp, true); } CloneDirectory(InputApp.FullName, IntermediateApp.FullName); // Rename the executable string GameName = Path.GetFileNameWithoutExtension(ExeName); FileReference.Move(FileReference.Combine(IntermediateApp, "Contents", "MacOS", "BootstrapPackagedGame"), FileReference.Combine(IntermediateApp, "Contents", "MacOS", GameName)); // Copy the icon string SrcInfoPlistPath = CombinePaths(TargetFile, "Contents", "Info.plist"); string SrcInfoPlist = File.ReadAllText(SrcInfoPlistPath); string IconName = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleIconFile"); if (!string.IsNullOrEmpty(IconName)) { string IconPath = CombinePaths(TargetFile, "Contents", "Resources", IconName + ".icns"); InternalUtils.SafeCreateDirectory(CombinePaths(IntermediateApp.FullName, "Contents", "Resources")); File.Copy(IconPath, CombinePaths(IntermediateApp.FullName, "Contents", "Resources", IconName + ".icns")); } // Update Info.plist contents string DestInfoPlistPath = CombinePaths(IntermediateApp.FullName, "Contents", "Info.plist"); string DestInfoPlist = File.ReadAllText(DestInfoPlistPath); string AppIdentifier = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleIdentifier"); if (AppIdentifier == "com.epicgames.UE4Game") { AppIdentifier = ""; } string Copyright = GetValueFromInfoPlist(SrcInfoPlist, "NSHumanReadableCopyright"); string BundleVersion = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleVersion", "1"); string ShortVersion = GetValueFromInfoPlist(SrcInfoPlist, "CFBundleShortVersionString", "1.0"); DestInfoPlist = DestInfoPlist.Replace("com.epicgames.BootstrapPackagedGame", string.IsNullOrEmpty(AppIdentifier) ? "com.epicgames." + GameName + "_bootstrap" : AppIdentifier + "_bootstrap"); DestInfoPlist = DestInfoPlist.Replace("BootstrapPackagedGame", GameName); DestInfoPlist = DestInfoPlist.Replace("__UE4_ICON_FILE__", IconName); DestInfoPlist = DestInfoPlist.Replace("__UE4_APP_TO_LAUNCH__", StagedRelativeTargetPath); DestInfoPlist = DestInfoPlist.Replace("__UE4_COMMANDLINE__", StagedArguments); DestInfoPlist = DestInfoPlist.Replace("__UE4_COPYRIGHT__", Copyright); DestInfoPlist = DestInfoPlist.Replace("__UE4_BUNDLE_VERSION__", BundleVersion); DestInfoPlist = DestInfoPlist.Replace("__UE4_SHORT_VERSION__", ShortVersion); File.WriteAllText(DestInfoPlistPath, DestInfoPlist); StageAppBundle(SC, IntermediateApp, new StagedDirectoryReference(ExeName)); } }
public override void ExecuteBuild() { var Params = new ProjectParams ( Command: this, // Shared RawProjectPath: ProjectPath ); Log("********** CRYPTOKEYS COMMAND STARTED **********"); string UE4EditorExe = HostPlatform.Current.GetUE4ExePath(Params.UE4Exe); if (!FileExists(UE4EditorExe)) { throw new AutomationException("Missing " + UE4EditorExe + " executable. Needs to be built first."); } bool bCycleAllKeys = ParseParam("updateallkeys"); bool bCycleEncryptionKey = bCycleAllKeys || ParseParam("updateencryptionkey"); bool bCycleSigningKey = bCycleAllKeys || ParseParam("updatesigningkey"); if (!bCycleAllKeys && !bCycleEncryptionKey && !bCycleSigningKey) { throw new Exception("A target for key cycling must be specified when using the cryptokeys automation script\n\t-updateallkeys: Update all keys\n\t-updateencryptionkey: Update encryption key\n\t-updatesigningkey: Update signing key"); } FileReference OutputFile = FileReference.Combine(ProjectPath.Directory, "Config", "DefaultCrypto.ini"); FileReference NoRedistOutputFile = FileReference.Combine(ProjectPath.Directory, "Config", "NoRedist", "DefaultCrypto.ini"); FileReference DestinationFile = OutputFile; // If the project has a DefaultCrypto.ini in a NoRedist folder, we want to copy the newly generated file into that location if (FileReference.Exists(NoRedistOutputFile)) { DestinationFile = NoRedistOutputFile; } string ChangeDescription = "Automated update of "; if (bCycleEncryptionKey) { ChangeDescription += "encryption"; } if (bCycleSigningKey) { if (bCycleEncryptionKey) { ChangeDescription += " and "; } ChangeDescription += "signing"; } ChangeDescription += " key"; if (bCycleEncryptionKey && bCycleSigningKey) { ChangeDescription += "s"; } ChangeDescription += " for project " + Params.ShortProjectName; P4Connection SubmitP4 = null; int NewCL = 0; if (CommandUtils.P4Enabled) { SubmitP4 = CommandUtils.P4; NewCL = SubmitP4.CreateChange(Description: ChangeDescription); SubmitP4.Revert(String.Format("-k \"{0}\"", DestinationFile.FullName)); SubmitP4.Sync(String.Format("-k \"{0}\"", DestinationFile.FullName), AllowSpew: false); SubmitP4.Add(NewCL, String.Format("\"{0}\"", DestinationFile.FullName)); SubmitP4.Edit(NewCL, String.Format("\"{0}\"", DestinationFile.FullName)); } else { Log(ChangeDescription); FileReference.MakeWriteable(OutputFile); } string CommandletParams = ""; if (bCycleAllKeys) { CommandletParams = "-updateallkeys"; } else if (bCycleEncryptionKey) { CommandletParams = "-updateencryptionkey"; } else if (bCycleSigningKey) { CommandletParams = "-updatesigningkey"; } RunCommandlet(ProjectPath, UE4EditorExe, "CryptoKeys", CommandletParams); if (DestinationFile != OutputFile) { File.Delete(DestinationFile.FullName); FileReference.Move(OutputFile, DestinationFile); } if (SubmitP4 != null) { int ActualCL; SubmitP4.Submit(NewCL, out ActualCL); } }
public override bool PublishSymbols(DirectoryReference SymbolStoreDirectory, List <FileReference> Files, string Product, string BuildVersion = null) { // Get the SYMSTORE.EXE path, using the latest SDK version we can find. FileReference SymStoreExe = GetSymStoreExe(); List <FileReference> FilesToAdd = Files.Where(x => x.HasExtension(".pdb") || x.HasExtension(".exe") || x.HasExtension(".dll")).ToList(); if (FilesToAdd.Count > 0) { DateTime Start = DateTime.Now; DirectoryReference TempSymStoreDir = DirectoryReference.Combine(RootDirectory, "Saved", "SymStore"); DirectoryReference.CreateDirectory(TempSymStoreDir); DeleteDirectoryContents(TempSymStoreDir); string TempFileName = Path.GetTempFileName(); try { File.WriteAllLines(TempFileName, FilesToAdd.Select(x => x.FullName), Encoding.ASCII); // Copy everything to the temp symstore ProcessStartInfo StartInfo = new ProcessStartInfo(); StartInfo.FileName = SymStoreExe.FullName; StartInfo.Arguments = string.Format("add /f \"@{0}\" /s \"{1}\" /t \"{2}\" /compress", TempFileName, TempSymStoreDir, Product); StartInfo.UseShellExecute = false; StartInfo.CreateNoWindow = true; if (Utils.RunLocalProcessAndLogOutput(StartInfo) != 0) { return(false); } } finally { File.Delete(TempFileName); } DateTime CompressDone = DateTime.Now; LogInformation("Took {0}s to compress the symbol files", (CompressDone - Start).TotalSeconds); // Take each new compressed file made and try and copy it to the real symstore. Exclude any symstore admin files foreach (FileReference File in DirectoryReference.EnumerateFiles(TempSymStoreDir, "*.*", SearchOption.AllDirectories).Where(File => File.HasExtension(".dl_") || File.HasExtension(".ex_") || File.HasExtension(".pd_"))) { string RelativePath = File.MakeRelativeTo(DirectoryReference.Combine(TempSymStoreDir)); FileReference ActualDestinationFile = FileReference.Combine(SymbolStoreDirectory, RelativePath); // Try and add a version file. Do this before checking to see if the symbol is there already in the case of exact matches (multiple builds could use the same pdb, for example) if (!string.IsNullOrWhiteSpace(BuildVersion)) { FileReference BuildVersionFile = FileReference.Combine(ActualDestinationFile.Directory, string.Format("{0}.version", BuildVersion)); // Attempt to create the file. Just continue if it fails. try { DirectoryReference.CreateDirectory(BuildVersionFile.Directory); FileReference.WriteAllText(BuildVersionFile, string.Empty); } catch (Exception Ex) { LogWarning("Failed to write the version file, reason {0}", Ex.ToString()); } } // Don't bother copying the temp file if the destination file is there already. if (FileReference.Exists(ActualDestinationFile)) { LogInformation("Destination file {0} already exists, skipping", ActualDestinationFile.FullName); continue; } FileReference TempDestinationFile = new FileReference(ActualDestinationFile.FullName + Guid.NewGuid().ToString()); try { CommandUtils.CopyFile(File.FullName, TempDestinationFile.FullName); } catch (Exception Ex) { throw new AutomationException("Couldn't copy the symbol file to the temp store! Reason: {0}", Ex.ToString()); } // Move the file in the temp store over. try { FileReference.Move(TempDestinationFile, ActualDestinationFile); } catch (Exception Ex) { // If the file is there already, it was likely either copied elsewhere (and this is an ioexception) or it had a file handle open already. // Either way, it's fine to just continue on. if (FileReference.Exists(ActualDestinationFile)) { LogInformation("Destination file {0} already exists or was in use, skipping.", ActualDestinationFile.FullName); continue; } // If it doesn't exist, we actually failed to copy it entirely. else { LogWarning("Couldn't move temp file {0} to the symbol store at location {1}! Reason: {2}", TempDestinationFile.FullName, ActualDestinationFile.FullName, Ex.ToString()); } } // Delete the temp one no matter what, don't want them hanging around in the symstore finally { FileReference.Delete(TempDestinationFile); } } LogInformation("Took {0}s to copy the symbol files to the store", (DateTime.Now - CompressDone).TotalSeconds); } return(true); }
public override void ExecuteBuild() { string BucketName = ParseRequiredStringParam("Bucket"); FileReference CredentialsFile = ParseRequiredFileReferenceParam("CredentialsFile"); string CredentialsKey = ParseRequiredStringParam("CredentialsKey"); DirectoryReference CacheDir = ParseRequiredDirectoryReferenceParam("CacheDir"); DirectoryReference FilterDir = ParseRequiredDirectoryReferenceParam("FilterDir"); int Days = ParseParamInt("Days", 7); int MaxFileSize = ParseParamInt("MaxFileSize", 0); string RootManifestPath = ParseRequiredStringParam("Manifest"); string KeyPrefix = ParseParamValue("KeyPrefix", ""); bool bReset = ParseParam("Reset"); // The credentials to upload with AWSCredentials Credentials; // Try to get the credentials by the key passed in from the script CredentialProfileStoreChain CredentialsChain = new CredentialProfileStoreChain(CredentialsFile.FullName); if (!CredentialsChain.TryGetAWSCredentials(CredentialsKey, out Credentials)) { throw new AutomationException("Unknown credentials key: {0}", CredentialsKey); } // Create the new client using (AmazonS3Client Client = new AmazonS3Client(Credentials, Region)) { using (SemaphoreSlim RequestSemaphore = new SemaphoreSlim(4)) { // Read the filters HashSet <string> Paths = new HashSet <string>(); foreach (FileInfo FilterFile in FilterDir.ToDirectoryInfo().EnumerateFiles("*.txt")) { TimeSpan Age = DateTime.UtcNow - FilterFile.LastWriteTimeUtc; if (Age < TimeSpan.FromDays(3)) { Log.TraceInformation("Reading {0}", FilterFile.FullName); string[] Lines = File.ReadAllLines(FilterFile.FullName); foreach (string Line in Lines) { string TrimLine = Line.Trim().Replace('\\', '/'); if (TrimLine.Length > 0) { Paths.Add(TrimLine); } } } else if (Age > TimeSpan.FromDays(5)) { try { Log.TraceInformation("Deleting {0}", FilterFile.FullName); FilterFile.Delete(); } catch (Exception Ex) { Log.TraceWarning("Unable to delete: {0}", Ex.Message); Log.TraceLog(ExceptionUtils.FormatExceptionDetails(Ex)); } } } Log.TraceInformation("Found {0:n0} files", Paths.Count); // Enumerate all the files that are in the network DDC Log.TraceInformation(""); Log.TraceInformation("Filtering files in {0}...", CacheDir); List <DerivedDataFile> Files = ParallelExecute <string, DerivedDataFile>(Paths, (Path, FilesBag) => ReadFileInfo(CacheDir, Path, FilesBag)); // Filter to the maximum size if (MaxFileSize != 0) { int NumRemovedMaxSize = Files.RemoveAll(x => x.Info.Length > MaxFileSize); Log.TraceInformation(""); Log.TraceInformation("Removed {0} files above size limit ({1:n0} bytes)", NumRemovedMaxSize, MaxFileSize); } // Create the working directory DirectoryReference WorkingDir = DirectoryReference.Combine(EngineDirectory, "Saved", "UploadDDC"); DirectoryReference.CreateDirectory(WorkingDir); // Get the path to the manifest FileReference RootManifestFile = FileReference.Combine(CommandUtils.RootDirectory, RootManifestPath); // Read the old root manifest RootManifest OldRootManifest = new RootManifest(); if (FileReference.Exists(RootManifestFile)) { OldRootManifest.Read(JsonObject.Read(RootManifestFile)); } // Read the old bundle manifest BundleManifest OldBundleManifest = new BundleManifest(); if (OldRootManifest.Entries.Count > 0) { FileReference LocalBundleManifest = FileReference.Combine(WorkingDir, "OldBundleManifest.json"); if (TryDownloadFile(Client, BucketName, OldRootManifest.Entries.Last().Key, LocalBundleManifest)) { OldBundleManifest.Read(JsonObject.Read(LocalBundleManifest)); } } // Create the new manifest BundleManifest NewBundleManifest = new BundleManifest(); // Try to download the old manifest, and add all the bundles we want to keep to the new manifest if (!bReset) { foreach (BundleManifest.Entry Bundle in OldBundleManifest.Entries) { FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name); if (!FileReference.Exists(BundleFile)) { Log.TraceInformation("Downloading {0}", BundleFile); FileReference TempCompressedFile = new FileReference(BundleFile.FullName + ".incoming.gz"); if (!TryDownloadFile(Client, BucketName, Bundle.ObjectKey, TempCompressedFile)) { Log.TraceWarning("Unable to download {0}", Bundle.ObjectKey); continue; } FileReference TempUncompressedFile = new FileReference(BundleFile.FullName + ".incoming"); try { DecompressFile(TempCompressedFile, TempUncompressedFile); } catch (Exception Ex) { Log.TraceWarning("Unable to uncompress {0}: {1}", Bundle.ObjectKey, Ex.ToString()); continue; } FileReference.Move(TempUncompressedFile, BundleFile); } NewBundleManifest.Entries.Add(Bundle); } } // Figure out all the item digests that we already have Dictionary <BundleManifest.Entry, HashSet <ContentHash> > BundleToKeyHashes = new Dictionary <BundleManifest.Entry, HashSet <ContentHash> >(); foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries) { HashSet <ContentHash> KeyHashes = new HashSet <ContentHash>(); FileReference BundleFile = FileReference.Combine(WorkingDir, Bundle.Name); using (FileStream Stream = FileReference.Open(BundleFile, FileMode.Open, FileAccess.Read, FileShare.Read)) { BinaryReader Reader = new BinaryReader(Stream); uint Signature = Reader.ReadUInt32(); if (Signature != BundleSignatureV1) { throw new Exception(String.Format("Invalid signature for {0}", BundleFile)); } int NumEntries = Reader.ReadInt32(); for (int EntryIdx = 0; EntryIdx < NumEntries; EntryIdx++) { byte[] Digest = new byte[ContentHash.LengthSHA1]; if (Reader.Read(Digest, 0, ContentHash.LengthSHA1) != ContentHash.LengthSHA1) { throw new Exception("Unexpected EOF"); } KeyHashes.Add(new ContentHash(Digest)); Stream.Seek(4, SeekOrigin.Current); } } BundleToKeyHashes[Bundle] = KeyHashes; } // Calculate the download size of the manifest long DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength); // Remove any bundles which have less than the minimum required size in valid data. We don't mark the manifest as dirty yet; these // files will only be rewritten if new content is added, to prevent the last bundle being rewritten multiple times. foreach (KeyValuePair <BundleManifest.Entry, HashSet <ContentHash> > Pair in BundleToKeyHashes) { long ValidBundleSize = Files.Where(x => Pair.Value.Contains(x.KeyHash)).Sum(x => (long)x.Info.Length); if (ValidBundleSize < MinBundleSize) { NewBundleManifest.Entries.Remove(Pair.Key); } } // Find all the valid digests HashSet <ContentHash> ReusedKeyHashes = new HashSet <ContentHash>(); foreach (BundleManifest.Entry Bundle in NewBundleManifest.Entries) { ReusedKeyHashes.UnionWith(BundleToKeyHashes[Bundle]); } // Remove all the files which already exist int NumRemovedExist = Files.RemoveAll(x => ReusedKeyHashes.Contains(x.KeyHash)); if (NumRemovedExist > 0) { Log.TraceInformation(""); Log.TraceInformation("Removed {0:n0} files which already exist", NumRemovedExist); } // Read all the files we want to include List <Tuple <DerivedDataFile, byte[]> > FilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >(); if (Files.Count > 0) { Log.TraceInformation(""); Log.TraceInformation("Reading remaining {0:n0} files into memory ({1:n1}mb)...", Files.Count, (float)Files.Sum(x => (long)x.Info.Length) / (1024 * 1024)); FilesToInclude.AddRange(ParallelExecute <DerivedDataFile, Tuple <DerivedDataFile, byte[]> >(Files, (x, y) => ReadFileData(x, y))); } // Generate new data using (RNGCryptoServiceProvider Crypto = new RNGCryptoServiceProvider()) { // Flag for whether to update the manifest bool bUpdateManifest = false; // Upload the new bundle Log.TraceInformation(""); if (FilesToInclude.Count == 0) { Log.TraceInformation("No new files to add."); } else { // Sort the files to include by creation time. This will bias towards grouping older, more "permanent", items together. Log.TraceInformation("Sorting input files"); List <Tuple <DerivedDataFile, byte[]> > SortedFilesToInclude = FilesToInclude.OrderBy(x => x.Item1.Info.CreationTimeUtc).ToList(); // Get the target bundle size long TotalSize = SortedFilesToInclude.Sum(x => (long)x.Item2.Length); int NumBundles = (int)((TotalSize + (MaxBundleSize - 1)) / MaxBundleSize); long TargetBundleSize = TotalSize / NumBundles; // Split the input data into bundles List <List <Tuple <DerivedDataFile, byte[]> > > BundleFilesToIncludeList = new List <List <Tuple <DerivedDataFile, byte[]> > >(); long BundleSize = 0; for (int FileIdx = 0; FileIdx < SortedFilesToInclude.Count; BundleSize = BundleSize % TargetBundleSize) { List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = new List <Tuple <DerivedDataFile, byte[]> >(); for (; BundleSize < TargetBundleSize && FileIdx < SortedFilesToInclude.Count; FileIdx++) { BundleFilesToInclude.Add(SortedFilesToInclude[FileIdx]); BundleSize += SortedFilesToInclude[FileIdx].Item2.Length; } BundleFilesToIncludeList.Add(BundleFilesToInclude); } // Upload each bundle DateTime NewBundleTime = DateTime.UtcNow; for (int BundleIdx = 0; BundleIdx < BundleFilesToIncludeList.Count; BundleIdx++) { List <Tuple <DerivedDataFile, byte[]> > BundleFilesToInclude = BundleFilesToIncludeList[BundleIdx]; // Get the new bundle info string NewBundleSuffix = (BundleFilesToIncludeList.Count > 1) ? String.Format("-{0}_of_{1}", BundleIdx + 1, BundleFilesToIncludeList.Count) : ""; string NewBundleName = String.Format("Bundle-{0:yyyy.MM.dd-HH.mm}{1}.ddb", NewBundleTime.ToLocalTime(), NewBundleSuffix); // Create a random number for the object key string NewBundleObjectKey = KeyPrefix + "bulk/" + CreateObjectName(Crypto); // Create the bundle header byte[] Header; using (MemoryStream HeaderStream = new MemoryStream()) { BinaryWriter Writer = new BinaryWriter(HeaderStream); Writer.Write(BundleSignatureV1); Writer.Write(BundleFilesToInclude.Count); foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude) { Writer.Write(FileToInclude.Item1.KeyHash.Bytes, 0, ContentHash.LengthSHA1); Writer.Write((int)FileToInclude.Item2.Length); } Header = HeaderStream.ToArray(); } // Create the output file FileReference NewBundleFile = FileReference.Combine(WorkingDir, NewBundleName + ".gz"); Log.TraceInformation("Writing {0}", NewBundleFile); using (FileStream BundleStream = FileReference.Open(NewBundleFile, FileMode.Create, FileAccess.Write, FileShare.Read)) { using (GZipStream ZipStream = new GZipStream(BundleStream, CompressionLevel.Optimal, true)) { ZipStream.Write(Header, 0, Header.Length); foreach (Tuple <DerivedDataFile, byte[]> FileToInclude in BundleFilesToInclude) { ZipStream.Write(FileToInclude.Item2, 0, FileToInclude.Item2.Length); } } } // Upload the file long NewBundleCompressedLength = NewBundleFile.ToFileInfo().Length; long NewBundleUncompressedLength = Header.Length + BundleFilesToInclude.Sum(x => (long)x.Item2.Length); Log.TraceInformation("Uploading bundle to {0} ({1:n1}mb)", NewBundleObjectKey, NewBundleCompressedLength / (1024.0f * 1024.0f)); UploadFile(Client, BucketName, NewBundleFile, 0, NewBundleObjectKey, RequestSemaphore, null); // Add the bundle to the new manifest BundleManifest.Entry Bundle = new BundleManifest.Entry(); Bundle.Name = NewBundleName; Bundle.ObjectKey = NewBundleObjectKey; Bundle.Time = NewBundleTime; Bundle.CompressedLength = (int)NewBundleCompressedLength; Bundle.UncompressedLength = (int)NewBundleUncompressedLength; NewBundleManifest.Entries.Add(Bundle); // Mark the manifest as requiring an update bUpdateManifest = true; } } // Update the manifest if (bUpdateManifest) { DateTime UtcNow = DateTime.UtcNow; DateTime RemoveBundleManifestsBefore = UtcNow - TimeSpan.FromDays(3.0); // Update the root manifest RootManifest NewRootManifest = new RootManifest(); NewRootManifest.AccessKey = OldRootManifest.AccessKey; NewRootManifest.SecretKey = OldRootManifest.SecretKey; foreach (RootManifest.Entry Entry in OldRootManifest.Entries) { if (Entry.CreateTime >= RemoveBundleManifestsBefore) { NewRootManifest.Entries.Add(Entry); } } // Make sure there's an entry for the last 24h DateTime RequireBundleManifestAfter = UtcNow - TimeSpan.FromDays(1.0); if (!NewRootManifest.Entries.Any(x => x.CreateTime > RequireBundleManifestAfter)) { RootManifest.Entry NewEntry = new RootManifest.Entry(); NewEntry.CreateTime = UtcNow; NewEntry.Key = KeyPrefix + CreateObjectName(Crypto); NewRootManifest.Entries.Add(NewEntry); } // Save out the new bundle manifest FileReference NewBundleManifestFile = FileReference.Combine(WorkingDir, "NewBundleManifest.json"); NewBundleManifest.Save(NewBundleManifestFile); // Update all the bundle manifests still valid foreach (RootManifest.Entry Entry in NewRootManifest.Entries) { Log.TraceInformation("Uploading bundle manifest to {0}", Entry.Key); UploadFile(Client, BucketName, NewBundleManifestFile, 0, Entry.Key, RequestSemaphore, null); } // Overwrite all the existing manifests if (AllowSubmit) { List <string> ExistingFiles = P4.Files(CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName)); // Create a changelist containing the new manifest int ChangeNumber = P4.CreateChange(Description: "Updating DDC bundle manifest"); if (ExistingFiles.Count > 0) { P4.Edit(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName)); NewRootManifest.Save(RootManifestFile); } else { NewRootManifest.Save(RootManifestFile); P4.Add(ChangeNumber, CommandUtils.MakePathSafeToUseWithCommandLine(RootManifestFile.FullName)); } // Submit it int SubmittedChangeNumber; P4.Submit(ChangeNumber, out SubmittedChangeNumber, true); if (SubmittedChangeNumber <= 0) { throw new AutomationException("Failed to submit change"); } // Delete any bundles that are no longer referenced HashSet <string> KeepObjectKeys = new HashSet <string>(NewBundleManifest.Entries.Select(x => x.ObjectKey)); foreach (BundleManifest.Entry OldEntry in OldBundleManifest.Entries) { if (!KeepObjectKeys.Contains(OldEntry.ObjectKey)) { Log.TraceInformation("Deleting unreferenced bundle {0}", OldEntry.ObjectKey); DeleteFile(Client, BucketName, OldEntry.ObjectKey, RequestSemaphore, null); } } // Delete any bundle manifests which are no longer referenced HashSet <string> KeepManifestKeys = new HashSet <string>(NewRootManifest.Entries.Select(x => x.Key)); foreach (RootManifest.Entry OldEntry in OldRootManifest.Entries) { if (!KeepManifestKeys.Contains(OldEntry.Key)) { Log.TraceInformation("Deleting unreferenced manifest {0}", OldEntry.Key); DeleteFile(Client, BucketName, OldEntry.Key, RequestSemaphore, null); } } } else { // Skip submitting Log.TraceWarning("Skipping manifest submit due to missing -Submit argument."); } // Update the new download size DownloadSize = NewBundleManifest.Entries.Sum(x => (long)x.CompressedLength); } } // Print some stats about the final manifest Log.TraceInformation(""); Log.TraceInformation("Total download size {0:n1}mb", DownloadSize / (1024.0 * 1024.0)); Log.TraceInformation(""); } } }