public IpfsManager(PlatformDirectory platform, ILogger <IpfsManager> logger)
{
_logger = logger;
platformDirectory = platform;
IpfsDataPath = Path.Combine(platform.DtpServerDataPath, ".ipfs");
IpfsExePath = Path.Combine(platform.ExecutingPath, @"Platform\ipfs");
}
public IHost Init()
{
Platform = new PlatformDirectory();
Platform.EnsureDtpServerDirectory();
SetupConfiguration();
SetupLogger();
Log.Information("Getting the motors running...");
// Renaming BuildWebHost to InitWebHost avoids problems with add-migration command.
// IDesignTimeDbContextFactory implemented for add-migration specifically.
webHost = CreateHostBuilder(null).Build();
return(webHost);
}
public virtual void ConfigureDbContext(IServiceCollection services)
{
using (TimeMe.Track("ConfigureDbContext"))
{
var platform = new PlatformDirectory();
var dbName = "trust.db";
var dbDestination = "./trust.db";
if (_hostingEnv.IsProduction())
{
dbDestination = Path.Combine(platform.DatabaseDataPath, dbName);
platform.EnsureDtpServerDirectory();
if (!File.Exists(dbDestination))
{
File.Copy(Path.Combine(dbName), dbDestination);
}
}
services.AddDbContext <TrustDBContext>(options =>
options.UseSqlite($"Filename={dbDestination}", b => b.MigrationsAssembly("DtpCore"))
.ConfigureWarnings(warnings => warnings.Ignore(CoreEventId.IncludeIgnoredWarning))
);
}
}
public override void ExecuteBuild()
{
var UEProjectRoot = ParseParamValue("UEProjectRoot");
if (UEProjectRoot == null)
{
UEProjectRoot = CmdEnv.LocalRoot;
}
var UEProjectDirectory = ParseParamValue("UEProjectDirectory");
if (UEProjectDirectory == null)
{
throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'");
}
var UEProjectName = ParseParamValue("UEProjectName");
if (UEProjectName == null)
{
UEProjectName = "";
}
var LocalizationProjectNames = new List <string>();
{
var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames");
if (LocalizationProjectNamesStr != null)
{
foreach (var ProjectName in LocalizationProjectNamesStr.Split(','))
{
LocalizationProjectNames.Add(ProjectName.Trim());
}
}
}
var LocalizationProviderName = ParseParamValue("LocalizationProvider");
if (LocalizationProviderName == null)
{
LocalizationProviderName = "";
}
var LocalizationStepNames = new List <string>();
{
var LocalizationStepNamesStr = ParseParamValue("LocalizationSteps");
if (LocalizationStepNamesStr == null)
{
LocalizationStepNames.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" });
}
else
{
foreach (var StepName in LocalizationStepNamesStr.Split(','))
{
LocalizationStepNames.Add(StepName.Trim());
}
}
LocalizationStepNames.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do
}
var ShouldGatherPlugins = ParseParam("IncludePlugins");
var IncludePlugins = new List <string>();
var ExcludePlugins = new List <string>();
if (ShouldGatherPlugins)
{
var IncludePluginsStr = ParseParamValue("IncludePlugins");
if (IncludePluginsStr != null)
{
foreach (var PluginName in IncludePluginsStr.Split(','))
{
IncludePlugins.Add(PluginName.Trim());
}
}
var ExcludePluginsStr = ParseParamValue("ExcludePlugins");
if (ExcludePluginsStr != null)
{
foreach (var PluginName in ExcludePluginsStr.Split(','))
{
ExcludePlugins.Add(PluginName.Trim());
}
}
}
var ShouldGatherPlatforms = ParseParam("IncludePlatforms");
var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments");
if (AdditionalCommandletArguments == null)
{
AdditionalCommandletArguments = "";
}
var EnableParallelGather = ParseParam("ParallelGather");
var StartTime = DateTime.UtcNow;
var LocalizationBatches = new List <LocalizationBatch>();
// Add the static set of localization projects as a batch
if (LocalizationProjectNames.Count > 0)
{
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames));
}
// Build up any additional batches needed for platforms
if (ShouldGatherPlatforms)
{
var PlatformsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory, "Platforms"));
if (DirectoryReference.Exists(PlatformsRootDirectory))
{
foreach (DirectoryReference PlatformDirectory in DirectoryReference.EnumerateDirectories(PlatformsRootDirectory))
{
// Find the localization targets defined for this platform
var PlatformTargetNames = GetLocalizationTargetsFromDirectory(new DirectoryReference(CombinePaths(PlatformDirectory.FullName, "Config", "Localization")));
if (PlatformTargetNames.Count > 0)
{
var RootRelativePluginPath = PlatformDirectory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, "", PlatformTargetNames));
}
}
}
}
// Build up any additional batches needed for plugins
if (ShouldGatherPlugins)
{
var PluginsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory));
IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(PluginsRootDirectory, "Plugins", UEProjectName.Length == 0 ? PluginType.Engine : PluginType.Project);
// Add a batch for each plugin that meets our criteria
var AvailablePluginNames = new HashSet <string>();
foreach (var PluginInfo in AllPlugins)
{
AvailablePluginNames.Add(PluginInfo.Name);
bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name);
if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0)
{
var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
var PluginTargetNames = new List <string>();
foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets)
{
PluginTargetNames.Add(LocalizationTarget.Name);
}
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames));
}
}
// If we had an explicit list of plugins to include, warn if any were missing
foreach (string PluginName in IncludePlugins)
{
if (!AvailablePluginNames.Contains(PluginName))
{
LogWarning("The plugin '{0}' specified by -IncludePlugins wasn't found and will be skipped.", PluginName);
}
}
}
// Create a single changelist to use for all changes
int PendingChangeList = 0;
if (P4Enabled)
{
var ChangeListCommitMessage = String.Format("Localization Automation using CL {0}", P4Env.Changelist);
if (File.Exists(CombinePaths(CmdEnv.LocalRoot, @"Engine/Restricted/NotForLicensees/Build/EpicInternal.txt")))
{
ChangeListCommitMessage += "\n#okforgithub ignore";
}
PendingChangeList = P4.CreateChange(P4Env.Client, ChangeListCommitMessage);
}
// Prepare to process each localization batch
var LocalizationTasks = new List <LocalizationTask>();
foreach (var LocalizationBatch in LocalizationBatches)
{
var LocalizationTask = new LocalizationTask(LocalizationBatch, UEProjectRoot, LocalizationProviderName, PendingChangeList, this);
LocalizationTasks.Add(LocalizationTask);
// Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on
if (P4Enabled)
{
LogInformation("Sync necessary content to head revision");
P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Config/Localization/...");
P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Content/Localization/...");
}
// Generate the info we need to gather for each project
foreach (var ProjectName in LocalizationTask.Batch.LocalizationProjectNames)
{
LocalizationTask.ProjectInfos.Add(GenerateProjectInfo(LocalizationTask.RootLocalizationTargetDirectory, ProjectName, LocalizationStepNames));
}
}
// Hash the current PO files on disk so we can work out whether they actually change
Dictionary <string, byte[]> InitalPOFileHashes = null;
if (P4Enabled)
{
InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
}
// Download the latest translations from our localization provider
if (LocalizationStepNames.Contains("Download"))
{
foreach (var LocalizationTask in LocalizationTasks)
{
if (LocalizationTask.LocProvider != null)
{
foreach (var ProjectInfo in LocalizationTask.ProjectInfos)
{
LocalizationTask.LocProvider.DownloadProjectFromLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ImportInfo);
}
}
}
}
// Begin the gather command for each task
// These can run in parallel when ParallelGather is enabled
{
var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe");
// Set the common basic editor arguments
var EditorArguments = P4Enabled
? String.Format("-SCCProvider=Perforce -P4Port={0} -P4User={1} -P4Client={2} -P4Passwd={3} -P4Changelist={4} -EnableSCC -DisableSCCSubmit", P4Env.ServerAndPort, P4Env.User, P4Env.Client, P4.GetAuthenticationToken(), PendingChangeList)
: "-SCCProvider=None";
if (IsBuildMachine)
{
EditorArguments += " -BuildMachine";
}
EditorArguments += " -Unattended -LogLocalizationConflicts";
if (EnableParallelGather)
{
EditorArguments += " -multiprocess";
}
if (!String.IsNullOrEmpty(AdditionalCommandletArguments))
{
EditorArguments += " " + AdditionalCommandletArguments;
}
// Set the common process run options
var CommandletRunOptions = ERunOptions.Default | ERunOptions.NoLoggingOfRunCommand; // Disable logging of the run command as it will print the exit code which GUBP can pick up as an error (we do that ourselves later)
if (EnableParallelGather)
{
CommandletRunOptions |= ERunOptions.NoWaitForExit;
}
foreach (var LocalizationTask in LocalizationTasks)
{
var ProjectArgument = String.IsNullOrEmpty(UEProjectName) ? "" : String.Format("\"{0}\"", Path.Combine(LocalizationTask.RootWorkingDirectory, String.Format("{0}.uproject", UEProjectName)));
foreach (var ProjectInfo in LocalizationTask.ProjectInfos)
{
var LocalizationConfigFiles = new List <string>();
foreach (var LocalizationStep in ProjectInfo.LocalizationSteps)
{
if (LocalizationStepNames.Contains(LocalizationStep.Name))
{
LocalizationConfigFiles.Add(LocalizationStep.LocalizationConfigFile);
}
}
if (LocalizationConfigFiles.Count > 0)
{
var Arguments = String.Format("{0} -run=GatherText -config=\"{1}\" {2}", ProjectArgument, String.Join(";", LocalizationConfigFiles), EditorArguments);
LogInformation("Running localization commandlet for '{0}': {1}", ProjectInfo.ProjectName, Arguments);
LocalizationTask.GatherProcessResults.Add(Run(EditorExe, Arguments, null, CommandletRunOptions));
}
else
{
LocalizationTask.GatherProcessResults.Add(null);
}
}
}
}
// Wait for each commandlet process to finish and report the result.
// This runs even for non-parallel execution to log the exit state of the process.
foreach (var LocalizationTask in LocalizationTasks)
{
for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex)
{
var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex];
var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex];
if (RunResult != null)
{
RunResult.WaitForExit();
RunResult.OnProcessExited();
RunResult.DisposeProcess();
if (RunResult.ExitCode == 0)
{
LogInformation("The localization commandlet for '{0}' exited with code 0.", ProjectInfo.ProjectName);
}
else
{
LogWarning("The localization commandlet for '{0}' exited with code {1} which likely indicates a crash.", ProjectInfo.ProjectName, RunResult.ExitCode);
}
}
}
}
// Upload the latest sources to our localization provider
if (LocalizationStepNames.Contains("Upload"))
{
foreach (var LocalizationTask in LocalizationTasks)
{
if (LocalizationTask.LocProvider != null)
{
// Upload all text to our localization provider
for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex)
{
var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex];
var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex];
if (RunResult != null && RunResult.ExitCode == 0)
{
// Recalculate the split platform paths before doing the upload, as the export may have changed them
ProjectInfo.ExportInfo.CalculateSplitPlatformNames(LocalizationTask.RootLocalizationTargetDirectory);
LocalizationTask.LocProvider.UploadProjectToLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ExportInfo);
}
else
{
LogWarning("Skipping upload to the localization provider for '{0}' due to an earlier commandlet failure.", ProjectInfo.ProjectName);
}
}
}
}
}
// Clean-up the changelist so it only contains the changed files, and then submit it (if we were asked to)
if (P4Enabled)
{
// Revert any PO files that haven't changed aside from their header
{
var POFilesToRevert = new List <string>();
var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
foreach (var CurrentPOFileHashPair in CurrentPOFileHashes)
{
byte[] InitialPOFileHash;
if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value))
{
POFilesToRevert.Add(CurrentPOFileHashPair.Key);
}
}
if (POFilesToRevert.Count > 0)
{
var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString()));
using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename))
{
foreach (var POFileToRevert in POFilesToRevert)
{
P4RevertArgsWriter.WriteLine(POFileToRevert);
}
}
P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename));
DeleteFile_NoExceptions(P4RevertArgsFilename);
}
}
// Revert any other unchanged files
P4.RevertUnchanged(PendingChangeList);
// Submit that single changelist now
if (AllowSubmit)
{
int SubmittedChangeList;
P4.Submit(PendingChangeList, out SubmittedChangeList);
}
}
var RunDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds;
LogInformation("Localize command finished in {0} seconds", RunDuration / 1000);
}
public override void ExecuteBuild()
{
var UEProjectRoot = ParseParamValue("UEProjectRoot");
if (UEProjectRoot == null)
{
UEProjectRoot = CmdEnv.LocalRoot;
}
var UEProjectDirectory = ParseParamValue("UEProjectDirectory");
if (UEProjectDirectory == null)
{
throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'");
}
var UEProjectName = ParseParamValue("UEProjectName");
if (UEProjectName == null)
{
UEProjectName = "";
}
var LocalizationProjectNames = new List <string>();
{
var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames");
if (LocalizationProjectNamesStr != null)
{
foreach (var ProjectName in LocalizationProjectNamesStr.Split(','))
{
LocalizationProjectNames.Add(ProjectName.Trim());
}
}
}
var LocalizationProviderName = ParseParamValue("LocalizationProvider");
if (LocalizationProviderName == null)
{
LocalizationProviderName = "";
}
var LocalizationStepNames = new List <string>();
{
var LocalizationStepNamesStr = ParseParamValue("LocalizationSteps");
if (LocalizationStepNamesStr == null)
{
LocalizationStepNames.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" });
}
else
{
foreach (var StepName in LocalizationStepNamesStr.Split(','))
{
LocalizationStepNames.Add(StepName.Trim());
}
}
LocalizationStepNames.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do
}
var ShouldGatherPlugins = ParseParam("IncludePlugins");
var IncludePlugins = new List <string>();
var ExcludePlugins = new List <string>();
if (ShouldGatherPlugins)
{
var IncludePluginsStr = ParseParamValue("IncludePlugins");
if (IncludePluginsStr != null)
{
foreach (var PluginName in IncludePluginsStr.Split(','))
{
IncludePlugins.Add(PluginName.Trim());
}
}
var ExcludePluginsStr = ParseParamValue("ExcludePlugins");
if (ExcludePluginsStr != null)
{
foreach (var PluginName in ExcludePluginsStr.Split(','))
{
ExcludePlugins.Add(PluginName.Trim());
}
}
}
var ShouldGatherPlatforms = ParseParam("IncludePlatforms");
var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments");
if (AdditionalCommandletArguments == null)
{
AdditionalCommandletArguments = "";
}
var LocalizationBatches = new List <LocalizationBatch>();
// Add the static set of localization projects as a batch
if (LocalizationProjectNames.Count > 0)
{
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames));
}
// Build up any additional batches needed for platforms
if (ShouldGatherPlatforms)
{
var PlatformsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory, "Platforms"));
if (DirectoryReference.Exists(PlatformsRootDirectory))
{
foreach (DirectoryReference PlatformDirectory in DirectoryReference.EnumerateDirectories(PlatformsRootDirectory))
{
// Find the localization targets defined for this platform
var PlatformTargetNames = GetLocalizationTargetsFromDirectory(new DirectoryReference(CombinePaths(PlatformDirectory.FullName, "Config", "Localization")));
if (PlatformTargetNames.Count > 0)
{
var RootRelativePluginPath = PlatformDirectory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, "", PlatformTargetNames));
}
}
}
}
// Build up any additional batches needed for plugins
if (ShouldGatherPlugins)
{
var PluginsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory));
IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(PluginsRootDirectory, "Plugins", UEProjectName.Length == 0 ? PluginType.Engine : PluginType.Project);
// Add a batch for each plugin that meets our criteria
var AvailablePluginNames = new HashSet <string>();
foreach (var PluginInfo in AllPlugins)
{
AvailablePluginNames.Add(PluginInfo.Name);
bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name);
if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0)
{
var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
var PluginTargetNames = new List <string>();
foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets)
{
PluginTargetNames.Add(LocalizationTarget.Name);
}
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames));
}
}
// If we had an explicit list of plugins to include, warn if any were missing
foreach (string PluginName in IncludePlugins)
{
if (!AvailablePluginNames.Contains(PluginName))
{
LogWarning("The plugin '{0}' specified by -IncludePlugins wasn't found and will be skipped.", PluginName);
}
}
}
// Create a single changelist to use for all changes, and hash the current PO files on disk so we can work out whether they actually change
int PendingChangeList = 0;
Dictionary <string, byte[]> InitalPOFileHashes = null;
if (P4Enabled)
{
var ChangeListCommitMessage = "Localization Automation";
if (File.Exists(CombinePaths(CmdEnv.LocalRoot, @"Engine/Build/NotForLicensees/EpicInternal.txt")))
{
ChangeListCommitMessage += "\n#okforgithub ignore";
}
PendingChangeList = P4.CreateChange(P4Env.Client, ChangeListCommitMessage);
InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
}
// Process each localization batch
foreach (var LocalizationBatch in LocalizationBatches)
{
ProcessLocalizationProjects(LocalizationBatch, PendingChangeList, UEProjectRoot, UEProjectName, LocalizationProviderName, LocalizationStepNames, AdditionalCommandletArguments);
}
// Clean-up the changelist so it only contains the changed files, and then submit it (if we were asked to)
if (P4Enabled)
{
// Revert any PO files that haven't changed aside from their header
{
var POFilesToRevert = new List <string>();
var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
foreach (var CurrentPOFileHashPair in CurrentPOFileHashes)
{
byte[] InitialPOFileHash;
if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value))
{
POFilesToRevert.Add(CurrentPOFileHashPair.Key);
}
}
if (POFilesToRevert.Count > 0)
{
var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString()));
using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename))
{
foreach (var POFileToRevert in POFilesToRevert)
{
P4RevertArgsWriter.WriteLine(POFileToRevert);
}
}
P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename));
DeleteFile_NoExceptions(P4RevertArgsFilename);
}
}
// Revert any other unchanged files
P4.RevertUnchanged(PendingChangeList);
// Submit that single changelist now
if (AllowSubmit)
{
int SubmittedChangeList;
P4.Submit(PendingChangeList, out SubmittedChangeList);
}
}
}
public IPFSShell(PlatformDirectory platformDirectory)
{
this.platformDirectory = platformDirectory;
}
