public static async Task RunAsync( IInstallationTokenProvider installationTokenProvider, CompressImagesMessage compressImagesMessage, ICollector <OpenPrMessage> openPrMessages, IRepoChecks repoChecks, ILogger logger, ExecutionContext context) { logger.LogInformation("CompressImagesFunction: starting run for {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName); var installationTokenParameters = new InstallationTokenParameters { AccessTokensUrl = string.Format(KnownGitHubs.AccessTokensUrlFormat, compressImagesMessage.InstallationId), AppId = KnownGitHubs.AppId, }; var installationToken = await installationTokenProvider.GenerateAsync( installationTokenParameters, File.OpenText(Path.Combine(context.FunctionDirectory, $"../{KnownGitHubs.AppPrivateKey}"))); // check if repo is archived before starting work var isArchived = await repoChecks.IsArchived(new GitHubClientParameters { Password = installationToken.Token, RepoName = compressImagesMessage.RepoName, RepoOwner = compressImagesMessage.Owner }); if (isArchived) { logger.LogInformation("CompressImagesFunction: skipping archived repo {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName); return; } var compressImagesParameters = new CompressimagesParameters { CloneUrl = compressImagesMessage.CloneUrl, LocalPath = LocalPath.CloneDir(Environment.GetEnvironmentVariable("TMP") ?? "/private/tmp/", compressImagesMessage.RepoName), Password = installationToken.Token, RepoName = compressImagesMessage.RepoName, RepoOwner = compressImagesMessage.Owner, PgpPrivateKeyStream = File.OpenRead(Path.Combine(context.FunctionDirectory, $"../{KnownGitHubs.PGPPrivateKeyFilename}")), PgPPassword = File.ReadAllText(Path.Combine(context.FunctionDirectory, $"../{KnownGitHubs.PGPPasswordFilename}")), CompressImagesMessage = compressImagesMessage, }; var didCompress = await CompressImages.RunAsync(compressImagesParameters, logger); if (didCompress) { logger.LogInformation("CompressImagesFunction: Successfully compressed images for {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName); openPrMessages.Add(new OpenPrMessage { InstallationId = compressImagesMessage.InstallationId, RepoName = compressImagesMessage.RepoName, CloneUrl = compressImagesMessage.CloneUrl, }); } logger.LogInformation("CompressImagesFunction: finished run for {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName); }
public static async Task Run( [QueueTrigger("compressimagesmessage")] CompressImagesMessage compressImagesMessage, [Queue("openprmessage")] ICollector <OpenPrMessage> openPrMessages, ILogger logger, ExecutionContext context) { logger.LogInformation("CompressImagesFunction: starting run for {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName); var installationTokenParameters = new InstallationTokenParameters { AccessTokensUrl = string.Format(KnownGitHubs.AccessTokensUrlFormat, compressImagesMessage.InstallationId), AppId = KnownGitHubs.AppId, }; var installationToken = await InstallationToken.GenerateAsync( installationTokenParameters, File.OpenText(Path.Combine(context.FunctionDirectory, $"../{KnownGitHubs.AppPrivateKey}"))); var compressImagesParameters = new CompressimagesParameters { CloneUrl = compressImagesMessage.CloneUrl, LocalPath = LocalPath.CloneDir(Environment.GetEnvironmentVariable("TMP") ?? "/private/tmp/", compressImagesMessage.RepoName), Password = installationToken.Token, RepoName = compressImagesMessage.RepoName, RepoOwner = compressImagesMessage.Owner, PgpPrivateKeyStream = File.OpenRead(Path.Combine(context.FunctionDirectory, $"../{KnownGitHubs.PGPPrivateKeyFilename}")), PgPPassword = File.ReadAllText(Path.Combine(context.FunctionDirectory, $"../{KnownGitHubs.PGPPasswordFilename}")) }; var didCompress = CompressImages.Run(compressImagesParameters, logger); if (didCompress) { logger.LogInformation("CompressImagesFunction: Successfully compressed images for {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName); openPrMessages.Add(new OpenPrMessage { InstallationId = compressImagesMessage.InstallationId, RepoName = compressImagesMessage.RepoName, CloneUrl = compressImagesMessage.CloneUrl, }); } logger.LogInformation("CompressImagesFunction: finished run for {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName); }
public static async Task Run( [QueueTrigger("compressimagesmessage")] CompressImagesMessage compressImagesMessage, [Queue("openprmessage")] ICollector <OpenPrMessage> openPrMessages, TraceWriter log, ExecutionContext context) { var installationTokenParameters = new InstallationTokenParameters { AccessTokensUrl = compressImagesMessage.AccessTokensUrl, AppId = KnownGitHubs.AppId, }; var installationToken = await InstallationToken.GenerateAsync( installationTokenParameters, File.OpenText(Path.Combine(context.FunctionDirectory, $"../{KnownGitHubs.AppPrivateKey}"))); var compressImagesParameters = new CompressimagesParameters { CloneUrl = compressImagesMessage.CloneUrl, LocalPath = LocalPath.CloneDir(Environment.GetEnvironmentVariable("TMP") ?? "/private/tmp/", compressImagesMessage.RepoName), Password = installationToken.Token, RepoName = compressImagesMessage.RepoName, RepoOwner = compressImagesMessage.Owner, PgpPrivateKeyStream = File.OpenRead(Path.Combine(context.FunctionDirectory, $"../{KnownGitHubs.PGPPrivateKeyFilename}")), PgPPassword = File.ReadAllText(Path.Combine(context.FunctionDirectory, $"../{KnownGitHubs.PGPPasswordFilename}")) }; var didCompress = CompressImages.Run(compressImagesParameters); if (didCompress) { log.Info("Compressed images; Route to OpenPR"); openPrMessages.Add(new OpenPrMessage { InstallationId = compressImagesMessage.InstallationId, RepoName = compressImagesMessage.RepoName, CloneUrl = compressImagesMessage.CloneUrl, }); } log.Info("Completed run"); }
public static async Task <bool> RunAsync(CompressimagesParameters parameters, ILogger logger) { CredentialsHandler credentialsProvider = (url, user, cred) => new UsernamePasswordCredentials { Username = KnownGitHubs.Username, Password = parameters.Password }; // clone var cloneOptions = new CloneOptions { CredentialsProvider = credentialsProvider, }; try { Repository.Clone(parameters.CloneUrl, parameters.LocalPath, cloneOptions); } catch (Exception) { // issue cloning, let's try to queue to the linux box to see if it can clone // linux support is in preview github.com/Azure/Azure-Functions/wiki/Azure-Functions-on-Linux-Preview var linuxConnectionString = Environment.GetEnvironmentVariable("linux-connectionstring"); if (string.IsNullOrEmpty(linuxConnectionString) == false && parameters.CompressImagesMessage != null) { logger.LogInformation("bouncing to linux queue"); var linuxQueue = CloudStorageAccount .Parse(linuxConnectionString) .CreateCloudQueueClient() .GetQueueReference("compressimagesmessage"); await linuxQueue.CreateIfNotExistsAsync(); await linuxQueue.AddMessageAsync(new CloudQueueMessage(JsonConvert.SerializeObject(parameters.CompressImagesMessage))); return(false); } throw; } var repo = new Repository(parameters.LocalPath); var remote = repo.Network.Remotes["origin"]; // check if we have the branch already or this is empty repo try { if (repo.Network.ListReferences(remote, credentialsProvider).Any() == false) { logger.LogInformation("CompressImagesFunction: no references found for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); return(false); } if (repo.Network.ListReferences(remote, credentialsProvider).Any(x => x.CanonicalName == $"refs/heads/{KnownGitHubs.BranchName}")) { logger.LogInformation("CompressImagesFunction: branch already exists for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); return(false); } } catch (Exception e) { // log + ignore logger.LogWarning(e, "CompressImagesFunction: issue checking for existing branch or empty repo for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); } var repoConfiguration = new RepoConfiguration(); try { // see if .imgbotconfig exists in repo root var repoConfigJson = File.ReadAllText(parameters.LocalPath + Path.DirectorySeparatorChar + ".imgbotconfig"); if (!string.IsNullOrEmpty(repoConfigJson)) { repoConfiguration = JsonConvert.DeserializeObject <RepoConfiguration>(repoConfigJson); } } catch { // ignore } if (Schedule.ShouldOptimizeImages(repoConfiguration, repo) == false) { logger.LogInformation("CompressImagesFunction: skipping optimization due to schedule for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); return(false); } // check out the branch repo.CreateBranch(KnownGitHubs.BranchName); var branch = Commands.Checkout(repo, KnownGitHubs.BranchName); // reset any mean files repo.Reset(ResetMode.Mixed, repo.Head.Tip); // optimize images var imagePaths = ImageQuery.FindImages(parameters.LocalPath, repoConfiguration); var optimizedImages = OptimizeImages(repo, parameters.LocalPath, imagePaths, logger, repoConfiguration.AggressiveCompression); if (optimizedImages.Length == 0) { return(false); } // create commit message based on optimizations foreach (var image in optimizedImages) { Commands.Stage(repo, image.OriginalPath); } var commitMessage = CommitMessage.Create(optimizedImages); var signature = new Signature(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail, DateTimeOffset.Now); repo.Commit(commitMessage, signature, signature); // We just made a normal commit, now we are going to capture all the values generated from that commit // then rewind and make a signed commit var commitBuffer = Commit.CreateBuffer( repo.Head.Tip.Author, repo.Head.Tip.Committer, repo.Head.Tip.Message, repo.Head.Tip.Tree, repo.Head.Tip.Parents, true, null); var signedCommitData = CommitSignature.Sign(commitBuffer + "\n", parameters.PgpPrivateKeyStream, parameters.PgPPassword); repo.Reset(ResetMode.Soft, repo.Head.Commits.Skip(1).First().Sha); var commitToKeep = repo.ObjectDatabase.CreateCommitWithSignature(commitBuffer, signedCommitData); repo.Refs.UpdateTarget(repo.Refs.Head, commitToKeep); var branchAgain = Commands.Checkout(repo, KnownGitHubs.BranchName); repo.Reset(ResetMode.Hard, commitToKeep.Sha); // verify images are not corrupted by reading from git // see https://github.com/dabutvin/ImgBot/issues/273 try { foreach (var image in optimizedImages) { new MagickImage(image.OriginalPath).Dispose(); } } catch (MagickErrorException) { logger.LogError("Corrupt images after reset!"); return(false); } // push to GitHub repo.Network.Push(remote, $"refs/heads/{KnownGitHubs.BranchName}", new PushOptions { CredentialsProvider = credentialsProvider, }); return(true); }
public static async Task RunAsync( IInstallationTokenProvider installationTokenProvider, CompressImagesMessage compressImagesMessage, ICollector <OpenPrMessage> openPrMessages, IRepoChecks repoChecks, ILogger logger, ExecutionContext context) { logger.LogInformation("CompressImagesFunction: starting run for {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName); var installationTokenParameters = new InstallationTokenParameters { AccessTokensUrl = string.Format(KnownGitHubs.AccessTokensUrlFormat, compressImagesMessage.InstallationId), AppId = KnownGitHubs.AppId, }; var installationToken = await installationTokenProvider.GenerateAsync( installationTokenParameters, KnownEnvironmentVariables.APP_PRIVATE_KEY); // check if repo is archived before starting work var isArchived = await repoChecks.IsArchived(new GitHubClientParameters { Password = installationToken.Token, RepoName = compressImagesMessage.RepoName, RepoOwner = compressImagesMessage.Owner }); if (isArchived) { logger.LogInformation("CompressImagesFunction: skipping archived repo {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName); return; } // check if imgbot branch already exists before starting work var branchExists = await repoChecks.BranchExists(new GitHubClientParameters { Password = installationToken.Token, RepoName = compressImagesMessage.RepoName, RepoOwner = compressImagesMessage.Owner, }); if (branchExists) { logger.LogInformation("CompressImagesFunction: skipping repo {Owner}/{RepoName} as branch exists", compressImagesMessage.Owner, compressImagesMessage.RepoName); return; } var compressImagesParameters = new CompressimagesParameters { CloneUrl = compressImagesMessage.CloneUrl, LocalPath = LocalPath.CloneDir(KnownEnvironmentVariables.TMP ?? "/private/tmp/", compressImagesMessage.RepoName), Password = installationToken.Token, RepoName = compressImagesMessage.RepoName, RepoOwner = compressImagesMessage.Owner, PgpPrivateKey = KnownEnvironmentVariables.PGP_PRIVATE_KEY, PgPPassword = KnownEnvironmentVariables.PGP_PASSWORD, CompressImagesMessage = compressImagesMessage, }; var didCompress = CompressImages.Run(compressImagesParameters, logger); if (didCompress) { logger.LogInformation("CompressImagesFunction: Successfully compressed images for {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName); openPrMessages.Add(new OpenPrMessage { InstallationId = compressImagesMessage.InstallationId, RepoName = compressImagesMessage.RepoName, CloneUrl = compressImagesMessage.CloneUrl, }); } logger.LogInformation("CompressImagesFunction: finished run for {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName); }
public static bool Run(CompressimagesParameters parameters, ICollector <CompressImagesMessage> compressImagesMessages, ILogger logger) { CredentialsHandler credentialsProvider = (url, user, cred) => new UsernamePasswordCredentials { Username = KnownGitHubs.Username, Password = parameters.Password }; // clone var cloneOptions = new CloneOptions { CredentialsProvider = credentialsProvider, }; Repository.Clone(parameters.CloneUrl, parameters.LocalPath, cloneOptions); var repo = new Repository(parameters.LocalPath); var remote = repo.Network.Remotes["origin"]; var isWikiCompress = parameters.CloneUrl.EndsWith(".wiki.git"); // check if we have the branch already or this is empty repo try { if (repo.Network.ListReferences(remote, credentialsProvider).Any() == false) { logger.LogInformation("CompressImagesFunction: no references found for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); return(false); } if (repo.Network.ListReferences(remote, credentialsProvider).Any(x => x.CanonicalName == $"refs/heads/{KnownGitHubs.BranchName}")) { logger.LogInformation("CompressImagesFunction: branch already exists for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); return(false); } } catch (Exception e) { // log + ignore logger.LogWarning(e, "CompressImagesFunction: issue checking for existing branch or empty repo for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); } // check if we should switch away from the default branch if (!isWikiCompress && parameters.Settings != null && !string.IsNullOrEmpty(parameters.Settings.DefaultBranchOverride)) { logger.LogInformation( "CompressImagesFunction: default branch override for {Owner}/{RepoName} is {DefaultBranchOverride}", parameters.RepoOwner, parameters.RepoName, parameters.Settings.DefaultBranchOverride); var baseBranch = repo.Branches[$"refs/remotes/origin/{parameters.Settings.DefaultBranchOverride}"]; if (baseBranch == null) { logger.LogWarning( "CompressImagesFunction: default branch ({DefaultBranchOverride}) not found for {Owner}/{RepoName}", parameters.Settings.DefaultBranchOverride, parameters.RepoOwner, parameters.RepoName); return(false); } Commands.Checkout(repo, baseBranch); } var repoConfiguration = new RepoConfiguration(); try { // see if .imgbotconfig exists in repo root var repoConfigJson = File.ReadAllText(parameters.LocalPath + Path.DirectorySeparatorChar + ".imgbotconfig"); if (!string.IsNullOrEmpty(repoConfigJson)) { repoConfiguration = JsonConvert.DeserializeObject <RepoConfiguration>(repoConfigJson); } } catch { // ignore } // Add new compressMessage if we should compress Wiki if (repoConfiguration.CompressWiki && isWikiCompress == false) { logger.LogInformation("CompressImagesFunction: Adding Wiki image compression to queue for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); compressImagesMessages.Add(new CompressImagesMessage() { InstallationId = parameters.CompressImagesMessage.InstallationId, RepoName = parameters.CompressImagesMessage.RepoName, Owner = parameters.RepoOwner, CloneUrl = $"https://github.com/{parameters.RepoOwner}/{parameters.RepoName}.wiki.git" }); } if (Schedule.ShouldOptimizeImages(repoConfiguration, repo) == false) { logger.LogInformation("CompressImagesFunction: skipping optimization due to schedule for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); return(false); } // Should not create branch if we are compressing Wiki if (isWikiCompress == false) { // check out the branch repo.CreateBranch(KnownGitHubs.BranchName); var branch = Commands.Checkout(repo, KnownGitHubs.BranchName); } // reset any mean files repo.Reset(ResetMode.Mixed, repo.Head.Tip); // optimize images var imagePaths = ImageQuery.FindImages(parameters.LocalPath, repoConfiguration); var optimizedImages = OptimizeImages(repo, parameters.LocalPath, imagePaths, logger, repoConfiguration.AggressiveCompression); if (optimizedImages.Length == 0) { return(false); } // create commit message based on optimizations foreach (var image in optimizedImages) { Commands.Stage(repo, image.OriginalPath); } var commitMessage = CommitMessage.Create(optimizedImages); var signature = new Signature(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail, DateTimeOffset.Now); repo.Commit(commitMessage, signature, signature); // We just made a normal commit, now we are going to capture all the values generated from that commit // then rewind and make a signed commit var commitBuffer = Commit.CreateBuffer( repo.Head.Tip.Author, repo.Head.Tip.Committer, repo.Head.Tip.Message, repo.Head.Tip.Tree, repo.Head.Tip.Parents, true, null); var signedCommitData = CommitSignature.Sign(commitBuffer + "\n", parameters.PgpPrivateKey, parameters.PgPPassword); repo.Reset(ResetMode.Soft, repo.Head.Commits.Skip(1).First().Sha); var commitToKeep = repo.ObjectDatabase.CreateCommitWithSignature(commitBuffer, signedCommitData); repo.Refs.UpdateTarget(repo.Refs.Head, commitToKeep); // Should use "master" if we are compressing Wiki if (isWikiCompress) { var branchAgain = Commands.Checkout(repo, "master"); } else { var branchAgain = Commands.Checkout(repo, KnownGitHubs.BranchName); } repo.Reset(ResetMode.Hard, commitToKeep.Sha); // verify images are not corrupted by reading from git // see https://github.com/dabutvin/ImgBot/issues/273 try { foreach (var image in optimizedImages) { if (image.OriginalPath.EndsWith(".svg")) { // do not use ImageMagick to verify SVGs continue; } new MagickImage(image.OriginalPath).Dispose(); } } catch (MagickErrorException) { logger.LogError("Corrupt images after reset!"); return(false); } // push to GitHub if (isWikiCompress) { repo.Network.Push(remote, "refs/heads/master", new PushOptions { CredentialsProvider = credentialsProvider, }); } else { repo.Network.Push(remote, $"refs/heads/{KnownGitHubs.BranchName}", new PushOptions { CredentialsProvider = credentialsProvider, }); } return(true); }
public static bool Run(CompressimagesParameters parameters) { CredentialsHandler credentialsProvider = (url, user, cred) => new UsernamePasswordCredentials { Username = KnownGitHubs.Username, Password = parameters.Password }; // clone var cloneOptions = new CloneOptions { CredentialsProvider = credentialsProvider, }; Repository.Clone(parameters.CloneUrl, parameters.LocalPath, cloneOptions); var repo = new Repository(parameters.LocalPath); var remote = repo.Network.Remotes["origin"]; // check if we have the branch already or this is empty repo try { if (repo.Network.ListReferences(remote, credentialsProvider).Any() == false) { return(false); } if (repo.Network.ListReferences(remote, credentialsProvider).Any(x => x.CanonicalName == $"refs/heads/{KnownGitHubs.BranchName}")) { return(false); } } catch { // ignore } var repoConfiguration = new RepoConfiguration(); try { // see if .imgbotconfig exists in repo root var repoConfigJson = File.ReadAllText(parameters.LocalPath + Path.DirectorySeparatorChar + ".imgbotconfig"); if (!string.IsNullOrEmpty(repoConfigJson)) { repoConfiguration = JsonConvert.DeserializeObject <RepoConfiguration>(repoConfigJson); } } catch { // ignore } if (Schedule.ShouldOptimizeImages(repoConfiguration, repo) == false) { return(false); } // check out the branch repo.CreateBranch(KnownGitHubs.BranchName); var branch = Commands.Checkout(repo, KnownGitHubs.BranchName); // reset any mean files repo.Reset(ResetMode.Mixed, repo.Head.Tip); // optimize images var imagePaths = ImageQuery.FindImages(parameters.LocalPath, repoConfiguration); var optimizedImages = OptimizeImages(repo, parameters.LocalPath, imagePaths); if (optimizedImages.Length == 0) { return(false); } // create commit message based on optimizations var commitMessage = CommitMessage.Create(optimizedImages); // commit var signature = new Signature(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail, DateTimeOffset.Now); repo.Commit(commitMessage, signature, signature); // We just made a normal commit, now we are going to capture all the values generated from that commit // then rewind and make a signed commit var commitBuffer = Commit.CreateBuffer( repo.Head.Tip.Author, repo.Head.Tip.Committer, repo.Head.Tip.Message, repo.Head.Tip.Tree, repo.Head.Tip.Parents, true, null); var signedCommitData = CommitSignature.Sign(commitBuffer + "\n", parameters.PgpPrivateKeyStream, parameters.PgPPassword); repo.Reset(ResetMode.Soft, repo.Head.Commits.Skip(1).First().Sha); var commitToKeep = repo.ObjectDatabase.CreateCommitWithSignature(commitBuffer, signedCommitData); repo.Refs.UpdateTarget(repo.Refs.Head, commitToKeep); var branchAgain = Commands.Checkout(repo, KnownGitHubs.BranchName); repo.Reset(ResetMode.Hard, commitToKeep.Sha); // push to GitHub repo.Network.Push(remote, $"refs/heads/{KnownGitHubs.BranchName}", new PushOptions { CredentialsProvider = credentialsProvider, }); return(true); }
public static bool Run(CompressimagesParameters parameters, ICollector <CompressImagesMessage> compressImagesMessages, ILogger logger) { CredentialsHandler credentialsProvider = (url, user, cred) => new UsernamePasswordCredentials { Username = KnownGitHubs.Username, Password = parameters.Password }; // clone var cloneOptions = new CloneOptions { CredentialsProvider = credentialsProvider, }; Repository.Clone(parameters.CloneUrl, parameters.LocalPath, cloneOptions); var repo = new Repository(parameters.LocalPath); var remote = repo.Network.Remotes["origin"]; var isWikiCompress = parameters.CloneUrl.EndsWith(".wiki.git"); // check if we have the branch already or this is empty repo try { if (repo.Network.ListReferences(remote, credentialsProvider).Any() == false) { logger.LogInformation("CompressImagesFunction: no references found for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); return(false); } if (!parameters.IsRebase && repo.Network.ListReferences(remote, credentialsProvider).Any(x => x.CanonicalName == $"refs/heads/{KnownGitHubs.BranchName}")) { logger.LogInformation("CompressImagesFunction: branch already exists for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); return(false); } } catch (Exception e) { // log + ignore logger.LogWarning(e, "CompressImagesFunction: issue checking for existing branch or empty repo for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); } // check if we should switch away from the default branch if (!isWikiCompress && parameters.Settings != null && !string.IsNullOrEmpty(parameters.Settings.DefaultBranchOverride)) { logger.LogInformation( "CompressImagesFunction: default branch override for {Owner}/{RepoName} is {DefaultBranchOverride}", parameters.RepoOwner, parameters.RepoName, parameters.Settings.DefaultBranchOverride); var baseBranch = repo.Branches[$"refs/remotes/origin/{parameters.Settings.DefaultBranchOverride}"]; if (baseBranch == null) { logger.LogWarning( "CompressImagesFunction: default branch ({DefaultBranchOverride}) not found for {Owner}/{RepoName}", parameters.Settings.DefaultBranchOverride, parameters.RepoOwner, parameters.RepoName); return(false); } Commands.Checkout(repo, baseBranch); } var repoConfiguration = new RepoConfiguration(); try { // see if .imgbotconfig exists in repo root var repoConfigJson = File.ReadAllText(parameters.LocalPath + Path.DirectorySeparatorChar + ".imgbotconfig"); if (!string.IsNullOrEmpty(repoConfigJson)) { repoConfiguration = JsonConvert.DeserializeObject <RepoConfiguration>(repoConfigJson); } } catch { // ignore } // Add new compressMessage if we should compress Wiki if (repoConfiguration.CompressWiki && isWikiCompress == false) { logger.LogInformation("CompressImagesFunction: Adding Wiki image compression to queue for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); compressImagesMessages.Add(new CompressImagesMessage() { InstallationId = parameters.CompressImagesMessage.InstallationId, RepoName = parameters.CompressImagesMessage.RepoName, Owner = parameters.RepoOwner, CloneUrl = $"https://github.com/{parameters.RepoOwner}/{parameters.RepoName}.wiki.git" }); } if (Schedule.ShouldOptimizeImages(repoConfiguration, repo) == false) { logger.LogInformation("CompressImagesFunction: skipping optimization due to schedule for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); return(false); } // Should not create branch if we are compressing Wiki or performing rebase if (isWikiCompress == false && !parameters.IsRebase) { // check out the branch repo.CreateBranch(KnownGitHubs.BranchName); var branch = Commands.Checkout(repo, KnownGitHubs.BranchName); } // reset any mean files repo.Reset(ResetMode.Mixed, repo.Head.Tip); // optimize images string[] imagePaths; List <string> addedOrModifiedImagePaths = new List <string>(); List <string> deletedImagePaths = new List <string>(); if (parameters.IsRebase) { var refspec = string.Format("{0}:{0}", KnownGitHubs.BranchName); Commands.Fetch(repo, "origin", new List <string> { refspec }, null, "fetch"); var diff = repo.Diff.Compare <TreeChanges>(repo.Branches[KnownGitHubs.BranchName].Commits.ElementAt(1).Tree, repo.Head.Tip.Tree); if (diff == null) { logger.LogInformation("Something went wrong while doing rebase"); return(false); } foreach (TreeEntryChanges c in diff) { if (KnownImgPatterns.ImgExtensions.Contains(Path.GetExtension(c.Path))) { var path = parameters.LocalPath + "/" + c.Path; var oldpath = parameters.LocalPath + "/" + c.OldPath; switch (c.Status) { case ChangeKind.Added: case ChangeKind.Modified: addedOrModifiedImagePaths.Add(path.Replace("\\", "/")); break; case ChangeKind.Renamed: addedOrModifiedImagePaths.Add(path.Replace("\\", "/")); deletedImagePaths.Add(oldpath.Replace("\\", "/")); break; case ChangeKind.Deleted: deletedImagePaths.Add(path.Replace("\\", "/")); break; } } } imagePaths = ImageQuery.FilterOutIgnoredFiles(addedOrModifiedImagePaths, repoConfiguration); } else { imagePaths = ImageQuery.FindImages(parameters.LocalPath, repoConfiguration); } var optimizedImages = OptimizeImages(repo, parameters.LocalPath, imagePaths, logger, repoConfiguration.AggressiveCompression); if (optimizedImages.Length == 0) { return(false); } // create commit message based on optimizations foreach (var image in optimizedImages) { Commands.Stage(repo, image.OriginalPath); } var commitMessage = CommitMessage.Create(optimizedImages); var signature = new Signature(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail, DateTimeOffset.Now); repo.Commit(commitMessage, signature, signature); if (parameters.IsRebase) { var baseBranch = repo.Head; var newCommit = baseBranch.Tip; var oldCommit = repo.Branches[KnownGitHubs.BranchName].Tip; // we need to reset the default branch so that we can // rebase to it later. repo.Reset(ResetMode.Hard, repo.Head.Commits.ElementAt(1)); // checkout to imgbot branch. TODO: remove because this is needed earlier on diff Commands.Checkout(repo, KnownGitHubs.BranchName); // cherry-pick var cherryPickOptions = new CherryPickOptions() { MergeFileFavor = MergeFileFavor.Theirs, }; var cherryPickResult = repo.CherryPick(newCommit, signature, cherryPickOptions); if (cherryPickResult.Status == CherryPickStatus.Conflicts) { var status = repo.RetrieveStatus(new LibGit2Sharp.StatusOptions() { }); foreach (var item in status) { if (item.State == FileStatus.Conflicted) { Commands.Stage(repo, item.FilePath); } } repo.Commit(commitMessage, signature, signature); } // New commit message creation var previousCommitResults = CompressionResult.ParseCommitMessage(oldCommit.Message); var mergedResults = CompressionResult.Merge(optimizedImages, previousCommitResults); var filteredResults = CompressionResult.Filter(mergedResults, deletedImagePaths.ToArray()); var squashCommitMessage = CommitMessage.Create(filteredResults); // squash var baseCommit = repo.Head.Commits.ElementAt(2); repo.Reset(ResetMode.Soft, baseCommit); repo.Commit(squashCommitMessage, signature, signature); // rebase var rebaseOptions = new RebaseOptions() { FileConflictStrategy = CheckoutFileConflictStrategy.Theirs, }; var rebaseResult = repo.Rebase.Start(repo.Head, baseBranch, null, new Identity(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail), rebaseOptions); while (rebaseResult.Status == RebaseStatus.Conflicts) { var status = repo.RetrieveStatus(new LibGit2Sharp.StatusOptions() { }); foreach (var item in status) { if (item.State == FileStatus.Conflicted) { if (imagePaths.Contains(parameters.LocalPath + "/" + item.FilePath)) { Commands.Stage(repo, item.FilePath); } else { Commands.Remove(repo, item.FilePath); } } } rebaseResult = repo.Rebase.Continue(new Identity(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail), rebaseOptions); } } // We just made a normal commit, now we are going to capture all the values generated from that commit // then rewind and make a signed commit var commitBuffer = Commit.CreateBuffer( repo.Head.Tip.Author, repo.Head.Tip.Committer, repo.Head.Tip.Message, repo.Head.Tip.Tree, repo.Head.Tip.Parents, true, null); var signedCommitData = CommitSignature.Sign(commitBuffer + "\n", parameters.PgpPrivateKey, parameters.PgPPassword); repo.Reset(ResetMode.Soft, repo.Head.Commits.Skip(1).First().Sha); var commitToKeep = repo.ObjectDatabase.CreateCommitWithSignature(commitBuffer, signedCommitData); repo.Refs.UpdateTarget(repo.Refs.Head, commitToKeep); // Should use "master" if we are compressing Wiki if (isWikiCompress) { var branchAgain = Commands.Checkout(repo, "master"); } else { var branchAgain = Commands.Checkout(repo, KnownGitHubs.BranchName); } repo.Reset(ResetMode.Hard, commitToKeep.Sha); // verify images are not corrupted by reading from git // see https://github.com/dabutvin/ImgBot/issues/273 try { foreach (var image in optimizedImages) { if (image.OriginalPath.EndsWith(".svg")) { // do not use ImageMagick to verify SVGs continue; } new MagickImage(image.OriginalPath).Dispose(); } } catch (MagickErrorException) { logger.LogError("Corrupt images after reset!"); return(false); } // push to GitHub if (isWikiCompress) { repo.Network.Push(remote, "refs/heads/master", new PushOptions { CredentialsProvider = credentialsProvider, }); } else { var refs = $"refs/heads/{KnownGitHubs.BranchName}"; if (parameters.IsRebase) { refs = refs.Insert(0, "+"); } logger.LogInformation("refs: {refs}", refs); repo.Network.Push(remote, refs, new PushOptions { CredentialsProvider = credentialsProvider, }); } return(true); }
public static async Task RunAsync( IInstallationTokenProvider installationTokenProvider, CompressImagesMessage compressImagesMessage, ICollector<OpenPrMessage> openPrMessages, ICollector<CompressImagesMessage> compressImagesMessages, CloudTable settingsTable, IRepoChecks repoChecks, ILogger logger, ExecutionContext context) { logger.LogInformation("CompressImagesFunction: starting run for {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName); var installationTokenParameters = new InstallationTokenParameters { AccessTokensUrl = string.Format(KnownGitHubs.AccessTokensUrlFormat, compressImagesMessage.InstallationId), AppId = KnownGitHubs.AppId, }; var installationToken = await installationTokenProvider.GenerateAsync( installationTokenParameters, KnownEnvironmentVariables.APP_PRIVATE_KEY); // check if repo is archived before starting work var isArchived = await repoChecks.IsArchived(new GitHubClientParameters { Password = installationToken.Token, RepoName = compressImagesMessage.RepoName, RepoOwner = compressImagesMessage.Owner }); if (isArchived) { logger.LogInformation("CompressImagesFunction: skipping archived repo {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName); return; } // check if imgbot branch already exists before starting work var branchExists = await repoChecks.BranchExists(new GitHubClientParameters { Password = installationToken.Token, RepoName = compressImagesMessage.RepoName, RepoOwner = compressImagesMessage.Owner, }); if (branchExists && !compressImagesMessage.IsRebase) { logger.LogInformation("CompressImagesFunction: skipping repo {Owner}/{RepoName} as branch exists", compressImagesMessage.Owner, compressImagesMessage.RepoName); return; } var compressImagesParameters = new CompressimagesParameters { CloneUrl = compressImagesMessage.CloneUrl, LocalPath = LocalPath.CloneDir(KnownEnvironmentVariables.TMP ?? "/private/tmp/", compressImagesMessage.RepoName), Password = installationToken.Token, RepoName = compressImagesMessage.RepoName, RepoOwner = compressImagesMessage.Owner, IsRebase = compressImagesMessage.IsRebase, PgpPrivateKey = KnownEnvironmentVariables.PGP_PRIVATE_KEY, PgPPassword = KnownEnvironmentVariables.PGP_PASSWORD, CompressImagesMessage = compressImagesMessage, Settings = await Common.TableModels.SettingsHelper.GetSettings(settingsTable, compressImagesMessage.InstallationId, compressImagesMessage.RepoName), }; var didCompress = CompressImages.Run(compressImagesParameters, compressImagesMessages, logger); if (didCompress && compressImagesParameters.CloneUrl.Contains(".wiki.git")) { logger.LogInformation("CompressImagesFunction: Successfully compressed images for {Owner}/{RepoName}/wiki", compressImagesMessage.Owner, compressImagesMessage.RepoName); } else if (didCompress) { var update = compressImagesMessage.IsRebase; logger.LogInformation("CompressImagesFunction: Successfully compressed images for {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName); openPrMessages.Add(new OpenPrMessage { InstallationId = compressImagesMessage.InstallationId, RepoName = compressImagesMessage.RepoName, CloneUrl = compressImagesMessage.CloneUrl, Update = compressImagesMessage.IsRebase, }); } try { Directory.Delete(compressImagesParameters.LocalPath, recursive: true); } catch (Exception exception) { logger.LogError(exception, "Error cleaning up local directory"); } logger.LogInformation("CompressImagesFunction: finished run for {Owner}/{RepoName}", compressImagesMessage.Owner, compressImagesMessage.RepoName); }
public static bool Run(CompressimagesParameters parameters, ICollector <CompressImagesMessage> compressImagesMessages, ILogger logger) { var storageAccount = CloudStorageAccount.Parse(Common.KnownEnvironmentVariables.AzureWebJobsStorage); var paidPlan = PaidPlan(storageAccount, parameters.RepoOwner); CredentialsHandler credentialsProvider = (url, user, cred) => new UsernamePasswordCredentials { Username = KnownGitHubs.Username, Password = parameters.Password }; // clone var cloneOptions = new CloneOptions { CredentialsProvider = credentialsProvider, }; Repository.Clone(parameters.CloneUrl, parameters.LocalPath, cloneOptions); var repo = new Repository(parameters.LocalPath); var remote = repo.Network.Remotes["origin"]; var isWikiCompress = parameters.CloneUrl.EndsWith(".wiki.git"); // check if we have the branch already or this is empty repo try { if (repo.Network.ListReferences(remote, credentialsProvider).Any() == false) { logger.LogInformation("CompressImagesFunction: no references found for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); return(false); } } catch (Exception e) { // log + ignore logger.LogWarning(e, "CompressImagesFunction: issue checking for existing branch or empty repo for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); } // check if the branch exists and has been modified by the user if (parameters.IsRebase && repo.Branches[$"refs/remotes/origin/{KnownGitHubs.BranchName}"].Tip.Author.Name != KnownGitHubs.ImgBotLogin) { logger.LogInformation("CompressImagesFunction: imgbot branch has been modified by the user."); return(false); } // check if we should switch away from the default branch if (!isWikiCompress && parameters.Settings != null && !string.IsNullOrEmpty(parameters.Settings.DefaultBranchOverride)) { logger.LogInformation( "CompressImagesFunction: default branch override for {Owner}/{RepoName} is {DefaultBranchOverride}", parameters.RepoOwner, parameters.RepoName, parameters.Settings.DefaultBranchOverride); var baseBranch = repo.Branches[$"refs/remotes/origin/{parameters.Settings.DefaultBranchOverride}"]; if (baseBranch == null) { logger.LogWarning( "CompressImagesFunction: default branch ({DefaultBranchOverride}) not found for {Owner}/{RepoName}", parameters.Settings.DefaultBranchOverride, parameters.RepoOwner, parameters.RepoName); return(false); } Commands.Checkout(repo, baseBranch); } var repoConfiguration = new RepoConfiguration(); try { // see if .imgbotconfig exists in repo root var repoConfigJson = File.ReadAllText(parameters.LocalPath + Path.DirectorySeparatorChar + ".imgbotconfig"); if (!string.IsNullOrEmpty(repoConfigJson)) { repoConfiguration = JsonConvert.DeserializeObject <RepoConfiguration>(repoConfigJson); // for now we are not adding the labels functionality || repoConfiguration.Labels.Any() TODO: add it when adding the labels feature if (paidPlan && (repoConfiguration.PrBody != null || repoConfiguration.PrTitle != null)) { var settingsTable = storageAccount.CreateCloudTableClient().GetTableReference("settings"); // Labels = repoConfiguration.Labels TODO: add it when adding the labels feature var settings = new Common.TableModels.Settings( parameters.CompressImagesMessage.InstallationId.ToString(), parameters.CompressImagesMessage.RepoName) { PrBody = repoConfiguration.PrBody, PrTitle = repoConfiguration.PrTitle, }; settingsTable.ExecuteAsync(TableOperation.InsertOrReplace(settings)).Wait(); } } } catch { // ignore } // Add new compressMessage if we should compress Wiki if (repoConfiguration.CompressWiki && isWikiCompress == false) { logger.LogInformation("CompressImagesFunction: Adding Wiki image compression to queue for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); compressImagesMessages.Add(new CompressImagesMessage() { InstallationId = parameters.CompressImagesMessage.InstallationId, RepoName = parameters.CompressImagesMessage.RepoName, Owner = parameters.RepoOwner, CloneUrl = $"https://github.com/{parameters.RepoOwner}/{parameters.RepoName}.wiki.git" }); } if (Schedule.ShouldOptimizeImages(repoConfiguration, repo) == false) { logger.LogInformation("CompressImagesFunction: skipping optimization due to schedule for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName); return(false); } // Should not create branch if we are compressing Wiki or performing rebase if (isWikiCompress == false && !parameters.IsRebase) { // check out the branch repo.CreateBranch(KnownGitHubs.BranchName); var branch = Commands.Checkout(repo, KnownGitHubs.BranchName); } else if (parameters.IsRebase) { // if rebasing, fetch the branch var refspec = string.Format("{0}:{0}", KnownGitHubs.BranchName); Commands.Fetch(repo, "origin", new List <string> { refspec }, null, "fetch"); } // reset any mean files repo.Reset(ResetMode.Mixed, repo.Head.Tip); // optimize images var imagePaths = ImageQuery.FindImages(parameters.LocalPath, repoConfiguration); var optimizedImages = OptimizeImages(repo, parameters.LocalPath, imagePaths, logger, repoConfiguration.AggressiveCompression); if (optimizedImages.Length == 0) { return(false); } if (!Threshold.MeetsThreshold(repoConfiguration, optimizedImages)) { logger.LogInformation($"Did not meet threshold. {parameters.RepoOwner}/{parameters.RepoName}"); return(false); } // create commit message based on optimizations foreach (var image in optimizedImages) { Commands.Stage(repo, image.OriginalPath); } var commitMessage = CommitMessage.Create(optimizedImages); var signature = new Signature(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail, DateTimeOffset.Now); repo.Commit(commitMessage, signature, signature); if (parameters.IsRebase) { var baseBranch = repo.Head; var newCommit = baseBranch.Tip; // we need to reset the default branch so that we can // rebase to it later. repo.Reset(ResetMode.Hard, repo.Head.Commits.ElementAt(1)); Commands.Checkout(repo, KnownGitHubs.BranchName); // reset imgbot branch removing old commit repo.Reset(ResetMode.Hard, repo.Head.Commits.ElementAt(1)); // cherry-pick var cherryPickOptions = new CherryPickOptions() { MergeFileFavor = MergeFileFavor.Theirs, }; var cherryPickResult = repo.CherryPick(newCommit, signature, cherryPickOptions); if (cherryPickResult.Status == CherryPickStatus.Conflicts) { var status = repo.RetrieveStatus(new LibGit2Sharp.StatusOptions() { }); foreach (var item in status) { if (item.State == FileStatus.Conflicted) { Commands.Stage(repo, item.FilePath); } } repo.Commit(commitMessage, signature, signature); } // rebase var rebaseOptions = new RebaseOptions() { FileConflictStrategy = CheckoutFileConflictStrategy.Theirs, }; var rebaseResult = repo.Rebase.Start(repo.Head, baseBranch, null, new Identity(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail), rebaseOptions); while (rebaseResult.Status == RebaseStatus.Conflicts) { var status = repo.RetrieveStatus(new LibGit2Sharp.StatusOptions() { }); foreach (var item in status) { if (item.State == FileStatus.Conflicted) { if (imagePaths.Contains(Path.Combine(parameters.LocalPath, item.FilePath))) { Commands.Stage(repo, item.FilePath); } else { Commands.Remove(repo, item.FilePath); } } } rebaseResult = repo.Rebase.Continue(new Identity(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail), rebaseOptions); } } // We just made a normal commit, now we are going to capture all the values generated from that commit // then rewind and make a signed commit var commitBuffer = Commit.CreateBuffer( repo.Head.Tip.Author, repo.Head.Tip.Committer, repo.Head.Tip.Message, repo.Head.Tip.Tree, repo.Head.Tip.Parents, true, null); var signedCommitData = CommitSignature.Sign(commitBuffer + "\n", parameters.PgpPrivateKey, parameters.PgPPassword); repo.Reset(ResetMode.Soft, repo.Head.Commits.Skip(1).First().Sha); var commitToKeep = repo.ObjectDatabase.CreateCommitWithSignature(commitBuffer, signedCommitData); repo.Refs.UpdateTarget(repo.Refs.Head, commitToKeep); // Should use "master" if we are compressing Wiki if (isWikiCompress) { var branchAgain = Commands.Checkout(repo, "master"); } else { var branchAgain = Commands.Checkout(repo, KnownGitHubs.BranchName); } repo.Reset(ResetMode.Hard, commitToKeep.Sha); // verify images are not corrupted by reading from git // see https://github.com/dabutvin/ImgBot/issues/273 try { foreach (var image in optimizedImages) { if (image.OriginalPath.EndsWith(".svg")) { // do not use ImageMagick to verify SVGs continue; } new MagickImage(image.OriginalPath).Dispose(); } } catch (MagickErrorException) { logger.LogError("Corrupt images after reset!"); return(false); } // push to GitHub if (isWikiCompress) { repo.Network.Push(remote, "refs/heads/master", new PushOptions { CredentialsProvider = credentialsProvider, }); } else { var refs = $"refs/heads/{KnownGitHubs.BranchName}"; if (parameters.IsRebase) { refs = refs.Insert(0, "+"); } logger.LogInformation("refs: {refs}", refs); repo.Network.Push(remote, refs, new PushOptions { CredentialsProvider = credentialsProvider, }); } try { Directory.Delete(parameters.LocalPath, true); } catch (Exception e) { logger.LogError(e, $"Delete issue with repository {parameters.LocalPath}"); } return(true); }