/// <summary>
/// Validate the object.
/// </summary>
/// <exception cref="ValidationException">
/// Thrown if validation fails
/// </exception>
public virtual void Validate()
{
if (Identity != null)
{
Identity.Validate();
}
if (RepoConfiguration != null)
{
RepoConfiguration.Validate();
}
if (GlobalParameters != null)
{
foreach (var valueElement in GlobalParameters.Values)
{
if (valueElement != null)
{
valueElement.Validate();
}
}
}
if (Encryption != null)
{
Encryption.Validate();
}
}
public static string[] FindImages(string localPath, RepoConfiguration repoConfiguration)
{
var images = Directory.EnumerateFiles(localPath, "*.*", SearchOption.AllDirectories)
.Where(x => KnownImgPatterns.ImgExtensions.Contains(Path.GetExtension(x).ToLower()));
if (repoConfiguration.IgnoredFiles != null)
{
// find all the ignored files and exclude them from the found images
var ignoredFiles = repoConfiguration.IgnoredFiles
.Select(x => x.Replace('\\', Path.DirectorySeparatorChar).Replace('/', Path.DirectorySeparatorChar))
.AsParallel()
.SelectMany(pattern =>
{
try
{
return(Directory.EnumerateFiles(localPath, pattern, SearchOption.AllDirectories));
}
catch
{
// ignore issues enumerating files
return(Enumerable.Empty <string>());
}
});
images = images.Except(ignoredFiles);
}
return(images.ToArray());
}
public void GivenDefaultConfiguration_ShouldNotOptimizeImages()
{
var compressionResults = new CompressionResult[] { };
var configuration = new RepoConfiguration();
var shouldOptimize = Threshold.MeetsThreshold(configuration, compressionResults);
Assert.IsFalse(shouldOptimize);
}
public static string[] FindImages(string localPath, RepoConfiguration repoConfiguration)
{
var images = Directory.EnumerateFiles(localPath, "*.*", SearchOption.AllDirectories)
.Where(x => KnownImgPatterns.ImgExtensions.Contains(Path.GetExtension(x).ToLower()))
.Select(x => x.Replace("\\", "/"));
return(FilterOutIgnoredFiles(images, repoConfiguration));
}
public void Given0_ShouldOptimizeImages()
{
var compressionResults = new CompressionResult[] { };
var configuration = new RepoConfiguration
{
MinKBReduced = 0
};
var shouldOptimize = Threshold.MeetsThreshold(configuration, compressionResults);
Assert.IsTrue(shouldOptimize);
}
/// <summary>
/// Using the compressionResults and the repoConfiguration determine whether
/// the optimization warrants a PR at this time.
/// </summary>
/// <returns>True when the images are compressed enough to warrant a PR.</returns>
public static bool MeetsThreshold(RepoConfiguration repoConfiguration, CompressionResult[] compressionResults)
{
if (repoConfiguration.MinKBReduced == null || repoConfiguration.MinKBReduced <= 0)
{
// no threshold specified - let's continue
return(true);
}
// determine total KB reduced
var totalKBReduced = compressionResults.Sum(x => x.SizeBefore - x.SizeAfter);
return(repoConfiguration.MinKBReduced <= totalKBReduced);
}
/// <summary>
/// Using the repo and the repoConfiguration determine whether
/// the schedule permits an optimization at this time.
/// </summary>
/// <param name="repoConfiguration">The configuration for the repository.</param>
/// <param name="repo">The repository.</param>
/// <returns>True when the images can be optimized.</returns>
public static bool ShouldOptimizeImages(RepoConfiguration repoConfiguration, LibGit2Sharp.IRepository repo)
{
if (string.IsNullOrEmpty(repoConfiguration.Schedule))
{
// no schedule specified - let's optimize those images
return(true);
}
// determine backofftime from keywords
TimeSpan backofftime;
switch (repoConfiguration.Schedule)
{
case KnownScheduleSettings.Daily:
backofftime = TimeSpan.FromDays(1);
break;
case KnownScheduleSettings.Weekly:
backofftime = TimeSpan.FromDays(7);
break;
case KnownScheduleSettings.Monthly:
backofftime = TimeSpan.FromDays(30);
break;
default:
backofftime = TimeSpan.Zero;
break;
}
// find the last time imgbot committed here
var imgbotCommit = repo.Commits.FirstOrDefault(x => x.Author.Email == KnownGitHubs.ImgBotEmail);
if (imgbotCommit == null)
{
// no imgbot commit in history - let's optimize those images
return(true);
}
if (DateTimeOffset.Now - imgbotCommit.Author.When > backofftime)
{
// Now minus the last imgbot commit is greater than the backoff time - let's optimize those images
return(true);
}
return(false);
}
public static string[] FindImages(string localPath, RepoConfiguration repoConfiguration)
{
var images = Directory.EnumerateFiles(localPath, "*.*", SearchOption.AllDirectories)
.Where(x => KnownImgPatterns.ImgExtensions.Contains(Path.GetExtension(x).ToLower()))
.Select(x => x.Replace("\\", "/"));
if (repoConfiguration.IgnoredFiles != null)
{
foreach (var ignorePattern in repoConfiguration.IgnoredFiles)
{
var pattern = new Regex(NormalizePattern(ignorePattern), RegexOptions.IgnoreCase);
images = images.Where(x => !pattern.IsMatch(x));
}
}
return(images.ToArray());
}
public static string[] FindImages(string localPath, RepoConfiguration repoConfiguration)
{
var images = KnownImgPatterns.ImgPatterns
.AsParallel()
.SelectMany(pattern => Directory.EnumerateFiles(localPath, pattern, SearchOption.AllDirectories));
if (repoConfiguration.IgnoredFiles != null)
{
// find all the ignored files and exclude them from the found images
var ignoredFiles = repoConfiguration.IgnoredFiles
.AsParallel()
.SelectMany(pattern => Directory.EnumerateFiles(localPath, pattern, SearchOption.AllDirectories));
images = images.Except(ignoredFiles);
}
return(images.ToArray());
}
public void GivenAboveThreshold_ShouldNotOptimizeImages()
{
var compressionResults = new CompressionResult[]
{
new CompressionResult
{
SizeBefore = 5000,
SizeAfter = 4900,
},
new CompressionResult
{
SizeBefore = 5000,
SizeAfter = 4999,
},
};
var configuration = new RepoConfiguration
{
MinKBReduced = 500
};
var shouldOptimize = Threshold.MeetsThreshold(configuration, compressionResults);
Assert.IsFalse(shouldOptimize);
}
public static async Task <bool> RunAsync(CompressimagesParameters parameters, ILogger logger)
{
CredentialsHandler credentialsProvider =
(url, user, cred) =>
new UsernamePasswordCredentials {
Username = KnownGitHubs.Username, Password = parameters.Password
};
// clone
var cloneOptions = new CloneOptions
{
CredentialsProvider = credentialsProvider,
};
try
{
Repository.Clone(parameters.CloneUrl, parameters.LocalPath, cloneOptions);
}
catch (Exception)
{
// issue cloning, let's try to queue to the linux box to see if it can clone
// linux support is in preview github.com/Azure/Azure-Functions/wiki/Azure-Functions-on-Linux-Preview
var linuxConnectionString = Environment.GetEnvironmentVariable("linux-connectionstring");
if (string.IsNullOrEmpty(linuxConnectionString) == false && parameters.CompressImagesMessage != null)
{
logger.LogInformation("bouncing to linux queue");
var linuxQueue = CloudStorageAccount
.Parse(linuxConnectionString)
.CreateCloudQueueClient()
.GetQueueReference("compressimagesmessage");
await linuxQueue.CreateIfNotExistsAsync();
await linuxQueue.AddMessageAsync(new CloudQueueMessage(JsonConvert.SerializeObject(parameters.CompressImagesMessage)));
return(false);
}
throw;
}
var repo = new Repository(parameters.LocalPath);
var remote = repo.Network.Remotes["origin"];
// check if we have the branch already or this is empty repo
try
{
if (repo.Network.ListReferences(remote, credentialsProvider).Any() == false)
{
logger.LogInformation("CompressImagesFunction: no references found for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
return(false);
}
if (repo.Network.ListReferences(remote, credentialsProvider).Any(x => x.CanonicalName == $"refs/heads/{KnownGitHubs.BranchName}"))
{
logger.LogInformation("CompressImagesFunction: branch already exists for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
return(false);
}
}
catch (Exception e)
{
// log + ignore
logger.LogWarning(e, "CompressImagesFunction: issue checking for existing branch or empty repo for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
}
var repoConfiguration = new RepoConfiguration();
try
{
// see if .imgbotconfig exists in repo root
var repoConfigJson = File.ReadAllText(parameters.LocalPath + Path.DirectorySeparatorChar + ".imgbotconfig");
if (!string.IsNullOrEmpty(repoConfigJson))
{
repoConfiguration = JsonConvert.DeserializeObject <RepoConfiguration>(repoConfigJson);
}
}
catch
{
// ignore
}
if (Schedule.ShouldOptimizeImages(repoConfiguration, repo) == false)
{
logger.LogInformation("CompressImagesFunction: skipping optimization due to schedule for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
return(false);
}
// check out the branch
repo.CreateBranch(KnownGitHubs.BranchName);
var branch = Commands.Checkout(repo, KnownGitHubs.BranchName);
// reset any mean files
repo.Reset(ResetMode.Mixed, repo.Head.Tip);
// optimize images
var imagePaths = ImageQuery.FindImages(parameters.LocalPath, repoConfiguration);
var optimizedImages = OptimizeImages(repo, parameters.LocalPath, imagePaths, logger, repoConfiguration.AggressiveCompression);
if (optimizedImages.Length == 0)
{
return(false);
}
// create commit message based on optimizations
foreach (var image in optimizedImages)
{
Commands.Stage(repo, image.OriginalPath);
}
var commitMessage = CommitMessage.Create(optimizedImages);
var signature = new Signature(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail, DateTimeOffset.Now);
repo.Commit(commitMessage, signature, signature);
// We just made a normal commit, now we are going to capture all the values generated from that commit
// then rewind and make a signed commit
var commitBuffer = Commit.CreateBuffer(
repo.Head.Tip.Author,
repo.Head.Tip.Committer,
repo.Head.Tip.Message,
repo.Head.Tip.Tree,
repo.Head.Tip.Parents,
true,
null);
var signedCommitData = CommitSignature.Sign(commitBuffer + "\n", parameters.PgpPrivateKeyStream, parameters.PgPPassword);
repo.Reset(ResetMode.Soft, repo.Head.Commits.Skip(1).First().Sha);
var commitToKeep = repo.ObjectDatabase.CreateCommitWithSignature(commitBuffer, signedCommitData);
repo.Refs.UpdateTarget(repo.Refs.Head, commitToKeep);
var branchAgain = Commands.Checkout(repo, KnownGitHubs.BranchName);
repo.Reset(ResetMode.Hard, commitToKeep.Sha);
// verify images are not corrupted by reading from git
// see https://github.com/dabutvin/ImgBot/issues/273
try
{
foreach (var image in optimizedImages)
{
new MagickImage(image.OriginalPath).Dispose();
}
}
catch (MagickErrorException)
{
logger.LogError("Corrupt images after reset!");
return(false);
}
// push to GitHub
repo.Network.Push(remote, $"refs/heads/{KnownGitHubs.BranchName}", new PushOptions
{
CredentialsProvider = credentialsProvider,
});
return(true);
}
public static bool Run(CompressimagesParameters parameters, ICollector <CompressImagesMessage> compressImagesMessages, ILogger logger)
{
CredentialsHandler credentialsProvider =
(url, user, cred) =>
new UsernamePasswordCredentials {
Username = KnownGitHubs.Username, Password = parameters.Password
};
// clone
var cloneOptions = new CloneOptions
{
CredentialsProvider = credentialsProvider,
};
Repository.Clone(parameters.CloneUrl, parameters.LocalPath, cloneOptions);
var repo = new Repository(parameters.LocalPath);
var remote = repo.Network.Remotes["origin"];
var isWikiCompress = parameters.CloneUrl.EndsWith(".wiki.git");
// check if we have the branch already or this is empty repo
try
{
if (repo.Network.ListReferences(remote, credentialsProvider).Any() == false)
{
logger.LogInformation("CompressImagesFunction: no references found for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
return(false);
}
if (repo.Network.ListReferences(remote, credentialsProvider).Any(x => x.CanonicalName == $"refs/heads/{KnownGitHubs.BranchName}"))
{
logger.LogInformation("CompressImagesFunction: branch already exists for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
return(false);
}
}
catch (Exception e)
{
// log + ignore
logger.LogWarning(e, "CompressImagesFunction: issue checking for existing branch or empty repo for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
}
// check if we should switch away from the default branch
if (!isWikiCompress && parameters.Settings != null && !string.IsNullOrEmpty(parameters.Settings.DefaultBranchOverride))
{
logger.LogInformation(
"CompressImagesFunction: default branch override for {Owner}/{RepoName} is {DefaultBranchOverride}",
parameters.RepoOwner,
parameters.RepoName,
parameters.Settings.DefaultBranchOverride);
var baseBranch = repo.Branches[$"refs/remotes/origin/{parameters.Settings.DefaultBranchOverride}"];
if (baseBranch == null)
{
logger.LogWarning(
"CompressImagesFunction: default branch ({DefaultBranchOverride}) not found for {Owner}/{RepoName}",
parameters.Settings.DefaultBranchOverride,
parameters.RepoOwner,
parameters.RepoName);
return(false);
}
Commands.Checkout(repo, baseBranch);
}
var repoConfiguration = new RepoConfiguration();
try
{
// see if .imgbotconfig exists in repo root
var repoConfigJson = File.ReadAllText(parameters.LocalPath + Path.DirectorySeparatorChar + ".imgbotconfig");
if (!string.IsNullOrEmpty(repoConfigJson))
{
repoConfiguration = JsonConvert.DeserializeObject <RepoConfiguration>(repoConfigJson);
}
}
catch
{
// ignore
}
// Add new compressMessage if we should compress Wiki
if (repoConfiguration.CompressWiki && isWikiCompress == false)
{
logger.LogInformation("CompressImagesFunction: Adding Wiki image compression to queue for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
compressImagesMessages.Add(new CompressImagesMessage()
{
InstallationId = parameters.CompressImagesMessage.InstallationId,
RepoName = parameters.CompressImagesMessage.RepoName,
Owner = parameters.RepoOwner,
CloneUrl = $"https://github.com/{parameters.RepoOwner}/{parameters.RepoName}.wiki.git"
});
}
if (Schedule.ShouldOptimizeImages(repoConfiguration, repo) == false)
{
logger.LogInformation("CompressImagesFunction: skipping optimization due to schedule for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
return(false);
}
// Should not create branch if we are compressing Wiki
if (isWikiCompress == false)
{
// check out the branch
repo.CreateBranch(KnownGitHubs.BranchName);
var branch = Commands.Checkout(repo, KnownGitHubs.BranchName);
}
// reset any mean files
repo.Reset(ResetMode.Mixed, repo.Head.Tip);
// optimize images
var imagePaths = ImageQuery.FindImages(parameters.LocalPath, repoConfiguration);
var optimizedImages = OptimizeImages(repo, parameters.LocalPath, imagePaths, logger, repoConfiguration.AggressiveCompression);
if (optimizedImages.Length == 0)
{
return(false);
}
// create commit message based on optimizations
foreach (var image in optimizedImages)
{
Commands.Stage(repo, image.OriginalPath);
}
var commitMessage = CommitMessage.Create(optimizedImages);
var signature = new Signature(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail, DateTimeOffset.Now);
repo.Commit(commitMessage, signature, signature);
// We just made a normal commit, now we are going to capture all the values generated from that commit
// then rewind and make a signed commit
var commitBuffer = Commit.CreateBuffer(
repo.Head.Tip.Author,
repo.Head.Tip.Committer,
repo.Head.Tip.Message,
repo.Head.Tip.Tree,
repo.Head.Tip.Parents,
true,
null);
var signedCommitData = CommitSignature.Sign(commitBuffer + "\n", parameters.PgpPrivateKey, parameters.PgPPassword);
repo.Reset(ResetMode.Soft, repo.Head.Commits.Skip(1).First().Sha);
var commitToKeep = repo.ObjectDatabase.CreateCommitWithSignature(commitBuffer, signedCommitData);
repo.Refs.UpdateTarget(repo.Refs.Head, commitToKeep);
// Should use "master" if we are compressing Wiki
if (isWikiCompress)
{
var branchAgain = Commands.Checkout(repo, "master");
}
else
{
var branchAgain = Commands.Checkout(repo, KnownGitHubs.BranchName);
}
repo.Reset(ResetMode.Hard, commitToKeep.Sha);
// verify images are not corrupted by reading from git
// see https://github.com/dabutvin/ImgBot/issues/273
try
{
foreach (var image in optimizedImages)
{
if (image.OriginalPath.EndsWith(".svg"))
{
// do not use ImageMagick to verify SVGs
continue;
}
new MagickImage(image.OriginalPath).Dispose();
}
}
catch (MagickErrorException)
{
logger.LogError("Corrupt images after reset!");
return(false);
}
// push to GitHub
if (isWikiCompress)
{
repo.Network.Push(remote, "refs/heads/master", new PushOptions
{
CredentialsProvider = credentialsProvider,
});
}
else
{
repo.Network.Push(remote, $"refs/heads/{KnownGitHubs.BranchName}", new PushOptions
{
CredentialsProvider = credentialsProvider,
});
}
return(true);
}
public static string[] FilterOutIgnoredFiles(IEnumerable <string> imagePaths, RepoConfiguration repoConfiguration)
{
if (repoConfiguration.IgnoredFiles != null)
{
foreach (var ignorePattern in repoConfiguration.IgnoredFiles)
{
var pattern = new Regex(NormalizePattern(ignorePattern), RegexOptions.IgnoreCase);
imagePaths = imagePaths.Where(x => !pattern.IsMatch(x));
}
}
return(imagePaths.ToArray());
}
public static bool Run(CompressimagesParameters parameters)
{
CredentialsHandler credentialsProvider =
(url, user, cred) =>
new UsernamePasswordCredentials {
Username = KnownGitHubs.Username, Password = parameters.Password
};
// clone
var cloneOptions = new CloneOptions
{
CredentialsProvider = credentialsProvider,
};
Repository.Clone(parameters.CloneUrl, parameters.LocalPath, cloneOptions);
var repo = new Repository(parameters.LocalPath);
var remote = repo.Network.Remotes["origin"];
// check if we have the branch already or this is empty repo
try
{
if (repo.Network.ListReferences(remote, credentialsProvider).Any() == false)
{
return(false);
}
if (repo.Network.ListReferences(remote, credentialsProvider).Any(x => x.CanonicalName == $"refs/heads/{KnownGitHubs.BranchName}"))
{
return(false);
}
}
catch
{
// ignore
}
var repoConfiguration = new RepoConfiguration();
try
{
// see if .imgbotconfig exists in repo root
var repoConfigJson = File.ReadAllText(parameters.LocalPath + Path.DirectorySeparatorChar + ".imgbotconfig");
if (!string.IsNullOrEmpty(repoConfigJson))
{
repoConfiguration = JsonConvert.DeserializeObject <RepoConfiguration>(repoConfigJson);
}
}
catch
{
// ignore
}
if (Schedule.ShouldOptimizeImages(repoConfiguration, repo) == false)
{
return(false);
}
// check out the branch
repo.CreateBranch(KnownGitHubs.BranchName);
var branch = Commands.Checkout(repo, KnownGitHubs.BranchName);
// reset any mean files
repo.Reset(ResetMode.Mixed, repo.Head.Tip);
// optimize images
var imagePaths = ImageQuery.FindImages(parameters.LocalPath, repoConfiguration);
var optimizedImages = OptimizeImages(repo, parameters.LocalPath, imagePaths);
if (optimizedImages.Length == 0)
{
return(false);
}
// create commit message based on optimizations
var commitMessage = CommitMessage.Create(optimizedImages);
// commit
var signature = new Signature(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail, DateTimeOffset.Now);
repo.Commit(commitMessage, signature, signature);
// We just made a normal commit, now we are going to capture all the values generated from that commit
// then rewind and make a signed commit
var commitBuffer = Commit.CreateBuffer(
repo.Head.Tip.Author,
repo.Head.Tip.Committer,
repo.Head.Tip.Message,
repo.Head.Tip.Tree,
repo.Head.Tip.Parents,
true,
null);
var signedCommitData = CommitSignature.Sign(commitBuffer + "\n", parameters.PgpPrivateKeyStream, parameters.PgPPassword);
repo.Reset(ResetMode.Soft, repo.Head.Commits.Skip(1).First().Sha);
var commitToKeep = repo.ObjectDatabase.CreateCommitWithSignature(commitBuffer, signedCommitData);
repo.Refs.UpdateTarget(repo.Refs.Head, commitToKeep);
var branchAgain = Commands.Checkout(repo, KnownGitHubs.BranchName);
repo.Reset(ResetMode.Hard, commitToKeep.Sha);
// push to GitHub
repo.Network.Push(remote, $"refs/heads/{KnownGitHubs.BranchName}", new PushOptions
{
CredentialsProvider = credentialsProvider,
});
return(true);
}
public static bool Run(CompressimagesParameters parameters, ICollector <CompressImagesMessage> compressImagesMessages, ILogger logger)
{
CredentialsHandler credentialsProvider =
(url, user, cred) =>
new UsernamePasswordCredentials {
Username = KnownGitHubs.Username, Password = parameters.Password
};
// clone
var cloneOptions = new CloneOptions
{
CredentialsProvider = credentialsProvider,
};
Repository.Clone(parameters.CloneUrl, parameters.LocalPath, cloneOptions);
var repo = new Repository(parameters.LocalPath);
var remote = repo.Network.Remotes["origin"];
var isWikiCompress = parameters.CloneUrl.EndsWith(".wiki.git");
// check if we have the branch already or this is empty repo
try
{
if (repo.Network.ListReferences(remote, credentialsProvider).Any() == false)
{
logger.LogInformation("CompressImagesFunction: no references found for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
return(false);
}
if (!parameters.IsRebase && repo.Network.ListReferences(remote, credentialsProvider).Any(x => x.CanonicalName == $"refs/heads/{KnownGitHubs.BranchName}"))
{
logger.LogInformation("CompressImagesFunction: branch already exists for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
return(false);
}
}
catch (Exception e)
{
// log + ignore
logger.LogWarning(e, "CompressImagesFunction: issue checking for existing branch or empty repo for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
}
// check if we should switch away from the default branch
if (!isWikiCompress && parameters.Settings != null && !string.IsNullOrEmpty(parameters.Settings.DefaultBranchOverride))
{
logger.LogInformation(
"CompressImagesFunction: default branch override for {Owner}/{RepoName} is {DefaultBranchOverride}",
parameters.RepoOwner,
parameters.RepoName,
parameters.Settings.DefaultBranchOverride);
var baseBranch = repo.Branches[$"refs/remotes/origin/{parameters.Settings.DefaultBranchOverride}"];
if (baseBranch == null)
{
logger.LogWarning(
"CompressImagesFunction: default branch ({DefaultBranchOverride}) not found for {Owner}/{RepoName}",
parameters.Settings.DefaultBranchOverride,
parameters.RepoOwner,
parameters.RepoName);
return(false);
}
Commands.Checkout(repo, baseBranch);
}
var repoConfiguration = new RepoConfiguration();
try
{
// see if .imgbotconfig exists in repo root
var repoConfigJson = File.ReadAllText(parameters.LocalPath + Path.DirectorySeparatorChar + ".imgbotconfig");
if (!string.IsNullOrEmpty(repoConfigJson))
{
repoConfiguration = JsonConvert.DeserializeObject <RepoConfiguration>(repoConfigJson);
}
}
catch
{
// ignore
}
// Add new compressMessage if we should compress Wiki
if (repoConfiguration.CompressWiki && isWikiCompress == false)
{
logger.LogInformation("CompressImagesFunction: Adding Wiki image compression to queue for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
compressImagesMessages.Add(new CompressImagesMessage()
{
InstallationId = parameters.CompressImagesMessage.InstallationId,
RepoName = parameters.CompressImagesMessage.RepoName,
Owner = parameters.RepoOwner,
CloneUrl = $"https://github.com/{parameters.RepoOwner}/{parameters.RepoName}.wiki.git"
});
}
if (Schedule.ShouldOptimizeImages(repoConfiguration, repo) == false)
{
logger.LogInformation("CompressImagesFunction: skipping optimization due to schedule for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
return(false);
}
// Should not create branch if we are compressing Wiki or performing rebase
if (isWikiCompress == false && !parameters.IsRebase)
{
// check out the branch
repo.CreateBranch(KnownGitHubs.BranchName);
var branch = Commands.Checkout(repo, KnownGitHubs.BranchName);
}
// reset any mean files
repo.Reset(ResetMode.Mixed, repo.Head.Tip);
// optimize images
string[] imagePaths;
List <string> addedOrModifiedImagePaths = new List <string>();
List <string> deletedImagePaths = new List <string>();
if (parameters.IsRebase)
{
var refspec = string.Format("{0}:{0}", KnownGitHubs.BranchName);
Commands.Fetch(repo, "origin", new List <string> {
refspec
}, null, "fetch");
var diff = repo.Diff.Compare <TreeChanges>(repo.Branches[KnownGitHubs.BranchName].Commits.ElementAt(1).Tree, repo.Head.Tip.Tree);
if (diff == null)
{
logger.LogInformation("Something went wrong while doing rebase");
return(false);
}
foreach (TreeEntryChanges c in diff)
{
if (KnownImgPatterns.ImgExtensions.Contains(Path.GetExtension(c.Path)))
{
var path = parameters.LocalPath + "/" + c.Path;
var oldpath = parameters.LocalPath + "/" + c.OldPath;
switch (c.Status)
{
case ChangeKind.Added:
case ChangeKind.Modified:
addedOrModifiedImagePaths.Add(path.Replace("\\", "/"));
break;
case ChangeKind.Renamed:
addedOrModifiedImagePaths.Add(path.Replace("\\", "/"));
deletedImagePaths.Add(oldpath.Replace("\\", "/"));
break;
case ChangeKind.Deleted:
deletedImagePaths.Add(path.Replace("\\", "/"));
break;
}
}
}
imagePaths = ImageQuery.FilterOutIgnoredFiles(addedOrModifiedImagePaths, repoConfiguration);
}
else
{
imagePaths = ImageQuery.FindImages(parameters.LocalPath, repoConfiguration);
}
var optimizedImages = OptimizeImages(repo, parameters.LocalPath, imagePaths, logger, repoConfiguration.AggressiveCompression);
if (optimizedImages.Length == 0)
{
return(false);
}
// create commit message based on optimizations
foreach (var image in optimizedImages)
{
Commands.Stage(repo, image.OriginalPath);
}
var commitMessage = CommitMessage.Create(optimizedImages);
var signature = new Signature(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail, DateTimeOffset.Now);
repo.Commit(commitMessage, signature, signature);
if (parameters.IsRebase)
{
var baseBranch = repo.Head;
var newCommit = baseBranch.Tip;
var oldCommit = repo.Branches[KnownGitHubs.BranchName].Tip;
// we need to reset the default branch so that we can
// rebase to it later.
repo.Reset(ResetMode.Hard, repo.Head.Commits.ElementAt(1));
// checkout to imgbot branch. TODO: remove because this is needed earlier on diff
Commands.Checkout(repo, KnownGitHubs.BranchName);
// cherry-pick
var cherryPickOptions = new CherryPickOptions()
{
MergeFileFavor = MergeFileFavor.Theirs,
};
var cherryPickResult = repo.CherryPick(newCommit, signature, cherryPickOptions);
if (cherryPickResult.Status == CherryPickStatus.Conflicts)
{
var status = repo.RetrieveStatus(new LibGit2Sharp.StatusOptions()
{
});
foreach (var item in status)
{
if (item.State == FileStatus.Conflicted)
{
Commands.Stage(repo, item.FilePath);
}
}
repo.Commit(commitMessage, signature, signature);
}
// New commit message creation
var previousCommitResults = CompressionResult.ParseCommitMessage(oldCommit.Message);
var mergedResults = CompressionResult.Merge(optimizedImages, previousCommitResults);
var filteredResults = CompressionResult.Filter(mergedResults, deletedImagePaths.ToArray());
var squashCommitMessage = CommitMessage.Create(filteredResults);
// squash
var baseCommit = repo.Head.Commits.ElementAt(2);
repo.Reset(ResetMode.Soft, baseCommit);
repo.Commit(squashCommitMessage, signature, signature);
// rebase
var rebaseOptions = new RebaseOptions()
{
FileConflictStrategy = CheckoutFileConflictStrategy.Theirs,
};
var rebaseResult = repo.Rebase.Start(repo.Head, baseBranch, null, new Identity(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail), rebaseOptions);
while (rebaseResult.Status == RebaseStatus.Conflicts)
{
var status = repo.RetrieveStatus(new LibGit2Sharp.StatusOptions()
{
});
foreach (var item in status)
{
if (item.State == FileStatus.Conflicted)
{
if (imagePaths.Contains(parameters.LocalPath + "/" + item.FilePath))
{
Commands.Stage(repo, item.FilePath);
}
else
{
Commands.Remove(repo, item.FilePath);
}
}
}
rebaseResult = repo.Rebase.Continue(new Identity(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail), rebaseOptions);
}
}
// We just made a normal commit, now we are going to capture all the values generated from that commit
// then rewind and make a signed commit
var commitBuffer = Commit.CreateBuffer(
repo.Head.Tip.Author,
repo.Head.Tip.Committer,
repo.Head.Tip.Message,
repo.Head.Tip.Tree,
repo.Head.Tip.Parents,
true,
null);
var signedCommitData = CommitSignature.Sign(commitBuffer + "\n", parameters.PgpPrivateKey, parameters.PgPPassword);
repo.Reset(ResetMode.Soft, repo.Head.Commits.Skip(1).First().Sha);
var commitToKeep = repo.ObjectDatabase.CreateCommitWithSignature(commitBuffer, signedCommitData);
repo.Refs.UpdateTarget(repo.Refs.Head, commitToKeep);
// Should use "master" if we are compressing Wiki
if (isWikiCompress)
{
var branchAgain = Commands.Checkout(repo, "master");
}
else
{
var branchAgain = Commands.Checkout(repo, KnownGitHubs.BranchName);
}
repo.Reset(ResetMode.Hard, commitToKeep.Sha);
// verify images are not corrupted by reading from git
// see https://github.com/dabutvin/ImgBot/issues/273
try
{
foreach (var image in optimizedImages)
{
if (image.OriginalPath.EndsWith(".svg"))
{
// do not use ImageMagick to verify SVGs
continue;
}
new MagickImage(image.OriginalPath).Dispose();
}
}
catch (MagickErrorException)
{
logger.LogError("Corrupt images after reset!");
return(false);
}
// push to GitHub
if (isWikiCompress)
{
repo.Network.Push(remote, "refs/heads/master", new PushOptions
{
CredentialsProvider = credentialsProvider,
});
}
else
{
var refs = $"refs/heads/{KnownGitHubs.BranchName}";
if (parameters.IsRebase)
{
refs = refs.Insert(0, "+");
}
logger.LogInformation("refs: {refs}", refs);
repo.Network.Push(remote, refs, new PushOptions
{
CredentialsProvider = credentialsProvider,
});
}
return(true);
}
public static bool Run(CompressimagesParameters parameters, ICollector <CompressImagesMessage> compressImagesMessages, ILogger logger)
{
var storageAccount = CloudStorageAccount.Parse(Common.KnownEnvironmentVariables.AzureWebJobsStorage);
var paidPlan = PaidPlan(storageAccount, parameters.RepoOwner);
CredentialsHandler credentialsProvider =
(url, user, cred) =>
new UsernamePasswordCredentials {
Username = KnownGitHubs.Username, Password = parameters.Password
};
// clone
var cloneOptions = new CloneOptions
{
CredentialsProvider = credentialsProvider,
};
Repository.Clone(parameters.CloneUrl, parameters.LocalPath, cloneOptions);
var repo = new Repository(parameters.LocalPath);
var remote = repo.Network.Remotes["origin"];
var isWikiCompress = parameters.CloneUrl.EndsWith(".wiki.git");
// check if we have the branch already or this is empty repo
try
{
if (repo.Network.ListReferences(remote, credentialsProvider).Any() == false)
{
logger.LogInformation("CompressImagesFunction: no references found for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
return(false);
}
}
catch (Exception e)
{
// log + ignore
logger.LogWarning(e, "CompressImagesFunction: issue checking for existing branch or empty repo for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
}
// check if the branch exists and has been modified by the user
if (parameters.IsRebase && repo.Branches[$"refs/remotes/origin/{KnownGitHubs.BranchName}"].Tip.Author.Name != KnownGitHubs.ImgBotLogin)
{
logger.LogInformation("CompressImagesFunction: imgbot branch has been modified by the user.");
return(false);
}
// check if we should switch away from the default branch
if (!isWikiCompress && parameters.Settings != null && !string.IsNullOrEmpty(parameters.Settings.DefaultBranchOverride))
{
logger.LogInformation(
"CompressImagesFunction: default branch override for {Owner}/{RepoName} is {DefaultBranchOverride}",
parameters.RepoOwner,
parameters.RepoName,
parameters.Settings.DefaultBranchOverride);
var baseBranch = repo.Branches[$"refs/remotes/origin/{parameters.Settings.DefaultBranchOverride}"];
if (baseBranch == null)
{
logger.LogWarning(
"CompressImagesFunction: default branch ({DefaultBranchOverride}) not found for {Owner}/{RepoName}",
parameters.Settings.DefaultBranchOverride,
parameters.RepoOwner,
parameters.RepoName);
return(false);
}
Commands.Checkout(repo, baseBranch);
}
var repoConfiguration = new RepoConfiguration();
try
{
// see if .imgbotconfig exists in repo root
var repoConfigJson = File.ReadAllText(parameters.LocalPath + Path.DirectorySeparatorChar + ".imgbotconfig");
if (!string.IsNullOrEmpty(repoConfigJson))
{
repoConfiguration = JsonConvert.DeserializeObject <RepoConfiguration>(repoConfigJson);
// for now we are not adding the labels functionality || repoConfiguration.Labels.Any() TODO: add it when adding the labels feature
if (paidPlan && (repoConfiguration.PrBody != null || repoConfiguration.PrTitle != null))
{
var settingsTable = storageAccount.CreateCloudTableClient().GetTableReference("settings");
// Labels = repoConfiguration.Labels TODO: add it when adding the labels feature
var settings = new Common.TableModels.Settings(
parameters.CompressImagesMessage.InstallationId.ToString(),
parameters.CompressImagesMessage.RepoName)
{
PrBody = repoConfiguration.PrBody,
PrTitle = repoConfiguration.PrTitle,
};
settingsTable.ExecuteAsync(TableOperation.InsertOrReplace(settings)).Wait();
}
}
}
catch
{
// ignore
}
// Add new compressMessage if we should compress Wiki
if (repoConfiguration.CompressWiki && isWikiCompress == false)
{
logger.LogInformation("CompressImagesFunction: Adding Wiki image compression to queue for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
compressImagesMessages.Add(new CompressImagesMessage()
{
InstallationId = parameters.CompressImagesMessage.InstallationId,
RepoName = parameters.CompressImagesMessage.RepoName,
Owner = parameters.RepoOwner,
CloneUrl = $"https://github.com/{parameters.RepoOwner}/{parameters.RepoName}.wiki.git"
});
}
if (Schedule.ShouldOptimizeImages(repoConfiguration, repo) == false)
{
logger.LogInformation("CompressImagesFunction: skipping optimization due to schedule for {Owner}/{RepoName}", parameters.RepoOwner, parameters.RepoName);
return(false);
}
// Should not create branch if we are compressing Wiki or performing rebase
if (isWikiCompress == false && !parameters.IsRebase)
{
// check out the branch
repo.CreateBranch(KnownGitHubs.BranchName);
var branch = Commands.Checkout(repo, KnownGitHubs.BranchName);
}
else if (parameters.IsRebase)
{
// if rebasing, fetch the branch
var refspec = string.Format("{0}:{0}", KnownGitHubs.BranchName);
Commands.Fetch(repo, "origin", new List <string> {
refspec
}, null, "fetch");
}
// reset any mean files
repo.Reset(ResetMode.Mixed, repo.Head.Tip);
// optimize images
var imagePaths = ImageQuery.FindImages(parameters.LocalPath, repoConfiguration);
var optimizedImages = OptimizeImages(repo, parameters.LocalPath, imagePaths, logger, repoConfiguration.AggressiveCompression);
if (optimizedImages.Length == 0)
{
return(false);
}
if (!Threshold.MeetsThreshold(repoConfiguration, optimizedImages))
{
logger.LogInformation($"Did not meet threshold. {parameters.RepoOwner}/{parameters.RepoName}");
return(false);
}
// create commit message based on optimizations
foreach (var image in optimizedImages)
{
Commands.Stage(repo, image.OriginalPath);
}
var commitMessage = CommitMessage.Create(optimizedImages);
var signature = new Signature(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail, DateTimeOffset.Now);
repo.Commit(commitMessage, signature, signature);
if (parameters.IsRebase)
{
var baseBranch = repo.Head;
var newCommit = baseBranch.Tip;
// we need to reset the default branch so that we can
// rebase to it later.
repo.Reset(ResetMode.Hard, repo.Head.Commits.ElementAt(1));
Commands.Checkout(repo, KnownGitHubs.BranchName);
// reset imgbot branch removing old commit
repo.Reset(ResetMode.Hard, repo.Head.Commits.ElementAt(1));
// cherry-pick
var cherryPickOptions = new CherryPickOptions()
{
MergeFileFavor = MergeFileFavor.Theirs,
};
var cherryPickResult = repo.CherryPick(newCommit, signature, cherryPickOptions);
if (cherryPickResult.Status == CherryPickStatus.Conflicts)
{
var status = repo.RetrieveStatus(new LibGit2Sharp.StatusOptions()
{
});
foreach (var item in status)
{
if (item.State == FileStatus.Conflicted)
{
Commands.Stage(repo, item.FilePath);
}
}
repo.Commit(commitMessage, signature, signature);
}
// rebase
var rebaseOptions = new RebaseOptions()
{
FileConflictStrategy = CheckoutFileConflictStrategy.Theirs,
};
var rebaseResult = repo.Rebase.Start(repo.Head, baseBranch, null, new Identity(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail), rebaseOptions);
while (rebaseResult.Status == RebaseStatus.Conflicts)
{
var status = repo.RetrieveStatus(new LibGit2Sharp.StatusOptions()
{
});
foreach (var item in status)
{
if (item.State == FileStatus.Conflicted)
{
if (imagePaths.Contains(Path.Combine(parameters.LocalPath, item.FilePath)))
{
Commands.Stage(repo, item.FilePath);
}
else
{
Commands.Remove(repo, item.FilePath);
}
}
}
rebaseResult = repo.Rebase.Continue(new Identity(KnownGitHubs.ImgBotLogin, KnownGitHubs.ImgBotEmail), rebaseOptions);
}
}
// We just made a normal commit, now we are going to capture all the values generated from that commit
// then rewind and make a signed commit
var commitBuffer = Commit.CreateBuffer(
repo.Head.Tip.Author,
repo.Head.Tip.Committer,
repo.Head.Tip.Message,
repo.Head.Tip.Tree,
repo.Head.Tip.Parents,
true,
null);
var signedCommitData = CommitSignature.Sign(commitBuffer + "\n", parameters.PgpPrivateKey, parameters.PgPPassword);
repo.Reset(ResetMode.Soft, repo.Head.Commits.Skip(1).First().Sha);
var commitToKeep = repo.ObjectDatabase.CreateCommitWithSignature(commitBuffer, signedCommitData);
repo.Refs.UpdateTarget(repo.Refs.Head, commitToKeep);
// Should use "master" if we are compressing Wiki
if (isWikiCompress)
{
var branchAgain = Commands.Checkout(repo, "master");
}
else
{
var branchAgain = Commands.Checkout(repo, KnownGitHubs.BranchName);
}
repo.Reset(ResetMode.Hard, commitToKeep.Sha);
// verify images are not corrupted by reading from git
// see https://github.com/dabutvin/ImgBot/issues/273
try
{
foreach (var image in optimizedImages)
{
if (image.OriginalPath.EndsWith(".svg"))
{
// do not use ImageMagick to verify SVGs
continue;
}
new MagickImage(image.OriginalPath).Dispose();
}
}
catch (MagickErrorException)
{
logger.LogError("Corrupt images after reset!");
return(false);
}
// push to GitHub
if (isWikiCompress)
{
repo.Network.Push(remote, "refs/heads/master", new PushOptions
{
CredentialsProvider = credentialsProvider,
});
}
else
{
var refs = $"refs/heads/{KnownGitHubs.BranchName}";
if (parameters.IsRebase)
{
refs = refs.Insert(0, "+");
}
logger.LogInformation("refs: {refs}", refs);
repo.Network.Push(remote, refs, new PushOptions
{
CredentialsProvider = credentialsProvider,
});
}
try
{
Directory.Delete(parameters.LocalPath, true);
}
catch (Exception e)
{
logger.LogError(e, $"Delete issue with repository {parameters.LocalPath}");
}
return(true);
}
