public void CloneHg(
string quotedHgCloneUrl,
string quotedCloneDirectoryPath,
MirroringSettings settings,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
RunRemoteHgCommandAndLogOutput(
"hg clone --noupdate " + quotedHgCloneUrl + " " + quotedCloneDirectoryPath,
settings);
}
catch (CommandException ex) when(ex.IsHgConnectionTerminatedError())
{
_eventLog.WriteEntry(
"Cloning from the Mercurial repo " + quotedHgCloneUrl + " failed because the server terminated the connection. Re-trying by pulling revision by revision.",
EventLogEntryType.Warning);
RunRemoteHgCommandAndLogOutput(
"hg clone --noupdate --rev 0 " + quotedHgCloneUrl + " " + quotedCloneDirectoryPath,
settings);
cancellationToken.ThrowIfCancellationRequested();
PullPerRevisionsHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
}
}
protected override void OnStart(string[] args)
{
if (!EventLog.Exists("Git-hg Mirror Daemon"))
{
EventLog.CreateEventSource(new EventSourceCreationData("GitHgMirror.Daemon", "Git-hg Mirror Daemon"));
}
// Keep in mind that the event log consumes memory so unless you keep this well beyond what you can spare
// the server will run out of RAM.
serviceEventLog.MaximumKilobytes = 196608; // 192MB
serviceEventLog.WriteEntry("GitHgMirrorDaemon started.");
_settings = new MirroringSettings
{
ApiEndpointUrl = new Uri(ConfigurationManager.AppSettings[Constants.ApiEndpointUrl]),
ApiPassword = ConfigurationManager.ConnectionStrings[Constants.ApiPasswordKey]?.ConnectionString ?? string.Empty,
RepositoriesDirectoryPath = ConfigurationManager.AppSettings[Constants.RepositoriesDirectoryPath],
BatchSize = int.Parse(ConfigurationManager.AppSettings[Constants.BatchSize], CultureInfo.InvariantCulture),
};
_startTimer = new System.Timers.Timer(10000);
_startTimer.Elapsed += StartTimerElapsed;
_startTimer.Enabled = true;
_cleaner = new UntouchedRepositoriesCleaner(_settings, serviceEventLog);
_cleanTimer = new System.Timers.Timer(3_600_000 * 2); // Two hours
_cleanTimer.Elapsed += (sender, e) => _cleaner.Clean(_cancellationTokenSource.Token);
_cleanTimer.Enabled = true;
}
/// <summary>
/// Runs the specified command for a git repo in hg.
/// </summary>
/// <param name="gitCloneUri">The git clone URI.</param>
/// <param name="command">
/// The command, including an optional placeholder for the git URL in form of {url}, e.g.: "clone --noupdate
/// {url}".
/// </param>
private void RunGitRepoCommand(Uri gitCloneUri, string command, MirroringSettings settings)
{
var gitUriBuilder = new UriBuilder(gitCloneUri);
var userName = gitUriBuilder.UserName;
var password = gitUriBuilder.Password;
gitUriBuilder.UserName = null;
gitUriBuilder.Password = null;
var gitUri = gitUriBuilder.Uri;
var quotedGitCloneUrl = gitUri.ToString().EncloseInQuotes();
command = command.Replace("{url}", quotedGitCloneUrl);
// Would be quite ugly that way.
#pragma warning disable S3240 // The simplest possible condition syntax should be used
if (!string.IsNullOrEmpty(userName))
#pragma warning restore S3240 // The simplest possible condition syntax should be used
{
RunRemoteHgCommandAndLogOutput(
PrefixHgCommandWithHgGitConfig(
"--config auth.rc.prefix=" +
("https://" + gitUri.Host).EncloseInQuotes() +
" --config auth.rc.username="******" --config auth.rc.password="******" " +
command),
settings);
}
else
{
RunRemoteHgCommandAndLogOutput(PrefixHgCommandWithHgGitConfig(command), settings);
}
}
/// <summary>
/// Runs the specified command for a git repo in hg.
/// </summary>
/// <param name="gitCloneUri">The git clone URI.</param>
/// <param name="command">
/// The command, including an optional placeholder for the git URL in form of {url}, e.g.: "clone --noupdate {url}".
/// </param>
private void RunGitRepoCommand(Uri gitCloneUri, string command, MirroringSettings settings)
{
var gitUriBuilder = new UriBuilder(gitCloneUri);
var userName = gitUriBuilder.UserName;
var password = gitUriBuilder.Password;
gitUriBuilder.UserName = null;
gitUriBuilder.Password = null;
var gitUri = gitUriBuilder.Uri;
var quotedGitCloneUrl = gitUri.ToString().EncloseInQuotes();
command = command.Replace("{url}", quotedGitCloneUrl);
if (!string.IsNullOrEmpty(userName))
{
RunRemoteHgCommandAndLogOutput(
PrefixHgCommandWithHgGitConfig(
"--config auth.rc.prefix=" +
("https://" + gitUri.Host).EncloseInQuotes() +
" --config auth.rc.username="******" --config auth.rc.password="******" " +
command),
settings);
}
else
{
RunRemoteHgCommandAndLogOutput(PrefixHgCommandWithHgGitConfig(command), settings);
}
}
public bool IsCloned(MirroringConfiguration configuration, MirroringSettings settings)
{
var repositoryDirectoryName = GetCloneDirectoryName(configuration);
// A subfolder per clone dir start letter.
var cloneDirectoryParentPath = Path.Combine(settings.RepositoriesDirectoryPath, repositoryDirectoryName[0].ToString());
var cloneDirectoryPath = Path.Combine(cloneDirectoryParentPath, repositoryDirectoryName);
// Also checking if the directory is empty. If yes, it was a failed attempt and really the repo is not cloned.
return(Directory.Exists(cloneDirectoryPath) && Directory.EnumerateFileSystemEntries(cloneDirectoryPath).Any());
}
public void ExportHistoryToGit(
string quotedCloneDirectoryPath,
MirroringSettings settings,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
CdDirectory(quotedCloneDirectoryPath);
RunHgCommandAndLogOutput(PrefixHgCommandWithHgGitConfig("gexport"), settings);
}
public void PushWithBookmarks(
string quotedHgCloneUrl,
string quotedCloneDirectoryPath,
MirroringSettings settings,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
CdDirectory(quotedCloneDirectoryPath);
var bookmarksOutput = RunHgCommandAndLogOutput("hg bookmarks", settings);
// There will be at least one bookmark, "master" with a git repo. However with hg-hg mirroring maybe there
// are no bookmarks.
if (bookmarksOutput.Contains("no bookmarks set"))
{
RunRemoteHgCommandAndLogOutput("hg push --new-branch --force " + quotedHgCloneUrl, settings);
}
else
{
var bookmarks = bookmarksOutput
.Split(Environment.NewLine.ToArray())
.Skip(1) // The first line is the command itself
.Where(line => !string.IsNullOrEmpty(line))
.Select(line => Regex.Match(line, @"\s([a-zA-Z0-9/.\-_]+)\s", RegexOptions.IgnoreCase).Groups[1].Value)
.Where(line => !string.IsNullOrEmpty(line))
.Select(line => "-B " + line);
// Pushing a lot of bookmarks at once would result in a "RuntimeError: maximum recursion depth exceeded"
// error.
const int batchSize = 29;
var bookmarksBatch = bookmarks.Take(batchSize);
var skip = 0;
var bookmarkCount = bookmarks.Count();
while (skip < bookmarkCount)
{
cancellationToken.ThrowIfCancellationRequested();
RunRemoteHgCommandAndLogOutput(
"hg push --new-branch --force " + string.Join(" ", bookmarksBatch) + " " + quotedHgCloneUrl,
settings);
skip += batchSize;
bookmarksBatch = bookmarks.Skip(skip).Take(batchSize);
if (bookmarksBatch.Any())
{
// Bitbucket throttles such requests so we need to slow down. Otherwise we'd get this error:
// "remote: Push throttled (max allowable rate: 30 per 60 seconds)." However, this is wrong as
// the actual limit is lower at 29.
Thread.Sleep(61000);
}
}
}
}
public static void Main()
{
// If true then a unique event log will be used for all copies of this executable. This helps if you want to
// run the app in multiple instances from source and not let the events show up across copies.
const bool useUniqueEventlog = true;
var eventLogName = "Git-hg Mirror Daemon";
var eventSourceName = "GitHgMirror.Tester";
if (useUniqueEventlog)
{
var suffix = "-" + typeof(Program).Assembly.Location.GetHashCode();
// "Only the first eight characters of a custom log name are significant" so we need to make the name
// unique withing 8 characters.
eventLogName = "GHM" + suffix;
eventSourceName += suffix;
}
if (!EventLog.Exists(eventLogName))
{
EventLog.CreateEventSource(new EventSourceCreationData(eventSourceName, eventLogName));
}
using var eventLog = new EventLog(eventLogName, ".", eventSourceName)
{
EnableRaisingEvents = true,
};
eventLog.EntryWritten += (sender, e) => Console.WriteLine(e.Entry.Message);
var settings = new MirroringSettings
{
ApiEndpointUrl = new Uri("http://githgmirror.com.127-0-0-1.org.uk/api/GitHgMirror.Common/Mirrorings"),
ApiPassword = "******",
RepositoriesDirectoryPath = @"C:\GitHgMirror\Repos",
MaxDegreeOfParallelism = 1,
BatchSize = 1,
};
// Uncomment if you want to also test repo cleaning.
////new UntouchedRepositoriesCleaner(settings, eventLog).Clean(new CancellationTokenSource().Token);
using var runner = new MirrorRunner(settings, eventLog);
// On exit with Ctrl+C
Console.CancelKeyPress += (sender, e) => runner.Stop();
runner.Start();
_waitHandle.WaitOne();
}
private string RunRemoteHgCommandAndLogOutput(string hgCommand, MirroringSettings settings, int retryCount = 0)
{
var output = string.Empty;
try
{
if (settings.MercurialSettings.UseInsecure)
{
hgCommand += " --insecure";
}
if (settings.MercurialSettings.UseDebugForRemoteCommands && !settings.MercurialSettings.UseDebug)
{
hgCommand += " --debug";
}
output = RunHgCommandAndLogOutput(hgCommand, settings);
return(output);
}
catch (CommandException ex)
{
// We'll randomly get such errors when interacting with Mercurial as well as with Git, otherwise
// properly running repos. So we re-try the operation a few times, maybe it'll work...
if (ex.Error.Contains("EOF occurred in violation of protocol"))
{
if (retryCount >= 5)
{
throw new IOException(
"Couldn't run the following Mercurial command successfully even after " + retryCount +
" tries due to an \"EOF occurred in violation of protocol\" error: " + hgCommand,
ex);
}
// Let's wait a bit before re-trying so our prayers can heal the connection in the meantime.
Thread.Sleep(10000);
return(RunRemoteHgCommandAndLogOutput(hgCommand, settings, ++retryCount));
}
// Catching warning-level "bitbucket.org certificate with fingerprint .. not verified (check
// hostfingerprints or web.cacerts config setting)" kind of errors that happen when mirroring happens
// accessing an insecure host.
if (!ex.Error.Contains("not verified (check hostfingerprints or web.cacerts config setting)"))
{
throw;
}
return(output);
}
}
private string RunHgCommandAndLogOutput(string hgCommand, MirroringSettings settings)
{
if (settings.MercurialSettings.UseDebug)
{
hgCommand += " --debug";
}
if (settings.MercurialSettings.UseTraceback)
{
hgCommand += " --traceback";
}
return(RunCommandAndLogOutput(hgCommand));
}
public void CloneGit(
Uri gitCloneUri,
string quotedCloneDirectoryPath,
MirroringSettings settings,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
CdDirectory(quotedCloneDirectoryPath);
// Cloning a large git repo will work even when (after cloning the corresponding hg repo) pulling it in will
// fail with a "the connection was forcibly closed by remote host"-like error. This is why we start with
// cloning the git repo.
RunGitRepoCommand(gitCloneUri, "clone --noupdate {url} " + quotedCloneDirectoryPath, settings);
}
static void Main(string[] args)
{
if (!EventLog.Exists("Git-hg Mirror Daemon"))
{
EventLog.CreateEventSource(new EventSourceCreationData("GitHgMirror.Tester", "Git-hg Mirror Daemon"));
}
using (var eventLog = new EventLog("Git-hg Mirror Daemon", ".", "GitHgMirror.Tester"))
{
eventLog.EnableRaisingEvents = true;
eventLog.EntryWritten += (sender, e) =>
{
Console.WriteLine(e.Entry.Message);
};
var settings = new MirroringSettings
{
ApiEndpointUrl = new Uri("http://githgmirror.com.127-0-0-1.org.uk/api/GitHgMirror.Common/Mirrorings"),
ApiPassword = "******",
RepositoriesDirectoryPath = @"C:\GitHgMirror\Repos",
MaxDegreeOfParallelism = 1,
BatchSize = 1
};
// Uncomment if you want to also test repo cleaning.
//new UntouchedRepositoriesCleaner(settings, eventLog).Clean(new CancellationTokenSource().Token);
var runner = new MirrorRunner(settings, eventLog);
// On exit with Ctrl+C
Console.CancelKeyPress += (sender, e) =>
{
runner.Stop();
};
runner.Start();
_waitHandle.WaitOne();
}
}
public void MirrorRepositories(
MirroringConfiguration configuration,
MirroringSettings settings,
CancellationToken cancellationToken)
{
var descriptor = GetMirroringDescriptor(configuration);
var loggedDescriptor = descriptor + " (#" + configuration.Id.ToString(CultureInfo.InvariantCulture) + ")";
Debug.WriteLine("Starting mirroring: " + loggedDescriptor);
_eventLog.WriteEntry("Starting mirroring: " + loggedDescriptor);
var repositoryDirectoryName = GetCloneDirectoryName(configuration);
// A subfolder per clone dir start letter:
var cloneDirectoryParentPath = Path.Combine(settings.RepositoriesDirectoryPath, repositoryDirectoryName[0].ToString());
var cloneDirectoryPath = Path.Combine(cloneDirectoryParentPath, repositoryDirectoryName);
var repositoryLockFilePath = GetRepositoryLockFilePath(cloneDirectoryPath);
try
{
if (File.Exists(repositoryLockFilePath))
{
var logEntryStart =
"An existing lock was found for the mirroring configuration " + loggedDescriptor + ". ";
var lastUpdatedTimeUtc = RepositoryInfoFileHelper.GetLastUpdatedDateTimeUtc(cloneDirectoryPath);
if (lastUpdatedTimeUtc >= DateTime.UtcNow.AddSeconds(-settings.MirroringTimoutSeconds))
{
_eventLog.WriteEntry(
logEntryStart +
"This can mean that the number of configurations was reduced and thus while a mirroring was" +
" running a new process for the same repositories was started. We'll let the initial process finish.");
return;
}
else
{
_eventLog.WriteEntry(
logEntryStart +
"Additionally the directory was last touched at " + lastUpdatedTimeUtc.ToString(CultureInfo.InvariantCulture) +
" UTC which is older than the allowed mirroring timeout (" + settings.MirroringTimoutSeconds +
"s). Thus the lock is considered abandoned and mirroring will continue.",
EventLogEntryType.Warning);
}
}
if (configuration.HgCloneUri.Scheme.Equals("ssh", StringComparison.OrdinalIgnoreCase) ||
configuration.GitCloneUri.Scheme.Equals("ssh", StringComparison.OrdinalIgnoreCase))
{
throw new MirroringException("SSH protocol is not supported, only HTTPS.");
}
if (!Directory.Exists(cloneDirectoryParentPath))
{
Directory.CreateDirectory(cloneDirectoryParentPath);
}
File.Create(repositoryLockFilePath).Dispose();
// Changing directory to other drive if necessary.
RunCommandAndLogOutput(Path.GetPathRoot(cloneDirectoryPath).Replace("\\", string.Empty));
var quotedHgCloneUrl = configuration.HgCloneUri.ToString().EncloseInQuotes();
var quotedGitCloneUrl = configuration.GitCloneUri.ToString().EncloseInQuotes();
var quotedCloneDirectoryPath = cloneDirectoryPath.EncloseInQuotes();
var isCloned = IsCloned(configuration, settings);
if (!isCloned)
{
DeleteDirectoryIfExists(cloneDirectoryPath);
Directory.CreateDirectory(cloneDirectoryPath);
}
RepositoryInfoFileHelper.CreateOrUpdateFile(cloneDirectoryPath, descriptor);
// Mirroring between two git repos is supported, but in a hacked-in way at the moment. This needs a
// clean-up. Also, do note that only GitToHg and TwoWay is implemented. It would make the whole thing
// even more messy to duplicate the logic in HgToGit.
var hgUrlIsGitUrl = configuration.HgCloneUri.Scheme == "git+https";
cancellationToken.ThrowIfCancellationRequested();
// It'll be fine for now.
#pragma warning disable S1151 // "switch case" clauses should not have too many lines of code
switch (configuration.Direction)
{
case MirroringDirection.GitToHg:
if (hgUrlIsGitUrl)
{
RunGitCommandAndMarkException(() => _gitCommandExecutor
.FetchOrCloneFromGit(configuration.GitCloneUri, cloneDirectoryPath, true, cancellationToken));
_gitCommandExecutor.PushToGit(configuration.HgCloneUri, cloneDirectoryPath, cancellationToken);
}
else
{
if (isCloned)
{
if (configuration.GitUrlIsHgUrl)
{
_hgCommandExecutor
.PullHg(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
}
else
{
RunGitCommandAndMarkException(() => _gitCommandExecutor
.FetchOrCloneFromGit(configuration.GitCloneUri, cloneDirectoryPath, true, cancellationToken));
_hgCommandExecutor.ImportHistoryFromGit(quotedCloneDirectoryPath, settings, cancellationToken);
}
}
else
{
if (configuration.GitUrlIsHgUrl)
{
_hgCommandExecutor
.CloneHg(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
}
else
{
_hgCommandExecutor
.CloneGit(configuration.GitCloneUri, quotedCloneDirectoryPath, settings, cancellationToken);
}
}
_hgCommandExecutor
.PushWithBookmarks(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
}
break;
case MirroringDirection.HgToGit:
if (isCloned)
{
_hgCommandExecutor.PullHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
}
else
{
_hgCommandExecutor.CloneHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
}
if (configuration.GitUrlIsHgUrl)
{
_hgCommandExecutor
.PushWithBookmarks(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
}
else
{
_hgCommandExecutor.CreateOrUpdateBookmarksForBranches(quotedCloneDirectoryPath, settings, cancellationToken);
_hgCommandExecutor.ExportHistoryToGit(quotedCloneDirectoryPath, settings, cancellationToken);
RunGitCommandAndMarkException(() =>
_gitCommandExecutor.PushToGit(configuration.GitCloneUri, cloneDirectoryPath, cancellationToken));
}
break;
case MirroringDirection.TwoWay:
Action syncHgAndGitHistories = () =>
{
_hgCommandExecutor
.CreateOrUpdateBookmarksForBranches(quotedCloneDirectoryPath, settings, cancellationToken);
_hgCommandExecutor.ExportHistoryToGit(quotedCloneDirectoryPath, settings, cancellationToken);
// This will clear all commits in the git repo that aren't in the git remote repo but
// add changes that were added to the git repo.
RunGitCommandAndMarkException(() => _gitCommandExecutor
.FetchOrCloneFromGit(configuration.GitCloneUri, cloneDirectoryPath, false, cancellationToken));
_hgCommandExecutor.ImportHistoryFromGit(quotedCloneDirectoryPath, settings, cancellationToken);
// Updating bookmarks which may have shifted after importing from git. This way the
// export to git will create a git repo with history identical to the hg repo.
_hgCommandExecutor.CreateOrUpdateBookmarksForBranches(quotedCloneDirectoryPath, settings, cancellationToken);
_hgCommandExecutor.ExportHistoryToGit(quotedCloneDirectoryPath, settings, cancellationToken);
RunGitCommandAndMarkException(() =>
_gitCommandExecutor.PushToGit(configuration.GitCloneUri, cloneDirectoryPath, cancellationToken));
};
if (hgUrlIsGitUrl)
{
// The easiest solution to do two-way git mirroring is to sync separately, with two clones.
// Otherwise when e.g. repository A adds a new commit, then repository B is pulled in, the
// head of the branch will be at the point where it is in B. Thus pushing to A will fail
// with "Cannot push non-fastforwardable reference". There are other similar errors that can
// arise but can't easily be fixed automatically in a safe way. So first pulling both repos
// then pushing them won't work.
var gitDirectoryPath = GitCommandExecutor.GetGitDirectoryPath(cloneDirectoryPath);
var secondToFirstClonePath = Path.Combine(gitDirectoryPath, "secondToFirst");
void PullSecondPushToFirst()
{
RunGitCommandAndMarkException(() => _gitCommandExecutor
.FetchOrCloneFromGit(configuration.HgCloneUri, secondToFirstClonePath, true, cancellationToken));
RunGitCommandAndMarkException(() => _gitCommandExecutor
.PushToGit(configuration.GitCloneUri, secondToFirstClonePath, cancellationToken));
}
var firstToSecondClonePath = Path.Combine(gitDirectoryPath, "firstToSecond");
RunGitCommandAndMarkException(() => _gitCommandExecutor
.FetchOrCloneFromGit(configuration.GitCloneUri, firstToSecondClonePath, true, cancellationToken));
try
{
RunGitCommandAndMarkException(() => _gitCommandExecutor
.PushToGit(configuration.HgCloneUri, firstToSecondClonePath, cancellationToken));
PullSecondPushToFirst();
}
catch (LibGit2SharpException ex)
when(ex.Message.Contains("Cannot push because a reference that you are trying to update on the remote contains commits that are not present locally."))
{
PullSecondPushToFirst();
// This exception can happen when the second repo contains changes not present in the
// first one. Then we need to update the first repo with the second's changes and pull-
// push again.
RunGitCommandAndMarkException(() => _gitCommandExecutor
.FetchOrCloneFromGit(configuration.GitCloneUri, firstToSecondClonePath, true, cancellationToken));
RunGitCommandAndMarkException(() => _gitCommandExecutor
.PushToGit(configuration.HgCloneUri, firstToSecondClonePath, cancellationToken));
}
}
else
{
if (isCloned)
{
_hgCommandExecutor.PullHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
if (configuration.GitUrlIsHgUrl)
{
_hgCommandExecutor
.PullHg(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
}
else
{
syncHgAndGitHistories();
}
}
else
{
if (configuration.GitUrlIsHgUrl)
{
_hgCommandExecutor
.CloneHg(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
_hgCommandExecutor
.PullHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
}
else
{
// We need to start with cloning the hg repo. Otherwise cloning the git repo, then
// pulling from the hg repo would yield a "repository unrelated" error, even if the
// git repo was created from the hg repo. For an explanation see:
// http://stackoverflow.com/questions/17240852/hg-git-clone-from-github-gives-abort-repository-is-unrelated
_hgCommandExecutor
.CloneHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
syncHgAndGitHistories();
}
}
_hgCommandExecutor
.PushWithBookmarks(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
if (configuration.GitUrlIsHgUrl)
{
_hgCommandExecutor
.PushWithBookmarks(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
}
}
break;
default:
throw new NotSupportedException("Not supported MirroringDirection.");
}
#pragma warning restore S1151 // "switch case" clauses should not have too many lines of code
Debug.WriteLine("Finished mirroring: " + loggedDescriptor);
_eventLog.WriteEntry("Finished mirroring: " + loggedDescriptor);
}
catch (Exception ex) when(!ex.IsFatalOrCancellation())
{
// We should dispose the command runners so the folder is not locked by the command line.
Dispose();
// Waiting a bit for any file locks or leases to be disposed even though CommandRunners and processes
// were killed.
Thread.Sleep(10000);
var exceptionMessage =
$"An error occured while running commands when mirroring the repositories {configuration.HgCloneUri} " +
$"and {configuration.GitCloneUri} in direction {configuration.Direction}. Mirroring will be re-started next time.";
try
{
// Re-cloning a repo is costly. During local debugging you can flip this variable from the Immediate
// Window to prevent it if necessary too.
var continueWithRepoFolderDelete = true;
// These git exceptions are caused by hg errors in a way, so despite them coming from git the whole
// repo folder should be removed.
var isHgOriginatedGitException =
ex.Message.Contains("does not match any existing object") ||
ex.Message.Contains("Object not found - failed to find pack entry");
if (ex.Data.Contains("IsGitException") && !isHgOriginatedGitException)
{
exceptionMessage += " The error was a git error.";
try
{
DeleteDirectoryIfExists(GitCommandExecutor.GetGitDirectoryPath(cloneDirectoryPath));
exceptionMessage += " Thus just the git folder was removed.";
continueWithRepoFolderDelete = false;
}
catch (Exception gitDirectoryDeleteException) when(!gitDirectoryDeleteException.IsFatalOrCancellation())
{
exceptionMessage +=
" While the removal of just the git folder was attempted it failed with the following " +
"exception, thus the deletion of the whole repository folder will be attempted: " +
gitDirectoryDeleteException;
// We'll continue with the repo folder removal below.
}
}
if (continueWithRepoFolderDelete)
{
DeleteDirectoryIfExists(cloneDirectoryPath);
RepositoryInfoFileHelper.DeleteFileIfExists(cloneDirectoryPath);
}
}
catch (Exception directoryDeleteException) when(!directoryDeleteException.IsFatalOrCancellation())
{
try
{
// This most possibly means that for some reason some process is still locking the folder
// although it shouldn't (mostly, but not definitely the cause of IOException) or there are
// read-only files (git pack files commonly are) which can be (but not always) behind
// UnauthorizedAccessException.
if (directoryDeleteException is IOException || directoryDeleteException is UnauthorizedAccessException)
{
var killResult = DirectoryUtil.KillProcessesLockingFiles(cloneDirectoryPath);
DeleteDirectoryIfExists(cloneDirectoryPath);
RepositoryInfoFileHelper.DeleteFileIfExists(cloneDirectoryPath);
exceptionMessage +=
" While deleting the folder of the mirror initially failed, after trying to kill processes " +
"that were locking files in it and setting all files not to be read-only the folder could be successfully deleted. " +
"Processes killed: " + (killResult.KilledProcesseFileNames.Any() ? string.Join(", ", killResult.KilledProcesseFileNames) : "no processes") +
" Read-only files: " + (killResult.ReadOnlyFilePaths.Any() ? string.Join(", ", killResult.ReadOnlyFilePaths) : "no files");
throw new MirroringException(exceptionMessage, ex, directoryDeleteException);
}
}
catch (Exception forcedCleanUpException)
when(!forcedCleanUpException.IsFatalOrCancellation() && !(forcedCleanUpException is MirroringException))
{
throw new MirroringException(
exceptionMessage + " Subsequently clean-up after the error failed as well, also the attempt " +
"to kill processes that were locking the mirror's folder and clearing all read-only files.",
ex,
directoryDeleteException,
forcedCleanUpException);
}
throw new MirroringException(
exceptionMessage + " Subsequently clean-up after the error failed as well.",
ex,
directoryDeleteException);
}
throw new MirroringException(exceptionMessage, ex);
}
finally
{
if (File.Exists(repositoryLockFilePath))
{
File.Delete(repositoryLockFilePath);
}
}
}
/// <summary>
/// Pulling chunks a repo history in chunks of revisions. This will be slow but surely work, even if one
/// changeset is huge like this one: http://hg.openjdk.java.net/openjfx/9-dev/rt/rev/86d5cbe0c60f (~100MB, 11000
/// files).
/// </summary>
private void PullPerRevisionsHg(
string quotedHgCloneUrl,
string quotedCloneDirectoryPath,
MirroringSettings settings,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
CdDirectory(quotedCloneDirectoryPath);
var startRevision = int.Parse(
RunHgCommandAndLogOutput("hg identify --rev tip --num", settings).Split(new[] { Environment.NewLine }, StringSplitOptions.None)[1],
CultureInfo.InvariantCulture);
var revision = startRevision + 1;
var finished = false;
var pullRetryCount = 0;
while (!finished)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
var output = RunRemoteHgCommandAndLogOutput(
"hg pull --rev " + revision + " " + quotedHgCloneUrl,
settings);
finished = output.Contains("no changes found");
// Let's try a normal pull every 300 revisions. If it succeeds then the mirroring can finish faster
// (otherwise it could even time out).
if (!finished && revision - startRevision >= 300)
{
PullHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
return;
}
revision++;
pullRetryCount = 0;
}
catch (CommandException pullException)
{
// This error happens when we try to go beyond existing revisions and it means we reached the end of
// the repo history. Maybe the hg identify command could be used to retrieve the latest revision
// number instead (see: https://selenic.com/hg/help/identify) although it says "can't query remote
// revision number, branch, or tag" (and even if it could, what if new changes are being pushed?).
// So using exceptions for now.
if (pullException.Error.Contains("abort: unknown revision "))
{
finished = true;
}
else if (pullException.IsHgConnectionTerminatedError() && pullRetryCount < 2)
{
// If such a pull fails then we can't fall back more, have to retry.
// It is used though.
#pragma warning disable S1854 // Unused assignments should be removed
pullRetryCount++;
#pragma warning restore S1854 // Unused assignments should be removed
// Letting temporary issues resolve themselves.
Thread.Sleep(30000);
}
else
{
throw;
}
}
}
}
public void CreateOrUpdateBookmarksForBranches(
string quotedCloneDirectoryPath,
MirroringSettings settings,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
CdDirectory(quotedCloneDirectoryPath);
// Adding bookmarks for all branches so they appear as proper git branches.
var branchesOutput = RunHgCommandAndLogOutput("hg branches --closed", settings);
var branches = branchesOutput
.Split(Environment.NewLine.ToArray())
.Skip(1) // The first line is the command itself
.Where(line => !string.IsNullOrEmpty(line))
.Select(line => Regex.Match(line, @"(.+?)\s+\d+:[a-z0-9]+").Groups[1].Value);
foreach (var branch in branches)
{
cancellationToken.ThrowIfCancellationRequested();
// Need to strip spaces from branch names, see:
// https://bitbucket.org/durin42/hg-git/issues/163/gexport-fails-on-bookmarks-with-spaces-in
var bookmark = branch.Replace(' ', '-');
// Need to strip multiple slashes from branch names, see:
// https://bitbucket.org/durin42/hg-git/issues/225/gexport-fails-on-bookmarks-with-multiple
if (bookmark.Count(character => character == '/') > 1)
{
var firstSlashIndex = bookmark.IndexOf('/');
bookmark = bookmark.Substring(0, firstSlashIndex) + bookmark.Substring(firstSlashIndex).Replace('/', '-');
}
if (branch == "default")
{
bookmark = "master" + GitBookmarkSuffix;
}
else
{
bookmark += GitBookmarkSuffix;
}
// Don't move the bookmark if on the changeset there is already a git bookmark, because this means that
// there was a branch created in git. E.g. we shouldn't move the master bookmark to the default head
// since with a new git branch there will be two default heads (since git branches are converted to
// bookmarks on default) and we'd wrongly move the master head.
var changesetLogOutput = RunHgCommandAndLogOutput("hg log -r " + branch.EncloseInQuotes(), settings);
// For hg log this is needed, otherwise the next command would return an empty line.
RunCommandAndLogOutput(Environment.NewLine);
var existingBookmarks = changesetLogOutput
.Split(Environment.NewLine.ToArray())
.Skip(1) // The first line is the command itself
.Where(line => !string.IsNullOrEmpty(line) && line.StartsWith("bookmark:", StringComparison.InvariantCulture))
.Select(line => Regex.Match(line, @"bookmark:\s+(.+)(\s|$)").Groups[1].Value);
if (!existingBookmarks.Any(existingBookmark => existingBookmark.EndsWith(GitBookmarkSuffix, StringComparison.InvariantCulture)))
{
// Need --force so it moves the bookmark if it already exists.
RunHgCommandAndLogOutput(
"hg bookmark -r " + branch.EncloseInQuotes() + " " + bookmark + " --force",
settings);
}
}
}