Пример #1
0
        public void CloneHg(
            string quotedHgCloneUrl,
            string quotedCloneDirectoryPath,
            MirroringSettings settings,
            CancellationToken cancellationToken)
        {
            cancellationToken.ThrowIfCancellationRequested();

            try
            {
                RunRemoteHgCommandAndLogOutput(
                    "hg clone --noupdate " + quotedHgCloneUrl + " " + quotedCloneDirectoryPath,
                    settings);
            }
            catch (CommandException ex) when(ex.IsHgConnectionTerminatedError())
            {
                _eventLog.WriteEntry(
                    "Cloning from the Mercurial repo " + quotedHgCloneUrl + " failed because the server terminated the connection. Re-trying by pulling revision by revision.",
                    EventLogEntryType.Warning);

                RunRemoteHgCommandAndLogOutput(
                    "hg clone --noupdate --rev 0 " + quotedHgCloneUrl + " " + quotedCloneDirectoryPath,
                    settings);

                cancellationToken.ThrowIfCancellationRequested();

                PullPerRevisionsHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
            }
        }
Пример #2
0
        protected override void OnStart(string[] args)
        {
            if (!EventLog.Exists("Git-hg Mirror Daemon"))
            {
                EventLog.CreateEventSource(new EventSourceCreationData("GitHgMirror.Daemon", "Git-hg Mirror Daemon"));
            }

            // Keep in mind that the event log consumes memory so unless you keep this well beyond what you can spare
            // the server will run out of RAM.
            serviceEventLog.MaximumKilobytes = 196608; // 192MB
            serviceEventLog.WriteEntry("GitHgMirrorDaemon started.");

            _settings = new MirroringSettings
            {
                ApiEndpointUrl            = new Uri(ConfigurationManager.AppSettings[Constants.ApiEndpointUrl]),
                ApiPassword               = ConfigurationManager.ConnectionStrings[Constants.ApiPasswordKey]?.ConnectionString ?? string.Empty,
                RepositoriesDirectoryPath = ConfigurationManager.AppSettings[Constants.RepositoriesDirectoryPath],
                BatchSize = int.Parse(ConfigurationManager.AppSettings[Constants.BatchSize], CultureInfo.InvariantCulture),
            };

            _startTimer          = new System.Timers.Timer(10000);
            _startTimer.Elapsed += StartTimerElapsed;
            _startTimer.Enabled  = true;

            _cleaner             = new UntouchedRepositoriesCleaner(_settings, serviceEventLog);
            _cleanTimer          = new System.Timers.Timer(3_600_000 * 2); // Two hours
            _cleanTimer.Elapsed += (sender, e) => _cleaner.Clean(_cancellationTokenSource.Token);
            _cleanTimer.Enabled  = true;
        }
Пример #3
0
        /// <summary>
        /// Runs the specified command for a git repo in hg.
        /// </summary>
        /// <param name="gitCloneUri">The git clone URI.</param>
        /// <param name="command">
        /// The command, including an optional placeholder for the git URL in form of {url}, e.g.: "clone --noupdate
        /// {url}".
        /// </param>
        private void RunGitRepoCommand(Uri gitCloneUri, string command, MirroringSettings settings)
        {
            var gitUriBuilder = new UriBuilder(gitCloneUri);
            var userName      = gitUriBuilder.UserName;
            var password      = gitUriBuilder.Password;

            gitUriBuilder.UserName = null;
            gitUriBuilder.Password = null;
            var gitUri            = gitUriBuilder.Uri;
            var quotedGitCloneUrl = gitUri.ToString().EncloseInQuotes();

            command = command.Replace("{url}", quotedGitCloneUrl);

            // Would be quite ugly that way.
#pragma warning disable S3240 // The simplest possible condition syntax should be used
            if (!string.IsNullOrEmpty(userName))
#pragma warning restore S3240 // The simplest possible condition syntax should be used
            {
                RunRemoteHgCommandAndLogOutput(
                    PrefixHgCommandWithHgGitConfig(
                        "--config auth.rc.prefix=" +
                        ("https://" + gitUri.Host).EncloseInQuotes() +
                        " --config auth.rc.username="******" --config auth.rc.password="******" " +
                        command),
                    settings);
            }
            else
            {
                RunRemoteHgCommandAndLogOutput(PrefixHgCommandWithHgGitConfig(command), settings);
            }
        }
Пример #4
0
        /// <summary>
        /// Runs the specified command for a git repo in hg.
        /// </summary>
        /// <param name="gitCloneUri">The git clone URI.</param>
        /// <param name="command">
        /// The command, including an optional placeholder for the git URL in form of {url}, e.g.: "clone --noupdate {url}".
        /// </param>
        private void RunGitRepoCommand(Uri gitCloneUri, string command, MirroringSettings settings)
        {
            var gitUriBuilder = new UriBuilder(gitCloneUri);
            var userName      = gitUriBuilder.UserName;
            var password      = gitUriBuilder.Password;

            gitUriBuilder.UserName = null;
            gitUriBuilder.Password = null;
            var gitUri            = gitUriBuilder.Uri;
            var quotedGitCloneUrl = gitUri.ToString().EncloseInQuotes();

            command = command.Replace("{url}", quotedGitCloneUrl);

            if (!string.IsNullOrEmpty(userName))
            {
                RunRemoteHgCommandAndLogOutput(
                    PrefixHgCommandWithHgGitConfig(
                        "--config auth.rc.prefix=" +
                        ("https://" + gitUri.Host).EncloseInQuotes() +
                        " --config auth.rc.username="******" --config auth.rc.password="******" " +
                        command),
                    settings);
            }
            else
            {
                RunRemoteHgCommandAndLogOutput(PrefixHgCommandWithHgGitConfig(command), settings);
            }
        }
Пример #5
0
        public bool IsCloned(MirroringConfiguration configuration, MirroringSettings settings)
        {
            var repositoryDirectoryName = GetCloneDirectoryName(configuration);
            // A subfolder per clone dir start letter.
            var cloneDirectoryParentPath = Path.Combine(settings.RepositoriesDirectoryPath, repositoryDirectoryName[0].ToString());
            var cloneDirectoryPath       = Path.Combine(cloneDirectoryParentPath, repositoryDirectoryName);

            // Also checking if the directory is empty. If yes, it was a failed attempt and really the repo is not cloned.
            return(Directory.Exists(cloneDirectoryPath) && Directory.EnumerateFileSystemEntries(cloneDirectoryPath).Any());
        }
Пример #6
0
        public void ExportHistoryToGit(
            string quotedCloneDirectoryPath,
            MirroringSettings settings,
            CancellationToken cancellationToken)
        {
            cancellationToken.ThrowIfCancellationRequested();

            CdDirectory(quotedCloneDirectoryPath);
            RunHgCommandAndLogOutput(PrefixHgCommandWithHgGitConfig("gexport"), settings);
        }
Пример #7
0
        public void PushWithBookmarks(
            string quotedHgCloneUrl,
            string quotedCloneDirectoryPath,
            MirroringSettings settings,
            CancellationToken cancellationToken)
        {
            cancellationToken.ThrowIfCancellationRequested();

            CdDirectory(quotedCloneDirectoryPath);

            var bookmarksOutput = RunHgCommandAndLogOutput("hg bookmarks", settings);

            // There will be at least one bookmark, "master" with a git repo. However with hg-hg mirroring maybe there
            // are no bookmarks.
            if (bookmarksOutput.Contains("no bookmarks set"))
            {
                RunRemoteHgCommandAndLogOutput("hg push --new-branch --force " + quotedHgCloneUrl, settings);
            }
            else
            {
                var bookmarks = bookmarksOutput
                                .Split(Environment.NewLine.ToArray())
                                .Skip(1) // The first line is the command itself
                                .Where(line => !string.IsNullOrEmpty(line))
                                .Select(line => Regex.Match(line, @"\s([a-zA-Z0-9/.\-_]+)\s", RegexOptions.IgnoreCase).Groups[1].Value)
                                .Where(line => !string.IsNullOrEmpty(line))
                                .Select(line => "-B " + line);

                // Pushing a lot of bookmarks at once would result in a "RuntimeError: maximum recursion depth exceeded"
                // error.
                const int batchSize      = 29;
                var       bookmarksBatch = bookmarks.Take(batchSize);
                var       skip           = 0;
                var       bookmarkCount  = bookmarks.Count();
                while (skip < bookmarkCount)
                {
                    cancellationToken.ThrowIfCancellationRequested();

                    RunRemoteHgCommandAndLogOutput(
                        "hg push --new-branch --force " + string.Join(" ", bookmarksBatch) + " " + quotedHgCloneUrl,
                        settings);
                    skip          += batchSize;
                    bookmarksBatch = bookmarks.Skip(skip).Take(batchSize);
                    if (bookmarksBatch.Any())
                    {
                        // Bitbucket throttles such requests so we need to slow down. Otherwise we'd get this error:
                        // "remote: Push throttled (max allowable rate: 30 per 60 seconds)." However, this is wrong as
                        // the actual limit is lower at 29.
                        Thread.Sleep(61000);
                    }
                }
            }
        }
Пример #8
0
        public static void Main()
        {
            // If true then a unique event log will be used for all copies of this executable. This helps if you want to
            // run the app in multiple instances from source and not let the events show up across copies.
            const bool useUniqueEventlog = true;

            var eventLogName    = "Git-hg Mirror Daemon";
            var eventSourceName = "GitHgMirror.Tester";

            if (useUniqueEventlog)
            {
                var suffix = "-" + typeof(Program).Assembly.Location.GetHashCode();
                // "Only the first eight characters of a custom log name are significant" so we need to make the name
                // unique withing 8 characters.
                eventLogName     = "GHM" + suffix;
                eventSourceName += suffix;
            }

            if (!EventLog.Exists(eventLogName))
            {
                EventLog.CreateEventSource(new EventSourceCreationData(eventSourceName, eventLogName));
            }

            using var eventLog = new EventLog(eventLogName, ".", eventSourceName)
                  {
                      EnableRaisingEvents = true,
                  };

            eventLog.EntryWritten += (sender, e) => Console.WriteLine(e.Entry.Message);

            var settings = new MirroringSettings
            {
                ApiEndpointUrl            = new Uri("http://githgmirror.com.127-0-0-1.org.uk/api/GitHgMirror.Common/Mirrorings"),
                ApiPassword               = "******",
                RepositoriesDirectoryPath = @"C:\GitHgMirror\Repos",
                MaxDegreeOfParallelism    = 1,
                BatchSize = 1,
            };

            // Uncomment if you want to also test repo cleaning.
            ////new UntouchedRepositoriesCleaner(settings, eventLog).Clean(new CancellationTokenSource().Token);

            using var runner = new MirrorRunner(settings, eventLog);

            // On exit with Ctrl+C
            Console.CancelKeyPress += (sender, e) => runner.Stop();

            runner.Start();

            _waitHandle.WaitOne();
        }
Пример #9
0
        private string RunRemoteHgCommandAndLogOutput(string hgCommand, MirroringSettings settings, int retryCount = 0)
        {
            var output = string.Empty;

            try
            {
                if (settings.MercurialSettings.UseInsecure)
                {
                    hgCommand += " --insecure";
                }

                if (settings.MercurialSettings.UseDebugForRemoteCommands && !settings.MercurialSettings.UseDebug)
                {
                    hgCommand += " --debug";
                }

                output = RunHgCommandAndLogOutput(hgCommand, settings);

                return(output);
            }
            catch (CommandException ex)
            {
                // We'll randomly get such errors when interacting with Mercurial as well as with Git, otherwise
                // properly running repos. So we re-try the operation a few times, maybe it'll work...
                if (ex.Error.Contains("EOF occurred in violation of protocol"))
                {
                    if (retryCount >= 5)
                    {
                        throw new IOException(
                                  "Couldn't run the following Mercurial command successfully even after " + retryCount +
                                  " tries due to an \"EOF occurred in violation of protocol\" error: " + hgCommand,
                                  ex);
                    }

                    // Let's wait a bit before re-trying so our prayers can heal the connection in the meantime.
                    Thread.Sleep(10000);

                    return(RunRemoteHgCommandAndLogOutput(hgCommand, settings, ++retryCount));
                }

                // Catching warning-level "bitbucket.org certificate with fingerprint .. not verified (check
                // hostfingerprints or web.cacerts config setting)" kind of errors that happen when mirroring happens
                // accessing an insecure host.
                if (!ex.Error.Contains("not verified (check hostfingerprints or web.cacerts config setting)"))
                {
                    throw;
                }

                return(output);
            }
        }
Пример #10
0
        private string RunHgCommandAndLogOutput(string hgCommand, MirroringSettings settings)
        {
            if (settings.MercurialSettings.UseDebug)
            {
                hgCommand += " --debug";
            }

            if (settings.MercurialSettings.UseTraceback)
            {
                hgCommand += " --traceback";
            }

            return(RunCommandAndLogOutput(hgCommand));
        }
Пример #11
0
        public void CloneGit(
            Uri gitCloneUri,
            string quotedCloneDirectoryPath,
            MirroringSettings settings,
            CancellationToken cancellationToken)
        {
            cancellationToken.ThrowIfCancellationRequested();

            CdDirectory(quotedCloneDirectoryPath);
            // Cloning a large git repo will work even when (after cloning the corresponding hg repo) pulling it in will
            // fail with a "the connection was forcibly closed by remote host"-like error. This is why we start with
            // cloning the git repo.
            RunGitRepoCommand(gitCloneUri, "clone --noupdate {url} " + quotedCloneDirectoryPath, settings);
        }
Пример #12
0
        static void Main(string[] args)
        {
            if (!EventLog.Exists("Git-hg Mirror Daemon"))
            {
                EventLog.CreateEventSource(new EventSourceCreationData("GitHgMirror.Tester", "Git-hg Mirror Daemon"));
            }

            using (var eventLog = new EventLog("Git-hg Mirror Daemon", ".", "GitHgMirror.Tester"))
            {
                eventLog.EnableRaisingEvents = true;

                eventLog.EntryWritten += (sender, e) =>
                {
                    Console.WriteLine(e.Entry.Message);
                };


                var settings = new MirroringSettings
                {
                    ApiEndpointUrl            = new Uri("http://githgmirror.com.127-0-0-1.org.uk/api/GitHgMirror.Common/Mirrorings"),
                    ApiPassword               = "******",
                    RepositoriesDirectoryPath = @"C:\GitHgMirror\Repos",
                    MaxDegreeOfParallelism    = 1,
                    BatchSize = 1
                };

                // Uncomment if you want to also test repo cleaning.
                //new UntouchedRepositoriesCleaner(settings, eventLog).Clean(new CancellationTokenSource().Token);

                var runner = new MirrorRunner(settings, eventLog);

                // On exit with Ctrl+C
                Console.CancelKeyPress += (sender, e) =>
                {
                    runner.Stop();
                };

                runner.Start();

                _waitHandle.WaitOne();
            }
        }
Пример #13
0
        public void MirrorRepositories(
            MirroringConfiguration configuration,
            MirroringSettings settings,
            CancellationToken cancellationToken)
        {
            var descriptor       = GetMirroringDescriptor(configuration);
            var loggedDescriptor = descriptor + " (#" + configuration.Id.ToString(CultureInfo.InvariantCulture) + ")";

            Debug.WriteLine("Starting mirroring: " + loggedDescriptor);
            _eventLog.WriteEntry("Starting mirroring: " + loggedDescriptor);

            var repositoryDirectoryName = GetCloneDirectoryName(configuration);
            // A subfolder per clone dir start letter:
            var cloneDirectoryParentPath = Path.Combine(settings.RepositoriesDirectoryPath, repositoryDirectoryName[0].ToString());
            var cloneDirectoryPath       = Path.Combine(cloneDirectoryParentPath, repositoryDirectoryName);
            var repositoryLockFilePath   = GetRepositoryLockFilePath(cloneDirectoryPath);

            try
            {
                if (File.Exists(repositoryLockFilePath))
                {
                    var logEntryStart =
                        "An existing lock was found for the mirroring configuration " + loggedDescriptor + ". ";
                    var lastUpdatedTimeUtc = RepositoryInfoFileHelper.GetLastUpdatedDateTimeUtc(cloneDirectoryPath);

                    if (lastUpdatedTimeUtc >= DateTime.UtcNow.AddSeconds(-settings.MirroringTimoutSeconds))
                    {
                        _eventLog.WriteEntry(
                            logEntryStart +
                            "This can mean that the number of configurations was reduced and thus while a mirroring was" +
                            " running a new process for the same repositories was started. We'll let the initial process finish.");

                        return;
                    }
                    else
                    {
                        _eventLog.WriteEntry(
                            logEntryStart +
                            "Additionally the directory was last touched at " + lastUpdatedTimeUtc.ToString(CultureInfo.InvariantCulture) +
                            " UTC which is older than the allowed mirroring timeout (" + settings.MirroringTimoutSeconds +
                            "s). Thus the lock is considered abandoned and mirroring will continue.",
                            EventLogEntryType.Warning);
                    }
                }

                if (configuration.HgCloneUri.Scheme.Equals("ssh", StringComparison.OrdinalIgnoreCase) ||
                    configuration.GitCloneUri.Scheme.Equals("ssh", StringComparison.OrdinalIgnoreCase))
                {
                    throw new MirroringException("SSH protocol is not supported, only HTTPS.");
                }

                if (!Directory.Exists(cloneDirectoryParentPath))
                {
                    Directory.CreateDirectory(cloneDirectoryParentPath);
                }

                File.Create(repositoryLockFilePath).Dispose();

                // Changing directory to other drive if necessary.
                RunCommandAndLogOutput(Path.GetPathRoot(cloneDirectoryPath).Replace("\\", string.Empty));

                var quotedHgCloneUrl         = configuration.HgCloneUri.ToString().EncloseInQuotes();
                var quotedGitCloneUrl        = configuration.GitCloneUri.ToString().EncloseInQuotes();
                var quotedCloneDirectoryPath = cloneDirectoryPath.EncloseInQuotes();
                var isCloned = IsCloned(configuration, settings);

                if (!isCloned)
                {
                    DeleteDirectoryIfExists(cloneDirectoryPath);
                    Directory.CreateDirectory(cloneDirectoryPath);
                }

                RepositoryInfoFileHelper.CreateOrUpdateFile(cloneDirectoryPath, descriptor);

                // Mirroring between two git repos is supported, but in a hacked-in way at the moment. This needs a
                // clean-up. Also, do note that only GitToHg and TwoWay is implemented. It would make the whole thing
                // even more messy to duplicate the logic in HgToGit.
                var hgUrlIsGitUrl = configuration.HgCloneUri.Scheme == "git+https";

                cancellationToken.ThrowIfCancellationRequested();

                // It'll be fine for now.
#pragma warning disable S1151 // "switch case" clauses should not have too many lines of code
                switch (configuration.Direction)
                {
                case MirroringDirection.GitToHg:
                    if (hgUrlIsGitUrl)
                    {
                        RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                      .FetchOrCloneFromGit(configuration.GitCloneUri, cloneDirectoryPath, true, cancellationToken));
                        _gitCommandExecutor.PushToGit(configuration.HgCloneUri, cloneDirectoryPath, cancellationToken);
                    }
                    else
                    {
                        if (isCloned)
                        {
                            if (configuration.GitUrlIsHgUrl)
                            {
                                _hgCommandExecutor
                                .PullHg(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                            }
                            else
                            {
                                RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                              .FetchOrCloneFromGit(configuration.GitCloneUri, cloneDirectoryPath, true, cancellationToken));
                                _hgCommandExecutor.ImportHistoryFromGit(quotedCloneDirectoryPath, settings, cancellationToken);
                            }
                        }
                        else
                        {
                            if (configuration.GitUrlIsHgUrl)
                            {
                                _hgCommandExecutor
                                .CloneHg(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                            }
                            else
                            {
                                _hgCommandExecutor
                                .CloneGit(configuration.GitCloneUri, quotedCloneDirectoryPath, settings, cancellationToken);
                            }
                        }

                        _hgCommandExecutor
                        .PushWithBookmarks(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                    }

                    break;

                case MirroringDirection.HgToGit:
                    if (isCloned)
                    {
                        _hgCommandExecutor.PullHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                    }
                    else
                    {
                        _hgCommandExecutor.CloneHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                    }

                    if (configuration.GitUrlIsHgUrl)
                    {
                        _hgCommandExecutor
                        .PushWithBookmarks(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                    }
                    else
                    {
                        _hgCommandExecutor.CreateOrUpdateBookmarksForBranches(quotedCloneDirectoryPath, settings, cancellationToken);
                        _hgCommandExecutor.ExportHistoryToGit(quotedCloneDirectoryPath, settings, cancellationToken);
                        RunGitCommandAndMarkException(() =>
                                                      _gitCommandExecutor.PushToGit(configuration.GitCloneUri, cloneDirectoryPath, cancellationToken));
                    }

                    break;

                case MirroringDirection.TwoWay:
                    Action syncHgAndGitHistories = () =>
                    {
                        _hgCommandExecutor
                        .CreateOrUpdateBookmarksForBranches(quotedCloneDirectoryPath, settings, cancellationToken);
                        _hgCommandExecutor.ExportHistoryToGit(quotedCloneDirectoryPath, settings, cancellationToken);

                        // This will clear all commits in the git repo that aren't in the git remote repo but
                        // add changes that were added to the git repo.
                        RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                      .FetchOrCloneFromGit(configuration.GitCloneUri, cloneDirectoryPath, false, cancellationToken));
                        _hgCommandExecutor.ImportHistoryFromGit(quotedCloneDirectoryPath, settings, cancellationToken);

                        // Updating bookmarks which may have shifted after importing from git. This way the
                        // export to git will create a git repo with history identical to the hg repo.
                        _hgCommandExecutor.CreateOrUpdateBookmarksForBranches(quotedCloneDirectoryPath, settings, cancellationToken);
                        _hgCommandExecutor.ExportHistoryToGit(quotedCloneDirectoryPath, settings, cancellationToken);

                        RunGitCommandAndMarkException(() =>
                                                      _gitCommandExecutor.PushToGit(configuration.GitCloneUri, cloneDirectoryPath, cancellationToken));
                    };

                    if (hgUrlIsGitUrl)
                    {
                        // The easiest solution to do two-way git mirroring is to sync separately, with two clones.
                        // Otherwise when e.g. repository A adds a new commit, then repository B is pulled in, the
                        // head of the branch will be at the point where it is in B. Thus pushing to A will fail
                        // with "Cannot push non-fastforwardable reference". There are other similar errors that can
                        // arise but can't easily be fixed automatically in a safe way. So first pulling both repos
                        // then pushing them won't work.

                        var gitDirectoryPath = GitCommandExecutor.GetGitDirectoryPath(cloneDirectoryPath);

                        var secondToFirstClonePath = Path.Combine(gitDirectoryPath, "secondToFirst");
                        void PullSecondPushToFirst()
                        {
                            RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                          .FetchOrCloneFromGit(configuration.HgCloneUri, secondToFirstClonePath, true, cancellationToken));
                            RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                          .PushToGit(configuration.GitCloneUri, secondToFirstClonePath, cancellationToken));
                        }

                        var firstToSecondClonePath = Path.Combine(gitDirectoryPath, "firstToSecond");
                        RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                      .FetchOrCloneFromGit(configuration.GitCloneUri, firstToSecondClonePath, true, cancellationToken));
                        try
                        {
                            RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                          .PushToGit(configuration.HgCloneUri, firstToSecondClonePath, cancellationToken));

                            PullSecondPushToFirst();
                        }
                        catch (LibGit2SharpException ex)
                            when(ex.Message.Contains("Cannot push because a reference that you are trying to update on the remote contains commits that are not present locally."))
                            {
                                PullSecondPushToFirst();

                                // This exception can happen when the second repo contains changes not present in the
                                // first one. Then we need to update the first repo with the second's changes and pull-
                                // push again.
                                RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                              .FetchOrCloneFromGit(configuration.GitCloneUri, firstToSecondClonePath, true, cancellationToken));
                                RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                              .PushToGit(configuration.HgCloneUri, firstToSecondClonePath, cancellationToken));
                            }
                    }
                    else
                    {
                        if (isCloned)
                        {
                            _hgCommandExecutor.PullHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);

                            if (configuration.GitUrlIsHgUrl)
                            {
                                _hgCommandExecutor
                                .PullHg(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                            }
                            else
                            {
                                syncHgAndGitHistories();
                            }
                        }
                        else
                        {
                            if (configuration.GitUrlIsHgUrl)
                            {
                                _hgCommandExecutor
                                .CloneHg(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                                _hgCommandExecutor
                                .PullHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                            }
                            else
                            {
                                // We need to start with cloning the hg repo. Otherwise cloning the git repo, then
                                // pulling from the hg repo would yield a "repository unrelated" error, even if the
                                // git repo was created from the hg repo. For an explanation see:
                                // http://stackoverflow.com/questions/17240852/hg-git-clone-from-github-gives-abort-repository-is-unrelated
                                _hgCommandExecutor
                                .CloneHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);

                                syncHgAndGitHistories();
                            }
                        }

                        _hgCommandExecutor
                        .PushWithBookmarks(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);

                        if (configuration.GitUrlIsHgUrl)
                        {
                            _hgCommandExecutor
                            .PushWithBookmarks(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                        }
                    }

                    break;

                default:
                    throw new NotSupportedException("Not supported MirroringDirection.");
                }
#pragma warning restore S1151 // "switch case" clauses should not have too many lines of code

                Debug.WriteLine("Finished mirroring: " + loggedDescriptor);
                _eventLog.WriteEntry("Finished mirroring: " + loggedDescriptor);
            }
            catch (Exception ex) when(!ex.IsFatalOrCancellation())
            {
                // We should dispose the command runners so the folder is not locked by the command line.
                Dispose();
                // Waiting a bit for any file locks or leases to be disposed even though CommandRunners and processes
                // were killed.
                Thread.Sleep(10000);

                var exceptionMessage =
                    $"An error occured while running commands when mirroring the repositories {configuration.HgCloneUri} " +
                    $"and {configuration.GitCloneUri} in direction {configuration.Direction}. Mirroring will be re-started next time.";

                try
                {
                    // Re-cloning a repo is costly. During local debugging you can flip this variable from the Immediate
                    // Window to prevent it if necessary too.
                    var continueWithRepoFolderDelete = true;

                    // These git exceptions are caused by hg errors in a way, so despite them coming from git the whole
                    // repo folder should be removed.
                    var isHgOriginatedGitException =
                        ex.Message.Contains("does not match any existing object") ||
                        ex.Message.Contains("Object not found - failed to find pack entry");
                    if (ex.Data.Contains("IsGitException") && !isHgOriginatedGitException)
                    {
                        exceptionMessage += " The error was a git error.";

                        try
                        {
                            DeleteDirectoryIfExists(GitCommandExecutor.GetGitDirectoryPath(cloneDirectoryPath));

                            exceptionMessage            += " Thus just the git folder was removed.";
                            continueWithRepoFolderDelete = false;
                        }
                        catch (Exception gitDirectoryDeleteException) when(!gitDirectoryDeleteException.IsFatalOrCancellation())
                        {
                            exceptionMessage +=
                                " While the removal of just the git folder was attempted it failed with the following " +
                                "exception, thus the deletion of the whole repository folder will be attempted: " +
                                gitDirectoryDeleteException;

                            // We'll continue with the repo folder removal below.
                        }
                    }

                    if (continueWithRepoFolderDelete)
                    {
                        DeleteDirectoryIfExists(cloneDirectoryPath);
                        RepositoryInfoFileHelper.DeleteFileIfExists(cloneDirectoryPath);
                    }
                }
                catch (Exception directoryDeleteException) when(!directoryDeleteException.IsFatalOrCancellation())
                {
                    try
                    {
                        // This most possibly means that for some reason some process is still locking the folder
                        // although it shouldn't (mostly, but not definitely the cause of IOException) or there are
                        // read-only files (git pack files commonly are) which can be (but not always) behind
                        // UnauthorizedAccessException.
                        if (directoryDeleteException is IOException || directoryDeleteException is UnauthorizedAccessException)
                        {
                            var killResult = DirectoryUtil.KillProcessesLockingFiles(cloneDirectoryPath);

                            DeleteDirectoryIfExists(cloneDirectoryPath);
                            RepositoryInfoFileHelper.DeleteFileIfExists(cloneDirectoryPath);

                            exceptionMessage +=
                                " While deleting the folder of the mirror initially failed, after trying to kill processes " +
                                "that were locking files in it and setting all files not to be read-only the folder could be successfully deleted. " +
                                "Processes killed: " + (killResult.KilledProcesseFileNames.Any() ? string.Join(", ", killResult.KilledProcesseFileNames) : "no processes") +
                                " Read-only files: " + (killResult.ReadOnlyFilePaths.Any() ? string.Join(", ", killResult.ReadOnlyFilePaths) : "no files");

                            throw new MirroringException(exceptionMessage, ex, directoryDeleteException);
                        }
                    }
                    catch (Exception forcedCleanUpException)
                        when(!forcedCleanUpException.IsFatalOrCancellation() && !(forcedCleanUpException is MirroringException))
                        {
                            throw new MirroringException(
                                      exceptionMessage + " Subsequently clean-up after the error failed as well, also the attempt " +
                                      "to kill processes that were locking the mirror's folder and clearing all read-only files.",
                                      ex,
                                      directoryDeleteException,
                                      forcedCleanUpException);
                        }

                    throw new MirroringException(
                              exceptionMessage + " Subsequently clean-up after the error failed as well.",
                              ex,
                              directoryDeleteException);
                }

                throw new MirroringException(exceptionMessage, ex);
            }
            finally
            {
                if (File.Exists(repositoryLockFilePath))
                {
                    File.Delete(repositoryLockFilePath);
                }
            }
        }
Пример #14
0
        /// <summary>
        /// Pulling chunks a repo history in chunks of revisions. This will be slow but surely work, even if one
        /// changeset is huge like this one: http://hg.openjdk.java.net/openjfx/9-dev/rt/rev/86d5cbe0c60f (~100MB, 11000
        /// files).
        /// </summary>
        private void PullPerRevisionsHg(
            string quotedHgCloneUrl,
            string quotedCloneDirectoryPath,
            MirroringSettings settings,
            CancellationToken cancellationToken)
        {
            cancellationToken.ThrowIfCancellationRequested();

            CdDirectory(quotedCloneDirectoryPath);

            var startRevision = int.Parse(
                RunHgCommandAndLogOutput("hg identify --rev tip --num", settings).Split(new[] { Environment.NewLine }, StringSplitOptions.None)[1],
                CultureInfo.InvariantCulture);
            var revision = startRevision + 1;

            var finished       = false;
            var pullRetryCount = 0;

            while (!finished)
            {
                cancellationToken.ThrowIfCancellationRequested();

                try
                {
                    var output = RunRemoteHgCommandAndLogOutput(
                        "hg pull --rev " + revision + " " + quotedHgCloneUrl,
                        settings);

                    finished = output.Contains("no changes found");

                    // Let's try a normal pull every 300 revisions. If it succeeds then the mirroring can finish faster
                    // (otherwise it could even time out).
                    if (!finished && revision - startRevision >= 300)
                    {
                        PullHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                        return;
                    }

                    revision++;
                    pullRetryCount = 0;
                }
                catch (CommandException pullException)
                {
                    // This error happens when we try to go beyond existing revisions and it means we reached the end of
                    // the repo history. Maybe the hg identify command could be used to retrieve the latest revision
                    // number instead (see: https://selenic.com/hg/help/identify) although it says "can't query remote
                    // revision number, branch, or tag" (and even if it could, what if new changes are being pushed?).
                    // So using exceptions for now.
                    if (pullException.Error.Contains("abort: unknown revision "))
                    {
                        finished = true;
                    }
                    else if (pullException.IsHgConnectionTerminatedError() && pullRetryCount < 2)
                    {
                        // If such a pull fails then we can't fall back more, have to retry.

                        // It is used though.
#pragma warning disable S1854 // Unused assignments should be removed
                        pullRetryCount++;
#pragma warning restore S1854 // Unused assignments should be removed

                        // Letting temporary issues resolve themselves.
                        Thread.Sleep(30000);
                    }
                    else
                    {
                        throw;
                    }
                }
            }
        }
Пример #15
0
        public void CreateOrUpdateBookmarksForBranches(
            string quotedCloneDirectoryPath,
            MirroringSettings settings,
            CancellationToken cancellationToken)
        {
            cancellationToken.ThrowIfCancellationRequested();

            CdDirectory(quotedCloneDirectoryPath);

            // Adding bookmarks for all branches so they appear as proper git branches.
            var branchesOutput = RunHgCommandAndLogOutput("hg branches --closed", settings);

            var branches = branchesOutput
                           .Split(Environment.NewLine.ToArray())
                           .Skip(1) // The first line is the command itself
                           .Where(line => !string.IsNullOrEmpty(line))
                           .Select(line => Regex.Match(line, @"(.+?)\s+\d+:[a-z0-9]+").Groups[1].Value);

            foreach (var branch in branches)
            {
                cancellationToken.ThrowIfCancellationRequested();

                // Need to strip spaces from branch names, see:
                // https://bitbucket.org/durin42/hg-git/issues/163/gexport-fails-on-bookmarks-with-spaces-in
                var bookmark = branch.Replace(' ', '-');

                // Need to strip multiple slashes from branch names, see:
                // https://bitbucket.org/durin42/hg-git/issues/225/gexport-fails-on-bookmarks-with-multiple
                if (bookmark.Count(character => character == '/') > 1)
                {
                    var firstSlashIndex = bookmark.IndexOf('/');
                    bookmark = bookmark.Substring(0, firstSlashIndex) + bookmark.Substring(firstSlashIndex).Replace('/', '-');
                }

                if (branch == "default")
                {
                    bookmark = "master" + GitBookmarkSuffix;
                }
                else
                {
                    bookmark += GitBookmarkSuffix;
                }

                // Don't move the bookmark if on the changeset there is already a git bookmark, because this means that
                // there was a branch created in git. E.g. we shouldn't move the master bookmark to the default head
                // since with a new git branch there will be two default heads (since git branches are converted to
                // bookmarks on default) and we'd wrongly move the master head.
                var changesetLogOutput = RunHgCommandAndLogOutput("hg log -r " + branch.EncloseInQuotes(), settings);
                // For hg log this is needed, otherwise the next command would return an empty line.
                RunCommandAndLogOutput(Environment.NewLine);

                var existingBookmarks = changesetLogOutput
                                        .Split(Environment.NewLine.ToArray())
                                        .Skip(1) // The first line is the command itself
                                        .Where(line => !string.IsNullOrEmpty(line) && line.StartsWith("bookmark:", StringComparison.InvariantCulture))
                                        .Select(line => Regex.Match(line, @"bookmark:\s+(.+)(\s|$)").Groups[1].Value);

                if (!existingBookmarks.Any(existingBookmark => existingBookmark.EndsWith(GitBookmarkSuffix, StringComparison.InvariantCulture)))
                {
                    // Need --force so it moves the bookmark if it already exists.
                    RunHgCommandAndLogOutput(
                        "hg bookmark -r " + branch.EncloseInQuotes() + " " + bookmark + " --force",
                        settings);
                }
            }
        }