コード例 #1
0
ファイル: Mirror.cs プロジェクト: Lombiq/Git-hg-Mirror-Daemon
        public void MirrorRepositories(
            MirroringConfiguration configuration,
            MirroringSettings settings,
            CancellationToken cancellationToken)
        {
            var descriptor       = GetMirroringDescriptor(configuration);
            var loggedDescriptor = descriptor + " (#" + configuration.Id.ToString(CultureInfo.InvariantCulture) + ")";

            Debug.WriteLine("Starting mirroring: " + loggedDescriptor);
            _eventLog.WriteEntry("Starting mirroring: " + loggedDescriptor);

            var repositoryDirectoryName = GetCloneDirectoryName(configuration);
            // A subfolder per clone dir start letter:
            var cloneDirectoryParentPath = Path.Combine(settings.RepositoriesDirectoryPath, repositoryDirectoryName[0].ToString());
            var cloneDirectoryPath       = Path.Combine(cloneDirectoryParentPath, repositoryDirectoryName);
            var repositoryLockFilePath   = GetRepositoryLockFilePath(cloneDirectoryPath);

            try
            {
                if (File.Exists(repositoryLockFilePath))
                {
                    var logEntryStart =
                        "An existing lock was found for the mirroring configuration " + loggedDescriptor + ". ";
                    var lastUpdatedTimeUtc = RepositoryInfoFileHelper.GetLastUpdatedDateTimeUtc(cloneDirectoryPath);

                    if (lastUpdatedTimeUtc >= DateTime.UtcNow.AddSeconds(-settings.MirroringTimoutSeconds))
                    {
                        _eventLog.WriteEntry(
                            logEntryStart +
                            "This can mean that the number of configurations was reduced and thus while a mirroring was" +
                            " running a new process for the same repositories was started. We'll let the initial process finish.");

                        return;
                    }
                    else
                    {
                        _eventLog.WriteEntry(
                            logEntryStart +
                            "Additionally the directory was last touched at " + lastUpdatedTimeUtc.ToString(CultureInfo.InvariantCulture) +
                            " UTC which is older than the allowed mirroring timeout (" + settings.MirroringTimoutSeconds +
                            "s). Thus the lock is considered abandoned and mirroring will continue.",
                            EventLogEntryType.Warning);
                    }
                }

                if (configuration.HgCloneUri.Scheme.Equals("ssh", StringComparison.OrdinalIgnoreCase) ||
                    configuration.GitCloneUri.Scheme.Equals("ssh", StringComparison.OrdinalIgnoreCase))
                {
                    throw new MirroringException("SSH protocol is not supported, only HTTPS.");
                }

                if (!Directory.Exists(cloneDirectoryParentPath))
                {
                    Directory.CreateDirectory(cloneDirectoryParentPath);
                }

                File.Create(repositoryLockFilePath).Dispose();

                // Changing directory to other drive if necessary.
                RunCommandAndLogOutput(Path.GetPathRoot(cloneDirectoryPath).Replace("\\", string.Empty));

                var quotedHgCloneUrl         = configuration.HgCloneUri.ToString().EncloseInQuotes();
                var quotedGitCloneUrl        = configuration.GitCloneUri.ToString().EncloseInQuotes();
                var quotedCloneDirectoryPath = cloneDirectoryPath.EncloseInQuotes();
                var isCloned = IsCloned(configuration, settings);

                if (!isCloned)
                {
                    DeleteDirectoryIfExists(cloneDirectoryPath);
                    Directory.CreateDirectory(cloneDirectoryPath);
                }

                RepositoryInfoFileHelper.CreateOrUpdateFile(cloneDirectoryPath, descriptor);

                // Mirroring between two git repos is supported, but in a hacked-in way at the moment. This needs a
                // clean-up. Also, do note that only GitToHg and TwoWay is implemented. It would make the whole thing
                // even more messy to duplicate the logic in HgToGit.
                var hgUrlIsGitUrl = configuration.HgCloneUri.Scheme == "git+https";

                cancellationToken.ThrowIfCancellationRequested();

                // It'll be fine for now.
#pragma warning disable S1151 // "switch case" clauses should not have too many lines of code
                switch (configuration.Direction)
                {
                case MirroringDirection.GitToHg:
                    if (hgUrlIsGitUrl)
                    {
                        RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                      .FetchOrCloneFromGit(configuration.GitCloneUri, cloneDirectoryPath, true, cancellationToken));
                        _gitCommandExecutor.PushToGit(configuration.HgCloneUri, cloneDirectoryPath, cancellationToken);
                    }
                    else
                    {
                        if (isCloned)
                        {
                            if (configuration.GitUrlIsHgUrl)
                            {
                                _hgCommandExecutor
                                .PullHg(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                            }
                            else
                            {
                                RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                              .FetchOrCloneFromGit(configuration.GitCloneUri, cloneDirectoryPath, true, cancellationToken));
                                _hgCommandExecutor.ImportHistoryFromGit(quotedCloneDirectoryPath, settings, cancellationToken);
                            }
                        }
                        else
                        {
                            if (configuration.GitUrlIsHgUrl)
                            {
                                _hgCommandExecutor
                                .CloneHg(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                            }
                            else
                            {
                                _hgCommandExecutor
                                .CloneGit(configuration.GitCloneUri, quotedCloneDirectoryPath, settings, cancellationToken);
                            }
                        }

                        _hgCommandExecutor
                        .PushWithBookmarks(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                    }

                    break;

                case MirroringDirection.HgToGit:
                    if (isCloned)
                    {
                        _hgCommandExecutor.PullHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                    }
                    else
                    {
                        _hgCommandExecutor.CloneHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                    }

                    if (configuration.GitUrlIsHgUrl)
                    {
                        _hgCommandExecutor
                        .PushWithBookmarks(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                    }
                    else
                    {
                        _hgCommandExecutor.CreateOrUpdateBookmarksForBranches(quotedCloneDirectoryPath, settings, cancellationToken);
                        _hgCommandExecutor.ExportHistoryToGit(quotedCloneDirectoryPath, settings, cancellationToken);
                        RunGitCommandAndMarkException(() =>
                                                      _gitCommandExecutor.PushToGit(configuration.GitCloneUri, cloneDirectoryPath, cancellationToken));
                    }

                    break;

                case MirroringDirection.TwoWay:
                    Action syncHgAndGitHistories = () =>
                    {
                        _hgCommandExecutor
                        .CreateOrUpdateBookmarksForBranches(quotedCloneDirectoryPath, settings, cancellationToken);
                        _hgCommandExecutor.ExportHistoryToGit(quotedCloneDirectoryPath, settings, cancellationToken);

                        // This will clear all commits in the git repo that aren't in the git remote repo but
                        // add changes that were added to the git repo.
                        RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                      .FetchOrCloneFromGit(configuration.GitCloneUri, cloneDirectoryPath, false, cancellationToken));
                        _hgCommandExecutor.ImportHistoryFromGit(quotedCloneDirectoryPath, settings, cancellationToken);

                        // Updating bookmarks which may have shifted after importing from git. This way the
                        // export to git will create a git repo with history identical to the hg repo.
                        _hgCommandExecutor.CreateOrUpdateBookmarksForBranches(quotedCloneDirectoryPath, settings, cancellationToken);
                        _hgCommandExecutor.ExportHistoryToGit(quotedCloneDirectoryPath, settings, cancellationToken);

                        RunGitCommandAndMarkException(() =>
                                                      _gitCommandExecutor.PushToGit(configuration.GitCloneUri, cloneDirectoryPath, cancellationToken));
                    };

                    if (hgUrlIsGitUrl)
                    {
                        // The easiest solution to do two-way git mirroring is to sync separately, with two clones.
                        // Otherwise when e.g. repository A adds a new commit, then repository B is pulled in, the
                        // head of the branch will be at the point where it is in B. Thus pushing to A will fail
                        // with "Cannot push non-fastforwardable reference". There are other similar errors that can
                        // arise but can't easily be fixed automatically in a safe way. So first pulling both repos
                        // then pushing them won't work.

                        var gitDirectoryPath = GitCommandExecutor.GetGitDirectoryPath(cloneDirectoryPath);

                        var secondToFirstClonePath = Path.Combine(gitDirectoryPath, "secondToFirst");
                        void PullSecondPushToFirst()
                        {
                            RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                          .FetchOrCloneFromGit(configuration.HgCloneUri, secondToFirstClonePath, true, cancellationToken));
                            RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                          .PushToGit(configuration.GitCloneUri, secondToFirstClonePath, cancellationToken));
                        }

                        var firstToSecondClonePath = Path.Combine(gitDirectoryPath, "firstToSecond");
                        RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                      .FetchOrCloneFromGit(configuration.GitCloneUri, firstToSecondClonePath, true, cancellationToken));
                        try
                        {
                            RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                          .PushToGit(configuration.HgCloneUri, firstToSecondClonePath, cancellationToken));

                            PullSecondPushToFirst();
                        }
                        catch (LibGit2SharpException ex)
                            when(ex.Message.Contains("Cannot push because a reference that you are trying to update on the remote contains commits that are not present locally."))
                            {
                                PullSecondPushToFirst();

                                // This exception can happen when the second repo contains changes not present in the
                                // first one. Then we need to update the first repo with the second's changes and pull-
                                // push again.
                                RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                              .FetchOrCloneFromGit(configuration.GitCloneUri, firstToSecondClonePath, true, cancellationToken));
                                RunGitCommandAndMarkException(() => _gitCommandExecutor
                                                              .PushToGit(configuration.HgCloneUri, firstToSecondClonePath, cancellationToken));
                            }
                    }
                    else
                    {
                        if (isCloned)
                        {
                            _hgCommandExecutor.PullHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);

                            if (configuration.GitUrlIsHgUrl)
                            {
                                _hgCommandExecutor
                                .PullHg(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                            }
                            else
                            {
                                syncHgAndGitHistories();
                            }
                        }
                        else
                        {
                            if (configuration.GitUrlIsHgUrl)
                            {
                                _hgCommandExecutor
                                .CloneHg(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                                _hgCommandExecutor
                                .PullHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                            }
                            else
                            {
                                // We need to start with cloning the hg repo. Otherwise cloning the git repo, then
                                // pulling from the hg repo would yield a "repository unrelated" error, even if the
                                // git repo was created from the hg repo. For an explanation see:
                                // http://stackoverflow.com/questions/17240852/hg-git-clone-from-github-gives-abort-repository-is-unrelated
                                _hgCommandExecutor
                                .CloneHg(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);

                                syncHgAndGitHistories();
                            }
                        }

                        _hgCommandExecutor
                        .PushWithBookmarks(quotedHgCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);

                        if (configuration.GitUrlIsHgUrl)
                        {
                            _hgCommandExecutor
                            .PushWithBookmarks(quotedGitCloneUrl, quotedCloneDirectoryPath, settings, cancellationToken);
                        }
                    }

                    break;

                default:
                    throw new NotSupportedException("Not supported MirroringDirection.");
                }
#pragma warning restore S1151 // "switch case" clauses should not have too many lines of code

                Debug.WriteLine("Finished mirroring: " + loggedDescriptor);
                _eventLog.WriteEntry("Finished mirroring: " + loggedDescriptor);
            }
            catch (Exception ex) when(!ex.IsFatalOrCancellation())
            {
                // We should dispose the command runners so the folder is not locked by the command line.
                Dispose();
                // Waiting a bit for any file locks or leases to be disposed even though CommandRunners and processes
                // were killed.
                Thread.Sleep(10000);

                var exceptionMessage =
                    $"An error occured while running commands when mirroring the repositories {configuration.HgCloneUri} " +
                    $"and {configuration.GitCloneUri} in direction {configuration.Direction}. Mirroring will be re-started next time.";

                try
                {
                    // Re-cloning a repo is costly. During local debugging you can flip this variable from the Immediate
                    // Window to prevent it if necessary too.
                    var continueWithRepoFolderDelete = true;

                    // These git exceptions are caused by hg errors in a way, so despite them coming from git the whole
                    // repo folder should be removed.
                    var isHgOriginatedGitException =
                        ex.Message.Contains("does not match any existing object") ||
                        ex.Message.Contains("Object not found - failed to find pack entry");
                    if (ex.Data.Contains("IsGitException") && !isHgOriginatedGitException)
                    {
                        exceptionMessage += " The error was a git error.";

                        try
                        {
                            DeleteDirectoryIfExists(GitCommandExecutor.GetGitDirectoryPath(cloneDirectoryPath));

                            exceptionMessage            += " Thus just the git folder was removed.";
                            continueWithRepoFolderDelete = false;
                        }
                        catch (Exception gitDirectoryDeleteException) when(!gitDirectoryDeleteException.IsFatalOrCancellation())
                        {
                            exceptionMessage +=
                                " While the removal of just the git folder was attempted it failed with the following " +
                                "exception, thus the deletion of the whole repository folder will be attempted: " +
                                gitDirectoryDeleteException;

                            // We'll continue with the repo folder removal below.
                        }
                    }

                    if (continueWithRepoFolderDelete)
                    {
                        DeleteDirectoryIfExists(cloneDirectoryPath);
                        RepositoryInfoFileHelper.DeleteFileIfExists(cloneDirectoryPath);
                    }
                }
                catch (Exception directoryDeleteException) when(!directoryDeleteException.IsFatalOrCancellation())
                {
                    try
                    {
                        // This most possibly means that for some reason some process is still locking the folder
                        // although it shouldn't (mostly, but not definitely the cause of IOException) or there are
                        // read-only files (git pack files commonly are) which can be (but not always) behind
                        // UnauthorizedAccessException.
                        if (directoryDeleteException is IOException || directoryDeleteException is UnauthorizedAccessException)
                        {
                            var killResult = DirectoryUtil.KillProcessesLockingFiles(cloneDirectoryPath);

                            DeleteDirectoryIfExists(cloneDirectoryPath);
                            RepositoryInfoFileHelper.DeleteFileIfExists(cloneDirectoryPath);

                            exceptionMessage +=
                                " While deleting the folder of the mirror initially failed, after trying to kill processes " +
                                "that were locking files in it and setting all files not to be read-only the folder could be successfully deleted. " +
                                "Processes killed: " + (killResult.KilledProcesseFileNames.Any() ? string.Join(", ", killResult.KilledProcesseFileNames) : "no processes") +
                                " Read-only files: " + (killResult.ReadOnlyFilePaths.Any() ? string.Join(", ", killResult.ReadOnlyFilePaths) : "no files");

                            throw new MirroringException(exceptionMessage, ex, directoryDeleteException);
                        }
                    }
                    catch (Exception forcedCleanUpException)
                        when(!forcedCleanUpException.IsFatalOrCancellation() && !(forcedCleanUpException is MirroringException))
                        {
                            throw new MirroringException(
                                      exceptionMessage + " Subsequently clean-up after the error failed as well, also the attempt " +
                                      "to kill processes that were locking the mirror's folder and clearing all read-only files.",
                                      ex,
                                      directoryDeleteException,
                                      forcedCleanUpException);
                        }

                    throw new MirroringException(
                              exceptionMessage + " Subsequently clean-up after the error failed as well.",
                              ex,
                              directoryDeleteException);
                }

                throw new MirroringException(exceptionMessage, ex);
            }
            finally
            {
                if (File.Exists(repositoryLockFilePath))
                {
                    File.Delete(repositoryLockFilePath);
                }
            }
        }
コード例 #2
0
ファイル: Mirror.cs プロジェクト: Lombiq/Git-hg-Mirror-Daemon
 public Mirror(EventLog eventLog)
     : base(eventLog)
 {
     _hgCommandExecutor  = new HgCommandExecutor(eventLog);
     _gitCommandExecutor = new GitCommandExecutor(eventLog);
 }