public void BuildArguments_should_add_reflog_if_requested(RefFilterOptions refFilterOptions, bool expected)
{
var args = _revisionReader.GetTestAccessor().BuildArgumentsBuildArguments(refFilterOptions, "", "", "");
if (expected)
{
args.ToString().Should().Contain(" --reflog ");
}
else
{
args.ToString().Should().NotContain(" --reflog ");
}
}
public void BuildArguments_check_parameters(RefFilterOptions refFilterOptions, string expectedToContain, string notExpectedToContain)
{
var args = _revisionReader.GetTestAccessor().BuildArgumentsBuildArguments(refFilterOptions, "my_*", "my_revision", "my_path");
if (expectedToContain is not null)
{
args.ToString().Should().Contain(expectedToContain);
}
if (notExpectedToContain is not null)
{
args.ToString().Should().NotContain(notExpectedToContain);
}
}
public void Execute(
GitModule module,
IObserver <GitRevision> subject,
RefFilterOptions refFilterOptions,
string branchFilter,
string revisionFilter,
string pathFilter,
[CanBeNull] Func <GitRevision, bool> revisionPredicate)
{
ThreadHelper.JoinableTaskFactory
.RunAsync(() => ExecuteAsync(module, subject, refFilterOptions, branchFilter, revisionFilter, pathFilter, revisionPredicate))
.FileAndForget(
ex =>
{
subject.OnError(ex);
return(false);
});
}
private ArgumentBuilder BuildArguments(RefFilterOptions refFilterOptions,
string branchFilter,
string revisionFilter,
string pathFilter)
{
return(new GitArgumentBuilder("log")
{
"-z",
branchFilter,
$"--pretty=format:\"{FullFormat}\"",
{
refFilterOptions.HasFlag(RefFilterOptions.FirstParent),
"--first-parent",
new ArgumentBuilder
{
{ AppSettings.ShowReflogReferences, "--reflog" },
{ AppSettings.SortByAuthorDate, "--author-date-order" },
{
refFilterOptions.HasFlag(RefFilterOptions.All),
"--all",
new ArgumentBuilder
{
{
refFilterOptions.HasFlag(RefFilterOptions.Branches) &&
!string.IsNullOrWhiteSpace(branchFilter) && branchFilter.IndexOfAny(new[] { '?', '*', '[' }) != -1,
"--branches=" + branchFilter
},
{ refFilterOptions.HasFlag(RefFilterOptions.Remotes), "--remotes" },
{ refFilterOptions.HasFlag(RefFilterOptions.Tags), "--tags" },
}.ToString()
},
{ refFilterOptions.HasFlag(RefFilterOptions.Boundary), "--boundary" },
{ refFilterOptions.HasFlag(RefFilterOptions.ShowGitNotes), "--not --glob=notes --not" },
{ refFilterOptions.HasFlag(RefFilterOptions.NoMerges), "--no-merges" },
{ refFilterOptions.HasFlag(RefFilterOptions.SimplifyByDecoration), "--simplify-by-decoration" }
}.ToString()
},
revisionFilter,
"--",
pathFilter
});
}
private async Task ExecuteAsync(
GitModule module,
IReadOnlyList <IGitRef> refs,
IObserver <GitRevision> subject,
RefFilterOptions refFilterOptions,
string branchFilter,
string revisionFilter,
string pathFilter,
[CanBeNull] Func <GitRevision, bool> revisionPredicate)
{
await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync();
var token = _cancellationTokenSequence.Next();
var revisionCount = 0;
await TaskScheduler.Default;
token.ThrowIfCancellationRequested();
var branchName = module.IsValidGitWorkingDir()
? module.GetSelectedBranch()
: "";
token.ThrowIfCancellationRequested();
UpdateSelectedRef(module, refs, branchName);
var refsByObjectId = refs.ToLookup(head => head.ObjectId);
token.ThrowIfCancellationRequested();
var arguments = BuildArguments(refFilterOptions, branchFilter, revisionFilter, pathFilter);
#if TRACE
var sw = Stopwatch.StartNew();
#endif
// This property is relatively expensive to call for every revision, so
// cache it for the duration of the loop.
var logOutputEncoding = module.LogOutputEncoding;
using (var process = module.GitCommandRunner.RunDetached(arguments, redirectOutput: true, outputEncoding: GitModule.LosslessEncoding))
{
token.ThrowIfCancellationRequested();
// Pool string values likely to form a small set: encoding, authorname, authoremail, committername, committeremail
var stringPool = new StringPool();
var buffer = new byte[4096];
foreach (var chunk in process.StandardOutput.BaseStream.ReadNullTerminatedChunks(ref buffer))
{
token.ThrowIfCancellationRequested();
if (TryParseRevision(module, chunk, stringPool, logOutputEncoding, out var revision))
{
if (revisionPredicate == null || revisionPredicate(revision))
{
// The full commit message body is used initially in InMemFilter, after which it isn't
// strictly needed and can be re-populated asynchronously.
//
// We keep full multiline message bodies within the last six months.
// Commits earlier than that have their properties set to null and the
// memory will be GCd.
if (DateTime.Now - revision.AuthorDate > TimeSpan.FromDays(30 * 6))
{
revision.Body = null;
}
// Look up any refs associated with this revision
revision.Refs = refsByObjectId[revision.ObjectId].AsReadOnlyList();
revisionCount++;
subject.OnNext(revision);
}
}
}
#if TRACE
Trace.WriteLine($"**** [{nameof(RevisionReader)}] Emitted {revisionCount} revisions in {sw.Elapsed.TotalMilliseconds:#,##0.#} ms. bufferSize={buffer.Length} poolCount={stringPool.Count}");
#endif
}
if (!token.IsCancellationRequested)
{
subject.OnCompleted();
}
}
public ArgumentBuilder BuildArgumentsBuildArguments(RefFilterOptions refFilterOptions,
string branchFilter, string revisionFilter, string pathFilter) =>
_revisionReader.BuildArguments(refFilterOptions, branchFilter, revisionFilter, pathFilter);
private async Task ExecuteAsync(
GitModule module,
IObserver <GitRevision> subject,
RefFilterOptions refFilterOptions,
string branchFilter,
string revisionFilter,
string pathFilter,
[CanBeNull] Func <GitRevision, bool> revisionPredicate)
{
ThreadHelper.ThrowIfNotOnUIThread();
var token = _cancellationTokenSequence.Next();
RevisionCount = 0;
await TaskScheduler.Default;
subject.OnNext(null);
token.ThrowIfCancellationRequested();
var branchName = module.IsValidGitWorkingDir()
? module.GetSelectedBranch()
: "";
token.ThrowIfCancellationRequested();
LatestRefs = module.GetRefs();
UpdateSelectedRef(module, LatestRefs, branchName);
var refsByObjectId = LatestRefs.ToLookup(head => head.Guid);
token.ThrowIfCancellationRequested();
const string fullFormat =
// These header entries can all be decoded from the bytes directly.
// Each hash is 20 bytes long. There is always a
/* Object ID */ "%H" +
/* Tree ID */ "%T" +
/* Parent IDs */ "%P%n" +
/* Author date */ "%at%n" +
/* Commit date */ "%ct%n" +
/* Encoding */ "%e%n" +
// Items below here must be decoded as strings to support non-ASCII
/* Author name */ "%aN%n" +
/* Author email */ "%aE%n" +
/* Committer name */ "%cN%n" +
/* Committer email */ "%cE%n" +
/* Commit subject */ "%s%n%n" +
/* Commit body */ "%b";
// TODO add AppBuilderExtensions support for flags enums, starting with RefFilterOptions, then use it in the below construction
var arguments = BuildArguments();
var sw = Stopwatch.StartNew();
// This property is relatively expensive to call for every revision, so
// cache it for the duration of the loop.
var logOutputEncoding = module.LogOutputEncoding;
using (var process = module.RunGitCmdDetached(arguments.ToString(), GitModule.LosslessEncoding))
{
token.ThrowIfCancellationRequested();
// Pool string values likely to form a small set: encoding, authorname, authoremail, committername, committeremail
var stringPool = new StringPool();
var buffer = new byte[4096];
foreach (var chunk in process.StandardOutput.BaseStream.ReadNullTerminatedChunks(ref buffer))
{
token.ThrowIfCancellationRequested();
if (TryParseRevision(module, chunk, stringPool, logOutputEncoding, out var revision))
{
if (revisionPredicate == null || revisionPredicate(revision))
{
// Remove full commit message to reduce memory consumption (28% for a repo with 69K commits)
// Full commit message is used in InMemFilter but later it's not needed
revision.Body = null;
// Look up any refs associate with this revision
revision.Refs = refsByObjectId[revision.Guid].AsReadOnlyList();
RevisionCount++;
subject.OnNext(revision);
}
}
}
Trace.WriteLine($"**** [{nameof(RevisionReader)}] Emitted {RevisionCount} revisions in {sw.Elapsed.TotalMilliseconds:#,##0.#} ms. bufferSize={buffer.Length} poolCount={stringPool.Count}");
}
await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync(token);
if (!token.IsCancellationRequested)
{
subject.OnCompleted();
}
ArgumentBuilder BuildArguments()
{
return(new ArgumentBuilder
{
"log",
"-z",
$"--pretty=format:\"{fullFormat}\"",
{ AppSettings.OrderRevisionByDate, "--date-order", "--topo-order" },
{ AppSettings.ShowReflogReferences, "--reflog" },
{
refFilterOptions.HasFlag(RefFilterOptions.All),
"--all",
new ArgumentBuilder
{
{
refFilterOptions.HasFlag(RefFilterOptions.Branches) &&
!branchFilter.IsNullOrWhiteSpace() &&
branchFilter.IndexOfAny(new[] { '?', '*', '[' }) != -1,
"--branches=" + branchFilter
},
{ refFilterOptions.HasFlag(RefFilterOptions.Remotes), "--remotes" },
{ refFilterOptions.HasFlag(RefFilterOptions.Tags), "--tags" },
}.ToString()
},
{ refFilterOptions.HasFlag(RefFilterOptions.Boundary), "--boundary" },
{ refFilterOptions.HasFlag(RefFilterOptions.ShowGitNotes), "--not --glob=notes --not" },
{ refFilterOptions.HasFlag(RefFilterOptions.NoMerges), "--no-merges" },
{ refFilterOptions.HasFlag(RefFilterOptions.FirstParent), "--first-parent" },
{ refFilterOptions.HasFlag(RefFilterOptions.SimplifyByDecoration), "--simplify-by-decoration" },
revisionFilter,
"--",
pathFilter
});
}
}
private async Task ExecuteAsync(
VsrModule module,
IReadOnlyList <IGitRef> refs,
IObserver <GitRevision> subject,
RefFilterOptions refFilterOptions,
string branchFilter,
string revisionFilter,
string pathFilter,
[CanBeNull] Func <GitRevision, bool> revisionPredicate)
{
await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync();
var token = _cancellationTokenSequence.Next();
var revisionCount = 0;
await TaskScheduler.Default;
token.ThrowIfCancellationRequested();
var branchName = module.IsValidVersionrWorkingDir()
? module.GetSelectedBranch()
: "";
token.ThrowIfCancellationRequested();
UpdateSelectedRef(module, refs, branchName);
var refsByObjectId = refs.ToLookup(head => head.ObjectId);
token.ThrowIfCancellationRequested();
var arguments = BuildArguments(refFilterOptions, branchFilter, revisionFilter, pathFilter);
#if TRACE
var sw = Stopwatch.StartNew();
#endif
var versions = module.GetLog(1000); // TODO: VSR - limit is in revisionFilter variable
foreach (var version in versions)
{
token.ThrowIfCancellationRequested();
var revision = new GitRevision(new ObjectId(version.ID))
{
ParentIds = version.Parent.HasValue ? new[] { new ObjectId(version.Parent.Value) } : null,
TreeGuid = null, // TODO: VSR
Author = version.Author,
AuthorEmail = version.Email,
AuthorDate = version.Timestamp, // TODO: VSR
Committer = version.Author, // TODO: VSR
CommitterEmail = version.Email, // TODO: VSR
CommitDate = version.Timestamp,
MessageEncoding = null, // TODO: VSR
Subject = version.Message,
Body = version.Message, // TODO: VSR
Name = version.ShortName, // TODO: VSR
HasMultiLineMessage = false, // TODO: VSR - !ReferenceEquals(Subject, Body),
HasNotes = false
};
if (revisionPredicate == null || revisionPredicate(revision))
{
// The full commit message body is used initially in InMemFilter, after which it isn't
// strictly needed and can be re-populated asynchronously.
//
// We keep full multiline message bodies within the last six months.
// Commits earlier than that have their properties set to null and the
// memory will be GCd.
if (DateTime.Now - revision.AuthorDate > TimeSpan.FromDays(30 * 6))
{
revision.Body = null;
}
// Look up any refs associated with this revision
revision.Refs = refsByObjectId[revision.ObjectId].AsReadOnlyList();
revisionCount++;
subject.OnNext(revision);
}
}
// This property is relatively expensive to call for every revision, so
// cache it for the duration of the loop.
var logOutputEncoding = module.LogOutputEncoding;
if (!token.IsCancellationRequested)
{
subject.OnCompleted();
}
}
private async Task ExecuteAsync(
GitModule module,
IReadOnlyList <IGitRef> refs,
IObserver <GitRevision> subject,
RefFilterOptions refFilterOptions,
string branchFilter,
string revisionFilter,
string pathFilter,
[CanBeNull] Func <GitRevision, bool> revisionPredicate)
{
await ThreadHelper.JoinableTaskFactory.SwitchToMainThreadAsync();
var token = _cancellationTokenSequence.Next();
var revisionCount = 0;
await TaskScheduler.Default;
token.ThrowIfCancellationRequested();
var branchName = module.IsValidGitWorkingDir()
? module.GetSelectedBranch()
: "";
token.ThrowIfCancellationRequested();
UpdateSelectedRef(module, refs, branchName);
var refsByObjectId = refs.ToLookup(head => head.ObjectId);
token.ThrowIfCancellationRequested();
const string fullFormat =
// These header entries can all be decoded from the bytes directly.
// Each hash is 20 bytes long.
/* Object ID */ "%H" +
/* Tree ID */ "%T" +
/* Parent IDs */ "%P%n" +
/* Author date */ "%at%n" +
/* Commit date */ "%ct%n" +
/* Encoding */ "%e%n" +
// Items below here must be decoded as strings to support non-ASCII.
/* Author name */ "%aN%n" +
/* Author email */ "%aE%n" +
/* Committer name */ "%cN%n" +
/* Committer email */ "%cE%n" +
/* Commit subject */ "%s%n%n" +
/* Commit body */ "%b";
var arguments = BuildArguments();
var sw = Stopwatch.StartNew();
// This property is relatively expensive to call for every revision, so
// cache it for the duration of the loop.
var logOutputEncoding = module.LogOutputEncoding;
using (var process = module.RunGitCmdDetached(arguments, redirectOutput: true, outputEncoding: GitModule.LosslessEncoding))
{
token.ThrowIfCancellationRequested();
// Pool string values likely to form a small set: encoding, authorname, authoremail, committername, committeremail
var stringPool = new StringPool();
var buffer = new byte[4096];
foreach (var chunk in process.StandardOutput.BaseStream.ReadNullTerminatedChunks(ref buffer))
{
token.ThrowIfCancellationRequested();
if (TryParseRevision(module, chunk, stringPool, logOutputEncoding, out var revision))
{
if (revisionPredicate == null || revisionPredicate(revision))
{
// The full commit message body is used initially in InMemFilter, after which it isn't
// strictly needed and can be re-populated asynchronously.
//
// We keep full multiline message bodies within the last six months.
// Commits earlier than that have their properties set to null and the
// memory will be GCd.
if (DateTime.Now - revision.AuthorDate > TimeSpan.FromDays(30 * 6))
{
revision.Body = null;
}
// Look up any refs associated with this revision
revision.Refs = refsByObjectId[revision.ObjectId].AsReadOnlyList();
revisionCount++;
subject.OnNext(revision);
}
}
}
Trace.WriteLine($"**** [{nameof(RevisionReader)}] Emitted {revisionCount} revisions in {sw.Elapsed.TotalMilliseconds:#,##0.#} ms. bufferSize={buffer.Length} poolCount={stringPool.Count}");
}
if (!token.IsCancellationRequested)
{
subject.OnCompleted();
}
ArgumentBuilder BuildArguments()
{
return(new GitArgumentBuilder("log")
{
"-z",
$"--pretty=format:\"{fullFormat}\"",
{ AppSettings.ShowReflogReferences, "--reflog" },
{
refFilterOptions.HasFlag(RefFilterOptions.All),
"--all",
new ArgumentBuilder
{
{
refFilterOptions.HasFlag(RefFilterOptions.Branches) &&
!branchFilter.IsNullOrWhiteSpace() &&
branchFilter.IndexOfAny(new[] { '?', '*', '[' }) != -1,
"--branches=" + branchFilter
},
{ refFilterOptions.HasFlag(RefFilterOptions.Remotes), "--remotes" },
{ refFilterOptions.HasFlag(RefFilterOptions.Tags), "--tags" },
}.ToString()
},
{ refFilterOptions.HasFlag(RefFilterOptions.Boundary), "--boundary" },
{ refFilterOptions.HasFlag(RefFilterOptions.ShowGitNotes), "--not --glob=notes --not" },
{ refFilterOptions.HasFlag(RefFilterOptions.NoMerges), "--no-merges" },
{ refFilterOptions.HasFlag(RefFilterOptions.FirstParent), "--first-parent" },
{ refFilterOptions.HasFlag(RefFilterOptions.SimplifyByDecoration), "--simplify-by-decoration" },
revisionFilter,
"--",
pathFilter
});
}
}
public RefFilterOptionsEventArgs(RefFilterOptions refFilterOptions)
{
RefFilterOptions = refFilterOptions;
}
