/// <summary>
/// Log file has format specified in GitCommandLine class
/// </summary>
private ChangeSetHistory ParseLog(Stream log, Graph graph)
{
var changeSets = new List <ChangeSet>();
using (var reader = new StreamReader(log))
{
var proceed = GoToNextRecord(reader);
if (!proceed)
{
return(new ChangeSetHistory(new List <ChangeSet>()));
// I found one case where git log --follow caused an empty output.
//throw new FormatException("The file does not contain any change sets.");
}
while (proceed)
{
var changeSet = ParseRecord(reader, graph);
changeSets.Add(changeSet);
proceed = GoToNextRecord(reader);
}
}
var history = new ChangeSetHistory(changeSets.OrderByDescending(x => x.Date).ToList());
return(history);
}
public void WriteDebugGraph(ChangeSetHistory history, Graph graph, string targetHash, string findMe)
{
var dbgGraph = graph.Clone();
dbgGraph.MinimizeTo(targetHash);
var highlightedNodes = new Dictionary <GraphNode, string>();
foreach (var node in dbgGraph.AllNodes)
{
var changeSet = history.ChangeSets.Single(cs => cs.Id == node.CommitHash);
var actionsDone = changeSet.Items.Where(item => item.ServerPath.Contains(findMe))
.Select(FormatItem).ToList();
if (actionsDone.Any())
{
highlightedNodes.Add(node, string.Join("\n", actionsDone));
}
}
var shortHash = targetHash.Substring(0, 5);
var file = Path.Combine(_directory, $"conflict_graph_{shortHash}_{findMe}.dot");
var graphWriter = new GraphWriter();
graphWriter.SaveGraphSimplified(file, dbgGraph, highlightedNodes);
}
/// <summary>
/// If the classifier returns string.EMPTY the according file is not used.
/// </summary>
public List <Coupling> CalculateClassifiedChangeCouplings(ChangeSetHistory history, Func <string, string> classifier)
{
_couplings.Clear();
_count.Clear();
foreach (var cs in history.ChangeSets)
{
var classifications = ClassifyItems(cs, classifier);
IncrementCommitCount(classifications);
for (var i = 0; i < classifications.Count - 1; i++) // Keep one for the last pair
{
// Make pairs of code classifiers.
for (var j = i + 1; j < classifications.Count; j++)
{
var class1 = classifications[i];
var class2 = classifications[j];
IncrementCoupling(class1, class2);
}
}
}
CalculateDegree();
return(_couplings.Values
// .Where(coupling => coupling.Couplings >= Thresholds.MinCouplingForChangeCoupling && coupling.Degree >= Thresholds.MinDegreeForChangeCoupling)
.OrderByDescending(coupling => coupling.Degree).ToList());
}
/// <summary>
/// Converts a given <see cref="T:XElement"/> into a <see cref="T:ChangeSetHistory"/>.
/// </summary>
/// <param name="element">The <see cref="T:XElement"/> to convert.</param>
/// <returns>The newly created <see cref="T:ChangeSetHistory"/>.</returns>
public ChangeSetHistory FromXElement(XElement element)
{
ChangeSetHistory history = new ChangeSetHistory();
foreach (XElement changeSetElement in element.Nodes()) {
ChangeSet changeSet = new ChangeSet();
// Get the change set details
foreach (XElement changeSetElementChild in changeSetElement.Nodes()) {
if (changeSetElementChild.Name.LocalName == "Applied") {
changeSet.Applied = DateTime.Parse(changeSetElementChild.Value);
} else if (changeSetElementChild.Name.LocalName == "Username") {
changeSet.Username = changeSetElementChild.Value;
} else if (changeSetElementChild.Name.LocalName == "Change") {
Change change = new Change();
foreach (XElement changeElement in changeSetElementChild.Nodes()) {
if (changeElement.Name.LocalName == "PropertyName") {
change.PropertyName = changeElement.Value;
} else if (changeElement.Name.LocalName == "OldValue") {
change.OldValue = changeElement.Value;
} else if (changeElement.Name.LocalName == "NewValue") {
change.NewValue = changeElement.Value;
}
}
changeSet.Changes.Add(change);
}
}
history.Append(changeSet);
}
return history;
}
private ChangeSetHistory ReadExportFile()
{
_tracking = new MovementTracker();
var result = new List <ChangeSet>();
using (var reader = XmlReader.Create(_logFile))
{
while (reader.Read())
{
if (reader.IsStartElement("logentry"))
{
ParseLogEntry(reader, result);
}
}
}
// First item is latest commit because we got the by ordering
Debug.Assert(result.First().Date >= result.Last().Date);
Warnings = _tracking.Warnings;
var history = new ChangeSetHistory(result);
history.CleanupHistory();
return(history);
}
private void LoadHistory()
{
if (_history == null)
{
// Assume the history was already cleaned such that it only contains tracked files and no deletes.
_history = _sourceProvider.QueryChangeSetHistory();
}
}
void LoadHistory()
{
if (_history == null)
{
var provider = Project.CreateProvider();
_history = provider.QueryChangeSetHistory();
Warnings = provider.Warnings;
// Remove all items that are deleted now.
_history.CleanupHistory();
}
}
public List <Coupling> CalculateChangeCouplings(ChangeSetHistory history, IFilter filter)
{
_couplings.Clear();
_count.Clear();
var idToLocalFile = BuildIdToLocalFileMap(history);
foreach (var cs in history.ChangeSets)
{
if (cs.Items.Count > Thresholds.MaxItemsInChangesetForChangeCoupling)
{
continue;
}
// Only accepted files
var itemIds = cs.Items.Where(item => filter.IsAccepted(item.LocalPath)).Select(item => item.Id).ToList();
// Do you have uncommitted changes.
// Do you have commit items not inside the base directory?
var missingFiles = itemIds.Select(id => idToLocalFile[id]).Where(file => !File.Exists(file));
// Debug.Assert(!missingFiles.Any());
IncrementCommitCount(itemIds);
for (var i = 0; i < itemIds.Count - 1; i++) // Keep one for the last pair
{
// Make pairs of files.
for (var j = i + 1; j < itemIds.Count; j++)
{
var id1 = itemIds[i];
var id2 = itemIds[j];
IncrementCoupling(id1, id2);
}
}
}
CalculateDegree();
return(_couplings.Values
.Where(coupling => coupling.Couplings >= Thresholds.MinCouplingForChangeCoupling && coupling.Degree >= Thresholds.MinDegreeForChangeCoupling)
.OrderByDescending(coupling => coupling.Degree)
.Select(c => new Coupling(idToLocalFile[c.Item1], idToLocalFile[c.Item2])
{
// Display coupling item with local path instead of identifier.
Degree = c.Degree,
Couplings = c.Couplings
}).ToList());
}
/// <summary>
/// returns the team communication path.
/// If no team classifier is provided, the developer himself is used.
/// </summary>
public Dictionary <OrderedPair, uint> AnalyzeTeamCommunication(ChangeSetHistory history, ITeamClassifier teamClassifier)
{
// file id -> dictionary{team, #commits}
var fileToCommitsPerTeam = new Dictionary <string, Dictionary <string, uint> >();
foreach (var cs in history.ChangeSets)
{
// Associated team (or developer)
var team = GetTeam(cs, teamClassifier);
foreach (var item in cs.Items)
{
// Add team to file
if (!fileToCommitsPerTeam.Keys.Contains(item.Id))
{
fileToCommitsPerTeam.Add(item.Id, new Dictionary <string, uint>());
}
var commitsPerTeam = fileToCommitsPerTeam[item.Id];
commitsPerTeam.AddToValue(team, 1);
}
}
// We know for each file which team did how many changes
// Each file that was accessed by two teams leads to a link between the teams.
// pair of teams -> number of commits.
var teamCommunicationPaths = new Dictionary <OrderedPair, uint>();
foreach (var file in fileToCommitsPerTeam)
{
var currentFileTeams = file.Value.Keys.ToList();
// Build team subsets that get counted as one path.
for (var index = 0; index < currentFileTeams.Count - 1; index++)
{
for (var index2 = 1; index2 < currentFileTeams.Count; index2++)
{
var teamPair = new OrderedPair(currentFileTeams[index], currentFileTeams[index2]);
// The team combination that worked together at least one time at the same file
// gets a point.
teamCommunicationPaths.AddToValue(teamPair, 1);
}
}
}
return(teamCommunicationPaths);
}
/// <summary>
/// Seed is the file name the file was first committed under.
/// </summary>
private void AssertFile(ChangeSetHistory history, string fileSeed, int changeSets, int add, int modify, int delete, int rename, int copy, string finalName)
{
var id = FindId(history, fileSeed);
Assert.IsNotNull(id);
var stats = Follow(history, id);
Assert.AreEqual(changeSets, stats.ChangesSets);
Assert.AreEqual(add, stats.Add, "Add");
Assert.AreEqual(modify, stats.Modify, "Modify");
Assert.AreEqual(rename, stats.Rename, "Rename");
Assert.AreEqual(copy, stats.Copy, "Copy");
Assert.AreEqual(delete, stats.Delete, "Delete");
Assert.AreEqual(finalName, stats.FinalName, "FinalName");
}
public List <Coupling> CalculateChangeCouplings(ChangeSetHistory history, IFilter filter)
{
_couplings.Clear();
_count.Clear();
var idToLocalFile = BuildIdToLocalFileMap(history);
foreach (var cs in history.ChangeSets)
{
if (cs.Items.Count > Thresholds.MaxItemsInChangesetForChangeCoupling)
{
continue;
}
// Only accepted files
// Normally the files are already filtered when the history was created.
var itemIds = cs.Items.Where(item => filter.IsAccepted(item.LocalPath)).Select(item => item.Id).ToList();
IncrementCommitCount(itemIds);
for (var i = 0; i < itemIds.Count - 1; i++) // Keep one for the last pair
{
// Make pairs of files.
for (var j = i + 1; j < itemIds.Count; j++)
{
var id1 = itemIds[i];
var id2 = itemIds[j];
IncrementCoupling(id1, id2);
}
}
}
CalculateDegree();
return(_couplings.Values
.Where(coupling => coupling.Couplings >= Thresholds.MinCouplingForChangeCoupling && coupling.Degree >= Thresholds.MinDegreeForChangeCoupling)
.OrderByDescending(coupling => coupling.Degree)
.Select(c => new Coupling(idToLocalFile[c.Item1], idToLocalFile[c.Item2])
{
// Display coupling item with local path instead of identifier.
Degree = c.Degree,
Couplings = c.Couplings
}).ToList());
}
private static Dictionary <string, string> BuildIdToLocalFileMap(ChangeSetHistory history)
{
var idToLocalFile = new Dictionary <string, string>();
foreach (var cs in history.ChangeSets)
{
foreach (var item in cs.Items)
{
if (!idToLocalFile.ContainsKey(item.Id))
{
// Seen the first time means latest file.
idToLocalFile.Add(item.Id, item.LocalPath);
}
}
}
return(idToLocalFile);
}
/// <summary>
/// Converts a given <see cref="ChangeSetHistory"/> into an <see cref="T:XElement"/>.
/// </summary>
/// <param name="changeSetHistory">The <see cref="ChangeSetHistory"/> to convert.</param>
/// <returns>The newly created <see cref="T:XElement"/>.</returns>
public XElement ToXElement(ChangeSetHistory changeSetHistory)
{
XElement historyElement = new XElement("ChangeSetHistory");
foreach (ChangeSet changeSet in changeSetHistory) {
XElement changeSetElement = new XElement("ChangeSet",
new XElement("Applied", changeSet.Applied),
new XElement("Username", changeSet.Username));
foreach (Change change in changeSet.Changes) {
XElement changeElement = new XElement("Change",
new XElement("PropertyName", change.PropertyName),
new XElement("OldValue", change.OldValue),
new XElement("NewValue", change.NewValue));
changeSetElement.Add(changeElement);
}
historyElement.Add(changeSetElement);
}
return historyElement;
}
private void UpdateHistory(IProgress progress)
{
// Git graph
_graph = new Graph();
_idToLocalFile = new Dictionary <string, string>();
// Build the full graph
var fullLog = _gitCli.Log();
var parser = new Parser(_mapper);
var(_, graph) = parser.ParseLogString(fullLog);
_graph = graph;
// Save the original log for information (debugging)
//SaveFullLogToDisk();
// Build a virtual commit history
var localPaths = GetAllTrackedLocalFiles();
var changeSets = RebuildHistory(localPaths, progress);
// file id -> List of change set id
var filesToRemove = FindSharedHistory(changeSets);
// Cleanup history. We stop tracking a file if it starts sharing its history with another file.
// The history may skip some commits so we use the full graph to traverse everything.
DeleteSharedHistory(changeSets, filesToRemove);
// Write history file
var history = new ChangeSetHistory(changeSets);
history.CleanupHistory(); // Assume nothing is removed since no deletes should be in history.
var json = JsonConvert.SerializeObject(history, Formatting.Indented);
File.WriteAllText(_historyFile, json, Encoding.UTF8);
// Save the constructed log for information
SaveRecoveredLogToDisk(changeSets, _graph);
// Just a plausibility check
VerifyNoDuplicateServerPathsInChangeset(changeSets);
}
private static string FindId(ChangeSetHistory history, string fileSeed)
{
string id = null;
foreach (var cs in history.ChangeSets)
{
// old to new
foreach (var item in cs.Items)
{
if (item.ServerPath == fileSeed && item.IsAdd())
{
id = item.Id;
break;
}
}
}
return(id);
}
private GitProviderTests.FollowResult Follow(ChangeSetHistory history, string id)
{
var result = new GitProviderTests.FollowResult();
foreach (var cs in history.ChangeSets)
{
var ofId = cs.Items.Where(item => item.Id == id).ToList();
if (ofId.Any())
{
result.ChangesSets++;
}
foreach (var item in ofId)
{
IncrementOperations(result, item);
}
}
return(result);
}
public bool Verify(Graph graph, GraphNode node, ChangeSetHistory history)
{
var expectedServerPaths = GetAllTrackedFiles(node.CommitHash);
var actualServerPaths = node.Scope.GetAllFiles();
var intersect = new HashSet <string>(expectedServerPaths.Intersect(actualServerPaths));
var inGit = new HashSet <string>(expectedServerPaths.Except(intersect));
var inScope = new HashSet <string>(actualServerPaths.Except(intersect));
//var differences = expectedServerPaths;
//differences.SymmetricExceptWith(actualServerPaths);
var union = inScope.Union(inGit).ToList();
if (union.Any())
{
// Save differences
_debugLogFile.WriteLine("Deviation from scope and expected git tree");
WriteDifference(inGit, "Git");
WriteDifference(inScope, "Scope");
// Save graphs
foreach (var serverPath in union)
{
var fi = new FileInfo(serverPath);
WriteDebugGraph(history, graph, node.CommitHash, fi.Name);
}
// Write differences to a separate file
//File.WriteAllText(Path.Combine(_directory, $"conflict_diff_{shortHash}.txt"), builder.ToString());
return(false);
}
return(true);
}
/// <summary>
/// Log file has format specified in GitCommandLine class
/// </summary>
ChangeSetHistory ParseLog(Stream log)
{
var changeSets = new List <ChangeSet>();
using (var reader = new StreamReader(log))
{
var proceed = GoToNextRecord(reader);
if (!proceed)
{
throw new FormatException("The file does not contain any change sets.");
}
while (proceed)
{
var changeSet = ParseRecord(reader);
changeSets.Add(changeSet);
proceed = GoToNextRecord(reader);
}
}
var history = new ChangeSetHistory(changeSets.OrderByDescending(x => x.Date).ToList());
return(history);
}
/// <summary>
/// Creates the history in a form that we can process further from initial commit to the last one.
/// </summary>
private ChangeSetHistory CreateHistory(Repository repo, Graph graph, IProgress progress)
{
var nodeCount = graph.AllNodes.Count();
var currentNode = 0;
var changeSets = new List <ChangeSet>();
var initialNodes = graph.AllNodes.Where(node => node.Parents.Any() is false).ToList();
Debug.Assert(initialNodes.Count == 1);
var initialNode = initialNodes.First();
var alreadyProcessed = new HashSet <string>();
var nodesToProcess = new Queue <GraphNode>();
nodesToProcess.Enqueue(initialNode);
while (nodesToProcess.Any())
{
var node = nodesToProcess.Dequeue();
if (IsMergeWithUnprocessedParents(node))
{
// We arrive here later again.
continue;
}
if (!alreadyProcessed.Add(node.CommitHash))
{
// When we arrive a merge node the first time both parents may be complete (or not).
// If so we would end up following the same path twice.
continue;
}
progress?.Message($"Processing commit {++currentNode} / {nodeCount}");
var commit = repo.Lookup <Commit>(node.CommitHash);
var differences = CalculateDiffs(repo, commit);
var(scope, deletedServerPathToId) = ApplyChangesToScope(node, differences);
node.Scope = scope;
var cs = CreateChangeSet(commit);
changeSets.Add(cs);
// Create change items
foreach (var change in differences.ChangesInCommit)
{
var item = CreateChangeItem(change, scope, deletedServerPathToId);
cs.Items.Add(item);
}
// Add children to processing queue
foreach (var childNode in node.Children)
{
nodesToProcess.Enqueue(childNode);
}
}
VerifyScope(graph.GetNode(repo.Head.Tip.Sha));
var history = new ChangeSetHistory(changeSets.OrderByDescending(cs => cs.Date).ToList());
return(history);
}
protected void SaveHistory(ChangeSetHistory history)
{
var json = JsonConvert.SerializeObject(history, Formatting.Indented);
File.WriteAllText(_historyFile, json, Encoding.UTF8);
}
internal void Clear()
{
_history = null;
_metrics = null;
_contributions = null;
}
