private int AddSectionCommitsForFile(ExcelWorksheet sheet, FileStat fileChange, int rowCounter)
{
sheet.Cells[rowCounter, 1].Value = fileChange.Filename;
sheet.Cells[rowCounter, 1].Style.Font.Size = 16;
rowCounter++;
sheet.Cells[rowCounter, 1].Value = "Latest commit";
sheet.Cells[rowCounter, 1].Style.HorizontalAlignment = OfficeOpenXml.Style.ExcelHorizontalAlignment.Right;
sheet.Cells[rowCounter, 2].Value = fileChange.LatestCommit;
rowCounter++;
sheet.Cells[rowCounter, 1].Value = "Commits";
sheet.Cells[rowCounter, 1].Style.HorizontalAlignment = OfficeOpenXml.Style.ExcelHorizontalAlignment.Right;
sheet.Cells[rowCounter, 2].Value = fileChange.CommitCount;
rowCounter++;
if (fileChange.FileExists)
{
var linesOfCode = fileChange.LinesOfCode > 0 ? fileChange.LinesOfCode.ToString() : "N/A";
sheet.Cells[rowCounter, 1].Style.HorizontalAlignment = OfficeOpenXml.Style.ExcelHorizontalAlignment.Right;
sheet.Cells[rowCounter, 2].Value = linesOfCode;
rowCounter++;
var cyclomaticComplexity = fileChange.CyclomaticComplexity > 0 ? fileChange.CyclomaticComplexity.ToString() : "N/A";
sheet.Cells[rowCounter, 1].Value = "Cyclomatic complexity";
sheet.Cells[rowCounter, 1].Style.HorizontalAlignment = OfficeOpenXml.Style.ExcelHorizontalAlignment.Right;
sheet.Cells[rowCounter, 2].Value = cyclomaticComplexity;
rowCounter++;
var methodCount = fileChange.MethodCount > 0 ? fileChange.MethodCount.ToString() : "N/A"; sheet.Cells[rowCounter, 1].Value = "Lines of code";
sheet.Cells[rowCounter, 1].Value = "Method count";
sheet.Cells[rowCounter, 1].Style.HorizontalAlignment = OfficeOpenXml.Style.ExcelHorizontalAlignment.Right;
sheet.Cells[rowCounter, 2].Value = methodCount;
rowCounter++;
}
else
{
sheet.Cells[rowCounter, 1].Value = "File has been deleted";
sheet.Cells[rowCounter, 1].Style.HorizontalAlignment = OfficeOpenXml.Style.ExcelHorizontalAlignment.Right;
rowCounter++;
}
TableHeader(sheet, rowCounter, 1, "Author", 25);
TableHeader(sheet, rowCounter, 2, "Commits", 9);
TableHeader(sheet, rowCounter, 3, "Percentage", 11);
TableHeader(sheet, rowCounter, 4, "Latest commit", 11);
rowCounter++;
foreach (var userfileChange in UserfileCommitsList.Where(ufc => ufc.Filename == fileChange.Filename))
{
sheet.Cells[rowCounter, 1].Value = userfileChange.Username;
sheet.Cells[rowCounter, 2].Value = userfileChange.CommitCount;
sheet.Cells[rowCounter, 3].Value = (double)userfileChange.CommitCount / (double)fileChange.CommitCount;
sheet.Cells[rowCounter, 3].Style.Numberformat.Format = "#,##0.00%";
var commitDatesOrdered = userfileChange.CommitDates.OrderBy(date => date);
sheet.Cells[rowCounter, 4].Value = commitDatesOrdered.Last().ToString("yyyy-MM-dd");
rowCounter++;
}
return(rowCounter);
}
private void AddSectionCommitsForEachFile(StringBuilder sb, FileStat fileChange, int sectionCounter)
{
sb.AppendLine("<div class=\"row\"><div class=\"col-md-6\">");
sb.AppendLine($"<h3>{WebUtility.HtmlEncode(fileChange.Filename)}</h3>");
sb.AppendLine("<table class=\"table pull-left\" style=\"width: 500px\">");
sb.AppendLine($"<tr><td class=\"text-right\">Latest commit</td><td>{fileChange.LatestCommit.ToString("yyyy-MM-dd")}</td></tr>");
sb.AppendLine($"<tr><td class=\"text-right\">Commits</td><td>{fileChange.CommitCount}</td></tr>");
if (fileChange.FileExists)
{
var linesOfCode = fileChange.LinesOfCode > 0 ? fileChange.LinesOfCode.ToString() : "N/A";
var cyclomaticComplexity = fileChange.CyclomaticComplexity > 0 ? fileChange.CyclomaticComplexity.ToString() : "N/A";
var methodCount = fileChange.MethodCount > 0 ? fileChange.MethodCount.ToString() : "N/A";
sb.AppendLine($"<tr><td class=\"text-right\">Lines of code</td><td>{linesOfCode}</td></tr>");
sb.AppendLine($"<tr><td class=\"text-right\">Cyclomatic Complexity</td><td>{cyclomaticComplexity}</td></tr>");
sb.AppendLine($"<tr><td class=\"text-right\">Method count</td><td>{methodCount}</td></tr>");
}
else
{
sb.AppendLine($"<tr><td class=\"text-right\">File has been deleted</td><td></td></tr>");
}
sb.AppendLine("</table>");
sb.AppendLine("</div>");
sb.AppendLine("<div class=\"col-md-6\">");
sb.AppendLine("</div>");
sb.AppendLine("</div>");
sb.AppendLine("<div class=\"row\"><div class=\"col-md-6\">");
sb.AppendLine("<table class=\"table pull-left\" style=\"width: 500px\">");
sb.AppendLine($"<tr><th>Name</th><th>Commits</th><th>Percentage</th><th>Latest commit</th></tr>");
var commitDates = new List <ScatterPoint>();
foreach (var userfileChange in UserfileCommitsList.Where(ufc => ufc.Filename == fileChange.Filename))
{
var username = WebUtility.HtmlEncode(userfileChange.Username);
var changeCount = string.Format("{0,3}", userfileChange.CommitCount);
var percentage = string.Format("{0,5:#0.00}", ((double)userfileChange.CommitCount / (double)fileChange.CommitCount) * 100);
var latestCommit = GenerateScatterPlotData(commitDates, userfileChange);
sb.AppendLine($"<tr><td class=\"text-right\">{username}</td><td>{changeCount}</td><td>{percentage}%</td><td>{latestCommit}</td></tr>");
}
sb.AppendLine("</table>");
sb.AppendLine("</div>");
sb.AppendLine("<div class=\"col-md-6\">");
AddScatterplotJavascript(sb, commitDates, sectionCounter);
sb.AppendLine($" <div id=\"scatterChart{sectionCounter}\" class=\"spinner\"></div>");
sb.AppendLine("</div>");
sb.AppendLine("</div>");
}
internal bool Run()
{
if (_keyValueDB.FileCollection.GetCount() == 0) return false;
_root = _keyValueDB.LastCommited;
var dontTouchGeneration = _keyValueDB.GetGeneration(_root.TrLogFileId);
InitFileStats(dontTouchGeneration);
CalculateFileUsefullness();
var totalWaste = CalcTotalWaste();
if (totalWaste < (ulong)_keyValueDB.MaxTrLogFileSize / 4)
{
if (_keyValueDB.DistanceFromLastKeyIndex(_root) < (ulong)_keyValueDB.MaxTrLogFileSize / 4) return false;
_keyValueDB.CreateIndexFile(_cancellation);
_keyValueDB.FileCollection.DeleteAllUnknownFiles();
return false;
}
_cancellation.ThrowIfCancellationRequested();
uint valueFileId;
var writer = _keyValueDB.StartPureValuesFile(out valueFileId);
_newPositionMap = new Dictionary<ulong, uint>();
var toRemoveFileIds = new List<uint>();
while (true)
{
var wastefullFileId = FindMostWastefullFile(_keyValueDB.MaxTrLogFileSize - writer.GetCurrentPosition());
if (wastefullFileId == 0) break;
MoveValuesContent(writer, wastefullFileId);
_fileStats[wastefullFileId] = new FileStat(0);
toRemoveFileIds.Add(wastefullFileId);
}
var valueFile = _keyValueDB.FileCollection.GetFile(valueFileId);
valueFile.HardFlush();
valueFile.Truncate();
var btreesCorrectInTransactionId = _keyValueDB.AtomicallyChangeBTree(root => root.RemappingIterate((uint oldFileId, uint oldOffset, out uint newFileId, out uint newOffset) =>
{
newFileId = valueFileId;
_cancellation.ThrowIfCancellationRequested();
return _newPositionMap.TryGetValue(((ulong)oldFileId << 32) | oldOffset, out newOffset);
}));
_keyValueDB.CreateIndexFile(_cancellation);
_keyValueDB.WaitForFinishingTransactionsBefore(btreesCorrectInTransactionId, _cancellation);
if (_newPositionMap.Count == 0)
{
toRemoveFileIds.Add(valueFileId);
}
_keyValueDB.MarkAsUnknown(toRemoveFileIds);
_keyValueDB.FileCollection.DeleteAllUnknownFiles();
return true;
}
public static FileStat Stat(this Node node, bool raw)
{
const long unixEpocMs = 62135622000000;
// build a VisualStudioCode:Stat object.
var ctime = node.Attributes.Items.Min(kv => kv.Timestamp);
var mtime = node.Attributes.Items.Max(kv => kv.Timestamp);
if (raw == true)
{
var rawsize = Node.Serializer.GetMaxSize(node);
var rawStat = new FileStat
{
Type = node.IsDirectory() ? 2 : 1, // 0 unknown, 64, symbolicLInk
Ctime = ctime / 10000 - unixEpocMs, //milliseconds elapsed since January 1, 1970 00:00:00 UTC
Mtime = mtime / 10000 - unixEpocMs, //milliseconds elapsed since January 1, 1970 00:00:00 UTC
Size = rawsize
};
return(rawStat);
}
var fileAttr = node.Attributes.Items.FirstOrDefault(kv => kv.Key.Equals(Attr.File));
var fileAttrSize = fileAttr?.Value.Kind == Primitive.ItemKind.TypeBytes &&
fileAttr.Value.TypeBytes.ByteArray != null
? fileAttr.Value.TypeBytes.ByteArray.Value.Length
: 0;
if (node.IsDirectory())
{
var children = node.Attributes.Items.FirstOrDefault(kv => kv.Key.Equals(Attr.Children));
mtime = children?.Timestamp ?? ctime;
fileAttrSize = children?.Value.Kind == Primitive.ItemKind.Array ? children?.Value.Array.Items.Count ?? 0 : 0;
}
var statObj = new FileStat
{
Type = node.IsDirectory() ? 2 : 1, // 0 unknown, 64, symbolicLInk
Ctime = ctime / 10000 - unixEpocMs, //milliseconds elapsed since January 1, 1970 00:00:00 UTC
Mtime = mtime / 10000 - unixEpocMs, //milliseconds elapsed since January 1, 1970 00:00:00 UTC
Size = fileAttrSize
};
return(statObj);
}
private string GenerateScatterPlotData(List <ScatterPoint> commitDates, FileStat fileStat)
{
var commitDatesOrdered = fileStat.CommitDates.OrderBy(date => date);
foreach (var commitDate in commitDatesOrdered)
{
var date = $"new Date('{commitDate.ToString("yyyy-MM-dd")}')";
var userId = UserNameKey[fileStat.Username];
if (!commitDates.Any(cd => cd.Date == date && cd.UserId == userId)) // Only add a point for each user for each date
{
commitDates.Add(new ScatterPoint
{
Date = date,
UserId = userId,
ToolTip = $"'{commitDate.ToString("yyyy-MM-dd")}, {fileStat.Username}'"
});
}
}
var latestCommitDate = commitDatesOrdered.Last();
return(latestCommitDate.ToString("yyyy-MM-dd"));
}
internal bool Run()
{
if (_keyValueDB.FileCollection.GetCount() == 0)
{
return(false);
}
_root = _keyValueDB.OldestRoot;
var dontTouchGeneration = _keyValueDB.GetGeneration(_root.TrLogFileId);
InitFileStats(dontTouchGeneration);
CalculateFileUsefullness();
MarkTotallyUselessFilesAsUnknown();
var totalWaste = CalcTotalWaste();
_keyValueDB.Logger?.CompactionStart(totalWaste);
if (IsWasteSmall(totalWaste))
{
if (_keyValueDB.DistanceFromLastKeyIndex(_root) > (ulong)(_keyValueDB.MaxTrLogFileSize / 4))
{
_keyValueDB.CreateIndexFile(_cancellation);
}
_keyValueDB.FileCollection.DeleteAllUnknownFiles();
return(false);
}
_cancellation.ThrowIfCancellationRequested();
uint valueFileId;
var writer = _keyValueDB.StartPureValuesFile(out valueFileId);
var toRemoveFileIds = new List <uint>();
_newPositionMap = new Dictionary <ulong, uint>();
while (true)
{
var wastefullFileId = FindMostWastefullFile(_keyValueDB.MaxTrLogFileSize - writer.GetCurrentPosition());
if (wastefullFileId == 0)
{
break;
}
MoveValuesContent(writer, wastefullFileId);
_fileStats[wastefullFileId] = new FileStat(0);
toRemoveFileIds.Add(wastefullFileId);
}
var valueFile = _keyValueDB.FileCollection.GetFile(valueFileId);
valueFile.HardFlush();
valueFile.Truncate();
_keyValueDB.Logger?.CompactionCreatedPureValueFile(valueFileId, valueFile.GetSize());
var btreesCorrectInTransactionId = _keyValueDB.AtomicallyChangeBTree(root => root.RemappingIterate((uint oldFileId, uint oldOffset, out uint newFileId, out uint newOffset) =>
{
newFileId = valueFileId;
_cancellation.ThrowIfCancellationRequested();
return(_newPositionMap.TryGetValue(((ulong)oldFileId << 32) | oldOffset, out newOffset));
}));
_keyValueDB.CreateIndexFile(_cancellation);
if (_newPositionMap.Count == 0)
{
toRemoveFileIds.Add(valueFileId);
}
if (_keyValueDB.AreAllTransactionsBeforeFinished(btreesCorrectInTransactionId))
{
_keyValueDB.MarkAsUnknown(toRemoveFileIds);
}
_keyValueDB.FileCollection.DeleteAllUnknownFiles();
return(true);
}
internal bool Run()
{
if (_keyValueDB.FileCollection.GetCount() == 0)
{
return(false);
}
_root = _keyValueDB.OldestRoot;
var dontTouchGeneration = _keyValueDB.GetGeneration(_root.TrLogFileId);
var preserveKeyIndexKey = _keyValueDB.CalculatePreserveKeyIndexKeyFromKeyIndexInfos(_keyValueDB.BuildKeyIndexInfos());
var preserveKeyIndexGeneration = _keyValueDB.CalculatePreserveKeyIndexGeneration(preserveKeyIndexKey);
InitFileStats(dontTouchGeneration);
long[] usedFilesFromOldGenerations = null;
if (preserveKeyIndexKey < uint.MaxValue)
{
var dontTouchGenerationDueToPreserve = -1L;
var fileInfo = _keyValueDB.FileCollection.FileInfoByIdx(preserveKeyIndexKey) as IKeyIndex;
if (fileInfo != null)
{
dontTouchGenerationDueToPreserve = fileInfo.Generation;
dontTouchGenerationDueToPreserve = Math.Min(dontTouchGenerationDueToPreserve, _keyValueDB.GetGeneration(fileInfo.TrLogFileId));
if (fileInfo.UsedFilesInOlderGenerations == null)
{
_keyValueDB.LoadUsedFilesFromKeyIndex(preserveKeyIndexKey, fileInfo);
}
usedFilesFromOldGenerations = fileInfo.UsedFilesInOlderGenerations;
}
dontTouchGeneration = Math.Min(dontTouchGeneration, dontTouchGenerationDueToPreserve);
}
var lastCommited = _keyValueDB.LastCommited;
if (_root != lastCommited)
{
ForbidDeleteOfFilesUsedByStillRunningOldTransaction(_root);
}
ForbidDeletePreservingHistory(dontTouchGeneration, usedFilesFromOldGenerations);
CalculateFileUsefullness(lastCommited);
MarkTotallyUselessFilesAsUnknown();
var totalWaste = CalcTotalWaste();
_keyValueDB.Logger?.CompactionStart(totalWaste);
if (IsWasteSmall(totalWaste))
{
if (_keyValueDB.DistanceFromLastKeyIndex(_root) > (ulong)(_keyValueDB.MaxTrLogFileSize / 4))
{
_keyValueDB.CreateIndexFile(_cancellation, preserveKeyIndexGeneration);
}
_keyValueDB.FileCollection.DeleteAllUnknownFiles();
return(false);
}
_cancellation.ThrowIfCancellationRequested();
uint valueFileId;
var writer = _keyValueDB.StartPureValuesFile(out valueFileId);
var toRemoveFileIds = new List <uint>();
_newPositionMap = new Dictionary <ulong, uint>();
while (true)
{
var wastefullFileId = FindMostWastefullFile(_keyValueDB.MaxTrLogFileSize - writer.GetCurrentPosition());
if (wastefullFileId == 0)
{
break;
}
MoveValuesContent(writer, wastefullFileId);
if (_fileStats[wastefullFileId].IsFreeToDelete())
{
toRemoveFileIds.Add(wastefullFileId);
}
_fileStats[wastefullFileId] = new FileStat(0);
}
var valueFile = _keyValueDB.FileCollection.GetFile(valueFileId);
valueFile.HardFlush();
valueFile.Truncate();
_keyValueDB.Logger?.CompactionCreatedPureValueFile(valueFileId, valueFile.GetSize());
var btreesCorrectInTransactionId = _keyValueDB.AtomicallyChangeBTree(root => root.RemappingIterate((uint oldFileId, uint oldOffset, out uint newFileId, out uint newOffset) =>
{
newFileId = valueFileId;
_cancellation.ThrowIfCancellationRequested();
return(_newPositionMap.TryGetValue(((ulong)oldFileId << 32) | oldOffset, out newOffset));
}));
_keyValueDB.CreateIndexFile(_cancellation, preserveKeyIndexGeneration);
if (_newPositionMap.Count == 0)
{
toRemoveFileIds.Add(valueFileId);
}
if (_keyValueDB.AreAllTransactionsBeforeFinished(btreesCorrectInTransactionId))
{
_keyValueDB.MarkAsUnknown(toRemoveFileIds);
}
_keyValueDB.FileCollection.DeleteAllUnknownFiles();
return(true);
}
