public void AddsDefaultEntry()
{
ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(initialContents: string.Empty);
modifiedPathsDatabase.Count.ShouldEqual(1);
modifiedPathsDatabase.Contains(DefaultEntry, isFolder: false).ShouldBeTrue();
}
public void ParsesExistingDataCorrectly()
{
ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(ExistingEntries);
modifiedPathsDatabase.Count.ShouldEqual(3);
modifiedPathsDatabase.Contains("file.txt", isFolder: false).ShouldBeTrue();
modifiedPathsDatabase.Contains("dir/file2.txt", isFolder: false).ShouldBeTrue();
modifiedPathsDatabase.Contains("dir1/dir2/file3.txt", isFolder: false).ShouldBeTrue();
}
private static void TestAddingPath(string pathToAdd, string pathInList, bool isFolder = false)
{
ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(initialContents: $"A {DefaultEntry}\r\n");
bool isRetryable;
modifiedPathsDatabase.TryAdd(pathToAdd, isFolder, out isRetryable);
modifiedPathsDatabase.Count.ShouldEqual(2);
modifiedPathsDatabase.Contains(pathInList, isFolder).ShouldBeTrue();
modifiedPathsDatabase.Contains(ToGitPathSeparators(pathInList), isFolder).ShouldBeTrue();
modifiedPathsDatabase.GetAllModifiedPaths().ShouldContainSingle(x => string.Compare(x, ToGitPathSeparators(pathInList), GVFSPlatform.Instance.Constants.PathComparison) == 0);
}
private static void TestAddingPath(string pathToAdd, string pathInList, bool isFolder = false)
{
ModifiedPathsDatabase mpd = CreateModifiedPathsDatabase(initialContents: $"A {DefaultEntry}\r\n");
bool isRetryable;
mpd.TryAdd(pathToAdd, isFolder, out isRetryable);
mpd.Count.ShouldEqual(2);
mpd.Contains(pathInList, isFolder).ShouldBeTrue();
mpd.Contains(ToGitPathSeparators(pathInList), isFolder).ShouldBeTrue();
mpd.GetAllModifiedPaths().ShouldContainSingle(x => string.Compare(x, ToGitPathSeparators(pathInList), StringComparison.OrdinalIgnoreCase) == 0);
}
public void BadDataFailsToLoad()
{
ConfigurableFileSystem configurableFileSystem = new ConfigurableFileSystem();
configurableFileSystem.ExpectedFiles.Add(MockEntryFileName, new ReusableMemoryStream("This is bad data!\r\n"));
string error;
ModifiedPathsDatabase modifiedPathsDatabase;
ModifiedPathsDatabase.TryLoadOrCreate(null, MockEntryFileName, configurableFileSystem, out modifiedPathsDatabase, out error).ShouldBeFalse();
modifiedPathsDatabase.ShouldBeNull();
}
public void RemoveEntriesWithParentFolderEntry()
{
ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(EntriesToCompress);
modifiedPathsDatabase.RemoveEntriesWithParentFolderEntry(new MockTracer());
modifiedPathsDatabase.Count.ShouldEqual(5);
modifiedPathsDatabase.Contains("file.txt", isFolder: false).ShouldBeTrue();
modifiedPathsDatabase.Contains("dir/", isFolder: true).ShouldBeTrue();
modifiedPathsDatabase.Contains("dir1/dir2", isFolder: false).ShouldBeTrue();
modifiedPathsDatabase.Contains("dir1/dir2/", isFolder: true).ShouldBeTrue();
modifiedPathsDatabase.Contains("dir1/file.txt", isFolder: false).ShouldBeTrue();
}
private static ModifiedPathsDatabase CreateModifiedPathsDatabase(string initialContents)
{
ConfigurableFileSystem configurableFileSystem = new ConfigurableFileSystem();
configurableFileSystem.ExpectedFiles.Add(MockEntryFileName, new ReusableMemoryStream(initialContents));
string error;
ModifiedPathsDatabase modifiedPathsDatabase;
ModifiedPathsDatabase.TryLoadOrCreate(null, MockEntryFileName, configurableFileSystem, out modifiedPathsDatabase, out error).ShouldBeTrue();
modifiedPathsDatabase.ShouldNotBeNull();
return(modifiedPathsDatabase);
}
public void EntryNotAddedIfParentDirectoryExists()
{
ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(initialContents: "A dir/\r\n");
modifiedPathsDatabase.Count.ShouldEqual(1);
modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue();
// Try adding a file for the directory that is in the modified paths
modifiedPathsDatabase.TryAdd("dir/file.txt", isFolder: false, isRetryable: out _);
modifiedPathsDatabase.Count.ShouldEqual(1);
modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue();
// Try adding a directory for the directory that is in the modified paths
modifiedPathsDatabase.TryAdd("dir/dir2", isFolder: true, isRetryable: out _);
modifiedPathsDatabase.Count.ShouldEqual(1);
modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue();
// Try adding a file for a directory that is not in the modified paths
modifiedPathsDatabase.TryAdd("dir2/file.txt", isFolder: false, isRetryable: out _);
modifiedPathsDatabase.Count.ShouldEqual(2);
modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue();
modifiedPathsDatabase.Contains("dir2/file.txt", isFolder: false).ShouldBeTrue();
// Try adding a directory for a the directory that is not in the modified paths
modifiedPathsDatabase.TryAdd("dir2/dir", isFolder: true, isRetryable: out _);
modifiedPathsDatabase.Count.ShouldEqual(3);
modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue();
modifiedPathsDatabase.Contains("dir2/file.txt", isFolder: false).ShouldBeTrue();
modifiedPathsDatabase.Contains("dir2/dir", isFolder: true).ShouldBeTrue();
// Try adding a file in a subdirectory that is in the modified paths
modifiedPathsDatabase.TryAdd("dir2/dir/file.txt", isFolder: false, isRetryable: out _);
modifiedPathsDatabase.Count.ShouldEqual(3);
modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue();
modifiedPathsDatabase.Contains("dir2/file.txt", isFolder: false).ShouldBeTrue();
modifiedPathsDatabase.Contains("dir2/dir", isFolder: true).ShouldBeTrue();
// Try adding a directory for a subdirectory that is in the modified paths
modifiedPathsDatabase.TryAdd("dir2/dir/dir3", isFolder: true, isRetryable: out _);
modifiedPathsDatabase.Count.ShouldEqual(3);
modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue();
modifiedPathsDatabase.Contains("dir2/file.txt", isFolder: false).ShouldBeTrue();
modifiedPathsDatabase.Contains("dir2/dir", isFolder: true).ShouldBeTrue();
}
public void Dispose()
{
if (this.BlobSizes != null)
{
this.BlobSizes.Dispose();
this.BlobSizes = null;
}
if (this.fileSystemVirtualizer != null)
{
this.fileSystemVirtualizer.Dispose();
this.fileSystemVirtualizer = null;
}
if (this.GitIndexProjection != null)
{
this.GitIndexProjection.Dispose();
this.GitIndexProjection = null;
}
if (this.modifiedPaths != null)
{
this.modifiedPaths.Dispose();
this.modifiedPaths = null;
}
if (this.gitStatusCache != null)
{
this.gitStatusCache.Dispose();
this.gitStatusCache = null;
}
if (this.backgroundFileSystemTaskRunner != null)
{
this.backgroundFileSystemTaskRunner.Dispose();
this.backgroundFileSystemTaskRunner = null;
}
if (this.context != null)
{
this.context.Dispose();
this.context = null;
}
}
private void DehydrateFolders(JsonTracer tracer, GVFSEnlistment enlistment, string[] folders)
{
List <string> foldersToDehydrate = new List <string>();
List <string> folderErrors = new List <string>();
if (!this.ShowStatusWhileRunning(
() =>
{
if (!ModifiedPathsDatabase.TryLoadOrCreate(
tracer,
Path.Combine(enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.ModifiedPaths),
this.fileSystem,
out ModifiedPathsDatabase modifiedPaths,
out string error))
{
this.WriteMessage(tracer, $"Unable to open modified paths database: {error}");
return(false);
}
using (modifiedPaths)
{
string ioError;
foreach (string folder in folders)
{
string normalizedPath = GVFSDatabase.NormalizePath(folder);
if (!this.IsFolderValid(normalizedPath))
{
this.WriteMessage(tracer, $"Cannot {this.ActionName} folder '{folder}': invalid folder path.");
}
else
{
// Need to check if parent folder is in the modified paths because
// dehydration will not do any good with a parent folder there
if (modifiedPaths.ContainsParentFolder(folder, out string parentFolder))
{
this.WriteMessage(tracer, $"Cannot {this.ActionName} folder '{folder}': Must {this.ActionName} parent folder '{parentFolder}'.");
}
else
{
string fullPath = Path.Combine(enlistment.WorkingDirectoryBackingRoot, folder);
if (this.fileSystem.DirectoryExists(fullPath))
{
if (!this.TryIO(tracer, () => this.fileSystem.DeleteDirectory(fullPath), $"Deleting '{fullPath}'", out ioError))
{
this.WriteMessage(tracer, $"Cannot {this.ActionName} folder '{folder}': removing '{folder}' failed.");
this.WriteMessage(tracer, "Ensure no applications are accessing the folder and retry.");
this.WriteMessage(tracer, $"More details: {ioError}");
folderErrors.Add($"{folder}\0{ioError}");
}
else
{
foldersToDehydrate.Add(folder);
}
}
else
{
this.WriteMessage(tracer, $"Cannot {this.ActionName} folder '{folder}': '{folder}' does not exist.");
// Still add to foldersToDehydrate so that any placeholders or modified paths get cleaned up
foldersToDehydrate.Add(folder);
}
}
}
}
}
return(true);
},
"Cleaning up folders"))
{
this.ReportErrorAndExit(tracer, $"{this.ActionName} for folders failed.");
}
this.Mount(tracer);
this.SendDehydrateMessage(tracer, enlistment, folderErrors, foldersToDehydrate.ToArray());
if (folderErrors.Count > 0)
{
foreach (string folderError in folderErrors)
{
this.ErrorOutput.WriteLine(folderError);
}
this.ReportErrorAndExit(tracer, ReturnCode.DehydrateFolderFailures, $"Failed to dehydrate {folderErrors.Count} folder(s).");
}
}
public FileSystemCallbacks(
GVFSContext context,
GVFSGitObjects gitObjects,
RepoMetadata repoMetadata,
BlobSizes blobSizes,
GitIndexProjection gitIndexProjection,
BackgroundFileSystemTaskRunner backgroundFileSystemTaskRunner,
FileSystemVirtualizer fileSystemVirtualizer,
GitStatusCache gitStatusCache = null)
{
this.logsHeadFileProperties = null;
this.postFetchJobLock = new object();
this.context = context;
this.gitObjects = gitObjects;
this.fileSystemVirtualizer = fileSystemVirtualizer;
this.placeHolderCreationCount = new ConcurrentDictionary <string, PlaceHolderCreateCounter>(StringComparer.OrdinalIgnoreCase);
this.newlyCreatedFileAndFolderPaths = new ConcurrentHashSet <string>(StringComparer.OrdinalIgnoreCase);
string error;
if (!ModifiedPathsDatabase.TryLoadOrCreate(
this.context.Tracer,
Path.Combine(this.context.Enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.ModifiedPaths),
this.context.FileSystem,
out this.modifiedPaths,
out error))
{
throw new InvalidRepoException(error);
}
this.BlobSizes = blobSizes;
this.BlobSizes.Initialize();
PlaceholderListDatabase placeholders;
if (!PlaceholderListDatabase.TryCreate(
this.context.Tracer,
Path.Combine(this.context.Enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.PlaceholderList),
this.context.FileSystem,
out placeholders,
out error))
{
throw new InvalidRepoException(error);
}
this.GitIndexProjection = gitIndexProjection ?? new GitIndexProjection(
context,
gitObjects,
this.BlobSizes,
repoMetadata,
fileSystemVirtualizer,
placeholders,
this.modifiedPaths);
if (backgroundFileSystemTaskRunner != null)
{
this.backgroundFileSystemTaskRunner = backgroundFileSystemTaskRunner;
this.backgroundFileSystemTaskRunner.SetCallbacks(
this.PreBackgroundOperation,
this.ExecuteBackgroundOperation,
this.PostBackgroundOperation);
}
else
{
this.backgroundFileSystemTaskRunner = new BackgroundFileSystemTaskRunner(
this.context,
this.PreBackgroundOperation,
this.ExecuteBackgroundOperation,
this.PostBackgroundOperation,
Path.Combine(context.Enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.BackgroundFileSystemTasks));
}
this.enableGitStatusCache = gitStatusCache != null;
// If the status cache is not enabled, create a dummy GitStatusCache that will never be initialized
// This lets us from having to add null checks to callsites into GitStatusCache.
this.gitStatusCache = gitStatusCache ?? new GitStatusCache(context, TimeSpan.Zero);
this.logsHeadPath = Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Logs.Head);
EventMetadata metadata = new EventMetadata();
metadata.Add("placeholders.Count", placeholders.EstimatedCount);
metadata.Add("background.Count", this.backgroundFileSystemTaskRunner.Count);
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(FileSystemCallbacks)} created");
this.context.Tracer.RelatedEvent(EventLevel.Informational, $"{nameof(FileSystemCallbacks)}_Constructor", metadata);
}
public override bool TryUpgrade(ITracer tracer, string enlistmentRoot)
{
ModifiedPathsDatabase modifiedPaths = null;
try
{
PhysicalFileSystem fileSystem = new PhysicalFileSystem();
string modifiedPathsDatabasePath = Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.Root, GVFSConstants.DotGVFS.Databases.ModifiedPaths);
string error;
if (!ModifiedPathsDatabase.TryLoadOrCreate(tracer, modifiedPathsDatabasePath, fileSystem, out modifiedPaths, out error))
{
tracer.RelatedError($"Unable to create the modified paths database. {error}");
return(false);
}
string sparseCheckoutPath = Path.Combine(enlistmentRoot, GVFSConstants.WorkingDirectoryRootName, GVFSConstants.DotGit.Info.SparseCheckoutPath);
bool isRetryable;
using (FileStream fs = File.OpenRead(sparseCheckoutPath))
using (StreamReader reader = new StreamReader(fs))
{
string entry = reader.ReadLine();
while (entry != null)
{
entry = entry.Trim();
if (!string.IsNullOrWhiteSpace(entry))
{
bool isFolder = entry.EndsWith(GVFSConstants.GitPathSeparatorString);
if (!modifiedPaths.TryAdd(entry.Trim(GVFSConstants.GitPathSeparator), isFolder, out isRetryable))
{
tracer.RelatedError("Unable to add to the modified paths database.");
return(false);
}
}
entry = reader.ReadLine();
}
}
string alwaysExcludePath = Path.Combine(enlistmentRoot, GVFSConstants.WorkingDirectoryRootName, GVFSConstants.DotGit.Info.AlwaysExcludePath);
if (fileSystem.FileExists(alwaysExcludePath))
{
string alwaysExcludeData = fileSystem.ReadAllText(alwaysExcludePath);
char[] carriageReturnOrLineFeed = new[] { '\r', '\n' };
int endPosition = alwaysExcludeData.Length;
while (endPosition > 0)
{
int startPosition = alwaysExcludeData.LastIndexOfAny(carriageReturnOrLineFeed, endPosition - 1);
if (startPosition < 0)
{
startPosition = 0;
}
string entry = alwaysExcludeData.Substring(startPosition, endPosition - startPosition).Trim();
if (entry.EndsWith("*"))
{
// This is the first entry using the old format and we don't want to process old entries
// because we would need folder entries since there isn't a file and that would cause sparse-checkout to
// recursively clear skip-worktree bits for everything under that folder
break;
}
// Substring will not return a null and the Trim will get rid of all the whitespace
// if there is a length it will be a valid path that we need to process
if (entry.Length > 0)
{
entry = entry.TrimStart('!');
bool isFolder = entry.EndsWith(GVFSConstants.GitPathSeparatorString);
if (!isFolder)
{
if (!modifiedPaths.TryAdd(entry.Trim(GVFSConstants.GitPathSeparator), isFolder, out isRetryable))
{
tracer.RelatedError("Unable to add to the modified paths database.");
return(false);
}
}
}
endPosition = startPosition;
}
}
modifiedPaths.ForceFlush();
fileSystem.WriteAllText(sparseCheckoutPath, "/.gitattributes" + Environment.NewLine);
fileSystem.DeleteFile(alwaysExcludePath);
}
catch (IOException ex)
{
tracer.RelatedError($"IOException: {ex.ToString()}");
return(false);
}
finally
{
if (modifiedPaths != null)
{
modifiedPaths.Dispose();
modifiedPaths = null;
}
}
if (!this.TryIncrementMajorVersion(tracer, enlistmentRoot))
{
return(false);
}
return(true);
}
public FileSystemCallbacks(
GVFSContext context,
GVFSGitObjects gitObjects,
RepoMetadata repoMetadata,
BlobSizes blobSizes,
GitIndexProjection gitIndexProjection,
BackgroundFileSystemTaskRunner backgroundFileSystemTaskRunner,
FileSystemVirtualizer fileSystemVirtualizer)
{
this.logsHeadFileProperties = null;
this.postFetchJobLock = new object();
this.context = context;
this.gitObjects = gitObjects;
this.fileSystemVirtualizer = fileSystemVirtualizer;
this.placeHolderCreationCount = new ConcurrentDictionary <string, PlaceHolderCreateCounter>(StringComparer.OrdinalIgnoreCase);
string error;
if (!ModifiedPathsDatabase.TryLoadOrCreate(
this.context.Tracer,
Path.Combine(this.context.Enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.ModifiedPaths),
this.context.FileSystem,
out this.modifiedPaths,
out error))
{
throw new InvalidRepoException(error);
}
this.BlobSizes = blobSizes;
this.BlobSizes.Initialize();
PlaceholderListDatabase placeholders;
if (!PlaceholderListDatabase.TryCreate(
this.context.Tracer,
Path.Combine(this.context.Enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.PlaceholderList),
this.context.FileSystem,
out placeholders,
out error))
{
throw new InvalidRepoException(error);
}
this.GitIndexProjection = gitIndexProjection ?? new GitIndexProjection(
context,
gitObjects,
this.BlobSizes,
repoMetadata,
fileSystemVirtualizer,
placeholders,
this.modifiedPaths);
this.backgroundFileSystemTaskRunner = backgroundFileSystemTaskRunner ?? new BackgroundFileSystemTaskRunner(
this.context,
this.PreBackgroundOperation,
this.ExecuteBackgroundOperation,
this.PostBackgroundOperation,
Path.Combine(context.Enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.BackgroundFileSystemTasks));
this.logsHeadPath = Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Logs.Head);
EventMetadata metadata = new EventMetadata();
metadata.Add("placeholders.Count", placeholders.EstimatedCount);
metadata.Add("background.Count", this.backgroundFileSystemTaskRunner.Count);
metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(FileSystemCallbacks)} created");
this.context.Tracer.RelatedEvent(EventLevel.Informational, $"{nameof(FileSystemCallbacks)}_Constructor", metadata);
}
public override bool TryUpgrade(ITracer tracer, string enlistmentRoot)
{
ModifiedPathsDatabase modifiedPaths = null;
try
{
PhysicalFileSystem fileSystem = new PhysicalFileSystem();
string modifiedPathsDatabasePath = Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.Root, GVFSConstants.DotGVFS.Databases.ModifiedPaths);
string error;
if (!ModifiedPathsDatabase.TryLoadOrCreate(tracer, modifiedPathsDatabasePath, fileSystem, out modifiedPaths, out error))
{
tracer.RelatedError($"Unable to create the modified paths database. {error}");
return(false);
}
string sparseCheckoutPath = Path.Combine(enlistmentRoot, GVFSConstants.WorkingDirectoryRootName, GVFSConstants.DotGit.Info.SparseCheckoutPath);
IEnumerable <string> sparseCheckoutLines = fileSystem.ReadAllText(sparseCheckoutPath).Split(new string[] { "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries);
bool isRetryable;
foreach (string entry in sparseCheckoutLines)
{
bool isFolder = entry.EndsWith(GVFSConstants.GitPathSeparatorString);
if (!modifiedPaths.TryAdd(entry.Trim(GVFSConstants.GitPathSeparator), isFolder, out isRetryable))
{
tracer.RelatedError("Unable to add to the modified paths database.");
return(false);
}
}
string alwaysExcludePath = Path.Combine(enlistmentRoot, GVFSConstants.WorkingDirectoryRootName, GVFSConstants.DotGit.Info.AlwaysExcludePath);
if (fileSystem.FileExists(alwaysExcludePath))
{
string[] alwaysExcludeLines = fileSystem.ReadAllText(alwaysExcludePath).Split(new string[] { "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries);
for (int i = alwaysExcludeLines.Length - 1; i >= 0; i--)
{
string entry = alwaysExcludeLines[i];
if (entry.EndsWith("*"))
{
// This is the first entry using the old format and we don't want to process old entries
// because we would need folder entries since there isn't a file and that would cause sparse-checkout to
// recursively clear skip-worktree bits for everything under that folder
break;
}
entry = entry.TrimStart('!');
bool isFolder = entry.EndsWith(GVFSConstants.GitPathSeparatorString);
if (!isFolder)
{
if (!modifiedPaths.TryAdd(entry.Trim(GVFSConstants.GitPathSeparator), isFolder, out isRetryable))
{
tracer.RelatedError("Unable to add to the modified paths database.");
return(false);
}
}
}
}
modifiedPaths.ForceFlush();
fileSystem.WriteAllText(sparseCheckoutPath, "/.gitattributes" + Environment.NewLine);
fileSystem.DeleteFile(alwaysExcludePath);
}
catch (IOException ex)
{
tracer.RelatedError($"IOException: {ex.ToString()}");
return(false);
}
finally
{
if (modifiedPaths != null)
{
modifiedPaths.Dispose();
modifiedPaths = null;
}
}
if (!this.TryIncrementMajorVersion(tracer, enlistmentRoot))
{
return(false);
}
return(true);
}