protected ManifestDirectoryInfo MakeInnerParentDirectory(
String hashedHashString,
ManifestDirectoryInfo dir)
{
if (dir.Files.Count < 256)
{
return(dir);
}
int nextDirLength = dir.Name.Length;
// Root dir is named ".", but pretend it is ""
if (nextDirLength == 1)
{
nextDirLength = 0;
}
// Increase specificity of next subdirectory name by 2 letters:
// For example a497 -> a4973e
nextDirLength += 2;
String nextDirName =
hashedHashString.Substring(0, nextDirLength);
if (dir.Subdirectories.ContainsKey(nextDirName))
{
return(MakeInnerParentDirectory(
hashedHashString,
dir.Subdirectories[nextDirName]));
}
return(new ManifestDirectoryInfo(nextDirName, dir));
}
protected void CheckDuplicatesRecursive(
ManifestDirectoryInfo currentDirectory,
Dictionary <FileHash, List <ManifestFileInfo> > fileDict)
{
foreach (ManifestFileInfo nextFileInfo in
currentDirectory.Files.Values)
{
if (fileDict.ContainsKey(nextFileInfo.FileHash) == false)
{
fileDict.Add(
nextFileInfo.FileHash,
new List <ManifestFileInfo>());
}
fileDict[nextFileInfo.FileHash].Add(nextFileInfo);
}
foreach (ManifestDirectoryInfo nextDirInfo in
currentDirectory.Subdirectories.Values)
{
CheckDuplicatesRecursive(
nextDirInfo,
fileDict);
}
}
protected void BuildHashedStringMap(
ManifestDirectoryInfo dir)
{
foreach (String nextFileName
in dir.Files.Keys)
{
if (nextFileName != DefaultOuterManifestFileName)
{
if (myHashedStringMap.ContainsKey(nextFileName) == true)
{
// TODO: Setup and use console delegate here instead
System.Console.WriteLine(
"WARNING: Duplicate filename \"" +
nextFileName +
"\" detected. Ignoring.");
}
else
{
myHashedStringMap.Add(
nextFileName,
dir.Files[nextFileName]);
}
}
}
foreach (ManifestDirectoryInfo nextDir in
dir.Subdirectories.Values)
{
BuildHashedStringMap(nextDir);
}
}
protected void CompareManifestsRecursiveSource(
ManifestDirectoryInfo sourceDir,
ManifestDirectoryInfo destDir,
HashSet <ManifestFileInfo> destFileMatch)
{
foreach (ManifestFileInfo sourceFile in sourceDir.Files.Values)
{
if (destDir != null &&
destDir.Files.ContainsKey(sourceFile.Name))
{
ManifestFileInfo destFile = destDir.Files[sourceFile.Name];
destFileMatch.Add(destFile);
if (sourceFile.FileHash.Equals(destFile.FileHash) == false)
{
ChangedFiles.Add(sourceFile, destFile);
}
else
{
if (Manifest.CompareManifestDates(
sourceFile.LastModifiedUtc,
destFile.LastModifiedUtc) == false)
{
LastModifiedDateFiles.Add(sourceFile, destFile);
}
if (Manifest.CompareManifestDates(
sourceFile.RegisteredUtc,
destFile.RegisteredUtc) == false)
{
RegisteredDateFiles.Add(sourceFile, destFile);
}
}
}
else
{
SourceOnlyFiles.Add(sourceFile);
}
}
foreach (ManifestDirectoryInfo nextSourceDir in
sourceDir.Subdirectories.Values)
{
ManifestDirectoryInfo nextDestDir = null;
if (destDir != null &&
destDir.Subdirectories.ContainsKey(nextSourceDir.Name))
{
nextDestDir = destDir.Subdirectories[nextSourceDir.Name];
}
CompareManifestsRecursiveSource(
nextSourceDir,
nextDestDir,
destFileMatch);
}
}
protected void SaveOuterManifest()
{
// Serialize the manifest to memory
MemoryStream serializedManifestStream =
new MemoryStream();
OuterManifest.WriteManifestStream(serializedManifestStream);
serializedManifestStream.Position = 0;
String tempFilePath =
Path.Combine(
InnerProxy.TempDirectory.FullName,
DefaultOuterManifestFileName);
// We use the inner GUID as salt for the outer manifest, so update
// it each time we write the outer manifest. The inner GUID is
// really useless anyways.
InnerProxy.Manifest.ChangeGUID();
byte[] outerKeyData = CryptUtilities.MakeKeyBytesFromString(
OuterKeyString,
InnerProxy.Manifest.Guid.ToByteArray());
byte[] cryptHash = WriteCryptFileAndHash(
serializedManifestStream,
outerKeyData,
tempFilePath);
// The new ManifestFileInfo is actually rooted in the inner
// Manifest object, but that is ok - although it is kind of a
// hack. The fact is that we don't maintain an actual Manifest
// object to mirror the inner manifest - and we know that the
// implementation of PutFile won't be affected by doing this.
ManifestDirectoryInfo parentDirectory =
InnerProxy.Manifest.RootDirectory;
ManifestFileInfo destManifestFile =
new ManifestFileInfo(
DefaultOuterManifestFileName,
parentDirectory);
destManifestFile.RegisteredUtc = DateTime.Now;
FileInfo outerManifestFileInfo = new FileInfo(tempFilePath);
destManifestFile.LastModifiedUtc =
outerManifestFileInfo.LastWriteTimeUtc;
destManifestFile.FileLength =
outerManifestFileInfo.Length;
destManifestFile.FileHash =
new FileHash(cryptHash, CryptUtilities.DefaultHashType);
InnerProxy.PutFile(ProxyToInner, destManifestFile);
}
protected long ValidateDir(
ManifestDirectoryInfo outerManDirInfo,
Utilities.Console console)
{
foreach (ManifestFileInfo outerManFileInfo in
outerManDirInfo.Files.Values)
{
ValidateFile(outerManFileInfo, console);
}
long fileCount = outerManDirInfo.Files.Count;
foreach (ManifestDirectoryInfo nextOuterDir in
outerManDirInfo.Subdirectories.Values)
{
fileCount += ValidateDir(nextOuterDir, console);
}
return(fileCount);
}
protected void CompareManifestsRecursiveDest(
ManifestDirectoryInfo destDir,
HashSet <ManifestFileInfo> destFileMatch)
{
foreach (ManifestFileInfo destFile in
destDir.Files.Values)
{
if (destFileMatch.Contains(destFile) == false)
{
DestOnlyFiles.Add(destFile);
}
}
foreach (ManifestDirectoryInfo nextDestDir in
destDir.Subdirectories.Values)
{
CompareManifestsRecursiveDest(
nextDestDir,
destFileMatch);
}
}
protected void BuildHashToInnerFileMap(
ManifestDirectoryInfo dir)
{
foreach (ManifestFileInfo nextFile in dir.Files.Values)
{
FileHash fileHash = nextFile.FileHash;
if (myHashToInnerFileMap.ContainsKey(fileHash))
{
continue;
}
// Using FileHash class for convenience
FileHash hashedHash = FileHash.ComputeHash(
fileHash.HashData);
String hashedHashString = hashedHash.ToString();
if (myHashedStringMap.Keys.Contains(
hashedHashString))
{
myHashToInnerFileMap[fileHash] =
myHashedStringMap[hashedHashString];
}
else
{
UnresolvedOuterFiles.Add(nextFile);
}
}
foreach (ManifestDirectoryInfo nextDir in
dir.Subdirectories.Values)
{
BuildHashToInnerFileMap(
nextDir);
}
}
protected ManifestFileInfo GetOrMakeManifestFileInfoFromParts(
Manifest manifest,
List <String> parts,
bool makeNewEntryIfNeeded = true)
{
lock (manifest)
{
ManifestDirectoryInfo currentParentThis =
manifest.RootDirectory;
int partIndex = 0;
for (; partIndex < parts.Count - 1; partIndex++)
{
String uriPart = parts[partIndex];
if (currentParentThis.Subdirectories.Keys.Contains(uriPart))
{
currentParentThis = currentParentThis.Subdirectories[uriPart];
}
else
{
if (makeNewEntryIfNeeded)
{
ManifestDirectoryInfo newParent =
new ManifestDirectoryInfo(
uriPart,
currentParentThis);
currentParentThis.Subdirectories[uriPart] =
newParent;
currentParentThis = newParent;
}
else
{
return(null);
}
}
}
String fileName = parts[partIndex];
if (currentParentThis.Files.Keys.Contains(fileName))
{
return(currentParentThis.Files[fileName]);
}
if (makeNewEntryIfNeeded)
{
ManifestFileInfo newManifestFile =
new ManifestFileInfo(
fileName,
currentParentThis);
currentParentThis.Files[newManifestFile.Name] =
newManifestFile;
return(newManifestFile);
}
return(null);
}
}
protected void UpdateRecursive(
DirectoryInfo currentDirectoryInfo,
ManifestDirectoryInfo currentManfestDirInfo)
{
// Setup data for current directory
Dictionary <String, FileInfo> fileDict =
new Dictionary <string, FileInfo>();
Dictionary <String, DirectoryInfo> dirDict =
new Dictionary <string, DirectoryInfo>();
if (currentDirectoryInfo != null)
{
FileInfo[] fileList = null;
try
{
fileList = currentDirectoryInfo.GetFiles();
}
catch (Exception)
{
WriteLine(Manifest.MakeStandardPathString(
currentManfestDirInfo));
if (IgnoreFile(Manifest.MakeStandardPathString(
currentManfestDirInfo)) == true)
{
// This was implemented primarily to allow the user to
// silence the process of skipping over inaccessible
// system directories by ignoring them. For example,
// in some cases the "$RECYCLE BIN" under Windows
// is not accessible and will generate an error. The
// user can now add such directories to the ignore list
// and they will be silently ignored. The special
// message for showProgress alerts the user that the
// directory is actually being skipped altogether
// since it can't be accessed. The only significant
// implication of this is that the ignored files won't
// be enumerated and counted as being ignored.
if (ShowProgress)
{
WriteLine(
Manifest.MakeStandardPathString(currentManfestDirInfo) +
" [IGNORED DIRECTORY AND CANNOT ACCESS]");
}
}
else
{
ForceWriteLine("Could not access contents of: " +
currentDirectoryInfo.FullName);
}
return;
}
foreach (FileInfo nextFileInfo in fileList)
{
fileDict.Add(nextFileInfo.Name.Normalize(), nextFileInfo);
}
DirectoryInfo[] dirList =
currentDirectoryInfo.GetDirectories();
foreach (DirectoryInfo nextDirInfo in dirList)
{
dirDict.Add(nextDirInfo.Name.Normalize(), nextDirInfo);
}
}
// Clone in case we modify during iteration
List <ManifestFileInfo> fileListClone =
new List <ManifestFileInfo>(currentManfestDirInfo.Files.Values);
// Iterate through existing manifest entries
foreach (ManifestFileInfo nextManFileInfo in fileListClone)
{
if (ShowProgress)
{
Write(Manifest.MakeStandardPathString(nextManFileInfo));
}
if (fileDict.ContainsKey(nextManFileInfo.Name))
{
FileCheckedCount++;
FileInfo nextFileInfo = fileDict[nextManFileInfo.Name];
if (IgnoreFile(Manifest.MakeStandardPathString(nextManFileInfo)))
{
Write(" [NEWLY IGNORED]");
currentManfestDirInfo.Files.Remove(
nextManFileInfo.Name);
NewlyIgnoredFiles.Add(nextManFileInfo);
}
else if (nextFileInfo.Length != nextManFileInfo.FileLength &&
Update == false &&
AlwaysCheckHash == false)
{
// Don't compute hash if we aren't doing an update
Write(" [DIFFERENT]");
ChangedFiles.Add(nextManFileInfo);
}
else if (AlwaysCheckHash == true ||
MakeNewHash == true ||
nextManFileInfo.FileHash == null ||
Manifest.CompareManifestDateToFilesystemDate(nextFileInfo.LastWriteTimeUtc, nextManFileInfo.LastModifiedUtc) == false ||
nextFileInfo.Length != nextManFileInfo.FileLength)
{
FileHash checkHash = null;
Exception exception = null;
try
{
string hashType = Manifest.DefaultHashMethod;
if (nextManFileInfo.FileHash != null)
{
hashType = nextManFileInfo.FileHash.HashType;
}
checkHash = FileHash.ComputeHash(
nextFileInfo,
hashType);
}
catch (Exception ex)
{
exception = ex;
}
if (exception != null)
{
WriteLine(" [ERROR]");
WriteLine(exception.ToString());
ErrorFiles.Add(nextManFileInfo);
}
else
{
if (nextManFileInfo.FileHash == null)
{
Write(" [NULL HASH IN MANIFEST]");
ChangedFiles.Add(nextManFileInfo);
}
else if (checkHash.Equals(nextManFileInfo.FileHash) == false)
{
Write(" [DIFFERENT]");
ChangedFiles.Add(nextManFileInfo);
}
else
{
if (Manifest.CompareManifestDateToFilesystemDate(
nextFileInfo.LastWriteTimeUtc,
nextManFileInfo.LastModifiedUtc) == false)
{
Write(" [LAST MODIFIED DATE]");
LastModifiedDateFiles.Add(nextManFileInfo);
if (BackDate == true)
{
nextFileInfo.LastWriteTimeUtc =
nextManFileInfo.LastModifiedUtc;
}
}
}
}
FileHash newHash = checkHash;
if (MakeNewHash)
{
newHash = FileHash.ComputeHash(
nextFileInfo,
GetNewHashType(Manifest));
}
// Update hash and last modified date accordingly
nextManFileInfo.FileHash = newHash;
nextManFileInfo.LastModifiedUtc = nextFileInfo.LastWriteTimeUtc;
nextManFileInfo.FileLength = nextFileInfo.Length;
}
else
{
Write(" [SKIPPED]");
}
}
else
{
Write(" [MISSING]");
currentManfestDirInfo.Files.Remove(nextManFileInfo.Name);
MissingFiles.Add(nextManFileInfo);
}
WriteLine("");
}
// Clone in case we modify during iteration
List <ManifestDirectoryInfo> directoryListClone =
new List <ManifestDirectoryInfo>(
currentManfestDirInfo.Subdirectories.Values);
foreach (ManifestDirectoryInfo nextManDirInfo in
directoryListClone)
{
DirectoryInfo nextDirInfo = null;
if (dirDict.ContainsKey(nextManDirInfo.Name))
{
nextDirInfo = dirDict[nextManDirInfo.Name];
}
UpdateRecursive(
nextDirInfo,
nextManDirInfo);
if (nextManDirInfo.Empty)
{
currentManfestDirInfo.Subdirectories.Remove(
nextManDirInfo.Name);
}
}
// Look for new files
foreach (String nextFileName in fileDict.Keys)
{
FileInfo nextFileInfo = fileDict[nextFileName];
if (currentManfestDirInfo.Files.ContainsKey(
nextFileName) == false)
{
ManifestFileInfo newManFileInfo =
new ManifestFileInfo(
nextFileName,
currentManfestDirInfo);
Write(Manifest.MakeStandardPathString(newManFileInfo));
if (IgnoreFile(Manifest.MakeStandardPathString(newManFileInfo)))
{
IgnoredFiles.Add(newManFileInfo);
// Don't groom the manifest file!
if (Manifest.MakeNativePathString(newManFileInfo) !=
ManifestNativeFilePath)
{
IgnoredFilesForGroom.Add(nextFileInfo);
}
Write(" [IGNORED]");
}
else
{
FileCheckedCount++;
bool checkHash = false;
if (Update == true ||
AlwaysCheckHash == true ||
TrackMoves == true)
{
checkHash = true;
}
Exception exception = null;
if (checkHash)
{
try
{
newManFileInfo.FileHash =
FileHash.ComputeHash(
nextFileInfo,
GetNewHashType(Manifest));
}
catch (Exception ex)
{
exception = ex;
}
}
if (checkHash && newManFileInfo.FileHash == null)
{
ErrorFiles.Add(newManFileInfo);
WriteLine(" [ERROR]");
WriteLine(exception.ToString());
}
else
{
NewFiles.Add(newManFileInfo);
NewFilesForGroom.Add(nextFileInfo);
Write(" [NEW]");
}
newManFileInfo.FileLength =
nextFileInfo.Length;
newManFileInfo.LastModifiedUtc =
nextFileInfo.LastWriteTimeUtc;
newManFileInfo.RegisteredUtc =
DateTime.Now.ToUniversalTime();
currentManfestDirInfo.Files.Add(
nextFileName,
newManFileInfo);
}
WriteLine("");
}
}
// Recurse looking for new directories
foreach (String nextDirName in dirDict.Keys)
{
DirectoryInfo nextDirInfo = dirDict[nextDirName];
if (currentManfestDirInfo.Subdirectories.ContainsKey(
nextDirName) == false)
{
ManifestDirectoryInfo nextManDirInfo =
new ManifestDirectoryInfo(
nextDirName,
currentManfestDirInfo);
currentManfestDirInfo.Subdirectories.Add(
nextDirName,
nextManDirInfo);
UpdateRecursive(
nextDirInfo,
nextManDirInfo);
if (nextManDirInfo.Empty)
{
currentManfestDirInfo.Subdirectories.Remove(
nextDirName);
}
}
}
}
public void PutFile(
IRepositoryProxy sourceRepository,
ManifestFileInfo sourceManifestFile)
{
// Name the inner file with the hash of the hash. We protect
// the hash in this way because it is used as the salt to
// encrypt the data in the file, and it might provide some
// benefit to a cryptographic attack.
FileHash hashedHash = FileHash.ComputeHash(
sourceManifestFile.FileHash.HashData);
String hashedHashString = hashedHash.ToString();
// Only add the file data if we don't have it already.
if (myHashedStringMap.ContainsKey(hashedHashString) == false)
{
FileInfo sourceFileInfo =
sourceRepository.GetFile(sourceManifestFile);
byte[] keyData = CryptUtilities.MakeKeyBytesFromString(
OuterKeyString,
sourceManifestFile.FileHash.HashData);
// Use the inner proxy temp directory because that is likely
// the ultimate destination of the file and we don't want to
// copy the data if we can avoid it. This is a minor break in
// encapsulation but has a significant impact on performance.
String destFilePath =
Path.Combine(
InnerProxy.TempDirectory.FullName,
hashedHashString);
Stream sourceFileStream = sourceFileInfo.OpenRead();
byte[] cryptHash = WriteCryptFileAndHash(
sourceFileStream,
keyData,
destFilePath);
FileInfo cryptFileInfo =
new FileInfo(destFilePath);
// Make a dummy parent manifest directory to give to the inner
// proxy. This is actually rooted in the inner manifest, but
// that is ok - although it is kind of a hack. The fact is
// that we don't maintain an actual manifest to mirror the
// inner manifest - and we know that the implementation of
// PutFile won't be affected by doing this.
ManifestDirectoryInfo parentDirectory =
MakeInnerParentDirectory(
hashedHashString,
InnerProxy.Manifest.RootDirectory);
ManifestFileInfo destManifestFile =
new ManifestFileInfo(
hashedHashString,
parentDirectory);
destManifestFile.RegisteredUtc =
DateTime.UtcNow;
destManifestFile.LastModifiedUtc =
cryptFileInfo.LastWriteTimeUtc;
destManifestFile.FileLength =
cryptFileInfo.Length;
destManifestFile.FileHash =
new FileHash(cryptHash, CryptUtilities.DefaultHashType);
InnerProxy.PutFile(ProxyToInner, destManifestFile);
myHashedStringMap.Add(hashedHashString, destManifestFile);
myNeedToRegenerateFileMap = true;
}
ManifestFileInfo outerManifestFileInfo =
Manifest.PutFileFromOtherManifest(sourceManifestFile);
myManifestChanged = true;
}
