private void ProcessChangesIfAny()
{
lock (DedupLock)
{
if (NewFiles.Any() || DeletedFiles.Any())
{
DedupTask = Task.Factory.StartNew(
ProcessChanges,
CancelSource.Token);
return;
}
if (!CurrentState.Settings.MonitorChanges)
{
operationStartTime = DateTime.MinValue;
OnOperationUpdate(
OperationType.Completed,
ProgressStyle.NoProgress);
OnLogged("Finished comparison.");
return;
}
operationStartTime = DateTime.Now;
OnOperationUpdate(
OperationType.Monitoring,
ProgressStyle.Marquee);
OnLogged("Monitoring for file changes...");
return;
}
}
private IEnumerable <VideoFile> GetVideoFileList(
IFolderSettings folderSettings)
{
operationStartTime = DateTime.Now;
OnOperationUpdate(OperationType.Searching, ProgressStyle.Marquee);
// Get all video files in source path.
var foundFiles = GetAllAccessibleFilesIn(
folderSettings.BasePath,
folderSettings.ExcludedDirectories,
folderSettings.Recursive)
.Where(f => folderSettings.FileExtensions.Contains(
Path.GetExtension(f),
StringComparer.InvariantCultureIgnoreCase))
.Select(f => new VideoFile(f))
.ToList();
if (CurrentState.VideoFiles == null)
{
return(foundFiles);
}
foreach (var file in foundFiles.Except(CurrentState.VideoFiles))
{
NewFiles.TryAdd(file, 0);
}
foreach (var file in CurrentState.VideoFiles.Except(foundFiles))
{
DeletedFiles.TryAdd(file, 0);
}
return(CurrentState.VideoFiles);
}
private void HandleDeletedFileEvent(string filePath)
{
if (!IsFilePathRelevant(filePath, CurrentState.Settings))
{
return;
}
_ = DeletedFiles.TryAdd(new VideoFile(filePath), 0);
OnLogged(string.Format(LogDeletedFile, filePath));
StartProcessingChanges();
}
private bool Undelete()
{
TextBlockControl.Visibility = Visibility.Visible;
if (DeletedFiles.Count == 0)
{
topTextBoxClass.messageDisplayStart("No more files to undelete.", 5);
return(false);
}
string LastDeleted = DeletedFiles.Pop();
if (LastDeleted == "")
{
topTextBoxClass.messageDisplayStart(LastDeleted + " UNDELETE ERROR.", 5);
return(false);
}
FolderItems folderItems = RecyclingBin.Items();
for (int i = 0; i < folderItems.Count; i++)
{
FolderItem FI = folderItems.Item(i);
string FileName = RecyclingBin.GetDetailsOf(FI, 0);
if (Path.GetExtension(FileName) == "")
{
FileName += Path.GetExtension(FI.Path);
}
//Necessary for systems with hidden file extensions.
string FilePath = RecyclingBin.GetDetailsOf(FI, 1);
if (String.Compare(LastDeleted, Path.Combine(FilePath, FileName), true) == 0)
{
FileInfo undelFile;
try
{
DoVerb(FI, "ESTORE");
undelFile = new FileInfo(LastDeleted);
}
catch
{
topTextBoxClass.messageDisplayStart(LastDeleted + " Could not be undeleted, file not found.", 5);
return(false);
}
topTextBoxClass.messageDisplayStart(LastDeleted + " Restored.", 5);
Array.Resize(ref ImageList, ImageList.Length + 1);
ImageList[ImageList.Length - 1] = undelFile;
Array.Resize(ref ImageIdxList, ImageIdxList.Length + 1);
ImageIdxList[ImageIdxList.Length - 1] = ImageIdxList.Length - 1;
ImagesNotNull++;
return(true);
}
}
return(false);
}
public void RemoveFile(int level, ulong fileNumber)
{
List <FileMetadata> levelFiles = GetFiles(level);
FileMetadata file = levelFiles.FirstOrDefault(f => f.FileNumber == fileNumber);
if (file == null)
{
throw new Exception($"Expected to find file {fileNumber} in level {level}, but did not");
}
Levels[level].Remove(file);
file.Table?.Dispose();
if (!DeletedFiles.ContainsKey(level))
{
DeletedFiles[level] = new List <ulong>();
}
DeletedFiles[level].Add(file.FileNumber);
}
private void ProcessChanges()
{
var cancelToken = CancelSource.Token;
while (DeletedFiles.Any())
{
var deletedFile = DeletedFiles.First().Key;
_ = DeletedFiles.TryRemove(deletedFile, out var _);
if (CurrentState.VideoFiles.Remove(deletedFile))
{
CurrentState.SaveState();
OnLogged($"Removed file: {deletedFile.FilePath}");
}
else
{
OnLogged($"Deleted file not in VideoFile-List: " +
$"{deletedFile.FilePath}");
}
cancelToken.ThrowIfCancellationRequested();
}
operationStartTime = DateTime.Now;
OnOperationUpdate(OperationType.Comparing, 0, NewFiles.Count());
// We need to count for the ProgressUpdate since we shrink
// the Queue on every iteration.
var filesProcessed = 1;
while (NewFiles.Any())
{
var newFile = NewFiles.First().Key;
_ = NewFiles.TryRemove(newFile, out var _);
OnOperationUpdate(OperationType.Comparing, filesProcessed,
filesProcessed + NewFiles.Count());
filesProcessed++;
if (!newFile.WaitForFileAccess(cancelToken))
{
OnLogged($"Unable to access new file: {newFile.FileName}");
cancelToken.ThrowIfCancellationRequested();
continue;
}
cancelToken.ThrowIfCancellationRequested();
if (newFile.Duration == TimeSpan.Zero)
{
OnLogged($"New file has no duration: {newFile.FilePath}");
continue;
}
cancelToken.ThrowIfCancellationRequested();
if (!CurrentState.VideoFiles.Contains(newFile))
{
OnLogged($"New file added to VideoFile-List: {newFile.FilePath}");
CurrentState.VideoFiles.Add(newFile);
}
else
{
OnLogged($"New file already in VideoFile-List: {newFile.FilePath}");
continue;
}
cancelToken.ThrowIfCancellationRequested();
FindDuplicatesOf(CurrentState.VideoFiles, newFile, cancelToken);
CurrentState.SaveState();
// Cleanup in case of cancel
foreach (var file in CurrentState.VideoFiles)
{
file.DisposeImages();
}
cancelToken.ThrowIfCancellationRequested();
}
ProcessChangesIfAny();
}
public async Task <UpdateImpactResults> Analyze(
Stream appxBlockMap1,
Stream appxBlockMap2,
CancellationToken cancellationToken = default,
IProgress <ProgressData> progressData = default)
{
var progressReadFiles1 = new RangeProgress(progressData, 0, 30);
var progressReadFiles2 = new RangeProgress(progressData, 30, 60);
var progressCalculatingStatus = new RangeProgress(progressData, 60, 75);
var progressDuplicates = new RangeProgress(progressData, 75, 90);
var progressRemainingCalculation = new RangeProgress(progressData, 90, 100);
var filesInOldPackage = new SortedDictionary <string, AppxFile>();
var filesInNewPackage = new SortedDictionary <string, AppxFile>();
var blocksInPackage1 = new Dictionary <string, AppxBlock>();
var blocksInPackage2 = new Dictionary <string, AppxBlock>();
progressReadFiles1.Report(new ProgressData(0, "Reading the first package..."));
var files1 = await this.GetFiles(appxBlockMap1, cancellationToken, progressReadFiles1).ConfigureAwait(false);
foreach (var file1 in files1)
{
foreach (var block in file1.Blocks)
{
blocksInPackage1[block.Hash] = block;
}
filesInOldPackage.Add(file1.Name, file1);
}
progressReadFiles2.Report(new ProgressData(0, "Reading the second package..."));
var files2 = await this.GetFiles(appxBlockMap2, cancellationToken, progressReadFiles2).ConfigureAwait(false);
foreach (var file2 in files2)
{
foreach (var block in file2.Blocks)
{
blocksInPackage2[block.Hash] = block;
}
filesInNewPackage.Add(file2.Name, file2);
}
progressCalculatingStatus.Report(new ProgressData(0, "Calculating file status (1/2)..."));
foreach (var file in filesInOldPackage)
{
cancellationToken.ThrowIfCancellationRequested();
if (!filesInNewPackage.TryGetValue(file.Key, out var fileFromPackage2))
{
file.Value.Status = ComparisonStatus.Old;
file.Value.UpdateImpact = 0; // file removed = no update on impact
file.Value.SizeDifference = -file.Value.UncompressedSize;
continue;
}
if (file.Value.UncompressedSize == fileFromPackage2.UncompressedSize && file.Value.Blocks.Select(b => b.Hash).SequenceEqual(fileFromPackage2.Blocks.Select(b => b.Hash)))
{
file.Value.Status = ComparisonStatus.Unchanged;
file.Value.UpdateImpact = 0; // file unchanged = no update on impact
file.Value.SizeDifference = 0;
fileFromPackage2.Status = ComparisonStatus.Unchanged;
fileFromPackage2.UpdateImpact = 0; // file unchanged = no update on impact
fileFromPackage2.SizeDifference = 0;
}
else
{
file.Value.Status = ComparisonStatus.Changed;
file.Value.UpdateImpact = 0; // file changed = show no update impact. The impact should be shown in the other package.
file.Value.SizeDifference = file.Value.UncompressedSize - fileFromPackage2.UncompressedSize;
var blocksOnlyInPackage2 = fileFromPackage2.Blocks.Select(b => b.Hash).Except(file.Value.Blocks.Select(b => b.Hash));
fileFromPackage2.Status = ComparisonStatus.Changed;
fileFromPackage2.UpdateImpact = blocksOnlyInPackage2.Select(b => blocksInPackage2[b]).Sum(b => b.CompressedSize);
fileFromPackage2.SizeDifference = fileFromPackage2.UncompressedSize - file.Value.UncompressedSize;
}
}
foreach (var block1KeyValuePair in blocksInPackage1)
{
cancellationToken.ThrowIfCancellationRequested();
var block1 = block1KeyValuePair.Value;
if (!blocksInPackage2.TryGetValue(block1KeyValuePair.Key, out var block2))
{
block1.Status = ComparisonStatus.Old;
block1.UpdateImpact = 0; // block removed, no update impact
}
else
{
block1.Status = ComparisonStatus.Unchanged;
block1.UpdateImpact = 0;
block2.Status = ComparisonStatus.Unchanged;
block2.UpdateImpact = 0;
}
}
progressCalculatingStatus.Report(new ProgressData(50, "Calculating file status (2/2)..."));
foreach (var file in filesInNewPackage)
{
cancellationToken.ThrowIfCancellationRequested();
if (!filesInOldPackage.TryGetValue(file.Key, out _))
{
file.Value.Status = ComparisonStatus.New;
file.Value.UpdateImpact = file.Value.Blocks.Sum(b => b.CompressedSize); // sum of all blocks
file.Value.SizeDifference = file.Value.UncompressedSize;
}
}
foreach (var block2KeyValuePair in blocksInPackage2)
{
cancellationToken.ThrowIfCancellationRequested();
var block2 = block2KeyValuePair.Value;
if (!blocksInPackage1.ContainsKey(block2KeyValuePair.Key))
{
block2.Status = ComparisonStatus.New;
block2.UpdateImpact = block2.CompressedSize;
}
}
var duplicates1 = new Dictionary <string, IList <AppxFile> >();
var duplicates2 = new Dictionary <string, IList <AppxFile> >();
progressDuplicates.Report(new ProgressData(0, "Finding duplicates..."));
using (var md5 = MD5.Create())
{
foreach (var file in filesInOldPackage.Values)
{
cancellationToken.ThrowIfCancellationRequested();
var hash = string.Join(Environment.NewLine, file.Blocks.Select(b => b.Hash));
var allBlocksHash = System.Text.Encoding.ASCII.GetString(md5.ComputeHash(System.Text.Encoding.ASCII.GetBytes(hash)));
if (!duplicates1.TryGetValue(allBlocksHash, out var list))
{
list = new List <AppxFile>();
duplicates1[allBlocksHash] = list;
}
list.Add(file);
}
foreach (var file in filesInNewPackage.Values)
{
cancellationToken.ThrowIfCancellationRequested();
var hash = string.Join(System.Environment.NewLine, file.Blocks.Select(b => b.Hash));
var allBlocksHash = System.Text.Encoding.ASCII.GetString(md5.ComputeHash(System.Text.Encoding.ASCII.GetBytes(hash)));
if (!duplicates2.TryGetValue(allBlocksHash, out var list))
{
list = new List <AppxFile>();
duplicates2[allBlocksHash] = list;
}
list.Add(file);
}
}
var duplicatedFiles1 = new AppxDuplication
{
Duplicates = new List <ComparedDuplicate>(),
FileCount = duplicates1.Count(d => d.Value.Count > 1) * 2,
FileSize = duplicates1.Where(d => d.Value.Count > 1).Sum(d => d.Value[0].UncompressedSize)
};
var duplicatedFiles2 = new AppxDuplication
{
Duplicates = new List <ComparedDuplicate>(),
FileCount = duplicates2.Count(d => d.Value.Count > 1) * 2,
FileSize = duplicates2.Where(d => d.Value.Count > 1).Sum(d => d.Value[0].UncompressedSize)
};
progressDuplicates.Report(new ProgressData(55, "Analyzing duplication impact (1/3)..."));
foreach (var file in duplicates1.Where(d => d.Value.Count > 1))
{
cancellationToken.ThrowIfCancellationRequested();
var duplicate = new ComparedDuplicate
{
Files = new List <ComparedDuplicateFile>()
};
foreach (var df in file.Value)
{
var mdf = new ComparedDuplicateFile
{
Name = df.Name,
PossibleSizeReduction = df.UncompressedSize,
PossibleImpactReduction = df.Blocks.Sum(d => blocksInPackage1[d.Hash].CompressedSize)
};
duplicate.Files.Add(mdf);
}
duplicate.PossibleSizeReduction = duplicate.Files[0].PossibleSizeReduction * (duplicate.Files.Count - 1);
duplicate.PossibleImpactReduction = duplicate.Files[0].PossibleImpactReduction * (duplicate.Files.Count - 1);
duplicatedFiles1.Duplicates.Add(duplicate);
}
progressDuplicates.Report(new ProgressData(70, "Analyzing duplication impact (2/3)..."));
foreach (var file in duplicates2.Where(d => d.Value.Count > 1))
{
cancellationToken.ThrowIfCancellationRequested();
var duplicate = new ComparedDuplicate
{
Files = new List <ComparedDuplicateFile>()
};
foreach (var df in file.Value)
{
var mdf = new ComparedDuplicateFile
{
Name = df.Name,
PossibleSizeReduction = df.UncompressedSize,
PossibleImpactReduction = df.Blocks.Sum(d => blocksInPackage2[d.Hash].CompressedSize)
};
duplicate.Files.Add(mdf);
}
duplicate.PossibleSizeReduction = duplicate.Files[0].PossibleSizeReduction * (duplicate.Files.Count - 1);
duplicate.PossibleImpactReduction = duplicate.Files[0].PossibleImpactReduction * (duplicate.Files.Count - 1);
duplicatedFiles2.Duplicates.Add(duplicate);
}
progressDuplicates.Report(new ProgressData(85, "Analyzing duplication impact (3/3)..."));
duplicatedFiles1.PossibleImpactReduction = duplicatedFiles1.Duplicates.Sum(d => d.PossibleImpactReduction);
duplicatedFiles1.PossibleSizeReduction = duplicatedFiles1.Duplicates.Sum(d => d.PossibleSizeReduction);
duplicatedFiles2.PossibleImpactReduction = duplicatedFiles2.Duplicates.Sum(d => d.PossibleImpactReduction);
duplicatedFiles2.PossibleSizeReduction = duplicatedFiles2.Duplicates.Sum(d => d.PossibleSizeReduction);
cancellationToken.ThrowIfCancellationRequested();
progressRemainingCalculation.Report(new ProgressData(0, "Analyzing changed files..."));
var changedFiles = new ChangedFiles
{
// Shared parameters
UpdateImpact = filesInNewPackage.Values.Where(b => b.Status == ComparisonStatus.Changed).SelectMany(b => b.Blocks).Sum(b => b.UpdateImpact),
ActualUpdateImpact = blocksInPackage2.Values.Where(b => b.Status == ComparisonStatus.Changed).Sum(b => b.UpdateImpact),
SizeDifference = filesInNewPackage.Values.Where(b => b.Status == ComparisonStatus.Changed).Sum(b => b.SizeDifference),
FileCount = filesInOldPackage.Values.Count(b => b.Status == ComparisonStatus.Changed),
// Old package
OldPackageFiles = filesInOldPackage.Values.Where(f => f.Status == ComparisonStatus.Changed).ToList(),
OldPackageFileSize = filesInOldPackage.Values.Where(b => b.Status == ComparisonStatus.Changed).Sum(b => b.UncompressedSize),
OldPackageBlockCount = filesInOldPackage.Values.Where(b => b.Status == ComparisonStatus.Changed).Sum(b => b.Blocks.Count),
OldPackageBlockSize = filesInOldPackage.Values.Where(b => b.Status == ComparisonStatus.Changed).SelectMany(b => b.Blocks).Sum(b => b.CompressedSize),
// New package
NewPackageFiles = filesInNewPackage.Values.Where(f => f.Status == ComparisonStatus.Changed).ToList(),
NewPackageFileSize = filesInNewPackage.Values.Where(b => b.Status == ComparisonStatus.Changed).Sum(b => b.UncompressedSize),
NewPackageBlockCount = filesInNewPackage.Values.Where(b => b.Status == ComparisonStatus.Changed).Sum(b => b.Blocks.Count),
NewPackageBlockSize = filesInNewPackage.Values.Where(b => b.Status == ComparisonStatus.Changed).SelectMany(b => b.Blocks).Sum(b => b.CompressedSize)
};
cancellationToken.ThrowIfCancellationRequested();
progressRemainingCalculation.Report(new ProgressData(25, "Analyzing added files..."));
var addedFiles = new AddedFiles
{
UpdateImpact = filesInNewPackage.Values.Where(b => b.Status == ComparisonStatus.New).SelectMany(b => b.Blocks).Sum(b => b.UpdateImpact),
ActualUpdateImpact = blocksInPackage2.Values.Where(b => b.Status == ComparisonStatus.New).Sum(b => b.UpdateImpact),
BlockCount = blocksInPackage2.Values.Count(b => b.Status == ComparisonStatus.New),
BlockSize = blocksInPackage2.Values.Where(b => b.Status == ComparisonStatus.New).Sum(b => b.CompressedSize),
FileCount = filesInNewPackage.Values.Count(s => s.Status == ComparisonStatus.New),
SizeDifference = filesInNewPackage.Values.Where(s => s.Status == ComparisonStatus.New).Sum(f => f.SizeDifference),
FileSize = filesInNewPackage.Values.Where(s => s.Status == ComparisonStatus.New).Sum(f => f.UncompressedSize),
Files = filesInNewPackage.Values.Where(f => f.Status == ComparisonStatus.New).ToList()
};
cancellationToken.ThrowIfCancellationRequested();
progressRemainingCalculation.Report(new ProgressData(50, "Analyzing deleted files..."));
var deletedFiles = new DeletedFiles
{
FileSize = filesInOldPackage.Values.Where(s => s.Status == ComparisonStatus.Old).Sum(f => f.UncompressedSize),
SizeDifference = filesInOldPackage.Values.Where(s => s.Status == ComparisonStatus.Old).Sum(f => f.SizeDifference),
FileCount = filesInOldPackage.Values.Count(s => s.Status == ComparisonStatus.Old),
BlockSize = blocksInPackage1.Values.Where(b => b.Status == ComparisonStatus.Old).Sum(b => b.CompressedSize),
BlockCount = blocksInPackage1.Values.Count(b => b.Status == ComparisonStatus.Old),
ActualUpdateImpact = blocksInPackage1.Values.Where(s => s.Status == ComparisonStatus.Old).Sum(f => f.UpdateImpact),
Files = filesInOldPackage.Values.Where(f => f.Status == ComparisonStatus.Old).ToList()
};
deletedFiles.UpdateImpact = deletedFiles.ActualUpdateImpact;
cancellationToken.ThrowIfCancellationRequested();
progressRemainingCalculation.Report(new ProgressData(75, "Analyzing unchanged files..."));
var unchangedFiles = new UnchangedFiles
{
FileCount = filesInNewPackage.Values.Count(b => b.Status == ComparisonStatus.Unchanged),
BlockCount = blocksInPackage2.Values.Count(b => b.Status == ComparisonStatus.Unchanged),
BlockSize = blocksInPackage2.Values.Where(b => b.Status == ComparisonStatus.Unchanged).Sum(b => b.CompressedSize),
FileSize = filesInNewPackage.Values.Where(b => b.Status == ComparisonStatus.Unchanged).Sum(b => b.UncompressedSize),
Files = filesInOldPackage.Values.Where(f => f.Status == ComparisonStatus.Unchanged).ToList()
};
cancellationToken.ThrowIfCancellationRequested();
progressRemainingCalculation.Report(new ProgressData(90, "Please wait..."));
var comparisonResult = new UpdateImpactResults
{
OldPackageLayout = new AppxLayout
{
FileSize = filesInOldPackage.Sum(f => f.Value.UncompressedSize),
FileCount = filesInOldPackage.Count,
BlockSize = blocksInPackage1.Sum(f => f.Value.CompressedSize),
BlockCount = blocksInPackage1.Count,
Layout = new PackageLayout
{
Blocks = this.GetChartForBlocks(filesInOldPackage),
Files = this.GetChartForFiles(filesInOldPackage)
}
},
NewPackageLayout = new AppxLayout
{
FileSize = filesInNewPackage.Sum(f => f.Value.UncompressedSize),
FileCount = filesInNewPackage.Count,
BlockSize = blocksInPackage2.Sum(f => f.Value.CompressedSize),
BlockCount = blocksInPackage2.Count,
Layout = new PackageLayout
{
Blocks = this.GetChartForBlocks(filesInNewPackage),
Files = this.GetChartForFiles(filesInNewPackage)
}
},
OldPackageDuplication = duplicatedFiles1,
NewPackageDuplication = duplicatedFiles2,
UpdateImpact =
blocksInPackage2.Where(b => b.Value.Status == ComparisonStatus.New).Sum(b => b.Value.UpdateImpact) +
blocksInPackage1.Sum(b => b.Value.UpdateImpact),
ChangedFiles = changedFiles,
DeletedFiles = deletedFiles,
AddedFiles = addedFiles,
UnchangedFiles = unchangedFiles
};
comparisonResult.OldPackageLayout.Size = comparisonResult.OldPackageLayout.BlockSize + filesInOldPackage.Values.Sum(f => f.HeaderSize);
comparisonResult.NewPackageLayout.Size = comparisonResult.NewPackageLayout.BlockSize + filesInNewPackage.Values.Sum(f => f.HeaderSize);
comparisonResult.SizeDifference = comparisonResult.AddedFiles.SizeDifference + comparisonResult.ChangedFiles.SizeDifference + comparisonResult.DeletedFiles.SizeDifference;
comparisonResult.ActualUpdateImpact = comparisonResult.UpdateImpact;
return(comparisonResult);
}
