// copies file entries of unmodified folders from existing EFU to new EFU
void EFUReindexUnchangedEntries(StreamWriter newEFU)
{
long lines = 0;
int lastpc = -1;
using (StreamReader sr = new StreamReader(EFUpath, Encoding.UTF8, false, 1 << 20))
{
sr.ReadLine(); // header
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
++lines;
int pc = (int)(100 * sr.BaseStream.Position / sr.BaseStream.Length);
if (Program.ShowProgress && (lines % 10000 == 0 || pc != lastpc))
{
Console.Write($" Done {pc}%\r");
}
lastpc = pc;
DirEntry file = ParseEntry(line, false);
if (file.isFolder)
{
continue;
}
else
{
if (isIncluded(file.Path, false)) // process included/excluded files
{
var folder = Path.GetDirectoryName(file.Path);
if (dirIndex.TryGetValue(folder.ToLower(), out var subdir) && subdir.Verified)
{
if (!subdir.Exists)
{
continue;
}
if (!subdir.Changed)
{
fileCount++;
totalSize += file.Size;
subdir.Size += file.Size;
newEFU.WriteLine(line);
}
}
}
else
{
exFileCount++;
exSize += file.Size;
}
}
}
}
if (Program.ShowProgress)
{
Console.Write(" \r");
}
}
// checks each known folder timestamp to determine if it has changed
void EFUFindChangedFolders()
{
int curr = 0;
int count = dirIndex.Count;
int changed = 0;
int missing = 0;
DirEntry prev = null;
int lastpc = -1;
foreach (var entry in dirIndex.Values)
{
int pc = ++curr * 100 / count;
if (Program.ShowProgress && (pc != lastpc || curr % 100 == 0))
{
string log = $" {curr}/{count} ({pc}%) {entry.Path}";
if (log.Length > Console.BufferWidth - 1)
{
log = log.Substring(0, Console.BufferWidth - 4) + "...";
}
Console.Write($"{log.PadRight(Console.BufferWidth - 1)}\r");
lastpc = pc;
}
entry.Verified = true;
if (prev != null && !prev.Exists && Path.GetDirectoryName(entry.Path) == prev.Path) // if parent doesn't exist, this one doesn't either
{
missing++;
continue;
}
prev = entry;
var di = new DirectoryInfo(entry.Path);
if (di.Exists)
{
entry.Exists = true;
if (di.LastWriteTime != entry.Modified || di.Attributes != entry.Attributes || di.Attributes.HasFlag(FileAttributes.ReparsePoint) || entry.Parent == null)
{
entry.Modified = di.LastWriteTime;
entry.Created = di.CreationTime;
entry.Attributes = di.Attributes;
entry.Size = 0;
entry.Changed = true;
changed++;
}
}
else
{
missing++;
}
}
if (Program.ShowProgress)
{
Console.Write($"\r{new string(' ', Console.BufferWidth - 1)}\r");
}
Console.WriteLine($" {count} folders checked, {changed} changed and {missing} missing");
}
public void Add(DirEntry entry)
{
if (Contents == null)
{
Contents = new List <DirEntry>();
}
entry.Parent = this;
Contents.Add(entry);
}
public DirEntry(DirEntry parent, string path, long size, DateTime modified, DateTime created, FileAttributes attributes)
{
Parent = parent;
Path = path;
Size = size;
Modified = modified;
Created = created;
Attributes = attributes;
}
public DirEntry(DirEntry parent, FileData dir)
{
Parent = parent;
Path = dir.FullPath;
Size = dir.Size;
Modified = dir.LastWriteTime;
Created = dir.CreationTime;
Attributes = dir.Attributes;
Exists = true;
Verified = true;
}
public DirEntry(DirEntry parent, DirectoryInfo dir)
{
Parent = parent;
Path = dir.FullName;
Size = 0;
Modified = dir.LastWriteTime;
Created = dir.CreationTime;
Attributes = dir.Attributes;
Exists = dir.Exists;
Verified = true;
}
// recursively update total folder size
// before calling this, each folder size is assumed to be just the sum of sizes of the files on that folder (not including subfolders)
long getSubSize(DirEntry dir)
{
if (dir.Contents != null)
{
foreach (var sub in dir.Contents)
{
if (sub.Exists)
{
dir.Size += getSubSize(sub);
}
}
}
return(dir.Size);
}
// get existing DirEntry or create new one
DirEntry getDirEntry(string path)
{
string key = path?.ToLower();
if (dirIndex.TryGetValue(key, out DirEntry entry))
{
return(entry);
}
string root = Path.GetPathRoot(path).TrimEnd('\\');
string dir = Path.GetDirectoryName(path)?.TrimEnd('\\') ?? "";
//string name = path == root ? root : Path.GetFileName(path);
DirEntry de = new DirEntry(getDirEntry(dir), dir, 0, DateTime.MinValue, DateTime.MinValue, 0);
dirIndex[key] = de;
return(de);
}
bool CheckRoots()
{
for (int i = 0; i < roots.Count; i++)
{
string path = Path.GetFullPath(roots[i]); // warning: case is not corrected
DirectoryInfo di = new DirectoryInfo(path);
if (!di.Exists)
{
Console.WriteLine($"Folder not found: {path}");
return(false);
}
DirEntry entry = new DirEntry(null, di);
entry.Verified = true;
entry.Changed = true;
entry.Exists = di.Exists;
dirIndex[path.ToLower()] = entry;
}
return(true);
}
void EFUScanFolderRecursive(StreamWriter newEFU, DirEntry dir)
{
if (Program.ShowProgress && dirnum++ % 100 == 0)
{
string log = $" Scanning: {dir.Path}";
if (log.Length > Console.BufferWidth - 1)
{
log = log.Substring(0, Console.BufferWidth - 4) + "...";
}
Console.Write($"{log.PadRight(Console.BufferWidth - 1)}\r");
}
int depth = dir.Path.Length < 3 ? 0 : dir.Path.Length - dir.Path.Substring(2).Replace("\\", "").Length;
if (depth > maxDepth)
{
maxDepth = depth;
}
FileData[] contents = FastDirectory.GetFiles(dir.Path, "*", SearchOption.TopDirectoryOnly);
var subs = contents.Where(e => e.Attributes.HasFlag(FileAttributes.Directory)).ToArray();
var files = contents.Where(e => !e.Attributes.HasFlag(FileAttributes.Directory)).ToArray();
if (Program.depthLimit > 0 && depth >= Program.depthLimit)
{
subs = new FileData[] { }
}
;
foreach (var f in files)
{
if (isIncluded(f.FullPath, false))
{
newEFU.WriteLine($"\"{f.FullPath}\",{f.Size},{f.LastWriteTime.ToFileTime()},{f.CreationTime.ToFileTime()},{(int)f.Attributes}");
dir.Size += f.Size;
totalSize += f.Size;
fileCount++;
}
else
{
exFileCount++;
exSize += f.Size;
}
}
foreach (var s in subs)
{
if (dirIndex.ContainsKey(s.FullPath.ToLower())) // skip folders already indexed
{
continue;
}
if (isIncluded(s.FullPath, true))
{
DirEntry sub = new DirEntry(dir, s)
{
Changed = true
};
dir.Add(sub);
dirIndex[s.FullPath.ToLower()] = sub;
EFUScanFolderRecursive(newEFU, sub);
//dir.Size += sub.Size;
}
else
{
exDirCount++;
}
}
}
// recursively update total folder size
// before calling this, each folder size is assumed to be just the sum of sizes of the files on that folder (not including subfolders)
long getSubSize(DirEntry dir)
{
if (dir.Contents != null)
{
foreach (var sub in dir.Contents)
{
if (sub.Exists)
{
dir.Size += getSubSize(sub);
}
}
}
return(dir.Size);
}
// writes folder info to EFU file
void EFUFinalize(StreamWriter newEFU)
{
foreach (var dir in dirIndex.Values)
{
if (dir.Exists)
{
dirCount++;
int depth = dir.Path.Length < 3 ? 0 : dir.Path.Length - dir.Path.Substring(2).Replace("\\", "").Length;
if (depth > maxDepth)
{
maxDepth = depth;
}
newEFU.WriteLine($"\"{dir.Path}\",{dir.Size},{dir.Modified.ToFileTime()},{dir.Created.ToFileTime()},{(int)dir.Attributes}");
}
}
}
// copies file entries of unmodified folders from existing EFU to new EFU
void EFUReindexUnchangedEntries(StreamWriter newEFU)
{
long lines = 0;
int lastpc = -1;
using (StreamReader sr = new StreamReader(EFUpath, Encoding.UTF8, false, 1 << 20))
{
sr.ReadLine(); // header
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
++lines;
int pc = (int)(100 * sr.BaseStream.Position / sr.BaseStream.Length);
if (Program.ShowProgress && (lines % 10000 == 0 || pc != lastpc))
{
Console.Write($" Done {pc}%\r");
}
lastpc = pc;
DirEntry file = ParseEntry(line, false);
if (file.isFolder)
{
continue;
}
else
{
if (isIncluded(file.Path, false)) // process included/excluded files
{
var folder = Path.GetDirectoryName(file.Path);
if (dirIndex.TryGetValue(folder.ToLower(), out var subdir) && subdir.Verified)
{
if (!subdir.Exists)
{
continue;
}
if (!subdir.Changed)
{
fileCount++;
totalSize += file.Size;
subdir.Size += file.Size;
newEFU.WriteLine(line);
}
}
}
else
{
exFileCount++;
exSize += file.Size;
}
}
}
}
if (Program.ShowProgress)
{
Console.Write(" \r");
}
}
// loads an EFU file, keeping only directory info and stats (files are ignored)
void EFULoad(bool Checked = false)
{
long lines = 0;
int dirs = 0;
int files = 0;
long size = 0;
int depth = 0;
int lastpc = -1;
using (StreamReader sr = new StreamReader(EFUpath, Encoding.UTF8, false, 1 << 20))
{
sr.ReadLine(); // header
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
++lines;
int pc = (int)(100 * sr.BaseStream.Position / sr.BaseStream.Length);
if (Program.ShowProgress && (lines % 10000 == 0 || pc != lastpc))
{
Console.Write($" Loaded {pc}%\r");
}
lastpc = pc;
DirEntry dir = ParseEntry(line, true); // dirs only
if (dir != null)
{
int fd = dir.Path.Length - dir.Path.Substring(1).Replace("\\", "").Length;
if (fd > depth)
{
depth = fd;
}
dirs++;
if (isIncluded(dir.Path, true)) // filter
{
dirIndex[dir.Path.ToLower()] = dir;
DirEntry parDir = null;
var parent = Path.GetDirectoryName(dir.Path);
if (parent != null && dirIndex.TryGetValue(parent.ToLower(), out parDir))
{
parDir.Add(dir);
}
if (parDir == null)
{
size += dir.Size; // root folder size
}
dir.Size = 0; // will be recalculated
}
else
{
exDirCount++; // size is added during elsewhere
}
}
else
{
files++;
}
}
}
//if (Program.ShowProgress) Console.Write(" \r");
Console.WriteLine($" Contents: {Util.FormatSize(size)} in {files:n0} files, {dirs:n0} folders [{depth} depth]");
}
// copies entries that pass exclude filter from existing EFU to new EFU
void EFUFilter(StreamWriter newEFU)
{
long lines = 0;
int lastpc = -1;
using (StreamReader sr = new StreamReader(EFUpath, Encoding.UTF8, false, 1 << 20))
{
sr.ReadLine(); // header
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
++lines;
int pc = (int)(100 * sr.BaseStream.Position / sr.BaseStream.Length);
if (Program.ShowProgress && (lines % 10000 == 0 || pc != lastpc))
{
Console.Write($" Done {pc}%\r");
}
lastpc = pc;
DirEntry file = ParseEntry(line, false);
if (file.isFolder)
{
continue;
}
else
{
var folder = Path.GetDirectoryName(file.Path);
if (folder != null && dirIndex.TryGetValue(folder.ToLower(), out var subdir))
{
if (isIncluded(file.Path, file.isFolder))
{
fileCount++;
totalSize += file.Size;
subdir.Size += file.Size;
newEFU.WriteLine(line);
continue;
}
}
exFileCount++;
exSize += file.Size;
}
}
}
if (Program.ShowProgress)
{
Console.Write(" \r");
}
}
// loads an EFU file, keeping only directory info and stats (files are ignored)
void getStats()
{
long lines = 0;
int lastpc = -1;
using (StreamReader sr = new StreamReader(EFUpath, Encoding.UTF8, false, 1 << 20))
{
sr.ReadLine(); // header
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
++lines;
int pc = (int)(100 * sr.BaseStream.Position / sr.BaseStream.Length);
if (Program.ShowProgress && (lines % 10000 == 0 || pc != lastpc))
{
Console.Write($" Loaded {pc}%\r");
}
lastpc = pc;
DirEntry dir = ParseEntry(line, false); // dirs only
if (isIncluded(dir.Path, dir.isFolder))
{
if (dir.isFolder)
{
dirCount++;
int depth = dir.Path.Length < 3 ? 0 : dir.Path.Length - dir.Path.Substring(2).Replace("\\", "").Length;
if (depth > maxDepth)
{
maxDepth = depth;
}
}
else
{
fileCount++;
totalSize += dir.Size;
string ext = (Path.GetExtension(dir.Path) ?? "").ToLower();
if (extensionCount.TryGetValue(ext, out var count))
{
extensionCount[ext] = new Tuple <int, long>(count.Item1 + 1, count.Item2 + dir.Size);
}
else
{
extensionCount[ext] = new Tuple <int, long>(1, dir.Size);
}
}
}
else
{
if (dir.isFolder)
{
exDirCount++;
}
else
{
exFileCount++;
exSize += dir.Size;
}
}
}
}
if (Program.ShowProgress)
{
Console.Write(" \r");
}
}
// uses string.split (2x faster than regex for this task)
DirEntry ParseEntry(string line, bool dirsOnly = false)
{
if (dirsOnly && (line.EndsWith(",0") || line.EndsWith(",32")))
{
return(null);
}
var items = line.Split(',');
if (items.Length >= 5)
{
FileAttributes attr = (FileAttributes)int.Parse(items[items.Length - 1]);
if (dirsOnly && !attr.HasFlag(FileAttributes.Directory))
{
return(null);
}
string path = items.Length == 5 ? items[0].Trim('\"') : string.Join(",", items.Take(items.Length - 4).ToList()).Trim('\"');
long size = long.Parse(items[items.Length - 4]);
DateTime mdate = DateTime.FromFileTime(long.Parse(items[items.Length - 3]));
DateTime cdate = DateTime.FromFileTime(long.Parse(items[items.Length - 2]));
return(new DirEntry(null, path, size, mdate, cdate, attr));
}
return(null);
}
// get existing DirEntry or create new one
DirEntry getDirEntry(string path)
{
string key = path?.ToLower();
if (dirIndex.TryGetValue(key, out DirEntry entry))
{
return(entry);
}
string root = Path.GetPathRoot(path).TrimEnd('\\');
string dir = Path.GetDirectoryName(path)?.TrimEnd('\\') ?? "";
//string name = path == root ? root : Path.GetFileName(path);
DirEntry de = new DirEntry(getDirEntry(dir), dir, 0, DateTime.MinValue, DateTime.MinValue, 0);
dirIndex[key] = de;
return(de);
}
}
// loads an EFU file, keeping only directory info and stats (files are ignored)
void getStats()
{
long lines = 0;
int lastpc = -1;
using (StreamReader sr = new StreamReader(EFUpath, Encoding.UTF8, false, 1 << 20))
{
sr.ReadLine(); // header
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
++lines;
int pc = (int)(100 * sr.BaseStream.Position / sr.BaseStream.Length);
if (Program.ShowProgress && (lines % 10000 == 0 || pc != lastpc))
{
Console.Write($" Loaded {pc}%\r");
}
lastpc = pc;
DirEntry dir = ParseEntry(line, false); // dirs only
if (isIncluded(dir.Path, dir.isFolder))
{
if (dir.isFolder)
{
dirCount++;
int depth = dir.Path.Length < 3 ? 0 : dir.Path.Length - dir.Path.Substring(2).Replace("\\", "").Length;
if (depth > maxDepth)
{
maxDepth = depth;
}
}
else
{
fileCount++;
totalSize += dir.Size;
string ext = (Path.GetExtension(dir.Path) ?? "").ToLower();
if (extensionCount.TryGetValue(ext, out var count))
{
extensionCount[ext] = new Tuple <int, long>(count.Item1 + 1, count.Item2 + dir.Size);
}
else
{
extensionCount[ext] = new Tuple <int, long>(1, dir.Size);
}
}
}
else
{
if (dir.isFolder)
{
exDirCount++;
}
else
{
exFileCount++;
exSize += dir.Size;
}
}
}
}
if (Program.ShowProgress)
{
Console.Write(" \r");
}
}
// loads an EFU file, keeping only directory info and stats (files are ignored)
void EFULoad(bool Checked = false)
{
long lines = 0;
int dirs = 0;
int files = 0;
long size = 0;
int depth = 0;
int lastpc = -1;
using (StreamReader sr = new StreamReader(EFUpath, Encoding.UTF8, false, 1 << 20))
{
sr.ReadLine(); // header
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
++lines;
int pc = (int)(100 * sr.BaseStream.Position / sr.BaseStream.Length);
if (Program.ShowProgress && (lines % 10000 == 0 || pc != lastpc))
{
Console.Write($" Loaded {pc}%\r");
}
lastpc = pc;
DirEntry dir = ParseEntry(line, true); // dirs only
if (dir != null)
{
int fd = dir.Path.Length - dir.Path.Substring(1).Replace("\\", "").Length;
if (fd > depth)
{
depth = fd;
}
dirs++;
if (isIncluded(dir.Path, true)) // filter
{
dirIndex[dir.Path.ToLower()] = dir;
DirEntry parDir = null;
var parent = Path.GetDirectoryName(dir.Path);
if (parent != null && dirIndex.TryGetValue(parent.ToLower(), out parDir))
{
parDir.Add(dir);
}
if (parDir == null)
{
size += dir.Size; // root folder size
}
dir.Size = 0; // will be recalculated
}
else
{
exDirCount++; // size is added during elsewhere
}
}
else
{
files++;
}
}
}
//if (Program.ShowProgress) Console.Write(" \r");
Console.WriteLine($" Contents: {Util.FormatSize(size)} in {files:n0} files, {dirs:n0} folders [{depth} depth]");
}
void EFUScanFolderRecursive(StreamWriter newEFU, DirEntry dir)
{
if (Program.ShowProgress && dirnum++ % 100 == 0)
{
string log = $" Scanning: {dir.Path}";
if (log.Length > Console.BufferWidth - 1)
{
log = log.Substring(0, Console.BufferWidth - 4) + "...";
}
Console.Write($"{log.PadRight(Console.BufferWidth - 1)}\r");
}
int depth = dir.Path.Length < 3 ? 0 : dir.Path.Length - dir.Path.Substring(2).Replace("\\", "").Length;
if (depth > maxDepth)
{
maxDepth = depth;
}
FileData[] contents = FastDirectory.GetFiles(dir.Path, "*.*", SearchOption.TopDirectoryOnly);
var subs = contents.Where(e => e.Attributes.HasFlag(FileAttributes.Directory)).ToArray();
var files = contents.Where(e => !e.Attributes.HasFlag(FileAttributes.Directory)).ToArray();
foreach (var f in files)
{
if (isIncluded(f.FullPath, false))
{
newEFU.WriteLine($"\"{f.FullPath}\",{f.Size},{f.LastWriteTime.ToFileTime()},{f.CreationTime.ToFileTime()},{(int)f.Attributes}");
dir.Size += f.Size;
totalSize += f.Size;
fileCount++;
}
else
{
exFileCount++;
exSize += f.Size;
}
}
foreach (var s in subs)
{
if (dirIndex.ContainsKey(s.FullPath.ToLower())) // skip folders already indexed
{
continue;
}
if (isIncluded(s.FullPath, true))
{
DirEntry sub = new DirEntry(dir, s)
{
Changed = true
};
dir.Add(sub);
dirIndex[s.FullPath.ToLower()] = sub;
EFUScanFolderRecursive(newEFU, sub);
//dir.Size += sub.Size;
}
else
{
exDirCount++;
}
}
}
