private static FsNode <T> GetDirectory <T>(string path, Dictionary <string, FsNode <T> > dict, out string filename)
{
var lastSlash = path.LastIndexOf('\\');
if (lastSlash == -1)
{
lastSlash = 0;
}
var directoryPath = path.Substring(0, lastSlash);
filename = lastSlash != 0 ? path.Substring(lastSlash + 1) : path;
if (!dict.TryGetValue(directoryPath, out var directory))
{
string currname;
var parent = GetDirectory(directoryPath, dict, out currname);
if (parent.Children == null)
{
parent.Children = new List <FsNode <T> >();
}
directory = new FsNode <T>();
directory.Name = currname;
if (directory.Name.Length == 2 && directory.Name[1] == ':')
{
directory.Name = currname[0].ToString();
}
parent.Children.Add(directory);
dict[directoryPath] = directory;
}
return(directory);
}
bool isDirectory(FsNode <FileInfo> info)
{
if (info.Info.Exists || File.Exists(info.FullName))
{
if (info.Name.ToLower().EndsWith(".rar"))
{
return(true);
}
if (info.Name.ToLower().EndsWith(".zip"))
{
return(true);
}
if (info.Name.ToLower().EndsWith(".7z"))
{
return(true);
}
if (info.Name.ToLower().EndsWith(".iso"))
{
return(true);
}
}
if (info.Tag is FsNode <RarArchiveEntry> )
{
//var children = info.GetChildrenDelegate();
if (((FsNode <RarArchiveEntry>)info.Tag).Info != null && ((FsNode <RarArchiveEntry>)info.Tag).Info.IsDirectory)
{
return(true);
}
return(false);
}
return((info.Info.Attributes & FileAttributes.Directory) > 0);
}
private FileInformation GetFileInformation(FsNode <ArchiveFileInfo> item)
{
return(new FileInformation()
{
Attributes = item == root ? FileAttributes.Directory : (FileAttributes)item.Info.Attributes,
CreationTime = item.Info.CreationTime,
FileName = item.Name,
LastAccessTime = item.Info.LastAccessTime,
LastWriteTime = item.Info.LastWriteTime,
Length = (long)item.Info.Size
});
}
private FileInformation GetFileInformation(FsNode <RarArchiveEntry> item)
{
return(new FileInformation()
{
Attributes = item == root ? FileAttributes.Directory : FileAttributes.ReadOnly,
CreationTime = item.Info.CreatedTime,
FileName = item.Name,
LastAccessTime = item.Info.LastAccessedTime,
LastWriteTime = item.Info.LastModifiedTime,
Length = (long)item.Info.Size
});
}
protected static FsNode <T> CreateTree <T>(IEnumerable <T> allfiles, Func <T, string> getPath, Func <T, bool> isDirectory = null, Action <FsNode <T> > postprocess = null)
{
var directories = new Dictionary <string, FsNode <T> >(StringComparer.OrdinalIgnoreCase);
var root = new FsNode <T>();
root.Name = "(Root)";
directories[string.Empty] = root;
foreach (var file in allfiles)
{
string name;
var path = getPath(file);
var directory = GetDirectory(path, directories, out name);
if (directory.Children == null)
{
directory.Children = new List <FsNode <T> >();
}
FsNode <T> f;
if (isDirectory != null && isDirectory(file))
{
if (directories.TryGetValue(path, out var ff))
{
f = ff;
}
else
{
f = new FsNode <T>();
directories[path] = f;
directory.Children.Add(f);
}
}
else
{
f = new FsNode <T>();
directory.Children.Add(f);
}
f.Name = name;
f.Info = file;
f.FullName = path;
if (postprocess != null)
{
postprocess(f);
}
}
return(root);
}
void CheckDirectorys(FsNode <ArchiveFileInfo> myroot)
{
foreach (var item in myroot.Children)
{
if (item.Children != null && item.Children.Count > 0)
{
var info = ((ArchiveFileInfo)item.Info);
info.Attributes = (int)FileAttributes.Directory;
item.Info = info;
CheckDirectorys(item);
}
}
}
public SharpCompressFs(string path)
{
lock (this)
{
zipfile = path;
root = CreateTree <RarArchiveEntry>(extractor.Entries, x => x.Key, x => x.IsDirectory);
CheckDirectorys(root);
extractor.Dispose();
extractor = null;
}
}
public SevenZipFs(string path)
{
zipfile = path;
extractor = new SevenZipExtractor(path);
root = CreateTree <ArchiveFileInfo>(extractor.ArchiveFileData, x => x.FileName, x => IsDirectory(x.Attributes));
cache = new MemoryStreamCache <FsNode <ArchiveFileInfo> >((item, stream) =>
{
lock (readerLock)
{
extractor.ExtractFile(item.Info.Index, stream);
}
});
}
public MyMirror(string path)
{
this.path = path.TrimEnd('\\');
root = GetFileInfo(this.path);
FileSystemWatcher fsw = new FileSystemWatcher(this.path, "*");
fsw.IncludeSubdirectories = true;
//fsw.Changed += Fsw_Changed;
fsw.Created += Fsw_Changed;
fsw.Renamed += Fsw_Changed;
fsw.Deleted += Fsw_Changed;
fsw.EnableRaisingEvents = true;
new Thread(() =>
{
while (true)
{
Thread.Sleep(30000);
Console.WriteLine("******* " + (GC.GetTotalMemory(false) / 1024.0 / 1024.0) + " - " +
(Process.GetCurrentProcess().WorkingSet64 / 1024.0 / 1024.0));
var dogc = false;
//Status();
foreach (var item in cache.ToArray())
{
logger.Debug("cache: {0} name: {1} lastaccess: {2}", item.Key, item.Value.SimpleMountName,
item.Value.LastAccessed);
if (item.Value.LastAccessed.AddMinutes(10) < DateTime.Now)
{
item.Value.extractor = null;
SharpCompressFs temp;
cache.TryRemove(item.Key, out temp);
dogc = true;
}
}
if (dogc)
{
GC.Collect();
}
}
})
{
IsBackground = true
}.Start();
}
FileInformation GetFileInformation(FsNode <FileInfo> item)
{
logger.Debug(Thread.CurrentThread.ManagedThreadId + " GetFileInformation<FsNode>: {0} {1}", item.FullName, item.Info.Attributes);
if (item == null)
{
return(new FileInformation());
}
var isdir = isDirectory(item);
long length = isdir ? 0 : item.Info.Exists ? item.Info.Length : 0;
FileAttributes attrib = item.Info.Attributes;
var archive = item.Tag as FsNode <RarArchiveEntry>;
if (archive != null && archive.Info != null)
{
length = (long)archive.Info.Size;
attrib = FileAttributes.Normal;
return(new FileInformation()
{
FileName = item.Info.Name,
Length = length,
Attributes = isdir ? (FileAttributes.Directory) : attrib,
CreationTime = archive.Info.CreatedTime,
LastAccessTime = archive.Info.LastAccessedTime,
LastWriteTime = archive.Info.LastModifiedTime,
});
}
var lastwrite = DateTime.MinValue;
try
{
lastwrite = item.Info.LastWriteTime;
}
catch { }
return(new FileInformation()
{
FileName = item.Info.Name,
Length = length,
Attributes = isdir ? (FileAttributes.Directory) : attrib,
CreationTime = item.Info.CreationTime,
LastAccessTime = item.Info.LastAccessTime,
LastWriteTime = lastwrite,
});
}
public SevenZipFs(string path)
{
zipfile = path;
extractor = new SevenZipExtractor(path);
root = CreateTree <ArchiveFileInfo>(extractor.ArchiveFileData, x => x.FileName, x => IsDirectory(x.Attributes));
CheckDirectorys(root);
extractor.Dispose();
extractor = null;
cache = new MemoryStreamCache <FsNode <ArchiveFileInfo> >((item, stream) =>
{
var th = new Thread(action =>
{
lock (readerLock)
{
try
{
extractor = new SevenZipExtractor(path);
extractor.Extracting += Extractor_Extracting;
extractor.FileExtractionStarted += Extractor_FileExtractionStarted;
extractor.ExtractFile(item.Info.Index, stream);
extractor.Dispose();
extractor = null;
}
catch (Exception ex)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(ex);
Console.ForegroundColor = ConsoleColor.White;
}
}
});
th.Start();
th.Join();
});
}
protected static FsNode <T> GetNode <T>(FsNode <T> root, string path)
{
var components = path.SplitFast('\\', StringSplitOptions.RemoveEmptyEntries);
var current = root;
foreach (var item in components)
{
if (current.Children == null)
{
return(null);
}
current = current.Children.FirstOrDefault(x => x.Name.Equals(item, StringComparison.OrdinalIgnoreCase));
if (current == null)
{
return(null);
}
}
return(current);
}
private FileInformation GetFileInformation(FsNode <WarcItem> item, bool precise)
{
var date = item.Info?.LastModified ?? item.Info?.Date;
long len = 0;
if (item.Info != null)
{
var l = cache.TryGetLength(item);
if (l != null)
{
len = l.Value;
}
else
{
if (item.Tag == TagVirtual)
{
if (precise)
{
len = cache.GetLength(item);
}
else
{
if (item.Info.PayloadLength != -1)
{
len = item.Info.PayloadLength;
}
else
{
len = 9999;
}
}
}
else
{
if (item.Info.PayloadLength != -1)
{
len = item.Info.PayloadLength;
}
else
{
if (precise)
{
len = cache.GetLength(item);
}
else
{
len = 9999;
}
}
}
}
}
return(new FileInformation()
{
Attributes = item.Info != null ? FileAttributes.Archive : FileAttributes.Directory,
CreationTime = date,
FileName = item.Name,
LastAccessTime = date,
LastWriteTime = date,
Length = len
});
}
public WarcFs(string cdx)
{
this.cdx = cdx;
byte[] fileNameBytes = null;
string fileNameString = null;
var folder = Path.GetDirectoryName(cdx);
this.Root = CreateTree <WarcItem>(WarcCdxItemRaw.Read(cdx).Select(x =>
{
var response = x.ResponseCode;
if (response.Length != 0)
{
var responseCode = Utf8Utils.ParseInt32(response);
if (responseCode < 200 || responseCode >= 300)
{
return(null);
}
}
return(x.ToWarcItem(folder, ref fileNameBytes, ref fileNameString));
}).Where(x => x != null), x =>
{
var url = new Uri(x.Url);
var keep = -1;
if (url.AbsolutePath.StartsWith("/w/images/"))
{
keep = 2;
}
else if (url.AbsolutePath.StartsWith("/wiki/"))
{
keep = 1;
}
else if (url.Host.EndsWith(".fbcdn.net"))
{
keep = 0;
}
else if (url.Host.EndsWith(".media.tumblr.com"))
{
keep = 0;
}
else if (url.Host.EndsWith(".bp.blogspot.com"))
{
keep = 0;
}
else if (url.Host.EndsWith(".reddit.com") && url.AbsolutePath.Contains("/comments/"))
{
keep = 3;
}
else if (url.Host.EndsWith(".staticflickr.com"))
{
keep = 0;
}
else if (url.Host.EndsWith(".giphy.com") && url.Host.Contains("media"))
{
keep = 0;
}
var path = WebsiteScraper.GetPathInternal(null, url, x.ContentType, keep);
path = path.Replace('/', '\\');
if (path.Length > 150)
{
var z = path.IndexOf('‽');
if (z != -1)
{
path = path.Substring(0, z) + "‽{" + Math.Abs((long)path.GetHashCode()) + "}" + Path.GetExtension(path);
}
}
if (url.IsHostedOn("facebook.com") && url.AbsolutePath.StartsWith("/pages_reaction_units/"))
{
path = path.TrimEnd(".js");
path += ".html";
}
return(path);
}, null, x =>
{
x.Tag = TagVirtual;
if (x.Info != null)
{
urlToFsNode[x.Info.Url] = x;
}
});
FsNode <WarcItem> rawRoot = null;
rawRoot = new FsNode <WarcItem>()
{
Name = "_raw", GetChildrenDelegate = CreateGetChildrenDelegate(this.Root)
};
Func <List <FsNode <WarcItem> > > CreateGetChildrenDelegate(FsNode <WarcItem> reference)
{
if (reference.Children == null)
{
return(() => null);
}
return(new Func <List <FsNode <WarcItem> > >(() =>
{
return reference.Children.Where(x => x != rawRoot).Select(x =>
{
var k = new FsNode <WarcItem>()
{
Info = x.Info,
Name = x.Name,
GetChildrenDelegate = CreateGetChildrenDelegate(x),
Tag = null,
FullName = x.FullName != null ? "_raw\\" + x.FullName : null
};
return k;
}).ToList();
}));
}
this.Root.Children.Add(rawRoot);
cache = new MemoryStreamCache <FsNode <WarcItem> >((item, dest) =>
{
if (item.Tag == TagVirtual)
{
var ct = item.Info.ContentType;
if (ct != null && ct.Contains("/html") || item.Info.Url.Contains("facebook.com/pages_reaction_units/"))
{
HtmlNode doc;
var pagePath = item.FullName;
if (item.Info.Url.Contains("/pages_reaction_units/"))
{
var jsontext = item.Info.ReadText();
var idx = jsontext.IndexOf('{');
var json = (JObject)HttpUtils.ReadJsonToken(jsontext, idx);
doc = new HtmlDocument("<!doctype html><html><head><meta charset=\"utf-8\"></head><body></body></html>").DocumentNode;
doc.OwnerDocument.SetPageUrl(item.Info.Url.AsUri());
var body = doc.Descendants("body").First();
foreach (var domop in (JArray)json["domops"])
{
var html = ((JArray)domop).First(x => x is JObject)["__html"].Value <string>();
body.AppendChild(html.AsHtmlNode());
}
}
else
{
doc = item.Info.ReadHtml();
}
ProcessHtml(ref doc, pagePath);
var simpleStyle = doc.OwnerDocument.CreateElement("link");
simpleStyle.SetAttributeValue("rel", "stylesheet");
simpleStyle.SetAttributeValue("href", @"file:///C:\Users\Andrea\Desktop\facebook-simple-css.css");
(doc.FindSingle("head") ?? doc).AppendChild(simpleStyle);
using (var sw = new StreamWriter(dest, Encoding.UTF8, 16 * 1024, true))
{
doc.WriteTo(sw);
}
return;
}
}
using (var k = item.Info.OpenStream())
{
k.CopyTo(dest);
}
});
}
private FsNode <FileInfo> GetFileInfo(string fileName, FileInfo prefileinfo = null)
{
logger.Debug(Thread.CurrentThread.ManagedThreadId + " GetFileInfo: " + fileName);
var lowerfilename = fileName.ToLower();
// check extension first, then if it exists
if (lowerfilename.EndsWith(".rar") || lowerfilename.EndsWith(".zip") ||
(lowerfilename.Contains(".rar") || lowerfilename.Contains(".zip")) && !File.Exists(fileName) && !Directory.Exists(fileName))
{
if (fileName.ToLower().Contains(".rar") || fileName.ToLower().Contains(".zip"))
{
try
{
var info2 = new FileInfo(fileName);
var index = fileName.ToLower().IndexOf(".rar");
if (index == -1)
{
index = fileName.ToLower().IndexOf(".zip");
}
var file = fileName.Substring(0, index + 4);
var subpath = fileName.Substring(index + 4);
SharpCompressFs fs = null;
cache.TryGetValue(file.ToLower(), out fs);
if (fs == null)
{
logger.Debug("SevenZipFs: get list " + file);
try
{
fs = new SharpCompressFs(file);
}
catch
{
return(null);
}
}
cache[file.ToLower()] = fs;
var fsnodeinfo = fs.GetFile(subpath);
if (fsnodeinfo == null)
{
return(null);
}
var answerarchive = new FsNode <FileInfo>()
{
Tag = fsnodeinfo,
Info = info2,
Name = info2.Name,
FullName = info2.FullName
};
answerarchive.GetChildrenDelegate = () =>
{
answerarchive.Tag = fsnodeinfo;
return(fsnodeinfo?.Children?
.Select(x => GetFileInfo((
file + Path.DirectorySeparatorChar + subpath +
Path.DirectorySeparatorChar +
x).Replace("" + Path.DirectorySeparatorChar + Path.DirectorySeparatorChar,
"" + Path.DirectorySeparatorChar)))?.Where(a => a != null).ToList());
};
return(answerarchive);
}
catch (Exception ex)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(ex);
Console.ForegroundColor = ConsoleColor.White;
return(null);
}
}
return(null);
}
FileInfo info;
if (prefileinfo != null)
{
info = prefileinfo;
}
else
{
info = new FileInfo(fileName);
}
var answer = new FsNode <FileInfo>()
{
Info = info,
Name = info.Name,
FullName = info.FullName,
GetChildrenDelegate = () =>
{
DirectoryInfo dirinfo = new DirectoryInfo(info.FullName);
if (dirinfo.Exists)
{
try
{
// get dirs
var dirs = new ConcurrentBag <FsNode <FileInfo> >();
Parallel.ForEach(dirinfo.GetDirectories("*", SearchOption.TopDirectoryOnly), x =>
{
if (CheckHasValidChildren(x, "*.rar"))
{
dirs.Add(GetFileInfo(x.FullName));
}
});
// get files
var files = new ConcurrentBag <FsNode <FileInfo> >();
var fileInfos = dirinfo.GetFiles("*.rar", SearchOption.TopDirectoryOnly);
Parallel.ForEach(fileInfos, x => { files.Add(GetFileInfo(x.FullName, x)); });
// filter files
var filefilter = files.Where(a =>
a != null &&
(a.Name.ToLower().Contains("part01.rar") || !a.Name.ToLower().Contains(".part")));
// combine to one list
var combined = filefilter.Concat(dirs).Where(a => a != null);
// return result
return(combined.ToList());
}
catch (Exception ex)
{
logger.Error(ex.ToString());
return(null);
}
}
return(null);
}
};
return(answer);
}
public IEnumerable <String> EnumerateFiles(string szDriveLetter)
{
try
{
var usnRecord = default(UsnRecord);
var mft = default(MftEnumData);
var dwRetBytes = 0;
var cb = 0;
var dicFrnLookup = new Dictionary <long, FsNode>();
var bIsFile = false;
// This shouldn't be called more than once.
if (_mBuffer.ToInt32() != 0)
{
throw new Exception("invalid buffer");
}
// Assign buffer size
_mBufferSize = 65536;
//64KB
// Allocate a buffer to use for reading records.
_mBuffer = Marshal.AllocHGlobal(_mBufferSize);
// correct path
szDriveLetter = szDriveLetter.TrimEnd('\\');
// Open the volume handle
_m_HCj = OpenVolume(szDriveLetter);
// Check if the volume handle is valid.
if (_m_HCj == InvalidHandleValue)
{
string errorMsg = "Couldn't open handle to the volume.";
if (!IsAdministrator())
{
errorMsg += "Current user is not administrator";
}
throw new Exception(errorMsg);
}
mft.StartFileReferenceNumber = 0;
mft.LowUsn = 0;
mft.HighUsn = long.MaxValue;
do
{
if (DeviceIoControl(_m_HCj, FsctlEnumUsnData, ref mft, Marshal.SizeOf(mft), _mBuffer, _mBufferSize, ref dwRetBytes, IntPtr.Zero))
{
cb = dwRetBytes;
// Pointer to the first record
IntPtr pUsnRecord = new IntPtr(_mBuffer.ToInt32() + 8);
while ((dwRetBytes > 8))
{
// Copy pointer to USN_RECORD structure.
usnRecord = (UsnRecord)Marshal.PtrToStructure(pUsnRecord, usnRecord.GetType());
// The filename within the USN_RECORD.
string fileName = Marshal.PtrToStringUni(new IntPtr(pUsnRecord.ToInt32() + usnRecord.FileNameOffset), usnRecord.FileNameLength / 2);
bIsFile = !usnRecord.FileAttributes.HasFlag(FileAttributes.Directory);
dicFrnLookup.Add(usnRecord.FileReferenceNumber, new FsNode(usnRecord.FileReferenceNumber, usnRecord.ParentFileReferenceNumber, fileName, bIsFile));
// Pointer to the next record in the buffer.
pUsnRecord = new IntPtr(pUsnRecord.ToInt32() + usnRecord.RecordLength);
dwRetBytes -= usnRecord.RecordLength;
}
// The first 8 bytes is always the start of the next USN.
mft.StartFileReferenceNumber = Marshal.ReadInt64(_mBuffer, 0);
}
else
{
break; // TODO: might not be correct. Was : Exit Do
}
} while (!(cb <= 8));
// Resolve all paths for Files
foreach (FsNode oFsNode in dicFrnLookup.Values.Where(o => o.IsFile))
{
string sFullPath = oFsNode.FileName;
FsNode oParentFsNode = oFsNode;
while (dicFrnLookup.TryGetValue(oParentFsNode.ParentFrn, out oParentFsNode))
{
sFullPath = string.Concat(oParentFsNode.FileName, @"\", sFullPath);
}
sFullPath = string.Concat(szDriveLetter, @"\", sFullPath);
yield return(sFullPath);
}
}
finally
{
//// cleanup
Cleanup();
}
}
