public void SameEqual()
{
var a = new FileSignature(new HashValue(new byte[] { 0x99, 0xE9, 0xD8, 0x51, 0x37, 0xDB, 0x46, 0xEF }), 355);
var b = new FileSignature(new HashValue(new byte[] { 0x99, 0xE9, 0xD8, 0x51, 0x37, 0xDB, 0x46, 0xEF }), 355);
a.Equals(b).Should().BeTrue("FileSignature with same properties should be equal");
}
public async Task CheckLocationTest()
{
var uri = new Uri(Path.Combine(Path.GetFullPath("."), "temp\\"));
var lss = new LocalStorageService(uri);
var fileSignature =
new FileSignature(new HashValue(new byte[] { 0x99, 0xE9, 0xD8, 0x51, 0x37, 0xDB, 0x46, 0xEF }), 1UL);
if (await lss.IsFileAvailable(fileSignature))
{
await lss.Delete(fileSignature);
}
(await lss.IsFileAvailable(fileSignature)).Should().BeFalse();
using (var stream = await lss.Write(fileSignature))
{
stream.WriteByte(0xff);
}
(await lss.IsFileAvailable(fileSignature)).Should().BeTrue();
using (var stream = await lss.Read(fileSignature))
{
stream.ReadByte().Should().Be(0xff);
stream.ReadByte().Should().Be(-1);
}
var filename = (await lss.GetFileInfo(fileSignature)).FullName;
filename.Should().StartWith(Path.Combine(Path.GetFullPath("."), "temp\\"));
filename.Should().ContainEquivalentOf(fileSignature.Hash.ToString());
await lss.Delete(fileSignature);
(await lss.IsFileAvailable(fileSignature)).Should().BeFalse();
await Assert.ThrowsAsync <FileNotFoundException>(async() => await lss.Read(fileSignature));
}
public static bool GuessFormatFromSignature(Stream stream, out SevenZipFormat format)
{
FileSignature longestSig = FileSignatures.Values.OrderByDescending(v => v.Magic.Length + v.Offset).First();
int longestSignatureLen = longestSig.Magic.Length + longestSig.Offset;
byte[] archiveFileSignature = new byte[longestSignatureLen];
int bytesRead = stream.Read(archiveFileSignature, 0, longestSignatureLen);
stream.Position -= bytesRead; // go back o beginning
if (bytesRead != longestSignatureLen)
{
format = SevenZipFormat.Undefined;
return(false);
}
foreach (KeyValuePair <SevenZipFormat, FileSignature> pair in FileSignatures)
{
if (archiveFileSignature.Skip(pair.Value.Offset).Take(pair.Value.Magic.Length).SequenceEqual(pair.Value.Magic))
{
format = pair.Key;
return(true);
}
}
format = SevenZipFormat.Undefined;
return(false);
}
public OnlineFile(FileSignature fileSignature, Uri uri)
{
FileSignature = fileSignature;
Uri = uri;
if (!Uri.IsAbsoluteUri)
{
throw new UriNotAbsoluteException();
}
}
Stream IFileAccessService.Write(FileSignature fileSignature, bool temporary)
{
var file = GetFileInfo(fileSignature, temporary);
return(Policy
.Handle <IOException>()
.WaitAndRetry(5, i => Math.Pow(2, i).Seconds(),
(exception, duration) => LogTo.Warning(exception,
"Opening file {file} for write failed after {duration}", file.FullName, duration))
.Execute(() => file.Open(FileMode.Create, FileAccess.Write, FileShare.None)));
}
public Uri GetDownloadUri(FileSignature fileSignature)
{
foreach (var repo in Repos.Items)
{
if (repo.Files.TryGetValue(fileSignature, out var relativeUri))
{
return(new Uri(repo.BaseUri, relativeUri));
}
}
throw new KeyNotFoundException($"Key {fileSignature} was not found in a Files dictionary of any Repo");
}
/// <summary>
/// 在指定的目录中搜寻完全一样的文件。
/// </summary>
/// <param name="fileUrl">需要进行匹配的样本文件的文件地址。</param>
/// <param name="directory">需要进行搜寻的目录，即匹配的范围。</param>
/// <param name="isSorted">是否将匹配结果进行排序之后并作为返回值提交给用户。</param>
/// <returns>如果搜寻到完全一样的文件，则会返回这些文件地址所构成的List实例。</returns>
/// <remarks>该方法的原理是基于哈希验证的，但为了提高函数的效率，因此采用了文件名称对比、文件大小对比和文件SHA1代码对比来实现更加高效的运行，因为在哈希代码上完全相同的文件，其文件名称（不包含目录）和文件大小是完全相同的，但是当需要进行匹配的文件大小过大（比如说一个Windows离线安装镜像），则该操作将会非常的消耗时间，因此为了避免这种情况导致的应用程序性能下降，建议使用异步操作模式保证其他诸如UI线程的正常工作。</remarks>
public static List <string> SearchIdenticalFiles(string fileUrl, string directory, bool isSorted)
{
List <string> searchResult = new List <string>();
List <string> traversedFiles = TraverseWith(directory, false);
const int CRITICAL_FILE_SIZE = 50000000;
Parallel.For(0, traversedFiles.Count, (index, interrupt) =>
{
try
{
if (GetFileName(traversedFiles[index]) == GetFileName(fileUrl))
{
long sourceSize = GetFileSize(fileUrl);
long targetSize = GetFileSize(traversedFiles[index]);
if (sourceSize == targetSize)
{
string srcHashCode = string.Empty;
string tgtHashCode = srcHashCode;
bool fileSizeCondition = sourceSize >= CRITICAL_FILE_SIZE || targetSize >= CRITICAL_FILE_SIZE;
Action getSrcHash = () => srcHashCode = new FileSignature(fileUrl).GetSHA1String();
Action getTgtHash = () => tgtHashCode = new FileSignature(traversedFiles[index]).GetSHA1String();
if (fileSizeCondition)
{
Parallel.Invoke(new Action[] { getSrcHash, getTgtHash });
}
else
{
getSrcHash.Invoke();
getTgtHash.Invoke();
}
if (srcHashCode.Equals(tgtHashCode))
{
searchResult.Add(traversedFiles[index]);
}
}
}
}
catch (Exception throwedException)
{
if (throwedException != null)
{
throw throwedException.InnerException;
}
interrupt.Stop();
}
});
if (isSorted)
{
searchResult.Sort();
}
return(searchResult);
}
public async Task <(bool available, Lazy <Task <Stream> > stream)> WriteIfMissingOrInvalid(
FileSignature fileSignature)
{
var fi = await GetFileInfo(fileSignature);
var exists = await IsFileAvailable(fileSignature);
if (exists == false || fileSignature.Length != Convert.ToUInt64(fi.Length))
{
return(false, new Lazy <Task <Stream> >(() => Write(fileSignature)));
}
return(true, null);
}
/// <summary>
/// Check if specified file is trusted
/// </summary>
/// <param name="file"><see cref="File"/> to verify</param>
/// <param name="signature"><see cref="FileSignature"/> of file we are verifying provided</param>
/// <returns>true when file comes from trusted source</returns>
public bool VerifyFile(File file, FileSignature signature)
{
IssuerInformation information = null;
if (_issuers.TryGetValue(signature.IssuerName, out information))
{
return(information.IssuerAlgorithm.Verify(file.Data, signature.Signature) &&
file.FileName == signature.FileName);
}
else
{
return(false);
}
}
// @pixelOffset: Offset from start of file to first pixel in image
public static BmpHeader Create(uint fileSize, uint pixelOffset)
{
BmpHeader header = new BmpHeader();
header.fileSignature = new byte[FileSignature.Count];
FileSignature.CopyTo(header.fileSignature, 0);
//System.Array.Copy(FileSignature, header.fileSignature, FileSignature.Count);
header.size = fileSize;
header.reserved1 = DefaultReserved1;
header.reserved2 = DefaultReserved2;
header.pixelOffset = pixelOffset;
return(header);
}
public void verification_against_trusted_issuer_with_incorrect_file()
{
var signature = new FileSignature()
{
FileName = "Nomad.dll",
IssuerName = "Nomad",
Signature = new byte[] { 65, 64, 63, 62, 61 }
};
var file = new File()
{
FileName = "Nomad.dll", Data = new byte[] { 65, 64, 63, 62, 60, 60 }
};
Assert.IsFalse(_fileSignatureVerificator.VerifyFile(file, signature));
}
public void TemporaryFinished(FileSignature fileSignature)
{
var temp = GetFileInfo(fileSignature, true);
var final = GetFileInfo(fileSignature, false);
LogTo.Verbose("Renaming file {src} to {dst}", temp, final);
if (final.Exists)
{
final.Delete();
}
Policy
.Handle <IOException>()
.WaitAndRetry(5, i => Math.Pow(2, i).Seconds(),
(exception, duration) => LogTo.Warning(exception,
"Moving file from {src} to {dst} has failed after {duration}", temp.FullName, final.FullName,
duration))
.Execute(() => { temp.MoveTo(final.FullName); });
}
/// <summary>
/// 移动占用空间比较大的文件。
/// </summary>
/// <param name="sourceFileUrl">需要被移动的文件的文件地址。</param>
/// <param name="targetFileUrl">移动操作的目标地址。</param>
/// <param name="transmissionSize">每一次传输的大小，通常这个值设定为1024即可。</param>
/// <param name="isCheckFileIntegrity">移动文件之后是否执行完整性检验。</param>
/// <exception cref="AbortedCheckOperationException">当文件完整性检查失败或者结果不符合要求时，则会抛出这个异常。</exception>
public static void MoveBigFile(string sourceFileUrl, string targetFileUrl, int transmissionSize, bool isCheckFileIntegrity)
{
FileSignature sourceSignature = new FileSignature(sourceFileUrl);
FileSignature targetSignature = new FileSignature(targetFileUrl);
ExString srcMd5Value = string.Empty;
ExString targetMd5Value = string.Empty;
CopyBigFile(sourceFileUrl, targetFileUrl, transmissionSize);
if (isCheckFileIntegrity)
{
srcMd5Value = sourceSignature.GetMD5String();
targetMd5Value = targetSignature.GetMD5String();
if (!srcMd5Value.Equals(targetMd5Value))
{
throw new AbortedCheckOperationException();
}
}
DeleteFile(sourceFileUrl);
}
public async Task <(bool Success, string ErrorMsg, byte[] FileSignature)> Validate(string fileName, Stream stream)
{
var extensionName = Path.GetExtension(fileName);
if (!Configure.AllowFileExtension.Contains(extensionName))
{
return(false, "Illegal file format.", null);
}
if (!FileSignature.GetSignature(extensionName, out var signatures))
{
return(false, $"未找到{extensionName}文件的签名，通过FileSignature.AddSignature()方法添加签名。", null);
}
var maxLen = signatures.Max(m => m.Signature.Length + m.Offset);
var headerBytes = new byte[maxLen];
stream.Read(headerBytes, 0, maxLen);
var success = signatures.Any(signature => headerBytes.Skip(signature.Offset).Take(signature.Signature.Length).SequenceEqual(signature.Signature));
return(await Task.FromResult((success, success ? "" : "Illegal file format.", headerBytes)));
}
private void SaveFile(FileSignature signature, string path)
{
const int bufsize = 0x100000;
var remains = signature.FileSize;
_volume.SeekFileArea(signature.Offset);
path = path + "/" + signature.FileName;
var uniquePath = Utility.UniqueFileName(path);
using (FileStream file = new FileStream(uniquePath, FileMode.Create))
{
while (remains > 0)
{
var read = Math.Min(remains, bufsize);
remains -= read;
byte[] buf = new byte[read];
_volume.GetReader().Read(buf, (int)read);
file.Write(buf, 0, (int)read);
}
}
}
private static async Task <Mod> CreateMod(DirectoryInfo modFolder, Action <FileSignature, Uri> fileAction)
{
Console.WriteLine($"Processing {modFolder.FullName}");
var hashing = new HashingService(new XXHash64());
var obs = hashing.GetFileHashes(modFolder, CancellationToken.None);
var mod = new Mod
{
Files = new Dictionary <Uri, FileSignature>(),
Name = modFolder.Name,
Version = "1.0"
};
foreach (var lazy in obs)
{
var fileHash = await lazy.Value;
Console.WriteLine($"Processing {fileHash.File.FullName}");
var fileSig = new FileSignature(fileHash.Hash, fileHash.File.Length);
mod.Files.Add(new Uri(modFolder.FullName).MakeRelativeUri(new Uri(fileHash.File.FullName)),
fileSig);
fileAction(fileSig, new Uri(fileHash.File.FullName));
}
return(mod);
}
public async Task Delete(FileSignature fileSignature)
{
var fi = await GetFileInfo(fileSignature);
await Task.Run(() => fi.Delete());
}
bool NeedToScanAssembly(FileInfo fileInfo)
{
string fileName = fileInfo.FullName.ToLowerInvariant();
if (!_Assemblies.ContainsKey(fileName))
return true;
if (!File.Exists(fileName))
return false;
TypeToAssemblyNamespaceMap assemblyMap = _Assemblies[fileName];
FileSignature fileSignature = assemblyMap._AssemblySignature;
FileSignature actualSignature = new FileSignature(fileName);
if (!actualSignature.Match(fileSignature))
return true;
return false;
}
private FileInfo GetFileInfo(FileSignature fileSignature, bool temporary)
{
return(new FileInfo(GetFileUri(fileSignature, temporary).FullName));
}
private Stream GetTemporaryFileStream(FileSignature argFileSignature)
{
return(fileAccessService.Write(argFileSignature, true));
}
private void AddNewFile(FileSignature fileSignature)
{
fileAccessService.TemporaryFinished(fileSignature);
LogTo.Verbose("File {name} is now available", fileSignature.Hash);
filesAvailable.AddOrUpdate(fileSignature);
}
private string GetFileName(FileSignature fileSignature, bool temporary)
{
return(fileSignature.Hash + (temporary ? ".tmp" : string.Empty));
}
public Stream StartDownload(FileSignature fileSignature)
{
return fileAccessService.Write(fileSignature, true);
}
public void FinishDownload(FileSignature fileSignature)
{
fileAccessService.TemporaryFinished(fileSignature);
availableFiles.AddOrUpdate(fileSignature);
}
public QueuedDownload(FileSignature fileSignature, Uri source)
{
FileSignature = fileSignature;
Source = source;
}
public async Task <Stream> Write(FileSignature fileSignature)
{
var file = await GetFileInfo(fileSignature);
return(await Task.Run(() => file.Open(FileMode.Create, FileAccess.Write)));
}
public async Task <Stream> Read(FileSignature fileSignature)
{
var file = await GetFileInfo(fileSignature);
return(await Task.Run(() => file.Open(FileMode.Open, FileAccess.Read)));
}
public async Task <bool> IsFileAvailable(FileSignature fileSignature)
{
var fi = await GetFileInfo(fileSignature);
return(await Task.Run(() => fi.Exists));
}
public Uri GetFileUri(FileSignature fileSignature)
{
return(new Uri(localPath, GetFileName(fileSignature)));
}
public string GetFileName(FileSignature fileSignature)
{
return(fileSignature.Hash.ToString());
}
public async Task <FileInfo> GetFileInfo(FileSignature fileSignature)
{
return(await Task.Run(() => new FileInfo(GetFileUri(fileSignature).LocalPath)));
}
