public static Byte[] CalculateCrc(Byte[] data)
{
using (var crc = new Crc32CAlgorithm())
{
return(crc.ComputeHash(data));
}
}
private async void GetResourceAsync(DownloadFile file)
{
try
{
log.Debug("zaczynam pobierac " + file.Name);
using (var client = new WebClient())
{
var responseBody = await client.DownloadStringTaskAsync(file.Url);
byte[] bytes = Encoding.ASCII.GetBytes(responseBody);
performanceCounter.IncrementBy(bytes.Length);
uint crc = Crc32CAlgorithm.Compute(bytes);
log.Debug(file.Name + " zasob pobrany pomyslnie.");
if (file.Status == Status.Success)
{
if (crc != file.Crc)
{
log.Info(file.Name + " rozni sie od poprzednio pobranego.");
}
}
file.Status = Status.Success;
file.Crc = crc;
file.LastAccess = DateTime.Now;
}
}
catch (WebException e)
{
file.Status = Status.Failed;
log.Error(file.Name + " cos poszlo nie tak " + e.Message);
}
}
public async Task <ScrapeJobEvent> ExecuteAsync(Uri url, string searchPattern)
{
if (url == null)
{
return(null);
}
var response = await GetResponseAsync(url);
var eventType = ScrapeJobEventType.Error;
string fingerprint = "";
if (!string.IsNullOrWhiteSpace(response.HtmlString))
{
var matches = MatchesPattern(response.HtmlString, searchPattern);
eventType = matches ? ScrapeJobEventType.Match : ScrapeJobEventType.NoMatch;
fingerprint = Crc32CAlgorithm.Compute(response.HtmlBytes).ToString(CultureInfo.InvariantCulture);
}
return(new ScrapeJobEvent
{
HttpResponseCode = response.StatusCode,
HttpResponseTimeInMs = response.ResponseTimeInMs,
TimeStamp = DateTime.UtcNow,
Type = eventType,
Url = url.AbsoluteUri,
Fingerprint = fingerprint
});
}
public void CRC32PerformanceTest()
{
var data = new byte[65536];
var random = new Random();
random.NextBytes(data);
/*Crc32.Net*/
long total = 0;
var stopwatch = new Stopwatch();
stopwatch.Start();
while (stopwatch.Elapsed < TimeSpan.FromSeconds(3))
{
Crc32.NET.Crc32Algorithm.Compute(data, 0, data.Length);
total += data.Length;
}
stopwatch.Stop();
Console.WriteLine($@"Crc32.Net Throughput: {total / stopwatch.Elapsed.TotalSeconds / 1024 / 1024:0.0}MB/s");
/*Crc32C.Net*/
total = 0;
stopwatch = new Stopwatch();
stopwatch.Start();
while (stopwatch.Elapsed < TimeSpan.FromSeconds(3))
{
Crc32CAlgorithm.Compute(data, 0, data.Length);
total += data.Length;
}
stopwatch.Stop();
Console.WriteLine($@"Crc32C.Net Throughput: {total / stopwatch.Elapsed.TotalSeconds / 1024 / 1024:0.0}MB/s");
}
/// <summary>
/// Write a single record.
/// </summary>
/// <param name="stream">Stream containing record in binary form</param>
public void Write(Stream stream)
{
ulong length = (ulong)stream.Length;
uint crcLength = Crc32CAlgorithm.Compute(BitConverter.GetBytes(length));
uint maskLength = MaskCrc32(crcLength);
_fileStream.Write(BitConverter.GetBytes(length), 0, 8); // uint64 length
_fileStream.Write(BitConverter.GetBytes(maskLength), 0, 4); // uint32 masked_crc32_of_length
stream.Seek(0, SeekOrigin.Begin); // Read from head
byte[] buffer = new byte[BufferSize];
int count = 0;
int readSize = stream.Read(buffer, 0, buffer.Length);
uint crcData = Crc32CAlgorithm.Compute(buffer, 0, readSize);
for (bool firstRun = true; readSize > 0;
count += readSize, readSize = stream.Read(buffer, 0, buffer.Length), firstRun = false)
{
if (!firstRun)
{
crcData = Crc32CAlgorithm.Append(crcData, buffer, 0, readSize);
}
_fileStream.Write(buffer, 0, readSize); // byte data[length]
}
if (count != (int)length)
{
throw new Exception("Stream length does not equal to read length.");
}
uint maskCrcData = MaskCrc32(crcData);
_fileStream.Write(BitConverter.GetBytes(maskCrcData), 0, 4); // uint32 masked_crc32_of_data
}
public static uint GetCrc32FromFile(string fileName)
{
try
{
using (FileStream file = new FileStream(fileName, FileMode.Open))
{
const int NumBytes = 10000;
var bytes = new byte[NumBytes];
var numRead = file.Read(bytes, 0, NumBytes);
if (numRead == 0)
{
return(0);
}
var crc = 0u;
while (numRead > 0)
{
Crc32CAlgorithm.Append(crc, bytes, 0, numRead);
numRead = file.Read(bytes, 0, NumBytes);
}
return(crc);
}
}
catch (Exception ex) when(ex is UnauthorizedAccessException || ex is IOException)
{
return(0);
}
}
public override async System.Threading.Tasks.Task <List <FileModel> > getFilesOutDatedAsync()
{
WebClient client = new WebClient();
Uri uri = new Uri(config.BaseURL + Routes.UPDATE_FILE_LIST);
String updateJson;
try
{
updateJson = await client.DownloadStringTaskAsync(uri);
}
catch (Exception e)
{
throw new CheckUpdateError(e.Message);
}
List <FileModel> list = JsonConvert.DeserializeObject <List <FileModel> >(updateJson);
List <FileModel> lists = list.FindAll(delegate(FileModel model) {
if (!File.Exists(model.ClientPath))
{
return(true);
}
byte[] clientFile = File.ReadAllBytes(model.ClientPath);
uint crc = Crc32CAlgorithm.Compute(clientFile);
return(!model.cCRC.Equals(crc.ToString()));
});
return(lists);
}
public static uint ToCrc32(this string selfStr)
{
var bytes = Encoding.UTF8.GetBytes(selfStr);
uint crc32 = Crc32CAlgorithm.Compute(bytes);
return(crc32);
}
public static string GetChecksum(this TypeDefinition type, TextualRepresentationOptions options)
{
uint lastHash = 0;
StringBuilder checksumBuilder = new StringBuilder(2 * ChecksumThreshold);
Action <string> appendAction = (s) =>
{
checksumBuilder.Append(s);
if (checksumBuilder.Length > ChecksumThreshold)
{
byte[] bytes = Encoding.ASCII.GetBytes(checksumBuilder.ToString());
lastHash = Crc32CAlgorithm.Append(lastHash, bytes);
checksumBuilder.Clear();
}
};
type.GetTextualRepresentation(appendAction, options);
// clear if something left in StringBuilder
if (checksumBuilder.Length > 0)
{
byte[] bytes = Encoding.ASCII.GetBytes(checksumBuilder.ToString());
lastHash = Crc32CAlgorithm.Append(lastHash, bytes);
checksumBuilder.Clear();
}
return(lastHash.ToString());
}
/// <summary>
/// 文字列のCRC32を計算して文字列で返す
/// </summary>
/// <param name="srcStr"></param>
/// <returns></returns>
public static string CalcCRC32(string srcStr)
{
byte[] bytes = new UTF8Encoding().GetBytes(srcStr);
uint crc32 = Crc32CAlgorithm.Compute(bytes);
return(Convert.ToString(crc32, 16));
}
public void MethodConsistency(string text, int offset, int tail, int split)
{
var checksums = new List <uint>();
var array = Encoding.ASCII.GetBytes(text);
var padded = Enumerable.Repeat <byte>(17, offset).Concat(array).Concat(Enumerable.Repeat <byte>(93, tail)).ToArray();
var half1 = array.Take(split).ToArray();
var half2 = array.Skip(split).ToArray();
checksums.Add(Crc32CAlgorithm.Compute(array));
checksums.Add(Crc32CAlgorithm.Compute(padded, offset, array.Length));
checksums.Add(Crc32CAlgorithm.Append(0, array));
checksums.Add(Crc32CAlgorithm.Append(0, padded, offset, array.Length));
checksums.Add(Crc32CAlgorithm.Append(Crc32CAlgorithm.Append(0, half1), half2));
checksums.Add(Crc32CAlgorithm.Append(Crc32CAlgorithm.Append(0, padded, offset, split), padded, offset + split, array.Length - split));
using (var hash = new Crc32CAlgorithm())
checksums.Add(BitConverter.ToUInt32(hash.ComputeHash(array), 0));
using (var hash = new Crc32CAlgorithm())
checksums.Add(BitConverter.ToUInt32(hash.ComputeHash(padded, offset, array.Length), 0));
using (var stream = new MemoryStream(array))
using (var hash = new Crc32CAlgorithm())
checksums.Add(BitConverter.ToUInt32(hash.ComputeHash(stream), 0));
if (text.Length == 0)
{
Assert.AreEqual(0, checksums[0]);
}
foreach (var checksum in checksums)
{
Assert.AreEqual(checksums[0], checksum);
}
}
public void Write(IProgress <Tuple <string, int> > progress)
{
List <uint> md5s = new List <uint>();
using (FileStream arc = new FileStream(Filename, FileMode.Create))
using (MemoryStream header = new MemoryStream())
using (BinaryWriter writer = new BinaryWriter(arc))
using (BinaryWriter headerWriter = new BinaryWriter(header))
{
writer.Write(Encoding.ASCII.GetBytes(ExtendedArchive.Magic));
writer.Write(ExtendedArchive.Version);
writer.Write((ushort)1);
byte[] title = Encoding.Unicode.GetBytes(Name);
writer.Write((ushort)title.Length);
writer.Write(title);
writer.Write((uint)Files.Count);
int headerLength = Files.Sum(x => x.HeaderLength);
writer.Write((uint)headerLength);
long headerPos = writer.BaseStream.Position;
writer.BaseStream.Position = headerLength + 1024; //a 1kb space is left incase the header needs to be shifted later
int i = 0;
foreach (var file in Files)
{
uint crc = Crc32CAlgorithm.Compute(file.Source.Md5);
i++;
try
{
file.WriteTo(writer, headerWriter, md5s.Contains(crc));
}
catch
{
progress.Report(new Tuple <string, int>(
"[" + i + " / " + Files.Count + "] Stopped writing " + file.Name + "\n",
100 * i / Files.Count));
throw;
}
progress.Report(new Tuple <string, int>(
"[" + i + " / " + Files.Count + "] Written " + file.Name + "... (" + file.Source.Size + " bytes)\n",
100 * i / Files.Count));
md5s.Add(crc);
}
writer.BaseStream.Position = headerPos;
writer.Write(header.ToArray());
}
progress.Report(new Tuple <string, int>("Finished.\n", 100));
}
public bool VerifyFile(DistributionFile file, FileInfo fileInfo)
{
var bytes = File.ReadAllBytes(fileInfo.FullName);
return
(Crc32CAlgorithm.Compute(bytes) == file.Hash &&
_signatureTool.Verify(file.Signature, bytes));
}
private static string CalculateCrc32c(Stream stream)
{
var algorithm = new Crc32CAlgorithm();
var hash = algorithm.ComputeHash(stream);
stream.Seek(0, SeekOrigin.Begin);
Array.Reverse(hash);
return(Convert.ToBase64String(hash));
}
public void ResultConsistency(string text, int offset)
{
var bytes = Encoding.ASCII.GetBytes(text);
var crc1 = Crc32CAlgorithm.Append(123456789, bytes, offset, bytes.Length - offset);
CoreSwitchHelper.SetCoreAlgType("Crc32C.SafeProxy");
var crc2 = Crc32CAlgorithm.Append(123456789, bytes, offset, bytes.Length - offset);
Assert.That(crc2, Is.EqualTo(crc1));
}
private static Guid GetHash(byte[] b1, byte[] b2)
{
uint i1 = Crc32Algorithm.Compute(b1);
uint i2 = Crc32CAlgorithm.Compute(b2);
byte[] i3 = BitConverter.GetBytes(Crc32Algorithm.Compute(b1));
byte[] i4 = BitConverter.GetBytes(Crc32CAlgorithm.Compute(b2));
return(new Guid(i1, (ushort)i2, (ushort)(i2 >> 16), i3[0], i3[1], i3[2], i3[3], i4[0], i4[1], i4[2], i4[3]));
}
/// <summary>
/// Sha256 object hash
/// </summary>
/// <param name="value">Object to hash</param>
/// <returns>String hash</returns>
public static string Crc32(object value)
{
if (value == null)
{
return(string.Empty);
}
var bytes = Byte(value);
return(Crc32CAlgorithm.Compute(bytes).ToString());
}
public static string GetCRCVersion(string filePath)
{
byte[] buffer = File.ReadAllBytes(filePath);
//FileStream stream = new FileStream( filePath, FileMode.Open );
//byte[] buffer = new byte[stream.Length];
//stream.Read( buffer, 0, (int)stream.Length );
//stream.Close();
uint crc32 = Crc32CAlgorithm.Compute(buffer);
return("?v=" + crc32);
}
public static UInt32 GetHash(string Name)
{
try
{
return(Crc32CAlgorithm.Compute(File.ReadAllBytes(Name)));
}
catch
{
MessageBox.Show(Name + " cannot be opened.");
return(new UInt32());
}
}
public void ResultConsistencyLong()
{
var bytes = new byte[30000];
new Random().NextBytes(bytes);
var crc1 = Crc32CAlgorithm.Append(123456789, bytes, 0, bytes.Length);
CoreSwitchHelper.SetCoreAlgType("Crc32C.SafeProxy");
var crc2 = Crc32CAlgorithm.Append(123456789, bytes, 0, bytes.Length);
Assert.That(crc2, Is.EqualTo(crc1));
}
public static string GetFileCRC32(string path)
{
// Read by 512 bytes
// No idea if this is accurate for detecting changes in files
var file = new FileStream(path, FileMode.Open);
var data = new byte[512];
file.Read(data, 0, 512);
file.Close();
return(Crc32CAlgorithm.Compute(data).ToString());
}
/// <summary>
/// Reads a protocol message from the specified byte-field and calls appropriate methods to process this message.
/// </summary>
/// <remarks>
/// This function checks all applicable checksums and validates that the message is complete before calling one of the specialized
/// methods to handle actual decoding and processing.
/// </remarks>
/// <param name="recv_buffer">Byte-field with an Ixian protocol message.</param>
/// <param name="endpoint">Remote endpoint from where the message was received.</param>
public static void readProtocolMessage(QueueMessageRaw raw_message, MessagePriority priority, RemoteEndpoint endpoint)
{
if (endpoint == null)
{
Logging.error("Endpoint was null. readProtocolMessage");
return;
}
ProtocolMessageCode code = raw_message.code;
// Filter messages
if (endpoint.presence == null)
{
// Check for presence and only accept hello and bye messages if there is no presence.
if (code != ProtocolMessageCode.hello &&
code != ProtocolMessageCode.helloData &&
code != ProtocolMessageCode.bye)
{
return;
}
}
if (raw_message.legacyChecksum != null)
{
// Compute checksum of received data
byte[] local_checksum = Crypto.sha512sqTrunc(raw_message.data, 0, 0, 32);
// Verify the checksum before proceeding
if (local_checksum.SequenceEqual(raw_message.legacyChecksum) == false)
{
Logging.error("Dropped message (invalid legacy checksum)");
return;
}
}
else
{
// Compute checksum of received data
uint local_checksum = Crc32CAlgorithm.Compute(raw_message.data);
// Verify the checksum before proceeding
if (local_checksum != raw_message.checksum)
{
Logging.error("Dropped message (invalid checksum)");
return;
}
}
// Can proceed to parse the data parameter based on the protocol message code.
// Data can contain multiple elements.
//parseProtocolMessage(code, data, socket, endpoint);
NetworkQueue.receiveProtocolMessage(code, raw_message.data, Crc32CAlgorithm.Compute(raw_message.data), priority, endpoint);
}
public static byte[] Calculate(string str)
{
byte[] hash;
Crc32CAlgorithm crc = new Crc32CAlgorithm();
hash = crc.ComputeHash(Encoding.UTF8.GetBytes(str));
//Invert the hash, since Hal does that for some reason
for (int i = 0; i < hash.Length; i++)
{
hash[i] = (byte)(255 - hash[i]);
}
return(hash);
}
public void Compute(string asciiChars, uint expectedResult)
{
// Arrange
var bytes = Encoding.ASCII.GetBytes(asciiChars);
// Act
var result = Crc32CAlgorithm.Compute(bytes);
// Assert
Assert.Equal(expectedResult, result);
}
public static uint Append(BytesView bv, uint initial)
{
uint crc = initial;
while (bv != null)
{
if (bv.len > 0)
{
crc = Crc32CAlgorithm.Append(crc, bv.bytes, bv.offset, bv.len);
}
bv = bv.nextNode;
}
return(crc);
}
public static string GetCRCVersion(string filePath)
{
//var inputArray = new byte[realDataLength + 4];
//// write real data to inputArray
//Crc32Algorithm.ComputeAndWriteToEnd( inputArray ); // last 4 bytes contains CRC
// // transferring data or writing reading, and checking as final operation
//if( !Crc32Algorithm.IsValidWithCrcAtEnd( inputArray ) )
// throw new InvalidOperationException( "Data was tampered" );
byte[] zipdata = File.ReadAllBytes(filePath);
uint crc = Crc32CAlgorithm.Compute(zipdata);
return("?v=" + crc);
}
public void Exceptions()
{
Assert.Throws <ArgumentNullException>(() => Crc32CAlgorithm.Compute(null));
Assert.Throws <ArgumentNullException>(() => Crc32CAlgorithm.Compute(null, 0, 0));
Assert.Throws <ArgumentNullException>(() => Crc32CAlgorithm.Append(0, null));
Assert.Throws <ArgumentNullException>(() => Crc32CAlgorithm.Append(0, null, 0, 0));
var buffer = new byte[10];
Assert.Throws <ArgumentOutOfRangeException>(() => Crc32CAlgorithm.Compute(buffer, -1, 5));
Assert.Throws <ArgumentOutOfRangeException>(() => Crc32CAlgorithm.Compute(buffer, 0, -1));
Assert.Throws <ArgumentOutOfRangeException>(() => Crc32CAlgorithm.Compute(buffer, 5, 6));
Assert.Throws <ArgumentOutOfRangeException>(() => Crc32CAlgorithm.Append(0, buffer, -1, 5));
Assert.Throws <ArgumentOutOfRangeException>(() => Crc32CAlgorithm.Append(0, buffer, 0, -1));
Assert.Throws <ArgumentOutOfRangeException>(() => Crc32CAlgorithm.Append(0, buffer, 5, 6));
}
internal bool VerifyChecksum()
{
uint crc = 0;
using (Stream source = GetStream())
{
byte[] buffer = new byte[PPeXCore.Settings.BufferSize];
int length = 0;
while ((length = source.Read(buffer, 0, (int)PPeXCore.Settings.BufferSize)) > 0)
{
crc = Crc32CAlgorithm.Append(crc, buffer, 0, length);
}
}
return(crc == Crc);
}
private string GetHash(string Name)
{
if (Name == string.Empty)
{
return(null);
}
try
{
return(Crc32CAlgorithm.Compute(File.ReadAllBytes(Name)).ToString("x2"));
}
catch
{
MessageBox.Show(Name + " cannot be opened.");
return(null);
}
}
private void CalculateCrc32(PhotoModel item)
{
var crc = Crc32CAlgorithm.Compute(Encoding.Default.GetBytes(item.LocalPath.ToLowerInvariant()));
var buffer = new byte[8096];
using (var stream = File.OpenRead(item.LocalPath))
{
while (true)
{
var read = stream.Read(buffer, 0, 8096);
crc = Crc32CAlgorithm.Append(crc, buffer, 0, read);
if (read < 8096)
{
item.Crc32 = crc.ToString();
return;
}
}
}
}
