public static int GetCrc32(byte[] bytes, int offset, int length)
{
if (bytes == null)
{
Warning("Bytes is invalid.");
}
if (offset < 0 || length < 0 || offset + length > bytes.Length)
{
Warning("Offset or length is invalid.");
}
s_Algorithm.HashCore(bytes, offset, length);
int result = (int)s_Algorithm.HashFinal();
s_Algorithm.Initialize();
return(result);
}
public static void WriteMetadata(string fileName, Dictionary <string, string> data)
{
using (var fs = new FileStream(fileName, FileMode.Append, FileAccess.Write))
using (var bw = new EndianBinaryWriter(EndianBitConverter.Big, fs))
{
foreach (var text in data)
{
int chunkLen = text.Key.Length + 1 + text.Value.Length;
bw.Write((uint)chunkLen);
bw.Write(Encoding.ASCII.GetBytes("tEXt"), 0, 4);
var _toCrc = new List <byte>();
_toCrc.AddRange(Encoding.ASCII.GetBytes(text.Key));
_toCrc.Add(0);
_toCrc.AddRange(Encoding.ASCII.GetBytes(text.Value));
bw.Write(Encoding.ASCII.GetBytes(text.Key));
bw.Write((byte)0);
bw.Write(Encoding.ASCII.GetBytes(text.Value));
var crc = new Crc32();
crc.Initialize();
crc.ComputeHash(_toCrc.ToArray());
bw.Write(crc.CrcValue);
}
}
}
public void TestCrc32(string name, uint polynomial, uint initialState, uint xorOutput, bool reverseInput, bool reverseOutput, uint check)
{
using (Crc32 crc = new Crc32(polynomial, initialState, xorOutput, reverseInput, reverseOutput))
{
crc.Initialize();
var bytes = Encoding.UTF8.GetBytes(data);
for (int i = 0; i < bytes.Length; i++)
{
crc.Append(bytes[i]);
}
uint output = crc.CurrentOutput;
Assert.AreEqual(check, output, 0);
}
}
private int _current_io_file_flag; //norm:0, pause:1, abort:2
private void _io_thread_callback()
{
while (_io_thread_flags == 0)
{
bool queue_empty = true;
string next_path = string.Empty;
lock (_io_queue_lock)
{
if (_io_queue.Count > 0)
{
queue_empty = false;
next_path = _io_queue[0];
_io_queue.RemoveAt(0);
}
}
if (queue_empty || _current_io_file_flag == 1)
{
_io_wait.Reset();
_io_wait.Wait(120000);
lock (_sql_lock)
{
_sql_trs.Commit(); //committing changes every 2 minutes
_sql_trs = _sql_con.BeginTransaction();
}
continue;
}
FileSystemWatcher fs_watcher = null;
try
{
var fileinfo = new FileInfo(next_path);
_current_io_file = next_path;
var origin_file_length = fileinfo.Length;
if (fileinfo.Exists == false)
{
Tracer.GlobalTracer.TraceWarning("File " + next_path + " no longer exists, ignored");
continue;
}
fs_watcher = new FileSystemWatcher(fileinfo.DirectoryName, fileinfo.Name);
bool is_file_changed = false;
fs_watcher.Deleted += (s, e) =>
{
Tracer.GlobalTracer.TraceWarning("File deleted while reading " + next_path);
is_file_changed = true;
};
fs_watcher.Changed += (s, e) =>
{
Tracer.GlobalTracer.TraceWarning("File changed while reading " + next_path);
is_file_changed = true;
};
fs_watcher.EnableRaisingEvents = true;
byte[] path_sha1 = null;
#pragma warning disable CS0162
if (_FILE_SYSTEM_CASE_SENSITIVE)
{
path_sha1 = SHA1.ComputeHash(next_path);
}
else
{
path_sha1 = SHA1.ComputeHash(next_path.ToLower());
}
#pragma warning restore
Tracer.GlobalTracer.TraceInfo("File hash started: " + next_path);
//calculation (parallel access)
var md5_calc = new MD5(); //speed: ~36s/GB
var sha1_calc = new SHA1(); //speed: ~55s/GB
var md5_slice_calc = new MD5(); //speed: ~45ms (256KB)
var crc32_calc = new Crc32(); //speed: ~11s/GB
long md5_pos = 0, sha1_pos = 0, crc32_pos = 0;
//loading pre-calculated data from sql
#region loading data
lock (_sql_lock)
{
_sql_cmd.CommandText = "select MD5_Serialized, SHA1_Serialized, MD5_Slice_Serialized, CRC32_serialized from FileIOState where Path_SHA1 = @Path_SHA1";
_sql_cmd.Parameters.Add("@Path_SHA1", System.Data.DbType.Binary);
_sql_cmd.Parameters["@Path_SHA1"].Value = path_sha1;
var dr = _sql_cmd.ExecuteReader();
var suc = dr.Read();
if (suc)
{
var ms_md5 = new MemoryStream((byte[])dr[0]);
var ms_sha1 = new MemoryStream((byte[])dr[1]);
var ms_md5_slice = new MemoryStream((byte[])dr[2]);
var ms_crc32 = new MemoryStream((byte[])dr[3]);
md5_calc = MD5.Deserialize(ms_md5);
sha1_calc = SHA1.Deserialize(ms_sha1);
md5_slice_calc = MD5.Deserialize(ms_md5_slice);
crc32_calc = Crc32.Deserialize(ms_crc32);
md5_pos = md5_calc.Length;
sha1_pos = sha1_calc.Length;
crc32_pos = crc32_calc.Length;
}
dr.Close();
_sql_cmd.Parameters.Clear();
}
#endregion
int io_state = 0;
//5 thread parallel read
Parallel.For(0, 5, (i) =>
{
//debug test
var str = new string[] { "MD5", "SHA1", "CRC32", "Slice_MD5", "Event" };
var sw = new System.Diagnostics.Stopwatch();
//Tracer.GlobalTracer.TraceInfo(str[i] + " calculation thread started");
sw.Start();
FileStream fs = null;
var buffer = new byte[65536];
long length = 0;
int current = 0;
try
{
fs = new FileStream(next_path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
switch (i)
{
case 0:
if (md5_calc.Length > origin_file_length)
{
md5_calc.Initialize();
}
fs.Seek(md5_calc.Length, SeekOrigin.Begin);
break;
case 1:
if (sha1_calc.Length > origin_file_length)
{
sha1_calc.Initialize();
}
fs.Seek(sha1_calc.Length, SeekOrigin.Begin);
break;
case 2:
if (crc32_calc.Length > origin_file_length)
{
crc32_calc.Initialize();
}
fs.Seek(crc32_calc.Length, SeekOrigin.Begin);
break;
case 3:
if (md5_slice_calc.Length > origin_file_length)
{
md5_slice_calc.Initialize();
}
fs.Seek(md5_slice_calc.Length, SeekOrigin.Begin);
break;
case 4:
break;
default:
throw new InvalidOperationException();
}
length = fs.Position;
do
{
try
{
switch (i)
{
case 0:
current = fs.Read(buffer, 0, buffer.Length);
md5_calc.TransformBlock(buffer, 0, current);
md5_pos += current;
break;
case 1:
current = fs.Read(buffer, 0, buffer.Length);
sha1_calc.TransformBlock(buffer, 0, current);
sha1_pos += current;
break;
case 2:
current = fs.Read(buffer, 0, buffer.Length);
crc32_calc.TransformBlock(buffer, 0, current);
crc32_pos += current;
break;
case 3:
current = fs.Read(buffer, 0, buffer.Length);
if (length >= BaiduPCS.VALIDATE_SIZE)
{
return;
}
current = (int)Math.Min(BaiduPCS.VALIDATE_SIZE - length, current);
md5_slice_calc.TransformBlock(buffer, 0, current);
break;
case 4:
if (io_state == 0xf)
{
return;
}
current = 1;
long a = md5_pos, b = sha1_pos, c = crc32_pos;
var min = Math.Min(Math.Min(a, b), c);
LocalFileIOUpdate?.Invoke(next_path, min, origin_file_length);
Thread.Sleep(10);
break;
default:
throw new InvalidOperationException();
}
length += current;
}
catch { break; }
} while (is_file_changed == false && _current_io_file_flag == 0 && _io_thread_flags == 0 && current > 0 && fs.CanRead);
}
catch (Exception ex)
{
Tracer.GlobalTracer.TraceError(ex);
}
finally
{
try
{
if (fs != null)
{
fs.Close();
fs.Dispose();
}
}
catch { }
if ((i != 3 && length == origin_file_length) || (i == 3 && length == Math.Min(BaiduPCS.VALIDATE_SIZE, origin_file_length)))
{
Interlocked.Add(ref io_state, 1 << i);
}
else if (i != 4)
{
Tracer.GlobalTracer.TraceWarning("invalid io state detected in " + str[i] + " calculation");
}
//debug test
sw.Stop();
//Tracer.GlobalTracer.TraceInfo(str[i] + " calculation finished (" + sw.ElapsedMilliseconds + " ms ellapsed)");
}
});
//handling file changed
if (is_file_changed)
{
try { LocalFileIOAbort?.Invoke(next_path); } catch { }
continue;
}
//handling io finish state
#region handling finish state
if (io_state == 0xf)
{
//handling file data
var temp_zero_bytes = new byte[0];
md5_calc.TransformFinalBlock(temp_zero_bytes, 0, 0);
sha1_calc.TransformFinalBlock(temp_zero_bytes, 0, 0);
md5_slice_calc.TransformFinalBlock(temp_zero_bytes, 0, 0);
var local_data = new LocalFileData();
local_data.CRC32 = crc32_calc.Hash;
local_data.CTime = fileinfo.CreationTime;
local_data.MD5 = util.Hex(md5_calc.Hash);
local_data.MTime = fileinfo.LastWriteTime;
local_data.Path = next_path;
local_data.Path_SHA1 = util.Hex(path_sha1);
local_data.SHA1 = util.Hex(sha1_calc.Hash);
local_data.Size = fileinfo.Length;
local_data.Slice_MD5 = util.Hex(md5_slice_calc.Hash);
//writing to sql
lock (_sql_lock)
{
_sql_cmd.CommandText = "select count(*) from FileList where Path_SHA1 = @Path_SHA1";
_sql_cmd.Parameters.Add("@Path_SHA1", System.Data.DbType.Binary);
_sql_cmd.Parameters["@Path_SHA1"].Value = path_sha1;
_sql_cmd.Parameters.Add("@CRC32", System.Data.DbType.Int32);
_sql_cmd.Parameters["@CRC32"].Value = (int)local_data.CRC32;
_sql_cmd.Parameters.Add("@CTime", System.Data.DbType.Int64);
_sql_cmd.Parameters["@CTime"].Value = (long)util.ToUnixTimestamp(local_data.CTime);
_sql_cmd.Parameters.Add("@MD5", System.Data.DbType.Binary);
_sql_cmd.Parameters["@MD5"].Value = md5_calc.Hash;
_sql_cmd.Parameters.Add("@MTime", System.Data.DbType.Int64);
_sql_cmd.Parameters["@MTime"].Value = (long)util.ToUnixTimestamp(local_data.MTime);
_sql_cmd.Parameters.Add("@Path", System.Data.DbType.String);
_sql_cmd.Parameters["@Path"].Value = next_path;
_sql_cmd.Parameters.Add("@SHA1", System.Data.DbType.Binary);
_sql_cmd.Parameters["@SHA1"].Value = sha1_calc.Hash;
_sql_cmd.Parameters.Add("@Slice_MD5", System.Data.DbType.Binary);
_sql_cmd.Parameters["@Slice_MD5"].Value = md5_slice_calc.Hash;
_sql_cmd.Parameters.Add("@Size", System.Data.DbType.Int64);
_sql_cmd.Parameters["@Size"].Value = local_data.Size;
var count = Convert.ToInt32(_sql_cmd.ExecuteScalar());
if (count == 0)
{
_sql_cmd.CommandText = "insert into FileList(Path, Path_SHA1, CRC32, CTime, MD5, MTime, SHA1, Slice_MD5, Size) values (@Path, @Path_SHA1, @CRC32, @CTime, @MD5, @MTime, @SHA1, @Slice_MD5, @Size)";
}
else
{
_sql_cmd.CommandText = "update FileList set Path = @Path, CRC32 = @CRC32, CTime = @CTime, MD5 = @MD5, MTime = @MTime, SHA1 = @SHA1, Slice_MD5 = @Slice_MD5, Size = @Size where Path_SHA1 = @Path_SHA1";
}
_sql_cmd.ExecuteNonQuery();
_sql_cmd.CommandText = "delete from FileIOState where Path_SHA1 = @Path_SHA1";
_sql_cmd.ExecuteNonQuery();
_sql_cmd.Parameters.Clear();
}
try { LocalFileIOFinish?.Invoke(local_data); }
catch { }
}
#endregion
//handling thread flags
#region handling unfinished state
if (_io_thread_flags == _IO_THREAD_ABORT_REQUEST || _current_io_file_flag == 1)
{
//handling non-completed io calculation
if (io_state != 0xf)
{
lock (_sql_lock)
{
_sql_cmd.CommandText = "select count(*) from FileIOState where Path_SHA1 = @Path_SHA1";
_sql_cmd.Parameters.Add("@Path_SHA1", System.Data.DbType.Binary);
_sql_cmd.Parameters["@Path_SHA1"].Value = path_sha1;
_sql_cmd.Parameters.Add("@Path", System.Data.DbType.String);
_sql_cmd.Parameters["@Path"].Value = next_path;
var ms_md5 = new MemoryStream();
md5_calc.Serialize(ms_md5);
ms_md5.Position = 0;
var ms_sha1 = new MemoryStream();
sha1_calc.Serialize(ms_sha1);
ms_sha1.Position = 0;
var ms_crc32 = new MemoryStream();
crc32_calc.Serialize(ms_crc32);
ms_crc32.Position = 0;
var ms_md5_slice = new MemoryStream();
md5_slice_calc.Serialize(ms_md5_slice);
ms_md5_slice.Position = 0;
_sql_cmd.Parameters.Add("@MD5", System.Data.DbType.Binary);
_sql_cmd.Parameters["@MD5"].Value = util.ReadBytes(ms_md5, (int)ms_md5.Length);
_sql_cmd.Parameters.Add("@SHA1", System.Data.DbType.Binary);
_sql_cmd.Parameters["@SHA1"].Value = util.ReadBytes(ms_sha1, (int)ms_sha1.Length);
_sql_cmd.Parameters.Add("@CRC32", System.Data.DbType.Binary);
_sql_cmd.Parameters["@CRC32"].Value = util.ReadBytes(ms_crc32, (int)ms_crc32.Length);
_sql_cmd.Parameters.Add("@MD5_Slice", System.Data.DbType.Binary);
_sql_cmd.Parameters["@MD5_Slice"].Value = util.ReadBytes(ms_md5_slice, (int)ms_md5_slice.Length);
var count = Convert.ToInt32(_sql_cmd.ExecuteScalar());
if (count == 0)
{
_sql_cmd.CommandText = "insert into FileIOState(Path, Path_SHA1, MD5_Serialized, SHA1_Serialized, CRC32_Serialized, MD5_Slice_Serialized) values(@Path, @Path_SHA1, @MD5, @SHA1, @CRC32, @MD5_Slice)";
}
else
{
_sql_cmd.CommandText = "update FileIOState set Path = @Path, MD5_Serialized = @MD5, SHA1_Serialized = @SHA1, CRC32_Serialized = @CRC32, MD5_Slice_Serialized = @MD5_Slice where Path_SHA1 = @Path_SHA1";
}
_sql_cmd.ExecuteNonQuery();
_sql_cmd.Parameters.Clear();
}
try { LocalFileIOAbort?.Invoke(next_path); }
catch { }
}
//commiting sql data
lock (_sql_lock)
{
_sql_trs.Commit();
_sql_trs = _sql_con.BeginTransaction();
}
_io_thread_flags = _IO_THREAD_ABORTED;
return;
}
#endregion
fs_watcher.EnableRaisingEvents = false;
fs_watcher.Dispose();
fs_watcher = null;
}
catch (Exception ex)
{
Tracer.GlobalTracer.TraceError(ex);
}
finally
{
if (fs_watcher != null)
{
fs_watcher.Dispose();
fs_watcher = null;
}
}
}
}
/// <summary>
/// Encodes the binary data in the specified stream into yEnc-encoded text
/// using the specified character encoding.
/// </summary>
/// <param name="header">The yEnc header.</param>
/// <param name="stream">The stream containing the binary data to encode.</param>
/// <param name="encoding">The character encoding to use.</param>
/// <returns>The yEnc-encoded text.</returns>
public static IEnumerable <string> Encode(YencHeader header, Stream stream, Encoding encoding)
{
yield return(GetHeaderLine(header));
if (header.IsFilePart)
{
yield return(GetPartHeaderLine(header));
}
var encodedBytes = new byte[1024];
var encodedOffset = 0;
int lastCol = header.LineLength - 1;
uint checksum = Crc32.Initialize();
for (var offset = 0; offset < header.PartSize; offset++)
{
int @byte = stream.ReadByte();
if (@byte == -1)
{
// end of stream
break;
}
checksum = Crc32.Calculate(checksum, @byte);
int val = (@byte + 42) % 256;
// encode dot only in first column
bool encodeDot = encodedOffset == 0;
// encode white space only in first and last column
bool encodeWhitespace = encodedOffset == 0 || encodedOffset == lastCol;
// encode critical characters
if (val == YencCharacters.Null ||
val == YencCharacters.Lf ||
val == YencCharacters.Cr ||
val == YencCharacters.Equal ||
val == YencCharacters.Dot && encodeDot ||
val == YencCharacters.Space && encodeWhitespace ||
val == YencCharacters.Tab && encodeWhitespace)
{
encodedBytes[encodedOffset++] = YencCharacters.Equal;
val = (val + 64) % 256;
}
encodedBytes[encodedOffset++] = (byte)val;
if (encodedOffset < header.LineLength)
{
continue;
}
// return encoded line
yield return(encoding.GetString(encodedBytes, 0, encodedOffset));
// reset offset
encodedOffset = 0;
}
if (encodedOffset > 0)
{
// return remainder
yield return(encoding.GetString(encodedBytes, 0, encodedOffset));
}
checksum = Crc32.Finalize(checksum);
yield return(GetFooterLine(header, checksum));
}
