public override async Task Generate(IEnumerable <IEnumerable <char> > tableOfTableOfChars, MD5 hasherMD5, SHA256 hasherSHA256, Func <string, string, string, long, long, bool> shouldBreakFunc, Stopwatch stopwatch = null, int batchInsertCount = 200, int batchTransactionCommitCount = 20000) { string last_key_entry = await GetLastKeyEntry(); double?nextPause = null; if (stopwatch != null) { stopwatch.Start(); nextPause = stopwatch.Elapsed.TotalMilliseconds + 1000; //next check after 1sec } long counter = 0, last_pause_counter = 0, tps = 0; var tran = await Conn.BeginTransactionAsync(IsolationLevel.ReadUncommitted); string insert_into = "INSERT INTO Hashes(`Key`, hashMD5, hashSHA256) VALUES"; MySqlCommand cmd = new MySqlCommand("", Conn, tran); cmd.CommandType = CommandType.Text; cmd.CommandText = insert_into; string comma = ""; int param_counter = 0; foreach (var chars_table in tableOfTableOfChars) { var key = string.Concat(chars_table); if (!string.IsNullOrEmpty(last_key_entry) && last_key_entry.CompareTo(key) >= 0) { continue; } var(hashMD5, hashSHA256) = CalculateTwoHashes(hasherMD5, hasherSHA256, key); //dbase.Insert(value, hash); cmd.CommandText += $"{comma}(@Key{param_counter}, @hashMD5{param_counter}, @hashSHA256{param_counter}){Environment.NewLine}"; comma = ","; MySqlParameter param; if (cmd.Parameters.Contains($"@Key{param_counter}")) { param = cmd.Parameters[$"@Key{param_counter}"]; param.Value = key; } else { param = new MySqlParameter(); param.ParameterName = $"@Key{param_counter}"; param.DbType = DbType.String; param.Size = 20; param.Value = key; cmd.Parameters.Add(param); } if (cmd.Parameters.Contains($"@hashMD5{param_counter}")) { param = cmd.Parameters[$"@hashMD5{param_counter}"]; param.Value = hashMD5; } else { param = new MySqlParameter(); param.ParameterName = $"@hashMD5{param_counter}"; param.DbType = DbType.String; param.Size = 32; param.Value = hashMD5; cmd.Parameters.Add(param); } if (cmd.Parameters.Contains($"@hashSHA256{param_counter}")) { param = cmd.Parameters[$"@hashSHA256{param_counter}"] as MySqlParameter; param.Value = hashSHA256; } else { param = new MySqlParameter(); param.ParameterName = $"@hashSHA256{param_counter}"; param.DbType = DbType.String; param.Size = 64; param.Value = hashSHA256; cmd.Parameters.Add(param); } param_counter++; if (counter % batchInsertCount == 0) { cmd.CommandText += ";"; await cmd.ExecuteNonQueryAsync(); cmd.Parameters.Clear(); cmd.Dispose(); cmd = new MySqlCommand("", Conn, tran); cmd.CommandType = CommandType.Text; cmd.CommandText = insert_into; cmd.Connection = Conn; cmd.Transaction = tran; param_counter = 0; comma = ""; } if (counter % batchTransactionCommitCount == 0) { if (cmd != null && !string.IsNullOrEmpty(cmd.CommandText) && cmd.CommandText != insert_into) { cmd.CommandText += ";"; await cmd.ExecuteNonQueryAsync(); cmd.Parameters.Clear(); cmd.Dispose(); cmd = new MySqlCommand("", Conn, tran); cmd.CommandType = CommandType.Text; cmd.CommandText = insert_into; cmd.Connection = Conn; cmd.Transaction = tran; param_counter = 0; comma = ""; } await tran.CommitAsync(); tran.Dispose(); tran = null; //Console.WriteLine($"MD5({key})={hashMD5},SHA256({key})={hashSHA256},counter={counter},tps={tps}"); //if (Console.KeyAvailable) // break; if (shouldBreakFunc(key, hashMD5, hashSHA256, counter, tps)) { break; } cmd.Transaction = tran = await Conn.BeginTransactionAsync(IsolationLevel.ReadUncommitted); } if (stopwatch != null && stopwatch.Elapsed.TotalMilliseconds >= nextPause) { if (last_pause_counter > 0) { tps = counter - last_pause_counter; nextPause = stopwatch.Elapsed.TotalMilliseconds + 1000; } last_pause_counter = counter; } counter++; } if (cmd != null && !string.IsNullOrEmpty(cmd.CommandText) && cmd.CommandText != insert_into) { cmd.CommandText += ";"; await cmd.ExecuteNonQueryAsync(); cmd.Parameters.Clear(); cmd.Dispose(); } if (tran != null) { await tran.CommitAsync(); tran.Dispose(); } }
public override async Task Generate(IEnumerable <IEnumerable <char> > tableOfTableOfChars, MD5 hasherMD5, SHA256 hasherSHA256, Func <string, string, string, long, long, bool> shouldBreakFunc, Stopwatch stopwatch = null, int batchInsertCount = 50, int batchTransactionCommitCount = 200) { string last_key_entry = await GetLastKeyEntry(); double?nextPause = null; if (stopwatch != null) { stopwatch.Start(); nextPause = stopwatch.Elapsed.TotalMilliseconds + 1000; //next check after 1sec } long counter = 0, last_pause_counter = 0, tps = 0; var tran = await Conn.BeginTransactionAsync() as SqliteTransaction; SqliteCommand cmd = new SqliteCommand("", Conn, tran); cmd.CommandType = CommandType.Text; int param_counter = 0; foreach (var chars_table in tableOfTableOfChars) { var key = string.Concat(chars_table); if (!string.IsNullOrEmpty(last_key_entry) && last_key_entry.CompareTo(key) >= 0) { continue; } var(hashMD5, hashSHA256) = CalculateTwoHashes(hasherMD5, hasherSHA256, key); //dbase.Insert(value, hash); cmd.CommandText += $"insert into hashes([key], hashMD5, hashSHA256)" + $"values(@key{param_counter}, @MD5{param_counter}, @SHA256{param_counter});{Environment.NewLine}"; SqliteParameter param; if (cmd.Parameters.Contains($"@key{param_counter}")) { param = cmd.Parameters[$"@key{param_counter}"]; param.Value = key; } else { param = new SqliteParameter($"@key{param_counter}", SqliteType.Text, 20); param.Value = key; cmd.Parameters.Add(param); } if (cmd.Parameters.Contains($"@MD5{param_counter}")) { param = cmd.Parameters[$"@MD5{param_counter}"]; param.Value = hashMD5; } else { param = new SqliteParameter($"@MD5{param_counter}", SqliteType.Text, 32); param.Value = hashMD5; cmd.Parameters.Add(param); } if (cmd.Parameters.Contains($"@SHA256{param_counter}")) { param = cmd.Parameters[$"@SHA256{param_counter}"]; param.Value = hashSHA256; } else { param = new SqliteParameter($"@SHA256{param_counter}", SqliteType.Text, 64); param.Value = hashSHA256; cmd.Parameters.Add(param); } param_counter++; if (counter % batchInsertCount == 0) { //cmd.Prepare(); await cmd.ExecuteNonQueryAsync(); cmd.Parameters.Clear(); cmd.Dispose(); cmd.CommandText = ""; cmd.Connection = Conn; cmd.Transaction = tran; param_counter = 0; } if (counter % batchTransactionCommitCount == 0) { if (cmd != null && !string.IsNullOrEmpty(cmd.CommandText)) { //cmd.Prepare(); await cmd.ExecuteNonQueryAsync(); cmd.Parameters.Clear(); cmd.Dispose(); cmd.CommandText = ""; cmd.Connection = Conn; cmd.Transaction = tran; param_counter = 0; } await tran.CommitAsync(); tran.Dispose(); tran = null; //Console.WriteLine($"MD5({key})={hashMD5},SHA256({key})={hashSHA256},counter={counter},tps={tps}"); //if (Console.KeyAvailable) // break; if (shouldBreakFunc(key, hashMD5, hashSHA256, counter, tps)) { break; } cmd.Transaction = tran = await Conn.BeginTransactionAsync() as SqliteTransaction; } if (stopwatch != null && stopwatch.Elapsed.TotalMilliseconds >= nextPause) { if (last_pause_counter > 0) { tps = counter - last_pause_counter; nextPause = stopwatch.Elapsed.TotalMilliseconds + 1000; } last_pause_counter = counter; } counter++; } if (cmd != null && !string.IsNullOrEmpty(cmd.CommandText)) { //cmd.Prepare(); await cmd.ExecuteNonQueryAsync(); cmd.Parameters.Clear(); cmd.Dispose(); } if (tran != null) { await tran.CommitAsync(); tran.Dispose(); } }
public override async Task Generate(IEnumerable <IEnumerable <char> > tableOfTableOfChars, MD5 hasherMD5, SHA256 hasherSHA256, Func <string, string, string, long, long, bool> shouldBreakFunc, Stopwatch stopwatch = null, int batchInsertCount = 200, int batchTransactionCommitCount = 20_000) { string last_key_entry = await GetLastKeyEntry(); double?nextPause = null; if (stopwatch != null) { stopwatch.Start(); nextPause = stopwatch.Elapsed.TotalMilliseconds + 1_000; //next check after 1sec } long counter = 0, last_pause_counter = 0, tps = 0; var tran = await Conn.BeginTransactionAsync(IsolationLevel.ReadCommitted) as OracleTransaction; OracleCommand cmd = new OracleCommand(@" CREATE PRIVATE TEMPORARY TABLE ORA$PTT_hashes ( ""Key"" VARCHAR2(20 BYTE), ""hashMD5"" CHAR(32 BYTE), ""hashSHA256"" CHAR(64 BYTE) ) ON COMMIT PRESERVE DEFINITION", Conn) { CommandType = CommandType.Text, Transaction = tran }; await cmd.ExecuteNonQueryAsync(); cmd = new OracleCommand("", Conn) { CommandType = CommandType.Text, Transaction = tran }; int param_counter = 0; foreach (var chars_table in tableOfTableOfChars) { var key = string.Concat(chars_table); if (!string.IsNullOrEmpty(last_key_entry) && last_key_entry.CompareTo(key) >= 0) { continue; } var(hashMD5, hashSHA256) = CalculateTwoHashes(hasherMD5, hasherSHA256, key); //dbase.Insert(value, hash); cmd.CommandText += $"INSERT /*+APPEND*/ INTO ORA$PTT_hashes(\"Key\", \"hashMD5\", \"hashSHA256\")" + $" VALUES(:key{param_counter}, :hashMD5{param_counter}, :hashSHA256{param_counter});{Environment.NewLine}"; OracleParameter param; if (cmd.Parameters.Contains($"key{param_counter}")) { param = cmd.Parameters[$"key{param_counter}"]; param.Value = key; } else { param = new OracleParameter($"key{param_counter}", OracleDbType.Varchar2, 20); param.Value = key; cmd.Parameters.Add(param); } if (cmd.Parameters.Contains($"hashMD5{param_counter}")) { param = cmd.Parameters[$"hashMD5{param_counter}"]; param.Value = hashMD5; } else { param = new OracleParameter($"hashMD5{param_counter}", OracleDbType.Char, 32); param.Value = hashMD5; cmd.Parameters.Add(param); } if (cmd.Parameters.Contains($"hashSHA256{param_counter}")) { param = cmd.Parameters[$"hashSHA256{param_counter}"]; param.Value = hashSHA256; } else { param = new OracleParameter($"hashSHA256{param_counter}", OracleDbType.Char, 64); param.Value = hashSHA256; cmd.Parameters.Add(param); } param_counter++; if (counter % batchInsertCount == 0) { cmd.CommandText = $@"BEGIN {cmd.CommandText} INSERT /*+APPEND*/ INTO ""Hashes"" SELECT ""Key"", ""hashMD5"", ""hashSHA256"" FROM ORA$PTT_hashes ORDER BY ""Key""; COMMIT; DELETE FROM ORA$PTT_hashes; END;"; await cmd.ExecuteNonQueryAsync(); cmd.Parameters.Clear(); cmd.Dispose(); cmd = new OracleCommand("", Conn) { CommandType = CommandType.Text, Transaction = tran }; param_counter = 0; } if (counter % batchTransactionCommitCount == 0) { if (cmd != null && !string.IsNullOrEmpty(cmd.CommandText)) { //cmd.Prepare(); cmd.CommandText = $@"BEGIN {cmd.CommandText} INSERT /*+APPEND*/ INTO ""Hashes"" SELECT ""Key"", ""hashMD5"", ""hashSHA256"" FROM ORA$PTT_hashes ORDER BY ""Key""; COMMIT; DELETE FROM ORA$PTT_hashes; END;"; await cmd.ExecuteNonQueryAsync(); cmd.Parameters.Clear(); cmd.Dispose(); cmd = new OracleCommand("", Conn) { CommandType = CommandType.Text, Transaction = tran }; param_counter = 0; } await tran.CommitAsync(); tran.Dispose(); tran = null; //Console.WriteLine($"MD5({key})={hashMD5},SHA256({key})={hashSHA256},counter={counter},tps={tps}"); //if (Console.KeyAvailable) // break; if (shouldBreakFunc(key, hashMD5, hashSHA256, counter, tps)) { break; } cmd.Transaction = tran = await Conn.BeginTransactionAsync(IsolationLevel.ReadCommitted) as OracleTransaction; } if (stopwatch != null && stopwatch.Elapsed.TotalMilliseconds >= nextPause) { if (last_pause_counter > 0) { tps = counter - last_pause_counter; nextPause = stopwatch.Elapsed.TotalMilliseconds + 1000; } last_pause_counter = counter; } counter++; } if (cmd != null && !string.IsNullOrEmpty(cmd.CommandText)) { cmd.CommandText = $@"BEGIN {cmd.CommandText} INSERT /*+APPEND*/ INTO ""Hashes"" SELECT ""Key"", ""hashMD5"", ""hashSHA256"" FROM ORA$PTT_hashes ORDER BY ""Key""; COMMIT; DELETE FROM ORA$PTT_hashes; END;"; await cmd.ExecuteNonQueryAsync(); cmd.Parameters.Clear(); cmd.Dispose(); } if (tran != null) { await tran.CommitAsync(); tran.Dispose(); } }