public override async Task Generate(IEnumerable <IEnumerable <char> > tableOfTableOfChars, MD5 hasherMD5, SHA256 hasherSHA256,
Func <string, string, string, long, long, bool> shouldBreakFunc, Stopwatch stopwatch = null,
int batchInsertCount = 200, int batchTransactionCommitCount = 20000)
{
string last_key_entry = await GetLastKeyEntry();
double?nextPause = null;
if (stopwatch != null)
{
stopwatch.Start();
nextPause = stopwatch.Elapsed.TotalMilliseconds + 1000; //next check after 1sec
}
long counter = 0, last_pause_counter = 0, tps = 0;
var tran = await Conn.BeginTransactionAsync(IsolationLevel.ReadUncommitted);
string insert_into = "INSERT INTO Hashes(`Key`, hashMD5, hashSHA256) VALUES";
MySqlCommand cmd = new MySqlCommand("", Conn, tran);
cmd.CommandType = CommandType.Text;
cmd.CommandText = insert_into;
string comma = "";
int param_counter = 0;
foreach (var chars_table in tableOfTableOfChars)
{
var key = string.Concat(chars_table);
if (!string.IsNullOrEmpty(last_key_entry) && last_key_entry.CompareTo(key) >= 0)
{
continue;
}
var(hashMD5, hashSHA256) = CalculateTwoHashes(hasherMD5, hasherSHA256, key);
//dbase.Insert(value, hash);
cmd.CommandText += $"{comma}(@Key{param_counter}, @hashMD5{param_counter}, @hashSHA256{param_counter}){Environment.NewLine}";
comma = ",";
MySqlParameter param;
if (cmd.Parameters.Contains($"@Key{param_counter}"))
{
param = cmd.Parameters[$"@Key{param_counter}"];
param.Value = key;
}
else
{
param = new MySqlParameter();
param.ParameterName = $"@Key{param_counter}";
param.DbType = DbType.String;
param.Size = 20;
param.Value = key;
cmd.Parameters.Add(param);
}
if (cmd.Parameters.Contains($"@hashMD5{param_counter}"))
{
param = cmd.Parameters[$"@hashMD5{param_counter}"];
param.Value = hashMD5;
}
else
{
param = new MySqlParameter();
param.ParameterName = $"@hashMD5{param_counter}";
param.DbType = DbType.String;
param.Size = 32;
param.Value = hashMD5;
cmd.Parameters.Add(param);
}
if (cmd.Parameters.Contains($"@hashSHA256{param_counter}"))
{
param = cmd.Parameters[$"@hashSHA256{param_counter}"] as MySqlParameter;
param.Value = hashSHA256;
}
else
{
param = new MySqlParameter();
param.ParameterName = $"@hashSHA256{param_counter}";
param.DbType = DbType.String;
param.Size = 64;
param.Value = hashSHA256;
cmd.Parameters.Add(param);
}
param_counter++;
if (counter % batchInsertCount == 0)
{
cmd.CommandText += ";";
await cmd.ExecuteNonQueryAsync();
cmd.Parameters.Clear();
cmd.Dispose();
cmd = new MySqlCommand("", Conn, tran);
cmd.CommandType = CommandType.Text;
cmd.CommandText = insert_into;
cmd.Connection = Conn;
cmd.Transaction = tran;
param_counter = 0;
comma = "";
}
if (counter % batchTransactionCommitCount == 0)
{
if (cmd != null && !string.IsNullOrEmpty(cmd.CommandText) && cmd.CommandText != insert_into)
{
cmd.CommandText += ";";
await cmd.ExecuteNonQueryAsync();
cmd.Parameters.Clear();
cmd.Dispose();
cmd = new MySqlCommand("", Conn, tran);
cmd.CommandType = CommandType.Text;
cmd.CommandText = insert_into;
cmd.Connection = Conn;
cmd.Transaction = tran;
param_counter = 0;
comma = "";
}
await tran.CommitAsync(); tran.Dispose();
tran = null;
//Console.WriteLine($"MD5({key})={hashMD5},SHA256({key})={hashSHA256},counter={counter},tps={tps}");
//if (Console.KeyAvailable)
// break;
if (shouldBreakFunc(key, hashMD5, hashSHA256, counter, tps))
{
break;
}
cmd.Transaction = tran = await Conn.BeginTransactionAsync(IsolationLevel.ReadUncommitted);
}
if (stopwatch != null && stopwatch.Elapsed.TotalMilliseconds >= nextPause)
{
if (last_pause_counter > 0)
{
tps = counter - last_pause_counter;
nextPause = stopwatch.Elapsed.TotalMilliseconds + 1000;
}
last_pause_counter = counter;
}
counter++;
}
if (cmd != null && !string.IsNullOrEmpty(cmd.CommandText) && cmd.CommandText != insert_into)
{
cmd.CommandText += ";";
await cmd.ExecuteNonQueryAsync();
cmd.Parameters.Clear();
cmd.Dispose();
}
if (tran != null)
{
await tran.CommitAsync(); tran.Dispose();
}
}
public override async Task Generate(IEnumerable <IEnumerable <char> > tableOfTableOfChars, MD5 hasherMD5, SHA256 hasherSHA256,
Func <string, string, string, long, long, bool> shouldBreakFunc,
Stopwatch stopwatch = null, int batchInsertCount = 50, int batchTransactionCommitCount = 200)
{
string last_key_entry = await GetLastKeyEntry();
double?nextPause = null;
if (stopwatch != null)
{
stopwatch.Start();
nextPause = stopwatch.Elapsed.TotalMilliseconds + 1000; //next check after 1sec
}
long counter = 0, last_pause_counter = 0, tps = 0;
var tran = await Conn.BeginTransactionAsync() as SqliteTransaction;
SqliteCommand cmd = new SqliteCommand("", Conn, tran);
cmd.CommandType = CommandType.Text;
int param_counter = 0;
foreach (var chars_table in tableOfTableOfChars)
{
var key = string.Concat(chars_table);
if (!string.IsNullOrEmpty(last_key_entry) && last_key_entry.CompareTo(key) >= 0)
{
continue;
}
var(hashMD5, hashSHA256) = CalculateTwoHashes(hasherMD5, hasherSHA256, key);
//dbase.Insert(value, hash);
cmd.CommandText += $"insert into hashes([key], hashMD5, hashSHA256)" +
$"values(@key{param_counter}, @MD5{param_counter}, @SHA256{param_counter});{Environment.NewLine}";
SqliteParameter param;
if (cmd.Parameters.Contains($"@key{param_counter}"))
{
param = cmd.Parameters[$"@key{param_counter}"];
param.Value = key;
}
else
{
param = new SqliteParameter($"@key{param_counter}", SqliteType.Text, 20);
param.Value = key;
cmd.Parameters.Add(param);
}
if (cmd.Parameters.Contains($"@MD5{param_counter}"))
{
param = cmd.Parameters[$"@MD5{param_counter}"];
param.Value = hashMD5;
}
else
{
param = new SqliteParameter($"@MD5{param_counter}", SqliteType.Text, 32);
param.Value = hashMD5;
cmd.Parameters.Add(param);
}
if (cmd.Parameters.Contains($"@SHA256{param_counter}"))
{
param = cmd.Parameters[$"@SHA256{param_counter}"];
param.Value = hashSHA256;
}
else
{
param = new SqliteParameter($"@SHA256{param_counter}", SqliteType.Text, 64);
param.Value = hashSHA256;
cmd.Parameters.Add(param);
}
param_counter++;
if (counter % batchInsertCount == 0)
{
//cmd.Prepare();
await cmd.ExecuteNonQueryAsync();
cmd.Parameters.Clear();
cmd.Dispose();
cmd.CommandText = "";
cmd.Connection = Conn;
cmd.Transaction = tran;
param_counter = 0;
}
if (counter % batchTransactionCommitCount == 0)
{
if (cmd != null && !string.IsNullOrEmpty(cmd.CommandText))
{
//cmd.Prepare();
await cmd.ExecuteNonQueryAsync();
cmd.Parameters.Clear();
cmd.Dispose();
cmd.CommandText = "";
cmd.Connection = Conn;
cmd.Transaction = tran;
param_counter = 0;
}
await tran.CommitAsync(); tran.Dispose();
tran = null;
//Console.WriteLine($"MD5({key})={hashMD5},SHA256({key})={hashSHA256},counter={counter},tps={tps}");
//if (Console.KeyAvailable)
// break;
if (shouldBreakFunc(key, hashMD5, hashSHA256, counter, tps))
{
break;
}
cmd.Transaction = tran = await Conn.BeginTransactionAsync() as SqliteTransaction;
}
if (stopwatch != null && stopwatch.Elapsed.TotalMilliseconds >= nextPause)
{
if (last_pause_counter > 0)
{
tps = counter - last_pause_counter;
nextPause = stopwatch.Elapsed.TotalMilliseconds + 1000;
}
last_pause_counter = counter;
}
counter++;
}
if (cmd != null && !string.IsNullOrEmpty(cmd.CommandText))
{
//cmd.Prepare();
await cmd.ExecuteNonQueryAsync();
cmd.Parameters.Clear();
cmd.Dispose();
}
if (tran != null)
{
await tran.CommitAsync(); tran.Dispose();
}
}
public override async Task Generate(IEnumerable <IEnumerable <char> > tableOfTableOfChars, MD5 hasherMD5, SHA256 hasherSHA256,
Func <string, string, string, long, long, bool> shouldBreakFunc, Stopwatch stopwatch = null,
int batchInsertCount = 200, int batchTransactionCommitCount = 20_000)
{
string last_key_entry = await GetLastKeyEntry();
double?nextPause = null;
if (stopwatch != null)
{
stopwatch.Start();
nextPause = stopwatch.Elapsed.TotalMilliseconds + 1_000; //next check after 1sec
}
long counter = 0, last_pause_counter = 0, tps = 0;
var tran = await Conn.BeginTransactionAsync(IsolationLevel.ReadCommitted) as OracleTransaction;
OracleCommand cmd = new OracleCommand(@"
CREATE PRIVATE TEMPORARY TABLE ORA$PTT_hashes
(
""Key"" VARCHAR2(20 BYTE),
""hashMD5"" CHAR(32 BYTE),
""hashSHA256"" CHAR(64 BYTE)
)
ON COMMIT PRESERVE DEFINITION", Conn)
{
CommandType = CommandType.Text,
Transaction = tran
};
await cmd.ExecuteNonQueryAsync();
cmd = new OracleCommand("", Conn)
{
CommandType = CommandType.Text,
Transaction = tran
};
int param_counter = 0;
foreach (var chars_table in tableOfTableOfChars)
{
var key = string.Concat(chars_table);
if (!string.IsNullOrEmpty(last_key_entry) && last_key_entry.CompareTo(key) >= 0)
{
continue;
}
var(hashMD5, hashSHA256) = CalculateTwoHashes(hasherMD5, hasherSHA256, key);
//dbase.Insert(value, hash);
cmd.CommandText += $"INSERT /*+APPEND*/ INTO ORA$PTT_hashes(\"Key\", \"hashMD5\", \"hashSHA256\")" +
$" VALUES(:key{param_counter}, :hashMD5{param_counter}, :hashSHA256{param_counter});{Environment.NewLine}";
OracleParameter param;
if (cmd.Parameters.Contains($"key{param_counter}"))
{
param = cmd.Parameters[$"key{param_counter}"];
param.Value = key;
}
else
{
param = new OracleParameter($"key{param_counter}", OracleDbType.Varchar2, 20);
param.Value = key;
cmd.Parameters.Add(param);
}
if (cmd.Parameters.Contains($"hashMD5{param_counter}"))
{
param = cmd.Parameters[$"hashMD5{param_counter}"];
param.Value = hashMD5;
}
else
{
param = new OracleParameter($"hashMD5{param_counter}", OracleDbType.Char, 32);
param.Value = hashMD5;
cmd.Parameters.Add(param);
}
if (cmd.Parameters.Contains($"hashSHA256{param_counter}"))
{
param = cmd.Parameters[$"hashSHA256{param_counter}"];
param.Value = hashSHA256;
}
else
{
param = new OracleParameter($"hashSHA256{param_counter}", OracleDbType.Char, 64);
param.Value = hashSHA256;
cmd.Parameters.Add(param);
}
param_counter++;
if (counter % batchInsertCount == 0)
{
cmd.CommandText = $@"BEGIN {cmd.CommandText}
INSERT /*+APPEND*/ INTO ""Hashes""
SELECT ""Key"", ""hashMD5"", ""hashSHA256"" FROM ORA$PTT_hashes ORDER BY ""Key"";
COMMIT;
DELETE FROM ORA$PTT_hashes;
END;";
await cmd.ExecuteNonQueryAsync();
cmd.Parameters.Clear();
cmd.Dispose();
cmd = new OracleCommand("", Conn)
{
CommandType = CommandType.Text,
Transaction = tran
};
param_counter = 0;
}
if (counter % batchTransactionCommitCount == 0)
{
if (cmd != null && !string.IsNullOrEmpty(cmd.CommandText))
{
//cmd.Prepare();
cmd.CommandText = $@"BEGIN {cmd.CommandText}
INSERT /*+APPEND*/ INTO ""Hashes""
SELECT ""Key"", ""hashMD5"", ""hashSHA256"" FROM ORA$PTT_hashes ORDER BY ""Key"";
COMMIT;
DELETE FROM ORA$PTT_hashes;
END;";
await cmd.ExecuteNonQueryAsync();
cmd.Parameters.Clear();
cmd.Dispose();
cmd = new OracleCommand("", Conn)
{
CommandType = CommandType.Text,
Transaction = tran
};
param_counter = 0;
}
await tran.CommitAsync(); tran.Dispose();
tran = null;
//Console.WriteLine($"MD5({key})={hashMD5},SHA256({key})={hashSHA256},counter={counter},tps={tps}");
//if (Console.KeyAvailable)
// break;
if (shouldBreakFunc(key, hashMD5, hashSHA256, counter, tps))
{
break;
}
cmd.Transaction = tran = await Conn.BeginTransactionAsync(IsolationLevel.ReadCommitted) as OracleTransaction;
}
if (stopwatch != null && stopwatch.Elapsed.TotalMilliseconds >= nextPause)
{
if (last_pause_counter > 0)
{
tps = counter - last_pause_counter;
nextPause = stopwatch.Elapsed.TotalMilliseconds + 1000;
}
last_pause_counter = counter;
}
counter++;
}
if (cmd != null && !string.IsNullOrEmpty(cmd.CommandText))
{
cmd.CommandText = $@"BEGIN {cmd.CommandText}
INSERT /*+APPEND*/ INTO ""Hashes""
SELECT ""Key"", ""hashMD5"", ""hashSHA256"" FROM ORA$PTT_hashes ORDER BY ""Key"";
COMMIT;
DELETE FROM ORA$PTT_hashes;
END;";
await cmd.ExecuteNonQueryAsync();
cmd.Parameters.Clear();
cmd.Dispose();
}
if (tran != null)
{
await tran.CommitAsync(); tran.Dispose();
}
}