public void AddDataRecords(IEnumerable<DataRecord> data, out int newData, out int NewTags)
{
newData = 0;
NewTags = 0;
lock (_conw)
{
if (cts.IsCancellationRequested) return;
StoppedEvent.Reset();
if (_conw.State != System.Data.ConnectionState.Open)
_conw.Open();
try
{
var ttTc = Helpers.TickCount;
long diTc = 0, mTc = 0, tTc = 0, tiTc = 0, tc = 0;
bool failed = false;
using (var tr = _conw.BeginTransaction(_writeTransactionOptions))
{
//var removed = new List<long>();
try
{
var md5 = new Dictionary<int, long>();
using (FbCommand cmd = new FbCommand(string.Empty, _conw, tr)/*, cmd1_5 = new FbCommand(string.Empty, _con, tr)*/)
{
long mc = cmd.GetGeneratorValue("GEN_MD5_INDEX");
List<string> tgs = new List<string>();
var mlMD5 = cmd.CreateParameter();
var mlRating = cmd.CreateParameter();
var mlMD5Id = cmd.CreateParameter();
mlMD5Id.Direction = System.Data.ParameterDirection.Output;
cmd.Parameters.AddRange(new FbParameter[] { mlMD5, mlRating, mlMD5Id });
cmd.CommandText = "update or insert into MD5_LIST (MD5, RATING) values (?,?) MATCHING (MD5) returning MD5_ID";
using (
FbCommand cmd2 = new FbCommand("update or insert into SERVERS_LIST (SERVER_NAME, SERVER_GROUP_ID) values (?,?) matching (SERVER_NAME) returning SERVER_ID", _conw, tr),
cmd3 = new FbCommand("update or insert into DATA_INFO (DATA_MD5_ID, DATA_SERVER_ID, DATA_POST_NUMBER, DATA_EXT_ID, DATA_SIZE, AUTOR_ID, CHILD_POST, PARENT_POST) values (?,?,?,?,?,?,?,?) matching (DATA_MD5_ID, DATA_SERVER_ID)", _conw, tr),
cmd4 = new FbCommand("update or insert into EXT_LIST (EXT) values (?) matching (EXT) returning EXT_ID", _conw, tr),
cmd5 = new FbCommand("update or insert into AUTORS (AUTOR) values (?) matching (AUTOR) returning AUTOR_ID", _conw, tr)
)
{
#region parameters
#region SERVERS_LIST
var slServerName = cmd2.CreateParameter();
var slServerGroup = cmd2.CreateParameter();
var slServerId = cmd2.CreateParameter();
slServerId.Direction = System.Data.ParameterDirection.Output;
cmd2.Parameters.AddRange(new[] { slServerName, slServerGroup, slServerId });
#endregion
#region DATA_INFO
var diMD5Id = cmd3.CreateParameter();
var diServerId = cmd3.CreateParameter();
var diPost = cmd3.CreateParameter();
var diExtId = cmd3.CreateParameter();
var diFileSize = cmd3.CreateParameter();
var diAutorId = cmd3.CreateParameter();
var diParentPost = cmd3.CreateParameter();
var diChildPost = cmd3.CreateParameter();
cmd3.Parameters.AddRange(new[] { diMD5Id, diServerId, diPost, diExtId, diFileSize, diAutorId, diChildPost, diParentPost });
#endregion
#region EXT_LIST
var elExt = cmd4.CreateParameter();
var elExtId = cmd4.CreateParameter();
elExtId.Direction = System.Data.ParameterDirection.Output;
cmd4.Parameters.AddRange(new[] { elExt, elExtId });
#endregion
#region AUTORS
var aAutor = cmd5.CreateParameter();
var aAutorId = cmd5.CreateParameter();
aAutorId.Direction = System.Data.ParameterDirection.Output;
cmd5.Parameters.AddRange(new[] { aAutor, aAutorId });
#endregion
#endregion
var autors = new Dictionary<string, int>();
var exts = new Dictionary<string, int>();
var servers = new Dictionary<string, int>();
int i = 0;
foreach (var d in data)
if (!string.IsNullOrWhiteSpace(d.MD5) && d.Servers != null)
{
i++;
mlMD5.Value = d.MD5;
mlRating.Value = (Char)d.Rating;
tc = Helpers.TickCount;
if (cmd.ExecuteNonQuery() > 0)
{
mTc += Helpers.TickCount - tc;
long id = md5[i] = (long)mlMD5Id.Value;
diMD5Id.Value = id;
if (mc < id)
newData++;
int indx = 0;
#region preparing tags list
if (d.Tags != null)
foreach (var t in d.Tags)
if (t.Length <= TAGS_LENGTH)
if (0 > (indx = tgs.BinarySearch(t)))
tgs.Insert(~indx, t);
#endregion
#region fill datainfo
tc = Helpers.TickCount;
foreach (var srv in d.Servers)
{
int si = -1;
//trying to get server id from storage
if (!servers.TryGetValue(srv.Server, out si))
{
slServerName.Value = srv.Server;
slServerGroup.Value = null;
if (cmd2.ExecuteNonQuery() > 0)
servers[srv.Server] = si = (int)slServerId.Value;
else
continue;
}
//fill subservers if exists
if (srv.subServers != null)
foreach (var ss in srv.subServers)
if (!servers.ContainsKey(ss))
{
slServerName.Value = ss;
slServerGroup.Value = si;
cmd2.ExecuteNonQuery();
servers[ss] = (int)slServerId.Value;
}
int tmpId = 0;
#region Ext data
diExtId.Value = null;
if (!string.IsNullOrWhiteSpace(srv.Ext))
{
if (exts.TryGetValue(srv.Ext, out tmpId))
diExtId.Value = tmpId;
else
{
elExt.Value = srv.Ext;
if (cmd4.ExecuteNonQuery() > 0)
{
diExtId.Value = elExtId.Value;
exts[srv.Ext] = (int)elExtId.Value;
}
}
}
#endregion
#region Autors data
diAutorId.Value = null;
if (!string.IsNullOrWhiteSpace(srv.Autor))
{
if (autors.TryGetValue(srv.Autor, out tmpId))
diAutorId.Value = tmpId;
else
{
aAutor.Value = srv.Autor;
if (cmd5.ExecuteNonQuery() > 0)
{
diAutorId.Value = aAutorId.Value;
autors[srv.Autor] = (int)aAutorId.Value;
}
}
}
#endregion
diServerId.Value = si;
diFileSize.Value = srv.Size < 0 ? null : (object)srv.Size;
diPost.Value = srv.Post < 0 ? null : (object)srv.Post;
diParentPost.Value = srv.ParentPost < 0 ? null : (object)srv.ParentPost;
diChildPost.Value = srv.ChildPost < 0 ? null : (object)srv.ChildPost;
cmd3.ExecuteNonQuery();
}
diTc += Helpers.TickCount - tc;
#endregion
}
}
}
tc = Helpers.TickCount;
var tags = AddTags(tr, tgs, out NewTags);
tTc = Helpers.TickCount - tc;
//fill tag<->md5 joining table
cmd.Parameters.Clear();
var tMD5Id = cmd.CreateParameter();
var tTagId = cmd.CreateParameter();
cmd.Parameters.AddRange(new[] { tMD5Id, tTagId });
cmd.CommandText = "update or insert into TAGS (MD5_ID, TAG_ID) values (?,?) MATCHING (MD5_ID, TAG_ID)";
mlMD5.Direction = System.Data.ParameterDirection.Input;
int j = 0;
tc = Helpers.TickCount;
foreach (var d in data)
{
j++;
if (md5.ContainsKey(j))
{
tMD5Id.Value = md5[j];
long id;
foreach (var t in d.Tags)
if (tags.TryGetValue(t, out id))
{
tTagId.Value = id;
cmd.ExecuteNonQuery();
}
}
}
tiTc = Helpers.TickCount - tc;
}
tr.Commit();
Statistics = string.Format("Total: {0}; MD5: {1}; DataInfo: {2}, Tags: {3}; MD5-Tags: {4}", Helpers.TickCount - ttTc, mTc, diTc, tTc, tiTc);
}
catch (Exception e)
{
tr.Rollback();
failed = true;
//EmptyMD5Slots.InsertRange(0, removed);
Helpers.ConsoleWrite(e.ToString(), ConsoleColor.DarkRed);
throw new DataSourceAddRecordsException();
}
}
if (failed)
ResetMD5Generator();
}
finally
{
StoppedEvent.Set();
}
}
}