private void LoadDataFromDB()
{
using (dblpData dataDBLP = new dblpData())
{
List<dblp_author> listAuthors = dataDBLP.dblp_author.ToList();
List<dblp_conference> listConferences = dataDBLP.dblp_conference.ToList();
List<dblp_inproceedings> listInproceedings = dataDBLP.dblp_inproceedings.ToList();
listAs.Clear();
listIs.Clear();
listCs.Clear();
AuthorDBLP aDBLP;
foreach (dblp_author a in listAuthors)
{
aDBLP = new AuthorDBLP()
{
OldValue = 0,
CurrentValue = Convert.ToInt32(txtStartValue.Text),
Id = a.id,
Name = a.name,
CountInproceedings = a.count == null ? 0 : (int)a.count,
Inproceedings = new List<int>()
};
foreach (string id in a.inproceedings.TrimStart('|').Split('|'))
{
aDBLP.Inproceedings.Add(Convert.ToInt32(id));
}
aDBLP.InproceedingsID = aDBLP.Inproceedings.Aggregate("", (cur, next) => cur + "|" + next);
listAs.Add(aDBLP);
}
InproceedingsDBLP iDBLP;
foreach (dblp_inproceedings i in listInproceedings)
{
iDBLP = new InproceedingsDBLP()
{
Id = i.id,
Title = i.title,
Key = i.key,
Year = i.year,
Pages = i.pages,
Crossref = i.crossref,
Conference = i.conference,
AuthorsID = "",
ConferenceID = i.conference_id == null ? 0 : (int)i.conference_id,
CountAuthors = i.count == null ? 0 : (int)i.count,
Authors = new List<int>()
};
foreach (string id in i.authors.TrimStart('|').Split('|'))
{
iDBLP.Authors.Add(Convert.ToInt32(id));
}
iDBLP.AuthorsID = iDBLP.Authors.Aggregate("", (cur, next) => cur + "|" + next);
listIs.Add(iDBLP);
}
ConferenceDBLP cDBLP;
foreach (dblp_conference c in listConferences)
{
cDBLP = new ConferenceDBLP()
{
Id = c.id,
Name = c.name,
CountInproceedings = c.count == null ? 0 : (int)c.count,
Inproceedings = new List<int>()
};
foreach (string id in c.inproceedings.TrimStart('|').Split('|'))
{
cDBLP.Inproceedings.Add(Convert.ToInt32(id));
}
cDBLP.InproceedingsID = cDBLP.Inproceedings.Aggregate("", (cur, next) => cur + "|" + next);
listCs.Add(cDBLP);
}
}
}
private void CreateInproceedingsID()
{
this.Invoke((MethodInvoker)delegate
{
this.Cursor = Cursors.WaitCursor;
});
var sw = Stopwatch.StartNew();
int count = 0;
//var dicIns = from d in dicInproceedings.AsParallel().Where(di => di.Value.Id < 100) select d;
Parallel.ForEach(dicInproceedings, inproceeding =>
{
string[] authorsID = inproceeding.Value.AuthorsID.Split('|');
string[] authorsName = inproceeding.Value.AUTHORNAMEs.Split('|');
Parallel.For(0, authorsID.Length, i =>
{
int akey = 0;
if (int.TryParse(authorsID[i], out akey))
{
lock (lock_author)
{
if (dicAuthors.ContainsKey(akey))
{
AuthorDBLP updateAuthor = dicAuthors[akey];
updateAuthor.UpdateInproceedings(inproceeding.Key, inproceeding.Value);
}
else
{
AuthorDBLP newAuthor = new AuthorDBLP()
{
Id = dicAuthors.Count,
Author_Keys = authorsID[i],
Key = "person/hashcode/" + authorsID[i],
Name = authorsName[i],
InproceedingsID = "|" + inproceeding.Value.Id,
CountInproceedings = 1,
LineIndex = dicAuthors.Count,
FileIndex = fileAuthors.Count,
OldValue = Convert.ToInt32(txtStartValue.Text),
CurrentValue = Convert.ToInt32(txtStartValue.Text)
};
dicAuthors.Add(akey, newAuthor);
}
}
}
});
if (count++ % 100000 == 0)
{
this.Invoke((MethodInvoker)delegate
{
txtMsgLog.Text += (string.Format("{0} Inproceeding={1} - {2} - Count Authors={3}\r\n", count, inproceeding.Value.Id, inproceeding.Value.Key, inproceeding.Value.CountAuthors));
txtMsgLog.SelectionStart = txtMsgLog.Text.Length;
txtMsgLog.ScrollToCaret();
if (txtMsgLog.Text.Length > 1000)
{
txtMsgLog.Text = "";
}
});
}
});
if (chkSaveAuthor.Checked)
{
dicInproceedings.Clear();
GC.Collect();
GC.WaitForPendingFinalizers();
SaveAuthorsData();
ParseInproceedings(Convert.ToInt32(txtYearTo.Text));
}
sw.Stop();
this.Invoke((MethodInvoker)delegate
{
dgvAuthors.Invoke((MethodInvoker)delegate
{
dgvAuthors.DataSource = dicAuthors.Values.ToList();
dgvAuthors.Refresh();
});
dgvInproceedings.Invoke((MethodInvoker)delegate
{
dgvInproceedings.DataSource = dicInproceedings.Values.ToList();
dgvInproceedings.Refresh();
});
dgvConferences.Invoke((MethodInvoker)delegate
{
dgvConferences.DataSource = dicConferences.Values.ToList();
dgvConferences.Refresh();
});
txtMsgLog.Text += (string.Format("Done Authors={0}, Inproceedings={1}, Conferences={2}\r\n", dicAuthors.Count, dicInproceedings.Count, dicConferences.Count));
txtMsgLog.Text += string.Format("Total time: {0}:{1}:{2}\r\n", sw.Elapsed.Hours, sw.Elapsed.Minutes, sw.Elapsed.Seconds);
txtMsgLog.SelectionStart = txtMsgLog.Text.Length;
txtMsgLog.ScrollToCaret();
this.Cursor = Cursors.Default;
});
}
private void AssignAuthorPaperList(table_www w, ref List<AuthorDBLP> listAuthors, ref List<InproceedingsDBLP> listInDBLP,
ref List<ConferenceDBLP> listConferences)
{
List<table_inproceedings> listIns = new List<table_inproceedings>();
AuthorDBLP authorDBLP = null;
string author = w.author_keys;
var sw = Stopwatch.StartNew();
var id = w.ID;
using (dblpData dataDBLP = new dblpData())
{
try
{
if (dataDBLP.table_inproceedings.Count() > 0)
{
List<table_inproceedings> listI = dataDBLP.table_inproceedings.Where(
a => a.author_keys.Contains(author + "|") || a.author_keys.EndsWith(author)).ToList();
if (listI.Count > 0)
{
listIns = listIns.Union(listI).ToList();
var inproceedings =
from l in listIns select new InproceedingsDBLP(l.ID, l.KEY, l.TITLE, l.BOOKTITLE, Convert.ToString(l.YEAR), l.author_keys, (int)l.COUNT, 0);
var conferences =
from l in listIns.GroupBy(i => i.BOOKTITLE).Select(
g => new
{
key = g.Key,
count = g.Count(),
inproceedings = g.Aggregate(string.Empty, (x, i) => x + "|" + i.ID)
})
select new ConferenceDBLP(l.key, (int)l.count, l.inproceedings);
lock (lock_w)
{
listInDBLP = listInDBLP.Union(inproceedings).ToList();
listConferences.AddRange(conferences);
}
table_www www = dataDBLP.table_www.Where(a => a.ID == id).FirstOrDefault();
www.inproceedings_count = w.inproceedings_count = listIns.Count;
authorDBLP = new AuthorDBLP()
{
Id = id,
CountInproceedings = listIns.Count,
InproceedingsID = www.inproceedings_key = w.inproceedings_key = listIns.Aggregate("", (x, y) => x + "|" + y.ID),
CurrentValue = 0,
OldValue = 0
};
dataDBLP.SaveChanges();
listAuthors.Add(authorDBLP);
}
sw.Stop();
// running on worker thread
if (this.IsHandleCreated)
{
this.Invoke((MethodInvoker)delegate
{
txtMsg.Text += string.Format("wwwID -{0} -{1}: {2:F2}s\r\n", id, author, sw.Elapsed.TotalSeconds);
if (txtMsg.Text.Length > 10000)
txtMsg.Text = "";
});
}
else
{
Thread.CurrentThread.Abort();
}
Console.WriteLine("wwwID -{0} -{1}: {2:F2}s\r\n", id, author, sw.Elapsed.TotalSeconds);
}
}
catch (Exception ex)
{
// running on worker thread
if (this.IsHandleCreated)
{
this.Invoke((MethodInvoker)delegate
{
txtMsg.Text += string.Format("wwwID - {0} - {1} - msg:{2}\r\n", id, author, ex.StackTrace);// runs on UI thread
});
}
else
{
Thread.CurrentThread.Abort();
}
Console.WriteLine("wwwID -{0} -{1}: {2:F2}s\r\n", id, author, sw.Elapsed.TotalSeconds);
}
}
this.Invoke((MethodInvoker)delegate
{
dgvAuthors.Invoke((MethodInvoker)delegate
{
dgvAuthors.DataSource = listAs;
dgvAuthors.Refresh();
});
dgvInproceedings.Invoke((MethodInvoker)delegate
{
dgvInproceedings.DataSource = listIs;
dgvInproceedings.Refresh();
});
dgvConferences.Invoke((MethodInvoker)delegate
{
dgvConferences.DataSource = listCs;
dgvConferences.Refresh();
});
});
}
