public static void processDB(string filename)
{
try {
Dictionary <int, Action> inserts = new Dictionary <int, Action>();
HashSet <int> discussionsIds = new HashSet <int>();
using (StreamReader reader = new StreamReader(filename)) {
int i = 0;
while (!reader.EndOfStream)
{
string line = reader.ReadLine().Trim();
if (line == "")
{
continue;
}
if (i % 1000 == 0)
{
System.Console.Write("[" + (int)(i / 1000) + "]");
}
Dictionary <string, string> data;
try {
data = DictionaryConverter.FromDump(line);
} catch (Exception e) {
System.Console.Error.WriteLine("Error while trying to parse line: " + e.GetType().FullName + ": " + e.Message);
System.Console.Error.WriteLine(e.StackTrace);
continue;
}
int postId = int.Parse(data["Number"]);
try {
if (inserts.ContainsKey(postId))
{
System.Console.Write("-");
}
else if (Config.instance.mainConnection.GetCountByConditions(Post.TableSpec.instance, new ComparisonCondition(Post.TableSpec.instance.getIdSpec(), ComparisonType.EQUAL, postId.ToString())) > 0)
{
/* Post post = Post.LoadById(postId);
* if(post.title.StartsWith("%") || post.title.StartsWith("Re%3A") || post.body.StartsWith("%") || (post.thread.firstPost.id == post.id && post.thread.title.StartsWith("%"))) {
* string title = data["Subject"];
* string body = data["Body"];
* inserts[postId] = () => {
* List<AbstractChange> changes = new List<AbstractChange> {
* new UpdateChange(
* Post.TableSpec.instance,
* new Dictionary<string, AbstractFieldValue> {
* { Post.TableSpec.FIELD_TITLE, new ScalarFieldValue(title) },
* { Post.TableSpec.FIELD_BODY, new ScalarFieldValue(body) },
* },
* post.id
* )
* };
* if(post.thread.firstPost.id == post.id) {
* changes.Add(
* new UpdateChange(
* Thread.TableSpec.instance,
* new Dictionary<string, AbstractFieldValue> {
* { Thread.TableSpec.FIELD_TITLE, new ScalarFieldValue(title) },
* },
* post.thread.id
* )
* );
* }
*
* ChangeSetUtil.ApplyChanges(changes.ToArray());
* };
* Console.Write("+");
* } else {
* Console.Write("-");
* }*/
System.Console.Write("-");
}
else
{
int localMain = int.Parse(data["Local_Main"]);
int main = int.Parse(data["Main"]);
int UnixTime;
try {
UnixTime = int.Parse(data["UnixTime"]);
} catch (OverflowException) {
UnixTime = 1000 * 1000 * 1000;
}
if (UnixTime <= 0)
{
UnixTime = 1000 * 1000 * 1000;
}
DateTime date = UNIX.AddSeconds(UnixTime).ToLocalTime();
User user;
string username = data["Username"];
try {
user = User.LoadByName(username);
} catch (NotFoundInDBException) {
System.Console.Error.WriteLine("Cannot find user '" + username + "'; creating one...");
ChangeSetUtil.ApplyChanges(
new InsertChange(
User.TableSpec.instance,
new Dictionary <string, AbstractFieldValue> {
{ User.TableSpec.FIELD_NAME, new ScalarFieldValue(username) },
{ User.TableSpec.FIELD_REGDATE, new ScalarFieldValue(date.ToUTCString()) },
{ User.TableSpec.FIELD_SHOWPOSTSTOUSERS, new ScalarFieldValue(User.ENUM_SHOWPOSTSTOUSERS_ALL) },
{ User.TableSpec.FIELD_BIOGRAPHY, new ScalarFieldValue("") },
{ User.TableSpec.FIELD_BIOGRAPHYUBB, new ScalarFieldValue("") },
{ User.TableSpec.FIELD_LOCATION, new ScalarFieldValue("") },
{ User.TableSpec.FIELD_SIGNATURE, new ScalarFieldValue("") },
{ User.TableSpec.FIELD_SIGNATUREUBB, new ScalarFieldValue("") },
{ User.TableSpec.FIELD_TITLE, new ScalarFieldValue("") },
{ User.TableSpec.FIELD_TOTALPOSTS, new ScalarFieldValue("0") },
{ User.TableSpec.FIELD_USERGROUPID, new ScalarFieldValue("1") },
}
)
);
user = User.LoadByName(data["Username"]);
}
string title = data["Subject"];
string body = data["Body"];
PostLayer layer = PostLayer.LoadById(1);
if (data.ContainsKey("Layer"))
{
layer = PostLayer.LoadById(int.Parse(data["Layer"]));
}
inserts[postId] = () => {
bool isDiscussion = false;
if (postId == main || postId == localMain)
{
//first post in the thread
string legacyBoardName;
if (discussions.ContainsKey(main) || (localMain != 0 && (localMain != postId || localMain != main)))
{
discussionsIds.Add(main);
legacyBoardName = discussions[main];
isDiscussion = true;
}
else
{
legacyBoardName = data["Board"];
}
Board board;
try {
board = Board.LoadByLegacyName(legacyBoardName);
} catch (NotFoundInDBException) {
throw new ApplicationException("Cannot find board '" + legacyBoardName + "'");
}
board.CreateThread(user, title, body, layer, date, postId);
}
else
{
int parentId = int.Parse(data["Parent"]);
if (parentId == 0)
{
parentId = localMain;
if (parentId == 0)
{
parentId = main;
}
}
Post parent;
try {
parent = Post.LoadById(parentId);
} catch (NotFoundInDBException) {
throw new ApplicationException("Cannot find parent post #" + parentId);
}
if (!isDiscussion && parent.thread.firstPostId != localMain)
{
System.Console.Write("d");
}
else
{
parent.Reply(user, title, body, layer, date, postId);
}
}
};
System.Console.Write("+");
}
} catch (Exception e) {
System.Console.Error.WriteLine("Cannot process post #" + postId + ": " + e.GetType().FullName + ": " + e.Message);
System.Console.Error.WriteLine(e.StackTrace);
System.Console.Write("!");
// Console.ReadLine();
} finally {
i++;
if ((i % 50000) == 0)
{
Web.Core.RegistryCleaner.CleanRegistry <int, Post>();
Web.Core.RegistryCleaner.CleanRegistry <int, Thread>();
GC.Collect();
System.Console.Error.WriteLine();
System.Console.Error.WriteLine("Registry cleaned; garbage collected");
System.Console.Error.WriteLine();
}
}
}
}
System.Console.WriteLine();
System.Console.WriteLine("Finished parsing");
System.Console.WriteLine("Total inserts: " + inserts.Count);
System.Console.ReadLine();
int j = 0;
foreach (var insert in inserts)
{
if (j % 1000 == 0)
{
System.Console.Write("[" + (int)(j / 1000) + "]");
}
try {
insert.Value();
System.Console.Write("+");
} catch (Exception e) {
System.Console.Error.WriteLine("Cannot process post #" + insert.Key + ": " + e.GetType().FullName + ": " + e.Message);
System.Console.Error.WriteLine(e.StackTrace);
System.Console.Write("!");
// Console.ReadLine();
} finally {
j++;
}
}
System.Console.WriteLine("Not found discussions:");
foreach (int discussionId in discussionsIds.OrderBy(id => id))
{
System.Console.WriteLine(discussionId);
}
} catch (Exception e) {
System.Console.Error.WriteLine(e.GetType().FullName + ": " + e.Message);
System.Console.Error.WriteLine(e.StackTrace);
}
}