예제 #1
0
        public static void processDB(string filename)
        {
            try {
                Dictionary <int, Action> inserts        = new Dictionary <int, Action>();
                HashSet <int>            discussionsIds = new HashSet <int>();
                using (StreamReader reader = new StreamReader(filename)) {
                    int i = 0;
                    while (!reader.EndOfStream)
                    {
                        string line = reader.ReadLine().Trim();
                        if (line == "")
                        {
                            continue;
                        }
                        if (i % 1000 == 0)
                        {
                            System.Console.Write("[" + (int)(i / 1000) + "]");
                        }
                        Dictionary <string, string> data;
                        try {
                            data = DictionaryConverter.FromDump(line);
                        } catch (Exception e) {
                            System.Console.Error.WriteLine("Error while trying to parse line: " + e.GetType().FullName + ": " + e.Message);
                            System.Console.Error.WriteLine(e.StackTrace);
                            continue;
                        }
                        int postId = int.Parse(data["Number"]);
                        try {
                            if (inserts.ContainsKey(postId))
                            {
                                System.Console.Write("-");
                            }
                            else if (Config.instance.mainConnection.GetCountByConditions(Post.TableSpec.instance, new ComparisonCondition(Post.TableSpec.instance.getIdSpec(), ComparisonType.EQUAL, postId.ToString())) > 0)
                            {
/*							Post post = Post.LoadById(postId);
 *                                                      if(post.title.StartsWith("%") || post.title.StartsWith("Re%3A") || post.body.StartsWith("%") || (post.thread.firstPost.id == post.id && post.thread.title.StartsWith("%"))) {
 *                                                              string title = data["Subject"];
 *                                                              string body = data["Body"];
 *                                                              inserts[postId] = () => {
 *                                                                      List<AbstractChange> changes = new List<AbstractChange> {
 *                                                                              new UpdateChange(
 *                                                                                      Post.TableSpec.instance,
 *                                                                                      new Dictionary<string, AbstractFieldValue> {
 *                                                                                              { Post.TableSpec.FIELD_TITLE, new ScalarFieldValue(title) },
 *                                                                                              { Post.TableSpec.FIELD_BODY, new ScalarFieldValue(body) },
 *                                                                                      },
 *                                                                                      post.id
 *                                                                              )
 *                                                                      };
 *                                                                      if(post.thread.firstPost.id == post.id) {
 *                                                                              changes.Add(
 *                                                                                      new UpdateChange(
 *                                                                                              Thread.TableSpec.instance,
 *                                                                                              new Dictionary<string, AbstractFieldValue> {
 *                                                                                                      { Thread.TableSpec.FIELD_TITLE, new ScalarFieldValue(title) },
 *                                                                                              },
 *                                                                                              post.thread.id
 *                                                                                      )
 *                                                                              );
 *                                                                      }
 *
 *                                                                      ChangeSetUtil.ApplyChanges(changes.ToArray());
 *                                                              };
 *                                                              Console.Write("+");
 *                                                      } else {
 *                                                              Console.Write("-");
 *                                                      }*/
                                System.Console.Write("-");
                            }
                            else
                            {
                                int localMain = int.Parse(data["Local_Main"]);
                                int main      = int.Parse(data["Main"]);
                                int UnixTime;
                                try {
                                    UnixTime = int.Parse(data["UnixTime"]);
                                } catch (OverflowException) {
                                    UnixTime = 1000 * 1000 * 1000;
                                }
                                if (UnixTime <= 0)
                                {
                                    UnixTime = 1000 * 1000 * 1000;
                                }
                                DateTime date = UNIX.AddSeconds(UnixTime).ToLocalTime();
                                User     user;
                                string   username = data["Username"];
                                try {
                                    user = User.LoadByName(username);
                                } catch (NotFoundInDBException) {
                                    System.Console.Error.WriteLine("Cannot find user '" + username + "'; creating one...");
                                    ChangeSetUtil.ApplyChanges(
                                        new InsertChange(
                                            User.TableSpec.instance,
                                            new Dictionary <string, AbstractFieldValue> {
                                        { User.TableSpec.FIELD_NAME, new ScalarFieldValue(username) },
                                        { User.TableSpec.FIELD_REGDATE, new ScalarFieldValue(date.ToUTCString()) },
                                        { User.TableSpec.FIELD_SHOWPOSTSTOUSERS, new ScalarFieldValue(User.ENUM_SHOWPOSTSTOUSERS_ALL) },
                                        { User.TableSpec.FIELD_BIOGRAPHY, new ScalarFieldValue("") },
                                        { User.TableSpec.FIELD_BIOGRAPHYUBB, new ScalarFieldValue("") },
                                        { User.TableSpec.FIELD_LOCATION, new ScalarFieldValue("") },
                                        { User.TableSpec.FIELD_SIGNATURE, new ScalarFieldValue("") },
                                        { User.TableSpec.FIELD_SIGNATUREUBB, new ScalarFieldValue("") },
                                        { User.TableSpec.FIELD_TITLE, new ScalarFieldValue("") },
                                        { User.TableSpec.FIELD_TOTALPOSTS, new ScalarFieldValue("0") },
                                        { User.TableSpec.FIELD_USERGROUPID, new ScalarFieldValue("1") },
                                    }
                                            )
                                        );
                                    user = User.LoadByName(data["Username"]);
                                }
                                string    title = data["Subject"];
                                string    body  = data["Body"];
                                PostLayer layer = PostLayer.LoadById(1);
                                if (data.ContainsKey("Layer"))
                                {
                                    layer = PostLayer.LoadById(int.Parse(data["Layer"]));
                                }
                                inserts[postId] = () => {
                                    bool isDiscussion = false;
                                    if (postId == main || postId == localMain)
                                    {
                                        //first post in the thread
                                        string legacyBoardName;
                                        if (discussions.ContainsKey(main) || (localMain != 0 && (localMain != postId || localMain != main)))
                                        {
                                            discussionsIds.Add(main);
                                            legacyBoardName = discussions[main];
                                            isDiscussion    = true;
                                        }
                                        else
                                        {
                                            legacyBoardName = data["Board"];
                                        }
                                        Board board;
                                        try {
                                            board = Board.LoadByLegacyName(legacyBoardName);
                                        } catch (NotFoundInDBException) {
                                            throw new ApplicationException("Cannot find board '" + legacyBoardName + "'");
                                        }
                                        board.CreateThread(user, title, body, layer, date, postId);
                                    }
                                    else
                                    {
                                        int parentId = int.Parse(data["Parent"]);
                                        if (parentId == 0)
                                        {
                                            parentId = localMain;
                                            if (parentId == 0)
                                            {
                                                parentId = main;
                                            }
                                        }
                                        Post parent;
                                        try {
                                            parent = Post.LoadById(parentId);
                                        } catch (NotFoundInDBException) {
                                            throw new ApplicationException("Cannot find parent post #" + parentId);
                                        }

                                        if (!isDiscussion && parent.thread.firstPostId != localMain)
                                        {
                                            System.Console.Write("d");
                                        }
                                        else
                                        {
                                            parent.Reply(user, title, body, layer, date, postId);
                                        }
                                    }
                                };
                                System.Console.Write("+");
                            }
                        } catch (Exception e) {
                            System.Console.Error.WriteLine("Cannot process post #" + postId + ": " + e.GetType().FullName + ": " + e.Message);
                            System.Console.Error.WriteLine(e.StackTrace);
                            System.Console.Write("!");
//						Console.ReadLine();
                        } finally {
                            i++;
                            if ((i % 50000) == 0)
                            {
                                Web.Core.RegistryCleaner.CleanRegistry <int, Post>();
                                Web.Core.RegistryCleaner.CleanRegistry <int, Thread>();
                                GC.Collect();
                                System.Console.Error.WriteLine();
                                System.Console.Error.WriteLine("Registry cleaned; garbage collected");
                                System.Console.Error.WriteLine();
                            }
                        }
                    }
                }

                System.Console.WriteLine();
                System.Console.WriteLine("Finished parsing");
                System.Console.WriteLine("Total inserts: " + inserts.Count);
                System.Console.ReadLine();
                int j = 0;
                foreach (var insert in inserts)
                {
                    if (j % 1000 == 0)
                    {
                        System.Console.Write("[" + (int)(j / 1000) + "]");
                    }
                    try {
                        insert.Value();
                        System.Console.Write("+");
                    } catch (Exception e) {
                        System.Console.Error.WriteLine("Cannot process post #" + insert.Key + ": " + e.GetType().FullName + ": " + e.Message);
                        System.Console.Error.WriteLine(e.StackTrace);
                        System.Console.Write("!");
//						Console.ReadLine();
                    } finally {
                        j++;
                    }
                }

                System.Console.WriteLine("Not found discussions:");
                foreach (int discussionId in discussionsIds.OrderBy(id => id))
                {
                    System.Console.WriteLine(discussionId);
                }
            } catch (Exception e) {
                System.Console.Error.WriteLine(e.GetType().FullName + ": " + e.Message);
                System.Console.Error.WriteLine(e.StackTrace);
            }
        }