// HOT COPPER POSTS, Threads, Authors... By Text file private void ThreadsByTextFile(object sender, EventArgs e) { listbox.Items.Add("[" + DateTime.Now + "] HOT COPPER by text file begins! Please wait for a few seconds."); DialogResult result = openFileDialog1.ShowDialog(); string links = ""; string returnedHtmlString = ""; if (result == DialogResult.OK) // Test result. { string file = openFileDialog1.FileName; try { links = File.ReadAllText(file); returnedHtmlString = WorkerClasses.getHtmlCodeFromFile(links); } catch (IOException) { listbox.Items.Add("ERROR: Not appropriate file selected!"); } } if (returnedHtmlString != null && returnedHtmlString.Trim() != "") { GetHotCopperThreads(returnedHtmlString); } else { MessageBox.Show("Failure in loading file"); } }
// Hot Copper Posts, Threads and Authors private void GetHotCopperThreadsFromUrl(object sender, EventArgs e) { string sourceCode = ""; url = textBox1.Text; if (url != null && url.Trim() != "") { // Retreive a source code from a webpage sourceCode = WorkerClasses.getSourceCode(url); } else { listbox.Items.Add("Please enter URL."); MessageBox.Show("Please enter URL."); } GetHotCopperThreads(sourceCode); }
public static int numOfPostsInAMonth(string authorPostLink) { try { string sourceCode = WorkerClasses.getSourceCode(authorPostLink); if (sourceCode == "invalid") { throw new UriFormatException(); } int startIndex = 0; int endIndex = 0; int count = 0; if (sourceCode.IndexOf("Next</a>") == -1) { while (sourceCode.IndexOf("listblock time") != -1) { startIndex = sourceCode.IndexOf("listblock time") + 14; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf(">") + 1; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf(">") + 1; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); endIndex = sourceCode.IndexOf("</"); string timeStr = sourceCode.Substring(0, endIndex); DateTime postTime = Convert.ToDateTime(timeStr); int postMonth = postTime.Month; int nowMonth = DateTime.Now.Month; if (postMonth == nowMonth) { count++; } } } else { while (sourceCode.IndexOf("Next</a>") != -1) { while (sourceCode.IndexOf("listblock time") != -1) { startIndex = sourceCode.IndexOf("listblock time") + 14; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf(">") + 1; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf(">") + 1; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); endIndex = sourceCode.IndexOf("</"); string timeStr = sourceCode.Substring(0, endIndex); DateTime postTime = Convert.ToDateTime(timeStr); int postMonth = postTime.Month; int nowMonth = DateTime.Now.Month; if (postMonth == nowMonth) { count++; } } endIndex = sourceCode.IndexOf("Next</a>"); sourceCode = sourceCode.Substring(0, endIndex); startIndex = sourceCode.LastIndexOf("</a>") + 4; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf("a href") + 1; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf("\"") + 1; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); endIndex = sourceCode.IndexOf("\""); string nextPageLink = "http://hotcopper.com.au/" + sourceCode.Substring(0, endIndex).Replace("&", "&"); if (nextPageLink.Contains("disabled")) { break; } else { sourceCode = WorkerClasses.getSourceCode(nextPageLink); if (sourceCode == "invalid") { throw new UriFormatException(); } } } } return(count); } catch { return(0); } }
// Positive and Negative words private void SentimentAnalysisButton(object sender, EventArgs e) { var db = new HCDB(); int duplicates = 0; int updatedData = 0; // Retreive a source code from a webpage // e.g. http://www3.nd.edu/~mcdonald/Word_Lists.html try { listbox.Items.Add("Process Starts. Please wait for a few minutes."); string word = ""; int posCount = 0; int negCount = 0; // Positive Words string positiveWordLink = "http://www3.nd.edu/~mcdonald/Data/Finance_Word_Lists/LoughranMcDonald_Positive.csv"; string positiveCode = WorkerClasses.getSourceCode(positiveWordLink).ToLower().Replace("\n", ""); if (positiveCode == "invalid") { throw new UriFormatException(); } string positiveCode_copy = positiveCode; // Negative Words string negativeWordLink = "http://www3.nd.edu/~mcdonald/Data/Finance_Word_Lists/LoughranMcDonald_Negative.csv"; string negativeCode = WorkerClasses.getSourceCode(negativeWordLink).ToLower().Replace("\n", ""); if (negativeCode == "invalid") { throw new UriFormatException(); } string negativeCode_copy = negativeCode; // Check with the article's story if a word in the list is contained in a story // if it is contained, count how many times. var postsList = (from u in db.Posts select u).ToList(); if (postsList.Count != 0) { foreach (var u in postsList) { if (u.PosWords == null && u.NegWords == null) { string content = u.Content; string[] contentSplit = content.Split(new char[] { '.', '?', '!', ' ', ';', ':', ',' }, StringSplitOptions.RemoveEmptyEntries); // Select a word (read by line) in the list while (positiveCode.IndexOf("\r") != -1) { int endIndex = positiveCode.IndexOf("\r"); word = positiveCode.Substring(0, endIndex); positiveCode = positiveCode.Substring(endIndex + 1, positiveCode.Length - endIndex - 1); if (content.Contains(word)) { var matchQuery = from words in contentSplit where words.ToLowerInvariant() == word.ToLowerInvariant() select words; posCount += matchQuery.Count(); } } while (negativeCode.IndexOf("\r") != -1) { int endIndex = negativeCode.IndexOf("\r"); word = negativeCode.Substring(0, endIndex); negativeCode = negativeCode.Substring(endIndex + 1, negativeCode.Length - endIndex - 1); if (content.Contains(word)) { var matchQuery = from words in contentSplit where words.ToLowerInvariant() == word.ToLowerInvariant() select words; negCount += matchQuery.Count(); } } u.PosWords = posCount; u.NegWords = negCount; db.SaveChanges(); updatedData++; // Reset components for next loop positiveCode = positiveCode_copy; negativeCode = negativeCode_copy; posCount = 0; negCount = 0; } else { duplicates++; } } } listbox.Items.Add("\n[" + DateTime.Now + "] HotCopper Pos/Neg Word Process Ended.\n" + updatedData + " updated and " + duplicates + " duplicates Found."); } catch (Exception ex) { if (ex.Message.Contains("UriFormatException")) { listbox.Items.Add("Invalid URL!"); MessageBox.Show("Invalid URL!"); } else { listbox.Items.Add(ex.Message); } } }
// MARKET DATA private void MarketDataButton(object sender, EventArgs e) { var db = new HCDB(); int newData = 0; // Retreive a source code from a webpage string url = textBox1.Text; // e.g. http://hotcopper.com.au/asx/anz#.VI98gSuUfJI MessageBox.Show("This runs"); if (url != null && url.Trim() != "") { try { string sourceCode = WorkerClasses.getSourceCode(url); if (sourceCode == "invalid") { throw new UriFormatException(); } listbox.Items.Add("[" + DateTime.Now + "] Process Starts. Please wait for a few minutes."); /* TAG */ string groupWord = textBox2.Text; if (groupWord == "") { groupWord = WorkerClasses.getGroupWord(url); } int startIndex = 0; int endIndex = 0; startIndex = sourceCode.IndexOf("stock-pricing"); sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); // Open startIndex = sourceCode.IndexOf("class=\"primary\""); sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf(">") + 1; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); endIndex = sourceCode.IndexOf("</"); string temp = sourceCode.Substring(0, endIndex).Replace(",", ""); MessageBox.Show("Open is currently " + temp); temp = temp.Replace("$", ""); if (temp.Contains("")) { temp = temp.Replace("¢", ""); temp = temp.Replace("M", ""); temp = Convert.ToString(Convert.ToDouble(temp) / 100); } Decimal openValue = Convert.ToDecimal(temp); // High startIndex = sourceCode.IndexOf("class=\"high\""); sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf(">") + 1; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); endIndex = sourceCode.IndexOf("</"); temp = sourceCode.Substring(0, endIndex).Replace(",", ""); temp = temp.Replace("$", ""); if (temp.Contains("")) { temp = temp.Replace("¢", ""); temp = Convert.ToString(Convert.ToDouble(temp) / 100); } Decimal highValue = Convert.ToDecimal(temp); // Low startIndex = sourceCode.IndexOf("class=\"low\""); sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf(">") + 1; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); endIndex = sourceCode.IndexOf("</"); temp = sourceCode.Substring(0, endIndex).Replace(",", ""); temp = temp.Replace("$", ""); if (temp.Contains("")) { temp = temp.Replace("¢", ""); temp = Convert.ToString(Convert.ToDouble(temp) / 100); } Decimal lowValue = Convert.ToDecimal(temp); // Value startIndex = sourceCode.IndexOf("class=\"primary\""); sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf(">") + 1; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); endIndex = sourceCode.IndexOf("</"); temp = sourceCode.Substring(0, endIndex).Replace(",", ""); temp = temp.Replace("$", ""); if (temp.Contains("")) { temp = temp.Replace("¢", ""); temp = temp.Replace("M", ""); temp = Convert.ToString(Convert.ToDouble(temp) / 100); } Decimal lastValue = Convert.ToDecimal(temp); // Volume startIndex = sourceCode.IndexOf("class=\"primary\""); sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf(">") + 1; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); endIndex = sourceCode.IndexOf("</"); temp = sourceCode.Substring(0, endIndex).Replace(",", ""); temp = temp.Replace("$", ""); if (temp.Contains("")) { temp = temp.Replace("¢", ""); temp = temp.Replace("M", ""); temp = Convert.ToString(Convert.ToDouble(temp) / 100); } Decimal marketPrice = Convert.ToDecimal(temp); /*// Volume (Millions) * startIndex = sourceCode.IndexOf("class=\"primary\""); * sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); * startIndex = sourceCode.IndexOf(">") + 1; * sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); * endIndex = sourceCode.IndexOf("</"); * temp = sourceCode.Substring(0, endIndex).Replace(",", ""); * if (temp.Contains("m") || temp.Contains("M")) * temp = temp.Replace("m", "").Replace("M", ""); * else if (temp.Contains("b") || temp.Contains("B")) * { * temp = temp.Replace("b", "").Replace("B", ""); * temp = Convert.ToString(Convert.ToDecimal(temp) * 1000); * } * else if (temp.Contains("k") || temp.Contains("K")) * { * temp = temp.Replace("k", "").Replace("K", ""); * temp = Convert.ToString(Convert.ToDecimal(temp) / 1000); * } * else * temp = Convert.ToString(Convert.ToDecimal(temp) / 1000000); * Decimal volume = Convert.ToDecimal(temp); * * // Value (Millions) * startIndex = sourceCode.IndexOf("class=\"primary\""); * sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); * startIndex = sourceCode.IndexOf("$") + 1; * sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); * endIndex = sourceCode.IndexOf("</"); * temp = sourceCode.Substring(0, endIndex).Replace(",", ""); * if (temp.Contains("m") || temp.Contains("M")) * temp = temp.Replace("m", "").Replace("M", ""); * else if (temp.Contains("b") || temp.Contains("B")) * { * temp = temp.Replace("b", "").Replace("B", ""); * temp = Convert.ToString(Convert.ToDecimal(temp) * 1000); * } * else if (temp.Contains("k") || temp.Contains("K")) * { * temp = temp.Replace("k", "").Replace("K", ""); * temp = Convert.ToString(Convert.ToDecimal(temp) / 1000); * } * else * temp = Convert.ToString(Convert.ToDecimal(temp) / 1000); * Decimal value = Convert.ToDecimal(temp); * * // Market Cap (Billions) * startIndex = sourceCode.IndexOf("class=\"primary\""); * sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); * startIndex = sourceCode.IndexOf("$") + 1; * sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); * endIndex = sourceCode.IndexOf("</"); * temp = sourceCode.Substring(0, endIndex).Replace(",", ""); * if (temp.Contains("b") || temp.Contains("B")) * temp = temp.Replace("b", "").Replace("B", ""); * else if (temp.Contains("m") || temp.Contains("M")) * { * temp = temp.Replace("m", "").Replace("M", ""); * temp = Convert.ToString(Convert.ToDecimal(temp) / 1000); * } * else if (temp.Contains("k") || temp.Contains("K")) * { * temp = temp.Replace("k", "").Replace("K", ""); * temp = Convert.ToString(Convert.ToDecimal(temp) / 1000000); * } * else * temp = Convert.ToString(Convert.ToDecimal(temp) / 1000000000); * Decimal marketCap = Convert.ToDecimal(temp); * */ db.MarketData.Add(new HCDB_MarketData { Tag = groupWord, Date = DateTime.Now, High = highValue, Low = lowValue, Open = openValue, Last = lastValue, Market_Price = marketPrice, Volume__Millions_ = 0M, Value__Millions_ = 0M, Market_Cap__Billions_ = 0M }); db.SaveChanges(); newData++; listbox.Items.Add("\n[" + DateTime.Now + "] Market Data completed.\n" + newData + " saved."); } catch (UriFormatException) { listbox.Items.Add("Invalid URL!"); MessageBox.Show("Invalid URL!"); textBox1.Text = ""; } catch (Exception ex) { listbox.Items.Add("Error found: " + ex); MessageBox.Show("Error found: " + ex); textBox1.Text = ""; } } }
private void GetHotCopperThreads(string pageSource) { var db = new HCDB(); int duplicates = 0; int newData = 0; int postNewData = 0; if (pageSource != null) { try { string stock = textBox2.Text; if (stock == null || stock.Trim() == "") { int identifier = url.IndexOf("hotcopper.com.au") + 16; if (url != "") { stock = url.Substring(identifier, url.Length - identifier); if (stock.Contains("#")) { stock = stock.Substring(0, stock.IndexOf("#")); } } } string sourceCode = pageSource; if (sourceCode == "invalid") { throw new UriFormatException(); } listbox.Items.Add("Process Starts. Please wait for a few minutes."); /* Group */ string groupWord = textBox2.Text; if (groupWord == "") { groupWord = WorkerClasses.getGroupWord(url); } #region HotCopper THREADS results only try { /* First of ALL, Save Threads Links */ while (sourceCode.IndexOf("listblock tags") != -1) { // TAG int startIndex = sourceCode.IndexOf("listblock tags"); sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); int endIndex = sourceCode.IndexOf("</div"); string relCode = sourceCode.Substring(0, endIndex); string tags = ""; while (relCode.Contains("<a href")) { startIndex = relCode.IndexOf("<a href"); relCode = relCode.Substring(startIndex, relCode.Length - startIndex); startIndex = relCode.IndexOf(">") + 1; relCode = relCode.Substring(startIndex, relCode.Length - startIndex); endIndex = relCode.IndexOf("</"); tags += " " + relCode.Substring(0, endIndex); } tags = tags.Replace("&", "&").Trim(); // Subject startIndex = sourceCode.IndexOf("listblock subject"); sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf("<h"); sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf(">") + 1; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); endIndex = sourceCode.IndexOf("</"); string subject = sourceCode.Substring(0, endIndex); string threadLink = ""; if (subject.Contains("<a")) { // Thread Link string tempCode = sourceCode; startIndex = tempCode.IndexOf("<a href"); tempCode = tempCode.Substring(startIndex, tempCode.Length - startIndex); startIndex = tempCode.IndexOf("\"") + 1; tempCode = tempCode.Substring(startIndex, tempCode.Length - startIndex); endIndex = tempCode.IndexOf("\""); threadLink = "http://hotcopper.com.au/" + tempCode.Substring(0, endIndex); // Remove "<a>" from Subject endIndex = subject.IndexOf("<a"); string firstPart = subject.Substring(0, endIndex); subject = subject.Substring(endIndex + 2, subject.Length - (endIndex + 2)); startIndex = subject.IndexOf(">") + 1; string secondPart = subject.Substring(startIndex, subject.Length - startIndex); subject = firstPart + "\n" + secondPart; } subject = subject.Replace("</a>", "").Trim(); subject = subject.Replace("'", "'"); subject = subject.Replace("&", "'"); // Author startIndex = sourceCode.IndexOf("listblock author "); sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf(">") + 1; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); endIndex = sourceCode.IndexOf("</"); string author = sourceCode.Substring(0, endIndex); // Author's Posts Link startIndex = sourceCode.IndexOf("a href"); sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf("\"") + 1; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); endIndex = sourceCode.IndexOf("\""); string authorPostLink = "https://hotcopper.com.au/" + sourceCode.Substring(0, endIndex); authorPostLink = authorPostLink.Replace("&", "&"); while (author.Contains("<a")) { endIndex = author.IndexOf("<a"); string firstPart = author.Substring(0, endIndex); author = author.Substring(endIndex + 2, author.Length - (endIndex + 2)); startIndex = author.IndexOf(">") + 1; string secondPart = author.Substring(startIndex, author.Length - startIndex); author = firstPart + "\n" + secondPart; } author = author.Replace("</a>", "").Trim(); author = author.Replace("'", "'"); // Check the subject if it is not a reply and not an announcement (First Post only) if (!subject.Substring(0, 4).ToLower().Contains("re:") && !subject.Substring(0, 4).ToLower().Contains("ann:")) { // Views startIndex = sourceCode.IndexOf("listblock stats "); sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf("</span>") + 7; if (startIndex == -1) { startIndex = sourceCode.IndexOf(">") + 1; } sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); endIndex = sourceCode.IndexOf("</"); string viewStr = sourceCode.Substring(0, endIndex).Trim(); viewStr = viewStr.Replace(",", ""); int view = Convert.ToInt32(viewStr); //// Immediate DateTime startIndex = sourceCode.IndexOf("listblock time"); sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf(">") + 1; sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); startIndex = sourceCode.IndexOf(">") + 1; endIndex = sourceCode.IndexOf("</"); if (startIndex < endIndex && startIndex != -1) { sourceCode = sourceCode.Substring(startIndex, sourceCode.Length - startIndex); } endIndex = sourceCode.IndexOf("</"); string datetimeStr = sourceCode.Substring(0, endIndex).Trim(); DateTime threadDate = Convert.ToDateTime(datetimeStr); // Access inside the thread if (threadLink != "") { string threadCode = WorkerClasses.getSourceCode(threadLink); if (threadCode == "invalid") { throw new UriFormatException(); } // Date of the thread startIndex = threadCode.IndexOf("icon left icon-clock"); threadCode = threadCode.Substring(startIndex, threadCode.Length - startIndex); startIndex = threadCode.IndexOf("</span>") + 7; threadCode = threadCode.Substring(startIndex, threadCode.Length - startIndex); endIndex = threadCode.IndexOf("</"); string threadBeginStr = threadCode.Substring(0, endIndex).Trim(); DateTime threadBegin = Convert.ToDateTime(datetimeStr); // if Nav to First does not exist, this post is the last one, save as it is. int totalPosts = 1; string lastPoster = author; string lastPost = threadLink; string threadCodeOrg = threadCode; startIndex = threadCode.IndexOf("rel=\"start\""); // Nav to First // if first exists, Check 'PageNav', find a total number, then go to last post, // find save last post link and last poster. //................................. UPDATE AREA ................................// if (startIndex != -1) { // Total Number of posts startIndex = threadCode.IndexOf("PageNav"); threadCode = threadCode.Substring(startIndex, threadCode.Length - startIndex); startIndex = threadCode.IndexOf("data-last") + 9; threadCode = threadCode.Substring(startIndex, threadCode.Length - startIndex); startIndex = threadCode.IndexOf("\"") + 1; threadCode = threadCode.Substring(startIndex, threadCode.Length - startIndex); endIndex = threadCode.IndexOf("\""); totalPosts = Convert.ToInt32(threadCode.Substring(0, endIndex)); // ------:> Navigate to Last Post startIndex = threadCode.IndexOf("Next</a>"); threadCode = threadCode.Substring(startIndex, threadCode.Length - startIndex); startIndex = threadCode.IndexOf("a href"); threadCode = threadCode.Substring(startIndex, threadCode.Length - startIndex); startIndex = threadCode.IndexOf("\"") + 1; threadCode = threadCode.Substring(startIndex, threadCode.Length - startIndex); endIndex = threadCode.IndexOf("\""); string lastPostLink = "http://hotcopper.com.au/" + threadCode.Substring(0, endIndex); string lastPostCode = WorkerClasses.getSourceCode(lastPostLink); if (lastPostCode == "invalid") { throw new UriFormatException(); } // Last Post lastPost = lastPostLink; // Last Poster startIndex = lastPostCode.IndexOf("user-wrap"); lastPostCode = lastPostCode.Substring(startIndex, lastPostCode.Length - startIndex); startIndex = lastPostCode.IndexOf("<h"); lastPostCode = lastPostCode.Substring(startIndex, lastPostCode.Length - startIndex); startIndex = lastPostCode.IndexOf(">") + 1; lastPostCode = lastPostCode.Substring(startIndex, lastPostCode.Length - startIndex); startIndex = lastPostCode.IndexOf(">") + 1; endIndex = lastPostCode.IndexOf("</"); lastPoster = lastPostCode.Substring(0, endIndex); while (lastPoster.Contains("<a")) { endIndex = lastPoster.IndexOf("<a"); string firstPart = lastPoster.Substring(0, endIndex); lastPoster = lastPoster.Substring(endIndex + 2, lastPoster.Length - (endIndex + 2)); startIndex = lastPoster.IndexOf(">") + 1; string secondPart = lastPoster.Substring(startIndex, lastPoster.Length - startIndex); lastPoster = firstPart + "\n" + secondPart; } lastPoster = lastPoster.Replace("</a>", "").Trim(); lastPoster = lastPoster.Replace("'", "'"); } if (!db.Threads.Any(f => f.Subject == subject && f.Begin_Date == threadBegin)) { db.Threads.Add(new HCDB_Threads { Stock = stock, Tags = tags, Subject = subject, Num_of_Posts = totalPosts, Num_of_Views = view, First_Poster = author, Begin_Date = threadBegin, Last_Post = lastPost, Last_Poster = lastPoster }); db.SaveChanges(); //MessageBox.Show("Count " + newData); Console.WriteLine("Count " + newData); newData++; } else { var existing = (from u in db.Threads where u.Subject == subject && u.Begin_Date == threadBegin select u).FirstOrDefault(); if (existing != null) { if (existing.Last_Post != lastPost) { existing.Last_Post = lastPost; } if (existing.Last_Poster != lastPoster) { existing.Last_Poster = lastPoster; } if (existing.Num_of_Posts != totalPosts) { existing.Num_of_Posts = totalPosts; } if (existing.Num_of_Views != view) { existing.Num_of_Views = view; } db.SaveChanges(); duplicates++; } } // Save Page Data postNewData += WorkerClasses.savePageData(stock, subject, threadCodeOrg, threadLink, authorPostLink); } } // Second, Third, Forth and after posts... else if (subject.Substring(0, 4).ToLower().Contains("re:")) { string threadCodeOrg = WorkerClasses.getSourceCode(threadLink); if (threadCodeOrg == "invalid") { throw new UriFormatException(); } // Save Page Data postNewData += WorkerClasses.savePageData(stock, subject, threadCodeOrg, threadLink, authorPostLink); } } } catch (Exception ex) { MessageBox.Show(ex.Message); MessageBox.Show(ex.ToString()); listbox.Items.Add("Error: " + ex); } listbox.Items.Add("\n[" + DateTime.Now + "] HotCopper Threads completed.\n" + newData + " saved and " + duplicates + " duplicates Updated."); listbox.Items.Add("\n[" + DateTime.Now + "] HotCopper Posts completed.\n" + postNewData + " saved."); #endregion } catch (Exception ex) { MessageBox.Show(ex.Message); MessageBox.Show(ex.ToString()); listbox.Items.Add("Invalid URL!"); MessageBox.Show("Invalid URL!"); textBox1.Text = ""; } } else { listbox.Items.Add("Please enter URL."); MessageBox.Show("Please enter URL."); } }