public string process_stopword_stem(string raw_text) // return string after stem and remove stopword
{
Iveonik.Stemmers.EnglishStemmer stemer = new Iveonik.Stemmers.EnglishStemmer();
var strArr = Regex.Split(raw_text, @"\W+");
int size_strArr = strArr.Length;
List <string> result = new List <string>();
for (int i = 0; i < size_strArr; i++)
{
bool flag = false; //<=> strArr[i] not in stopwords
// remove stopword
foreach (var sword in this.stopwords)
{
if (String.Compare(strArr[i], sword, true) == 0)
{
flag = true;
break;
}
}
// if flag = false, stem(strArr[i]) and then push List result
if (flag == false)
{
string word = stemer.Stem(strArr[i]);
result.Add(word);
}
}
return(String.Join(" ", result).Trim().ToLower());
}
public void retrieveWordsEnglish()
{
Iveonik.Stemmers.IStemmer stemmer = new Iveonik.Stemmers.EnglishStemmer();
string[] words = sentenceText.Split(new char[] { ' ', '\r', '\n', '\t', '.', ',', ':', ';', '!', '?', '\"', '_', '-', '(', ')', '|', '$', '%', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' }, StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < words.Length; i++)
{
words[i] = stemmer.Stem(words[i]);
}
Words = words.ToList();
}
public static void MineTokenizeTweet1Gram(string targetSQLConnectionString, List <TweetText> tweets, string englishWordDictionaryPath, string stopWordsFilePath)
{
logger.Info("MineTokenizeTweet1Gram start");
List <tmToken1Gram> textMiningResults = new List <tmToken1Gram>();
Iveonik.Stemmers.EnglishStemmer englishStemmer = new Iveonik.Stemmers.EnglishStemmer();
string englishWordDictionary = File.ReadAllText(englishWordDictionaryPath);
string stopWordsFile = File.ReadAllText(stopWordsFilePath);
foreach (var tweet in tweets)
{
var words = tmRemoveSpecialCharactersFromText(tweet.Text, true).Split(' ');
foreach (var word in words)
{
tmToken1Gram result = new tmToken1Gram();
result.TweetID = tweet.ID;
result.Token = word;
if (tmRemoveEnglishStopWords(word, stopWordsFile).Length > 0)
{
result.IsStopWord = true;
}
else
{
result.IsStopWord = false;
}
if (tmRemoveNonEnglishWords(word, englishWordDictionary).Length > 0)
{
result.IsEnglishWord = true;
}
else
{
result.IsEnglishWord = false;
}
if (tmRemoveNonEnglishWordsAndStopWords(word, englishWordDictionary, stopWordsFile).Length > 0)
{
result.IsNotEnglishWordAndNotStopWord = true;
}
else
{
result.IsNotEnglishWordAndNotStopWord = false;
}
result.IsNumber = tmCheckIfNumber(word);
if (word.StartsWith("@"))
{
result.IsAccountName = true;
}
else
{
result.IsAccountName = false;
}
if (word.StartsWith("#"))
{
result.IsHashtag = true;
}
else
{
result.IsHashtag = false;
}
if (word.StartsWith("www") || word.StartsWith("http") || word.StartsWith("https"))
{
result.IsWebsiteUrl = true;
}
else
{
result.IsWebsiteUrl = false;
}
if (result.IsEnglishWord == true && result.IsStopWord == false)
{
try
{
result.TokenRootWord = englishStemmer.Stem(word);
}
catch { }
finally
{
if (!(result.TokenRootWord.Length > 0))
{
result.TokenRootWord = "N/A";
}
}
}
textMiningResults.Add(result);
}
}
SqlConnection conn = new SqlConnection(targetSQLConnectionString);
SqlCommand cmd;
try
{
conn.Open();
}
catch (Exception exc)
{
logger.Error(exc);
}
if (conn.State == ConnectionState.Open)
{
foreach (var result in textMiningResults)
{
cmd = new SqlCommand("[Internal].[sp_InsertToken1Gram]", conn);
cmd.CommandType = CommandType.StoredProcedure;
cmd.Parameters.Add("@TweetID", SqlDbType.Int).Value = result.TweetID;
cmd.Parameters.Add("@Token", SqlDbType.NVarChar, 500).Value = result.Token;
cmd.Parameters.Add("@TokenRootWord", SqlDbType.NVarChar, 500).Value = result.TokenRootWord;
cmd.Parameters.Add("@IsEnglishWord", SqlDbType.Bit).Value = result.IsEnglishWord;
cmd.Parameters.Add("@IsStopWord", SqlDbType.Bit).Value = result.IsStopWord;
cmd.Parameters.Add("@IsNotEnglishWordAndNotStopWord", SqlDbType.Bit).Value = result.IsNotEnglishWordAndNotStopWord;
cmd.Parameters.Add("@IsHashtag", SqlDbType.Bit).Value = result.IsHashtag;
cmd.Parameters.Add("@IsAccountName", SqlDbType.Bit).Value = result.IsAccountName;
cmd.Parameters.Add("@IsNumber", SqlDbType.Bit).Value = result.IsNumber;
cmd.Parameters.Add("@IsWebsiteUrl", SqlDbType.Bit).Value = result.IsWebsiteUrl;
try
{
cmd.ExecuteNonQuery();
}
catch (Exception exc)
{
logger.Error(exc);
}
}
conn.Close();
logger.Info("MineTokenizeTweet1Gram done");
}
}
