public void AnalyzeWords(List <string> words, ComboSource <string, ThreeTuple <ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> > combo, Dictionary <string, List <KeyValuePair <double, double> > > sentencesV, Dictionary <string, List <KeyValuePair <double, double> > > sentencesA, Dictionary <string, List <KeyValuePair <double, double> > > sentencesD) { List <KeyValuePair <double, double> > wordsV = new List <KeyValuePair <double, double> >(), wordsA = new List <KeyValuePair <double, double> >(), wordsD = new List <KeyValuePair <double, double> >(); foreach (string word in words) { if (word.StartsWith(" ") || word.Length <= 2) { continue; } ThreeTuple <ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> vad; if (!TryGetWordOrStem(combo, word, out vad)) { continue; } wordsV.Add(new KeyValuePair <double, double>(vad.one.Mean, 1 / vad.one.Variance)); wordsA.Add(new KeyValuePair <double, double>(vad.two.Mean, 1 / vad.two.Variance)); wordsD.Add(new KeyValuePair <double, double>(vad.three.Mean, 1 / vad.three.Variance)); } double vmean = WeightedStatistics.Mean(wordsV), amean = WeightedStatistics.Mean(wordsA), dmean = WeightedStatistics.Mean(wordsD); double vvar = WeightedStatistics.Variance(wordsV, vmean, true), avar = WeightedStatistics.Variance(wordsA, amean, true), dvar = WeightedStatistics.Variance(wordsD, dmean, true); if (double.IsNaN(vmean) || double.IsNaN(amean) || double.IsNaN(dmean)) { return; } foreach (string word in words) { if (word.StartsWith(" ") || word.Length <= 2) { continue; } ThreeTuple <ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> vad; if (TryGetWordOrStem(source, word, out vad)) { continue; } string stem = stemmer.stemTerm(word); AddTextNumerDenom(stem, sentencesV, vmean, vvar); AddTextNumerDenom(stem, sentencesA, amean, avar); AddTextNumerDenom(stem, sentencesD, dmean, dvar); } }
//------------------------------------------------- // コンストラクタ //------------------------------------------------- /// <summary> /// コンストラクタ /// </summary> /// <param name="dispatcher"></param> public MainViewModel(Dispatcher dispatcher) { base.Dispatcher = dispatcher; //------------------------------------------------- // コンボボックスの設定値 //------------------------------------------------- // UIスレッド渡す //ComboSource = new BindingListAsync<MainViewModelCombo>(dispatcher); ComboSource.Add(new MainViewModelCombo(0, "PID_HOGE")); ComboSource.Add(new MainViewModelCombo(1, "PID_FUGA")); ComboSource.Add(new MainViewModelCombo(2, "PID_HEGE")); }
//------------------------------------------------- // コンストラクタ //------------------------------------------------- /// <summary> /// コンストラクタ /// </summary> /// <param name="dispatcher"></param> public MainViewModel(Dispatcher dispatcher) { //------------------------------------------------- // 画面コントロールの初期値 //------------------------------------------------- BtnStartServiceEnabled = true; BtnStopServiceEnabled = false; //------------------------------------------------- // コンボボックスの設定値 //------------------------------------------------- // UIスレッド渡す //ComboSource = new BindingListAsync<MainViewModelCombo>(dispatcher); ComboSource.Add(new MainViewModelCombo(0, "PID_HOGE")); ComboSource.Add(new MainViewModelCombo(1, "PID_FUGA")); ComboSource.Add(new MainViewModelCombo(2, "PID_HEGE")); //------------------------------------------------- // コールバック登録受付通知受信のための準備 // ※NuGetでMVVMLightLibs取得 //------------------------------------------------- Messenger.Default.Register <CallbackRegistNty>(this, onReceivedMessage); }
ImputeEmotionalContentFromFile(string filename, uint column, uint repeats, string imputesave) { MemorySource <string, ThreeTuple <ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> > imputed = new MemorySource <string, ThreeTuple <ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> >(); ComboSource <string, ThreeTuple <ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> > combo = new ComboSource <string, ThreeTuple <ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> >(source, imputed); // Check for existing imputed file DataReader imputereader = new DataReader(imputesave); uint kk = 0; for (string[] row = imputereader.ReadRow(); row != null; row = imputereader.ReadRow()) { kk++; if (kk % 1000 == 0) { Console.WriteLine("#" + kk); } double meanv = double.Parse(row[1]), varv = double.Parse(row[2]), meana = double.Parse(row[3]), vara = double.Parse(row[4]), meand = double.Parse(row[5]), vard = double.Parse(row[6]); ContinuousDistribution valence = new ClippedGaussianDistribution(meanv, varv, 0, 1); ContinuousDistribution arousal = new ClippedGaussianDistribution(meana, vara, 0, 1); ContinuousDistribution dominance = new ClippedGaussianDistribution(meand, vard, 0, 1); imputed[row[0]] = new ThreeTuple <ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>(valence, arousal, dominance); } imputereader.Close(); for (uint ii = 0; ii < repeats; ii++) { Dictionary <string, List <KeyValuePair <double, double> > > sentencesV = new Dictionary <string, List <KeyValuePair <double, double> > >(), sentencesA = new Dictionary <string, List <KeyValuePair <double, double> > >(), sentencesD = new Dictionary <string, List <KeyValuePair <double, double> > >(); DataReader reader = new DataReader(filename); uint jj = 0; for (string[] row = reader.ReadRow(); row != null; row = reader.ReadRow()) { jj++; if (jj % 1000 == 0) { Console.WriteLine("#" + jj + ": " + sentencesV.Count + ", " + imputed.Count); } List <string> words = TwitterUtilities.SplitWords(row[column].ToLower()); AnalyzeWords(words, combo, sentencesV, sentencesA, sentencesD); } reader.Close(); AnalyzeSentences(imputed, sentencesV, sentencesA, sentencesD, imputesave); } source = combo; return(imputed); }
/*public IDataSource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>> ImputeEmotionalContent(List<List<string>> texts, uint repeats) { * MemorySource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>> inputed = new MemorySource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>>(); * ComboSource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>> combo = new ComboSource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>>(source, inputed); * * for (uint ii = 0; ii < repeats; ii++) { * Dictionary<string, double> wordVnumers = new Dictionary<string, double>(), wordVdenoms = new Dictionary<string, double>(), * wordAnumers = new Dictionary<string, double>(), wordAdenoms = new Dictionary<string, double>(), * wordDnumers = new Dictionary<string, double>(), wordDdenoms = new Dictionary<string, double>(), * wordVsumvar = new Dictionary<string, double>(), wordVcounts = new Dictionary<string, double>(), * wordAsumvar = new Dictionary<string, double>(), wordAcounts = new Dictionary<string, double>(), * wordDsumvar = new Dictionary<string, double>(), wordDcounts = new Dictionary<string, double>(); * * uint jj = 0; * foreach (List<string> words in texts) { * jj++; * if (jj % 1000 == 0) * Console.WriteLine("#" + jj); * * double textVnumer = 0, textVdenom = 0, textAnumer = 0, textAdenom = 0, textDnumer = 0, textDdenom = 0; * double textVsumvar = 0, textVcount = 0, textAsumvar = 0, textAcount = 0, textDsumvar = 0, textDcount = 0; * foreach (string word in words) { * if (word.StartsWith(" ") || word.Length <= 2) * continue; * * ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> vad; * if (!TryGetWordOrStem(combo, word, out vad)) * continue; * * textVnumer += vad.one.Mean / vad.one.Variance; * textVdenom += 1 / vad.one.Variance; * textVsumvar += vad.one.Variance; * textVcount++; * textAnumer += vad.two.Mean / vad.two.Variance; * textAdenom += 1 / vad.two.Variance; * textAsumvar += vad.two.Variance; * textAcount++; * textDnumer += vad.three.Mean / vad.three.Variance; * textDdenom += 1 / vad.three.Variance; * textDsumvar += vad.three.Variance; * textDcount++; * } * * double vmean = textVnumer / textVdenom, amean = textAnumer / textAdenom, dmean = textDnumer / textDdenom; * double vvar = textVsumvar / textVcount, avar = textAsumvar / textAcount, dvar = textDsumvar / textDcount; * * if (double.IsNaN(vmean) || double.IsNaN(amean) || double.IsNaN(dmean)) * continue; * * foreach (string word in words) { * if (word.StartsWith(" ") || word.Length <= 2) * continue; * * ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> vad; * if (TryGetWordOrStem(source, word, out vad)) * continue; * * string stem = stemmer.stemTerm(word); * * AddTextNumerDenom(stem, wordVnumers, wordVdenoms, wordVsumvar, wordVcounts, vmean, vvar); * AddTextNumerDenom(stem, wordAnumers, wordAdenoms, wordAsumvar, wordAcounts, amean, avar); * AddTextNumerDenom(stem, wordDnumers, wordDdenoms, wordDsumvar, wordDcounts, dmean, dvar); * } * } * * foreach (string stem in wordVnumers.Keys) { * ContinuousDistribution valence = new ClippedGaussianDistribution(wordVnumers[stem] / wordVdenoms[stem], wordVsumvar[stem] / wordVcounts[stem], 0, 1); * ContinuousDistribution arousal = new ClippedGaussianDistribution(wordAnumers[stem] / wordAdenoms[stem], wordAsumvar[stem] / wordAcounts[stem], 0, 1); * ContinuousDistribution dominance = new ClippedGaussianDistribution(wordDnumers[stem] / wordDdenoms[stem], wordDsumvar[stem] / wordDcounts[stem], 0, 1); * * inputed[stem] = new ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>(valence, arousal, dominance); * } * } * * source = combo; * return inputed; * } * * public void AddTextNumerDenom(string stem, Dictionary<string, double> wordXnumers, * Dictionary<string, double> wordXdenoms, Dictionary<string, double> wordXsumvar, * Dictionary<string, double> wordXcounts, double xmean, double xvar) { * double numer, denom, sumvar, count; * if (!wordXnumers.TryGetValue(stem, out numer)) { * numer = 0; * denom = 0; * sumvar = 0; * count = 0; * } else { * wordXdenoms.TryGetValue(stem, out denom); * wordXsumvar.TryGetValue(stem, out sumvar); * wordXcounts.TryGetValue(stem, out count); * xvar += (xmean - numer / denom) * (xmean - numer / denom); * } * * numer += xmean / xvar; * denom += 1 / xvar; * sumvar += xvar; * count++; * * wordXnumers[stem] = numer; * wordXdenoms[stem] = denom; * wordXsumvar[stem] = sumvar; * wordXcounts[stem] = count; * }*/ public IDataSource <string, ThreeTuple <ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> > ImputeEmotionalContent(List <List <string> > texts, uint repeats, string imputesave) { MemorySource <string, ThreeTuple <ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> > imputed = new MemorySource <string, ThreeTuple <ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> >(); ComboSource <string, ThreeTuple <ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> > combo = new ComboSource <string, ThreeTuple <ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> >(source, imputed); for (uint ii = 0; ii < repeats; ii++) { Dictionary <string, List <KeyValuePair <double, double> > > sentencesV = new Dictionary <string, List <KeyValuePair <double, double> > >(), sentencesA = new Dictionary <string, List <KeyValuePair <double, double> > >(), sentencesD = new Dictionary <string, List <KeyValuePair <double, double> > >(); uint jj = 0; foreach (List <string> words in texts) { jj++; if (jj % 1000 == 0) { Console.WriteLine("#" + jj); } AnalyzeWords(words, combo, sentencesV, sentencesA, sentencesD); } AnalyzeSentences(imputed, sentencesV, sentencesA, sentencesD, imputesave); } source = combo; return(imputed); }
public IDataSource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>> ImputeEmotionalContentFromFile(string filename, uint column, uint repeats, string imputesave) { MemorySource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>> imputed = new MemorySource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>>(); ComboSource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>> combo = new ComboSource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>>(source, imputed); // Check for existing imputed file DataReader imputereader = new DataReader(imputesave); uint kk = 0; for (string[] row = imputereader.ReadRow(); row != null; row = imputereader.ReadRow()) { kk++; if (kk % 1000 == 0) Console.WriteLine("#" + kk); double meanv = double.Parse(row[1]), varv = double.Parse(row[2]), meana = double.Parse(row[3]), vara = double.Parse(row[4]), meand = double.Parse(row[5]), vard = double.Parse(row[6]); ContinuousDistribution valence = new ClippedGaussianDistribution(meanv, varv, 0, 1); ContinuousDistribution arousal = new ClippedGaussianDistribution(meana, vara, 0, 1); ContinuousDistribution dominance = new ClippedGaussianDistribution(meand, vard, 0, 1); imputed[row[0]] = new ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>(valence, arousal, dominance); } imputereader.Close(); for (uint ii = 0; ii < repeats; ii++) { Dictionary<string, List<KeyValuePair<double, double>>> sentencesV = new Dictionary<string, List<KeyValuePair<double, double>>>(), sentencesA = new Dictionary<string, List<KeyValuePair<double, double>>>(), sentencesD = new Dictionary<string, List<KeyValuePair<double, double>>>(); DataReader reader = new DataReader(filename); uint jj = 0; for (string[] row = reader.ReadRow(); row != null; row = reader.ReadRow()) { jj++; if (jj % 1000 == 0) Console.WriteLine("#" + jj + ": " + sentencesV.Count + ", " + imputed.Count); List<string> words = TwitterUtilities.SplitWords(row[column].ToLower()); AnalyzeWords(words, combo, sentencesV, sentencesA, sentencesD); } reader.Close(); AnalyzeSentences(imputed, sentencesV, sentencesA, sentencesD, imputesave); } source = combo; return imputed; }
/*public IDataSource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>> ImputeEmotionalContent(List<List<string>> texts, uint repeats) { MemorySource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>> inputed = new MemorySource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>>(); ComboSource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>> combo = new ComboSource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>>(source, inputed); for (uint ii = 0; ii < repeats; ii++) { Dictionary<string, double> wordVnumers = new Dictionary<string, double>(), wordVdenoms = new Dictionary<string, double>(), wordAnumers = new Dictionary<string, double>(), wordAdenoms = new Dictionary<string, double>(), wordDnumers = new Dictionary<string, double>(), wordDdenoms = new Dictionary<string, double>(), wordVsumvar = new Dictionary<string, double>(), wordVcounts = new Dictionary<string, double>(), wordAsumvar = new Dictionary<string, double>(), wordAcounts = new Dictionary<string, double>(), wordDsumvar = new Dictionary<string, double>(), wordDcounts = new Dictionary<string, double>(); uint jj = 0; foreach (List<string> words in texts) { jj++; if (jj % 1000 == 0) Console.WriteLine("#" + jj); double textVnumer = 0, textVdenom = 0, textAnumer = 0, textAdenom = 0, textDnumer = 0, textDdenom = 0; double textVsumvar = 0, textVcount = 0, textAsumvar = 0, textAcount = 0, textDsumvar = 0, textDcount = 0; foreach (string word in words) { if (word.StartsWith(" ") || word.Length <= 2) continue; ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> vad; if (!TryGetWordOrStem(combo, word, out vad)) continue; textVnumer += vad.one.Mean / vad.one.Variance; textVdenom += 1 / vad.one.Variance; textVsumvar += vad.one.Variance; textVcount++; textAnumer += vad.two.Mean / vad.two.Variance; textAdenom += 1 / vad.two.Variance; textAsumvar += vad.two.Variance; textAcount++; textDnumer += vad.three.Mean / vad.three.Variance; textDdenom += 1 / vad.three.Variance; textDsumvar += vad.three.Variance; textDcount++; } double vmean = textVnumer / textVdenom, amean = textAnumer / textAdenom, dmean = textDnumer / textDdenom; double vvar = textVsumvar / textVcount, avar = textAsumvar / textAcount, dvar = textDsumvar / textDcount; if (double.IsNaN(vmean) || double.IsNaN(amean) || double.IsNaN(dmean)) continue; foreach (string word in words) { if (word.StartsWith(" ") || word.Length <= 2) continue; ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> vad; if (TryGetWordOrStem(source, word, out vad)) continue; string stem = stemmer.stemTerm(word); AddTextNumerDenom(stem, wordVnumers, wordVdenoms, wordVsumvar, wordVcounts, vmean, vvar); AddTextNumerDenom(stem, wordAnumers, wordAdenoms, wordAsumvar, wordAcounts, amean, avar); AddTextNumerDenom(stem, wordDnumers, wordDdenoms, wordDsumvar, wordDcounts, dmean, dvar); } } foreach (string stem in wordVnumers.Keys) { ContinuousDistribution valence = new ClippedGaussianDistribution(wordVnumers[stem] / wordVdenoms[stem], wordVsumvar[stem] / wordVcounts[stem], 0, 1); ContinuousDistribution arousal = new ClippedGaussianDistribution(wordAnumers[stem] / wordAdenoms[stem], wordAsumvar[stem] / wordAcounts[stem], 0, 1); ContinuousDistribution dominance = new ClippedGaussianDistribution(wordDnumers[stem] / wordDdenoms[stem], wordDsumvar[stem] / wordDcounts[stem], 0, 1); inputed[stem] = new ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>(valence, arousal, dominance); } } source = combo; return inputed; } public void AddTextNumerDenom(string stem, Dictionary<string, double> wordXnumers, Dictionary<string, double> wordXdenoms, Dictionary<string, double> wordXsumvar, Dictionary<string, double> wordXcounts, double xmean, double xvar) { double numer, denom, sumvar, count; if (!wordXnumers.TryGetValue(stem, out numer)) { numer = 0; denom = 0; sumvar = 0; count = 0; } else { wordXdenoms.TryGetValue(stem, out denom); wordXsumvar.TryGetValue(stem, out sumvar); wordXcounts.TryGetValue(stem, out count); xvar += (xmean - numer / denom) * (xmean - numer / denom); } numer += xmean / xvar; denom += 1 / xvar; sumvar += xvar; count++; wordXnumers[stem] = numer; wordXdenoms[stem] = denom; wordXsumvar[stem] = sumvar; wordXcounts[stem] = count; }*/ public IDataSource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>> ImputeEmotionalContent(List<List<string>> texts, uint repeats, string imputesave) { MemorySource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>> imputed = new MemorySource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>>(); ComboSource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>> combo = new ComboSource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>>(source, imputed); for (uint ii = 0; ii < repeats; ii++) { Dictionary<string, List<KeyValuePair<double, double>>> sentencesV = new Dictionary<string, List<KeyValuePair<double, double>>>(), sentencesA = new Dictionary<string, List<KeyValuePair<double, double>>>(), sentencesD = new Dictionary<string, List<KeyValuePair<double, double>>>(); uint jj = 0; foreach (List<string> words in texts) { jj++; if (jj % 1000 == 0) Console.WriteLine("#" + jj); AnalyzeWords(words, combo, sentencesV, sentencesA, sentencesD); } AnalyzeSentences(imputed, sentencesV, sentencesA, sentencesD, imputesave); } source = combo; return imputed; }
public void AnalyzeWords(List<string> words, ComboSource<string, ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution>> combo, Dictionary<string, List<KeyValuePair<double, double>>> sentencesV, Dictionary<string, List<KeyValuePair<double, double>>> sentencesA, Dictionary<string, List<KeyValuePair<double, double>>> sentencesD) { List<KeyValuePair<double, double>> wordsV = new List<KeyValuePair<double, double>>(), wordsA = new List<KeyValuePair<double, double>>(), wordsD = new List<KeyValuePair<double, double>>(); foreach (string word in words) { if (word.StartsWith(" ") || word.Length <= 2) continue; ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> vad; if (!TryGetWordOrStem(combo, word, out vad)) continue; wordsV.Add(new KeyValuePair<double, double>(vad.one.Mean, 1/vad.one.Variance)); wordsA.Add(new KeyValuePair<double, double>(vad.two.Mean, 1/vad.two.Variance)); wordsD.Add(new KeyValuePair<double, double>(vad.three.Mean, 1/vad.three.Variance)); } double vmean = WeightedStatistics.Mean(wordsV), amean = WeightedStatistics.Mean(wordsA), dmean = WeightedStatistics.Mean(wordsD); double vvar = WeightedStatistics.Variance(wordsV, vmean, true), avar = WeightedStatistics.Variance(wordsA, amean, true), dvar = WeightedStatistics.Variance(wordsD, dmean, true); if (double.IsNaN(vmean) || double.IsNaN(amean) || double.IsNaN(dmean)) return; foreach (string word in words) { if (word.StartsWith(" ") || word.Length <= 2) continue; ThreeTuple<ContinuousDistribution, ContinuousDistribution, ContinuousDistribution> vad; if (TryGetWordOrStem(source, word, out vad)) continue; string stem = stemmer.stemTerm(word); AddTextNumerDenom(stem, sentencesV, vmean, vvar); AddTextNumerDenom(stem, sentencesA, amean, avar); AddTextNumerDenom(stem, sentencesD, dmean, dvar); } }