//return the gradient of -log{P(y*|x,w)} as follows: E_{P(y|x)}(F(x,y)) - F(x,y*)
virtual public double getGrad(List <double> vecGrad, model m, dataSeq x, baseHashSet <int> idSet)
{
if (idSet != null)
{
idSet.Clear();
}
int nTag = m.NTag;
//compute beliefs
belief bel = new belief(x.Count, nTag);
belief belMasked = new belief(x.Count, nTag);
_inf.getBeliefs(bel, m, x, false);
_inf.getBeliefs(belMasked, m, x, true);
double ZGold = belMasked.Z;
double Z = bel.Z;
List <featureTemp> fList;
for (int i = 0; i < x.Count; i++)
{
fList = _fGene.getFeatureTemp(x, i);
for (int j = 0; j < fList.Count; j++)
{
featureTemp im = fList[j];
int id = im.id;
double v = im.val;
for (int s = 0; s < nTag; s++)
{
int f = _fGene.getNodeFeatID(id, s);
if (idSet != null)
{
idSet.Add(f);
}
vecGrad[f] += bel.belState[i][s] * v;
vecGrad[f] -= belMasked.belState[i][s] * v;
}
}
}
for (int i = 1; i < x.Count; i++)
{
for (int s = 0; s < nTag; s++)
{
for (int sPre = 0; sPre < nTag; sPre++)
{
int f = _fGene.getEdgeFeatID(sPre, s);
if (idSet != null)
{
idSet.Add(f);
}
vecGrad[f] += bel.belEdge[i][sPre, s];
vecGrad[f] -= belMasked.belEdge[i][sPre, s];
}
}
}
return(Z - ZGold);
}
//the scalar version
virtual public double getGradCRF(List <double> vecGrad, double scalar, model m, dataSeq x, baseHashSet <int> idSet)
{
idSet.Clear();
int nTag = m.NTag;
//compute beliefs
belief bel = new belief(x.Count, nTag);
belief belMasked = new belief(x.Count, nTag);
_inf.getBeliefs(bel, m, x, scalar, false);
_inf.getBeliefs(belMasked, m, x, scalar, true);
double ZGold = belMasked.Z;
double Z = bel.Z;
List <featureTemp> fList;
//Loop over nodes to compute features and update the gradient
for (int i = 0; i < x.Count; i++)
{
fList = _fGene.getFeatureTemp(x, i);
foreach (featureTemp im in fList)
{
for (int s = 0; s < nTag; s++)
{
int f = _fGene.getNodeFeatID(im.id, s);
idSet.Add(f);
vecGrad[f] += bel.belState[i][s] * im.val;
vecGrad[f] -= belMasked.belState[i][s] * im.val;
}
}
}
//Loop over edges to compute features and update the gradient
for (int i = 1; i < x.Count; i++)
{
for (int s = 0; s < nTag; s++)
{
for (int sPre = 0; sPre < nTag; sPre++)
{
int f = _fGene.getEdgeFeatID(sPre, s);
idSet.Add(f);
vecGrad[f] += bel.belEdge[i][sPre, s];
vecGrad[f] -= belMasked.belEdge[i][sPre, s];
}
}
}
return(Z - ZGold);//-log{P(y*|x,w)}
}
public static List <double> getFscore(List <string> goldTagList, List <string> resTagList, List <double> infoList)
{
List <double> scoreList = new List <double>();
if (resTagList.Count != goldTagList.Count)
{
throw new Exception("error");
}
//convert original tags to 3 tags: B(x), I, O
getNewTagList(Global.chunkTagMap, ref goldTagList);
getNewTagList(Global.chunkTagMap, ref resTagList);
List <string> goldChunkList = getChunks(goldTagList);
List <string> resChunkList = getChunks(resTagList);
int gold_chunk = 0, res_chunk = 0, correct_chunk = 0;
for (int i = 0; i < goldChunkList.Count; i++)
{
string res = resChunkList[i];
string gold = goldChunkList[i];
string[] resChunkAry = res.Split(Global.commaAry, StringSplitOptions.RemoveEmptyEntries);
string[] goldChunkAry = gold.Split(Global.commaAry, StringSplitOptions.RemoveEmptyEntries);
gold_chunk += goldChunkAry.Length;
res_chunk += resChunkAry.Length;
baseHashSet <string> goldChunkSet = new baseHashSet <string>();
foreach (string im in goldChunkAry)
{
goldChunkSet.Add(im);
}
foreach (string im in resChunkAry)
{
if (goldChunkSet.Contains(im))
{
correct_chunk++;
}
}
}
double pre = (double)correct_chunk / (double)res_chunk * 100;
double rec = (double)correct_chunk / (double)gold_chunk * 100;
double f1 = 2 * pre * rec / (pre + rec);
scoreList.Add(f1);
scoreList.Add(pre);
scoreList.Add(rec);
infoList.Add(gold_chunk);
infoList.Add(res_chunk);
infoList.Add(correct_chunk);
return(scoreList);
}
//the mini-batch version
public double getGrad_SGD_miniBatch(List <double> g, model m, List <dataSeq> X, baseHashSet <int> idset)
{
if (idset != null)
{
idset.Clear();
}
double error = 0;
foreach (dataSeq x in X)
{
baseHashSet <int> idset2 = new baseHashSet <int>();
error += getGradCRF(g, m, x, idset2);
if (idset != null)
{
foreach (int i in idset2)
{
idset.Add(i);
}
}
}
return(error);
}
public void getMaps(string file)
{
if (!File.Exists(file))
{
Console.WriteLine("file {0} no exist!", file);
return;
}
Console.WriteLine("file {0} converting...", file);
StreamReader sr = new StreamReader(file);
baseHashMap <string, int> featureFreqMap = new baseHashMap <string, int>();
baseHashSet <string> tagSet = new baseHashSet <string>();
//get feature-freq info and tagset
int nFeatTemp = 0;
while (!sr.EndOfStream)
{
string line = sr.ReadLine();
line = line.Replace("\t", " ");
line = line.Replace("\r", "");
if (line == "")
{
continue;
}
string[] ary = line.Split(Global.blankAry, StringSplitOptions.RemoveEmptyEntries);
nFeatTemp = ary.Length - 2;
for (int i = 1; i < ary.Length - 1; i++)
{
if (ary[i] == "/")//no feature here
{
continue;
}
string[] ary2 = ary[i].Split(Global.slashAry, StringSplitOptions.RemoveEmptyEntries);//for real-value features
string feature = i.ToString() + "." + ary2[0];
if (featureFreqMap.ContainsKey(feature) == false)
{
featureFreqMap[feature] = 1;
}
else
{
featureFreqMap[feature]++;
}
}
string tag = ary[ary.Length - 1];
tagSet.Add(tag);
}
//sort features
List <string> sortList = new List <string>();
foreach (baseHashMap <string, int> .KeyValuePair kv in featureFreqMap)
{
sortList.Add(kv.Key + " " + kv.Value);
}
if (Global.regMode == "GL")//sort based on feature templates
{
sortList.Sort(listSortFunc.compareKV_key);
//sortList.Reverse();
Global.groupStart = new List <int>();
Global.groupEnd = new List <int>();
Global.groupStart.Add(0);
for (int k = 1; k < sortList.Count; k++)
{
string[] thisAry = sortList[k].Split(Global.dotAry, StringSplitOptions.RemoveEmptyEntries);
string[] preAry = sortList[k - 1].Split(Global.dotAry, StringSplitOptions.RemoveEmptyEntries);
string str = thisAry[0], preStr = preAry[0];
if (str != preStr)
{
Global.groupStart.Add(k);
Global.groupEnd.Add(k);
}
}
Global.groupEnd.Add(sortList.Count);
}
else//sort based on feature frequency
{
sortList.Sort(listSortFunc.compareKV_value);//sort feature based on freq, for 1)compress .txt file 2)better edge features
sortList.Reverse();
}
if (Global.regMode == "GL")
{
if (nFeatTemp != Global.groupStart.Count)
{
throw new Exception("inconsistent # of features per line, check the feature file for consistency!");
}
}
//feature index should begin from 0
StreamWriter swFeat = new StreamWriter("featureIndex.txt");
for (int i = 0; i < sortList.Count; i++)
{
string[] ary = sortList[i].Split(Global.blankAry);
featureIndexMap[ary[0]] = i;
swFeat.WriteLine("{0} {1}", ary[0], i);
}
swFeat.Close();
//label index should begin from 0
StreamWriter swTag = new StreamWriter("tagIndex.txt");
List <string> tagSortList = new List <string>();
foreach (string tag in tagSet)
{
tagSortList.Add(tag);
}
tagSortList.Sort();//sort tags
for (int i = 0; i < tagSortList.Count; i++)
{
tagIndexMap[tagSortList[i]] = i;
swTag.WriteLine("{0} {1}", tagSortList[i], i);
}
swTag.Close();
sr.Close();
}
public static void loadData_multi(List <dataSet> XList, dataSet X, List <dataSet> XXList)
{
XList.Clear();
XXList.Clear();
//load train data
baseHashSet <int> checkSet = new baseHashSet <int>();
for (int i = 0; i < Global.nTask; i++)
{
string dat_i = i.ToString() + Global.fFeatureTrain;
string tag_i = i.ToString() + Global.fGoldTrain;
dataSet Xi = new dataSet(dat_i, tag_i);
dataSizeScale(Xi);
checkSet.Add(Xi.NFeatureTemp);
XList.Add(Xi);
}
if (checkSet.Count > 1)
{
throw new Exception("inconsistent features among multi tasks!");
}
//make nTag consistent among different tasks
int maxNTag = 0;
foreach (dataSet Xi in XList)
{
if (maxNTag < Xi.NTag)
{
maxNTag = Xi.NTag;
}
}
for (int i = 0; i < Global.nTask; i++)
{
XList[i].NTag = maxNTag;
}
//add to merged data
X.NTag = XList[0].NTag;
X.NFeatureTemp = XList[0].NFeatureTemp;
foreach (dataSet Xi in XList)
{
foreach (dataSeq im in Xi)
{
X.Add(im);
}
}
Global.swLog.WriteLine("data sizes (1, ..., T):");
for (int i = 0; i < Global.nTask; i++)
{
dataSet Xi = XList[i];
Global.swLog.WriteLine(" " + Xi.Count.ToString());
}
Global.swLog.WriteLine();
//load test data
for (int i = 0; i < Global.nTask; i++)
{
string dat_i = i.ToString() + Global.fFeatureTest;
string tag_i = i.ToString() + Global.fGoldTest;
dataSet Xtest = new dataSet(dat_i, tag_i);
XXList.Add(Xtest);
}
for (int i = 0; i < Global.nTask; i++)
{
XXList[i].NTag = maxNTag;
}
}
override public double getGradCRF(List <double> gradList, model m, dataSeq x, baseHashSet <int> idSet)
{
if (idSet != null)
{
idSet.Clear();
}
int nTag = m.NTag;
//compute beliefs
belief bel = new belief(x.Count, nTag);
belief belMasked = new belief(x.Count, nTag);
//store the YY and Y
List <dMatrix> YYlist = new List <dMatrix>(), maskYYlist = new List <dMatrix>();
List <List <double> > Ylist = new List <List <double> >(), maskYlist = new List <List <double> >();
_inf.getYYandY(m, x, YYlist, Ylist, maskYYlist, maskYlist);
_inf.getBeliefs(bel, m, x, YYlist, Ylist);
_inf.getBeliefs(belMasked, m, x, maskYYlist, maskYlist);
double ZGold = belMasked.Z;
double Z = bel.Z;
List <featureTemp> fList;
//Loop over nodes to compute features and update the gradient
for (int i = 0; i < x.Count; i++)
{
fList = _fGene.getFeatureTemp(x, i);
foreach (featureTemp im in fList)
{
for (int s = 0; s < nTag; s++)
{
int f = _fGene.getNodeFeatID(im.id, s);
if (idSet != null)
{
idSet.Add(f);
}
gradList[f] += bel.belState[i][s] * im.val;
gradList[f] -= belMasked.belState[i][s] * im.val;
}
}
}
//Loop over edges to compute features and update the gradient
for (int i = 1; i < x.Count; i++)
{
//non-rich
if (Global.useTraditionalEdge)
{
for (int s = 0; s < nTag; s++)
{
for (int sPre = 0; sPre < nTag; sPre++)
{
int f = _fGene.getEdgeFeatID(sPre, s);
if (idSet != null)
{
idSet.Add(f);
}
gradList[f] += bel.belEdge[i][sPre, s];
gradList[f] -= belMasked.belEdge[i][sPre, s];
}
}
}
//rich
fList = _fGene.getFeatureTemp(x, i);
foreach (featureTemp im in fList)
{
int id = im.id;
if (id < _fGene.getNRichFeatTemp())
{
for (int s = 0; s < nTag; s++)
{
for (int sPre = 0; sPre < nTag; sPre++)
{
int f = _fGene.getEdgeFeatID(id, sPre, s);
if (idSet != null)
{
idSet.Add(f);
}
gradList[f] += bel.belEdge[i][sPre, s] * im.val;
gradList[f] -= belMasked.belEdge[i][sPre, s] * im.val;
}
}
}
}
}
return(Z - ZGold);//-log{P(y*|x,w)}
}
