public float[] FillSmall(AttentionParameter p) { m_blob_bottom.Reshape(2, 1, 1, 1); // timesteps = 2, batch = 1, input = 1 float[] rgData = convertF(m_blob_bottom.mutable_cpu_data); // shape (2, 1, 1, 1) // timestep 1, batch 1 rgData[0] = 1.11f; rgData[1] = 2.11f; m_blob_bottom.mutable_cpu_data = convert(rgData); m_blobState.Reshape(new List <int>() { 1, (int)p.dim }); List <int> rgShape = Utility.Clone <int>(m_blob_bottom.shape()); while (rgShape.Count > 2) { rgShape.RemoveAt(rgShape.Count - 1); } m_blobClip.Reshape(rgShape); m_blobClip.SetData(1); BottomVec.Clear(); BottomVec.Add(m_blob_bottom); BottomVec.Add(m_blobState); BottomVec.Add(m_blobClip); return(rgData); }
private float[] calculateAttention(AttentionParameter p, float[] rgData, float[] rgState, int nT, int nB, int nI, int nS) { int nM = nT * nB; int nN = (int)p.dim; int nK = nI; float[] rgDataT = SimpleDatum.Transpose(rgData, nT, nB, nI); float[] rgStateT = SimpleDatum.Transpose(rgState, 1, nB, nS); // IP input data with rgUa wts. create_weights(ref m_rgUa, nN * nK); create_weights(ref m_rgUab, nN, 0.1f); create_weights(ref m_rgBiasMult, nM, 1.0f); float[] rgUh = gemm(false, false, nM, nN, nK, 1.0f, rgDataT, m_rgUa, 0.0f); float[] rgUhb = gemm(false, false, nM, nN, 1, 1.0f, m_rgBiasMult, m_rgUab, 0.0f); rgUh = add(rgUh, rgUhb); // IP rgFullState with rgWa wts. nM = nB; nK = (int)p.dim; create_weights(ref m_rgWa, nN * nK); create_weights(ref m_rgWab, nN, 0.1f); m_rgBiasMult = null; create_weights(ref m_rgBiasMult, nM, 1.0f); float[] rgWc = gemm(false, false, nM, nN, nK, 1.0f, rgState, m_rgWa, 0.0f); float[] rgWcb = gemm(false, false, nM, nN, 1, 1.0f, m_rgBiasMult, m_rgWab, 0.0f); rgWc = add(rgWc, rgWcb); // Copy rgWc across all T. float[] rgFullWc = expand(rgWc, nT); // Add uh + wc float[] rgUhWc = add(rgUh, rgFullWc); // rgGg = Tanh(un + wc); float[] rgGg = tanh(rgUhWc); // rgAa = IP rgGg with rgV wts. nM = nT * nB; nN = 1; nK = nS; create_weights(ref m_rgV, nN * nK); m_rgAa = gemm(false, false, nM, nN, nK, 1.0f, rgGg, m_rgV, 0.0f); // Softmax over time steps T. m_rgSoftmax = softmax(m_rgAa, nB, nT); // Multiply softmax vector with input data. float[] rgFocusInput = mul(rgDataT, m_rgSoftmax, nB, nT, nI); // Sum across all T. float[] rgContext = sum(rgFocusInput, nB, nT, nI); return(rgContext); }
public float[] Fill2(AttentionParameter p) { m_blob_bottom.Reshape(3, 1, 32, 1); // timesteps = 3, batch = 1, input = 32 float[] rgData = convertF(m_blob_bottom.mutable_cpu_data); for (int t = 0; t < m_blob_bottom.num; t++) { for (int i = 0; i < 32; i++) { int nIdx = t * 32 + i; rgData[nIdx] = (t + 1) + (i * 0.01f); } } m_blob_bottom.mutable_cpu_data = convert(rgData); m_blobState.Reshape(new List <int>() { 1, (int)p.dim }); List <int> rgShape = Utility.Clone <int>(m_blob_bottom.shape()); while (rgShape.Count > 2) { rgShape.RemoveAt(rgShape.Count - 1); } m_blobClip.Reshape(rgShape); m_blobClip.SetData(1); BottomVec.Clear(); BottomVec.Add(m_blob_bottom); BottomVec.Add(m_blobState); BottomVec.Add(m_blobClip); return(rgData); }