protected internal virtual void Load(BinaryReader br) { var length = br.ReadInt32(); _keywords = new string[length]; for (int i = 0; i < length; i++) { _keywords[i] = br.ReadString(); } length = br.ReadInt32(); var bs = br.ReadBytes(length); _dict = ByteArrToUshortArr(bs); length = br.ReadInt32(); bs = br.ReadBytes(length); _first = ByteArrToIntArr(bs); length = br.ReadInt32(); bs = br.ReadBytes(length); _end = ByteArrToIntArr(bs); length = br.ReadInt32(); bs = br.ReadBytes(length); _resultIndex = ByteArrToIntArr(bs); var dictLength = br.ReadInt32(); _nextIndex = new IntDictionary[dictLength]; _max = new ushort[dictLength]; _min = new ushort[dictLength]; for (int i = 0; i < dictLength; i++) { length = br.ReadInt32(); bs = br.ReadBytes(length); var keys = ByteArrToUshortArr(bs); length = br.ReadInt32(); bs = br.ReadBytes(length); var values = ByteArrToIntArr(bs); IntDictionary dictionary = new IntDictionary(keys, values); _nextIndex[i] = dictionary; if (length == 0) { _min[i] = ushort.MaxValue; } else { _max[i] = keys[keys.Length - 1]; _min[i] = keys[0]; } } }
protected override void SetKeywords2(List <string> keywords) { List <TrieNode> allNode = BuildFirstLayerTrieNode(keywords); TrieNode root = allNode[0]; StringBuilder stringBuilder = new StringBuilder(); for (int i = 1; i < allNode.Count; i++) { stringBuilder.Append(allNode[i].Char); } var length = CreateDict(stringBuilder.ToString()); stringBuilder = null; var allNode2 = new List <TrieNode3Ex>(); for (int i = 0; i < allNode.Count; i++) { allNode2.Add(new TrieNode3Ex() { Index = i });; } for (int i = 0; i < allNode2.Count; i++) { var oldNode = allNode[i]; var newNode = allNode2[i]; foreach (var item in oldNode.m_values) { var key = _dict[item.Key]; var index = item.Value.Index; if (key == 0) { newNode.HasWildcard = true; newNode.WildcardNode = allNode2[index]; continue; } newNode.Add((char)key, allNode2[index]); } foreach (var item in oldNode.Results) { if (oldNode.IsWildcard) { if (keywords[item].Length > oldNode.WildcardLayer) { newNode.SetResults(item); } } else { newNode.SetResults(item); } //newNode.SetResults(item); } var failure = oldNode.Failure; while (failure != root) { if (oldNode.IsWildcard && failure.Layer <= oldNode.WildcardLayer) { break; } foreach (var item in failure.m_values) { var key = _dict[item.Key]; var index = item.Value.Index; if (key == 0) { newNode.HasWildcard = true; if (newNode.WildcardNode == null) { newNode.WildcardNode = allNode2[index]; } continue; } if (newNode.HasKey((char)key) == false) { newNode.Add((char)key, allNode2[index]); } } foreach (var item in failure.Results) { if (oldNode.IsWildcard) { if (keywords[item].Length > oldNode.WildcardLayer) { newNode.SetResults(item); } } else { newNode.SetResults(item); } } failure = failure.Failure; } } allNode.Clear(); allNode = null; root = null; var min = new List <ushort>(); var max = new List <ushort>(); var wildcard = new List <int>(); var nextIndexs = new List <Dictionary <ushort, int> >(); var end = new List <int>() { 0 }; var resultIndex = new List <int>(); for (int i = 0; i < allNode2.Count; i++) { var dict = new Dictionary <ushort, int>(); var node = allNode2[i]; min.Add(node.minflag); max.Add(node.maxflag); if (node.HasWildcard) { wildcard.Add(node.WildcardNode.Index); } else { wildcard.Add(0); } if (i > 0) { foreach (var item in node.m_values) { dict[item.Key] = item.Value.Index; } } foreach (var item in node.Results) { resultIndex.Add(item); } end.Add(resultIndex.Count); nextIndexs.Add(dict); } var first = new int[Char.MaxValue + 1]; foreach (var item in allNode2[0].m_values) { first[item.Key] = item.Value.Index; } _firstIndex = first; _min = min.ToArray(); _max = max.ToArray(); _nextIndex = new IntDictionary[nextIndexs.Count]; for (int i = 0; i < nextIndexs.Count; i++) { IntDictionary dictionary = new IntDictionary(nextIndexs[i]); //dictionary.SetDictionary(nextIndexs[i]); _nextIndex[i] = dictionary; } _wildcard = wildcard.ToArray(); _end = end.ToArray(); _resultIndex = resultIndex.ToArray(); allNode2.Clear(); allNode2 = null; }
protected internal virtual void Load(BinaryReader br) { var length = br.ReadInt32(); _keywords = new string[length]; for (int i = 0; i < length; i++) { _keywords[i] = br.ReadString(); } length = br.ReadInt32(); var bs = br.ReadBytes(length); _dict = ByteArrToIntArr(bs); length = br.ReadInt32(); bs = br.ReadBytes(length); _first = ByteArrToIntArr(bs); length = br.ReadInt32(); bs = br.ReadBytes(length); _end = ByteArrToIntArr(bs); length = br.ReadInt32(); bs = br.ReadBytes(length); _resultIndex = ByteArrToIntArr(bs); var dictLength = br.ReadInt32(); _nextIndex = new IntDictionary[dictLength]; List <int> max = new List <int>(); List <int> min = new List <int>(); for (int i = 0; i < dictLength; i++) { length = br.ReadInt32(); bs = br.ReadBytes(length); var keys = ByteArrToIntArr(bs); bs = br.ReadBytes(length); var values = ByteArrToIntArr(bs); var dict = new Dictionary <int, int>(); for (int j = 0; j < keys.Length; j++) { dict[keys[j]] = values[j]; } IntDictionary dictionary = new IntDictionary(); dictionary.SetDictionary(dict); _nextIndex[i] = dictionary; if (length == 0) { max.Add(0); min.Add(int.MaxValue); } else { max.Add(keys.Last()); min.Add(keys[0]); } } _max = max.ToArray(); _min = min.ToArray(); }
private void SetKeywords() { var root = new TrieNode(); Dictionary <int, List <TrieNode> > allNodeLayers = new Dictionary <int, List <TrieNode> >(); for (int i = 0; i < _keywords.Length; i++) { var p = _keywords[i]; var nd = root; for (int j = 0; j < p.Length; j++) { nd = nd.Add((char)p[j]); if (nd.Layer == 0) { nd.Layer = j + 1; List <TrieNode> trieNodes; if (allNodeLayers.TryGetValue(nd.Layer, out trieNodes) == false) { trieNodes = new List <TrieNode>(); allNodeLayers[nd.Layer] = trieNodes; } trieNodes.Add(nd); } } nd.SetResults(i); } List <TrieNode> allNode = new List <TrieNode>(); allNode.Add(root); foreach (var trieNodes in allNodeLayers) { foreach (var nd in trieNodes.Value) { allNode.Add(nd); } } allNodeLayers = null; for (int i = 1; i < allNode.Count; i++) { var nd = allNode[i]; nd.Index = i; TrieNode r = nd.Parent.Failure; char c = nd.Char; while (r != null && !r.m_values.ContainsKey(c)) { r = r.Failure; } if (r == null) { nd.Failure = root; } else { nd.Failure = r.m_values[c]; foreach (var result in nd.Failure.Results) { nd.SetResults(result); } } } root.Failure = root; StringBuilder stringBuilder = new StringBuilder(); for (int i = 1; i < allNode.Count; i++) { stringBuilder.Append(allNode[i].Char); } var length = CreateDict(stringBuilder.ToString()); stringBuilder = null; var allNode2 = new List <TrieNode2Ex>(); for (int i = 0; i < allNode.Count; i++) { allNode2.Add(new TrieNode2Ex() { Index = i }); } for (int i = 0; i < allNode2.Count; i++) { var oldNode = allNode[i]; var newNode = allNode2[i]; foreach (var item in oldNode.m_values) { var key = (char)_dict[item.Key]; var index = item.Value.Index; newNode.Add(key, allNode2[index]); } foreach (var item in oldNode.Results) { newNode.SetResults(item); } oldNode = oldNode.Failure; while (oldNode != root) { foreach (var item in oldNode.m_values) { var key = (char)_dict[item.Key]; var index = item.Value.Index; if (newNode.HasKey(key) == false) { newNode.Add(key, allNode2[index]); } } foreach (var item in oldNode.Results) { newNode.SetResults(item); } oldNode = oldNode.Failure; } } allNode.Clear(); allNode = null; root = null; var min = new List <int>(); var max = new List <int>(); var nextIndexs = new List <Dictionary <int, int> >(); var end = new List <int>() { 0 }; var resultIndex = new List <int>(); for (int i = 0; i < allNode2.Count; i++) { var dict = new Dictionary <int, int>(); var node = allNode2[i]; min.Add(node.minflag); max.Add(node.maxflag); if (i > 0) { foreach (var item in node.m_values) { dict[item.Key] = item.Value.Index; } } foreach (var item in node.Results) { resultIndex.Add(item); } end.Add(resultIndex.Count); nextIndexs.Add(dict); } var first = new int[Char.MaxValue + 1]; foreach (var item in allNode2[0].m_values) { first[item.Key] = item.Value.Index; } _first = first; _min = min.ToArray(); _max = max.ToArray(); _nextIndex = new IntDictionary[nextIndexs.Count]; for (int i = 0; i < nextIndexs.Count; i++) { IntDictionary dictionary = new IntDictionary(); dictionary.SetDictionary(nextIndexs[i]); _nextIndex[i] = dictionary; } _end = end.ToArray(); _resultIndex = resultIndex.ToArray(); allNode2.Clear(); allNode2 = null; }