/// <summary>
/// High First
/// </summary>
/// <param name="path"></param>
/// <param name="map"></param>
/// <returns></returns>
public static bool SaveDat(string path, SortedDictionary <string, string> map)
{
var fs = new FileStream(path + ".bi" + Predefine.BIN_EXT, FileMode.Create, FileAccess.Write);
try
{
var values = map.Values;
var bytes = ByteUtil.Int2Byte_HighFirst(values.Count);
fs.Write(bytes, 0, 4);
foreach (var v in values)
{
bytes = ByteUtil.UTF2Byte_HighFirst(v);
fs.Write(bytes, 0, bytes.Length);
}
return(_trie.Save(fs));
}
catch (Exception e)
{
return(false);
}
finally
{
fs.Close();
}
}
public void BuildTest()
{
var path = Path.Combine(Config.DataRootPath, _testFile);
if (File.Exists(path))
{
File.Delete(path);
}
DoubleArrayTrie <string> trie = new DoubleArrayTrie <string>();
var errorCount = trie.Build(_mockData.Keys.ToList(), _mockData.Values.ToList());
Assert.Equal(errorCount, 0);
trie.Save(Path.Combine(Config.DataRootPath, _testFile));
}
private static bool SaveDat(SortedDictionary <string, bool> map)
{
var fs = new FileStream(Config.Translated_Person_Dict_Path + Predefine.BIN_EXT, FileMode.Create, FileAccess.Write);
try
{
return(_trie.Save(fs));
}
catch (Exception e)
{
return(false);
}
finally
{
fs.Close();
}
}
public static void Load(string path)
{
_trie = new DoubleArrayTrie <int>();
var valueArr = LoadDat(path + ".value.dat");
if (valueArr != null)
{
if (_trie.Load(path + ".trie.dat", valueArr))
{
return;
}
}
var map = new SortedDictionary <string, int>(StrComparer.Default);
foreach (var line in File.ReadLines(path))
{
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
var segs = line.Split(new[] { ' ', '\t' }, StringSplitOptions.RemoveEmptyEntries);
map[segs[0]] = int.Parse(segs[1]);
}
_trie = new DoubleArrayTrie <int>();
_trie.Build(map);
valueArr = new int[map.Count];
int m = 0;
foreach (var v in map.Values)
{
valueArr[m++] = v;
}
var fs = new FileStream(path + ".trie.dat", FileMode.Create, FileAccess.Write);
_trie.Save(fs);
fs.Close();
SaveDat(path + ".value.dat", valueArr);
}
/// <summary>
/// 保存二进制数据到文件
/// 先保存values，然后保存keys
/// </summary>
/// <param name="path"></param>
/// <param name="dict"></param>
/// <returns></returns>
private static bool SaveDat(string path, SortedDictionary <string, WordAttr> dict)
{
var fs = new FileStream(path + Predefine.BIN_EXT, FileMode.Create, FileAccess.Write);
try
{
var bytes = BitConverter.GetBytes(dict.Count);
fs.Write(bytes, 0, 4);
// -------------- save values ---------------
foreach (var p in dict)
{
var attr = p.Value;
bytes = BitConverter.GetBytes(attr.totalFreq);
fs.Write(bytes, 0, 4);
bytes = BitConverter.GetBytes(attr.natures.Length);
for (int i = 0; i < attr.natures.Length; i++)
{
bytes = BitConverter.GetBytes((int)attr.natures[i]);
fs.Write(bytes, 0, 4);
bytes = BitConverter.GetBytes(attr.freqs[i]);
fs.Write(bytes, 0, 4);
}
}
// ------------- value saving finish ------------------
// -------------- save keys --------------------------
return(_trie.Save(fs));
}
catch (Exception e)
{
// log warning "save failed"
return(false);
}
finally
{
fs.Close();
}
}
public static bool Load(string path)
{
try
{
_trie = new DoubleArrayTrie <AreaInfo>();
var valueArr = LoadDat(path + ".value.dat");
if (valueArr != null)
{
if (_trie.Load(path + ".trie.dat", valueArr))
{
return(true);
}
}
// 读取txt文件
var map = new SortedDictionary <string, AreaInfo>(StrComparer.Default);
foreach (var line in File.ReadLines(path))
{
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
var segs = line.Split(new[] { ' ', '\t' }, StringSplitOptions.RemoveEmptyEntries);
var code = segs[0];
for (int i = 1; i < segs.Length; i++)
{
var name = segs[i];
if (Invalids.Contains(name))
{
continue; // 跳过无效地区名
}
if (name.Length == 2)
{
AddInMap(name, "", code, map);
}
else
{
var lastChar = name[name.Length - 1];
if ("市省县区州旗盟".Contains(lastChar))
{
AddInMap(name.Substring(0, name.Length - 1), lastChar.ToString(), code, map);
}
else if (name.Length < 9)
{
AddInMap(name, "", code, map);
}
var lastTwo = name.Substring(2);
var prevs = name.Substring(0, name.Length - 2);
if (Invalids.Contains(lastTwo))
{
AddInMap(prevs, lastTwo, code, map);
if (prevs.Length == 3 && "市省".Contains(prevs[2]))
{
AddInMap(name.Substring(0, 2), lastTwo, code, map);
}
}
if (lastChar == '旗')
{
var sublast = name[2];
if ("前后左中右特".Contains(sublast))
{
AddInMap(prevs, "旗", code, map);
}
}
var subLastTwo = name.Substring(name.Length - 3, 2);
if (subLastTwo == "自治")
{
prevs = name.Substring(0, name.Length - 3);
var ends = name.Substring(name.Length - 3);
AddInMap(prevs, ends, code, map);
if (prevs.Length >= 4)
{
for (int k = 2; k < prevs.Length - 1; k++)
{
if (k < prevs.Length - 3)
{
if (Nationalities.Contains(prevs.Substring(k, 4)))
{
AddInMap(prevs.Substring(0, k), ends, code, map);
AddInMap(prevs.Substring(0, k) + "自治", lastChar.ToString(), code, map);
break;
}
}
if (k < prevs.Length - 2)
{
if (Nationalities.Contains(prevs.Substring(k, 3)))
{
AddInMap(prevs.Substring(0, k), ends, code, map);
AddInMap(prevs.Substring(0, k) + "自治", lastChar.ToString(), code, map);
break;
}
}
if (Nationalities.Contains(prevs.Substring(k, 2)))
{
AddInMap(prevs.Substring(0, k), ends, code, map);
AddInMap(prevs.Substring(0, k) + "自治", lastChar.ToString(), code, map);
break;
}
}
}
}
}
}
}
_trie = new DoubleArrayTrie <AreaInfo>();
_trie.Build(map);
valueArr = new AreaInfo[map.Count];
int m = 0;
foreach (var v in map.Values)
{
valueArr[m++] = v;
}
var fs = new FileStream(path + ".trie.dat", FileMode.Create, FileAccess.Write);
_trie.Save(fs);
fs.Close();
SaveDat(path + ".value.dat", valueArr);
return(true);
}
catch (Exception e)
{
return(false);
}
}
