internal Word(
StringPointerBuilder stringPointerBuilder,
string hanzi, string traditional, string thumbPinyin, string thumbTranslation, string radicals, string link,
MeaningListBuilder meaningBuilder, StringPointerBuilder tagBuilder, ulong pinyinMask)
{
WordStart = stringPointerBuilder.StringPointers.Count;
stringPointerBuilder.Add(hanzi);
stringPointerBuilder.Add(traditional);
stringPointerBuilder.Add(thumbPinyin);
stringPointerBuilder.Add(thumbTranslation);
stringPointerBuilder.Add(radicals);
stringPointerBuilder.Add(link);
stringPointerBuilder.Append(meaningBuilder.Pinyins);
stringPointerBuilder.Append(meaningBuilder.Translations);
stringPointerBuilder.Append(tagBuilder);
PinyinListLength = (byte)meaningBuilder.Pinyins.StringPointers.Count;
TranslationListLength = (byte)meaningBuilder.Translations.StringPointers.Count;
TagListLength = (byte)tagBuilder.StringPointers.Count;
MeaningsMemory = new MeaningListMemory(meaningBuilder);
PinyinMask = pinyinMask;
}
private async Task LoadDictionaryTask()
{
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
// spawn jobs
List <DictionaryLoadJob> dictionaryLoadJobs = await GetDictionaryLoadJobs();
List <Task> jobTasks = new List <Task>();
foreach (DictionaryLoadJob dictionaryLoadJob in dictionaryLoadJobs)
{
jobTasks.Add(Task.Run((Action)dictionaryLoadJob.LoadDictionaryPart, LoadTaskCanceller.Token));
}
// wait for all jobs to complete
Task.WaitAll(jobTasks.ToArray(), LoadTaskCanceller.Token);
// merge results
StringPointerBuilder stringBuilder = new StringPointerBuilder();
List <MeaningMemory> meaningMemoryBuilder = new List <MeaningMemory>();
List <Word> words = new List <Word>();
foreach (DictionaryLoadJob dictionaryLoadJob in dictionaryLoadJobs)
{
foreach (Word word in dictionaryLoadJob.Words)
{
word.WordStart += stringBuilder.StringPointers.Count;
word.MeaningsMemory.MeaningStart += meaningMemoryBuilder.Count;
}
words.AddRange(dictionaryLoadJob.Words);
stringBuilder.Append(dictionaryLoadJob.StringBuilder);
meaningMemoryBuilder.AddRange(dictionaryLoadJob.MeaningBuilder.MeaningMemory);
}
WordDictionary.SetDictionary(
DictionaryName,
words,
stringBuilder.StringBuilder.ToString(),
stringBuilder.StringPointers.ToArray(),
meaningMemoryBuilder.ToArray());
foreach (DictionaryLoadJob dictionaryLoadJob in dictionaryLoadJobs)
{
dictionaryLoadJob.Dispose();
}
SortDictionaryWords();
await BuildIndex();
stopwatch.Stop();
Debug.WriteLine("Loaded dictionary {1} in 0.{0:fffffff}", stopwatch.Elapsed, DictionaryName);
WordDictionary.Loading.IsCompleted = true;
WordDictionary.Loading.DictionaryLoadedNotifier.Set();
}
protected virtual void SortDictionaryWords()
{
// sort
WordDictionary._Words.Sort();
// repack everything...
StringPointerBuilder stringBuilder = new StringPointerBuilder();
MeaningListBuilder meaningBuilder = new MeaningListBuilder();
StringPointerBuilder pinyinBuilder = new StringPointerBuilder();
StringPointerBuilder translationBuilder = new StringPointerBuilder();
StringPointerBuilder tagBuilder = new StringPointerBuilder();
List <Word> newWords = new List <Word>();
foreach (Word word in WordDictionary.Words)
{
foreach (Meaning meaning in word.Meanings)
{
foreach (StringPointer pinyin in meaning.Pinyins)
{
pinyinBuilder.Add(pinyin);
}
foreach (StringPointer translation in meaning.Translations)
{
translationBuilder.Add(translation);
}
meaningBuilder.Add(pinyinBuilder, translationBuilder);
pinyinBuilder.Clear();
translationBuilder.Clear();
}
foreach (StringPointer tag in word.Tags)
{
tagBuilder.Add(tag);
}
newWords.Add(new Word(stringBuilder,
word.Hanzi, word.Traditional, word.ThumbPinyin, word.ThumbTranslation, word.Radicals, word.Link, meaningBuilder, tagBuilder, word.PinyinMask));
meaningBuilder.Clear();
tagBuilder.Clear();
}
WordDictionary.SetDictionary(
DictionaryName,
newWords,
stringBuilder.StringBuilder.ToString(),
stringBuilder.StringPointers.ToArray(),
meaningBuilder.MeaningMemory.ToArray());
}
internal MeaningMemory(StringPointerBuilder pinyins, StringPointerBuilder translations)
{
PinyinCount = (byte)pinyins.StringPointers.Count;
TranslationCount = (byte)translations.StringPointers.Count;
}
public override void LoadDictionaryPart()
{
BinaryReader reader = new BinaryReader(LoadStream, Encoding.UTF8);
StringPointerBuilder pinyinsBuilder = new StringPointerBuilder();
StringPointerBuilder translationsBuilder = new StringPointerBuilder();
StringPointerBuilder tagsBuilder = new StringPointerBuilder();
List <string> thumbPinyinBuilder = new List <string>();
List <string> thumbTranslationBuilder = new List <string>();
while (LoadStream.Position < LoadStream.Length)
{
string hanzi = string.Empty;
string traditional = string.Empty;
string radicals = string.Empty;
string link = string.Empty;
string thumbPinyin = string.Empty;
string thumbTranslation = string.Empty;
hanzi = reader.ReadString();
traditional = reader.ReadString();
if (Version >= 1)
{
radicals = reader.ReadString();
}
link = reader.ReadString();
thumbPinyin = reader.ReadString();
thumbTranslation = reader.ReadString();
byte meaningCount = reader.ReadByte();
for (byte i = 0; i < meaningCount; ++i)
{
byte pinyinCount = reader.ReadByte();
for (byte j = 0; j < pinyinCount; ++j)
{
string pinyin = reader.ReadString();
pinyinsBuilder.Add(pinyin);
thumbPinyinBuilder.Add(pinyin);
}
byte translationCount = reader.ReadByte();
for (byte j = 0; j < translationCount; ++j)
{
string translation = reader.ReadString();
translationsBuilder.Add(translation);
if (MeaningBuilder.MeaningMemory.Count == MeaningBuilder.MeaningStart)
{
thumbTranslationBuilder.Add(translation);
}
}
MeaningBuilder.Add(pinyinsBuilder, translationsBuilder);
pinyinsBuilder.Clear();
translationsBuilder.Clear();
}
byte tagCount = reader.ReadByte();
for (byte i = 0; i < tagCount; ++i)
{
tagsBuilder.Add(reader.ReadString());
}
if (thumbPinyin == string.Empty)
{
thumbPinyin = BuildThumb(thumbPinyinBuilder);
}
if (thumbTranslation == string.Empty)
{
thumbTranslation = BuildThumb(thumbTranslationBuilder);
}
ulong pinyinMask = 0;
if (Version >= 3)
{
pinyinMask = reader.ReadUInt64();
}
else
{
foreach (string pinyin in thumbPinyinBuilder)
{
pinyinMask |= pinyin.LetterMask();
}
}
Words.Add(new Word(StringBuilder,
hanzi, traditional, thumbPinyin, thumbTranslation, radicals, link,
MeaningBuilder, tagsBuilder, pinyinMask));
tagsBuilder.Clear();
MeaningBuilder.Clear();
thumbPinyinBuilder.Clear();
thumbTranslationBuilder.Clear();
}
}
public override void LoadDictionaryPart()
{
XmlReader reader = XmlReader.Create(LoadStream);
StringPointerBuilder pinyinBuilder = new StringPointerBuilder();
StringPointerBuilder translationBuilder = new StringPointerBuilder();
StringPointerBuilder tagBuilder = new StringPointerBuilder();
List <string> thumbPinyinBuilder = new List <string>();
List <string> thumbTranslationBuilder = new List <string>();
while (reader.ReadToFollowing("Word"))
{
JobCancellationToken.ThrowIfCancellationRequested();
string hanzi = string.Empty;
string traditional = string.Empty;
string radicals = string.Empty;
string link = string.Empty;
string thumbPinyin = string.Empty;
string thumbTranslation = string.Empty;
reader.ReadToFollowing("Hanzi");
hanzi = reader.ReadElementContentAsString();
Cursor cursor = Cursor.Traditional;
int wordDepth = reader.Depth;
while (reader.Read() && reader.Depth >= wordDepth)
{
if (reader.NodeType == XmlNodeType.Element)
{
string name = reader.Name;
if (cursor <= Cursor.Traditional && name == "Traditional")
{
traditional = reader.ReadElementContentAsString();
cursor = Cursor.Traditional + 1;
}
else if (cursor <= Cursor.Radicals && name == "Radicals")
{
radicals = reader.ReadElementContentAsString();
cursor = Cursor.Radicals + 1;
}
else if (cursor <= Cursor.Link && name == "Link")
{
link = reader.ReadElementContentAsString();
cursor = Cursor.Link + 1;
}
else if (cursor <= Cursor.ThumbPinyin && name == "ThumbPinyin")
{
thumbPinyin = reader.ReadElementContentAsString();
cursor = Cursor.ThumbPinyin + 1;
}
else if (cursor <= Cursor.ThumbTranslation && name == "ThumbTranslation")
{
thumbTranslation = reader.ReadElementContentAsString();
cursor = Cursor.ThumbTranslation + 1;
}
else if (cursor <= Cursor.Meaning && name == "Meaning")
{
int meaningDepth = reader.Depth;
cursor = Cursor.ThumbPinyin;
while (reader.Read() && reader.Depth > meaningDepth)
{
if (reader.NodeType == XmlNodeType.Element)
{
if (cursor <= Cursor.MeaningPinyin && reader.Name == "Pinyin")
{
string pinyin = reader.ReadElementContentAsString();
pinyinBuilder.Add(pinyin);
thumbPinyinBuilder.Add(pinyin);
}
else
{
cursor = Cursor.MeaningTranslation;
string translation = reader.ReadElementContentAsString();
translationBuilder.Add(translation);
if (MeaningBuilder.MeaningMemory.Count == MeaningBuilder.MeaningStart)
{
thumbTranslationBuilder.Add(translation);
}
}
}
}
MeaningBuilder.Add(pinyinBuilder, translationBuilder);
pinyinBuilder.Clear();
translationBuilder.Clear();
cursor = Cursor.Meaning;
}
else// if (cursor <= Cursor.Tag && name == "Tag")
{
// tag is the last element, we do not need to check
cursor = Cursor.Tag;
tagBuilder.Add(reader.ReadElementContentAsString());
}
}
}
if (thumbPinyin == string.Empty)
{
thumbPinyin = BuildThumb(thumbPinyinBuilder);
}
if (thumbTranslation == string.Empty)
{
thumbTranslation = BuildThumb(thumbTranslationBuilder);
}
ulong pinyinMask = 0;
foreach (string pinyin in thumbPinyinBuilder)
{
pinyinMask |= pinyin.LetterMask();
}
Words.Add(new Word(StringBuilder,
hanzi, traditional, thumbPinyin, thumbTranslation, radicals, link,
MeaningBuilder, tagBuilder, pinyinMask));
tagBuilder.Clear();
MeaningBuilder.Clear();
thumbPinyinBuilder.Clear();
thumbTranslationBuilder.Clear();
}
reader.Dispose();
}
