public static void Setup(TestContext context) { using (var charDefOutput = new MemoryStream()) using (var unkDicOutput = new MemoryStream()) using (var charDefResource = GetResource("char.def")) using (var unkDefResource = GetResource("unk.def")) { var charDefCompiler = new CharacterDefinitionsCompiler(charDefOutput); charDefCompiler.ReadCharacterDefinition(charDefResource, Encoding.GetEncoding("euc-jp")); charDefCompiler.Compile(); var categoryMap = charDefCompiler.MakeCharacterCategoryMap(); var unkDefCompiler = new UnknownDictionaryCompiler(unkDicOutput, categoryMap); unkDefCompiler.ReadUnknownDefinition(unkDefResource, Encoding.GetEncoding("euc-jp")); unkDefCompiler.Compile(); charDefOutput.Seek(0, SeekOrigin.Begin); unkDicOutput.Seek(0, SeekOrigin.Begin); var definitions = IntArrayIO.ReadSparseArray2D(charDefOutput); var mappings = IntArrayIO.ReadSparseArray2D(charDefOutput); var symbols = StringArrayIO.ReadArray(charDefOutput); CharacterDefinitions = new CharacterDefinitions(definitions, mappings, symbols); Costs = IntArrayIO.ReadArray2D(unkDicOutput); References = IntArrayIO.ReadArray2D(unkDicOutput); Features = StringArrayIO.ReadArray2D(unkDicOutput); UnknownDictionary = new UnknownDictionary(CharacterDefinitions, References, Costs, Features); } }
public WordIdMap(Stream input) { var arrays = IntArrayIO.ReadArrays(input, 2); Indices = arrays[0]; WordIds = arrays[1]; }
public static CharacterDefinitions NewInstance(IResourceResolver resolver) { using (var charDefInput = resolver.Resolve(CharacterDefinitionsFileName)) { var definitions = IntArrayIO.ReadSparseArray2D(charDefInput); var mappings = IntArrayIO.ReadSparseArray2D(charDefInput); var symbols = StringArrayIO.ReadArray(charDefInput); return(new CharacterDefinitions(definitions, mappings, symbols)); } }
public static UnknownDictionary NewInstance(IResourceResolver resolver, CharacterDefinitions characterDefinitions, int totalFeatures) { using (var unkDefInput = resolver.Resolve(UnknownDictionaryFileName)) { var costs = IntArrayIO.ReadArray2D(unkDefInput); var references = IntArrayIO.ReadArray2D(unkDefInput); var features = StringArrayIO.ReadArray2D(unkDefInput); return(new UnknownDictionary( characterDefinitions, references, costs, features, totalFeatures )); } }
public void Setup() { CharDefFileName = Path.GetTempFileName(); using (var fs = new FileStream(CharDefFileName, FileMode.Create, FileAccess.ReadWrite)) using (var resource = GetResource("char.def")) { var compiler = new CharacterDefinitionsCompiler(fs); compiler.ReadCharacterDefinition(resource, Encoding.GetEncoding("euc-jp")); CategoryIdMap = Invert(compiler.MakeCharacterCategoryMap()); compiler.Compile(); } using (var fs = new FileStream(CharDefFileName, FileMode.Open, FileAccess.Read)) { var definitions = IntArrayIO.ReadSparseArray2D(fs); var mappings = IntArrayIO.ReadSparseArray2D(fs); var symbols = StringArrayIO.ReadArray(fs); CharacterDefinition = new CharacterDefinitions(definitions, mappings, symbols); } }
public void Compile() { IntArrayIO.WriteSparseArray2D(Output, MakeCharacterDefinitions()); IntArrayIO.WriteSparseArray2D(Output, MakeCharacterMappings()); StringArrayIO.WriteArray(Output, MakeCharacterCategorySymbols()); }
public void Compile() { IntArrayIO.WriteArray2D(Output, MakeCosts()); IntArrayIO.WriteArray2D(Output, MakeCategoryReferences()); StringArrayIO.WriteArray2D(Output, MakeFeatures()); }
public void Write(Stream output) { Compile(); IntArrayIO.WriteArray(output, Indices); IntArrayIO.WriteArray(output, WordIdArray.GetArray()); }