public void SetUp() { if (Directory.Exists(TestUtils.CompiledPath)) { Directory.Delete(TestUtils.CompiledPath, true); } Directory.CreateDirectory(TestUtils.CompiledPath); using (var outputStream = File.Create(charDef)) { CharacterDefinitionsCompiler compiler = new CharacterDefinitionsCompiler(CodePagesEncodingProvider.Instance); string assetFileName = @"./Core/Resource/char.def"; using (var defStream = File.OpenRead(assetFileName)) { Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); compiler.ReadCharacterDefinition(defStream, "euc-jp"); categoryIdMap = Invert(compiler.MakeCharacterCategoryMap()); compiler.Compile(outputStream); } } using (var input = File.OpenRead(charDef)) using (BinaryReader reader = new BinaryReader(input)) { int[][] definitions = IntegerArrayIO.ReadSparseArray2D(reader); int[][] mappings = IntegerArrayIO.ReadSparseArray2D(reader); string[] symbols = StringArrayIO.ReadArray(reader); characterDefinition = new CharacterDefinitions(definitions, mappings, symbols); } }
public void Compile(Stream output) { using (BinaryWriter writer = new BinaryWriter(output)) { IntegerArrayIO.WriteArray2D(writer, MakeCosts()); IntegerArrayIO.WriteArray2D(writer, MakeCategoryReferences()); StringArrayIO.WriteArray2D(writer, MakeFeatures()); } }
public void SetUp() { SortedDictionary <string, int> categoryMap; using (var outputStream = File.Create(charDef)) { CharacterDefinitionsCompiler charDefCompiler = new CharacterDefinitionsCompiler(CodePagesEncodingProvider.Instance); string assetFileName = @"./Core/Resource/char.def"; using (var defStream = File.OpenRead(assetFileName)) { charDefCompiler.ReadCharacterDefinition(defStream, "euc-jp"); charDefCompiler.Compile(outputStream); } categoryMap = charDefCompiler.MakeCharacterCategoryMap(); } var unkDefFile = TestUtils.CompiledPath + Path.DirectorySeparatorChar + "kuromoji-unkdef-.bin"; using (var outputStream = File.Create(unkDefFile)) { UnknownDictionaryCompiler unkDefCompiler = new UnknownDictionaryCompiler(categoryMap); string assetFileName = @"./Core/Resource/unk.def"; using (var defStream = File.OpenRead(assetFileName)) { unkDefCompiler.ReadUnknownDefinition(defStream, "euc-jp"); unkDefCompiler.Compile(outputStream); } } using (var charDefInput = File.OpenRead(charDef)) using (var reader = new BinaryReader(charDefInput)) { int[][] definitions = IntegerArrayIO.ReadSparseArray2D(reader); int[][] mappings = IntegerArrayIO.ReadSparseArray2D(reader); string[] symbols = StringArrayIO.ReadArray(reader); characterDefinitions = new CharacterDefinitions(definitions, mappings, symbols); } using (var unkDefInput = File.OpenRead(unkDefFile)) using (var reader = new BinaryReader(unkDefInput)) { costs = IntegerArrayIO.ReadArray2D(reader); references = IntegerArrayIO.ReadArray2D(reader); features = StringArrayIO.ReadArray2D(reader); unknownDictionary = new UnknownDictionary(characterDefinitions, references, costs, features); } }
/// <summary> /// Different with the original java ver. We need to pass an output stream into this mehod /// </summary> public void Compile(Stream output) { try { using (BinaryWriter dataOut = new BinaryWriter(output)) { IntegerArrayIO.WriteSparseArray2D(dataOut, MakeCharacterDefinitions()); IntegerArrayIO.WriteSparseArray2D(dataOut, MakeCharacterMappings()); StringArrayIO.WriteArray(dataOut, MakeCharacterCategorySymbols()); } } catch (IOException ex) { throw new IOException("CharacterDefinitionsCompiler.Compile: " + ex.Message); } }
public WordIdMap(Stream input) { try { lock (input) { BinaryReader reader = new BinaryReader(input); int[][] arrays = IntegerArrayIO.ReadArrays(reader, 2); indices = arrays[0]; wordIds = arrays[1]; } } catch (Exception ex) { throw new IOException("WordIdMap Contructor: " + ex.Message); } }
public void Write(Stream output) { try { lock (output) { BinaryWriter writer = new BinaryWriter(output); Compile(null); IntegerArrayIO.WriteArray(writer, indices); IntegerArrayIO.WriteArray(writer, wordIdArray.GetArray()); } } catch (IOException ex) { throw new Exception("WordIdMapCompiler.Write: " + ex.Message); } }
public static CharacterDefinitions NewInstance(string resourceAbsolutePath) { try { using (Stream charDefInput = File.OpenRead(resourceAbsolutePath + Path.DirectorySeparatorChar + CHARACTER_DEFINITIONS_FILENAME)) using (BinaryReader reader = new BinaryReader(charDefInput)) { int[][] definitions = IntegerArrayIO.ReadSparseArray2D(reader); int[][] mappings = IntegerArrayIO.ReadSparseArray2D(reader); string[] symbols = StringArrayIO.ReadArray(reader); CharacterDefinitions characterDefinition = new CharacterDefinitions(definitions, mappings, symbols); return(characterDefinition); } } catch (IOException ex) { throw new IOException("CharacterDefinitions.NewInstance: " + ex.Message); } }
public static UnknownDictionary NewInstance(string absoluteFolderPath, CharacterDefinitions characterDefinitions, int totalFeatures) { string filePath = absoluteFolderPath + Path.DirectorySeparatorChar + UnknownDictionary.UNKNOWN_DICTIONARY_FILENAME; using (Stream unkDefInput = File.OpenRead(filePath)) using (BinaryReader reader = new BinaryReader(unkDefInput)) { int[][] costs = IntegerArrayIO.ReadArray2D(reader); int[][] references = IntegerArrayIO.ReadArray2D(reader); string[][] features = StringArrayIO.ReadArray2D(reader); UnknownDictionary unknownDictionary = new UnknownDictionary( characterDefinitions, references, costs, features, totalFeatures ); return(unknownDictionary); } }