public void SetUp()
{
if (Directory.Exists(TestUtils.CompiledPath))
{
Directory.Delete(TestUtils.CompiledPath, true);
}
Directory.CreateDirectory(TestUtils.CompiledPath);
using (var outputStream = File.Create(charDef))
{
CharacterDefinitionsCompiler compiler = new CharacterDefinitionsCompiler(CodePagesEncodingProvider.Instance);
string assetFileName = @"./Core/Resource/char.def";
using (var defStream = File.OpenRead(assetFileName))
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
compiler.ReadCharacterDefinition(defStream, "euc-jp");
categoryIdMap = Invert(compiler.MakeCharacterCategoryMap());
compiler.Compile(outputStream);
}
}
using (var input = File.OpenRead(charDef))
using (BinaryReader reader = new BinaryReader(input))
{
int[][] definitions = IntegerArrayIO.ReadSparseArray2D(reader);
int[][] mappings = IntegerArrayIO.ReadSparseArray2D(reader);
string[] symbols = StringArrayIO.ReadArray(reader);
characterDefinition = new CharacterDefinitions(definitions, mappings, symbols);
}
}
public static void Setup(TestContext context)
{
using (var charDefOutput = new MemoryStream())
using (var unkDicOutput = new MemoryStream())
using (var charDefResource = GetResource("char.def"))
using (var unkDefResource = GetResource("unk.def"))
{
var charDefCompiler = new CharacterDefinitionsCompiler(charDefOutput);
charDefCompiler.ReadCharacterDefinition(charDefResource, Encoding.GetEncoding("euc-jp"));
charDefCompiler.Compile();
var categoryMap = charDefCompiler.MakeCharacterCategoryMap();
var unkDefCompiler = new UnknownDictionaryCompiler(unkDicOutput, categoryMap);
unkDefCompiler.ReadUnknownDefinition(unkDefResource, Encoding.GetEncoding("euc-jp"));
unkDefCompiler.Compile();
charDefOutput.Seek(0, SeekOrigin.Begin);
unkDicOutput.Seek(0, SeekOrigin.Begin);
var definitions = IntArrayIO.ReadSparseArray2D(charDefOutput);
var mappings = IntArrayIO.ReadSparseArray2D(charDefOutput);
var symbols = StringArrayIO.ReadArray(charDefOutput);
CharacterDefinitions = new CharacterDefinitions(definitions, mappings, symbols);
Costs = IntArrayIO.ReadArray2D(unkDicOutput);
References = IntArrayIO.ReadArray2D(unkDicOutput);
Features = StringArrayIO.ReadArray2D(unkDicOutput);
UnknownDictionary = new UnknownDictionary(CharacterDefinitions, References, Costs, Features);
}
}
public void Compile(Stream output)
{
using (BinaryWriter writer = new BinaryWriter(output))
{
IntegerArrayIO.WriteArray2D(writer, MakeCosts());
IntegerArrayIO.WriteArray2D(writer, MakeCategoryReferences());
StringArrayIO.WriteArray2D(writer, MakeFeatures());
}
}
public static CharacterDefinitions NewInstance(IResourceResolver resolver)
{
using (var charDefInput = resolver.Resolve(CharacterDefinitionsFileName))
{
var definitions = IntArrayIO.ReadSparseArray2D(charDefInput);
var mappings = IntArrayIO.ReadSparseArray2D(charDefInput);
var symbols = StringArrayIO.ReadArray(charDefInput);
return(new CharacterDefinitions(definitions, mappings, symbols));
}
}
public void SetUp()
{
SortedDictionary <string, int> categoryMap;
using (var outputStream = File.Create(charDef))
{
CharacterDefinitionsCompiler charDefCompiler = new CharacterDefinitionsCompiler(CodePagesEncodingProvider.Instance);
string assetFileName = @"./Core/Resource/char.def";
using (var defStream = File.OpenRead(assetFileName))
{
charDefCompiler.ReadCharacterDefinition(defStream, "euc-jp");
charDefCompiler.Compile(outputStream);
}
categoryMap = charDefCompiler.MakeCharacterCategoryMap();
}
var unkDefFile = TestUtils.CompiledPath + Path.DirectorySeparatorChar + "kuromoji-unkdef-.bin";
using (var outputStream = File.Create(unkDefFile))
{
UnknownDictionaryCompiler unkDefCompiler = new UnknownDictionaryCompiler(categoryMap);
string assetFileName = @"./Core/Resource/unk.def";
using (var defStream = File.OpenRead(assetFileName))
{
unkDefCompiler.ReadUnknownDefinition(defStream, "euc-jp");
unkDefCompiler.Compile(outputStream);
}
}
using (var charDefInput = File.OpenRead(charDef))
using (var reader = new BinaryReader(charDefInput))
{
int[][] definitions = IntegerArrayIO.ReadSparseArray2D(reader);
int[][] mappings = IntegerArrayIO.ReadSparseArray2D(reader);
string[] symbols = StringArrayIO.ReadArray(reader);
characterDefinitions = new CharacterDefinitions(definitions, mappings, symbols);
}
using (var unkDefInput = File.OpenRead(unkDefFile))
using (var reader = new BinaryReader(unkDefInput))
{
costs = IntegerArrayIO.ReadArray2D(reader);
references = IntegerArrayIO.ReadArray2D(reader);
features = StringArrayIO.ReadArray2D(reader);
unknownDictionary = new UnknownDictionary(characterDefinitions, references, costs, features);
}
}
/// <summary>
/// Different with the original java ver. We need to pass an output stream into this mehod
/// </summary>
public void Compile(Stream output)
{
try
{
using (BinaryWriter dataOut = new BinaryWriter(output))
{
IntegerArrayIO.WriteSparseArray2D(dataOut, MakeCharacterDefinitions());
IntegerArrayIO.WriteSparseArray2D(dataOut, MakeCharacterMappings());
StringArrayIO.WriteArray(dataOut, MakeCharacterCategorySymbols());
}
}
catch (IOException ex)
{
throw new IOException("CharacterDefinitionsCompiler.Compile: " + ex.Message);
}
}
public static UnknownDictionary NewInstance(IResourceResolver resolver, CharacterDefinitions characterDefinitions, int totalFeatures)
{
using (var unkDefInput = resolver.Resolve(UnknownDictionaryFileName))
{
var costs = IntArrayIO.ReadArray2D(unkDefInput);
var references = IntArrayIO.ReadArray2D(unkDefInput);
var features = StringArrayIO.ReadArray2D(unkDefInput);
return(new UnknownDictionary(
characterDefinitions,
references,
costs,
features,
totalFeatures
));
}
}
public static CharacterDefinitions NewInstance(string resourceAbsolutePath)
{
try
{
using (Stream charDefInput = File.OpenRead(resourceAbsolutePath + Path.DirectorySeparatorChar + CHARACTER_DEFINITIONS_FILENAME))
using (BinaryReader reader = new BinaryReader(charDefInput))
{
int[][] definitions = IntegerArrayIO.ReadSparseArray2D(reader);
int[][] mappings = IntegerArrayIO.ReadSparseArray2D(reader);
string[] symbols = StringArrayIO.ReadArray(reader);
CharacterDefinitions characterDefinition = new CharacterDefinitions(definitions, mappings, symbols);
return(characterDefinition);
}
}
catch (IOException ex)
{
throw new IOException("CharacterDefinitions.NewInstance: " + ex.Message);
}
}
public void Setup()
{
CharDefFileName = Path.GetTempFileName();
using (var fs = new FileStream(CharDefFileName, FileMode.Create, FileAccess.ReadWrite))
using (var resource = GetResource("char.def"))
{
var compiler = new CharacterDefinitionsCompiler(fs);
compiler.ReadCharacterDefinition(resource, Encoding.GetEncoding("euc-jp"));
CategoryIdMap = Invert(compiler.MakeCharacterCategoryMap());
compiler.Compile();
}
using (var fs = new FileStream(CharDefFileName, FileMode.Open, FileAccess.Read))
{
var definitions = IntArrayIO.ReadSparseArray2D(fs);
var mappings = IntArrayIO.ReadSparseArray2D(fs);
var symbols = StringArrayIO.ReadArray(fs);
CharacterDefinition = new CharacterDefinitions(definitions, mappings, symbols);
}
}
public static UnknownDictionary NewInstance(string absoluteFolderPath,
CharacterDefinitions characterDefinitions,
int totalFeatures)
{
string filePath = absoluteFolderPath + Path.DirectorySeparatorChar + UnknownDictionary.UNKNOWN_DICTIONARY_FILENAME;
using (Stream unkDefInput = File.OpenRead(filePath))
using (BinaryReader reader = new BinaryReader(unkDefInput))
{
int[][] costs = IntegerArrayIO.ReadArray2D(reader);
int[][] references = IntegerArrayIO.ReadArray2D(reader);
string[][] features = StringArrayIO.ReadArray2D(reader);
UnknownDictionary unknownDictionary = new UnknownDictionary(
characterDefinitions,
references,
costs,
features,
totalFeatures
);
return(unknownDictionary);
}
}
public void Compile()
{
IntArrayIO.WriteSparseArray2D(Output, MakeCharacterDefinitions());
IntArrayIO.WriteSparseArray2D(Output, MakeCharacterMappings());
StringArrayIO.WriteArray(Output, MakeCharacterCategorySymbols());
}
public void Compile()
{
IntArrayIO.WriteArray2D(Output, MakeCosts());
IntArrayIO.WriteArray2D(Output, MakeCategoryReferences());
StringArrayIO.WriteArray2D(Output, MakeFeatures());
}
