private void cmdManaged_Click(object sender, EventArgs e)
{
//some lang eg. Thai, Lao, need dictionary breaking
//we use dic data from icu-project
//1. create dictionary based breaking engine
//TODO: dic should be read once
var dicProvider = new IcuSimpleTextFileDictionaryProvider()
{
DataDir = "../../../icu58/brkitr_src/dictionaries"
};
CustomBreakerBuilder.Setup(dicProvider);
CustomBreaker breaker1 = CustomBreakerBuilder.NewCustomBreaker();
char[] test = this.textBox1.Text.ToCharArray();
this.listBox1.Items.Clear();
breaker1.BreakWords(test, 0, test.Length);
foreach (BreakSpan span in breaker1.GetBreakSpanIter())
{
string s = new string(test, span.startAt, span.len);
this.listBox1.Items.Add(span.startAt + " " + s);
}
}
private void Form1_Load(object sender, EventArgs e)
{
InitIcuLib();
//thai
//currentLocale = "th-TH";
//string test1 = "ผู้ใหญ่บหาผ้าใหม่ให้สะใภ้ใช้คล้องคอใฝ่ใจเอาใส่ห่อมิหลงใหลใครขอดูจะใคร่ลงเรือใบดูน้ำใสและปลาปูสิ่งใดอยู่ในตู้มิใช่อยู่ใต้ตั่งเตียงบ้าใบถือใยบัวหูตามัวมาให้เคียงเล่าเท่าอย่าละเลี่ยงยี่สิบม้วนจำจงดี";
// string test1 = "ขาย =";
//string test1 = "แป้นพิมลาว";
//string test1 = "ผ้าใหม่";
//string test1 = "ประ";
//----------------
//
//lao
icu_currentLocale = "lo-LA";
//string test1 = "ສະບາຍດີແປ້ນພິມລາວ";
//string test1 = "ສາທາລະນະລັດ ປະຊາທິປະໄຕ ປະຊາຊົນລາວ";
string test1 = "ABCD1234567890ສາທາລະນະລັດ ປະຊາທິປະໄຕ ປະຊາຊົນລາວ ผู้ใหญ่หาผ้าใหม่";
//----------------
this.textBox1.Text = test1;
//-------
CustomBreakerBuilder.Setup("../../../icu61/brkitr/dictionaries");
}
private void cmdManaged_Click(object sender, EventArgs e)
{
CustomBreakerBuilder.DataDir = "../../../icu58/brkitr_src/dictionaries";
CustomBreaker breaker1 = CustomBreakerBuilder.NewCustomBreaker();
//2. create dictionary based breaking engine
//
char[] test = this.textBox1.Text.ToCharArray();
this.listBox1.Items.Clear();
breaker1.BreakWords(test, 0);
foreach (var span in breaker1.GetBreakSpanIter())
{
string s = new string(test, span.startAt, span.len);
this.listBox1.Items.Add(span.startAt + " " + s);
}
}
void ParseWithManaged(int ntimes)
{
//-------------------
CustomBreakerBuilder.DataDir = "../../../icu58/brkitr_src/dictionaries";
CustomBreaker breaker1 = CustomBreakerBuilder.NewCustomBreaker();
char[] test = this.textBox1.Text.ToCharArray();
//-------------
for (int i = ntimes - 1; i >= 0; --i)
{
breaker1.BreakWords(test, 0);
foreach (var span in breaker1.GetBreakSpanIter())
{
}
}
}
void InitNewCustomTextBreakerAndBreakWords(char[] inputBuffer)
{
//---------------------------
//we don't have to create a new text breaker everytime.
//we can reuse it.***
//this is just a demonstration.
//---------------------------
//some lang eg. Thai, Lao, need dictionary breaking
//we use dic data from icu-project
//1. create dictionary based breaking engine
//TODO: dic should be read once
var dicProvider = new IcuSimpleTextFileDictionaryProvider()
{
DataDir = "../../../icu62/brkitr"
};
CustomBreakerBuilder.Setup(dicProvider);
CustomBreaker breaker1 = CustomBreakerBuilder.NewCustomBreaker();
//when we want to break into a group of consecutive unicode ranges. (this does not use Dictionry breaker)
breaker1.EngBreakingEngine.SurrogatePairBreakingOption = (SurrogatePairBreakingOption)cmbSurrogatePairBreakOptions.SelectedItem;
breaker1.UseUnicodeRangeBreaker = chkUseUnicodeRangeBreaker.Checked;
breaker1.BreakNumberAfterText = true;
this.listBox1.Items.Clear();
breaker1.SetNewBreakHandler(vis =>
{
BreakSpan span = vis.GetBreakSpan();
string s = new string(inputBuffer, span.startAt, span.len);
this.listBox1.Items.Add(span.startAt + " " + s);
});
breaker1.BreakWords(inputBuffer, 0, inputBuffer.Length);
//foreach (BreakSpan span in breaker1.GetBreakSpanIter())
//{
//
// this.listBox1.Items.Add(span.startAt + " " + s);
//}
}
void ParseWithManaged(int ntimes)
{
//-------------------
var dicProvider = new IcuSimpleTextFileDictionaryProvider()
{
DataDir = "../../../icu58/brkitr_src"
};
CustomBreakerBuilder.Setup(dicProvider);
CustomBreaker breaker1 = CustomBreakerBuilder.NewCustomBreaker();
breaker1.SetNewBreakHandler(vis => { }); //just break, do nothing about result
char[] test = this.textBox1.Text.ToCharArray();
//-------------
for (int i = ntimes - 1; i >= 0; --i)
{
breaker1.BreakWords(test, 0, test.Length);
}
}
private void cmdManaged_Click(object sender, EventArgs e)
{
//some lang eg. Thai, Lao, need dictionary breaking
//we use dic data from icu-project
//1. create dictionary based breaking engine
CustomBreaker breaker1 = CustomBreakerBuilder.NewCustomBreaker();
char[] test = this.textBox1.Text.ToCharArray();
this.listBox1.Items.Clear();
breaker1.BreakWords(test, 0, test.Length);
foreach (BreakSpan span in breaker1.GetBreakSpanIter())
{
string s = new string(test, span.startAt, span.len);
this.listBox1.Items.Add(span.startAt + " " + s);
}
}
void ParseWithManaged(int ntimes)
{
//-------------------
var dicProvider = new IcuSimpleTextFileDictionaryProvider()
{
DataDir = "../../../icu58/brkitr_src/dictionaries"
};
CustomBreakerBuilder.Setup(dicProvider);
CustomBreaker breaker1 = CustomBreakerBuilder.NewCustomBreaker();
char[] test = this.textBox1.Text.ToCharArray();
//-------------
for (int i = ntimes - 1; i >= 0; --i)
{
breaker1.BreakWords(test, 0, test.Length);
foreach (var span in breaker1.GetBreakSpanIter())
{
}
}
}
void ParseWithManaged(int ntimes)
{
//-------------------
var dicProvider = new IcuSimpleTextFileDictionaryProvider()
{
DataDir = "../../../icu58/brkitr_src"
};
CustomBreakerBuilder.Setup(dicProvider);
CustomBreaker breaker1 = CustomBreakerBuilder.NewCustomBreaker();
breaker1.UseUnicodeRangeBreaker = chkUseUnicodeRangeBreaker.Checked;
breaker1.EngBreakingEngine.SurrogatePairBreakingOption = (SurrogatePairBreakingOption)cmbSurrogatePairBreakOptions.SelectedItem;
breaker1.SetNewBreakHandler(vis => { }); //just break, do nothing about result
char[] test = this.textBox1.Text.ToCharArray();
//-------------
for (int i = ntimes - 1; i >= 0; --i)
{
breaker1.BreakWords(test, 0, test.Length);
}
}
private void button1_Click(object sender, EventArgs e)
{
//char[] test = this.textBox1.Text.ToCharArray();
//string test_str = "حب";
this.listBox1.Items.Clear();
string test_str = "یہ ایک (car) ہے۔";
char[] test = test_str.ToCharArray();
var dicProvider = new IcuSimpleTextFileDictionaryProvider()
{
DataDir = "../../../icu58/brkitr_src"
};
CustomBreakerBuilder.Setup(dicProvider);
CustomBreaker breaker1 = CustomBreakerBuilder.NewCustomBreaker();
breaker1.SetNewBreakHandler(vis =>
{
BreakSpan span = vis.GetBreakSpan();
string s = new string(test, span.startAt, span.len);
this.listBox1.Items.Add(span.startAt + " " + s);
}); //just break, do nothing about result
breaker1.BreakWords(test);
//for (int i = 0; i < outputList.Count - 1; i++)
//{
// Assert.AreEqual
// (
// output[i],
// input.Substring(outputList[i], outputList[i + 1] - outputList[i])
// );
//}
}
public MyManagedTextBreaker()
{
//TODO: review config folder here
_textBreaker = CustomBreakerBuilder.NewCustomBreaker();
_textBreaker.SetNewBreakHandler(vis => _breakAtList.Add(vis.LatestBreakAt));
}
public MyManagedTextBreaker()
{
//TODO: review config folder here
myTextBreaker = CustomBreakerBuilder.NewCustomBreaker();
}
