public static void SendString(DataOutputStream stream, string message)
{
Task.Run(() =>
{
stream.WriteChars(message);
stream.WriteChar(END);
});
}
/// <exception cref="System.IO.IOException"/>
internal static Path GetPath(string location, FileSystem fs)
{
Path path = new Path(location);
// create a multi-block file on hdfs
DataOutputStream @out = fs.Create(path, true, 4096, (short)2, 512, null);
for (int i = 0; i < 1000; ++i)
{
@out.WriteChars("Hello\n");
}
@out.Close();
return(path);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.TimeoutException"/>
/// <exception cref="System.Exception"/>
private void CreateInputs(FileSystem fs, Path inDir, string fileName)
{
// create a multi-block file on hdfs
Path path = new Path(inDir, fileName);
short replication = 2;
DataOutputStream @out = fs.Create(path, true, 4096, replication, 512, null);
for (int i = 0; i < 1000; ++i)
{
@out.WriteChars("Hello\n");
}
@out.Close();
System.Console.Out.WriteLine("Wrote file");
DFSTestUtil.WaitReplication(fs, path, replication);
}
internal void FlattenData(Stream os)
{
DataOutputStream dos = new DataOutputStream(os);
int i;
// Remove comments and whitespace from the rules to make it smaller.
string strippedRules = RBBIRuleScanner.StripRules(fRules);
// Calculate the size of each section in the data in bytes.
// Sizes here are padded up to a multiple of 8 for better memory alignment.
// Sections sizes actually stored in the header are for the actual data
// without the padding.
//
int headerSize = 24 * 4; // align8(sizeof(RBBIDataHeader));
int forwardTableSize = Align8(fForwardTables.GetTableSize());
int reverseTableSize = Align8(fReverseTables.GetTableSize());
// int safeFwdTableSize = Align8(fSafeFwdTables.getTableSize());
int safeRevTableSize = Align8(fSafeRevTables.GetTableSize());
int trieSize = Align8(fSetBuilder.GetTrieSize());
int statusTableSize = Align8(fRuleStatusVals.Count * 4);
int rulesSize = Align8((strippedRules.Length) * 2);
int totalSize = headerSize
+ forwardTableSize
+ /* reverseTableSize */ 0
+ /* safeFwdTableSize */ 0
+ (safeRevTableSize > 0 ? safeRevTableSize : reverseTableSize)
+ statusTableSize + trieSize + rulesSize;
int outputPos = 0; // Track stream position, starting from RBBIDataHeader.
//
// Write out an ICU Data Header
//
ICUBinary.WriteHeader(RBBIDataWrapper.DATA_FORMAT, RBBIDataWrapper.FORMAT_VERSION, 0, dos);
//
// Write out the RBBIDataHeader
//
int[] header = new int[RBBIDataWrapper.DH_SIZE]; // sizeof struct RBBIDataHeader
header[RBBIDataWrapper.DH_MAGIC] = 0xb1a0;
header[RBBIDataWrapper.DH_FORMATVERSION] = RBBIDataWrapper.FORMAT_VERSION;
header[RBBIDataWrapper.DH_LENGTH] = totalSize; // fLength, the total size of all rule sections.
header[RBBIDataWrapper.DH_CATCOUNT] = fSetBuilder.NumCharCategories; // fCatCount.
// Only save the forward table and the safe reverse table,
// because these are the only ones used at run-time.
//
// For the moment, we still build the other tables if they are present in the rule source files,
// for backwards compatibility. Old rule files need to work, and this is the simplest approach.
//
// Additional backwards compatibility consideration: if no safe rules are provided, consider the
// reverse rules to actually be the safe reverse rules.
header[RBBIDataWrapper.DH_FTABLE] = headerSize; // fFTable
header[RBBIDataWrapper.DH_FTABLELEN] = forwardTableSize; // fTableLen
// Do not save Reverse Table.
header[RBBIDataWrapper.DH_RTABLE] = header[RBBIDataWrapper.DH_FTABLE] + forwardTableSize; // fRTable
header[RBBIDataWrapper.DH_RTABLELEN] = 0; // fRTableLen
// Do not save the Safe Forward table.
header[RBBIDataWrapper.DH_SFTABLE] = header[RBBIDataWrapper.DH_RTABLE]
+ 0; // fSTable
header[RBBIDataWrapper.DH_SFTABLELEN] = 0; // fSTableLen
// Safe reverse table. Use if present, otherwise save regular reverse table as the safe reverse.
header[RBBIDataWrapper.DH_SRTABLE] = header[RBBIDataWrapper.DH_SFTABLE]
+ 0; // fSRTable
if (safeRevTableSize > 0)
{
header[RBBIDataWrapper.DH_SRTABLELEN] = safeRevTableSize;
}
else
{
Debug.Assert(reverseTableSize > 0);
header[RBBIDataWrapper.DH_SRTABLELEN] = reverseTableSize;
}
header[RBBIDataWrapper.DH_TRIE] = header[RBBIDataWrapper.DH_SRTABLE]
+ header[RBBIDataWrapper.DH_SRTABLELEN]; // fTrie
header[RBBIDataWrapper.DH_TRIELEN] = fSetBuilder.GetTrieSize(); // fTrieLen
header[RBBIDataWrapper.DH_STATUSTABLE] = header[RBBIDataWrapper.DH_TRIE]
+ header[RBBIDataWrapper.DH_TRIELEN];
header[RBBIDataWrapper.DH_STATUSTABLELEN] = statusTableSize; // fStatusTableLen
header[RBBIDataWrapper.DH_RULESOURCE] = header[RBBIDataWrapper.DH_STATUSTABLE]
+ statusTableSize;
header[RBBIDataWrapper.DH_RULESOURCELEN] = strippedRules.Length * 2;
for (i = 0; i < header.Length; i++)
{
dos.WriteInt32(header[i]);
outputPos += 4;
}
// Write out the actual state tables.
short[] tableData;
tableData = fForwardTables.ExportTable();
Assert.Assrt(outputPos == header[4]);
for (i = 0; i < tableData.Length; i++)
{
dos.WriteInt16(tableData[i]);
outputPos += 2;
}
/* do not write the reverse table
* tableData = fReverseTables.exportTable();
* Assert.Assrt(outputPos == header[6]);
* for (i = 0; i < tableData.length; i++) {
* dos.WriteInt16(tableData[i]);
* outputPos += 2;
* }
*/
/* do not write safe forwards table
* Assert.Assrt(outputPos == header[8]);
* tableData = fSafeFwdTables.exportTable();
* for (i = 0; i < tableData.length; i++) {
* dos.WriteInt16(tableData[i]);
* outputPos += 2;
* }
*/
// Write the safe reverse table.
// If not present, write the plain reverse table (old style rule compatibility)
Assert.Assrt(outputPos == header[10]);
if (safeRevTableSize > 0)
{
tableData = fSafeRevTables.ExportTable();
}
else
{
tableData = fReverseTables.ExportTable();
}
for (i = 0; i < tableData.Length; i++)
{
dos.WriteInt16(tableData[i]);
outputPos += 2;
}
// write out the Trie table
Assert.Assrt(outputPos == header[12]);
fSetBuilder.SerializeTrie(os);
outputPos += header[13];
while (outputPos % 8 != 0)
{ // pad to an 8 byte boundary
dos.Write(0);
outputPos += 1;
}
// Write out the status {tag} table.
Assert.Assrt(outputPos == header[16]);
foreach (var val in fRuleStatusVals)
{
dos.WriteInt32(val);
outputPos += 4;
}
while (outputPos % 8 != 0)
{ // pad to an 8 byte boundary
dos.Write(0);
outputPos += 1;
}
// Write out the stripped rules (rules with extra spaces removed
// These go last in the data area, even though they are not last in the header.
Assert.Assrt(outputPos == header[14]);
dos.WriteChars(strippedRules);
outputPos += strippedRules.Length * 2;
while (outputPos % 8 != 0)
{ // pad to an 8 byte boundary
dos.Write(0);
outputPos += 1;
}
}
internal void FlattenData(Stream os)
{
DataOutputStream dos = new DataOutputStream(os);
int i;
// Remove comments and whitespace from the rules to make it smaller.
String strippedRules = IBM.ICU.Text.RBBIRuleScanner.StripRules(fRules);
// Calculate the size of each section in the data in bytes.
// Sizes here are padded up to a multiple of 8 for better memory
// alignment.
// Sections sizes actually stored in the header are for the actual data
// without the padding.
//
int headerSize = 24 * 4; // align8(sizeof(RBBIDataHeader));
int forwardTableSize = Align8(fForwardTables.GetTableSize());
int reverseTableSize = Align8(fReverseTables.GetTableSize());
int safeFwdTableSize = Align8(fSafeFwdTables.GetTableSize());
int safeRevTableSize = Align8(fSafeRevTables.GetTableSize());
int trieSize = Align8(fSetBuilder.GetTrieSize());
int statusTableSize = Align8(fRuleStatusVals.Count * 4);
int rulesSize = Align8((strippedRules.Length) * 2);
int totalSize = headerSize + forwardTableSize + reverseTableSize
+ safeFwdTableSize + safeRevTableSize + statusTableSize
+ trieSize + rulesSize;
int outputPos = 0; // Track stream position, starting from
// RBBIDataHeader.
//
// Write out an ICU Data Header
// TODO: actually create a real header, rather than just a placeholder.
// The empty placeholder is ok for compile-and-go from within ICU4J.
// Replicating the ICU4C genbrk tool for building .brk resources would
// need a real header.
//
byte[] ICUDataHeader = new byte[0x80];
dos.Write(ICUDataHeader, 0, ICUDataHeader.Length);
//
// Write out the RBBIDataHeader
//
int[] header = new int[IBM.ICU.Text.RBBIDataWrapper.DH_SIZE]; // sizeof struct
// RBBIDataHeader
header[IBM.ICU.Text.RBBIDataWrapper.DH_MAGIC] = 0xb1a0;
header[IBM.ICU.Text.RBBIDataWrapper.DH_FORMATVERSION] = 0x03010000; // uint8_t
// fFormatVersion[4];
header[IBM.ICU.Text.RBBIDataWrapper.DH_LENGTH] = totalSize; // fLength, the total
// size of all rule
// sections.
header[IBM.ICU.Text.RBBIDataWrapper.DH_CATCOUNT] = fSetBuilder
.GetNumCharCategories(); // fCatCount.
header[IBM.ICU.Text.RBBIDataWrapper.DH_FTABLE] = headerSize; // fFTable
header[IBM.ICU.Text.RBBIDataWrapper.DH_FTABLELEN] = forwardTableSize; // fTableLen
header[IBM.ICU.Text.RBBIDataWrapper.DH_RTABLE] = header[IBM.ICU.Text.RBBIDataWrapper.DH_FTABLE]
+ forwardTableSize; // fRTable
header[IBM.ICU.Text.RBBIDataWrapper.DH_RTABLELEN] = reverseTableSize; // fRTableLen
header[IBM.ICU.Text.RBBIDataWrapper.DH_SFTABLE] = header[IBM.ICU.Text.RBBIDataWrapper.DH_RTABLE]
+ reverseTableSize; // fSTable
header[IBM.ICU.Text.RBBIDataWrapper.DH_SFTABLELEN] = safeFwdTableSize; // fSTableLen
header[IBM.ICU.Text.RBBIDataWrapper.DH_SRTABLE] = header[IBM.ICU.Text.RBBIDataWrapper.DH_SFTABLE]
+ safeFwdTableSize; // fSRTable
header[IBM.ICU.Text.RBBIDataWrapper.DH_SRTABLELEN] = safeRevTableSize; // fSRTableLen
header[IBM.ICU.Text.RBBIDataWrapper.DH_TRIE] = header[IBM.ICU.Text.RBBIDataWrapper.DH_SRTABLE]
+ safeRevTableSize; // fTrie
header[IBM.ICU.Text.RBBIDataWrapper.DH_TRIELEN] = fSetBuilder.GetTrieSize(); // fTrieLen
header[IBM.ICU.Text.RBBIDataWrapper.DH_STATUSTABLE] = header[IBM.ICU.Text.RBBIDataWrapper.DH_TRIE]
+ header[IBM.ICU.Text.RBBIDataWrapper.DH_TRIELEN];
header[IBM.ICU.Text.RBBIDataWrapper.DH_STATUSTABLELEN] = statusTableSize; // fStatusTableLen
header[IBM.ICU.Text.RBBIDataWrapper.DH_RULESOURCE] = header[IBM.ICU.Text.RBBIDataWrapper.DH_STATUSTABLE]
+ statusTableSize;
header[IBM.ICU.Text.RBBIDataWrapper.DH_RULESOURCELEN] = strippedRules.Length * 2;
for (i = 0; i < header.Length; i++)
{
dos.WriteInt(header[i]);
outputPos += 4;
}
// Write out the actual state tables.
short[] tableData;
tableData = fForwardTables.ExportTable();
IBM.ICU.Impl.Assert.Assrt(outputPos == header[4]);
for (i = 0; i < tableData.Length; i++)
{
dos.WriteShort(tableData[i]);
outputPos += 2;
}
tableData = fReverseTables.ExportTable();
IBM.ICU.Impl.Assert.Assrt(outputPos == header[6]);
for (i = 0; i < tableData.Length; i++)
{
dos.WriteShort(tableData[i]);
outputPos += 2;
}
IBM.ICU.Impl.Assert.Assrt(outputPos == header[8]);
tableData = fSafeFwdTables.ExportTable();
for (i = 0; i < tableData.Length; i++)
{
dos.WriteShort(tableData[i]);
outputPos += 2;
}
IBM.ICU.Impl.Assert.Assrt(outputPos == header[10]);
tableData = fSafeRevTables.ExportTable();
for (i = 0; i < tableData.Length; i++)
{
dos.WriteShort(tableData[i]);
outputPos += 2;
}
// write out the Trie table
IBM.ICU.Impl.Assert.Assrt(outputPos == header[12]);
fSetBuilder.SerializeTrie(os);
outputPos += header[13];
while (outputPos % 8 != 0) // pad to an 8 byte boundary
{
dos.WriteByte((byte)0);
outputPos += 1;
}
// Write out the status {tag} table.
IBM.ICU.Impl.Assert.Assrt(outputPos == header[16]);
for (i = 0; i < fRuleStatusVals.Count; i++)
{
Int32 val = (Int32)fRuleStatusVals[i];
dos.WriteInt(val);
outputPos += 4;
}
while (outputPos % 8 != 0) // pad to an 8 byte boundary
{
dos.WriteByte((byte)0);
outputPos += 1;
}
// Write out the stripped rules (rules with extra spaces removed
// These go last in the data area, even though they are not last in the
// header.
IBM.ICU.Impl.Assert.Assrt(outputPos == header[14]);
dos.WriteChars(strippedRules);
outputPos += strippedRules.Length * 2;
while (outputPos % 8 != 0) // pad to an 8 byte boundary
{
dos.WriteByte((byte)0);
outputPos += 1;
}
}