/// <summary>
/// Returns true if this set contains one or more of the characters
/// and strings of the given set.
/// </summary>
/// <param name="set">This set.</param>
/// <param name="s">Set to be checked for containment.</param>
/// <returns>True if the condition is met.</returns>
/// <draft>ICU4N 60.1</draft>
/// <provisional>This API might change or be removed in a future release.</provisional>
public static bool ContainsSome(this UnicodeSet set, UnicodeSet s)
{
if (set == null)
{
throw new ArgumentNullException(nameof(set));
}
return(set.ContainsSome(s));
}
/// <summary>
/// Union the set of all characters that may output by this object into the
/// given set.
/// </summary>
///
/// <param name="toUnionTo">the set into which to union the output characters</param>
public virtual void AddReplacementSetTo(UnicodeSet toUnionTo)
{
// The output of this replacer varies; it is the source text between
// matchStart and matchLimit. Since this varies depending on the
// input text, we can't compute it here. We can either do nothing
// or we can add ALL characters to the set. It's probably more useful
// to do nothing.
}
public void ToChar()
{
const string pattern = "[A-GH-NO-ST-Z{TZ}]";
IEnumerable <string> unicodeSet = UnicodeSet.ToCharacters(pattern);
IEnumerable <string> expected = "A B C D E F G H I J K L M N O P Q R S T U V W X Y Z TZ".Split(' ');
Assert.That(unicodeSet, Is.EqualTo(expected));
}
/// <summary>
/// Retains only the elements in this set that are contained in the
/// specified set. In other words, removes from this set all of
/// its elements that are not contained in the specified set. This
/// operation effectively modifies this set so that its value is
/// the <i>intersection</i> of the two sets.
/// </summary>
/// <param name="set">This set.</param>
/// <param name="c">Set that defines which elements this set will retain.</param>
/// <stable>ICU 2.0</stable>
public static UnicodeSet RetainAll(this UnicodeSet set, UnicodeSet c)
{
if (set == null)
{
throw new ArgumentNullException(nameof(set));
}
return(set.RetainAll(c));
}
public void ToPattern()
{
List <string> unicodeSet = "A B C D E F G H I J K L M N O P Q R S TZ T U V W X Y Z".Split(' ').ToList();
const string expected = "[A-Z{TZ}]";
string result = UnicodeSet.ToPattern(unicodeSet);
Assert.That(result, Is.EqualTo(expected));
}
/// <summary>
/// Adds all characters in range (uses preferred naming convention).
/// </summary>
/// <param name="set">This set.</param>
/// <param name="start">The index of where to start on adding all characters.</param>
/// <param name="end">The index of where to end on adding all characters.</param>
/// <returns>A reference to this object.</returns>
/// <draft>ICU4N 60.1</draft>
/// <provisional>This API might change or be removed in a future release.</provisional>
public static UnicodeSet AddAll(this UnicodeSet set, int start, int end)
{
if (set == null)
{
throw new ArgumentNullException(nameof(set));
}
return(set.AddAll(start, end));
}
/// <summary>
/// Add a collection (as strings) into this <see cref="UnicodeSet"/>.
/// Uses standard naming convention.
/// </summary>
/// <param name="set">This set.</param>
/// <param name="source">Source collection to add into.</param>
/// <returns>A reference to this object.</returns>
/// <draft>ICU4N 60.1</draft>
/// <provisional>This API might change or be removed in a future release.</provisional>
// ICU4N specific overload to optimize for string
public static UnicodeSet AddAll(this UnicodeSet set, IEnumerable <string> source)
{
if (set == null)
{
throw new ArgumentNullException(nameof(set));
}
return(set.AddAll(source));
}
public CompoundRBTEntry(String theID, ArrayList theIDBlockVector,
ArrayList theDataVector, UnicodeSet theCompoundFilter)
{
ID = theID;
idBlockVector = theIDBlockVector;
dataVector = theDataVector;
compoundFilter = theCompoundFilter;
}
/// <seealso cref="Transliterator.AddSourceTargetSet(UnicodeSet, UnicodeSet, UnicodeSet)"/>
public override void AddSourceTargetSet(UnicodeSet inputFilter, UnicodeSet sourceSet, UnicodeSet targetSet)
{
// intersect myFilter with the input filter
UnicodeSet myFilter = GetFilterAsUnicodeSet(inputFilter);
sourceSet.AddAll(myFilter);
// do nothing with the target
}
public void ToCharacters_UpperPlane()
{
string pattern = @"[\U00010D00]";
string strForUpperPlaneCharacter = UnicodeSet.ToCharacters(pattern).First();
Assert.That(strForUpperPlaneCharacter, Is.EqualTo("𐴀"));
Assert.That(char.ConvertToUtf32(strForUpperPlaneCharacter, 0), Is.EqualTo(0x00010D00));
}
/// <summary>
/// Add the contents of the UnicodeSet (as strings) into a collection.
/// </summary>
/// <typeparam name="T">Collection type.</typeparam>
/// <param name="set">This set.</param>
/// <param name="target">Collection to add into.</param>
/// <draft>ICU4N 60.1</draft>
/// <provisional>This API might change or be removed in a future release.</provisional>
public static T AddAllTo <T>(this UnicodeSet set, T target) where T : ICollection <string>
{
if (set == null)
{
throw new ArgumentNullException(nameof(set));
}
return(set.AddAllTo(target));
}
/// <summary>
/// Complements the specified character in this set. The character
/// will be removed if it is in this set, or will be added if it is
/// not in this set.
/// </summary>
/// <draft>ICU4N 60.1</draft>
/// <provisional>This API might change or be removed in a future release.</provisional>
public static UnicodeSet Complement(this UnicodeSet set, int c)
{
if (set == null)
{
throw new ArgumentNullException(nameof(set));
}
return(set.Complement(c));
}
public void SinglePatternToChar()
{
const string pattern = "[A]";
IEnumerable <string> unicodeSet = UnicodeSet.ToCharacters(pattern);
IEnumerable <string> expected = "A".Split(' ');
Assert.That(unicodeSet, Is.EqualTo(expected));
}
public UnicodeSet GetMatchSet(UnicodeSet result)
{
if (result == null)
{
result = new UnicodeSet();
}
AddAll(matchIterator, result);
return(result);
}
// ---------------------------------------------------------------------------------
//
// scanSet Construct a UnicodeSet from the text at the current scan
// position. Advance the scan position to the first character
// after the set.
//
// A new RBBI setref node referring to the set is pushed onto the node
// stack.
//
// The scan position is normally under the control of the state machine
// that controls rule parsing. UnicodeSets, however, are parsed by
// the UnicodeSet constructor, not by the RBBI rule parser.
//
// ---------------------------------------------------------------------------------
internal void ScanSet() {
UnicodeSet uset = null;
int startPos;
ILOG.J2CsMapping.Text.ParsePosition pos = new ILOG.J2CsMapping.Text.ParsePosition(fScanIndex);
int i;
startPos = fScanIndex;
try {
uset = new UnicodeSet(fRB.fRules, pos, fSymbolTable,
IBM.ICU.Text.UnicodeSet.IGNORE_SPACE);
} catch (Exception e) { // TODO: catch fewer exception types.
// Repackage UnicodeSet errors as RBBI rule builder errors, with
// location info.
Error(IBM.ICU.Text.RBBIRuleBuilder.U_BRK_MALFORMED_SET);
}
// Verify that the set contains at least one code point.
//
if (uset.IsEmpty()) {
// This set is empty.
// Make it an error, because it almost certainly is not what the
// user wanted.
// Also, avoids having to think about corner cases in the tree
// manipulation code
// that occurs later on.
// TODO: this shouldn't be an error; it does happen.
Error(IBM.ICU.Text.RBBIRuleBuilder.U_BRK_RULE_EMPTY_SET);
}
// Advance the RBBI parse postion over the UnicodeSet pattern.
// Don't just set fScanIndex because the line/char positions maintained
// for error reporting would be thrown off.
i = pos.GetIndex();
for (;;) {
if (fNextIndex >= i) {
break;
}
NextCharLL();
}
RBBINode n;
n = PushNewNode(IBM.ICU.Text.RBBINode.setRef);
n.fFirstPos = startPos;
n.fLastPos = fNextIndex;
n.fText = fRB.fRules.Substring(n.fFirstPos,(n.fLastPos)-(n.fFirstPos));
// findSetFor() serves several purposes here:
// - Adopts storage for the UnicodeSet, will be responsible for
// deleting.
// - Mantains collection of all sets in use, needed later for
// establishing
// character categories for run time engine.
// - Eliminates mulitiple instances of the same set.
// - Creates a new uset node if necessary (if this isn't a duplicate.)
FindSetFor(n.fText, n, uset);
}
/// <summary>
/// Returns the next token in this string tokenizer's string. First, the set
/// of characters considered to be delimiters by this
/// <tt>StringTokenizer</tt> object is changed to be the characters in the
/// string <tt>delim</tt>. Then the next token in the string after the
/// current position is returned. The current position is advanced beyond the
/// recognized token. The new delimiter set remains the default after this
/// call.
/// </summary>
///
/// <param name="delim">the new delimiters.</param>
/// <returns>the next token, after switching to the new delimiter set.</returns>
/// <exception cref="NoSuchElementException">if there are no more tokens in this tokenizer's string.</exception>
/// @stable ICU 2.4
public String NextToken(String delim)
{
m_delimiters_ = EMPTY_DELIMITER_;
if (delim != null && delim.Length > 0)
{
m_delimiters_ = new UnicodeSet();
m_delimiters_.AddAll(delim);
}
return(NextToken(m_delimiters_));
}
/// <seealso cref="Transliterator.AddSourceTargetSet(UnicodeSet, UnicodeSet, UnicodeSet)"/>
#pragma warning disable 672
public override void AddSourceTargetSet(UnicodeSet inputFilter, UnicodeSet sourceSet, UnicodeSet targetSet)
{
#pragma warning restore 672
#pragma warning disable 612, 618
// intersect myFilter with the input filter
UnicodeSet myFilter = GetFilterAsUnicodeSet(inputFilter);
#pragma warning restore 612, 618
sourceSet.AddAll(myFilter);
// do nothing with the target
}
public BNF AddSet(String variable, UnicodeSet set)
{
if (set != null)
{
String body = set.ToString();
t.AddSymbol(variable, body, 0, body.Length);
AddPick(variable, IBM.ICU.Charset.Pick.CodePointMthd(set));
}
return(this);
}
// might want to add to UnicodeSet
private String getList(UnicodeSet set)
{
StringBuffer result = new StringBuffer();
for (UnicodeSetIterator it = new UnicodeSetIterator(set); it.Next();)
{
result.Append(it.GetString());
}
return(result.ToString());
}
/// <exclude/>
/// <summary>
/// Returns the set of all characters that may be generated as replacement
/// text by this transliterator.
/// </summary>
///
public override UnicodeSet GetTargetSet()
{
UnicodeSet set = new UnicodeSet();
for (int i = 0; i < trans.Length; ++i)
{
// This is a heuristic, and not 100% reliable.
set.AddAll(trans[i].GetTargetSet());
}
return(set);
}
/// <summary>
/// Returns the ending offset found by matching characters with testSet,
/// until a position is found that doen't match
/// </summary>
///
/// <param name="string"></param>
/// <param name="offset"></param>
/// <param name="testSet"></param>
/// <returns></returns>
public int Span(String str0, int offset, UnicodeSet testSet)
{
while (true)
{
int newOffset = testSet.MatchesAt(str0, offset);
if (newOffset < 0)
{
return(offset);
}
}
}
/// <summary>
/// Adds bunch o' codepoints; otherwise like put.
/// </summary>
///
/// <param name="codepoints"></param>
/// <param name="value"></param>
/// <returns>this (for chaining)</returns>
public UnicodeMap PutAll(UnicodeSet codepoints, Object value_ren)
{
// TODO optimize
UnicodeSetIterator it = new UnicodeSetIterator(codepoints);
while (it.NextRange())
{
_putAll(it.codepoint, it.codepointEnd, value_ren);
}
return(this);
}
//
// RBBISymbolTable::lookupMatcher This function from the abstract symbol
// table
// interface maps a single stand-in character to a
// pointer to a Unicode Set. The Unicode Set code uses this
// mechanism to get all references to the same $variable
// name to refer to a single common Unicode Set instance.
//
// This implementation cheats a little, and does not maintain a map of
// stand-in chars
// to sets. Instead, it takes advantage of the fact that the UnicodeSet
// constructor will always call this function right after calling lookup(),
// and we just need to remember what set to return between these two calls.
public virtual UnicodeMatcher LookupMatcher(int ch)
{
UnicodeSet retVal = null;
if (ch == 0xffff)
{
retVal = fCachedSetLookup;
fCachedSetLookup = null;
}
return(retVal);
}
/// <summary>
/// Adds a starting or ending string character to the <see cref="spanNotSet"/>
/// so that a character span ends before any string.
/// </summary>
private void AddToSpanNotSet(int c)
{
if (Utility.SameObjects(spanNotSet, null) || Utility.SameObjects(spanNotSet, spanSet))
{
if (spanSet.Contains(c))
{
return; // Nothing to do.
}
spanNotSet = spanSet.CloneAsThawed();
}
spanNotSet.Add(c);
}
// ----------------------------------------------------------------------------------------
//
// findSetFor given a String,
// - find the corresponding Unicode Set (uset node)
// (create one if necessary)
// - Set fLeftChild of the caller's node (should be a setRef node)
// to the uset node
// Maintain a hash table of uset nodes, so the same one is always used
// for the same string.
// If a "to adopt" set is provided and we haven't seen this key before,
// add the provided set to the hash table.
// If the string is one (32 bit) char in length, the set contains
// just one element which is the char in question.
// If the string is "any", return a set containing all chars.
//
// ----------------------------------------------------------------------------------------
internal void FindSetFor(String s, RBBINode node, UnicodeSet setToAdopt) {
RBBIRuleScanner.RBBISetTableEl el;
// First check whether we've already cached a set for this string.
// If so, just use the cached set in the new node.
// delete any set provided by the caller, since we own it.
el = (RBBIRuleScanner.RBBISetTableEl ) ILOG.J2CsMapping.Collections.Collections.Get(fSetTable,s);
if (el != null) {
node.fLeftChild = el.val;
IBM.ICU.Impl.Assert.Assrt(node.fLeftChild.fType == IBM.ICU.Text.RBBINode.uset);
return;
}
// Haven't seen this set before.
// If the caller didn't provide us with a prebuilt set,
// create a new UnicodeSet now.
if (setToAdopt == null) {
if (s.Equals(kAny)) {
setToAdopt = new UnicodeSet(0x000000, 0x10ffff);
} else {
int c;
c = IBM.ICU.Text.UTF16.CharAt(s, 0);
setToAdopt = new UnicodeSet(c, c);
}
}
//
// Make a new uset node to refer to this UnicodeSet
// This new uset node becomes the child of the caller's setReference
// node.
//
RBBINode usetNode = new RBBINode(IBM.ICU.Text.RBBINode.uset);
usetNode.fInputSet = setToAdopt;
usetNode.fParent = node;
node.fLeftChild = usetNode;
usetNode.fText = s;
//
// Add the new uset node to the list of all uset nodes.
//
ILOG.J2CsMapping.Collections.Generics.Collections.Add(fRB.fUSetNodes,usetNode);
//
// Add the new set to the set hash table.
//
el = new RBBIRuleScanner.RBBISetTableEl ();
el.key = s;
el.val = usetNode;
ILOG.J2CsMapping.Collections.Collections.Put(fSetTable,el.key,el);
return;
}
/// <summary>
/// Returns the next token in this string tokenizer's string. First, the set
/// of characters considered to be delimiters by this
/// <tt>StringTokenizer</tt> object is changed to be the characters in the
/// string <tt>delim</tt>. Then the next token in the string after the
/// current position is returned. The current position is advanced beyond the
/// recognized token. The new delimiter set remains the default after this
/// call.
/// </summary>
///
/// <param name="delim">the new delimiters.</param>
/// <returns>the next token, after switching to the new delimiter set.</returns>
/// <exception cref="NoSuchElementException">if there are no more tokens in this tokenizer's string.</exception>
/// @stable ICU 2.4
public String NextToken(UnicodeSet delim)
{
m_delimiters_ = delim;
CheckDelimiters();
m_tokenOffset_ = -1;
m_tokenSize_ = -1;
if (!m_returnDelimiters_)
{
m_nextOffset_ = GetNextNonDelimiter(m_nextOffset_);
}
return(NextToken());
}
private void FixSets()
{
if (syntax.ContainsSome(QUOTERS) || syntax.ContainsSome(whiteSpace))
{
syntax = ((UnicodeSet)syntax.Clone()).RemoveAll(QUOTERS)
.RemoveAll(whiteSpace);
}
if (whiteSpace.ContainsSome(QUOTERS))
{
whiteSpace = ((UnicodeSet)whiteSpace.Clone()).RemoveAll(QUOTERS);
}
non_string = new UnicodeSet(syntax).AddAll(whiteSpace);
}
/// <summary>
/// Returns the ending offset found by matching characters with testSet,
/// until a position is found that does match
/// </summary>
///
/// <param name="string"></param>
/// <param name="offset"></param>
/// <param name="testSet"></param>
/// <returns></returns>
public int SpanNot(String str0, int offset, UnicodeSet testSet)
{
while (true)
{
int newOffset = testSet.MatchesAt(str0, offset);
if (newOffset >= 0)
{
return(offset);
}
++offset; // try next character position
// we don't have to worry about surrogates for this.
}
}
/* Utilities */
public static bool VerifySetsIdentical(AbstractTestLog here, UnicodeSet set1, UnicodeSet set2)
{
if (set1.Equals(set2))
{
return(true);
}
TestFmwk.Errln("Sets differ:");
TestFmwk.Errln("UnicodeMap - HashMap");
TestFmwk.Errln(new UnicodeSet(set1).RemoveAll(set2).ToPattern(true));
TestFmwk.Errln("HashMap - UnicodeMap");
TestFmwk.Errln(new UnicodeSet(set2).RemoveAll(set1).ToPattern(true));
return(false);
}
private NamePrepTransform()
{
// load the resource bundle
//ICUResourceBundle bundle = (ICUResourceBundle)ICUResourceBundle.GetBundleInstance("com/ibm/icu/dev/data/testdata", "idna_rules", typeof(NamePrepTransform).GetTypeInfo().Assembly, true);
ICUResourceBundle bundle = (ICUResourceBundle)ICUResourceBundle.GetBundleInstance("Dev/Data/TestData", "idna_rules", typeof(NamePrepTransform).GetTypeInfo().Assembly, true);
String mapRules = bundle.GetString("MapNoNormalization");
mapRules += bundle.GetString("MapNFKC");
// disable
mapTransform = new MapTransform("CaseMap", mapRules, 0 /*Transliterator.FORWARD*/);
labelSeparatorSet = new UnicodeSet(bundle.GetString("LabelSeparatorSet"));
prohibitedSet = new UnicodeSet(bundle.GetString("ProhibitedSet"));
unassignedSet = new UnicodeSet(bundle.GetString("UnassignedSet"));
}
static Characters()
{
UCSCHAR = new UnicodeSet();
UCSCHAR.add(0xA0, 0xD7FF);
UCSCHAR.add(0xF900, 0xFDCF);
UCSCHAR.add(0xFDF0, 0xFFEF);
UCSCHAR.add(0x1000, 0x1FFFD);
UCSCHAR.add(0x2000, 0x2FFFD);
UCSCHAR.add(0x3000, 0x3FFFD);
UCSCHAR.add(0x4000, 0x4FFFD);
UCSCHAR.add(0x5000, 0x5FFFD);
UCSCHAR.add(0x6000, 0x6FFFD);
UCSCHAR.add(0x7000, 0x7FFFD);
UCSCHAR.add(0x8000, 0x8FFFD);
UCSCHAR.add(0x9000, 0x9FFFD);
UCSCHAR.add(0xA000, 0xAFFFD);
UCSCHAR.add(0xB000, 0xBFFFD);
UCSCHAR.add(0xC000, 0xCFFFD);
UCSCHAR.add(0xD000, 0xDFFFD);
UCSCHAR.add(0xE000, 0xEFFFD);
IUNRESERVED = new UnicodeSet();
IUNRESERVED.addAll("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~");
IUNRESERVED.addAll(UCSCHAR);
// ipchar = iunreserved / pct-encoded / sub-delims / ":" / "@"
IPCHAR = new UnicodeSet();
IPCHAR.addAll(IUNRESERVED);
IPCHAR.add(0x25); // '%'
IPCHAR.addAll(SUB_DELIMS); // sub-delims
IPCHAR.add(0x3A); // ':'
IPCHAR.add(0x40); // '@'
// xri-pchar = iunreserved / pct-encoded / xri-sub-delims / ":"
XRI_PCHAR = new UnicodeSet();
XRI_PCHAR.addAll(IUNRESERVED);
XRI_PCHAR.add(0x25); // '%'
XRI_PCHAR.addAll(XRI_SUB_DELIMS);
XRI_PCHAR.add(0x3A); // ':'
}
