public E <LocalStr> SendServerMessage(string message) { if (TsString.TokenLength(message) > TsConst.MaxSizeTextMessage) { return(new LocalStr(strings.error_ts_msg_too_long)); } return(ts3FullClient.SendServerMessage(message, 1).FormatLocal()); }
public void Split() { for (int i = 4; i < MaxSplit; i++) { var parts = LongTextTransform.Split(Str1, LongTextBehaviour.SplitHard, maxMessageSize: i).ToArray(); foreach (var part in parts) { Assert.LessOrEqual(TsString.TokenLength(part), i); } var joined = string.Concat(parts); Assert.AreEqual(Str1, joined); } }
/// <summary>Trims a string to have the given token count at max.</summary> /// <param name="value">The string to substring from the left side.</param> /// <param name="token">The max token count.</param> /// <returns>The new substring.</returns> private static string SubstringToken(string value, int token) { int tokens = 0; for (int i = 0; i < value.Length; i++) { int addToken = TsString.IsDoubleChar(value[i]) ? 2 : 1; if (tokens + addToken > token) { return(value.Substring(0, i)); } else { tokens += addToken; } } return(value); }
public static IEnumerable <string> Transform(string text, LongTextBehaviour behaviour, int limit = int.MaxValue, int maxMessageSize = TsConst.MaxSizeTextMessage) { if (maxMessageSize < 4) { throw new ArgumentOutOfRangeException(nameof(maxMessageSize), "The minimum split length must be at least 4 bytes to fit all utf8 characters"); } // Assuming worst case that each UTF-8 character which epands to 4 bytes. // If the message is still shorter we can safely return in 1 block. if (text.Length * 4 <= TsConst.MaxSizeTextMessage) { return new[] { text } } ; var bytes = Encoding.UTF8.GetBytes(text); // If the entire text UTF-8 encoded fits in one message we can return early. if (bytes.Length * 2 < TsConst.MaxSizeTextMessage) { return new[] { text } } ; var list = new List <string>(); Span <Ind> splitIndices = stackalloc Ind[SeparatorWeight.Length]; var block = bytes.AsSpan(); while (block.Length > 0) { int tokenCnt = 0; int i = 0; bool filled = false; for (; i < block.Length; i++) { tokenCnt += TsString.IsDoubleChar(block[i]) ? 2 : 1; if (tokenCnt > maxMessageSize) { if (behaviour == LongTextBehaviour.Drop) { return(Enumerable.Empty <string>()); } filled = true; break; } for (int j = 0; j < SeparatorWeight.Length; j++) { if (block[i] == SeparatorWeight[j]) { splitIndices[j] = new Ind(i, tokenCnt); } } } if (!filled) { list.Add(block.NewUtf8String()); break; } bool hasSplit = false; if (behaviour != LongTextBehaviour.SplitHard) { for (int j = 0; j < SeparatorWeight.Length; j++) { if (!hasSplit && splitIndices[j].i > 0) { list.Add(block.Slice(0, splitIndices[j].i + 1).NewUtf8String()); block = block.Slice(splitIndices[j].i + 1); hasSplit = true; } } splitIndices.Fill(new Ind()); } if (!hasSplit) { // UTF-8 adjustment while (i > 0 && (block[i] & 0xC0) == 0x80) { i--; } list.Add(block.Slice(0, i).NewUtf8String()); block = block.Slice(i); } if (--limit == 0) { break; } } return(list); }
public string ProcessQuery(IEnumerable <AudioLogEntry> entries, Func <AudioLogEntry, string> format) { //! entryLinesRev[0] is the most recent entry var entryLinesRev = entries.Select(e => { string finStr = format(e); return(new Line { Value = finStr, TokenLength = TsString.TokenLength(finStr) }); }); //! entryLines[n] is the most recent entry var entryLines = entryLinesRev.Reverse(); var queryTokenLen = entryLines.Sum(eL => eL.TokenLength + LineBreakLen); StringBuilder strb; // If the entire content fits within the ts3 limitation, we can concat and return it. if (queryTokenLen <= TsConst.MaxSizeTextMessage) { if (queryTokenLen == 0) { return("Nothing found!"); } strb = new StringBuilder(queryTokenLen, queryTokenLen); // we want the most recent entry at the bottom so we reverse the list foreach (var eL in entryLines) { strb.Append(eL.Value).Append(LineBreak); } return(strb.ToString()); } int spareToken = TsConst.MaxSizeTextMessage; int listStart = 0; // Otherwise we go iteratively through the list to test how many entries we can add with our token foreach (var eL in entryLinesRev) { // if we don't have enough token to fit in the next entry (even in shorted form) // then we break and use the last few tokens in the next step to fill up. if (spareToken < 0 || (spareToken < MinTokenLine && spareToken < eL.TokenLength)) { break; } // now the further execution is legal because of either of those cases // 1) !(spareToken < MinTokenLine): entry will be trimmed to MinTokenLine and fits // 2) !(spareToken < entryLines[i].TokenLength): entry already fits into spareTokens if (eL.TokenLength < MinTokenLine) { spareToken -= eL.TokenLength; listStart++; } else { spareToken -= MinTokenLine; listStart++; } } //! useList[0] is the most recent entry var useList = entryLinesRev.Take(listStart).ToList(); if (fairDistribute) { // If the fairDistribute option is active this loop will start out by trying to give each // entry an equal fraction of all spareToken. for (int i = 0; i < useList.Count; i++) { if (spareToken <= 0) { break; } int fairBonus = spareToken / (useList.Count - i); int available = Math.Min(fairBonus, useList[i].TokenLength); useList[i].BonusToken = available; spareToken -= available; } } else { // Now distribute the remaining tokens by first come first serve in reverse order // so the more recent an entry is the more token it gets foreach (var eL in useList) { if (spareToken <= 0) { break; } if (eL.TokenLength > UseableTokenLine) { int available = Math.Min(spareToken, eL.TokenLength - UseableTokenLine); eL.BonusToken = available; spareToken -= available; } } } // now we can just build our result and return strb = new StringBuilder(TsConst.MaxSizeTextMessage - spareToken, TsConst.MaxSizeTextMessage); for (int i = useList.Count - 1; i >= 0; i--) { var eL = useList[i]; if (eL.TokenLength < UseableTokenLine + eL.BonusToken) { strb.Append(eL.Value).Append(LineBreak); } else { strb.Append(SubstringToken(eL.Value, UseableTokenLine + eL.BonusToken)).Append(LineBreak); } } return(strb.ToString()); }