/// <summary>
/// Postavlja tokene u prosledjenu recenicu i vraca listu svih tokena
/// </summary>
/// <param name="content"></param>
/// <param name="sentence"></param>
/// <param name="language"></param>
/// <returns></returns>
internal static List <T> setTokensFromContent <T, TS>(params object[] resources)
where T : class, IContentToken, new()
where TS : IContentSubSentence, new()
{
//logSystem.log("set tokens from content Sentence: " + sentence.content, logType.Notification);
IContentSentence sentence = resources.getFirstOfType <IContentSentence>();
contentPreprocessFlag preprocessFlags = resources.getFirstOfType <contentPreprocessFlag>();
subsentenceDetectionFlag subflags = resources.getFirstOfType <subsentenceDetectionFlag>();
tokenDetectionFlag flags = resources.getFirstOfType <tokenDetectionFlag>();
//tokenDetectionFlag[] _flags
List <T> output = new List <T>();
try
{
//subsentenceDetectionFlags subflags = _subflags;
// tokenDetectionFlags flags = _flags;
string pcontent = preprocess.process(sentence.content, preprocessFlags);
contentMatchCollection subsentenceMatches = subsentenceDetection.setSubSentences <TS>(sentence, subflags);
foreach (contentMatch dt in subsentenceMatches.Values)
{
IContentSubSentence ss = dt.element as IContentSubSentence;
sentence.items.Add(ss);
foreach (T sst in ss.items)
{
output.Add(sst);
}
//output.AddRange(ss.items);
}
List <IContentToken> directTokens = new List <IContentToken>();
directTokens = setTokensForSentence <T>(sentence, true, flags, subsentenceMatches);
if (directTokens != null)
{
foreach (IContentToken dt in directTokens)
{
T tkn = dt as T;
if (tkn != null)
{
output.Add(tkn);
}
}
}
else
{
}
sentence.content = pcontent;
}
catch (Exception ex)
{
var isb = new StringBuilder();
isb.AppendLine("tokenDetection error");
isb.AppendLine("Target is: " + sentence.toStringSafe());
throw;
// devNoteManager.note(sentence, ex, isb.ToString(), "tokenDetection", devNoteType.tokenization);
}
// logSystem.log("set tokens from content Sentence done", logType.Notification);
return(output);
}
/// <summary>
/// Glavni metod za obradu sadrzaja jedne recenice >> prvo poziva setSubSentences, zatim setTokensForSentence
/// </summary>
/// <typeparam name="T"></typeparam>
/// <typeparam name="TS"></typeparam>
/// <param name="resources"> tokenDetectionFlags flags, contentTokenCollection contentTokenCollections</param>
/// <returns></returns>
public virtual contentTokenCollection setTokensFromContent <T, TS>(params object[] resources)
where T : class, IContentToken, new()
where TS : IContentSubSentence, new()
{
//logSystem.log("set tokens from content Sentence: " + sentence.content, logType.Notification);
IContentSentence sentence = this;
tokenDetectionFlag detection_flags = resources.getFirstOfType <tokenDetectionFlag>(); // new tokenDetectionFlags();
contentTokenCollection tokenCollections = resources.getFirstOfType <contentTokenCollection>();
if (tokenCollections == null)
{
tokenCollections = new contentTokenCollection();
}
contentMatchCollection subsentenceMatches = _setSubSentences <TS>(detection_flags, null);
try
{
int subCount = 0;
for (int dti = 0; dti < subsentenceMatches.Count; dti++)
{
contentMatch dt = subsentenceMatches[subsentenceMatches.Keys.imbGetItemAt(dti).ToString()]; // subsentenceMatches[dti];
contentSubSentence ss = dt.element as contentSubSentence;
contentTokenCollection subtkns = new contentTokenCollection();
//var cs = ss._setTokensForSentence<T>(subtkns, subsentenceMatches, flags);
var cs = ss._setTokensForSentence <T>(subtkns, detection_flags);
//var cs = ss._setTokensForSentence<T>(tokenCollections, flags);
//var cs = tokenCollectionsss._set
//var cs = ss._setTokensForSentence<T>(flags);
for (int ci = 0; ci < cs.Count; ci++)
{
ss.setItem(cs[ci]);
}
//cs = ss._setTokensForSentence<T>(subtkns, subsentenceMatches);
// ss.items.AddRange(cs);
// contentTokenCollection subtkns = ss.setTokensFromContent<T>(resources);
//ss.items.Add(ss);
//foreach (T sst in ss.items)
//{
// tokenCollections.Add(sst);
//}
//tokenCollections.Add(ss);
//dt.element = ss;
// subCount++;
}
List <IContentToken> directTokens = new List <IContentToken>();
directTokens = _setTokensForSentence <T>(subsentenceMatches, detection_flags, tokenCollections, directTokens);
if (directTokens != tokenCollections)
{
for (int dti = 0; dti < directTokens.Count; dti++)
{
IContentToken dt = directTokens[dti];
T tkn = dt as T;
if (tkn != null)
{
tokenCollections.Add(tkn);
}
}
}
}
catch (Exception ex)
{
var isb = new StringBuilder();
isb.AppendLine("tokenDetection error");
isb.AppendLine("Target is: " + sentence.toStringSafe());
throw;
// devNoteManager.note(sentence, ex, isb.ToString(), "tokenDetection", devNoteType.tokenization);
}
foreach (var tk in tokenCollections)
{
//subsentenceMatches.allocated(tk.)
setItem(tk);
}
// logSystem.log("set tokens from content Sentence done", logType.Notification);
return(tokenCollections);
}
