private static DataSet CreateDataSetFromGroups(string masterModelDir)
{
var tables = TextProcessor.GetTrTimeData(masterModelDir);
var ds = new DataSet();
foreach (var table in tables)
{
var dt = new DataTable(table.No.Value.ToString());
dt.Columns.Add(new DataColumn("Parameter"));
foreach (var data in table.Column)
{
dt.Columns.Add(new DataColumn(data.Time.Value.ToString(), typeof(double)));
}
DataRow row1 = dt.NewRow();
row1["Parameter"] = string.Format("Travel Time ({0})", table.VehType);
DataRow row2 = dt.NewRow();
row2["Parameter"] = "Nr. of Vehicles";
foreach (var data in table.Column)
{
row1[data.Time.Value.ToString()] = data.TravelTime;
row2[data.Time.Value.ToString()] = data.VehCount;
}
dt.Rows.Add(row1);
dt.Rows.Add(row2);
ds.Tables.Add(dt);
}
return(ds);
}
public string GetWordDefinition(string word)
{
var result = string.Empty;
var request = (HttpWebRequest)WebRequest.Create(string.Format(url, word));
using (var response = request.GetResponse()) {
using (var stream = response.GetResponseStream()) {
if (stream != null)
{
var reader = new StreamReader(stream);
var output = reader.ReadToEnd();
var resultTags = TextProcessor.ExtractTagContent("<ul>", "</ul>", output, true);
if (resultTags.Any())
{
var trimmedOutput = resultTags.First();
trimmedOutput = TextProcessor.RemoveTag("<a", "</a>", trimmedOutput);
trimmedOutput = TextProcessor.RemoveTag("<b", "</b>", trimmedOutput);
result = TextProcessor.ExtractTagContent("<li>", "</li>", trimmedOutput, true).First().Trim(' ', '(', ')', ',');
}
}
}
}
return(result);
}
public void openProgram(PO po)
{
if (po is TextProcessor)
{
TextProcessor text = po as TextProcessor;
text.open();
text.showText();
text.close();
}
if (po is Word)
{
Word word = po as Word;
word.open();
word.showText();
word.close();
}
if (po is Virus)
{
Virus virus = po as Virus;
virus.open();
virus.detect();
virus.close();
}
}
private void Host_GenerateTrTimesReport(object sender, EventArgs <DataTableEx, Dispatcher> e)
{
var list = new List <TwoStrings>();
var dt = e.Value.dt = new DataTable();
e.Param.Invoke(DispatcherPriority.Background, new ThreadStart(delegate()
{
list = (from m in Host.List
select new TwoStrings()
{
ModelName = m.ModelName,
ExperimentNumber = m.ExperimentNumber,
}).ToList();
}));
var dirs = from m in list
select new
{
ExpName = string.Format("{0}#{1}", m.ModelName, m.ExperimentNumber),
Path = clerk.GetExperimentModelDirName(m.ModelName, m.ExperimentNumber)
};
var experiments = from d in dirs
let raw = TextProcessor.GetTrTimeData2(d.Path)
from r in raw
from c in r.Column
select new
{
ExperimentName = d.ExpName,
No = r.No,
VehType = r.VehType,
TravelTime = c.TravelTime,
VehCount = c.VehCount,
TravelTimeSum = c.TravelTimeSum,
VehCountSum = c.VehCountSum,
NumberOfRuns = r.RunsCount,
};
// Travel Time (___) columns
var trTimeColumns = (from exp in experiments
select exp.VehType).Distinct();
// creating columns
dt.Columns.AddRange(new DataColumn[]
{
new DataColumn("Experiment"),
new DataColumn("Measur\x2024"),
new DataColumn("Nr\x2024_of_Vehicles"),
});
foreach (string colName in trTimeColumns)
{
dt.Columns.Add(new DataColumn(string.Format("Travel_Time_({0})", colName)));
}
var items = new List <ItemDescriptor>(experiments.Count() * 2);
// creating rows
foreach (var exp in experiments)
{
var dr = dt.NewRow();
dr["Experiment"] = exp.ExperimentName;
dr["Measur\x2024"] = exp.No;
dr[string.Format("Travel_Time_({0})", exp.VehType)] = exp.TravelTime;
dr["Nr\x2024_of_Vehicles"] = exp.VehCount;
dt.Rows.Add(dr);
int n = exp.NumberOfRuns.Value - 1;
double trTimeAverage = double.Parse(exp.TravelTime);
double trTimeStDev = n > 0 ? Math.Sqrt(exp.TravelTimeSum.Value / n) : 0;
double vehCountAverage = double.Parse(exp.VehCount);
double vehCountStDev = n > 0 ? Math.Sqrt(exp.VehCountSum.Value / n) : 0;
items.AddRange(new ItemDescriptor[]
{
new ItemDescriptor()
{
CounterId = exp.No.ToString(),
ColName = string.Format("Travel_Time_({0})", exp.VehType),
RowName = exp.ExperimentName,
Value = new DataCollectionItem()
{
Value = exp.TravelTime,
StandardDeviation = trTimeStDev.ToString("0.##"),
Confidence90 = (n > 0 ? Euclid.TDistribution(0.1, n) * (trTimeStDev / Math.Sqrt(n)) : 0).ToString("0.##"),
Confidence95 = (n > 0 ? Euclid.TDistribution(0.05, n) * (trTimeStDev / Math.Sqrt(n)) : 0).ToString("0.##"),
Confidence99 = (n > 0 ? Euclid.TDistribution(0.01, n) * (trTimeStDev / Math.Sqrt(n)) : 0).ToString("0.##"),
NumberOfRuns = exp.NumberOfRuns.ToString(),
}
},
new ItemDescriptor()
{
CounterId = exp.No.ToString(),
ColName = "Nr\x2024_of_Vehicles",
RowName = exp.ExperimentName,
Value = new DataCollectionItem()
{
Value = exp.VehCount,
StandardDeviation = vehCountStDev.ToString("0.##"),
Confidence90 = string.Format("±{0:0.##}", (n > 0 ? Euclid.TDistribution(0.1, n) * (vehCountStDev / Math.Sqrt(n)) : 0)),
Confidence95 = string.Format("±{0:0.##}", (n > 0 ? Euclid.TDistribution(0.05, n) * (vehCountStDev / Math.Sqrt(n)) : 0)),
Confidence99 = string.Format("±{0:0.##}", (n > 0 ? Euclid.TDistribution(0.01, n) * (vehCountStDev / Math.Sqrt(n)) : 0)),
NumberOfRuns = exp.NumberOfRuns.ToString(),
}
},
});
e.Value.Items = items;
}
}
public RequestPostsModel(UserTypeForRequest _userType, long _userId, int _countPost, TextProcessor x)
{
userType = _userType;
userId = _userId;
countPost = _countPost;
textProcessor = x;
}
public void CreateObjectWithTextNullOrEmpty_Valid_Exception()
{
TextProcessor actual = new TextProcessor(null);
}
public TextFormatter(TextProcessor next, ITagFormatter formatter) : base(next, formatter)
{
}
public InnerTextProcessor(TextProcessor next, ITagFormatter formatter) : base(next, formatter)
{
}
public SpecialHtmlRemover(TextProcessor next, ITagFormatter formatter) : base(next, formatter)
{
}
public TextProcessor(TextProcessor next, ITagFormatter formatter)
{
Next = next;
_formatter = formatter;
}
public static ITextProcessor CreateTextProcessor()
{
// Fist, define the calculator methods
Func <int, int, Task <int> > sumFunc = (a, b) => Task.FromResult(a + b);
Func <int, int, Task <int> > subtractFunc = (a, b) => Task.FromResult(a - b);
Func <int, int, Task <int> > multiplyFunc = (a, b) => Task.FromResult(a * b);
Func <int, int, Task <int> > divideFunc = (a, b) => Task.FromResult(a / b);
// After that, the syntaxes for all operations, using the CSDL parser:
// 1. Sum:
// a) The default syntax, for inputs like 'sum 1 and 2' or 'sum 3 4'
var sumSyntax = CsdlParser.Parse("operation+:Word(sum) a:Integer :Word?(and) b:Integer");
// b) The alternative syntax, for inputs like '3 plus 4'
var alternativeSumSyntax = CsdlParser.Parse("a:Integer :Word(plus,more) b:Integer");
// 2. Subtract:
// a) The default syntax, for inputs like 'subtract 2 from 3'
var subtractSyntax = CsdlParser.Parse("operation+:Word(subtract,sub) b:Integer :Word(from) a:Integer");
// b) The alternative syntax, for inputs like '5 minus 3'
var alternativeSubtractSyntax = CsdlParser.Parse("a:Integer :Word(minus) b:Integer");
// 3. Multiply:
// a) The default syntax, for inputs like 'multiply 3 and 3' or 'multiply 5 2'
var multiplySyntax = CsdlParser.Parse("operation+:Word(multiply,mul) a:Integer :Word?(and,by) b:Integer");
// b) The alternative syntax, for inputs like '6 times 2'
var alternativeMultiplySyntax = CsdlParser.Parse("a:Integer :Word(times) b:Integer");
// 4. Divide:
// a) The default syntax, for inputs like 'divide 3 by 3' or 'divide 10 2'
var divideSyntax = CsdlParser.Parse("operation+:Word(divide,div) a:Integer :Word?(by) b:Integer");
// b) The alternative syntax, for inputs like '6 by 2'
var alternativeDivideSyntax = CsdlParser.Parse("a:Integer :Word(by) b:Integer");
// Define a output processor that prints the command results to the console
var outputProcessor = new DelegateOutputProcessor <int>((o, context) => Console.WriteLine($"Result: {o}"));
// Now create the command processors, to bind the methods to the syntaxes
var sumCommandProcessor = new DelegateCommandProcessor(
sumFunc,
true,
outputProcessor,
sumSyntax,
alternativeSumSyntax
);
var subtractCommandProcessor = new DelegateCommandProcessor(
subtractFunc,
true,
outputProcessor,
subtractSyntax,
alternativeSubtractSyntax
);
var multiplyCommandProcessor = new DelegateCommandProcessor(
multiplyFunc,
true,
outputProcessor,
multiplySyntax,
alternativeMultiplySyntax
);
var divideCommandProcessor = new DelegateCommandProcessor(
divideFunc,
true,
outputProcessor,
divideSyntax,
alternativeDivideSyntax
);
// Finally, create the text processor and register all command processors
var textProcessor = new TextProcessor(new PunctuationTextSplitter());
textProcessor.CommandProcessors.Add(sumCommandProcessor);
textProcessor.CommandProcessors.Add(subtractCommandProcessor);
textProcessor.CommandProcessors.Add(multiplyCommandProcessor);
textProcessor.CommandProcessors.Add(divideCommandProcessor);
textProcessor.TextPreprocessors.Add(new TrimTextPreprocessor());
return(textProcessor);
}
public virtual bool PreprocessLine(ChattingLine chattingLine)
{
chattingLine.CleanedContent = TextProcessor.NaiveCleanText(chattingLine.RawContent);
return(!string.IsNullOrEmpty(chattingLine.CleanedContent));
}
public IActionResult ImportExcelSheet()
{
var lines = System.IO.File.ReadAllLines(@"C:\Users\peter\rand\inventory2.csv").ToList();
string newCsv = lines[0] + ",ID\n";
foreach (var item in lines.Skip(1))
{
var details = item.Split(",");
List <IEnumerable <InventoryPOS.DataStore.Daos.Interfaces.ProductAttribute> > atts = _service.GetProductAttributes();
var brandVal = TextProcessor.ToTitleCase(details[1]).Trim();
Brand brand = GetAttribute(atts, brandVal) as Brand;
if (brand == null)
{
var newBrand = new Brand {
Value = brandVal
};
_service.AddBrand(newBrand);
brand = newBrand;
}
var colourVal = TextProcessor.ToTitleCase(details[2]).Trim();
Colour colour = (Colour)GetAttribute(atts, colourVal);
if (colour == null)
{
var newColour = new Colour()
{
Value = colourVal
};
_service.AddColour(newColour);
colour = newColour;
}
var sizeVal = details[3].ToUpper().Trim();
Size size = (Size)GetAttribute(atts, sizeVal);
if (size == null)
{
var newSize = new Size()
{
Value = sizeVal
};
_service.AddSize(newSize);
size = newSize;
}
var categoryVal = TextProcessor.ToTitleCase(details[4]).Trim();
ItemCategory category = (ItemCategory)GetAttribute(atts, categoryVal);
if (category == null)
{
var newCategory = new ItemCategory()
{
Value = categoryVal
};
_service.AddItemCategory(newCategory);
category = newCategory;
}
string desc = (String.IsNullOrEmpty(details[0])) ? details[5] : (details[5] + " [" + details[0] + "]");
int qty;
bool successQty = int.TryParse(details[9], out qty);
if (!successQty)
{
qty = 1000;
}
var newProduct = new ProductDto()
{
Active = true,
BrandId = brand.Id,
ColourId = colour.Id,
ItemCategoryId = category.Id,
SizeId = size.Id,
Description = desc,
ManufactureCode = details[0],
Barcode = 0,
Qty = qty,
Price = Double.Parse(details[7])
};
string id = AddProduct(newProduct).Id.ToString();
newCsv += (item + ",,,, ID:" + id + ",\n");
}
System.IO.File.WriteAllText(@"C:\Users\peter\rand\newInventory.csv", newCsv);
return(Ok());
}
public Classifier()
{
tp = new TextProcessor();
}
public TextProcessorExtensionsTests()
{
_sut = Fixture.Create <TextProcessor>();
}
//private int GetUserInputOption(Win32Window excelHandle)
//{
// // ask for data input type
// using (var frmDESelectInput = new frmDESelectInput())
// {
// if (frmDESelectInput.ShowDialog(excelHandle) == DialogResult.Cancel)
// {
// frmDESelectInput.Close();
// // user pressed "Cancel"
// return 0;
// }
// // get what the user selected from the frmDESelectInput
// // 1 = Tabular Form radio button
// return frmDESelectInput.UserSelection;
// }
//}
//private void decideHowToProcessUserInput(Win32Window excelHandle)
//{
// switch (userInputOption)
// {
// // user selected the Tabular Form radio button
// case 1:
// // the user chose "Yes"
// processTabularFormInput(excelHandle);
// break;
// // user selected option 2 radio button
// case 2:
// processOption2Input(excelHandle);
// break;
// }
//}
/// <summary>
/// Take data from the current sheet of the Excel table, process it and dump the results to a csv file
/// </summary>
private void processTabularFormInput(Win32Window excelHandle, frmDESelectInput fDESelectInput)
{
//Interop.Workbook wb = AddInUtils.GetActiveWorkbook_IfExists();
//Interop.Worksheet activeWS = AddInUtils.GetActiveWorksheet_IfExists();
bool keepWindow2 = false;
try
{
//MessageBox.Show(excelHandle, $"You have selected radio button {userInputOption}", "", MessageBoxButtons.OK, MessageBoxIcon.Information);
string inputTableName;
Interop.ListObject inputTable = null;
char InputCSVlistSeparatorParsed = '!';
bool boolVarTableHeaderInExport;
string csvUsrIinputListSeparator = String.Empty;
bool AggregateTableRowsSwitch;
List <string> excelTableColumnsDataTypes = new List <string>();
var wb = AddInUtils.GetActiveWorkbook_IfExists();
var activeWS = AddInUtils.GetActiveWorksheet_IfExists();
// ask for table name
using (wb)
using (activeWS)
using (var frmDETInputTableName = new frmDETInputTableName())
{
do
{
if (frmDETInputTableName.ShowDialog(fDESelectInput) == DialogResult.Cancel)
{
frmDETInputTableName.Close();
_secondWindowClosed = true;
return;
}
inputTableName = frmDETInputTableName.GetInputTableName();
boolVarTableHeaderInExport = frmDETInputTableName.IncludeTableHeaderValuesInExportFile();
csvUsrIinputListSeparator = frmDETInputTableName.GetCSVlistSeparator();
AggregateTableRowsSwitch = frmDETInputTableName.GetAggregationSwitchValue();
// check the table name so it's not empty, whitespace or null
if (String.IsNullOrEmpty(inputTableName) || String.IsNullOrWhiteSpace(inputTableName))
{
MessageBox.Show(excelHandle, "Please enter a table name of at least 1 visible character.",
"", MessageBoxButtons.OK, MessageBoxIcon.Error);
keepWindow2 = true;
}
else
{
keepWindow2 = false;
}
if (keepWindow2 == false)
{
////MessageBox.Show(excelHandle, $"table name {inputTableName}", "", MessageBoxButtons.OK, MessageBoxIcon.Information);
InputCSVlistSeparatorParsed = Parser.GetListSeparatorFromUserInput(csvUsrIinputListSeparator);
var numOfTablesInCurrentWorkbook = AddInUtils.currentWorkbookTableCount(wb);
//MessageBox.Show(excelHandle, $"tables in workbook {numOfTablesInCurrentWorkbook}", "", MessageBoxButtons.OK, MessageBoxIcon.Information);
if (numOfTablesInCurrentWorkbook == 0 || numOfTablesInCurrentWorkbook == null)
{
MessageBox.Show(excelHandle, $"There are no tables in your current workbook. Please create at least one table in order to use this form of data input.",
"", MessageBoxButtons.OK,
MessageBoxIcon.Error);
keepWindow2 = true;
}
else
{
keepWindow2 = false;
}
if (keepWindow2 == false)
{
inputTable = AddInUtils.ReturnInputTableFromWorkbook(wb, inputTableName);
// if the input table was not found
if (inputTable == null)
{
MessageBox.Show(excelHandle, $"The input table could not be found. Please create at least one table with the specified name in order to use this form of data input.",
"", MessageBoxButtons.OK,
MessageBoxIcon.Error);
keepWindow2 = true;
}
else
{
keepWindow2 = false;
}
}
if (keepWindow2 == false)
{
// the column data types are in the excelTableColumnsDataTypes var
AddInUtils.GetExcelTableColumnDataTypes(excelHandle, wb, inputTable, out excelTableColumnsDataTypes);
// change from default type to new type
using (var frmDEShowExcelTableColumnsDataType = new frmDEShowExcelTableColumnsDataType(excelTableColumnsDataTypes,
AggregateTableRowsSwitch,
inputTable))
{
if (frmDEShowExcelTableColumnsDataType.ShowDialog(frmDETInputTableName) != DialogResult.Cancel)
{
keepWindow2 = false;
// close form 2
frmDETInputTableName.Close();
// user input for each column store in Transform.usrInputForAggregationProcess
// based on user input aggregate rows or not
TextProcessor.SaveExcelTableToCSVFile(inputTable, boolVarTableHeaderInExport, InputCSVlistSeparatorParsed,
AggregateTableRowsSwitch, wb, activeWS);
// erase all the user info about each column.
// From my brief search System.Collections.Generic.Dictionary does not implement IDisposable interface
Transform.usrInputForAggregationProcess.Clear();
}
else
{
keepWindow2 = true;
// close form 3
frmDEShowExcelTableColumnsDataType.Close();
}
}
}
}
}while (keepWindow2 == true);
// don't come back to window 1
_secondWindowClosed = false;
}
}
catch (Exception ex)
{
MessageBox.Show(excelHandle, "Error occured: " + ex.Message, "Oops!", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
finally
{
// NOT WORKING
//if (wb != null) Marshal.ReleaseComObject(wb);
//if (activeWS != null) Marshal.ReleaseComObject(activeWS);
}
}
public static ITextProcessor CreateTextProcessor()
{
// 1. Define the calendar syntaxes, using some LDWords for input flexibility
var addReminderSyntax = CsdlParser.Parse(
"^[:Word?(hey,ok) :LDWord?(calendar,agenda) :Word?(add,new,create) command:LDWord(remind,reminder) :Word?(me) :Word~(to,of) message:Text :Word?(for) when:LDWord?(today,tomorrow,someday)]");
var partialAddReminderSyntax = CsdlParser.Parse(
"^[:Word?(hey,ok) :LDWord?(calendar,agenda) :Word?(add,new,create) command+:LDWord(remind,reminder) :Word?(for,me) when+:LDWord?(today,tomorrow,someday)]");
var getRemindersSyntax = CsdlParser.Parse(
"[when:LDWord?(today,tomorrow,someday) :LDWord(reminders)]");
// 2. Now the output processors
var addReminderOutputProcessor = new DelegateOutputProcessor<Reminder>((reminder, context) =>
{
Console.WriteLine($"Reminder '{reminder.Message}' added successfully for '{reminder.When}'");
});
var getRemindersOutputProcessor = new DelegateOutputProcessor<IEnumerable<Reminder>>((reminders, context) =>
{
var remindersDictionary = reminders
.GroupBy(r => r.When)
.ToDictionary(r => r.Key, r => r.Select(reminder => reminder.Message));
foreach (var when in remindersDictionary.Keys)
{
Console.WriteLine($"Reminders for {when}:");
foreach (var reminderMessage in remindersDictionary[when])
{
Console.WriteLine($"* {reminderMessage}");
}
Console.WriteLine();
}
});
// 3. Create a instance of the processor object to be shared by all processors
var calendar = new Calendar();
// 4. Create the command processors
var addRemiderCommandProcessor = new ReflectionCommandProcessor(
calendar,
nameof(AddReminderAsync),
true,
addReminderOutputProcessor,
addReminderSyntax);
var partialAddRemiderCommandProcessor = new DelegateCommandProcessor(
new Func<string, Task>((when) =>
{
Console.Write($"What do you want to be reminded {when}?");
return Task.FromResult(0);
}),
syntaxes: partialAddReminderSyntax);
var getRemidersCommandProcessor = new ReflectionCommandProcessor(
calendar,
nameof(GetRemindersAsync),
true,
getRemindersOutputProcessor,
getRemindersSyntax);
// 5. Register the the processors
var textProcessor = new TextProcessor();
textProcessor.CommandProcessors.Add(addRemiderCommandProcessor);
textProcessor.CommandProcessors.Add(partialAddRemiderCommandProcessor);
textProcessor.CommandProcessors.Add(getRemidersCommandProcessor);
// 6. Add some preprocessors to normalize the input text
textProcessor.TextPreprocessors.Add(new TextNormalizerPreprocessor());
textProcessor.TextPreprocessors.Add(new ToLowerCasePreprocessor());
return textProcessor;
}
public TextboxState()
{
_textProcessor = new TextProcessor();
}
/// <summary> /// Creates and initializes domain-specific instance of ParagraphExtractor. /// </summary> /// <param name="next">Next processing object in the chain.</param> /// <returns>Instance of ParagraphExtractor specific for the domain supported /// by inherited class.</returns> protected abstract ParagraphExtractor GetParagraphExtractor(TextProcessor next, ITagFormatter formatter);
public static ITextProcessor CreateTextProcessor()
{
// 1. Define the calendar syntaxes, using some LDWords for input flexibility
var addReminderSyntax = CsdlParser.Parse(
"^[:Word?(hey,ok) :LDWord?(calendar,agenda) :Word?(add,new,create) command:LDWord(remind,reminder) :Word?(me) :Word~(to,of) message:Text :Word?(for) when:LDWord?(today,tomorrow,someday)]");
var partialAddReminderSyntax = CsdlParser.Parse(
"^[:Word?(hey,ok) :LDWord?(calendar,agenda) :Word?(add,new,create) command+:LDWord(remind,reminder) :Word?(for,me) when+:LDWord?(today,tomorrow,someday)]");
var getRemindersSyntax = CsdlParser.Parse(
"[when:LDWord?(today,tomorrow,someday) :LDWord(reminders)]");
// 2. Now the output processors
var addReminderOutputProcessor = new DelegateOutputProcessor <Reminder>((reminder, context) =>
{
Console.WriteLine($"Reminder '{reminder.Message}' added successfully for '{reminder.When}'");
});
var getRemindersOutputProcessor = new DelegateOutputProcessor <IEnumerable <Reminder> >((reminders, context) =>
{
var remindersDictionary = reminders
.GroupBy(r => r.When)
.ToDictionary(r => r.Key, r => r.Select(reminder => reminder.Message));
foreach (var when in remindersDictionary.Keys)
{
Console.WriteLine($"Reminders for {when}:");
foreach (var reminderMessage in remindersDictionary[when])
{
Console.WriteLine($"* {reminderMessage}");
}
Console.WriteLine();
}
});
// 3. Create a instance of the processor object to be shared by all processors
var calendar = new Calendar();
// 4. Create the command processors
var addRemiderCommandProcessor = new ReflectionCommandProcessor(
calendar,
nameof(AddReminderAsync),
true,
addReminderOutputProcessor,
addReminderSyntax);
var partialAddRemiderCommandProcessor = new DelegateCommandProcessor(
new Func <string, Task>((when) =>
{
Console.Write($"What do you want to be reminded {when}?");
return(Task.FromResult(0));
}),
syntaxes: partialAddReminderSyntax);
var getRemidersCommandProcessor = new ReflectionCommandProcessor(
calendar,
nameof(GetRemindersAsync),
true,
getRemindersOutputProcessor,
getRemindersSyntax);
// 5. Register the the processors
var textProcessor = new TextProcessor();
textProcessor.CommandProcessors.Add(addRemiderCommandProcessor);
textProcessor.CommandProcessors.Add(partialAddRemiderCommandProcessor);
textProcessor.CommandProcessors.Add(getRemidersCommandProcessor);
// 6. Add some preprocessors to normalize the input text
textProcessor.TextPreprocessors.Add(new TextNormalizerPreprocessor());
textProcessor.TextPreprocessors.Add(new ToLowerCasePreprocessor());
return(textProcessor);
}
protected abstract InnerTextProcessor GetInnerTextProcessor(TextProcessor next, ITagFormatter formatter);
public void Translate(List <ChattingLine> chattingLines)
{
try
{
// Build text
var textBuilder = new StringBuilder();
var settings = new XmlWriterSettings {
OmitXmlDeclaration = true
};
var textWriter = XmlWriter.Create(textBuilder, settings);
textWriter.WriteStartElement("TranslateArrayRequest");
{
textWriter.WriteElementString("AppId", null);
textWriter.WriteElementString("From", _langFrom);
textWriter.WriteStartElement("Options");
{
textWriter.WriteElementString("Category", NsService, "general");
textWriter.WriteElementString("ContentType", NsService, "text/html");
textWriter.WriteElementString("ReservedFlags", NsService, null);
textWriter.WriteElementString("State", NsService, "0");
textWriter.WriteElementString("Uri", NsService, "all");
textWriter.WriteElementString("User", NsService, "all");
textWriter.WriteEndElement();
}
textWriter.WriteStartElement("Texts");
{
foreach (var line in chattingLines)
{
textWriter.WriteElementString("string", NsArrays,
WebUtility.HtmlEncode(TextProcessor.NaiveCleanText(line.RawContent)));
}
textWriter.WriteEndElement();
}
textWriter.WriteElementString("To", _langTo);
textWriter.WriteEndElement();
}
textWriter.Flush();
textWriter.Close();
var requestBody = textBuilder.ToString();
Console.WriteLine("Request body is: {0}.", requestBody);
// Call Microsoft translate API.
DoRequest(GetAuthToken(), requestBody, out string responseBody, out HttpStatusCode statusCode);
// Parse result
switch (statusCode)
{
case HttpStatusCode.OK:
Console.WriteLine("Request status is OK. Response body is:");
Console.WriteLine(responseBody);
Console.WriteLine("Result of translate array method is:");
var doc = XDocument.Parse(responseBody);
var ns = XNamespace.Get(NsService);
var sourceTextCounter = 0;
foreach (var xe in doc.Descendants(ns + "TranslateArrayResponse"))
{
foreach (var node in xe.Elements(ns + "TranslatedText"))
{
var result = WebUtility.HtmlDecode(node.Value);
Console.WriteLine("\n\nSource text: {0}\nTranslated Text: {1}",
chattingLines[sourceTextCounter].RawContent, result);
chattingLines[sourceTextCounter].TranslatedContent = result;
}
sourceTextCounter++;
}
break;
case HttpStatusCode.Unauthorized:
throw new TranslateException(TranslateException.ExceptionReason.InvalidApiKey,
"Invalid API key",
null);
default:
Console.WriteLine("Request status code is: {0}.", statusCode);
Console.WriteLine("Request error message: {0}.", responseBody);
throw new TranslateException(TranslateException.ExceptionReason.GeneralServiceError,
responseBody, null);
}
}
catch (TranslateException)
{
throw;
}
catch (Exception ex)
{
throw new TranslateException(TranslateException.ExceptionReason.UnknownError, null, ex);
}
}
/// <summary> /// Creates and initializes domain-specific instance of TagRemover. /// </summary> /// <param name="next">Next processing object in the chain.</param> /// <returns>Instance of TagRemover specific for the domain supported /// by inherited class.</returns> protected abstract TagRemover GetTagRemover(TextProcessor next, ITagFormatter formatter);
public ProcessTextTests()
{
ILineReaderFactory lineReaderFactory = new LineReaderFactory();
textProcessor = new TextProcessor(lineReaderFactory);
}
protected void ReadConfiguration(TextProcessor chain)
{
var pathToConfig = GetConfigurationFileName();
_configSerializer.Deserialize(pathToConfig, chain);
}
protected override void DoWork(TranslateContext context)
{
var service = context.Service;
var provider = context.Provider;
int batchWait = 0;
var batchWorkingList = new List <ChattingLine>();
var availableLines = new List <ChattingLine>();
while (!WorkingThreadStopping)
{
if (batchWorkingList.Count > BatchThreshold || (batchWait > 1 && batchWorkingList.Count > 0))
{
batchWait = 0;
// Invoke translate service
try
{
// Make sure we do have something to translate
batchWorkingList.ForEach(it =>
{
if (provider.PreprocessLine(it))
{
availableLines.Add(it);
}
else
{
it.TranslatedContent = string.Empty;
}
});
if (availableLines.Count != 0)
{
provider.Translate(availableLines);
availableLines.Clear();
}
service._controller.NotifyBatchTranslateCompleted(false, new List <ChattingLine>(batchWorkingList));
// Render overlay text
var overlayContentRendered = TextProcessor.BuildRTF(
service._plugin,
service.AddTimestamp,
service.Timestamp24Hour,
batchWorkingList.Where(it => it.EventCode <= EventCode.Clipboard)); // Don't display test channels
service._controller.NotifyOverlayContentUpdated(false, overlayContentRendered);
}
catch (Exception ex)
{
service._controller.NotifyLogMessageAppend(false, ex + "\n");
}
finally
{
batchWorkingList.Clear();
}
}
else
{
lock (service._mainLock)
{
if (service._pendingLines.Count > 0)
{
batchWait = 0;
batchWorkingList.AddRange(service._pendingLines);
service._pendingLines.Clear();
}
else
{
if (batchWorkingList.Count > 0)
{
batchWait++;
}
Monitor.Wait(service._mainLock, 500);
}
}
}
}
}
public void Initialize()
{
this._textProcessor = new TextProcessor(TEST_TEXT);
}
public void Translate(List <ChattingLine> chattingLines)
{
try
{
// Build text
var textBuilder = new StringBuilder();
var settings = new XmlWriterSettings {
OmitXmlDeclaration = true
};
var textWriter = XmlWriter.Create(textBuilder, settings);
textWriter.WriteStartElement("lines");
foreach (var line in chattingLines)
{
textWriter.WriteElementString("line", TextProcessor.NaiveCleanText(line.RawContent));
}
textWriter.WriteEndElement();
textWriter.Flush();
textWriter.Close();
var text = textBuilder.ToString();
string textResponse;
using (var client = new HttpClient())
using (var request = new HttpRequestMessage(HttpMethod.Post, "/api/v1.5/tr.json/translate"))
{
client.BaseAddress = new Uri("https://translate.yandex.net");
var keyValues = new List <KeyValuePair <string, string> >();
keyValues.Add(new KeyValuePair <string, string>("key", _apiKey));
keyValues.Add(new KeyValuePair <string, string>("text", text));
keyValues.Add(new KeyValuePair <string, string>("lang", _lang));
keyValues.Add(new KeyValuePair <string, string>("options", "1"));
keyValues.Add(new KeyValuePair <string, string>("format", "html"));
request.Content = new FormUrlEncodedContent(keyValues);
var response = client.SendAsync(request).Result;
textResponse = response.Content.ReadAsStringAsync().Result;
}
// read json
var ser = new DataContractJsonSerializer(typeof(YandexTranslateResult));
var result =
(YandexTranslateResult)
ser.ReadObject(new MemoryStream(Encoding.UTF8.GetBytes(textResponse)));
switch (result.Code)
{
case 200:
break;
// Faild
case 401:
throw new TranslateException(TranslateException.ExceptionReason.InvalidApiKey,
"Invalid API key",
null);
case 402:
throw new TranslateException(TranslateException.ExceptionReason.InvalidApiKey,
"Blocked API key",
null);
case 404:
throw new TranslateException(TranslateException.ExceptionReason.ApiLimitExceed,
"Exceeded the daily limit on the amount of translated text", null);
case 413:
throw new TranslateException(TranslateException.ExceptionReason.ApiLimitExceed,
"Exceeded the maximum text size", null);
case 422:
throw new TranslateException(TranslateException.ExceptionReason.GeneralServiceError,
"The text cannot be translated", null);
case 501:
throw new TranslateException(TranslateException.ExceptionReason.DirectionNotSupported,
"The specified translation direction is not supported", null);
default:
throw new TranslateException(TranslateException.ExceptionReason.GeneralServiceError,
$"Code = {result.Code}, Msg = {result.Message}", null);
}
var translatedLinesXml = result.Text[0];
// Parse xml
var doc = new XmlDocument();
doc.LoadXml(translatedLinesXml);
var nodes = doc.SelectNodes("lines/line");
if (nodes == null || nodes.Count != chattingLines.Count)
{
// Error
throw new TranslateException(TranslateException.ExceptionReason.InternalError,
"Translate result count mismatch.", null);
}
foreach (var p in chattingLines.Zip(nodes.Cast <XmlNode>(),
(a, b) => new KeyValuePair <ChattingLine, XmlNode>(a, b)))
{
p.Key.TranslatedContent = p.Value.InnerText;
}
}
catch (TranslateException)
{
throw;
}
catch (Exception ex)
{
throw new TranslateException(TranslateException.ExceptionReason.UnknownError, null, ex);
}
}
/// <summary>
/// Apply the property replacement using the specified <see cref="IVariableSource"/>s for all
/// object in the supplied
/// <see cref="Spring.Objects.Factory.Config.IConfigurableListableObjectFactory"/>.
/// </summary>
/// <param name="factory">
/// The <see cref="Spring.Objects.Factory.Config.IConfigurableListableObjectFactory"/>
/// used by the application context.
/// </param>
/// <exception cref="Spring.Objects.ObjectsException">
/// If an error occured.
/// </exception>
protected virtual void ProcessProperties( IConfigurableListableObjectFactory factory )
{
CompositeVariableSource compositeVariableSource = new CompositeVariableSource(variableSourceList);
TextProcessor tp = new TextProcessor(this, compositeVariableSource);
ObjectDefinitionVisitor visitor = new ObjectDefinitionVisitor(new ObjectDefinitionVisitor.ResolveHandler(tp.ParseAndResolveVariables));
IList<string> objectDefinitionNames = factory.GetObjectDefinitionNames(includeAncestors);
for (int i = 0; i < objectDefinitionNames.Count; ++i)
{
string name = objectDefinitionNames[i];
IObjectDefinition definition = factory.GetObjectDefinition( name, includeAncestors );
if (definition == null)
continue;
try
{
visitor.VisitObjectDefinition( definition );
}
catch (ObjectDefinitionStoreException ex)
{
throw new ObjectDefinitionStoreException(
definition.ResourceDescription, name, ex.Message );
}
}
}
public TextProcessorTests()
{
_sut = Fixture.Create <TextProcessor>();
}
public static ITextProcessor CreateTextProcessor()
{
// Fist, define the calculator methods
Func<int, int, Task<int>> sumFunc = (a, b) => Task.FromResult(a + b);
Func<int, int, Task<int>> subtractFunc = (a, b) => Task.FromResult(a - b);
Func<int, int, Task<int>> multiplyFunc = (a, b) => Task.FromResult(a * b);
Func<int, int, Task<int>> divideFunc = (a, b) => Task.FromResult(a / b);
// After that, the syntaxes for all operations, using the CSDL parser:
// 1. Sum:
// a) The default syntax, for inputs like 'sum 1 and 2' or 'sum 3 4'
var sumSyntax = CsdlParser.Parse("operation+:Word(sum) a:Integer :Word?(and) b:Integer");
// b) The alternative syntax, for inputs like '3 plus 4'
var alternativeSumSyntax = CsdlParser.Parse("a:Integer :Word(plus,more) b:Integer");
// 2. Subtract:
// a) The default syntax, for inputs like 'subtract 2 from 3'
var subtractSyntax = CsdlParser.Parse("operation+:Word(subtract,sub) b:Integer :Word(from) a:Integer");
// b) The alternative syntax, for inputs like '5 minus 3'
var alternativeSubtractSyntax = CsdlParser.Parse("a:Integer :Word(minus) b:Integer");
// 3. Multiply:
// a) The default syntax, for inputs like 'multiply 3 and 3' or 'multiply 5 2'
var multiplySyntax = CsdlParser.Parse("operation+:Word(multiply,mul) a:Integer :Word?(and,by) b:Integer");
// b) The alternative syntax, for inputs like '6 times 2'
var alternativeMultiplySyntax = CsdlParser.Parse("a:Integer :Word(times) b:Integer");
// 4. Divide:
// a) The default syntax, for inputs like 'divide 3 by 3' or 'divide 10 2'
var divideSyntax = CsdlParser.Parse("operation+:Word(divide,div) a:Integer :Word?(by) b:Integer");
// b) The alternative syntax, for inputs like '6 by 2'
var alternativeDivideSyntax = CsdlParser.Parse("a:Integer :Word(by) b:Integer");
// Define a output processor that prints the command results to the console
var outputProcessor = new DelegateOutputProcessor<int>((o, context) => Console.WriteLine($"Result: {o}"));
// Now create the command processors, to bind the methods to the syntaxes
var sumCommandProcessor = new DelegateCommandProcessor(
sumFunc,
true,
outputProcessor,
sumSyntax,
alternativeSumSyntax
);
var subtractCommandProcessor = new DelegateCommandProcessor(
subtractFunc,
true,
outputProcessor,
subtractSyntax,
alternativeSubtractSyntax
);
var multiplyCommandProcessor = new DelegateCommandProcessor(
multiplyFunc,
true,
outputProcessor,
multiplySyntax,
alternativeMultiplySyntax
);
var divideCommandProcessor = new DelegateCommandProcessor(
divideFunc,
true,
outputProcessor,
divideSyntax,
alternativeDivideSyntax
);
// Finally, create the text processor and register all command processors
var textProcessor = new TextProcessor();
textProcessor.CommandProcessors.Add(sumCommandProcessor);
textProcessor.CommandProcessors.Add(subtractCommandProcessor);
textProcessor.CommandProcessors.Add(multiplyCommandProcessor);
textProcessor.CommandProcessors.Add(divideCommandProcessor);
return textProcessor;
}
private void Host_GenerateCountersReport(object sender, EventArgs <DataTableEx, Dispatcher> e)
{
var list = new List <TwoStrings>();
var dt = e.Value.dt = new DataTable();
dt.Columns.Add(new DataColumn("Experiment"));
dt.Columns.Add(new DataColumn("Measur\x2024"));
e.Param.Invoke(DispatcherPriority.Background, new ThreadStart(delegate()
{
list = (from m in Host.List
select new TwoStrings()
{
ModelName = m.ModelName,
ExperimentNumber = m.ExperimentNumber,
}).ToList();
}));
var dirs = from m in list
select new
{
ExpName = string.Format("{0}#{1}", m.ModelName, m.ExperimentNumber),
Path = clerk.GetExperimentModelDirName(m.ModelName, m.ExperimentNumber)
};
var experiments = from d in dirs
select new DataCollection()
{
ExperimentName = d.ExpName,
AggregatedData = TextProcessor.GetCountersData2(d.Path)
};
// columns
var parameters = from exp in experiments
from data in exp.AggregatedData
from cell in data.Cells
group cell by new { F = cell.Function, H = cell.Header, V = cell.VehType } into gr
select BuildHeader(new DataCollectionItem()
{
Function = gr.Key.F,
Header = gr.Key.H,
VehType = gr.Key.V,
});;
// creating columns
foreach (var param in parameters)
{
var col = new DataColumn(param);
dt.Columns.Add(col);
}
var xxx = from exp in experiments
from data in exp.AggregatedData
//group data by new { data.Id, exp.ExperimentName } into gr
select new
{
CounterId = data.Id.ToString(),
ExpName = exp.ExperimentName,
};
foreach (var item in xxx)
{
var row = dt.NewRow();
row[0] = item.ExpName;
row[1] = item.CounterId;
aex.Protect(() => dt.Rows.Add(row));
}
e.Value.Items = from exp in experiments
from data in exp.AggregatedData
from cell in data.Cells
select new ItemDescriptor()
{
CounterId = data.Id.ToString(),
ColName = BuildHeader(cell),
RowName = exp.ExperimentName,
Value = cell,
};
foreach (var item in e.Value.Items)
{
var row = dt.Select(string.Format("[Experiment] = '{0}' AND [Measur\x2024] = '{1}'"
, item.RowName
, item.CounterId)).FirstOrDefault();
row.SetField <DataCollectionItem>(item.ColName, item.Value);
}
}
private static Boolean KawigiEdit_RunTest(int testNum, string p0, Boolean hasAnswer, string p1)
{
Console.Write("Test " + testNum + ": [" + "\"" + p0 + "\"");
Console.WriteLine("]");
TextProcessor obj;
string answer;
obj = new TextProcessor();
DateTime startTime = DateTime.Now;
answer = obj.collectLetters(p0);
DateTime endTime = DateTime.Now;
Boolean res;
res = true;
Console.WriteLine("Time: " + (endTime - startTime).TotalSeconds + " seconds");
if (hasAnswer) {
Console.WriteLine("Desired answer:");
Console.WriteLine("\t" + "\"" + p1 + "\"");
}
Console.WriteLine("Your answer:");
Console.WriteLine("\t" + "\"" + answer + "\"");
if (hasAnswer) {
res = answer == p1;
}
if (!res) {
Console.WriteLine("DOESN'T MATCH!!!!");
} else if ((endTime - startTime).TotalSeconds >= 2) {
Console.WriteLine("FAIL the timeout");
res = false;
} else if (hasAnswer) {
Console.WriteLine("Match :-)");
} else {
Console.WriteLine("OK, but is it right?");
}
Console.WriteLine("");
return res;
}
private InputProcessor InitProcessor()
{
var inputFile = new FileInfo(InputFileName);
InputProcessor processor = null;
switch (inputFile.Extension)
{
case ".xls":
case ".xlsx":
processor = new ExcelProcessor(inputFile);
break;
case ".csv":
case ".txt":
processor = new TextProcessor(inputFile);
break;
default:
return(null);
}
return(processor);
}
