public void TestAddText()
{
string groupName;
_uniqueNames.TryPop(out groupName);
//string groupName = _uniqueNames.Pop();
IGroupModel groupModel = _modelFactory.GetGroupModel(groupName, UniversalConstants.CountSize);
_groupStore.Add(groupModel);
//string textName = _uniqueNames.Pop();
string textName;
_uniqueNames.TryPop(out textName);
StreamReader text = new StreamReader("../../SampleTextFiles/WordSpanningMultipleLines.txt");
ITextModel textModel = _modelFactory.GetTextModel(textName, text, UniversalConstants.CountSize);
_textStore.Add(textModel);
_groupStore.AddItem(groupModel, textModel);
groupModel = _groupStore.GetOne(x => x.Name == groupName);
List <ITextOrGroupViewModel> groupMembers = groupModel.GetMembers();
Assert.AreEqual(1, groupMembers.Count);
Assert.AreEqual(textName, groupMembers[0].GetName());
Assert.IsInstanceOfType(groupMembers[0], typeof(ITextModel));
CompareTextModels(textModel, (ITextModel)groupMembers[0]);
_groupStore.RemoveItem(groupModel, textModel);
_textStore.Delete(textModel);
_groupStore.Delete(groupModel);
}
public string Convert(ITextModel textModel)
{
if (textModel.SentenceList.Count == 0)
{
return(string.Empty);
}
var sb = new StringBuilder();
int maxWordCount = textModel.SentenceList.Max(sentence => sentence.WordList.Count);
for (int i = 1; i <= maxWordCount; i++)
{
sb.Append($", {_wordName} {i}");
}
sb.AppendLine();
int rowNumber = 1;
foreach (var sentence in textModel.SentenceList)
{
sb.Append($"{_sentenceName} {rowNumber++}, ");
sb.AppendLine(ConvertSentenceToCSV(sentence));
}
sb.Remove(sb.Length - 2, 2); // Remove Last NewLine
return(sb.ToString());
}
public void TestAddDuplicate()
{
ArgumentException expectedException = null;
//string name = _uniqueNames.Pop();
string name;
_uniqueNames.TryPop(out name);
try
{
StreamReader text1 = new StreamReader("../../SampleTextFiles/WordSpanningMultipleLines.txt");
ITextModel model1 = _modelFactory.GetTextModel(name, text1, UniversalConstants.CountSize);
_textStore.Add(model1);
StreamReader text2 = new StreamReader("../../SampleTextFiles/MismatchedQuotationMarks.txt");
ITextModel model2 = _modelFactory.GetTextModel(name, text2, UniversalConstants.CountSize);
_textStore.Add(model2);
}
catch (ArgumentException ex)
{
expectedException = ex;
}
finally
{
List <Text> textsToDelete = _db.Texts.Where(x => x.Name == name).ToList();
foreach (Text text in textsToDelete)
{
_db.Texts.Remove(text);
_db.SaveChanges();
}
}
if (expectedException != null)
{
throw expectedException;
}
}
public void Add(ITextModel model)
{
string name = model.GetName();
if (_db.Texts.Any(x => x.Name == name))
{
throw new ArgumentException($"Cannot add model since a model already exists in the database with name {model.GetName()}.");
}
WordCount withQuotes = TranslateCounts(model.GetCountsWithQuotes());
_db.WordCounts.Add(withQuotes);
_db.SaveChanges();
WordCount withoutQuotes = TranslateCounts(model.GetCountsWithoutQuotes());
_db.WordCounts.Add(withoutQuotes);
_db.SaveChanges();
Text text = new Text()
{
Name = model.GetName(),
Author = model.GetAuthor(),
WithQuotesId = withQuotes.Id,
WithoutQuotesId = withoutQuotes.Id,
IncludeQuotes = 1
};
_db.Texts.Add(text);
_db.SaveChanges();
}
public SaveAsJob(
SessionFactory sessionFactory,
QueryParser <string> queryParser,
ITextModel model,
ILogger logger,
string target,
string[] collections,
string[] fields,
string[] select,
string q,
bool and,
bool or,
int skip,
int take,
bool truncate)
: base(collections, fields, q, and, or)
{
_indexFieldNames = new HashSet <string>(select);
_sessionFactory = sessionFactory;
_queryParser = queryParser;
_logger = logger;
_model = model;
_target = target;
_skip = skip;
_take = take;
_select = select;
_truncate = truncate;
}
public CrawlJob(
SessionFactory sessionFactory,
QueryParser <string> queryParser,
ITextModel model,
ILogger logger,
string id,
string[] collection,
string[] field,
string q,
string job,
bool and,
bool or,
int skip,
int take)
: base(id, collection, field, q, job, and, or)
{
_sessionFactory = sessionFactory;
_queryParser = queryParser;
_logger = logger;
_model = model;
_wetStoredFieldNames = new HashSet <string> {
"url", "title", "description", "filename"
};
_wetIndexedFieldNames = new HashSet <string> {
"title", "description"
};
_skip = skip;
_take = take;
Status["download"] = 0;
Status["index"] = 0;
}
public void Write(HttpRequest request, ITextModel model)
{
var documents = Deserialize <IEnumerable <IDictionary <string, object> > >(request.Body);
var storedFieldNames = new HashSet <string>(request.Query["storedFields"].ToArray());
var indexedFieldNames = new HashSet <string>(request.Query["indexedFields"].ToArray());
if (request.Query.ContainsKey("collection"))
{
var collections = request.Query["collection"].ToArray();
foreach (var collection in collections)
{
_sessionFactory.Write(
new WriteJob(
collection.ToHash(),
documents,
model,
storedFieldNames,
indexedFieldNames));
}
}
else
{
_sessionFactory.Write(
documents,
model,
storedFieldNames,
indexedFieldNames);
}
}
public static void WriteWatSegment(
string fileName,
string collection,
ITextModel model,
ILogger logger,
string refFileName)
{
var time = Stopwatch.StartNew();
var collectionId = collection.ToHash();
var storeFieldNames = new HashSet <string>
{
"title", "description", "url", "filename"
};
var indexFieldNames = new HashSet <string>
{
"title", "description", "url"
};
using (var sessionFactory = new SessionFactory(new KeyValueConfiguration("sir.ini"), logger))
using (var writeSession = sessionFactory.CreateWriteSession(collectionId))
using (var indexSession = sessionFactory.CreateIndexSession(collectionId, model))
using (var queue = new ProducerConsumerQueue <IDictionary <string, object> >(1, (document =>
{
sessionFactory.Write(document, writeSession, indexSession, storeFieldNames, indexFieldNames);
})))
{
foreach (var document in ReadWatFile(fileName, refFileName))
{
queue.Enqueue(document);
}
}
logger.LogInformation($"indexed {fileName} in {time.Elapsed}");
}
public void TextModelWithoutWordsOfSetLengthWithСonsonantLetter(ITextModel textModel)// удаляет из текста слова заданной длины, начинающиеся на согласную
{
if (textModel != null && textModel.Text.Count > 0)
{
List <ISentenceElement> selectedWords = new List <ISentenceElement>();
Console.WriteLine("Для удаления из текста всех слова заданной длины, начинающиеся на согласную букву задайте длину слова");
bool iswordLengthSuccess = int.TryParse(Console.ReadLine(), out int wordLength);
if (iswordLengthSuccess)
{
string consonantsEnglish = "BCDFGHJKLMNPQRSTVWXYZbcdfghjklmnpqrstvwxyz";
foreach (var sentence in textModel.Text)
{
selectedWords.AddRange(sentence.SentenceElements.
Where(x => consonantsEnglish.Contains(x.Symbols[0].Character) && x.Symbols.Count() == wordLength));
selectedWords.ForEach(x => sentence.SentenceElements.Remove(x));
selectedWords.Clear();
}
}
else
{
Console.WriteLine("Ошибка. Введено не числовое значение");
}
}
else
{
Console.WriteLine("Внимание! При выполнении метода TextModelWithoutWordsOfSetLengthWithСonsonantLetter(ITextModel textModel): " +
"textModel равен нулю или количестово предложений в textModel равно нулю");
}
}
public void TestRemoveNonexistantItem()
{
ArgumentException expectedException = null;
//string groupName = _uniqueNames.Pop();
string groupName;
_uniqueNames.TryPop(out groupName);
IGroupModel groupModel = _modelFactory.GetGroupModel(groupName, UniversalConstants.CountSize);
_groupStore.Add(groupModel);
//string textName = _uniqueNames.Pop();
string textName;
_uniqueNames.TryPop(out textName);
StreamReader text = new StreamReader("../../SampleTextFiles/WordSpanningMultipleLines.txt");
ITextModel textModel = _modelFactory.GetTextModel(textName, text, UniversalConstants.CountSize);
groupModel = _groupStore.GetOne(x => x.Name == groupName);
try
{
_groupStore.RemoveItem(groupModel, textModel);
}
catch (ArgumentException ex)
{
expectedException = ex;
}
finally
{
_groupStore.Delete(groupModel);
}
if (expectedException != null)
{
throw expectedException;
}
}
public void TestRemoveItem()
{
//string groupName = _uniqueNames.Pop();
string groupName;
_uniqueNames.TryPop(out groupName);
IGroupModel groupModel = _modelFactory.GetGroupModel(groupName, UniversalConstants.CountSize);
_groupStore.Add(groupModel);
//string textName = _uniqueNames.Pop();
string textName;
_uniqueNames.TryPop(out textName);
StreamReader text = new StreamReader("../../SampleTextFiles/WordSpanningMultipleLines.txt");
ITextModel textModel = _modelFactory.GetTextModel(textName, text, UniversalConstants.CountSize);
_textStore.Add(textModel);
_groupStore.AddItem(groupModel, textModel);
groupModel = _groupStore.GetOne(x => x.Name == groupName);
Assert.IsTrue(_groupStore.Contains(groupModel, textModel));
_groupStore.RemoveItem(groupModel, textModel);
Assert.IsFalse(_groupStore.Contains(groupModel, textModel));
_textStore.Delete(textModel);
_groupStore.Delete(groupModel);
}
public void TestContainsTwoLevels()
{
//string name1 = _uniqueNames.Pop();
string name1;
_uniqueNames.TryPop(out name1);
IGroupModel group1 = _modelFactory.GetGroupModel(name1, UniversalConstants.CountSize);
_groupStore.Add(group1);
//string name2 = _uniqueNames.Pop();
string name2;
_uniqueNames.TryPop(out name2);
IGroupModel group2 = _modelFactory.GetGroupModel(name2, UniversalConstants.CountSize);
_groupStore.Add(group2);
//string textName = _uniqueNames.Pop();
string textName;
_uniqueNames.TryPop(out textName);
StreamReader text = new StreamReader("../../SampleTextFiles/WordSpanningMultipleLines.txt");
ITextModel textModel = _modelFactory.GetTextModel(textName, text, UniversalConstants.CountSize);
_textStore.Add(textModel);
Assert.IsFalse(_groupStore.Contains(group1, textModel));
_groupStore.AddItem(group2, textModel);
Assert.IsFalse(_groupStore.Contains(group1, textModel));
_groupStore.AddItem(group1, group2);
Assert.IsTrue(_groupStore.Contains(group1, textModel));
_groupStore.RemoveItem(group1, group2);
_groupStore.RemoveItem(group2, textModel);
_textStore.Delete(textModel);
_groupStore.Delete(group2);
_groupStore.Delete(group1);
}
public void TestGetMany()
{
//string name1 = _uniqueNames.Pop();
string name1;
_uniqueNames.TryPop(out name1);
//string name2 = _uniqueNames.Pop();
string name2;
_uniqueNames.TryPop(out name2);
IEnumerable <ITextModel> modelsFromDb = _textStore.GetMany(x => x.Name == name1);
List <ITextModel> models = modelsFromDb.ToList();
Assert.AreEqual(0, models.Count);
StreamReader text1 = new StreamReader("../../SampleTextFiles/WordSpanningMultipleLines.txt");
StreamReader text2 = new StreamReader("../../SampleTextFiles/MismatchedQuotationMarks.txt");
ITextModel model1 = _modelFactory.GetTextModel(name1, text1, UniversalConstants.CountSize);
ITextModel model2 = _modelFactory.GetTextModel(name2, text2, UniversalConstants.CountSize);
_textStore.Add(model1);
_textStore.Add(model2);
modelsFromDb = _textStore.GetMany(x => x.Name == name1 || x.Name == name2);
models = modelsFromDb.ToList();
CompareTextModels(model1, models[0]);
CompareTextModels(model2, models[1]);
_textStore.Delete(model1);
_textStore.Delete(model2);
}
private void UpdateFileSizes(ITextModel leftControl, ITextModel rightControl, double controlHeight)
{
if (leftControl == null || rightControl == null)
{
return;
}
int lineCountOrg = leftControl.ValidLineCount;
int lineCountCur = rightControl.ValidLineCount;
int maxLineCount = Math.Max(lineCountOrg, lineCountCur);
double leftHeight = 0;
double rightHeight = 0;
if (lineCountOrg != lineCountCur)
{
if (lineCountOrg == maxLineCount)
{
rightHeight += (double)(lineCountOrg - lineCountCur) / maxLineCount * controlHeight / 2.0;
}
else
{
leftHeight += (double)(lineCountCur - lineCountOrg) / maxLineCount * controlHeight / 2.0;
}
}
double leftOffset = (double)lineCountOrg / maxLineCount * controlHeight;
double rightOffset = (double)lineCountCur / maxLineCount * controlHeight;
_leftFileTop = Math.Floor(leftHeight);
_rightFileTop = Math.Floor(rightHeight);
_leftFileHeight = Math.Ceiling(leftHeight + leftOffset) - _leftFileTop;
_rightFileHeight = Math.Ceiling(rightHeight + rightOffset) - _rightFileTop;
}
private static void CompareBaseless(string first, string second, ITextModel model)
{
var doc1 = new VectorNode(model.Tokenize(first).First());
var doc2 = new VectorNode(model.Tokenize(second).First());
var angle = model.CosAngle(doc1.Vector, doc2.Vector);
Console.WriteLine($"similarity (baseless): {angle}");
}
public SearchController(
IHttpReader reader,
IConfigurationProvider config,
ITextModel model,
SessionFactory sessionFactory) : base(config, sessionFactory)
{
_reader = reader;
_model = model;
}
public void DeleteText(string name)
{
ITextModel model = _textStore.GetOne(x => x.Name == name);
if (model == null)
{
throw new ArgumentException($"Can't delete text named {name} since it doesn't exist.");
}
_textStore.Delete(model);
}
private void WriteTextModel(ITextModel textModel, string filePath)
{
if (textModel != null)
{
using (StreamWriter streamWriter = new StreamWriter(filePath))
{
textModel.Text.ForEach(x => x.SentenceElements.ForEach(x => x.Symbols.ForEach(x => streamWriter.Write(x.Character))));
}
}
}
private static void Compare(string first, string second, ITextModel model)
{
var baseVectorComponents = new List <float>(model.VectorWidth);
var baseVectors = new List <IVector>();
for (int i = 0; i < model.VectorWidth; i++)
{
baseVectorComponents.Add(i == 0 ? 1 : Convert.ToSingle(Math.Log10(i)));
var bvecs = new List <float>(model.VectorWidth);
for (int y = 0; y < model.VectorWidth; y++)
{
float value;
if (y == i)
{
value = 1;
}
else
{
value = 0;
}
bvecs.Add(value);
}
baseVectors.Add(new IndexedVector(bvecs));
}
var bvector = new IndexedVector(baseVectorComponents);
var doc1 = new VectorNode(model.Tokenize(first).First());
var doc2 = new VectorNode(model.Tokenize(second).First());
var angles1 = new List <float>();
var angles2 = new List <float>();
foreach (var bvec in baseVectors)
{
angles1.Add(Convert.ToSingle(model.CosAngle(doc1.Vector, bvec)));
angles2.Add(Convert.ToSingle(model.CosAngle(doc2.Vector, bvec)));
}
var docVector1 = new IndexedVector(angles1);
var docVector2 = new IndexedVector(angles2);
var angle = model.CosAngle(docVector1, docVector2);
var angle1 = model.CosAngle(docVector1, bvector);
var angle2 = model.CosAngle(docVector2, bvector);
Console.WriteLine($"similarity: {angle}");
Console.WriteLine($"bvector similarity 1: {angle1}");
Console.WriteLine($"bvector similarity 2: {angle2}");
Console.WriteLine($"base vector similarity: {Math.Min(angle1, angle2) / Math.Max(angle1, angle2)}");
}
public async Task <string> Format(HttpRequest request, ITextModel tokenizer)
{
var parser = new HttpStringQueryParser(new QueryParser <string>(_sessionFactory, tokenizer, _log));
var query = await parser.ParseRequest(request);
var dictionary = new Dictionary <string, object>();
parser.ParseQuery(query, dictionary);
return(JsonConvert.SerializeObject(dictionary, Formatting.Indented));
}
public void TestDeleteNonexistant()
{
//string name = _uniqueNames.Pop();
string name;
_uniqueNames.TryPop(out name);
StreamReader text = new StreamReader("../../SampleTextFiles/WordSpanningMultipleLines.txt");
ITextModel model = _modelFactory.GetTextModel(name, text, UniversalConstants.CountSize);
_textStore.Delete(model);
}
public WriteController(
IHttpWriter writer,
ITextModel tokenizer,
ILogger <WriteController> logger,
IConfigurationProvider config)
{
_writer = writer;
_model = tokenizer;
_logger = logger;
_config = config;
}
public StatusController(
IConfigurationProvider config,
SessionFactory sessionFactory,
ITextModel model,
QueryParser <string> queryParser,
JobQueue queue) : base(config, sessionFactory)
{
_queue = queue;
_model = model;
_queryParser = queryParser;
}
public List <ISentence> SentencesOrderByTheNumberOfWords(ITextModel textModel)//Предложения заданного текста в порядке возрастания количества слов в каждом из них.
{
if (textModel != null && textModel.Text.Count > 0)
{
var result = textModel.Text.Where(x => x.SentenceElements.
Where(x => x is Word).Count() > 0).OrderBy(x => x.SentenceElements.Where(x => (x is Word)).Count()).ToList();
return(result);
}
Console.WriteLine("Внимание! При выполнении метода SentencesOrderByTheNumberOfWords(ITextModel textModel): " +
"textModel равен нулю или количестово предложений в textModel равно нулю");
return(null);
}
public SaveAsController(
IConfigurationProvider config,
SessionFactory sessionFactory,
ITextModel model,
QueryParser <string> queryParser,
SaveAsJobQueue queue,
ILogger <SaveAsController> log) : base(config, sessionFactory)
{
_model = model;
_queryParser = queryParser;
_log = log;
}
public void ModifyIncludeQuotes(ITextModel model, bool includeQuotes)
{
string name = model.GetName();
Text text = _db.Texts.FirstOrDefault(x => x.Name == name);
if (text == null)
{
throw new ArgumentException("Cannot update model since no corresponding text exists in the database.");
}
text.IncludeQuotes = Convert.ToInt32(includeQuotes);
_db.SaveChanges();
}
/// <summary>
/// Initializes a new instance of the <see cref="TextToSpeechService" /> class.
/// </summary>
/// <param name="textModel">The text model (obtained through DI).</param>
/// <param name="eventAggregator">Provides pub/sub events (obtained through DI).</param>
public TextToSpeechService(ITextModel textModel, IEventAggregator eventAggregator)
{
this.TextModel = textModel;
this.SpeechSynthesisCommand =
new DelegateCommand(this.SayCurrentText, this.SpeechSynthesisCanExecute).ObservesProperty(
() => this.CurrentText);
Commands.SpeechSynthesisCommand.RegisterCommand(this.SpeechSynthesisCommand);
// TODO: Send to background thread.
eventAggregator.GetEvent <Events.TextUpdatedEvent>()
.Subscribe(() => this.CurrentText = this.TextModel.CurrentText);
}
public void WriteTextModelWithoutWordsOfSetLengthWithСonsonantLetter(ITextModel textModel)
{
try
{
string filePath = ConfigurationManager.AppSettings.Get("TextModelWithoutWordsOfSetLengthWithСonsonantLetter");
WriteTextModel(textModel, filePath);
}
catch (Exception ex)
{
Console.WriteLine($"Error in method WriteTextModelWithoutWordsOfSetLengthWithСonsonantLetter() :{ex}");
}
}
public void WriteTextModelExchangeWordOfSetLengthWhithString(ITextModel textModel)
{
try
{
string filePath = ConfigurationManager.AppSettings.Get("TextModelExchangeWordOfSetLengthWhithString");
WriteTextModel(textModel, filePath);
}
catch (Exception ex)
{
Console.WriteLine($"Error in method WriteTextModelExchangeWordOfSetLengthWhithString() :{ex}");
}
}
public string Convert(ITextModel textModel)
{
var sb = new StringBuilder();
sb.AppendLine("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>");
sb.AppendLine($"<{_textTag}>");
foreach (var sentence in textModel.SentenceList)
{
sb.AppendLine(ConvertSentenceToXML(sentence));
}
sb.Append($"</{_textTag}>");
return(sb.ToString());
}
