static void Main(string[] args)
{
var ioService = new IoService(new ReactiveProcessFactory());
var repoRoot = ioService.CurrentDirectory.Ancestors().First(ancestor => (ancestor / ".git").IsFolder());
var autoMapperExtensionsFilePath = repoRoot / "src" / "ComposableCollections.AutoMapper" / "AutoMapperExtensions.g.cs";
using (var streamWriter = autoMapperExtensionsFilePath.OpenWriter())
{
streamWriter.WriteLine(@"using System;
using System.Collections.Generic;
using AutoMapper;
using System.Linq;
using System.Linq.Expressions;
using ComposableCollections.Common;
using ComposableCollections.Dictionary;
using ComposableCollections.Dictionary.Adapters;
using ComposableCollections.Dictionary.Decorators;
using ComposableCollections.Dictionary.Interfaces;
using ComposableCollections.Dictionary.Sources;
using ComposableCollections.Dictionary.Transactional;
using ComposableCollections.Dictionary.WithBuiltInKey;
using ComposableCollections.Dictionary.WithBuiltInKey.Interfaces;
using UtilityDisposables;
namespace ComposableCollections
{
public static partial class AutoMapperExtensions
{");
GenerateWithMappingExtensionMethods(streamWriter);
streamWriter.WriteLine("}\n}");
}
}
public EventsTest()
{
IoService ioService = new IoService();
_subscription =
Observable
.FromEvent <IoEvent, IoEventArgs>(
a => ioService.IoEvent += a, a => ioService.IoEvent -= a)
.Subscribe(eve =>
{
switch (eve.PortNum)
{
case 0:
LongRunningMethod(eve.Description);
break;
case 1:
//invoke a long running method
break;
default:
break;
}
});
}
public void DirectoryIsValid_With_Valid_Dir_Should_Return_True()
{
var service = new IoService();
bool result = service.DirectoryIsValid("../../testdata");
Assert.IsTrue(result);
}
public void DirectoryIsValid_With_EmptyOrNull_Dir_Should_Return_False(String input)
{
var service = new IoService();
bool result = service.DirectoryIsValid(input);
Assert.IsFalse(result);
}
public void DirectoryIsValid_With_Invalid_Dir_Should_Return_False()
{
var service = new IoService();
bool result = service.DirectoryIsValid("invalid/path");
Assert.IsFalse(result);
}
public void UploadDirect()
{
//Guid fileGuid = Guid.NewGuid();
var fileInfo = IoService.OpenFileInfo();
UploadFile(fileInfo);
}
/// <summary>
/// Instantiates.
/// </summary>
/// <param name="service">the service this session belongs to</param>
/// <param name="processor">the processor to process this session</param>
/// <param name="socket">the associated socket</param>
/// <param name="readBuffer">the <see cref="SocketAsyncEventArgsBuffer"/> as reading buffer</param>
/// <param name="writeBuffer">the <see cref="SocketAsyncEventArgsBuffer"/> as writing buffer</param>
/// <param name="reuseBuffer">whether or not reuse internal buffer, see <seealso cref="SocketSession.ReuseBuffer"/> for more</param>
public AsyncSocketSession(IoService service, IoProcessor<SocketSession> processor, System.Net.Sockets.Socket socket,
SocketAsyncEventArgsBuffer readBuffer, SocketAsyncEventArgsBuffer writeBuffer, Boolean reuseBuffer)
: base(service, processor, new SessionConfigImpl(socket), socket, socket.LocalEndPoint, socket.RemoteEndPoint, reuseBuffer)
{
_readBuffer = readBuffer;
_readBuffer.SocketAsyncEventArgs.UserToken = this;
_writeBuffer = writeBuffer;
_writeBuffer.SocketAsyncEventArgs.UserToken = this;
_completeHandler = saea_Completed;
}
/// <summary>
/// Instantiates.
/// </summary>
/// <param name="service">the service this session belongs to</param>
/// <param name="processor">the processor to process this session</param>
/// <param name="socket">the associated socket</param>
/// <param name="readBuffer">the <see cref="SocketAsyncEventArgsBuffer"/> as reading buffer</param>
/// <param name="writeBuffer">the <see cref="SocketAsyncEventArgsBuffer"/> as writing buffer</param>
/// <param name="reuseBuffer">whether or not reuse internal buffer, see <seealso cref="SocketSession.ReuseBuffer"/> for more</param>
public AsyncSocketSession(IoService service, IoProcessor <SocketSession> processor, System.Net.Sockets.Socket socket,
SocketAsyncEventArgsBuffer readBuffer, SocketAsyncEventArgsBuffer writeBuffer, Boolean reuseBuffer)
: base(service, processor, new SessionConfigImpl(socket), socket, socket.LocalEndPoint, socket.RemoteEndPoint, reuseBuffer)
{
_readBuffer = readBuffer;
_readBuffer.SocketAsyncEventArgs.UserToken = this;
_writeBuffer = writeBuffer;
_writeBuffer.SocketAsyncEventArgs.UserToken = this;
_completeHandler = saea_Completed;
}
private bool LogIn()
{
IoService.Out.Write(Resources.InfoLogin);
string login = IoService.In.ReadLine()?.Trim(' ');
IoService.Out.Write(Resources.InfoPassword);
string password = IoService.ReadPassword();
bool valid = UserValidator.ValidPassword(password);
if (!valid)
{
IoService.Out.WriteLine(
Resources.ErrorNotValidPassword,
PasswordValidation.MinPasswordLength);
return(false);
}
if (Session.IsLogged)
{
IoService.Out.WriteLine(Resources.ErrorDoubleLogIn);
Logger.Error(Resources.ErrorDoubleLogIn);
throw new Exception(Resources.ErrorDoubleLogIn);
}
try
{
LoginDTO loginModel = new LoginDTO()
{
UserName = login,
Password = password
};
Response <SessionInfo> response = authenticator.LogIn(loginModel);
if (response.IsSuccessful)
{
Session = response.ResultTask.Result;
}
Logger.Info(Resources.InfoEndedSignUp);
LoggedIn?.Invoke(this, new EventArgs());
}
catch (Exception ex)
{
IoService.Out.WriteLine(ex.Message);
Logger.Error(ex.Message);
return(false);
}
return(true);
}
protected IEnumerable <AbsolutePath> GetChildren(AbsolutePath parent)
{
var parentString = parent.ToString();
if (Directory.Exists(parentString))
{
return(Directory.GetFileSystemEntries(parentString).Select(x => IoService.ParseAbsolutePath(x, _path.Flags)));
}
return(Enumerable.Empty <AbsolutePath>());
}
/// <summary>
/// Constructor for client-side session.
/// </summary>
public LoopbackSession(IoService service, LoopbackEndPoint localEP,
IoHandler handler, LoopbackPipe remoteEntry)
: base(service)
{
Config = new DefaultLoopbackSessionConfig();
_lock = new Byte[0];
_localEP = localEP;
_remoteEP = remoteEntry.Endpoint;
_filterChain = new LoopbackFilterChain(this);
_receivedMessageQueue = new ConcurrentQueue <Object>();
_remoteSession = new LoopbackSession(this, remoteEntry);
}
/// <summary>
/// Constructor for client-side session.
/// </summary>
public LoopbackSession(IoService service, LoopbackEndPoint localEP,
IoHandler handler, LoopbackPipe remoteEntry)
: base(service)
{
Config = new DefaultLoopbackSessionConfig();
_lock = new Byte[0];
_localEP = localEP;
_remoteEP = remoteEntry.Endpoint;
_filterChain = new LoopbackFilterChain(this);
_receivedMessageQueue = new ConcurrentQueue<Object>();
_remoteSession = new LoopbackSession(this, remoteEntry);
}
protected SocketSession(IoService service, IoProcessor processor, IoSessionConfig config,
System.Net.Sockets.Socket socket, EndPoint localEP, EndPoint remoteEP, Boolean reuseBuffer)
: base(service)
{
_socket = socket;
_localEP = localEP;
_remoteEP = remoteEP;
Config = config;
if (service.SessionConfig != null)
Config.SetAll(service.SessionConfig);
_processor = processor;
_filterChain = new DefaultIoFilterChain(this);
}
private void ChoseStockFile()
{
string selectedPath = IoService.OpenFileDialog(defaultPathToStockFile);
if (!string.IsNullOrEmpty(selectedPath))
{
PathToStockFile = selectedPath;
}
else
{
IsManualSelected = false;
}
}
/// <summary>
/// </summary>
protected AbstractIoSession(IoService service)
{
_service = service;
_handler = service.Handler;
_creationTime = DateTime.Now;
_lastThroughputCalculationTime = _creationTime;
_lastReadTime = _lastWriteTime = _creationTime;
_id = Interlocked.Increment(ref idGenerator);
_closeFuture = new DefaultCloseFuture(this);
_closeFuture.Complete += ResetCounter;
}
/// <summary>
/// </summary>
protected SocketSession(IoService service, IoProcessor processor, IoSessionConfig config,
System.Net.Sockets.Socket socket, EndPoint localEP, EndPoint remoteEP, Boolean reuseBuffer)
: base(service)
{
_socket = socket;
_localEP = localEP;
_remoteEP = remoteEP;
Config = config;
if (service.SessionConfig != null)
{
Config.SetAll(service.SessionConfig);
}
_processor = processor;
_filterChain = new DefaultIoFilterChain(this);
}
private string ReadAndValidatePassword()
{
bool valid;
string password;
do
{
IoService.Out.Write(Resources.InfoPassword);
password = IoService.ReadPassword();
valid = UserValidator.ValidPassword(password);
if (!valid)
{
IoService.Out.WriteLine(
Resources.ErrorNotValidPassword,
PasswordValidation.MinPasswordLength);
}
}while (!valid);
return(password);
}
public void InitSettings()
{
var settings = new GameSetting();
var overview = new ResultOverview();
var ioService = new IoService();
var eventManager = new EventController();
// Init the environment
loader.InitDecks(settings);
loader.InitResultOverview(overview, ioService);
// Adds internal events for the app
eventManager.Add(new Event(EventNewCardGame));
// Finally add it to the IOC container
var facade = FacadeFactory.Create();
facade.AddUnique(settings, Settings);
facade.AddUnique(overview, Results);
facade.AddUnique(ioService, IoService);
facade.AddUnique(eventManager, EventManager);
}
private static void Main(string[] args)
{
var service = new IoService(new ReactiveProcessFactory());
var repositoryRoot = service.CurrentDirectory.Ancestors().First(ancestor => (ancestor / ".git").Exists());
var todoRegex = new Regex(@"TODO:(?<todoDescription>[^\n]+)");
var changes = repositoryRoot.Descendants()
.ToLiveLinq()
.Where(path => path.HasExtension(".md") && path.GetPathType() == PathType.File)
.Select(path => path.AsTextFile().Read().Lines
.Select(line => todoRegex.Match(line))
.Where(match => match.Success)
.Select(match => new { match, path }))
.SelectMany(x => x);
changes.AsObservable().Subscribe(x =>
{
if (x.Type == CollectionChangeType.Add)
{
foreach (var item in x.Values)
{
Console.WriteLine($"A TODO item was added: {item.match.Groups["todoDescription"].Value.Trim()}");
}
}
else
{
foreach (var item in x.Values)
{
Console.WriteLine($"A TODO item was removed: {item.match.Groups["todoDescription"].Value.Trim()}");
}
}
});
Console.WriteLine("Press any key to exit");
Console.ReadKey();
}
public static void Sort(string input, string output)
{
IConfig config =
new Config(
input,
output);
IGroupsInfoMarger groupsInfoMarger =
new GroupsInfoMarger();
IBuffersPool buffersPool =
new BuffersPool(
config);
IIoService ioService =
new IoService(
buffersPool);
ITasksQueue tasksQueue =
new TasksQueue(
config);
IInputReaderFactory inputReaderFactory =
new InputReaderFactory(
ioService,
tasksQueue,
buffersPool,
config);
IGroupsLinesOutputFactory groupsLinessOutputFactory =
new GroupsLinesOutputFactory(
ioService,
tasksQueue,
buffersPool,
config);
IGrouperIOs grouperIOs =
new GrouperIOs(
inputReaderFactory,
groupsLinessOutputFactory,
ioService,
config);
IGrouper grouper =
new Grouper(
groupsInfoMarger,
grouperIOs,
tasksQueue,
config);
IGroupsLoaderFactory groupsLoaderFactory =
new GroupsLoaderFactory(
buffersPool,
ioService,
config);
ISortingSegmentsSupplier sortingSegmentsSupplier =
new SortingSegmentsSupplier(
config);
ILinesIndexesExtractor linesIndexesExtractor =
new LinesIndexesExtractor(
config);
IGroupSorter groupSorter =
new GroupSorter(
sortingSegmentsSupplier,
linesIndexesExtractor);
ISortedGroupWriterFactory sortedGroupWriterFactory =
new SortedGroupWriterFactory(
ioService,
config);
ISorter sorter =
new Sorter(
ioService,
grouper,
groupsLoaderFactory,
groupSorter,
sortedGroupWriterFactory,
config);
sorter.Sort();
}
public void DirectoryIsValid_With_Valid_Dir_Should_Return_True ()
{
var service = new IoService();
bool result = service.DirectoryIsValid("../../testdata");
Assert.IsTrue(result);
}
public void DirectoryIsValid_With_Invalid_Dir_Should_Return_False ()
{
var service = new IoService();
bool result = service.DirectoryIsValid("invalid/path");
Assert.IsFalse(result);
}
public void DirectoryIsValid_With_EmptyOrNull_Dir_Should_Return_False (String input)
{
var service = new IoService();
bool result = service.DirectoryIsValid(input);
Assert.IsFalse(result);
}
private ISetChanges <AbsolutePath> ToLiveLinqWithFsWatch(AbsolutePath root, bool includeFileContentChanges, PathObservationMethod observationMethod)
{
// TODO - add support for FSWatch events on Windows and Linux as well. Although I think I already support all the ones on Linux
// and the FileSystemWatcher class on Windows should be sufficient, it would be nice to have this support for
// completeness' sake.
ReactiveProcess proc;
var recursiveArg = IncludeSubdirectories ? "--recursive" : string.Empty;
if (observationMethod == PathObservationMethod.FsWatchDefault)
{
proc = IoService.ReactiveProcessFactory.Start("fswatch", $"-0 {recursiveArg} \"{root}\"");
}
else if (observationMethod == PathObservationMethod.FsWatchPollMonitor)
{
proc = IoService.ReactiveProcessFactory.Start("fswatch", $"--monitor=poll_monitor -0 {recursiveArg} \"{root}\"");
}
else if (observationMethod == PathObservationMethod.FsWatchFsEventsMonitor)
{
proc = IoService.ReactiveProcessFactory.Start("fswatch", $"--monitor=fsevents_monitor -0 {recursiveArg} \"{root}\"");
}
else if (observationMethod == PathObservationMethod.FsWatchKQueueMonitor)
{
proc = IoService.ReactiveProcessFactory.Start("fswatch", $"--monitor=kqueue_monitor -0 {recursiveArg} \"{root}\"");
}
else
{
throw new ArgumentException($"Unknown path observation method: {observationMethod}");
}
var initialState = this
.ToImmutableDictionary(x => x, x => x.GetPathType());
var resultObservable = proc.StandardOutput
.Scan(new { StringBuilder = new StringBuilder(), BuiltString = (string)null },
(state, ch) =>
{
if (ch == 0)
{
return(new
{
StringBuilder = new StringBuilder(), BuiltString = state.StringBuilder.ToString()
});
}
state.StringBuilder.Append(ch);
return(new { state.StringBuilder, BuiltString = (string)null });
}).Where(state => state.BuiltString != null).Select(state => state.BuiltString)
.Scan(new { State = initialState, LastEvents = (IDictionaryChangeStrict <AbsolutePath, PathType>[])null },
(state, itemString) =>
{
var item = IoService.TryParseAbsolutePath(itemString).Value;
if (state.State.ContainsKey(item))
{
if (File.Exists(itemString) || Directory.Exists(itemString))
{
if (includeFileContentChanges)
{
return(new
{
state.State,
LastEvents = new []
{
LiveLinq.Utility.DictionaryRemove(MoreCollections.Utility.KeyValuePair(item, state.State[item])),
LiveLinq.Utility.DictionaryAdd(MoreCollections.Utility.KeyValuePair(item, item.GetPathType()))
}
});
}
else
{
return(new
{
state.State,
LastEvents = new IDictionaryChangeStrict <AbsolutePath, PathType> [0]
});
}
}
else
{
// TODO - fix bug where when a directory is deleted, subdirectories and subfolders are not removed from the state.
return(new
{
State = state.State.Remove(item),
LastEvents = new IDictionaryChangeStrict <AbsolutePath, PathType>[]
{
LiveLinq.Utility.DictionaryRemove(MoreCollections.Utility.KeyValuePair(item, state.State[item])),
}
});
}
}
else
{
return(new
{
State = state.State.Add(item, item.GetPathType()),
LastEvents = new IDictionaryChangeStrict <AbsolutePath, PathType>[]
{
LiveLinq.Utility.DictionaryAdd(MoreCollections.Utility.KeyValuePair(item, item.GetPathType())),
}
});
}
})
.SelectMany(state => state.LastEvents);
resultObservable = Observable.Return(LiveLinq.Utility.DictionaryAdd(initialState))
.Concat(resultObservable);
var result = resultObservable.ToLiveLinq().KeysAsSet();
if (!string.IsNullOrWhiteSpace(_pattern))
{
var regex = IoService.FileNamePatternToRegex(_pattern);
result = result.Where(path => regex.IsMatch(path.Name));
}
return(result);
}
public IoServiceStatistics(IoService service)
{
_service = service;
}
public AsyncSocketSession(IoService service, IoProcessor <SocketSession> processor,
System.Net.Sockets.Socket socket, EndPoint localEP, EndPoint remoteEP, Boolean reuseBuffer)
: base(service, processor, new SessionConfigImpl(socket), socket, localEP, remoteEP, reuseBuffer)
{
_readBuffer = new Byte[service.SessionConfig.ReadBufferSize];
}
internal AsyncDatagramSession(IoService service, IoProcessor<AsyncDatagramSession> processor,
AsyncDatagramAcceptor.SocketContext ctx, EndPoint remoteEP, Boolean reuseBuffer)
: base(service, processor, new DefaultDatagramSessionConfig(), ctx.Socket, ctx.Socket.LocalEndPoint, remoteEP, reuseBuffer)
{
_socketContext = ctx;
}
static void Main(string[] args)
{
var service = new IoService(new ReactiveProcessFactory());
var repositoryRoot = service.CurrentDirectory.Ancestors().First(ancestor => (ancestor / ".git").Exists());
var documentationRoot = service.ParseAbsolutePath("/Users/family/Resilio Sync/NathanLoumFamily");
var markdownFiles = documentationRoot.Descendants()
.ToLiveLinq()
.Where(x => x.HasExtension(".md") && x.GetPathType() == PathType.File)
.Select(x => x.AsSmallTextFile());
var markdownHtmls = markdownFiles
.Select(markdownFile =>
{
var markdown = markdownFile.Read();
var html = "<html>" + Markdig.Markdown.ToHtml(markdown) + "</html>";
var document = new XmlDocument();
document.LoadXml(html);
return(new { markdownFile.Path, Html = document });
});
var docxFiles = documentationRoot.Descendants()
.ToLiveLinq()
.Where(x => x.HasExtension(".docx") && x.GetPathType() == PathType.File);
var docxHtmls = docxFiles
.Select(docxPath => new { Source = docxPath, Target = docxPath.WithExtension(".html") })
.Do(x => x.Source.PandocToHtml(x.Target), x => x.Target.DeleteFile())
.Select(x => new { Path = x.Source, Html = x.Target.AsXmlFile().Read() });
var htmls = markdownHtmls.Concat(docxHtmls, true);
var backLinks = htmls.SelectMany(html =>
{
var links = html.Html.SelectNodes("//a");
var backLinksForThisFile = new List <Tuple <AbsolutePath, AbsolutePath, string> >();
foreach (var link in links)
{
var linkEl = link as XmlElement;
var href = linkEl.Attributes["href"].InnerText;
var text = linkEl.InnerText;
if (href.Contains(":"))
{
continue;
}
var maybePath = service.TryParseRelativePath(href);
if (!maybePath.HasValue)
{
continue;
}
var linkTo = (html.Path.Parent() / maybePath.Value).Simplify();
backLinksForThisFile.Add(Tuple.Create(html.Path, linkTo, text));
}
var parentReadme = string.Equals(html.Path.Name, "readme.md", StringComparison.OrdinalIgnoreCase) ? html.Path.Ancestor(2) / "readme.md" : html.Path.Ancestor(1) / "readme.md";
backLinksForThisFile.Add(Tuple.Create(html.Path, parentReadme, "Parent"));
return(backLinksForThisFile.AsEnumerable());
})
.GroupBy(x => x.Item2)
.SelectValue(x => x.Select(y => Tuple.Create(y.Item1, y.Item3)))
.ToReadOnlyObservableDictionary();
var transformedHtmls = htmls.Select(html => backLinks.ToLiveLinq()[html.Path]
.SelectLatest(x =>
{
return(x.Select(y => y.ToObservableState()).Otherwise(() =>
Observable.Return(ImmutableHashSet <Tuple <AbsolutePath, string> > .Empty)));
})
.Select(backLinksForThisHtml =>
{
var links = html.Html.SelectNodes("//a");
foreach (var link in links)
{
var linkEl = link as XmlElement;
var href = linkEl.Attributes["href"].InnerText;
var text = linkEl.InnerText;
if (href.Contains(":"))
{
continue;
}
var maybePath = service.TryParseRelativePath(href);
if (!maybePath.HasValue)
{
continue;
}
if (linkEl.Attributes["href"].InnerText.EndsWith(".md"))
{
linkEl.Attributes["href"].InnerText =
linkEl.Attributes["href"].InnerText
.Substring(0, linkEl.Attributes["href"].InnerText.Length - 3) + ".html";
}
if (linkEl.Attributes["href"].InnerText.EndsWith(".docx"))
{
linkEl.Attributes["href"].InnerText =
linkEl.Attributes["href"].InnerText
.Substring(0, linkEl.Attributes["href"].InnerText.Length - 3) + ".html";
}
}
var backLinkHtml =
"<ul>" + string.Join("\n",
backLinksForThisHtml.Select(x =>
{
var relativePath = x.Item1.WithExtension(".html").RelativeTo(html.Path.Parent());
return
($"<li>This page is \"{x.Item2}\" of - <a href=\"{relativePath}\">{relativePath}</a></li>");
})) + "</ul>";
return(new { html.Path, Html = html.Html.OuterXml + backLinkHtml });
}));
transformedHtmls
.Subscribe(addedHtml => {
Console.WriteLine($"Adding a markdown file: {addedHtml.Path}");
addedHtml.Path.WithExtension(".html").WriteAllText(addedHtml.Html);
}, (removedHtml, removalMode) => {
Console.WriteLine($"Removing a markdown file: {removedHtml.Path}");
removedHtml.Path.WithExtension(".html").Delete();
});
}
public AsyncSocketSession(IoService service, IoProcessor<SocketSession> processor,
System.Net.Sockets.Socket socket, Boolean reuseBuffer)
: base(service, processor, new SessionConfigImpl(socket), socket, socket.LocalEndPoint, socket.RemoteEndPoint, reuseBuffer)
{
_readBuffer = new Byte[service.SessionConfig.ReadBufferSize];
}
/// <summary>
/// Initializes.
/// </summary>
public IoServiceStatistics(IoService service)
{
_service = service;
}
static TrainTestData <BasicMLDataSet> GetBasicMlDataSet(string connectionStr, bool recreateDb, ILogger logger, DirectoryInfo inputDirectory, int ommitStocksSmallerThan, int ommitDeadStocksDate, Random rnProvider, decimal ratioTrainingSet)
{
var context = new StockEfContext(connectionStr);
var unitOfWork = new StockEfUnitOfWork(context);
var stocksDeserialized = default(List <Company>);
var watch = Stopwatch.StartNew();
if (context.DbExists() && !recreateDb)
{
stocksDeserialized = unitOfWork.Stocks.GetAll().ToList();
logger.LogInfo(
$@"Found {stocksDeserialized.Count} companies in Db in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
}
else
{
if (context.DbExists())
{
context.DropDbIfExists();
logger.LogInfo($@"Dropped Db in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
}
context.CreateDbIfNotExists();
logger.LogInfo($@"Created Db in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
var directoryService = new IoService();
var stocksRaw = directoryService.ReadDirectory(inputDirectory);
logger.LogInfo($@"Read {stocksRaw.Count} in {watch.ElapsedMilliseconds.AsTime()} from {inputDirectory.Name}");
watch.Restart();
stocksDeserialized = new StocksBulkDeserializer(new StocksDeserializer(new StockQuoteCsvClassMap())).Deserialize(stocksRaw);
logger.LogInfo($@"Deserialized {stocksDeserialized.Count} in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
var bulkInserter = new CompanyBulkInserter(connectionStr);
bulkInserter.BulkInsert(stocksDeserialized);
logger.LogInfo($@"Saved {stocksDeserialized.Count} to {connectionStr} in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
}
var normalizer = new StockQuotesToNormalizedMatrix();
var allStocksNormalized = new List <BasicMLDataSet>();
var matrixConverter = new MatrixToMlData();
var ommitedDueToLength = 0;
var ommitedDueToInvalidity = 0;
foreach (var stock in stocksDeserialized)
{
if (stock.Quotes.Count < ommitStocksSmallerThan)
{
++ommitedDueToLength;
}
else if (stock.Quotes.Max(s => s.Date) < ommitDeadStocksDate)
{
++ommitedDueToInvalidity;
}
else
{
allStocksNormalized.Add(matrixConverter.ConvertToHighPred(normalizer.Convert(stock.Quotes.ToList())));
}
}
logger.LogInfo(
$@"Loaded, converted and normalized {allStocksNormalized.Count} ({allStocksNormalized.Sum(s => s.Count)} samples) in {watch.ElapsedMilliseconds.AsTime()}. Ommited {
stocksDeserialized.Count - allStocksNormalized.Count
}.{(ommitedDueToLength > 0 || ommitedDueToInvalidity > 0 ? " Reason:" : string.Empty)}{
(ommitedDueToLength > 0 ? $" {ommitedDueToLength} too small" : string.Empty)}{(ommitedDueToLength > 0 && ommitedDueToInvalidity > 0 ? "," : string.Empty)}{
(ommitedDueToInvalidity > 0 ? $" {ommitedDueToInvalidity} invalid" : string.Empty)}");
watch.Restart();
var trainDataSet = new BasicMLDataSet();
var testDataSet = new BasicMLDataSet();
var i = 0;
for (; i < allStocksNormalized.Count * ratioTrainingSet; ++i)
{
foreach (var mlDataPair in allStocksNormalized[i].Data)
{
trainDataSet.Add(mlDataPair);
}
}
for (; i < allStocksNormalized.Count; ++i)
{
foreach (var mlDataPair in allStocksNormalized[i].Data)
{
testDataSet.Add(mlDataPair);
}
}
logger.LogInfo($@"Constructed training and test datasets with {trainDataSet.Count} and {testDataSet.Count} samples in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
trainDataSet.Data.Shuffle(rnProvider);
logger.LogInfo($@"Finished shuffling trainDataSet ({trainDataSet.Count} samples) in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
testDataSet.Data.Shuffle(rnProvider);
logger.LogInfo($@"Finished shuffling testDataSet ({testDataSet.Count} samples) in {watch.ElapsedMilliseconds.AsTime()}");
watch.Restart();
return(new TrainTestData <BasicMLDataSet> {
TrainingSet = trainDataSet, TestSet = testDataSet
});
}
public void Test(
string inputFileSettings,
string groupsFile,
int bufferSize,
int enginesCount,
int maxThreadsCount,
bool clear)
{
string inputFilePath = null;
try
{
if (inputFileSettings.StartsWith(UseExistanceFile))
{
inputFilePath = SplitString(inputFileSettings, ": ")[1];
}
else
{
var fileGenerationSettings = SplitString(inputFileSettings, " ");
Generator.Generate(sizeData: fileGenerationSettings[0],
lineSettings: fileGenerationSettings[1],
path: fileGenerationSettings[2]);
inputFilePath = fileGenerationSettings[2];
}
var configMock = new Mock <IConfig>();
var physicalBufferLength = bufferSize + 1;
configMock
.SetupGet(o => o.PhysicalBufferLength)
.Returns(physicalBufferLength);
configMock
.SetupGet(o => o.UsingBufferLength)
.Returns(bufferSize);
configMock
.SetupGet(o => o.MaxRunningTasksCount)
.Returns(maxThreadsCount);
configMock
.SetupGet(o => o.GrouperEnginesCount)
.Returns(enginesCount);
configMock
.SetupGet(o => o.InputFilePath)
.Returns(inputFilePath);
configMock
.SetupGet(o => o.GroupsFilePath)
.Returns(groupsFile);
IGroupsInfoMarger groupsSummaryInfoMarger =
new GroupsInfoMarger();
ITasksQueue tasksQueue =
new TasksQueue(configMock.Object);
IBuffersPool buffersPool =
new InfinityBuffersPool(physicalBufferLength);
IIoService ioService =
new IoService(
buffersPool);
IInputReaderFactory inputReaderMaker =
new InputReaderFactory(
ioService,
tasksQueue,
buffersPool,
configMock.Object);
IGroupsLinesOutputFactory linesWriterFactory =
new GroupsLinesOutputFactory(
ioService,
tasksQueue,
buffersPool,
configMock.Object);
IGrouperIOs grouperIOs =
new GrouperIOs(
inputReaderMaker,
linesWriterFactory,
ioService,
configMock.Object);
ILinesIndexesExtractor linesIndexesExtractor =
new LinesIndexesExtractor(
configMock.Object);
IGroupsLoaderFactory groupsLoaderMaker =
new GroupsLoaderFactory(
buffersPool,
ioService,
configMock.Object);
var grouper = new Grouper(
groupsSummaryInfoMarger,
grouperIOs,
tasksQueue,
configMock.Object);
var trivialGrouper = new TrivialGrouper();
var expectedGroups = trivialGrouper.SplitToGroups(
ReadAllLinesFrom(inputFilePath));
var groupsInfo = grouper.SeparateInputToGroups();
var output = new IGroup[Consts.MaxGroupsCount];
var loader = groupsLoaderMaker.Create(groupsInfo, output);
loader.LoadNextGroups();
var expectedGroupIds = expectedGroups
.Select(o => o.Id)
.ToArray();
var actualGroupIds = groupsInfo
.Select((group, id) => new { group, id })
.Where(o => !GroupInfo.IsZero(o.group))
.Select(o => o.id)
.ToArray();
#region DEBUG
// #if DEBUG
// var expectedGroupPrefixes = expectedGroupIds
// .Select(ToPrefix)
// .ToArray();
//
// var actualGroupPrefixes = actualGroupIds
// .Select(ToPrefix)
// .ToArray();
//
// var expectedGroupPrefixesInLine =
// string.Join(" | ", expectedGroupPrefixes);
//
// var actualGroupPrefixesInLine =
// string.Join(" | ", actualGroupPrefixes);
//
// var actualIdsOnly = actualGroupIds
// .Except(expectedGroupIds)
// .Select(ToPrefix);
//
// var actualIdsOnlyInLine =
// string.Join(" | ", actualIdsOnly);
//
// var expectedIdsOnly = expectedGroupIds
// .Except(actualGroupIds)
// .Select(ToPrefix);
//
// var allPrefixes =
// new[]
// {
// new[] {string.Empty},
//
// Enumerable.Range(' ', '~' - ' ' + 1)
// .Select(o => ((char) o).ToString()),
//
// Enumerable.Join(
// Enumerable.Range(' ', '~' - ' ' + 1),
// Enumerable.Range(' ', '~' - ' ' + 1),
// _ => true,
// _ => true,
// (c1, c2) => new string(new []{(char)c1, (char)c2}))
// }
// .Aggregate(Enumerable.Concat)
// .ToArray();
//
// var allCalculatedIds = allPrefixes
// .Select(ToId)
// .OrderBy(o => o)
// .ToArray();
//
// var allCalculatedIdsDistinct =
// allCalculatedIds.Distinct().ToArray();
//
// var allCalculatedPrefixes = Enumerable
// .Range(0, Consts.MaxGroupsCount)
// .Select(ToPrefix)
// .ToArray();
//
// var allCalculatedPrefixesDistinct =
// allCalculatedPrefixes.Distinct().ToArray();
// #endif
#endregion
CollectionAssert.AreEqual(
expectedGroupIds,
actualGroupIds);
int j = 0;
for (int i = 0; i < Consts.MaxGroupsCount; i++)
{
var info = groupsInfo[i];
if (GroupInfo.IsZero(info))
{
continue;
}
var expectedInfo = expectedGroups[j];
Assert.AreEqual(expectedInfo.BytesCount, info.BytesCount);
Assert.AreEqual(expectedInfo.LinesCount, info.LinesCount);
linesIndexesExtractor.ExtractIndexes(output[i]);
var expectedLines = expectedInfo.Lines
.Select(o => o.Content)
.ToArray();
foreach (var line in expectedLines)
{
line[0] = Consts.EndLineByte1;
}
var expectedLinesDictionary =
new Dictionary <HashedBytesArray, int>(info.LinesCount);
for (int k = 0; k < info.LinesCount; k++)
{
var hashedLine = Hash(expectedLines[k]);
if (expectedLinesDictionary.ContainsKey(hashedLine))
{
++expectedLinesDictionary[hashedLine];
}
else
{
expectedLinesDictionary.Add(hashedLine, 1);
}
}
#region DEBUG
// #if DEBUG
// var linesCountInDictionary = expectedLinesDictionary
// .Values.Sum(o => o);
// #endif
#endregion
var lines = output[i].Lines;
for (int k = 0; k < info.LinesCount; k++)
{
var lineIndexes = lines.Array[lines.Offset + k];
var lineLength = lineIndexes.LettersCount
+ lineIndexes.DigitsCount
+ 3;
var buffers = output[i].Buffers;
var bufferIndex = lineIndexes.Start / bufferSize;
var indexInBuffer = lineIndexes.Start % bufferSize;
var line = new byte[lineLength];
if (indexInBuffer + lineLength <= bufferSize)
{
Array.Copy(buffers.Array[buffers.Offset + bufferIndex], indexInBuffer,
line, 0,
lineLength);
}
else
{
var bufferRightLength = bufferSize - indexInBuffer;
Array.Copy(buffers.Array[buffers.Offset + bufferIndex], indexInBuffer,
line, 0,
bufferRightLength);
Array.Copy(buffers.Array[buffers.Offset + bufferIndex + 1], 0,
line, bufferRightLength,
lineLength - bufferRightLength);
}
var actualHashedLine = Hash(line);
Assert.IsTrue(expectedLinesDictionary.ContainsKey(actualHashedLine));
--expectedLinesDictionary[actualHashedLine];
if (expectedLinesDictionary[actualHashedLine] == 0)
{
expectedLinesDictionary.Remove(actualHashedLine);
}
}
Assert.AreEqual(0, expectedLinesDictionary.Count);
++j;
}
Assert.AreEqual(expectedGroups.Length, j);
loader.Dispose();
}
finally
{
if (clear)
{
if (!inputFileSettings.StartsWith(UseExistanceFile) &&
inputFilePath != null &&
File.Exists(inputFilePath))
{
File.Delete(inputFilePath);
}
if (File.Exists(groupsFile))
{
File.Delete(groupsFile);
}
}
}
}
/// <summary>
/// Creates a new acceptor-side session instance.
/// </summary>
internal AsyncDatagramSession(IoService service, IoProcessor <AsyncDatagramSession> processor,
AsyncDatagramAcceptor.SocketContext ctx, EndPoint remoteEP, Boolean reuseBuffer)
: base(service, processor, new DefaultDatagramSessionConfig(), ctx.Socket, ctx.Socket.LocalEndPoint, remoteEP, reuseBuffer)
{
_socketContext = ctx;
}