public static void RegistRemotingObj(string channelName,
int port,
string typeName,
string assembly,
string url)
{
//Port
IChannel chnl = ChannelServices.GetChannel(channelName);
if (channel == null || chnl == null)
{
if (channel != null)
ChannelServices.UnregisterChannel(channel);
if (chnl != null)
ChannelServices.UnregisterChannel(chnl);
channel = new TcpServerChannel(channelName, port);
ChannelServices.RegisterChannel(channel, false);
}
//Object Regist
WellKnownServiceTypeEntry registerType = new WellKnownServiceTypeEntry(typeName, assembly, url, WellKnownObjectMode.Singleton);
RemotingConfiguration.RegisterWellKnownServiceType(registerType);
RemotingConfiguration.CustomErrorsMode = CustomErrorsModes.Off;
RemotingConfiguration.CustomErrorsEnabled(false);
}
/// <summary>
/// Constructs a new ActorContext.
/// </summary>
/// <param name="system">the ActorSystem within which the actor is created</param>
/// <param name="actorType">the Type of the actor</param>
/// <param name="actor">the Actor to create</param>
/// <param name="name">the string name to give the Actor</param>
/// <param name="props">the Props to pass individually as class arguments</param>
/// <param name="parent">the ActorRef of the parent of the Actor being created</param>
/// <param name="parentPath">the ActorPath of the Actor being created</param>
/// <param name="suspended">the bool indicating whether the actor is being created as suspended</param>
/// <param name="channel">the IChannel the actor will use</param>
/// <param name="fiber">the IFiber the actor will use</param>
internal ActorContext(
ActorSystem system,
Type actorType,
Actor actor,
string name,
Props props,
ActorRef parent,
ActorPath parentPath,
bool suspended,
IChannel<Delivery> channel,
IFiber fiber)
{
Actor = actor;
Actor.InternalContext = this;
Channel = channel;
Fiber = fiber;
Children = new List<ActorRef>(0);
Parent = parent;
Path = parentPath.WithName(name);
Props = props;
Receivers = new Stack<Receive>(1);
Self = new ActorRef(this);
Sender = ActorRef.NoSender;
_Suspended = suspended;
System = system;
_Terminated = false;
Type = actorType;
Start();
}
/// <summary>
/// </summary>
/// <param name="channel"> </param>
protected override void Play(IChannel channel)
{
// REALLY NEED TO CHANGE THIS IOC/TEMPLATE METHOD IN BASE CLASS
// http://blogs.msdn.com/danielfe/archive/2004/06/08/151212.aspx
// _channel = channel;
//if (channel == null) throw new ArgumentNullException("channel", "Must specify a channel to play. ");
var url = channel.CurrentTrack.TrackUrl;
if (!IsInstalled) return;
//Trace.WriteLine(string.Format("{0} will attempt to stream {1}", this.PlayerType.ToString(), url));
var thread = new Thread
(() =>
{
iTunesApp player = new iTunesAppClass();
player.OpenURL(ParseStreamUri(url).AbsoluteUri);
player.Play();
});
thread.Start();
//Parallel.Invoke(
// () => startPlayer(url));
}
/// <summary>
/// BSON 패킷 구조를 따르는 PacketBuffer을 BSON Data로 변환 시키는 메서드
/// </summary>
/// <param name="buffer">BSON 패킷 구조를 따르는 Packet Buffer</param>
/// <returns>BSON Data</returns>
public dynamic Decode(IChannel channel, PacketBuffer buffer)
{
//버퍼 읽기 시작을 알림
buffer.BeginBufferIndex();
if (buffer.AvailableBytes() < 5) //버퍼길이가 5미만이면 리턴
return null;
uint len = buffer.ReadUInt32();
if (len > buffer.AvailableBytes())
{
//버퍼의 길이가 실제 패킷 길이보다 모자름으로, 리셋후 리턴
buffer.ResetBufferIndex();
return null;
}
var data = new byte[len];
buffer.ReadBytes(data);
buffer.EndBufferIndex();
var stream = new MemoryStream(data);
dynamic res = Serializer.Deserialize(new BsonReader(stream));
stream.Dispose();
return res;
}
public void SetUp()
{
testSender = MockRepository.GenerateStub<ISendMessages>();
messagePersister = new InMemoryPersistence();
httpChannel = new HttpChannel(messagePersister)
{
ListenUrl = "http://localhost:8092/Gateway/",
ReturnAddress = "Gateway.Tests.Input"
};
httpChannel.MessageReceived += httpChannel_MessageReceived;
httpChannel.Start();
bus = Configure.With()
.DefaultBuilder()
.XmlSerializer()
.FileShareDataBus("./databus")
.InMemoryFaultManagement()
.UnicastBus()
.MsmqTransport()
.CreateBus()
.Start();
}
private static void Abort(IChannel channel, ChannelFactory channelFactory)
{
if (channel != null)
channel.Abort();
if (channelFactory != null)
channelFactory.Abort();
}
public Form1()
{
InitializeComponent();
log4net.Config.XmlConfigurator.Configure();
textBox1.Text = ConfigurationManager.AppSettings["Path"];
_application = new Application(ApplicationName);
_workerFiber = new PoolFiber();
_workerFiber.Start();
_importar = new Channel<string>();
_importar.Subscribe(_workerFiber, Work);
_notificationType = new NotificationType(SampleNotificationType, "Sample Notification");
_growl = new GrowlConnector { EncryptionAlgorithm = Cryptography.SymmetricAlgorithmType.AES };
_growl.NotificationCallback += GrowlNotificationCallback;
_growl.Register(_application, new[] { _notificationType });
folderWatch.Created += FolderWatchCreated;
folderWatch.Renamed += FolderWatchRenamed;
folderWatch.Deleted += FolderWatchDeleted;
InitDatabase();
if (!_autostart) return;
Operar(true,true);
}
public void AddRange(IChannel[] values)
{
foreach (IChannel chnl in values)
{
InnerList.Add(chnl);
}
}
public MessageConsumerBuilder(IChannel channel, string queueName)
{
_channel = channel;
_queueName = queueName;
_prefetchHigh = _channel.DefaultPrefetchHigh;
_prefetchLow = _channel.DefaultPrefetchLow;
}
public dynamic Decode(IChannel channel, PacketBuffer buffer)
{
buffer.BeginBufferIndex();
if (buffer.AvailableBytes() < 1)
{
buffer.ResetBufferIndex();
return null;
}
var data = new byte[buffer.AvailableBytes()];
buffer.ReadBytes(data);
string s = System.Text.Encoding.UTF8.GetString(data);
int len = s.IndexOf('\n');
if (len == -1)
{
buffer.ResetBufferIndex();
return null;
}
s = s.Substring(0, len + 1);
buffer.SetPosition(System.Text.Encoding.UTF8.GetByteCount(s));
buffer.EndBufferIndex();
return s;
}
protected AbstractSocketChannel(IChannel parent, Socket socket)
: base(parent)
{
Socket = socket;
_state = StateFlags.Open;
try
{
Socket.Blocking = false;
}
catch (SocketException ex)
{
try
{
socket.Close();
}
catch (SocketException ex2)
{
if (Logger.IsWarningEnabled)
{
Logger.Warning("Failed to close a partially initialized socket.", ex2);
}
}
throw new ChannelException("Failed to enter non-blocking mode.", ex);
}
}
public IpcChannel ()
{
if (IsUnix)
_innerChannel = new Unix.IpcChannel ();
else
_innerChannel = new Win32.IpcChannel ();
}
public IpcChannel (string portName)
{
if (IsUnix)
_innerChannel = new Unix.IpcChannel (portName);
else
_innerChannel = new Win32.IpcChannel (portName);
}
/// <summary>
/// Set up remoting communication channels.
/// </summary>
private void SetupChannels()
{
lock (m_hostLock)
{
// If channels are not set up yet, set them up.
if (m_serverChannel == null)
{
Debug.Assert(m_clientChannel == null);
// This channel is used for debugging session, when TA connects to authoring vs instance.
BinaryServerFormatterSinkProvider serverProvider = new BinaryServerFormatterSinkProvider();
serverProvider.TypeFilterLevel = TypeFilterLevel.Full; // Enable remoting objects as arguments.
Hashtable props = new Hashtable();
string channelNamePrefix = "ExcelTestHostAddin_";
string serverChannelName = channelNamePrefix + Guid.NewGuid() + "_ServerChannel";
props["name"] = serverChannelName;
props["portName"] = serverChannelName; // Must be different from client's port.
// Default IpcChannel security is: allow for all users who can authorize on this machine.
props["authorizedGroup"] = WindowsIdentity.GetCurrent().Name;
m_serverChannel = new IpcServerChannel(props, serverProvider);
ChannelServices.RegisterChannel(m_serverChannel, false);
// This channel is used for connecting to both authoring and host side VS.
m_clientChannel = new IpcClientChannel(channelNamePrefix + Guid.NewGuid() + "_ClientChannel", new BinaryClientFormatterSinkProvider());
ChannelServices.RegisterChannel(m_clientChannel, false);
}
}
}
public void Disconnected(IChannel channel)
{
channelLock.EnterWriteLock();
_channels.Remove(channel);
channelLock.ExitWriteLock();
SendMessage(channel + " 88" + "\r\n> ");
}
/// <summary>
/// Creates a MediaItem that represents a TV stream. The MediaItem also holds information about stream indices to provide PiP
/// functions (<paramref name="slotIndex"/>).
/// </summary>
/// <param name="slotIndex">Index of the slot (0/1)</param>
/// <param name="path">Path or URL of the stream</param>
/// <param name="channel"></param>
/// <returns></returns>
public static LiveTvMediaItem CreateMediaItem(int slotIndex, string path, IChannel channel)
{
if (!String.IsNullOrEmpty(path))
{
ISystemResolver systemResolver = ServiceRegistration.Get<ISystemResolver>();
IDictionary<Guid, MediaItemAspect> aspects = new Dictionary<Guid, MediaItemAspect>();
MediaItemAspect providerResourceAspect;
MediaItemAspect mediaAspect;
SlimTvResourceAccessor resourceAccessor = new SlimTvResourceAccessor(slotIndex, path);
aspects[ProviderResourceAspect.ASPECT_ID] = providerResourceAspect = new MediaItemAspect(ProviderResourceAspect.Metadata);
aspects[MediaAspect.ASPECT_ID] = mediaAspect = new MediaItemAspect(MediaAspect.Metadata);
// VideoAspect needs to be included to associate VideoPlayer later!
aspects[VideoAspect.ASPECT_ID] = new MediaItemAspect(VideoAspect.Metadata);
providerResourceAspect.SetAttribute(ProviderResourceAspect.ATTR_SYSTEM_ID, systemResolver.LocalSystemId);
String raPath = resourceAccessor.CanonicalLocalResourcePath.Serialize();
providerResourceAspect.SetAttribute(ProviderResourceAspect.ATTR_RESOURCE_ACCESSOR_PATH, raPath);
mediaAspect.SetAttribute(MediaAspect.ATTR_TITLE, "Live TV");
mediaAspect.SetAttribute(MediaAspect.ATTR_MIME_TYPE, "video/livetv"); // Custom mimetype for LiveTv
LiveTvMediaItem tvStream = new LiveTvMediaItem(new Guid(), aspects);
tvStream.AdditionalProperties[LiveTvMediaItem.SLOT_INDEX] = slotIndex;
tvStream.AdditionalProperties[LiveTvMediaItem.CHANNEL] = channel;
tvStream.AdditionalProperties[LiveTvMediaItem.TUNING_TIME] = DateTime.Now;
return tvStream;
}
return null;
}
private static void _connection_OnReceived(object Data, System.IO.Ports.SerialPort port, IChannel channel, DateTime Timestamp)
{
//receive the data
_display.Clear();
Font fontNinaB = Resources.GetFont(Resources.FontResources.NinaB);
var display = (string)Data;
try
{
var ary = (string[]) Data;
if (ary != null)
{
display = "";
for (int i = 0; i < ary.Length; i++)
{
display += ary[i] + ",";
}
}
}
catch (Exception)
{
}
_display.DrawText(display, fontNinaB, Color.White, 10, 64);
_display.Flush();
//echo it back
_connection.Write(Data);
}
///// <summary>
// /// Creates a new instance.
// ///
// /// @param parent
// /// the parent of this channel. {@code null} if there's no parent.
// /// </summary>
//protected AbstractChannel(IChannel parent)
// : this(DefaultChannelId.NewInstance())
//{
//}
/// <summary>
/// Creates a new instance.
///
//* @param parent
//* the parent of this channel. {@code null} if there's no parent.
/// </summary>
protected AbstractChannel(IChannel parent /*, ChannelId id*/)
{
this.Parent = parent;
//this.Id = id;
this.channelUnsafe = this.NewUnsafe();
this.pipeline = new DefaultChannelPipeline(this);
}
public dynamic Decode(IChannel channel, PacketBuffer buffer)
{
buffer.BeginBufferIndex();
if(buffer.AvailableBytes() > MaxBufferSize)
{
channel.Disconnect();
return null;
}
if (buffer.AvailableBytes() == 0)
return null;
Request request;
try
{
request = Request.Parse(buffer);
}
catch (Exception)
{
buffer.ResetBufferIndex();
return null;
}
buffer.EndBufferIndex();
return request;
}
internal void SetInstanceIdChannel(string instanceId, IChannel channel)
{
lock (this)
{
this.channels[instanceId] = channel;
}
}
public CreateProgramAMSConfigParams(IChannel channel, string aAssetlName, string aProgramlName, TimeSpan DVR_WindowLength)
{
_channel = channel;
_aAssetlName = aAssetlName;
_aProgramlName = aProgramlName;
_dvrWindowLength = DVR_WindowLength;
}
private void SetupFunctionalTestChannels()
{
_ftChannel = new IpcChannel("PowerPointLabsFT");
ChannelServices.RegisterChannel(_ftChannel, false);
RemotingConfiguration.RegisterWellKnownServiceType(typeof(PowerPointLabsFT),
"PowerPointLabsFT", WellKnownObjectMode.Singleton);
}
public object Decode(IChannel channel, PacketBuffer buffer)
{
//버퍼 읽기 시작을 알림
buffer.BeginBufferIndex();
if (buffer.AvailableBytes() < 6) //버퍼길이가 5미만이면 리턴
return null;
buffer.ReadByte();
uint len = buffer.ReadUInt32();
if (len > buffer.AvailableBytes())
{
//버퍼의 길이가 실제 패킷 길이보다 모자름으로, 리셋후 리턴
buffer.ResetBufferIndex();
return null;
}
var data = new byte[len];
buffer.ReadBytes(data);
buffer.EndBufferIndex();
var stream = new MemoryStream(data);
var res = Unpacker.Create(stream).Unpack<MessagePackObject>();
stream.Dispose();
return res;
}
/// <summary>Method abort</summary>
/// <param name="error"></param>
/// <param name="channel"></param>
/// <exception cref="BEEPException" />
public virtual void abort(BEEPError error, IChannel channel)
{
tuningChannels.Remove(channel);
log.debug("TuningProfile.abort");
// Log entry or something - throw an exception???
}
private bool IsCamAbleToDecryptChannel(IUser user, ITvCardHandler tvcard, IChannel tuningDetail, int decryptLimit)
{
if (!tuningDetail.FreeToAir)
{
bool isCamAbleToDecryptChannel = true;
if (decryptLimit > 0)
{
int camDecrypting = NumberOfChannelsDecrypting(tvcard);
//Check if the user is currently occupying a decoding slot and subtract that from the number of channels it is decoding.
if (user.CardId == tvcard.DataBaseCard.IdCard)
{
bool isFreeToAir = IsFreeToAir(tvcard, ref user);
if (!isFreeToAir)
{
int numberOfUsersOnCurrentChannel = GetNumberOfUsersOnCurrentChannel(tvcard, user);
//Only subtract the slot the user is currently occupying if he is the only user occupying the slot.
if (numberOfUsersOnCurrentChannel == 1)
{
--camDecrypting;
}
}
}
//check if cam is capable of descrambling an extra channel
isCamAbleToDecryptChannel = (camDecrypting < decryptLimit);
}
return isCamAbleToDecryptChannel;
}
return true;
}
protected Importar()
{
_cola = new PoolFiber();
_channel=new Channel<string>();
_cola.Start();
_channel.Subscribe(_cola, MoverArchivo);
}
public SecurityDuplexSessionChannel (ChannelFactoryBase factory, IChannel innerChannel, EndpointAddress remoteAddress, Uri via, InitiatorMessageSecurityBindingSupport security)
: base (factory, remoteAddress, via)
{
this.channel = innerChannel;
session = new SecurityDuplexSession (this);
InitializeSecurityFunctionality (security);
}
public IMessage GetReport(IChannel channel)
{
log.Debug("public Message getReport(Session session): called");
// Generate a dummy report, the coordinator expects a report but doesn't care what it is.
return channel.CreateTextMessage("Dummy Run, Ok.");
}
public void Run(ModuleInfo info, CancellationToken token = default(CancellationToken))
{
double a = 0;
double b = Math.PI/2;
double h = 0.00000001;
const int pointsNum = 2;
var points = new IPoint[pointsNum];
var channels = new IChannel[pointsNum];
for (int i = 0; i < pointsNum; ++i)
{
points[i] = info.CreatePoint();
channels[i] = points[i].CreateChannel();
points[i].ExecuteClass("FirstModule.IntegralModule");
}
double y = a;
for (int i = 0; i < pointsNum; ++i)
{
channels[i].WriteData(y);
channels[i].WriteData(y + (b - a) / pointsNum);
channels[i].WriteData(h);
y += (b - a) / pointsNum;
}
DateTime time = DateTime.Now;
Console.WriteLine("Waiting for result...");
double res = 0;
for (int i = pointsNum - 1; i >= 0; --i)
{
res += channels[i].ReadData(typeof(double));
}
Console.WriteLine("Result found: res = {0}, time = {1}", res, Math.Round((DateTime.Now - time).TotalSeconds,3));
}
public virtual void Init()
{
_logger.Info("public virtual void Init(): called");
// Create a connection to the broker.
IConnectionInfo connectionInfo = QpidConnectionInfo.FromUrl(DEFAULT_URI);
_connection = new AMQConnection(connectionInfo);
_logger.Info("Starting...");
// Register this to listen for exceptions on the test connection.
_exceptionDelegate = new ExceptionListenerDelegate(OnException);
_connection.ExceptionListener += _exceptionDelegate;
// Establish a session on the broker.
_channel = _connection.CreateChannel(false, AcknowledgeMode.AutoAcknowledge, 1);
// Create a durable, non-temporary, non-exclusive queue.
_queueName = _channel.GenerateUniqueName();
_channel.DeclareQueue(_queueName, true, false, false);
_channel.Bind(_queueName, ExchangeNameDefaults.TOPIC, _routingKey);
// Clear the most recent message and exception.
_lastException = null;
}
public TrainState(IChannel ch, int numFeatures, LinearModelParameters predictor, AveragedPerceptronTrainer parent)
: base(ch, numFeatures, predictor, parent)
{
}
public BgDaisyIslFiscalPrinter(IChannel channel, IDictionary <string, string>?options = null)
: base(channel, options)
{
}
public virtual IEnumerable <SuggestedRecipe> Apply(
TransformInference.InferenceResult transformInferenceResult, Type predictorType, IChannel ch)
{
TransformInference.SuggestedTransform[] transforms = GetSuggestedTransforms(
transformInferenceResult, predictorType);
if (transforms?.Length > 0)
{
foreach (var recipe in ApplyCore(predictorType, transforms))
{
yield return(recipe);
}
}
}
public ExecutionException(object request, IChannel channel, Exception cause) : base(channel, cause)
{
Request = request;
}
public ExecutionException(object request, IChannel channel, string msg) : base(channel, msg)
{
Request = request;
}
public ExecutionException(object request, IChannel channel, string message, Exception cause) : base(channel, message, cause)
{
Request = request;
}
void IAppSession.Initialize(IServerInfo server, IChannel channel)
{
Server = server;
StartTime = DateTimeOffset.Now;
_channel = channel;
}
private protected override void CheckAndUpdateParametersBeforeTraining(IChannel ch, RoleMappedData data, float[] labels, int[] groups) => GbmOptions["objective"] = "binary";
public InputDataManager(SymSgdClassificationTrainer trainer, FloatLabelCursor.Factory cursorFactory, IChannel ch)
{
_instIndices = new ArrayManager <int>(trainer, ch);
_instValues = new ArrayManager <float>(trainer, ch);
_instanceProperties = new List <InstanceProperties>();
_cursorFactory = cursorFactory;
_ch = ch;
_cursor = cursorFactory.Create();
_cursorMoveNext = _cursor.MoveNext();
_isFullyLoaded = true;
_instanceIndex = 0;
_trainer = trainer;
}
private protected override TrainStateBase MakeState(IChannel ch, int numFeatures, LinearModelParameters predictor)
{
return(new TrainState(ch, numFeatures, predictor, this));
}
private static StatAggregator CreateStatAggregator(IChannel ch, ColumnType type, ReplacementKind?kind, bool bySlot, IRowCursor cursor, int col)
{
ch.Assert(type.ItemType.IsNumber);
if (!type.IsVector)
{
// The type is a scalar.
if (kind == ReplacementKind.Mean)
{
switch (type.RawKind)
{
case DataKind.R4:
return(new R4.MeanAggregatorOne(ch, cursor, col));
case DataKind.R8:
return(new R8.MeanAggregatorOne(ch, cursor, col));
default:
break;
}
}
if (kind == ReplacementKind.Min || kind == ReplacementKind.Max)
{
switch (type.RawKind)
{
case DataKind.R4:
return(new R4.MinMaxAggregatorOne(ch, cursor, col, kind == ReplacementKind.Max));
case DataKind.R8:
return(new R8.MinMaxAggregatorOne(ch, cursor, col, kind == ReplacementKind.Max));
default:
break;
}
}
}
else if (bySlot)
{
// Imputation by slot.
// REVIEW: It may be more appropriate to have a warning here instead.
if (type.ValueCount == 0)
{
throw ch.Except("Imputation by slot is not allowed for vectors of unknown size.");
}
if (kind == ReplacementKind.Mean)
{
switch (type.ItemType.RawKind)
{
case DataKind.R4:
return(new R4.MeanAggregatorBySlot(ch, type, cursor, col));
case DataKind.R8:
return(new R8.MeanAggregatorBySlot(ch, type, cursor, col));
default:
break;
}
}
else if (kind == ReplacementKind.Min || kind == ReplacementKind.Max)
{
switch (type.ItemType.RawKind)
{
case DataKind.R4:
return(new R4.MinMaxAggregatorBySlot(ch, type, cursor, col, kind == ReplacementKind.Max));
case DataKind.R8:
return(new R8.MinMaxAggregatorBySlot(ch, type, cursor, col, kind == ReplacementKind.Max));
default:
break;
}
}
}
else
{
// Imputation across slots.
if (kind == ReplacementKind.Mean)
{
switch (type.ItemType.RawKind)
{
case DataKind.R4:
return(new R4.MeanAggregatorAcrossSlots(ch, cursor, col));
case DataKind.R8:
return(new R8.MeanAggregatorAcrossSlots(ch, cursor, col));
default:
break;
}
}
else if (kind == ReplacementKind.Min || kind == ReplacementKind.Max)
{
switch (type.ItemType.RawKind)
{
case DataKind.R4:
return(new R4.MinMaxAggregatorAcrossSlots(ch, cursor, col, kind == ReplacementKind.Max));
case DataKind.R8:
return(new R8.MinMaxAggregatorAcrossSlots(ch, cursor, col, kind == ReplacementKind.Max));
default:
break;
}
}
}
ch.Assert(false);
throw ch.Except("Internal error, unrecognized imputation method ReplacementKind '{0}' or unrecognized type '{1}' " +
"assigned in NAReplaceTransform.", kind, type);
}
private TPredictor TrainCore(IChannel ch, RoleMappedData data, LinearPredictor predictor, int weightSetCount)
{
int numFeatures = data.Schema.Feature.Type.VectorSize;
var cursorFactory = new FloatLabelCursor.Factory(data, CursOpt.Label | CursOpt.Features | CursOpt.Weight);
int numThreads = 1;
ch.CheckUserArg(numThreads > 0, nameof(_args.NumberOfThreads),
"The number of threads must be either null or a positive integer.");
var positiveInstanceWeight = _args.PositiveInstanceWeight;
VBuffer <float> weights = default;
float bias = 0.0f;
if (predictor != null)
{
predictor.GetFeatureWeights(ref weights);
VBufferUtils.Densify(ref weights);
bias = predictor.Bias;
}
else
{
weights = VBufferUtils.CreateDense <float>(numFeatures);
}
// Reference: Parasail. SymSGD.
bool tuneLR = _args.LearningRate == null;
var lr = _args.LearningRate ?? 1.0f;
bool tuneNumLocIter = (_args.UpdateFrequency == null);
var numLocIter = _args.UpdateFrequency ?? 1;
var l2Const = _args.L2Regularization;
var piw = _args.PositiveInstanceWeight;
// This is state of the learner that is shared with the native code.
State state = new State();
GCHandle stateGCHandle = default;
try
{
stateGCHandle = GCHandle.Alloc(state, GCHandleType.Pinned);
state.TotalInstancesProcessed = 0;
using (InputDataManager inputDataManager = new InputDataManager(this, cursorFactory, ch))
{
bool shouldInitialize = true;
using (var pch = Host.StartProgressChannel("Preprocessing"))
inputDataManager.LoadAsMuchAsPossible();
int iter = 0;
if (inputDataManager.IsFullyLoaded)
{
ch.Info("Data fully loaded into memory.");
}
using (var pch = Host.StartProgressChannel("Training"))
{
if (inputDataManager.IsFullyLoaded)
{
pch.SetHeader(new ProgressHeader(new[] { "iterations" }),
entry => entry.SetProgress(0, state.PassIteration, _args.NumberOfIterations));
// If fully loaded, call the SymSGDNative and do not come back until learned for all iterations.
Native.LearnAll(inputDataManager, tuneLR, ref lr, l2Const, piw, weights.Values, ref bias, numFeatures,
_args.NumberOfIterations, numThreads, tuneNumLocIter, ref numLocIter, _args.Tolerance, _args.Shuffle, shouldInitialize, stateGCHandle);
shouldInitialize = false;
}
else
{
pch.SetHeader(new ProgressHeader(new[] { "iterations" }),
entry => entry.SetProgress(0, iter, _args.NumberOfIterations));
// Since we loaded data in batch sizes, multiple passes over the loaded data is feasible.
int numPassesForABatch = inputDataManager.Count / 10000;
while (iter < _args.NumberOfIterations)
{
// We want to train on the final passes thoroughly (without learning on the same batch multiple times)
// This is for fine tuning the AUC. Experimentally, we found that 1 or 2 passes is enough
int numFinalPassesToTrainThoroughly = 2;
// We also do not want to learn for more passes than what the user asked
int numPassesForThisBatch = Math.Min(numPassesForABatch, _args.NumberOfIterations - iter - numFinalPassesToTrainThoroughly);
// If all of this leaves us with 0 passes, then set numPassesForThisBatch to 1
numPassesForThisBatch = Math.Max(1, numPassesForThisBatch);
state.PassIteration = iter;
Native.LearnAll(inputDataManager, tuneLR, ref lr, l2Const, piw, weights.Values, ref bias, numFeatures,
numPassesForThisBatch, numThreads, tuneNumLocIter, ref numLocIter, _args.Tolerance, _args.Shuffle, shouldInitialize, stateGCHandle);
shouldInitialize = false;
// Check if we are done with going through the data
if (inputDataManager.FinishedTheLoad)
{
iter += numPassesForThisBatch;
// Check if more passes are left
if (iter < _args.NumberOfIterations)
{
inputDataManager.RestartLoading(_args.Shuffle, Host);
}
}
// If more passes are left, load as much as possible
if (iter < _args.NumberOfIterations)
{
inputDataManager.LoadAsMuchAsPossible();
}
}
}
// Maps back the dense features that are mislocated
if (numThreads > 1)
{
Native.MapBackWeightVector(weights.Values, stateGCHandle);
}
Native.DeallocateSequentially(stateGCHandle);
}
}
}
finally
{
if (stateGCHandle.IsAllocated)
{
stateGCHandle.Free();
}
}
return(CreatePredictor(weights, bias));
}
internal abstract InternalRegressionTree TrainingIteration(IChannel ch, bool[] activeFeatures);
protected StatAggregatorAcrossSlots(IChannel ch, IRowCursor cursor, int col)
: base(ch, cursor, col)
{
}
private void HandlePingReq(IChannel channel)
{
channel.WriteAndFlushAsync(PingRespPacket.Instance);
}
protected StatAggregator(IChannel ch)
{
Contracts.AssertValue(ch);
Ch = ch;
}
/// <inheritdoc /> public async Task <IChannelMessageAck> WriteChatMessageAsync(IChannel channel, string content) => await WriteChatMessageAsync(channel.Id, content);
private static async Task TestStringEcho0(ServerBootstrap sb, Bootstrap cb, bool autoRead, ITestOutputHelper output)
{
sb.ChildOption(ChannelOption.AutoRead, autoRead);
cb.Option(ChannelOption.AutoRead, autoRead);
IPromise serverDonePromise = new TaskCompletionSource();
IPromise clientDonePromise = new TaskCompletionSource();
StringEchoHandler sh = new StringEchoHandler(autoRead, serverDonePromise, output);
StringEchoHandler ch = new StringEchoHandler(autoRead, clientDonePromise, output);
sb.ChildHandler(new ActionChannelInitializer <IChannel>(sch =>
{
sch.Pipeline.AddLast("framer", new DelimiterBasedFrameDecoder(512, Delimiters.LineDelimiter()));
sch.Pipeline.AddLast("decoder", new StringDecoder(Encoding.ASCII));
sch.Pipeline.AddBefore("decoder", "encoder", new StringEncoder(Encoding.ASCII));
sch.Pipeline.AddAfter("decoder", "handler", sh);
}));
cb.Handler(new ActionChannelInitializer <IChannel>(sch =>
{
sch.Pipeline.AddLast("framer", new DelimiterBasedFrameDecoder(512, Delimiters.LineDelimiter()));
sch.Pipeline.AddLast("decoder", new StringDecoder(Encoding.ASCII));
sch.Pipeline.AddBefore("decoder", "encoder", new StringEncoder(Encoding.ASCII));
sch.Pipeline.AddAfter("decoder", "handler", ch);
}));
IChannel sc = await sb.BindAsync();
IChannel cc = await cb.ConnectAsync(sc.LocalAddress);
for (int i = 0; i < s_data.Length; i++)
{
string element = s_data[i];
string delimiter = s_random.Next(0, 1) == 1 ? "\r\n" : "\n";
await cc.WriteAndFlushAsync(element + delimiter);
}
await ch._donePromise.Task;
await sh._donePromise.Task;
await sh._channel.CloseAsync();
await ch._channel.CloseAsync();
await sc.CloseAsync();
if (sh._exception.Value != null && !(sh._exception.Value is SocketException || (sh._exception.Value is ChannelException chexc && chexc.InnerException is OperationException) || sh._exception.Value is OperationException))
{
throw sh._exception.Value;
}
if (ch._exception.Value != null && !(ch._exception.Value is SocketException || (sh._exception.Value is ChannelException chexc1 && chexc1.InnerException is OperationException) || sh._exception.Value is OperationException))
{
throw ch._exception.Value;
}
if (sh._exception.Value != null)
{
throw sh._exception.Value;
}
if (ch._exception.Value != null)
{
throw ch._exception.Value;
}
}
/// <inheritdoc /> public async Task <IChannelMessageAck> RemoveChatMessageAsync(IChannel channel, string messageId) => await RemoveChatMessageAsync(channel.Id, messageId);
private void SendPingReq(IChannel channel)
{
channel.WriteAndFlushAsync(PingReqPacket.Instance);
}
protected override void PrintFoldResultsCore(IChannel ch, Dictionary <string, IDataView> metrics)
{
IDataView top;
if (!metrics.TryGetValue(AnomalyDetectionEvaluator.TopKResults, out top))
{
throw Host.Except("Did not find the top-k results data view");
}
var sb = new StringBuilder();
using (var cursor = top.GetRowCursor(col => true))
{
int index;
if (!top.Schema.TryGetColumnIndex(AnomalyDetectionEvaluator.TopKResultsColumns.Instance, out index))
{
throw Host.Except("Data view does not contain the 'Instance' column");
}
var instanceGetter = cursor.GetGetter <ReadOnlyMemory <char> >(index);
if (!top.Schema.TryGetColumnIndex(AnomalyDetectionEvaluator.TopKResultsColumns.AnomalyScore, out index))
{
throw Host.Except("Data view does not contain the 'Anomaly Score' column");
}
var scoreGetter = cursor.GetGetter <Single>(index);
if (!top.Schema.TryGetColumnIndex(AnomalyDetectionEvaluator.TopKResultsColumns.Label, out index))
{
throw Host.Except("Data view does not contain the 'Label' column");
}
var labelGetter = cursor.GetGetter <Single>(index);
bool hasRows = false;
while (cursor.MoveNext())
{
if (!hasRows)
{
sb.AppendFormat("{0} Top-scored Results", _topScored);
sb.AppendLine();
sb.AppendLine("=================================================");
sb.AppendLine("Instance Anomaly Score Labeled");
hasRows = true;
}
var name = default(ReadOnlyMemory <char>);
Single score = 0;
Single label = 0;
instanceGetter(ref name);
scoreGetter(ref score);
labelGetter(ref label);
sb.AppendFormat("{0,-10}{1,12:G4}{2,12}", name, score, label);
sb.AppendLine();
}
}
if (sb.Length > 0)
{
ch.Info(MessageSensitivity.UserData, sb.ToString());
}
IDataView overall;
if (!metrics.TryGetValue(MetricKinds.OverallMetrics, out overall))
{
throw Host.Except("No overall metrics found");
}
// Find the number of anomalies, and the thresholds.
int numAnomIndex;
if (!overall.Schema.TryGetColumnIndex(AnomalyDetectionEvaluator.OverallMetrics.NumAnomalies, out numAnomIndex))
{
throw Host.Except("Could not find the 'NumAnomalies' column");
}
int stratCol;
var hasStrat = overall.Schema.TryGetColumnIndex(MetricKinds.ColumnNames.StratCol, out stratCol);
int stratVal;
bool hasStratVals = overall.Schema.TryGetColumnIndex(MetricKinds.ColumnNames.StratVal, out stratVal);
Contracts.Assert(hasStrat == hasStratVals);
long numAnomalies = 0;
using (var cursor = overall.GetRowCursor(col => col == numAnomIndex ||
(hasStrat && col == stratCol)))
{
var numAnomGetter = cursor.GetGetter <long>(numAnomIndex);
ValueGetter <uint> stratGetter = null;
if (hasStrat)
{
var type = cursor.Schema.GetColumnType(stratCol);
stratGetter = RowCursorUtils.GetGetterAs <uint>(type, cursor, stratCol);
}
bool foundRow = false;
while (cursor.MoveNext())
{
uint strat = 0;
if (stratGetter != null)
{
stratGetter(ref strat);
}
if (strat > 0)
{
continue;
}
if (foundRow)
{
throw Host.Except("Found multiple non-stratified rows in overall results data view");
}
foundRow = true;
numAnomGetter(ref numAnomalies);
}
}
var kFormatName = string.Format(FoldDrAtKFormat, _k);
var pFormatName = string.Format(FoldDrAtPFormat, _p);
var numAnomName = string.Format(FoldDrAtNumAnomaliesFormat, numAnomalies);
(string Source, string Name)[] cols =
UpdateChatMessageAsync(IChannel channel, string messageId, string content) => await UpdateChatMessageAsync(channel.Id, messageId, content);
private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Factory cursorFactory, int featureCount)
{
Host.AssertValue(ch);
ch.AssertValue(cursorFactory);
int m = featureCount + 1;
// Check for memory conditions first.
if ((long)m * (m + 1) / 2 > int.MaxValue)
{
throw ch.Except("Cannot hold covariance matrix in memory with {0} features", m - 1);
}
// Track the number of examples.
long n = 0;
// Since we are accumulating over many values, we use Double even for the single precision build.
var xty = new Double[m];
// The layout of this algorithm is a packed row-major lower triangular matrix.
var xtx = new Double[m * (m + 1) / 2];
// Build X'X (lower triangular) and X'y incrementally (X'X+=X'X_i; X'y+=X'y_i):
using (var cursor = cursorFactory.Create())
{
while (cursor.MoveNext())
{
var yi = cursor.Label;
// Increment first element of X'y
xty[0] += yi;
// Increment first element of lower triangular X'X
xtx[0] += 1;
var values = cursor.Features.Values;
if (cursor.Features.IsDense)
{
int ioff = 1;
ch.Assert(cursor.Features.Count + 1 == m);
// Increment rest of first column of lower triangular X'X
for (int i = 1; i < m; i++)
{
ch.Assert(ioff == i * (i + 1) / 2);
var val = values[i - 1];
// Add the implicit first bias term to X'X
xtx[ioff++] += val;
// Add the remainder of X'X
for (int j = 0; j < i; j++)
{
xtx[ioff++] += val * values[j];
}
// X'y
xty[i] += val * yi;
}
ch.Assert(ioff == xtx.Length);
}
else
{
var fIndices = cursor.Features.Indices;
for (int ii = 0; ii < cursor.Features.Count; ++ii)
{
int i = fIndices[ii] + 1;
int ioff = i * (i + 1) / 2;
var val = values[ii];
// Add the implicit first bias term to X'X
xtx[ioff++] += val;
// Add the remainder of X'X
for (int jj = 0; jj <= ii; jj++)
{
xtx[ioff + fIndices[jj]] += val * values[jj];
}
// X'y
xty[i] += val * yi;
}
}
n++;
}
ch.Check(n > 0, "No training examples in dataset.");
if (cursor.BadFeaturesRowCount > 0)
{
ch.Warning("Skipped {0} instances with missing features/label during training", cursor.SkippedRowCount);
}
if (_l2Weight > 0)
{
// Skip the bias term for regularization, in the ridge regression case.
// So start at [1,1] instead of [0,0].
// REVIEW: There are two ways to view this, firstly, it is more
// user friendly ot make this scaling factor behave similarly regardless
// of data size, so that if you have the same parameters, you get the same
// model if you feed in your data than if you duplicate your data 10 times.
// This is what I have now. The alternate point of view is to view this
// L2 regularization parameter as providing some sort of prior, in which
// case duplication 10 times should in fact be treated differently! (That
// is, we should not multiply by n below.) Both interpretations seem
// correct, in their way.
Double squared = _l2Weight * _l2Weight * n;
int ioff = 0;
for (int i = 1; i < m; ++i)
{
xtx[ioff += i + 1] += squared;
}
ch.Assert(ioff == xtx.Length - 1);
}
}
if (!(_l2Weight > 0) && n < m)
{
throw ch.Except("Ordinary least squares requires more examples than parameters. There are {0} parameters, but {1} examples. To enable training, use a positive L2 weight so this behaves as ridge regression.", m, n);
}
Double yMean = n == 0 ? 0 : xty[0] / n;
ch.Info("Trainer solving for {0} parameters across {1} examples", m, n);
// Cholesky Decomposition of X'X into LL'
try
{
Mkl.Pptrf(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, m, xtx);
}
catch (DllNotFoundException)
{
// REVIEW: Is there no better way?
throw ch.ExceptNotSupp("The MKL library (libMklImports) or one of its dependencies is missing.");
}
// Solve for beta in (LL')beta = X'y:
Mkl.Pptrs(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, m, 1, xtx, xty, 1);
// Note that the solver overwrote xty so it contains the solution. To be more clear,
// we effectively change its name (through reassignment) so we don't get confused that
// this is somehow xty in the remaining calculation.
var beta = xty;
xty = null;
// Check that the solution is valid.
for (int i = 0; i < beta.Length; ++i)
{
ch.Check(FloatUtils.IsFinite(beta[i]), "Non-finite values detected in OLS solution");
}
var weights = VBufferUtils.CreateDense <float>(beta.Length - 1);
for (int i = 1; i < beta.Length; ++i)
{
weights.Values[i - 1] = (float)beta[i];
}
var bias = (float)beta[0];
if (!(_l2Weight > 0) && m == n)
{
// We would expect the solution to the problem to be exact in this case.
ch.Info("Number of examples equals number of parameters, solution is exact but no statistics can be derived");
return(new OlsLinearRegressionPredictor(Host, ref weights, bias, null, null, null, 1, float.NaN));
}
Double rss = 0; // residual sum of squares
Double tss = 0; // total sum of squares
using (var cursor = cursorFactory.Create())
{
var lrPredictor = new LinearRegressionPredictor(Host, ref weights, bias);
var lrMap = lrPredictor.GetMapper <VBuffer <float>, float>();
float yh = default;
while (cursor.MoveNext())
{
var features = cursor.Features;
lrMap(ref features, ref yh);
var e = cursor.Label - yh;
rss += e * e;
var ydm = cursor.Label - yMean;
tss += ydm * ydm;
}
}
var rSquared = ProbClamp(1 - (rss / tss));
// R^2 adjusted differs from the normal formula on account of the bias term, by Said's reckoning.
double rSquaredAdjusted;
if (n > m)
{
rSquaredAdjusted = ProbClamp(1 - (1 - rSquared) * (n - 1) / (n - m));
ch.Info("Coefficient of determination R2 = {0:g}, or {1:g} (adjusted)",
rSquared, rSquaredAdjusted);
}
else
{
rSquaredAdjusted = Double.NaN;
}
// The per parameter significance is compute intensive and may not be required for all practitioners.
// Also we can't estimate it, unless we can estimate the variance, which requires more examples than
// parameters.
if (!_perParameterSignificance || m >= n)
{
return(new OlsLinearRegressionPredictor(Host, ref weights, bias, null, null, null, rSquared, rSquaredAdjusted));
}
ch.Assert(!Double.IsNaN(rSquaredAdjusted));
var standardErrors = new Double[m];
var tValues = new Double[m];
var pValues = new Double[m];
// Invert X'X:
Mkl.Pptri(Mkl.Layout.RowMajor, Mkl.UpLo.Lo, m, xtx);
var s2 = rss / (n - m); // estimate of variance of y
for (int i = 0; i < m; i++)
{
// Initialize with inverse Hessian.
standardErrors[i] = (Single)xtx[i * (i + 1) / 2 + i];
}
if (_l2Weight > 0)
{
// Iterate through all entries of inverse Hessian to make adjustment to variance.
int ioffset = 1;
float reg = _l2Weight * _l2Weight * n;
for (int iRow = 1; iRow < m; iRow++)
{
for (int iCol = 0; iCol <= iRow; iCol++)
{
var entry = (Single)xtx[ioffset];
var adjustment = -reg * entry * entry;
standardErrors[iRow] -= adjustment;
if (0 < iCol && iCol < iRow)
{
standardErrors[iCol] -= adjustment;
}
ioffset++;
}
}
Contracts.Assert(ioffset == xtx.Length);
}
for (int i = 0; i < m; i++)
{
// sqrt of diagonal entries of s2 * inverse(X'X + reg * I) * X'X * inverse(X'X + reg * I).
standardErrors[i] = Math.Sqrt(s2 * standardErrors[i]);
ch.Check(FloatUtils.IsFinite(standardErrors[i]), "Non-finite standard error detected from OLS solution");
tValues[i] = beta[i] / standardErrors[i];
pValues[i] = (float)MathUtils.TStatisticToPValue(tValues[i], n - m);
ch.Check(0 <= pValues[i] && pValues[i] <= 1, "p-Value calculated outside expected [0,1] range");
}
return(new OlsLinearRegressionPredictor(Host, ref weights, bias, standardErrors, tValues, pValues, rSquared, rSquaredAdjusted));
}
/// <inheritdoc /> public async Task LeaveChatAsync(IChannel channel) => await LeaveChatAsync(channel.Id);
public ChatSession(uint hostId, IChannel channel, ProudServer server)
: base(hostId, channel, server)
{
}
/// <summary>Create a new instance</summary>
/// <param name="parent">
/// the <see cref="IChannel" /> which created this instance or <c>null</c> if it was created by the
/// user
/// </param>
/// <param name="socket">the <see cref="ISocketChannel" /> which will be used</param>
public TcpSocketChannel(IChannel parent, Socket socket)
: this(parent, socket, false)
{
}
public UpdatableDisplayFallback(IChannel channel)
{
this.channel = channel;
}
public async Task <IServiceChannelDispatcher> CreateServiceChannelDispatcherAsync(IChannel channel)
{
var sessionIdleManager = channel.GetProperty <ServiceChannel.SessionIdleManager>();
IChannelBinder binder = null;
if (channel is IReplyChannel)
{
var rcbinder = channel.GetProperty <ReplyChannelBinder>();
rcbinder.Init(channel as IReplyChannel, BaseAddress);
binder = rcbinder;
}
else if (channel is IDuplexSessionChannel)
{
var dcbinder = channel.GetProperty <DuplexChannelBinder>();
dcbinder.Init(channel as IDuplexSessionChannel, _requestReplyCorrelator, BaseAddress);
binder = dcbinder;
}
else if (channel is IInputChannel)
{
var icbinder = channel.GetProperty <InputChannelBinder>();
icbinder.Init(channel as IInputChannel, BaseAddress);
binder = icbinder;
}
// TODO: Wire up wasChannelThrottled
var channelHandler = new ChannelHandler(Binding.MessageVersion, binder, channel.GetProperty <ServiceThrottle>(),
this, /*wasChannelThrottled*/ false, sessionIdleManager);
var channelDispatcher = channelHandler.GetDispatcher();
// channel.ChannelDispatcher = channelDispatcher;
await channelHandler.OpenAsync();
return(channelDispatcher);
}
private static List <KeyToVectorMappingTransformer.ColumnInfo> ConvertFeatures(ColumnInfo[] feats, HashSet <string> featNames, List <KeyValuePair <string, string> > concatNames, IChannel ch,
out List <TypeConvertingTransformer.ColumnInfo> cvt, out int errCount)
{
Contracts.AssertValue(feats);
Contracts.AssertValue(featNames);
Contracts.AssertValue(concatNames);
Contracts.AssertValue(ch);
List <KeyToVectorMappingTransformer.ColumnInfo> ktv = null;
cvt = null;
errCount = 0;
foreach (var col in feats)
{
// Skip duplicates.
if (!featNames.Add(col.Name))
{
continue;
}
if (!col.Type.IsVector || col.Type.VectorSize > 0)
{
var type = col.Type.ItemType;
if (type.IsKey)
{
if (type.KeyCount > 0)
{
var colName = GetUniqueName();
concatNames.Add(new KeyValuePair <string, string>(col.Name, colName));
Utils.Add(ref ktv, new KeyToVectorMappingTransformer.ColumnInfo(col.Name, colName));
continue;
}
}
if (type.IsNumber || type.IsBool)
{
// Even if the column is R4 in training, we still want to add it to the conversion.
// The reason is that at scoring time, the column might have a slightly different type (R8 for example).
// This happens when the training is done on an XDF and the scoring is done on a data frame.
var colName = GetUniqueName();
concatNames.Add(new KeyValuePair <string, string>(col.Name, colName));
Utils.Add(ref cvt, new TypeConvertingTransformer.ColumnInfo(col.Name, colName, DataKind.R4));
continue;
}
}
ch.Error("The type of column '{0}' is not valid as a training feature: {1}", col.Name, col.Type);
errCount++;
}
return(ktv);
}
public ProudSession Create(uint hostId, IChannel channel, ProudServer server)
{
return(new ChatSession(hostId, channel, server));
}
