private void CallStack() { while (true) { if (m_dataQueue.Count > 0) { IDataBuffer <object> dataBuffer = null; if (m_dataQueue.TryDequeue(out dataBuffer)) { if (dataBuffer != null) { Callback(this, new LogMessageEventArgs(dataBuffer)); } } } if (m_dataQueue.IsEmpty) { Thread.Sleep(100); } else { Thread.Sleep(10); } } }
private void NewDataBuffer() { lock (m_newDataBufferLockObj) { m_dataBuffer = new DataBuffer(); } }
private void BatchSendMessage(object dataBuffer) { IDataBuffer buffer = dataBuffer as IDataBuffer; if (buffer != null && buffer.Count > 0) { List <PushMessage> list = buffer.GetList <PushMessage>(); try { Process.Info("消息通知批量发送接口", "BatchSendMessage", "", string.Format("批次【{0}】,数量【{1}】", ++callCount, list.Count), ""); JinRi.Notify.ServiceAgent.SenderSendServiceSOA.SendServiceClient client = new JinRi.Notify.ServiceAgent.SenderSendServiceSOA.SendServiceClient(); PushMessageResult result = client.ReceiveList(list.ToArray()); Process.Info("消息通知批量发送接口_返回", "BatchSendMessage", "", result.Success + "", ""); } catch (Exception ex) { //如果推送异常,去掉标示的推送缓存 foreach (PushMessage message in list) { string cacheKey = string.Format(CacheKeys.PushMessageToSender_Arg2, message.PushId, message.PushCount); DistributedCache.Delete(cacheKey); } Process.Error("消息通知批量发送接口", "BatchSendMessage", "", "异步发送推送消息到消息通知接口出现异常:" + ex.GetString(), "Error"); } } }
/// <inheritdoc /> public override void Fill(INDArray filler, INDArray arrayToFill) { IDataBuffer <float> arrayToFillData = InternaliseArray(arrayToFill).Data; IDataBuffer <float> fillerData = InternaliseArray(filler).Data; arrayToFillData.SetValues(fillerData.Data, fillerData.Offset, 0, Math.Min(arrayToFill.Length, filler.Length)); }
private void CallStack() { while (true) { IDataBuffer dataBuffer = null; lock (m_dataStackLockObj) { if (m_dataStack.Count > 0) { dataBuffer = m_dataStack.Pop(); Interlocked.Decrement(ref _count); } } if (dataBuffer != null) { Callback(dataBuffer); } if (1 == Interlocked.CompareExchange(ref _isFlush, 0, 0)) { if (0 == Interlocked.CompareExchange(ref _count, 0, 0)) { _hasAllPoped = 1; break; } } Thread.Sleep(10); } }
/// <inheritdoc /> public override void Fill(INDArray filler, INDArray arrayToFill, long[] sourceBeginIndices, long[] sourceEndIndices, long[] destinationBeginIndices, long[] destinationEndIndices) { IDataBuffer <float> fillerData = InternaliseArray(filler).Data; IDataBuffer <float> arrayToFillData = InternaliseArray(arrayToFill).Data; int sourceOffset = (int)NDArrayUtils.GetFlatIndex(filler.Shape, filler.Strides, sourceBeginIndices); int sourceLength = (int)NDArrayUtils.GetFlatIndex(filler.Shape, filler.Strides, sourceEndIndices) - sourceOffset + 1; // +1 because end is inclusive int destinationOffset = (int)NDArrayUtils.GetFlatIndex(arrayToFill.Shape, arrayToFill.Strides, destinationBeginIndices); int destinationLength = (int)NDArrayUtils.GetFlatIndex(arrayToFill.Shape, arrayToFill.Strides, destinationEndIndices) - destinationOffset + 1; // same here if (sourceLength < 0) { throw new ArgumentOutOfRangeException($"Source begin indices must be smaller than source end indices, but source length was {sourceLength}."); } if (destinationLength < 0) { throw new ArgumentOutOfRangeException($"Destination begin indices must be smaller than destination end indices, but destination length was {destinationLength}."); } if (sourceLength != destinationLength) { throw new ArgumentException($"Source and destination indices length must batch, but source length was {sourceLength} and destination legnth was {destinationLength}."); } Array.Copy(fillerData.Data, sourceOffset, arrayToFillData.Data, destinationOffset, sourceLength); }
/// <summary> /// 初始化程序(设置ServiceLocator,读取IDataBuffer) /// </summary> public static void InitProgram() { if (!m_init) { #if DEBUG Console.WriteLine("Please press any key to continue..."); Console.ReadLine(); #endif m_init = true; var p = DefaultServiceProvider.Instance; p.EnableLog(); p.EnableNHibernate(); p.EnableCache(); p.EnableScript(false); Microsoft.Practices.ServiceLocation.ServiceLocator.SetLocatorProvider(new Microsoft.Practices.ServiceLocation.ServiceLocatorProvider( delegate() { return(p); })); IDataBuffer db = ServiceProvider.GetService <IDataBuffer>(); if (db != null) { db.LoadData(); } } }
/// <summary> /// 初始化程序(设置ServiceLocator,读取IDataBuffer) /// </summary> public static void InitProgram() { if (!m_init) { m_init = true; Feng.Utils.XceedUtility.SetXceedLicense(); log4net.Config.XmlConfigurator.Configure(); //HibernatingRhinos.NHibernate.Profiler.Appender.NHibernateProfiler.Initialize(); //if (Microsoft.Practices.ServiceLocation.ServiceLocator.Current == null) // throw null { if (!SystemConfiguration.LiteMode) { SetSpringServiceLocator(); } ServiceProvider.SetDefaultService <ILogger>(new log4netLogger()); ServiceProvider.SetDefaultService <IExceptionProcess>(new LoggerExceptionProcess()); ServiceProvider.SetDefaultService <Feng.NH.ISessionFactoryManager>(new Feng.NH.NHibernateSessionFactoryManager()); ServiceProvider.SetDefaultService <IRepositoryFactory>(new NH.RepositoryFactory()); IPersistentCache c = new PersistentHashtableCache(); ServiceProvider.SetDefaultService <ICache>(c); ServiceProvider.SetDefaultService <IPersistentCache>(c); Feng.DBDef def = new Feng.DBDef(); ServiceProvider.SetDefaultService <IDefinition>(def); IDataBuffer buf = new Feng.DBDataBuffer(); ServiceProvider.SetDefaultService <IDataBuffer>(buf); IDataBuffers bufs = new DataBuffers(); bufs.AddDataBuffer(new Cache()); bufs.AddDataBuffer(buf); bufs.AddDataBuffer(def); ServiceProvider.SetDefaultService <IDataBuffers>(bufs); IEntityScript script = new PythonandEvalutionEngineScript(); ServiceProvider.SetDefaultService <IScript>(script); ServiceProvider.SetDefaultService <IEntityScript>(script); ServiceProvider.SetDefaultService <IMessageBox>(new EmptyMessageBox()); ServiceProvider.SetDefaultService <IControlCollectionFactory>(new Feng.ControlCollectionFactoryBase()); ServiceProvider.SetDefaultService <IManagerFactory>(new Feng.Utils.ManagerFactory()); IDataBuffer db = ServiceProvider.GetService <IDataBuffer>(); if (db != null) { db.LoadData(); } } } }
private void LogProcessBatchMessage(object obj) { IDataBuffer buffer = obj as IDataBuffer; if (buffer != null && buffer.Count > 0) { LogBatchMessage(buffer.GetEnumerator(), false); } }
private void LogHandleBatchMessage(object obj) { IDataBuffer buffer = obj as IDataBuffer; if (buffer != null && buffer.Count > 0) { LogBatchMessage(buffer.GetEnumerator(), true); } }
private void BatchSavePushMessage(object dataBuffer) { IDataBuffer buffer = dataBuffer as IDataBuffer; if (buffer != null && buffer.Count > 0) { DirectSavePushMessage(buffer.GetList <PushMessageModel>()); } }
private void BatchSaveNotifyMessage(object dataBuffer) { IDataBuffer buffer = dataBuffer as IDataBuffer; if (buffer != null && buffer.Count > 0) { DirectSaveNotifyMessage(buffer.GetList <NotifyMessage>()); } }
private void BatchLowPublisher(object dataBuffer) { IDataBuffer buffer = dataBuffer as IDataBuffer; if (buffer != null && buffer.Count > 0) { Process.Debug("批量发布消息到队列", "BatchLowPublisher", string.Format("条数:【{0}】", buffer.Count), ""); m_rabbitMQBus.Publisher(m_rabbitBusDic[MessagePriorityEnum.Low], MessagePriorityEnum.Low, buffer.GetList <NotifyMessage>()); } }
/// <summary> /// Copy constructor. /// </summary> /// <param name="other">The buffer to copy.</param> public DataBuffer(DataBuffer <T> other) { _underlyingBuffer = other._underlyingBuffer; _underlyingRootBuffer = other._underlyingRootBuffer; Type = other.Type; _data = other.Data; Offset = other.Offset; RelativeOffset = other.RelativeOffset; Length = other.Length; }
/// <summary> /// pool事件处理程序 /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private void OnSendRequest(object sender, LogMessageEventArgs e) { IDataBuffer <object> buffer = e.Message as IDataBuffer <object>; if (buffer != null && buffer.Count > 0) { List <DataTable> list = buffer.GetList().Cast <DataTable>().ToList(); RepeatBatchSaveLog(list, list[0].TableName.StartsWith("HandleLog")); } }
/// <summary> /// 缓冲池 /// </summary> /// <param name="dataBufferSize">缓冲池bufferPool的容量</param> /// <param name="dataBufferCount">单个缓冲buffer容量</param> /// <param name="autoFlushSeconds">刷新时间</param> /// <param name="isBlockMainThread">是否阻塞write线程</param> public DataBufferPool(int dataBufferSize, int dataBufferCount, TimeSpan autoFlushSeconds, bool isBlockMainThread) { m_isBlockMainThread = isBlockMainThread; m_BufferSize = dataBufferSize; m_DataCount = dataBufferCount; m_AutoFlushTime = autoFlushSeconds; m_dataBuffer = new DataBuffer(m_DataCount); //定时刷新 scheduler.Start(autoFlushSeconds, () => AutoFlush()); }
/// <summary> /// 批量生产消息,发消息队列 /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private void OnWrite(object sender, LogMessageEventArgs e) { IDataBuffer <object> data = e.Message as IDataBuffer <object>; if (data != null) { var list = data.GetList().Cast <DataObj>().ToList(); ServerInfo server = RabbitMQConfig.ServerInfoList[3]; EasyNetQHelper.SendAsync(server.Code, list); } }
/// <summary> /// Create a ndarray of a certain buffer and shape. /// </summary> /// <param name="buffer">The buffer to back this ndarray.</param> /// <param name="shape">The shape.</param> public ADNDArray(IDataBuffer <T> buffer, long[] shape) { if (buffer.Length < ArrayUtils.Product(shape)) { throw new ArgumentException($"Buffer must contain the entire shape, but buffer length was {buffer.Length} and total shape length {ArrayUtils.Product(shape)} (shape = {ArrayUtils.ToString(shape)})."); } Initialise(NDArrayUtils.CheckShape(shape), NDArrayUtils.GetStrides(shape)); Data = buffer; }
/// <inheritdoc /> public override void Fill <TOther>(TOther value, INDArray arrayToFill) { IDataBuffer <float> arrayToFillData = InternaliseArray(arrayToFill).Data; float floatValue = (float)System.Convert.ChangeType(value, typeof(float)); for (int i = 0; i < arrayToFillData.Length; i++) { arrayToFillData.Data.SetValue(floatValue, i); } }
/// <summary> /// Create an ndarray of a certain array (array will be COPIED into a data buffer) and shape. /// Total shape length must be smaller or equal than the data array length. /// </summary> /// <param name="data">The data to use to fill this ndarray.</param> /// <param name="shape">The shape.</param> public ADNDArray(T[] data, params long[] shape) { if (data.Length < ArrayUtils.Product(shape)) { throw new ArgumentException($"Data must contain the entire shape, but data length was {data.Length} and total shape length {ArrayUtils.Product(shape)} (shape = {ArrayUtils.ToString(shape)})."); } Initialise(NDArrayUtils.CheckShape(shape), NDArrayUtils.GetStrides(shape)); Data = new DataBuffer <T>(data, 0L, Length); }
protected void Button1_Click(object sender, EventArgs e) { //this.Label1.Text = this.Grid1.PageSize.ToString() + "/" + this.Grid1.PageCount.ToString() + "/" + this.Grid1.RecordCount.ToString() // + "/" + this.Grid1.GroupBy // + "/" + (int)Math.Ceiling((double)(((double)this.Grid1.RecordCount) / ((double)this.Grid1.PageSize))); ; IDataBuffer db = Microsoft.Practices.ServiceLocation.ServiceLocator.Current.GetInstance <IDataBuffer>(); if (db != null) { db.Reload(); } }
private void RepeatBatchSaveLog(object dataBuffer, bool isHandle) { string errMsg = ""; IDataBuffer buffer = dataBuffer as IDataBuffer; if (buffer != null && buffer.Count > 0) { List <DataTable> list = buffer.GetList <DataTable>(); foreach (DataTable table in list) { string key = table.TableName; int count = m_repeatBatchSaveTimes[key]; bool ret = InsertInternal(table, isHandle, out errMsg) > 0; if (!ret) { if (count < 5) { m_repeatBatchSaveTimes[key] = count + 1; if (isHandle) { _repeatBatchSaveHandlePool.Write(table); } else { _repeatBatchSaveProcessPool.Write(table); } } else { lock (m_repeatBatchSaveTimesLockObj) { m_repeatBatchSaveTimes.Remove(key); } Logger.Info(string.Format("日志补偿保存,表名:【{0}】,更新次数:【{1}】,更新结果:失败-超过补偿次数,影响数据条数:【{2}】", key, count + 1, table.Rows.Count)); } Logger.Info(string.Format("日志补偿保存,表名:【{0}】,更新次数:【{1}】,更新结果:【{2}】", key, count + 1, ret)); } else { lock (m_repeatBatchSaveTimesLockObj) { m_repeatBatchSaveTimes.Remove(key); } #if APPSERVER DeleteBackFile(table.TableName, isHandle); #endif } Thread.Sleep(100); } } }
/// <summary> /// Initializes the filesystem /// </summary> /// <param name="data">The raw file system data</param> /// <param name="fileSystemSectorIndex">The index of the file system information</param> public FileSystem(IDataBuffer data, int fileSystemSectorIndex) { _fileSystemData = data; //Load the file system information LoadFileSystemInfo(fileSystemSectorIndex); ClusterSize = Header.SectorsPerCluster * SectorSize; BaseClusterOffset = Header.FirstFileAllocationTablePosition - (3 * ClusterSize); //Load the partition and file allocation table LoadFileAllocationTable(); }
private void BuildBufferMessage(object dataBuffer) { IDataBuffer buffer = dataBuffer as IDataBuffer; if (buffer != null && buffer.Count > 0) { List <NotifyMessage> list = buffer.GetList <NotifyMessage>(); foreach (NotifyMessage message in list) { Build(message); } } }
private void SendBatch(object dataBuffer) { IDataBuffer buffer = dataBuffer as IDataBuffer; if (buffer != null && buffer.Count > 0) { Process.Debug("批量发送推送消息", "SendBatch", string.Format("数量【{0}】", buffer.Count), ""); List <PushMessage> list = buffer.GetList <PushMessage>(); foreach (PushMessage message in list) { Send(message, Callback); } } }
private void RepeatSaveMessage(object dataBuffer) { IDataBuffer buffer = dataBuffer as IDataBuffer; if (buffer != null && buffer.Count > 0) { List <object> list = buffer.GetList <object>(); foreach (object obj in list) { bool ret = false; string key = ""; if (obj is NotifyMessage) { key = ((NotifyMessage)obj).MessageId; ret = _notifyMessageBus.Save((NotifyMessage)obj); } else if (obj is PushMessageModel) { key = ((PushMessageModel)obj).PushId; ret = _pushMessageBus.Save((PushMessageModel)obj); } int count = m_repeatSaveTimes[key]; if (!ret) { if (count <= 5) { m_repeatSaveTimes[key] = count + 1; _repeatSavePool.Write(obj); } else { lock (m_repeatSaveTimesLockObj) { m_repeatSaveTimes.Remove(key); } } } else { lock (m_repeatSaveTimesLockObj) { m_repeatSaveTimes.Remove(key); } } Process.Debug(key, "消息补偿保存记录", "RepeatSaveMessage", key, string.Format("保存次数:【{0}】,更新结果:【{1}】", count + 1, ret), ""); Thread.Sleep(100); } } }
/// <inheritdoc /> public override void Fill <T>(T[] filler, INDArray arrayToFill, long[] destinationBeginIndices, long[] destinationEndIndices) { IDataBuffer <float> arrayToFillData = InternaliseArray(arrayToFill).Data; int destinationOffset = (int)NDArrayUtils.GetFlatIndex(arrayToFill.Shape, arrayToFill.Strides, destinationBeginIndices); int destinationLength = (int)NDArrayUtils.GetFlatIndex(arrayToFill.Shape, arrayToFill.Strides, destinationEndIndices) - destinationOffset + 1; // +1 because end is inclusive if (destinationLength < 0) { throw new ArgumentOutOfRangeException($"Destination begin indices must be smaller than destination end indices, but destination length was {destinationLength}."); } Array.Copy(filler, 0, arrayToFillData.Data, destinationOffset, destinationLength); }
public void TestDataBufferCopy() { long length = 400000L; DataBuffer <double> rootBuffer = new DataBuffer <double>(length); DataBuffer <double> childBufferL2 = new DataBuffer <double>(rootBuffer, 100L, length - 200L); IDataBuffer <double> childBufferL2Copy = childBufferL2.ShallowCopy(); Assert.AreSame(childBufferL2.Data, childBufferL2Copy.Data); Assert.AreSame(childBufferL2.GetUnderlyingBuffer(), childBufferL2Copy.GetUnderlyingBuffer()); Assert.AreSame(childBufferL2.GetUnderlyingRootBuffer(), childBufferL2Copy.GetUnderlyingRootBuffer()); Assert.AreEqual(childBufferL2.Length, childBufferL2Copy.Length); Assert.AreEqual(childBufferL2.Offset, childBufferL2Copy.Offset); Assert.AreEqual(childBufferL2.RelativeOffset, childBufferL2Copy.RelativeOffset); }
private void WriteFile(object sender, LogMessageEventArgs e) { string path = CreateFile(); IDataBuffer <object> data = e.Message as IDataBuffer <object>; if (data != null) { foreach (var d in data) { //File.AppendAllText(path, JsonConvert.SerializeObject(d) + Environment.NewLine); Debug.WriteLine(JsonConvert.SerializeObject(d)); tmpList.Add(d as DataObj); } } }
public object GetCellValue(Int2 gridCoord, LayerId layerId) { Int2 chunkCoord = GetChunkCoord(gridCoord, chunkWidth, chunkHeight); ChunkData chunk; if (TryGetChunkData(chunkCoord, out chunk)) { int index = GetCellIndex(gridCoord, chunkWidth, chunkHeight); IDataBuffer buffer = chunk.GetBuffer(layerId); return(buffer.GetValue(index)); } else { return(0); } }
/// <summary> /// /// </summary> /// <param name="buf"></param> public void AddDataBuffer(IDataBuffer buf) { m_buffers.Add(buf); }
protected MProto(IDataBuffer reader) { this.databuffer = reader; }
internal MBinaryProto(IDataBuffer buffer) : base(buffer) { }