private void CallStack()
{
while (true)
{
if (m_dataQueue.Count > 0)
{
IDataBuffer <object> dataBuffer = null;
if (m_dataQueue.TryDequeue(out dataBuffer))
{
if (dataBuffer != null)
{
Callback(this, new LogMessageEventArgs(dataBuffer));
}
}
}
if (m_dataQueue.IsEmpty)
{
Thread.Sleep(100);
}
else
{
Thread.Sleep(10);
}
}
}
private void NewDataBuffer()
{
lock (m_newDataBufferLockObj)
{
m_dataBuffer = new DataBuffer();
}
}
private void BatchSendMessage(object dataBuffer)
{
IDataBuffer buffer = dataBuffer as IDataBuffer;
if (buffer != null && buffer.Count > 0)
{
List <PushMessage> list = buffer.GetList <PushMessage>();
try
{
Process.Info("消息通知批量发送接口", "BatchSendMessage", "", string.Format("批次【{0}】，数量【{1}】", ++callCount, list.Count), "");
JinRi.Notify.ServiceAgent.SenderSendServiceSOA.SendServiceClient client = new JinRi.Notify.ServiceAgent.SenderSendServiceSOA.SendServiceClient();
PushMessageResult result = client.ReceiveList(list.ToArray());
Process.Info("消息通知批量发送接口_返回", "BatchSendMessage", "", result.Success + "", "");
}
catch (Exception ex)
{
//如果推送异常，去掉标示的推送缓存
foreach (PushMessage message in list)
{
string cacheKey = string.Format(CacheKeys.PushMessageToSender_Arg2, message.PushId, message.PushCount);
DistributedCache.Delete(cacheKey);
}
Process.Error("消息通知批量发送接口", "BatchSendMessage", "", "异步发送推送消息到消息通知接口出现异常：" + ex.GetString(), "Error");
}
}
}
/// <inheritdoc />
public override void Fill(INDArray filler, INDArray arrayToFill)
{
IDataBuffer <float> arrayToFillData = InternaliseArray(arrayToFill).Data;
IDataBuffer <float> fillerData = InternaliseArray(filler).Data;
arrayToFillData.SetValues(fillerData.Data, fillerData.Offset, 0, Math.Min(arrayToFill.Length, filler.Length));
}
private void CallStack()
{
while (true)
{
IDataBuffer dataBuffer = null;
lock (m_dataStackLockObj)
{
if (m_dataStack.Count > 0)
{
dataBuffer = m_dataStack.Pop();
Interlocked.Decrement(ref _count);
}
}
if (dataBuffer != null)
{
Callback(dataBuffer);
}
if (1 == Interlocked.CompareExchange(ref _isFlush, 0, 0))
{
if (0 == Interlocked.CompareExchange(ref _count, 0, 0))
{
_hasAllPoped = 1;
break;
}
}
Thread.Sleep(10);
}
}
/// <inheritdoc />
public override void Fill(INDArray filler, INDArray arrayToFill, long[] sourceBeginIndices, long[] sourceEndIndices, long[] destinationBeginIndices, long[] destinationEndIndices)
{
IDataBuffer <float> fillerData = InternaliseArray(filler).Data;
IDataBuffer <float> arrayToFillData = InternaliseArray(arrayToFill).Data;
int sourceOffset = (int)NDArrayUtils.GetFlatIndex(filler.Shape, filler.Strides, sourceBeginIndices);
int sourceLength = (int)NDArrayUtils.GetFlatIndex(filler.Shape, filler.Strides, sourceEndIndices) - sourceOffset + 1; // +1 because end is inclusive
int destinationOffset = (int)NDArrayUtils.GetFlatIndex(arrayToFill.Shape, arrayToFill.Strides, destinationBeginIndices);
int destinationLength = (int)NDArrayUtils.GetFlatIndex(arrayToFill.Shape, arrayToFill.Strides, destinationEndIndices) - destinationOffset + 1; // same here
if (sourceLength < 0)
{
throw new ArgumentOutOfRangeException($"Source begin indices must be smaller than source end indices, but source length was {sourceLength}.");
}
if (destinationLength < 0)
{
throw new ArgumentOutOfRangeException($"Destination begin indices must be smaller than destination end indices, but destination length was {destinationLength}.");
}
if (sourceLength != destinationLength)
{
throw new ArgumentException($"Source and destination indices length must batch, but source length was {sourceLength} and destination legnth was {destinationLength}.");
}
Array.Copy(fillerData.Data, sourceOffset, arrayToFillData.Data, destinationOffset, sourceLength);
}
/// <summary>
/// 初始化程序（设置ServiceLocator，读取IDataBuffer）
/// </summary>
public static void InitProgram()
{
if (!m_init)
{
#if DEBUG
Console.WriteLine("Please press any key to continue...");
Console.ReadLine();
#endif
m_init = true;
var p = DefaultServiceProvider.Instance;
p.EnableLog();
p.EnableNHibernate();
p.EnableCache();
p.EnableScript(false);
Microsoft.Practices.ServiceLocation.ServiceLocator.SetLocatorProvider(new Microsoft.Practices.ServiceLocation.ServiceLocatorProvider(
delegate()
{
return(p);
}));
IDataBuffer db = ServiceProvider.GetService <IDataBuffer>();
if (db != null)
{
db.LoadData();
}
}
}
/// <summary>
/// 初始化程序（设置ServiceLocator，读取IDataBuffer）
/// </summary>
public static void InitProgram()
{
if (!m_init)
{
m_init = true;
Feng.Utils.XceedUtility.SetXceedLicense();
log4net.Config.XmlConfigurator.Configure();
//HibernatingRhinos.NHibernate.Profiler.Appender.NHibernateProfiler.Initialize();
//if (Microsoft.Practices.ServiceLocation.ServiceLocator.Current == null) // throw null
{
if (!SystemConfiguration.LiteMode)
{
SetSpringServiceLocator();
}
ServiceProvider.SetDefaultService <ILogger>(new log4netLogger());
ServiceProvider.SetDefaultService <IExceptionProcess>(new LoggerExceptionProcess());
ServiceProvider.SetDefaultService <Feng.NH.ISessionFactoryManager>(new Feng.NH.NHibernateSessionFactoryManager());
ServiceProvider.SetDefaultService <IRepositoryFactory>(new NH.RepositoryFactory());
IPersistentCache c = new PersistentHashtableCache();
ServiceProvider.SetDefaultService <ICache>(c);
ServiceProvider.SetDefaultService <IPersistentCache>(c);
Feng.DBDef def = new Feng.DBDef();
ServiceProvider.SetDefaultService <IDefinition>(def);
IDataBuffer buf = new Feng.DBDataBuffer();
ServiceProvider.SetDefaultService <IDataBuffer>(buf);
IDataBuffers bufs = new DataBuffers();
bufs.AddDataBuffer(new Cache());
bufs.AddDataBuffer(buf);
bufs.AddDataBuffer(def);
ServiceProvider.SetDefaultService <IDataBuffers>(bufs);
IEntityScript script = new PythonandEvalutionEngineScript();
ServiceProvider.SetDefaultService <IScript>(script);
ServiceProvider.SetDefaultService <IEntityScript>(script);
ServiceProvider.SetDefaultService <IMessageBox>(new EmptyMessageBox());
ServiceProvider.SetDefaultService <IControlCollectionFactory>(new Feng.ControlCollectionFactoryBase());
ServiceProvider.SetDefaultService <IManagerFactory>(new Feng.Utils.ManagerFactory());
IDataBuffer db = ServiceProvider.GetService <IDataBuffer>();
if (db != null)
{
db.LoadData();
}
}
}
}
private void LogProcessBatchMessage(object obj)
{
IDataBuffer buffer = obj as IDataBuffer;
if (buffer != null && buffer.Count > 0)
{
LogBatchMessage(buffer.GetEnumerator(), false);
}
}
private void LogHandleBatchMessage(object obj)
{
IDataBuffer buffer = obj as IDataBuffer;
if (buffer != null && buffer.Count > 0)
{
LogBatchMessage(buffer.GetEnumerator(), true);
}
}
private void BatchSavePushMessage(object dataBuffer)
{
IDataBuffer buffer = dataBuffer as IDataBuffer;
if (buffer != null && buffer.Count > 0)
{
DirectSavePushMessage(buffer.GetList <PushMessageModel>());
}
}
private void BatchSaveNotifyMessage(object dataBuffer)
{
IDataBuffer buffer = dataBuffer as IDataBuffer;
if (buffer != null && buffer.Count > 0)
{
DirectSaveNotifyMessage(buffer.GetList <NotifyMessage>());
}
}
private void BatchLowPublisher(object dataBuffer)
{
IDataBuffer buffer = dataBuffer as IDataBuffer;
if (buffer != null && buffer.Count > 0)
{
Process.Debug("批量发布消息到队列", "BatchLowPublisher", string.Format("条数：【{0}】", buffer.Count), "");
m_rabbitMQBus.Publisher(m_rabbitBusDic[MessagePriorityEnum.Low], MessagePriorityEnum.Low, buffer.GetList <NotifyMessage>());
}
}
/// <summary>
/// Copy constructor.
/// </summary>
/// <param name="other">The buffer to copy.</param>
public DataBuffer(DataBuffer <T> other)
{
_underlyingBuffer = other._underlyingBuffer;
_underlyingRootBuffer = other._underlyingRootBuffer;
Type = other.Type;
_data = other.Data;
Offset = other.Offset;
RelativeOffset = other.RelativeOffset;
Length = other.Length;
}
/// <summary>
/// pool事件处理程序
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void OnSendRequest(object sender, LogMessageEventArgs e)
{
IDataBuffer <object> buffer = e.Message as IDataBuffer <object>;
if (buffer != null && buffer.Count > 0)
{
List <DataTable> list = buffer.GetList().Cast <DataTable>().ToList();
RepeatBatchSaveLog(list, list[0].TableName.StartsWith("HandleLog"));
}
}
/// <summary>
/// 缓冲池
/// </summary>
/// <param name="dataBufferSize">缓冲池bufferPool的容量</param>
/// <param name="dataBufferCount">单个缓冲buffer容量</param>
/// <param name="autoFlushSeconds">刷新时间</param>
/// <param name="isBlockMainThread">是否阻塞write线程</param>
public DataBufferPool(int dataBufferSize, int dataBufferCount, TimeSpan autoFlushSeconds, bool isBlockMainThread)
{
m_isBlockMainThread = isBlockMainThread;
m_BufferSize = dataBufferSize;
m_DataCount = dataBufferCount;
m_AutoFlushTime = autoFlushSeconds;
m_dataBuffer = new DataBuffer(m_DataCount);
//定时刷新
scheduler.Start(autoFlushSeconds, () => AutoFlush());
}
/// <summary>
/// 批量生产消息,发消息队列
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void OnWrite(object sender, LogMessageEventArgs e)
{
IDataBuffer <object> data = e.Message as IDataBuffer <object>;
if (data != null)
{
var list = data.GetList().Cast <DataObj>().ToList();
ServerInfo server = RabbitMQConfig.ServerInfoList[3];
EasyNetQHelper.SendAsync(server.Code, list);
}
}
/// <summary>
/// Create a ndarray of a certain buffer and shape.
/// </summary>
/// <param name="buffer">The buffer to back this ndarray.</param>
/// <param name="shape">The shape.</param>
public ADNDArray(IDataBuffer <T> buffer, long[] shape)
{
if (buffer.Length < ArrayUtils.Product(shape))
{
throw new ArgumentException($"Buffer must contain the entire shape, but buffer length was {buffer.Length} and total shape length {ArrayUtils.Product(shape)} (shape = {ArrayUtils.ToString(shape)}).");
}
Initialise(NDArrayUtils.CheckShape(shape), NDArrayUtils.GetStrides(shape));
Data = buffer;
}
/// <inheritdoc />
public override void Fill <TOther>(TOther value, INDArray arrayToFill)
{
IDataBuffer <float> arrayToFillData = InternaliseArray(arrayToFill).Data;
float floatValue = (float)System.Convert.ChangeType(value, typeof(float));
for (int i = 0; i < arrayToFillData.Length; i++)
{
arrayToFillData.Data.SetValue(floatValue, i);
}
}
/// <summary>
/// Create an ndarray of a certain array (array will be COPIED into a data buffer) and shape.
/// Total shape length must be smaller or equal than the data array length.
/// </summary>
/// <param name="data">The data to use to fill this ndarray.</param>
/// <param name="shape">The shape.</param>
public ADNDArray(T[] data, params long[] shape)
{
if (data.Length < ArrayUtils.Product(shape))
{
throw new ArgumentException($"Data must contain the entire shape, but data length was {data.Length} and total shape length {ArrayUtils.Product(shape)} (shape = {ArrayUtils.ToString(shape)}).");
}
Initialise(NDArrayUtils.CheckShape(shape), NDArrayUtils.GetStrides(shape));
Data = new DataBuffer <T>(data, 0L, Length);
}
protected void Button1_Click(object sender, EventArgs e)
{
//this.Label1.Text = this.Grid1.PageSize.ToString() + "/" + this.Grid1.PageCount.ToString() + "/" + this.Grid1.RecordCount.ToString()
// + "/" + this.Grid1.GroupBy
// + "/" + (int)Math.Ceiling((double)(((double)this.Grid1.RecordCount) / ((double)this.Grid1.PageSize))); ;
IDataBuffer db = Microsoft.Practices.ServiceLocation.ServiceLocator.Current.GetInstance <IDataBuffer>();
if (db != null)
{
db.Reload();
}
}
private void RepeatBatchSaveLog(object dataBuffer, bool isHandle)
{
string errMsg = "";
IDataBuffer buffer = dataBuffer as IDataBuffer;
if (buffer != null && buffer.Count > 0)
{
List <DataTable> list = buffer.GetList <DataTable>();
foreach (DataTable table in list)
{
string key = table.TableName;
int count = m_repeatBatchSaveTimes[key];
bool ret = InsertInternal(table, isHandle, out errMsg) > 0;
if (!ret)
{
if (count < 5)
{
m_repeatBatchSaveTimes[key] = count + 1;
if (isHandle)
{
_repeatBatchSaveHandlePool.Write(table);
}
else
{
_repeatBatchSaveProcessPool.Write(table);
}
}
else
{
lock (m_repeatBatchSaveTimesLockObj)
{
m_repeatBatchSaveTimes.Remove(key);
}
Logger.Info(string.Format("日志补偿保存，表名：【{0}】，更新次数：【{1}】，更新结果：失败-超过补偿次数，影响数据条数：【{2}】", key, count + 1, table.Rows.Count));
}
Logger.Info(string.Format("日志补偿保存，表名：【{0}】，更新次数：【{1}】，更新结果：【{2}】", key, count + 1, ret));
}
else
{
lock (m_repeatBatchSaveTimesLockObj)
{
m_repeatBatchSaveTimes.Remove(key);
}
#if APPSERVER
DeleteBackFile(table.TableName, isHandle);
#endif
}
Thread.Sleep(100);
}
}
}
/// <summary>
/// Initializes the filesystem
/// </summary>
/// <param name="data">The raw file system data</param>
/// <param name="fileSystemSectorIndex">The index of the file system information</param>
public FileSystem(IDataBuffer data, int fileSystemSectorIndex)
{
_fileSystemData = data;
//Load the file system information
LoadFileSystemInfo(fileSystemSectorIndex);
ClusterSize = Header.SectorsPerCluster * SectorSize;
BaseClusterOffset = Header.FirstFileAllocationTablePosition - (3 * ClusterSize);
//Load the partition and file allocation table
LoadFileAllocationTable();
}
private void BuildBufferMessage(object dataBuffer)
{
IDataBuffer buffer = dataBuffer as IDataBuffer;
if (buffer != null && buffer.Count > 0)
{
List <NotifyMessage> list = buffer.GetList <NotifyMessage>();
foreach (NotifyMessage message in list)
{
Build(message);
}
}
}
private void SendBatch(object dataBuffer)
{
IDataBuffer buffer = dataBuffer as IDataBuffer;
if (buffer != null && buffer.Count > 0)
{
Process.Debug("批量发送推送消息", "SendBatch", string.Format("数量【{0}】", buffer.Count), "");
List <PushMessage> list = buffer.GetList <PushMessage>();
foreach (PushMessage message in list)
{
Send(message, Callback);
}
}
}
private void RepeatSaveMessage(object dataBuffer)
{
IDataBuffer buffer = dataBuffer as IDataBuffer;
if (buffer != null && buffer.Count > 0)
{
List <object> list = buffer.GetList <object>();
foreach (object obj in list)
{
bool ret = false;
string key = "";
if (obj is NotifyMessage)
{
key = ((NotifyMessage)obj).MessageId;
ret = _notifyMessageBus.Save((NotifyMessage)obj);
}
else if (obj is PushMessageModel)
{
key = ((PushMessageModel)obj).PushId;
ret = _pushMessageBus.Save((PushMessageModel)obj);
}
int count = m_repeatSaveTimes[key];
if (!ret)
{
if (count <= 5)
{
m_repeatSaveTimes[key] = count + 1;
_repeatSavePool.Write(obj);
}
else
{
lock (m_repeatSaveTimesLockObj)
{
m_repeatSaveTimes.Remove(key);
}
}
}
else
{
lock (m_repeatSaveTimesLockObj)
{
m_repeatSaveTimes.Remove(key);
}
}
Process.Debug(key, "消息补偿保存记录", "RepeatSaveMessage", key, string.Format("保存次数：【{0}】，更新结果：【{1}】", count + 1, ret), "");
Thread.Sleep(100);
}
}
}
/// <inheritdoc />
public override void Fill <T>(T[] filler, INDArray arrayToFill, long[] destinationBeginIndices, long[] destinationEndIndices)
{
IDataBuffer <float> arrayToFillData = InternaliseArray(arrayToFill).Data;
int destinationOffset = (int)NDArrayUtils.GetFlatIndex(arrayToFill.Shape, arrayToFill.Strides, destinationBeginIndices);
int destinationLength = (int)NDArrayUtils.GetFlatIndex(arrayToFill.Shape, arrayToFill.Strides, destinationEndIndices) - destinationOffset + 1; // +1 because end is inclusive
if (destinationLength < 0)
{
throw new ArgumentOutOfRangeException($"Destination begin indices must be smaller than destination end indices, but destination length was {destinationLength}.");
}
Array.Copy(filler, 0, arrayToFillData.Data, destinationOffset, destinationLength);
}
public void TestDataBufferCopy()
{
long length = 400000L;
DataBuffer <double> rootBuffer = new DataBuffer <double>(length);
DataBuffer <double> childBufferL2 = new DataBuffer <double>(rootBuffer, 100L, length - 200L);
IDataBuffer <double> childBufferL2Copy = childBufferL2.ShallowCopy();
Assert.AreSame(childBufferL2.Data, childBufferL2Copy.Data);
Assert.AreSame(childBufferL2.GetUnderlyingBuffer(), childBufferL2Copy.GetUnderlyingBuffer());
Assert.AreSame(childBufferL2.GetUnderlyingRootBuffer(), childBufferL2Copy.GetUnderlyingRootBuffer());
Assert.AreEqual(childBufferL2.Length, childBufferL2Copy.Length);
Assert.AreEqual(childBufferL2.Offset, childBufferL2Copy.Offset);
Assert.AreEqual(childBufferL2.RelativeOffset, childBufferL2Copy.RelativeOffset);
}
private void WriteFile(object sender, LogMessageEventArgs e)
{
string path = CreateFile();
IDataBuffer <object> data = e.Message as IDataBuffer <object>;
if (data != null)
{
foreach (var d in data)
{
//File.AppendAllText(path, JsonConvert.SerializeObject(d) + Environment.NewLine);
Debug.WriteLine(JsonConvert.SerializeObject(d));
tmpList.Add(d as DataObj);
}
}
}
public object GetCellValue(Int2 gridCoord, LayerId layerId)
{
Int2 chunkCoord = GetChunkCoord(gridCoord, chunkWidth, chunkHeight);
ChunkData chunk;
if (TryGetChunkData(chunkCoord, out chunk))
{
int index = GetCellIndex(gridCoord, chunkWidth, chunkHeight);
IDataBuffer buffer = chunk.GetBuffer(layerId);
return(buffer.GetValue(index));
}
else
{
return(0);
}
}
/// <summary>
///
/// </summary>
/// <param name="buf"></param>
public void AddDataBuffer(IDataBuffer buf)
{
m_buffers.Add(buf);
}
protected MProto(IDataBuffer reader)
{
this.databuffer = reader;
}
internal MBinaryProto(IDataBuffer buffer)
: base(buffer)
{
}
