public void StartTask()
{
if (ProcessingTask != null)
{
ProcessingTask.Start();
}
}
public ProcessingTaskEntity(ProcessingTask task)
{
PartitionKey = task.UserId;
RowKey = task.Id;
TaskString = JsonConvert.SerializeObject(task);
}
public void StopTask()
{
if (ProcessingTask != null)
{
ProcessingTask.Wait();
}
}
void OnAddClick(object sender, int index, int protoId, int curCout)
{
if (!IsRunning)
{
return;
}
if (PeGameMgr.IsMulti)
{
_ColonyObj._Network.RPCServer(EPacketType.PT_CL_PRC_AddItem, protoId, curCout, index);
}
else
{
if (mTaskTable[index] == null)
{
mTaskTable[index] = new ProcessingTask();
}
//1.find same item
ProcessingTask pTask = mTaskTable[index];
if (pTask.AddItem(new ItemIdCount(protoId, curCout)))
{
//--to do: update UI
UpdateLineToUI(index);
}
}
}
public static void ParseData(byte[] data, CSProcessingData recordData)
{
using (MemoryStream ms = new MemoryStream(data))
using (BinaryReader reader = new BinaryReader(ms))
{
recordData.isAuto = BufferHelper.ReadBoolean(reader);
int taskCount = BufferHelper.ReadInt32(reader);
for (int i = 0; i < taskCount; i++)
{
int taskIndex = BufferHelper.ReadInt32(reader);
ProcessingTask pt = new ProcessingTask();
int itemCount = BufferHelper.ReadInt32(reader);
for (int j = 0; j < itemCount; j++)
{
ItemIdCount po = new ItemIdCount();
po.protoId = BufferHelper.ReadInt32(reader);
po.count = BufferHelper.ReadInt32(reader);
pt.itemList.Add(po);
}
pt.runCount = BufferHelper.ReadInt32(reader);
pt.m_CurTime = BufferHelper.ReadSingle(reader);
pt.m_Time = BufferHelper.ReadSingle(reader);
recordData.mTaskTable[taskIndex] = pt;
}
}
}
void OnSelectProcess(object sender, int index)
{
// if (!IsRunning)
// return;
selectProcess = mTaskTable[index];
UpdateSelectedTask();
}
public void SetCounter(int index, float cur, float final, int runCount)
{
if (mTaskTable[index] == null)
{
mTaskTable[index] = new ProcessingTask();
}
mTaskTable[index].StartCounterFromNet(cur, final, runCount);
}
public void AddNpcToTask(CSPersonnel npc, int index)
{
if (mTaskTable[index] == null)
{
mTaskTable[index] = new ProcessingTask();
}
mTaskTable[index].AddNpc(npc);
UpdateLineToUI(index);
}
/// <summary>
/// Выполнить архивацию файла.
/// </summary>
/// <param name="sourceStream">Исходный поток.</param>
/// <param name="targetStream">Архивированный поток.</param>
/// <param name="bufferSize">Размер буфера.</param>
private void CompressInternal(Stream sourceStream, Stream targetStream, long bufferSize)
{
var parts = Math.Ceiling((double)sourceStream.Length / bufferSize);
if (bufferSize > sourceStream.Length)
{
bufferSize = sourceStream.Length;
}
for (var portion = 0; portion < parts; portion++)
{
var currentPortion = portion;
var archivePart = new ArchivePart();
var size = bufferSize;
var offset = currentPortion * size;
if (offset + size > sourceStream.Length)
{
size = sourceStream.Length - offset;
}
archivePart.Offset = offset;
var compressingTask = new ProcessingTask(() =>
{
var portionArray = new byte[size];
lock (sourceStream)
{
sourceStream.Seek(archivePart.Offset, SeekOrigin.Begin);
sourceStream.Read(portionArray, 0, (int)size);
}
archivePart.Input = portionArray;
archivePart.Compress();
});
_queueManager.CompressQueue.Enqueue(compressingTask);
var writingTask = new ProcessingTask(() =>
{
while (true)
{
if (!archivePart.HasResult)
{
continue;
}
lock (targetStream)
archivePart.WriteResult(targetStream);
break;
}
});
_queueManager.WritingQueue.Enqueue(writingTask);
}
}
public async Task SaveTaskAsync(ProcessingTask task)
{
if (!MockWebApi.Tasks.ContainsKey(task.Id))
{
MockWebApi.Tasks.Add(task.Id, task);
}
else
{
MockWebApi.Tasks[task.Id] = task;
}
}
public async Task <ProcessingTaskQueueMessage> EnqueueConversionTaskAsync(ProcessingTask task)
{
var msg = new ProcessingTaskQueueMessage
{
Id = Guid.NewGuid().ToString(),
PopReceipt = "",
Task = task
};
MockWebApi.QueueMessages.Enqueue(msg);
return(msg);
}
public async Task SaveTaskAsync(ProcessingTask task)
{
if (null == task)
{
throw new ArgumentNullException("task");
}
ConfigurationOptions options = await _configuration.GetApplicationConfigurationAsync();
CloudTable table = await getTableReferenceAsync(options);
TableOperation operation = TableOperation.InsertOrReplace(new ProcessingTaskEntity(task));
await table.ExecuteAsync(operation);
}
protected override void SubscriptionStart(long sequenceId, Subscription subscription)
{
if (_externalSubscription != null)
{
throw new InvalidOperationException("Input edge subscription can be started only once.");
}
_externalSubscription = CreateExternalSubscription();
_externalSubscription.Start(sequenceId);
base.SubscriptionStart(sequenceId, subscription);
_task = new ProcessingTask(this);
_scheduler.Schedule(_task);
}
public async Task <ProcessingTaskQueueMessage> EnqueueConversionTaskAsync(ProcessingTask task)
{
ConfigurationOptions options = await _configuration.GetApplicationConfigurationAsync();
CloudQueue queue = await getQueueReferenceAsync(options);
string message = JsonConvert.SerializeObject(task);
var msg = new CloudQueueMessage(message);
await queue.AddMessageAsync(msg);
return(new ProcessingTaskQueueMessage {
Id = null, PopReceipt = null, Task = task
});
}
public async Task DoWorkAsync()
{
ProcessingTaskQueueMessage msg = await _queueService.DequeueConversionTaskAsync();
if (null == msg)
{
return;
}
ProcessingTask task = await _taskService.GetTaskAsync(msg.Task.Id);
if (null == task || null == task.SourceFile || string.IsNullOrWhiteSpace(task.SourceFile.BlobFileName))
{
return;
}
IFileProcessor fileProcessor = _fileProcessors[task.Operation];
if (null == fileProcessor)
{
return;
}
Trace.TraceInformation("Обрабатываем задание {0}", JsonConvert.SerializeObject(msg.Task));
var sourceStream = await _fileStorageService.GetSourceFileStreamAsync(task.SourceFile.BlobFileName);
var destinationStream = new MemoryStream((int)sourceStream.Length); // плохо, но не хочется делать файлы
await fileProcessor.ProcessFileAsync(sourceStream, destinationStream);
destinationStream.Seek(0, SeekOrigin.Begin);
var destinationFile = await _fileStorageService.SaveDestinationFileStreamAsync(destinationStream, task.SourceFile.MimeContentType);
destinationStream.Close();
task.DestinationFile = destinationFile;
task.Status = TaskStatus.Success;
task.Message = "Task Processed Succesfully";
await _taskService.SaveTaskAsync(task);
await _queueService.DeleteConversionTaskAsync(msg.Id, msg.PopReceipt);
Trace.TraceInformation("Задание обработано успешно");
}
//lz-2016.07.14 设置运行次数
void OnSetRunCount(object sender, int index, int runCount)
{
if (PeGameMgr.IsMulti)
{
//向服务器发送设置Rounds消息
_ColonyObj._Network.RPCServer(EPacketType.PT_CL_PRC_SetRound, index, runCount);
}
else
{
if (null == mTaskTable[index])
{
mTaskTable[index] = new ProcessingTask();
}
ProcessingTask pTaks = mTaskTable[index];
pTaks.SetRunCount(runCount);
UpdateLineToUI(index);
}
}
public void Initialize()
{
_rootElements = new OpenXmlElement[]
{
new Paragraph(new Run(new Text("first"))),
new Paragraph(new Run(new Text("second"))),
new BookmarkEnd(),
new Paragraph(new Run(new Text("third")))
};
_stream = new MemoryStream();
MockProcessor();
_instance = new ProcessingTask
{
RootElements = _rootElements,
Processor = _processor,
Out = new StreamWriter(_stream)
};
}
public void TaskAccomplished(ProcessingTask pTask, float percent = 1)
{
//get resource & renew task
GetTaskResult(percent, pTask.itemList);
int index = mTaskTable.ToList().FindIndex(it => it == pTask);
if (percent < 1)
{
pTask.ClearItem();
//--to do: updateUI
UpdateLineToUI(index);
if (pTask == selectProcess)
{
if (uiObj != null)
{
uiObj.ShowStartBtn(true);
}
}
pTask.SetRunCount(0);
}
else
{
if (pTask.runCount <= 1)
{
pTask.ClearItem();
//--to do: updateUI
UpdateLineToUI(index);
if (pTask == selectProcess)
{
if (uiObj != null)
{
uiObj.ShowStartBtn(true);
}
}
pTask.SetRunCount(0);
}
else
{
pTask.CountDownRepeat();
StartProcessing(index);
}
}
}
public void OnRemoveClick(object sender, int index, int protoId)
{
if (!IsRunning)
{
return;
}
if (PeGameMgr.IsMulti)
{
_ColonyObj._Network.RPCServer(EPacketType.PT_CL_PRC_RemoveItem, index, protoId);
}
else
{
if (mTaskTable[index] == null)
{
return;
}
ProcessingTask pTask = mTaskTable[index];
pTask.RemoveItem(protoId);
UpdateLineToUI(index);
}
}
public void AddItemToTask(int protoId, int count, int taskIndex)
{
if (PeGameMgr.IsMulti)
{
_ColonyObj._Network.RPCServer(EPacketType.PT_CL_PRC_AddItem, protoId, count, taskIndex);
}
else
{
if (mTaskTable[taskIndex] == null)
{
mTaskTable[taskIndex] = new ProcessingTask();
}
if (!mTaskTable[taskIndex].IsWorking())
{
mTaskTable[taskIndex].itemList.Add(new ItemIdCount(protoId, count));
}
else
{
//--to do: fail prompt
}
}
}
/// <summary>
/// Разархивировать файл.
/// </summary>
/// <param name="sourceStream">Архивированный поток.</param>
/// <param name="targetStream">Разархивированный поток.</param>
private void DecompressInternal(Stream sourceStream, Stream targetStream)
{
_archiveParts = GetArchiveParts(sourceStream);
foreach (var archivePart in _archiveParts)
{
var decompressingTask = new ProcessingTask(() =>
{
lock (sourceStream)
{
sourceStream.Seek(archivePart.Offset, SeekOrigin.Begin);
archivePart.Input = new byte[archivePart.Size];
sourceStream.Read(archivePart.Input, 0, archivePart.Size);
}
archivePart.Decompress();
});
_queueManager.CompressQueue.Enqueue(decompressingTask);
var writingTask = new ProcessingTask(() =>
{
while (true)
{
if (!archivePart.HasResult)
{
continue;
}
lock (targetStream)
archivePart.WriteResult(targetStream);
break;
}
});
_queueManager.WritingQueue.Enqueue(writingTask);
}
}
public List <ItemIdCount> GetItemsInProcessing()
{
List <ItemIdCount> processingItems = new List <ItemIdCount> ();
for (int i = 0; i < mTaskTable.Length; i++)
{
if (mTaskTable[i] != null)
{
if (mTaskTable[i].IsWorking())
{
ProcessingTask pt = mTaskTable[i];
if (pt.itemList.Count > 0)
{
foreach (ItemIdCount iic in pt.itemList)
{
CSUtils.AddItemIdCount(processingItems, iic.protoId, iic.count);
}
}
}
}
}
return(processingItems);
}
public async Task <HttpResponseMessage> Post()
{
Trace.TraceInformation(this.CreateLogEntry("Receiving a file"));
if (!Request.Content.IsMimeMultipartContent())
{
return(Request.CreateResponse(HttpStatusCode.BadRequest));
}
string operation = null;
HttpContent filePart = null;
MultipartMemoryStreamProvider bodyParts = await Request.Content.ReadAsMultipartAsync();
ProcessingTask task = null;
foreach (HttpContent part in bodyParts.Contents)
{
var contentDisposition = part.Headers.ContentDisposition;
if (contentDisposition == null || string.IsNullOrWhiteSpace(contentDisposition.Name))
{
continue;
}
string partName = contentDisposition.Name.Trim(new[] { ' ', '\t', '\"', '\'' });
if (string.Equals(partName, "operation", StringComparison.OrdinalIgnoreCase))
{
operation = await part.ReadAsStringAsync();
}
else if (null != part.Headers.ContentType &&
string.Equals(partName, "file", StringComparison.OrdinalIgnoreCase))
{
filePart = part;
}
if (!string.IsNullOrWhiteSpace(operation) && null != filePart)
{
break;
}
}
if (string.IsNullOrWhiteSpace(operation) || null == filePart)
{
return(Request.CreateResponse(HttpStatusCode.BadRequest));
}
string contentType = filePart.Headers.ContentType.MediaType;
string fileName = filePart.Headers.ContentDisposition.FileName;
fileName = fileName.Trim(new[] { ' ', '\t', '\"', '\'' });
FileMetadata fileMetadata = null;
using (Stream requestStream = await filePart.ReadAsStreamAsync())
{
fileMetadata = await _fileStorageService
.SaveSourceFileStreamAsync(requestStream, contentType);
}
task = new ProcessingTask
{
Status = TaskStatus.Created,
UserId = "*****@*****.**", // аутентификация - не в этом демо
Id = Guid.NewGuid().ToShortString(),
OriginalFileName = fileName,
Operation = operation,
Message = "Created processing task",
SourceFile = fileMetadata
};
// создали задачу
await _taskService.SaveTaskAsync(task);
// поставили ссылку в очередь
ProcessingTaskQueueMessage msg = await _queueService.EnqueueConversionTaskAsync(task);
task.Status = TaskStatus.Enqueued;
task.Message = "Enqueued task for processing";
// изменили статус задачи
await _taskService.SaveTaskAsync(task);
return(Request.CreateResponse(HttpStatusCode.OK, task));
}
public void CreateNewTaskWithItems(List <ItemIdCount> allItems)
{
//1.split task
//List<ItemIdCount> taskObjList = new List<ItemIdCount> ();
List <List <ItemIdCount> > taskList = new List <List <ItemIdCount> > ();
taskList.Add(new List <ItemIdCount> ());
int startIndex = 0; //first index has room
foreach (ItemIdCount iic in allItems)
{
int countMax = ProcessingObjInfo.GetPobMax(iic.protoId);
if (countMax < 0)
{
continue;
}
//find the first room
while (taskList[startIndex].Count >= ProcessingConst.OBJ_MAX)
{
startIndex++;
if (taskList.Count <= startIndex)
{
taskList.Add(new List <ItemIdCount> ());
}
}
int addIndex = startIndex; //current index to add;
while (iic.count > countMax)
{
if (taskList.Count <= addIndex)
{
taskList.Add(new List <ItemIdCount> ());
}
if (taskList[addIndex].Count < ProcessingConst.OBJ_MAX)
{
taskList[addIndex].Add(new ItemIdCount(iic.protoId, countMax));
iic.count -= countMax;
}
addIndex++;
}
if (taskList.Count <= addIndex)
{
taskList.Add(new List <ItemIdCount> ());
}
taskList[addIndex].Add(iic);
}
//2.assign npc
//1-find all free npc
List <CSPersonnel> freeNpc = GetFreeWorkers();
if (freeNpc.Count == 0)
{
return;
}
//2-get free Taks room
List <int> freeTask = new List <int> ();
for (int i = 0; i < ProcessingConst.TASK_NUM; i++)
{
if (mTaskTable[i] == null || mTaskTable[i].itemList.Count == 0)
{
freeTask.Add(i);
}
}
if (freeTask.Count == 0)
{
return;
}
int needTaskNum = taskList.Count;
int npcNum = freeNpc.Count;
int freeTaskNum = freeTask.Count;
//3-assign Npc
int runTaskNum = Mathf.Min(needTaskNum, npcNum, freeTaskNum);
int npcPerTask = npcNum / runTaskNum;
if (runTaskNum == 1 && npcNum > 1)
{
npcPerTask = npcNum - 1;
}
for (int i = 0; i < runTaskNum; i++)
{
int taskIndex = freeTask[i];
if (mTaskTable[taskIndex] == null)
{
mTaskTable[taskIndex] = new ProcessingTask();
}
for (int j = 0; j < npcPerTask; j++)
{
freeNpc[j].TrySetProcessingIndex(taskIndex);
}
freeNpc.RemoveRange(0, npcPerTask);
mTaskTable[taskIndex].itemList = taskList[i];
}
List <int> runRange = freeTask.GetRange(0, runTaskNum);
foreach (CSPersonnel csp in freeNpc)
{
if (runRange.Contains(csp.ProcessingIndex))
{
int changeToIndex = FindNextTaskNoStart(runRange);
if (changeToIndex >= 0)
{
csp.TrySetProcessingIndex(changeToIndex);
}
}
}
//3.start task
for (int i = 0; i < runTaskNum; i++)
{
int taskIndex = freeTask[i];
StartProcessing(taskIndex);
}
if (runTaskNum > 0)
{
CSAutocycleMgr.Instance.ShowProcessFor(taskList[0]);
}
//4.updateUI
UpdateDataToUI();
}
public void Sync()
{
ProcessingTask?.GetAwaiter().GetResult();
}
public async Task <ConsumptionAnalysisReport> AnalyzeConsumptionForDeletedResources(IList <UsageDetail> usage,
bool onlyWithOverages)
{
var report = new ConsumptionAnalysisReport();
// Get resources with non - zero costs
var processingPool = new ConcurrentQueue <ProcessingTask>(usage
.GroupBy(r => r.InstanceId.ToLowerInvariant())
.Select(u =>
new
{
ResourceId = u.First().InstanceId,
Costs = u.Sum(c => c.PretaxCost)
})
.Where(p => p.Costs > 0)
.Select(t => new ProcessingTask {
ResourceId = t.ResourceId
}));
var totalResources = processingPool.Count;
Log.Information(
$"Subscription ${_subscriptionId}. Retrieved {totalResources} resources with non-zero billing");
// Running processing in parallel threads
var processingThreads = processingPool.Count > MaxProcessingThreads
? MaxProcessingThreads
: processingPool.Count;
var tasks = new List <Task>(processingThreads);
var processedCount = 0;
for (var i = 0; i < processingThreads; i++)
{
var task = Task.Run(() =>
{
ProcessingTask task = null;
while (processingPool.TryDequeue(out task))
{
try
{
if (task.Exceptions.Count > ProcessingRetryCount)
{
Log.Warning($"Subscription ${_subscriptionId}. Failed to process {task.ResourceId}");
continue;
}
// Skip existing resources
var resourceExists = _resourceProvider.IsResourceExists(task.ResourceId).Result;
if (resourceExists == null)
{
Log.Information(
$"Item skipped. Billing Item {task.ResourceId} is not actually a resource");
continue;
}
if (resourceExists.Value)
{
Log.Information($"Item skipped. Item {task.ResourceId} exists at the moment");
continue;
}
// Delete event is missing for some resources, so use resource group deletion date
var deleteActivity = _activityLogProvider.GetResourceDeletionDate(task.ResourceId).Result ??
_activityLogProvider
.GetResourceGroupDeletionDate(
GetResourceGroupName(task.ResourceId)).Result;
if (deleteActivity == null || onlyWithOverages &&
!usage.Any(r =>
r.InstanceId == task.ResourceId && r.UsageStart > deleteActivity?.EventTimestamp))
{
processedCount++;
continue;
}
// Calculate overage as sum of billing records for dates past deletion
report.AddResource(new BilledResources
{
SubscriptionId = usage.First(r => r.InstanceId == task.ResourceId).SubscriptionGuid,
Currency = usage.First(r => r.InstanceId == task.ResourceId).Currency,
Id = task.ResourceId,
InstanceName = usage.First(r => r.InstanceId == task.ResourceId).InstanceName,
PretaxCost = usage.Where(r => r.InstanceId == task.ResourceId).Sum(u => u.PretaxCost),
ConsumedService = usage.First(r => r.InstanceId == task.ResourceId).ConsumedService,
SubscriptionName = usage.First(r => r.InstanceId == task.ResourceId).SubscriptionName,
UsageStart = usage.Where(r => r.InstanceId == task.ResourceId).Min(u => u.UsageStart),
UsageEnd = usage.Where(r => r.InstanceId == task.ResourceId).Max(u => u.UsageEnd),
ActivityDeleted = deleteActivity?.EventTimestamp,
DeleteOperationId = deleteActivity?.OperationId,
Overage = deleteActivity == null
? 0
: usage.Where(r =>
r.InstanceId == task.ResourceId &&
r.UsageStart > deleteActivity.EventTimestamp)
.Sum(o => o.PretaxCost)
});
processedCount++;
}
catch (Exception exception)
{
// Activity API sometimes fails with timeouts, need to retry
Log.Error(exception, $"Exception while processing resource {task.ResourceId}, retrying");
task.Exceptions.Add(exception);
processingPool.Enqueue(task);
}
}
}).ContinueWith(t =>
{
if (!t.IsCompletedSuccessfully)
{
Log.Error(t.Exception?.GetBaseException(), "Analysis thread crashed");
}
});
tasks.Add(task);
}
using var timer = new Timer(data =>
Log.Information(
$"Subscription {_subscriptionId}. Processed ... {processedCount} of {totalResources}"),
null, 0, 10000);
await Task.WhenAll(tasks);
return(report);
}
public void TaskPipelineTest()
{
var operationContent = new StringContent("ToUpper");
var cd = new ContentDispositionHeaderValue("inline");
cd.Name = "operation";
operationContent.Headers.ContentDisposition = cd;
var fileContent = new ByteArrayContent(File.ReadAllBytes(@"..\..\Source.txt"));
cd = new ContentDispositionHeaderValue("attachment");
cd.Name = "file";
cd.FileName = "Source.txt";
fileContent.Headers.ContentDisposition = cd;
var md = new MediaTypeHeaderValue("text/plain");
fileContent.Headers.ContentType = md;
var multipartContent = new MultipartContent();
multipartContent.Add(operationContent);
multipartContent.Add(fileContent);
using (var server = MockWebApi.CreateServer())
{
var response = server.CreateRequest("/api/ProcessingTask")
.And(r => r.Content = multipartContent)
.PostAsync()
.GetAwaiter()
.GetResult();
Assert.IsTrue(HttpStatusCode.OK == response.StatusCode, "StatusCode should be OK");
ProcessingTask task = response.Content.ReadAsAsync <ProcessingTask>().GetAwaiter().GetResult();
string taskId = task.Id;
Assert.IsTrue(TaskStatus.Enqueued == task.Status, "Task should be enqueued");
response = server.CreateRequest("/api/ProcessingTaskStatus?taskId=" + taskId)
.GetAsync()
.GetAwaiter()
.GetResult();
Assert.IsTrue(HttpStatusCode.OK == response.StatusCode, "StatusCode should be OK");
task = response.Content.ReadAsAsync <ProcessingTask>().GetAwaiter().GetResult();
Assert.IsTrue(TaskStatus.Enqueued == task.Status, "Task should be still enqueued");
var converters = new Dictionary <string, IFileProcessor>();
converters["ToUpper"] = new ToUpperFileProcessor();
ILifetimeAwareComponent <IFileProcessingWorker> app =
MockWebApi.CreateFileProcessingWorker(converters);
using (app.LifetimeScope)
{
app.Service.DoWorkAsync().GetAwaiter().GetResult();
}
response = server.CreateRequest("/api/ProcessingTaskStatus?taskId=" + taskId)
.GetAsync()
.GetAwaiter()
.GetResult();
Assert.IsTrue(HttpStatusCode.OK == response.StatusCode, "StatusCode should be OK");
task = response.Content.ReadAsAsync <ProcessingTask>().GetAwaiter().GetResult();
Assert.IsTrue(TaskStatus.Success == task.Status, "Task should be succeeded");
response = server.CreateRequest("/api/ProcessingResult?taskId=" + taskId)
.GetAsync()
.GetAwaiter()
.GetResult();
Assert.IsTrue(HttpStatusCode.OK == response.StatusCode, "StatusCode should be OK");
}
}
