public McpeWrapper GetBatch()
{
lock (_cacheSync)
{
if (!isDirty && _cachedBatch != null)
{
return(_cachedBatch);
}
ClearCache();
McpeFullChunkData fullChunkData = McpeFullChunkData.CreateObject();
fullChunkData.chunkX = x;
fullChunkData.chunkZ = z;
fullChunkData.chunkData = GetBytes();
byte[] bytes = fullChunkData.Encode();
fullChunkData.PutPool();
var batch = BatchUtils.CreateBatchPacket(bytes, 0, bytes.Length, CompressionLevel.Optimal, true);
batch.MarkPermanent();
batch.Encode();
_cachedBatch = batch;
_cache = null;
isDirty = false;
return(_cachedBatch);
}
}
internal static McpeWrapper CreateMcpeBatch(byte[] bytes)
{
McpeWrapper batch = BatchUtils.CreateBatchPacket(new Memory <byte>(bytes, 0, (int)bytes.Length), CompressionLevel.Optimal, true);
batch.MarkPermanent();
batch.Encode();
return(batch);
}
private IEnumerable <PayloadReaderTestDescriptor> CreateCrossReferenceTestDescriptors(CrossReferenceTestCase testCase, ReaderTestConfiguration testConfiguration)
{
ExceptionUtilities.CheckArgumentNotNull(testCase, "testCase");
var emptyPayload = new OData.Common.PayloadTestDescriptor()
{
PayloadEdmModel = new EdmModel().Fixup()
};
IEnumerable <OData.Common.PayloadTestDescriptor> operationPayloads = new[] { emptyPayload };
// One of the operations in the test case may specify a reference link value to use to generate payloads
string payloadReferenceLink = testCase.ChangeSets.SelectMany(cset => cset.Operations).Select(o => o.PayloadCrossReferenceLink).SingleOrDefault(s => !string.IsNullOrEmpty(s));
if (payloadReferenceLink != null)
{
EdmModel testModel = Test.OData.Utils.Metadata.TestModels.BuildTestModel();
operationPayloads =
GeneratePayloadElementsWithCrossReferenceLinks(payloadReferenceLink, testConfiguration).Select(
p => new OData.Common.PayloadTestDescriptor
{
PayloadElement = p,
PayloadEdmModel = testModel,
});
}
var testDescriptors = new List <PayloadReaderTestDescriptor>();
foreach (var payload in operationPayloads)
{
IEnumerable <IMimePart> requestChangesets = testCase.ChangeSets.Select(
c => (IMimePart)BatchUtils.GetRequestChangeset(
c.Operations.Select(o =>
{
// check whether we need to inject a payload into this operation
var operationPayload = string.IsNullOrEmpty(o.PayloadCrossReferenceLink) ? emptyPayload : payload;
ODataUri operationUri = new ODataUri(new[] { ODataUriBuilder.Unrecognized(o.Uri.OriginalString) });
var requestOperation = operationPayload.InRequestOperation(HttpVerb.Post, operationUri, this.RequestManager);
requestOperation.Headers.Add(HttpHeaders.ContentId, o.ContentId);
return((IMimePart)requestOperation);
}).ToArray(),
this.RequestManager));
var testDescriptor = new PayloadReaderTestDescriptor(this.PayloadReaderSettings)
{
DebugDescription = testCase.DebugDescription,
PayloadElement = PayloadBuilder.BatchRequestPayload(requestChangesets.ToArray()).AddAnnotation(new BatchBoundaryAnnotation("batch_foo")),
ExpectedException = testCase.ExpectedException,
SkipTestConfiguration = (testConfig) => !testConfig.IsRequest,
};
testDescriptors.Add(testDescriptor);
}
return(testDescriptors);
}
private void SendBuffered(int messageCount, MemoryStream memStream)
{
if (messageCount == 0)
{
return;
}
var batch = BatchUtils.CreateBatchPacket(memStream.GetBuffer(), 0, (int)memStream.Length, CompressionLevel.Fastest, false);
batch.Encode();
memStream.Position = 0;
memStream.SetLength(0);
Server.SendPackage(this, batch);
}
private bool RegenerateProjectUsingUVS(FileSystemPath uprojectFilePath, FileSystemPath engineRoot)
{
if (PlatformUtil.RuntimePlatform != PlatformUtil.Platform.Windows)
{
return(false);
}
var pathToUnrealVersionSelector =
engineRoot / "Engine" / "Binaries" / "Win64" / "UnrealVersionSelector.exe";
if (!pathToUnrealVersionSelector.ExistsFile)
{
myLogger.Info($"[UnrealLink]: {pathToUnrealVersionSelector} is not available");
return(false);
}
var commandLine =
GetPlatformCommandLine(pathToUnrealVersionSelector, "/projectFiles", $"\"{uprojectFilePath}\"");
try
{
lock (HACK_getMutexForUBT())
{
myLogger.Info($"[UnrealLink]: Regenerating project files: {commandLine}");
ErrorLevelException.ThrowIfNonZero(InvokeChildProcess.InvokeChildProcessIntoLogger(
BatchUtils.GetPathToCmd(),
commandLine,
LoggingLevel.INFO,
TimeSpan.FromMinutes(1),
InvokeChildProcess.TreatStderr.AsOutput,
pathToUnrealVersionSelector.Directory
));
}
}
catch (ErrorLevelException errorLevelException)
{
myLogger.Error(errorLevelException,
$"[UnrealLink]: Failed refresh project files: calling {pathToUnrealVersionSelector} {commandLine}");
return(false);
}
return(true);
}
private byte[] Compress(ICollection <Packet> packets)
{
long length = 0;
foreach (Packet packet in packets)
{
length += packet.Encode().Length;
}
var compressionLevel = _session.CompressionThreshold > -1 && length >= _session.CompressionThreshold ?
System.IO.Compression.CompressionLevel.Fastest : System.IO.Compression.CompressionLevel.NoCompression;
using (MemoryStream stream = MiNetServer.MemoryStreamManager.GetStream())
{
int checksum;
using (var compressStream = new DeflateStream(stream, compressionLevel, true))
{
foreach (Packet packet in packets)
{
byte[] bs = packet.Encode();
if (bs != null && bs.Length > 0)
{
BatchUtils.WriteLength(compressStream, bs.Length);
compressStream.Write(bs, 0, bs.Length);
}
packet.PutPool();
}
compressStream.Flush();
}
byte[] bytes = stream.ToArray();
return(bytes);
}
}
public void SendQueue()
{
if (_sendQueueNotConcurrent.Count == 0)
{
return;
}
if (!Monitor.TryEnter(_syncHack))
{
return;
}
try
{
using (MemoryStream memStream = new MemoryStream())
{
Queue <Packet> queue = _sendQueueNotConcurrent;
int messageCount = 0;
int lenght = queue.Count;
for (int i = 0; i < lenght; i++)
{
Packet packet = null;
lock (_queueSync)
{
if (queue.Count == 0)
{
break;
}
try
{
packet = queue.Dequeue();
}
catch (Exception e)
{
}
}
if (packet == null)
{
continue;
}
if (State == ConnectionState.Unconnected)
{
packet.PutPool();
continue;
}
if (lenght == 1)
{
Server.SendPacket(this, packet);
}
else if (packet is McpeWrapper)
{
SendBuffered(messageCount, memStream);
messageCount = 0;
Server.SendPacket(this, packet);
// The following is necessary if no throttling is done on chunk sending,
// but having it creates a lot of packet lag when using SpawnLevel. You can see it
// as players standing still, but having running particles.
//
//Thread.Sleep(1); // Really important to slow down speed a bit
}
else if (packet.NoBatch)
{
SendBuffered(messageCount, memStream);
messageCount = 0;
Server.SendPacket(this, packet);
}
else
{
if (messageCount == 0)
{
memStream.Position = 0;
memStream.SetLength(0);
}
byte[] bytes = packet.Encode();
if (bytes != null)
{
messageCount++;
BatchUtils.WriteLength(memStream, bytes.Length);
//memStream.Write(BitConverter.GetBytes(Endian.SwapInt32(bytes.Length)), 0, 4);
memStream.Write(bytes, 0, bytes.Length);
}
packet.PutPool();
}
}
if (State == ConnectionState.Unconnected)
{
return;
}
SendBuffered(messageCount, memStream);
}
}
finally
{
Monitor.Exit(_syncHack);
}
}
private static async Task MainAsync()
{
string poolId = "JobPrepReleaseSamplePool";
string jobId = "JobPrepReleaseSampleJob";
var settings = Config.LoadAccountSettings();
// Location of the file that the job tasks will work with, a text file in the
// node's "shared" directory.
string taskOutputFile = "$AZ_BATCH_NODE_SHARED_DIR/job_prep_and_release.txt";
// The job prep task will write the node ID to the text file in the shared directory
string jobPrepCmdLine = $@"/bin/bash -c ""echo $AZ_BATCH_NODE_ID tasks: > {taskOutputFile}""";
// Each task then echoes its ID to the same text file
string taskCmdLine = $@"/bin/bash -c ""echo $AZ_BATCH_TASK_ID >> {taskOutputFile}""";
// The job release task will then delete the text file from the shared directory
string jobReleaseCmdLine = $@"/bin/bash -c ""rm {taskOutputFile}""";
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(settings.BatchServiceUrl, settings.BatchAccountName, settings.BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
var pool = await BatchUtils.CreatePoolIfNotExistAsync(batchClient, poolId);
var prepTask = new JobPreparationTask {
CommandLine = jobPrepCmdLine
};
var releaseTask = new JobReleaseTask {
CommandLine = jobReleaseCmdLine
};
var job = await BatchUtils.CreateJobIfNotExistAsync(batchClient, pool.Id, jobId, prepTask : prepTask, releaseTask : releaseTask);
// Create the tasks that the job will execute
List <CloudTask> tasks = new List <CloudTask>();
for (int i = 1; i <= 8; i++)
{
string taskId = "task" + i.ToString().PadLeft(3, '0');
string taskCommandLine = taskCmdLine;
CloudTask task = new CloudTask(taskId, taskCommandLine);
tasks.Add(task);
}
// Add the tasks in one API call as opposed to a separate AddTask call for each. Bulk task
// submission helps to ensure efficient underlying API calls to the Batch service.
Console.WriteLine("Submitting tasks and awaiting completion...");
await batchClient.JobOperations.AddTaskAsync(job.Id, tasks);
// Wait for the tasks to complete before proceeding. The long timeout here is to allow time
// for the nodes within the pool to be created and started if the pool had not yet been created.
await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
job.ListTasks(),
TaskState.Completed,
TimeSpan.FromMinutes(30));
Console.WriteLine("All tasks completed.");
Console.WriteLine();
// Print the contents of the shared text file modified by the job preparation and other tasks.
ODATADetailLevel nodeDetail = new ODATADetailLevel(selectClause: "id, state");
IPagedEnumerable <ComputeNode> nodes = batchClient.PoolOperations.ListComputeNodes(poolId, nodeDetail);
await nodes.ForEachAsync(async (node) =>
{
// Check to ensure that the node is Idle before attempting to pull the text file.
// If the pool was just created, there is a chance that another node completed all
// of the tasks prior to the other node(s) completing their startup procedure.
if (node.State == ComputeNodeState.Idle)
{
var files = await node.ListNodeFiles().ToListAsync();
NodeFile sharedTextFile = await node.GetNodeFileAsync("shared/job_prep_and_release.txt");
Console.WriteLine("Contents of {0} on {1}:", sharedTextFile.Path, node.Id);
Console.WriteLine("-------------------------------------------");
Console.WriteLine(await sharedTextFile.ReadAsStringAsync());
}
});
// Terminate the job to mark it as Completed; this will initiate the Job Release Task on any node
// that executed job tasks. Note that the Job Release Task is also executed when a job is deleted,
// thus you need not call Terminate if you typically delete your jobs upon task completion.
await batchClient.JobOperations.TerminateJobAsync(job.Id);
// Wait for the job to reach state "Completed." Note that this wait is not typically necessary in
// production code, but is done here to enable the checking of the release tasks exit code below.
await BatchUtils.WaitForJobToReachStateAsync(batchClient, job.Id, JobState.Completed, TimeSpan.FromMinutes(2));
// Print the exit codes of the prep and release tasks by obtaining their execution info
List <JobPreparationAndReleaseTaskExecutionInformation> prepReleaseInfo = await batchClient.JobOperations.ListJobPreparationAndReleaseTaskStatus(job.Id).ToListAsync();
foreach (JobPreparationAndReleaseTaskExecutionInformation info in prepReleaseInfo)
{
Console.WriteLine();
Console.WriteLine("{0}: ", info.ComputeNodeId);
// If no tasks were scheduled to run on the node, the JobPreparationTaskExecutionInformation will be null
if (info.JobPreparationTaskExecutionInformation != null)
{
Console.WriteLine(" Prep task exit code: {0}", info.JobPreparationTaskExecutionInformation.ExitCode);
}
// If no tasks were scheduled to run on the node, the JobReleaseTaskExecutionInformation will be null
if (info.JobReleaseTaskExecutionInformation != null)
{
Console.WriteLine(" Release task exit code: {0}", info.JobReleaseTaskExecutionInformation.ExitCode);
}
}
// Clean up the resources we've created in the Batch account
Console.WriteLine();
Console.WriteLine("Delete job? [yes] no");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
// Note that deleting the job will execute the job release task if the job was not previously terminated
await batchClient.JobOperations.DeleteJobAsync(job.Id);
}
Console.WriteLine("Delete pool? [yes] no");
response = Console.ReadLine();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(poolId);
}
}
}
private static async Task MainAsync()
{
const string poolId = "FileHandlingPool";
const string jobId = "FileHandlingJobDemo";
var settings = Config.LoadAccountSettings();
SetupStorage(settings.StorageAccountName, settings.StorageAccountKey);
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(settings.BatchServiceUrl, settings.BatchAccountName, settings.BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
var pool = await BatchUtils.CreatePoolIfNotExistAsync(batchClient, poolId);
var job = await BatchUtils.CreateJobIfNotExistAsync(batchClient, poolId, jobId);
//set up auto storage file
ResourceFile autoStorageFile = ResourceFile.FromAutoStorageContainer(AutoStorageContainerName, AutoStorageFileName);
Console.WriteLine("\n[INFO] Autostorage resource File reference: ");
Console.WriteLine("AutoStorageContainer: " + autoStorageFile.AutoStorageContainerName);
Console.WriteLine("FilePath: " + autoStorageFile.FilePath);
//upload file to external storage and add it as a resource file
string storageConnectionString =
$"DefaultEndpointsProtocol=https;AccountName={settings.StorageAccountName};AccountKey={settings.StorageAccountKey}";
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer externalContainer = blobClient.GetContainerReference(ExternalStorageContainerName);
await externalContainer.CreateIfNotExistsAsync();
var externalFile = await UploadFileToContainer(blobClient, ExternalStorageContainerName, "resource_files/resource_file.txt", "resource_file.txt");
Console.WriteLine("\n[INFO] External storage resource File reference:");
Console.WriteLine("SAS Url: " + externalFile.HttpUrl);
Console.WriteLine("FilePath: " + externalFile.FilePath);
// using staging files API
var filesToStage = new List <IFileStagingProvider>();
StagingStorageAccount fileStagingStorageAccount = new StagingStorageAccount(
storageAccount: settings.StorageAccountName,
storageAccountKey: settings.StorageAccountKey,
blobEndpoint: storageAccount.BlobEndpoint.ToString());
FileToStage stagedFile = new FileToStage("resource_files/staged_file.txt", fileStagingStorageAccount);
Console.WriteLine("\n[INFO] Staged File added:");
Console.WriteLine("Local File: " + stagedFile.LocalFileToStage);
Console.WriteLine("Node File: " + stagedFile.NodeFileName);
filesToStage.Add(stagedFile);
// setup output files
// Generate SAS for outputcontainer
CloudBlobContainer outputContainer = blobClient.GetContainerReference(OutputContainerName);
await outputContainer.CreateIfNotExistsAsync();
string containerSas = outputContainer.GetSharedAccessSignature(new SharedAccessBlobPolicy()
{
Permissions = SharedAccessBlobPermissions.Write,
SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddDays(1)
});
string containerUrl = outputContainer.Uri.AbsoluteUri + containerSas;
Console.WriteLine("\n[INFO] Output container: " + containerUrl);
Console.WriteLine("\nPress return to continue...");
Console.ReadLine();
// Create tasks
List <CloudTask> tasks = new List <CloudTask>();
for (var i = 1; i <= 10; i++)
{
var taskId = i.ToString().PadLeft(3, '0');
var commandLine = $@"/bin/bash -c ""echo 'Hello from {taskId}' && printf 'root dir:\n' > output.txt && ls -la >> output.txt && printf '\ninput dir:\n' >> output.txt && ls -la input >> output.txt""";
var task = new CloudTask(taskId, commandLine);
// add resource files to task (one autostorage, one in external storage)
task.ResourceFiles = new[] { autoStorageFile, externalFile };
// add staged files
task.FilesToStage = filesToStage;
// add output files
var outputFiles = new List <OutputFile>
{
new OutputFile(
filePattern: @"../std*.txt",
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(
containerUrl: containerUrl,
path: taskId)),
uploadOptions: new OutputFileUploadOptions(
uploadCondition: OutputFileUploadCondition.TaskCompletion)),
new OutputFile(
filePattern: @"output.txt",
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(
containerUrl: containerUrl,
path: taskId + @"\output.txt")),
uploadOptions: new OutputFileUploadOptions(
uploadCondition: OutputFileUploadCondition.TaskCompletion)),
};
task.OutputFiles = outputFiles;
tasks.Add(task);
}
Console.WriteLine("Submitting tasks and awaiting completion...");
// Add all tasks to the job.
batchClient.JobOperations.AddTask(job.Id, tasks);
await BatchUtils.WaitForTasksAndPrintOutputAsync(batchClient, job.ListTasks(), TimeSpan.FromMinutes(30));
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.JobOperations.DeleteJob(jobId);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.PoolOperations.DeletePool(poolId);
}
}
}
/// <summary>
/// Creates several PayloadTestDescriptors containing Batch Requests
/// </summary>
/// <param name="requestManager">Used for building the requests</param>
/// <param name="model">The model to use for adding additional types.</param>
/// <param name="withTypeNames">Whether or not to use full type names.</param>
/// <returns>PayloadTestDescriptors</returns>
public static IEnumerable <PayloadTestDescriptor> CreateBatchRequestTestDescriptors(
IODataRequestManager requestManager,
EdmModel model,
bool withTypeNames = false)
{
ExceptionUtilities.CheckArgumentNotNull(requestManager, "requestManager");
EdmEntityType personType = null;
EdmComplexType carType = null;
EdmEntitySet personsEntitySet = null;
EdmEntityContainer container = model.EntityContainer as EdmEntityContainer;
if (model != null)
{
//TODO: Clone EdmModel
//model = model.Clone();
if (container == null)
{
container = new EdmEntityContainer("TestModel", "DefaultContainer");
model.AddElement(container);
}
personType = model.FindDeclaredType("TestModel.TFPerson") as EdmEntityType;
carType = model.FindDeclaredType("TestModel.TFCar") as EdmComplexType;
// Create the metadata types for the entity instance used in the entity set
if (carType == null)
{
carType = new EdmComplexType("TestModel", "TFCar");
model.AddElement(carType);
carType.AddStructuralProperty("Make", EdmPrimitiveTypeKind.String, true);
carType.AddStructuralProperty("Color", EdmPrimitiveTypeKind.String, true);
}
if (personType == null)
{
personType = new EdmEntityType("TestModel", "TFPerson");
model.AddElement(personType);
personType.AddKeys(personType.AddStructuralProperty("Id", EdmPrimitiveTypeKind.Int32));
personType.AddStructuralProperty("Name", EdmPrimitiveTypeKind.String, true);
personType.AddStructuralProperty("Car", carType.ToTypeReference());
container.AddEntitySet("Customers", personType);
}
personsEntitySet = container.AddEntitySet("People", personType);
}
ComplexInstance carInstance = PayloadBuilder.ComplexValue(withTypeNames ? "TestModel.TFCar" : null)
.Property("Make", PayloadBuilder.PrimitiveValue("Ford"))
.Property("Color", PayloadBuilder.PrimitiveValue("Blue"));
ComplexProperty carProperty = (ComplexProperty)PayloadBuilder.Property("Car", carInstance)
.WithTypeAnnotation(personType);
EntityInstance personInstance = PayloadBuilder.Entity(withTypeNames ? "TestModel.TFPerson" : null)
.Property("Id", PayloadBuilder.PrimitiveValue(1))
.Property("Name", PayloadBuilder.PrimitiveValue("John Doe"))
.Property("Car", carInstance)
.WithTypeAnnotation(personType);
var carPropertyPayload = new PayloadTestDescriptor()
{
PayloadElement = carProperty,
PayloadEdmModel = model
};
var emptyPayload = new PayloadTestDescriptor()
{
PayloadEdmModel = CreateEmptyEdmModel()
};
var personPayload = new PayloadTestDescriptor()
{
PayloadElement = personInstance,
PayloadEdmModel = model
};
var root = ODataUriBuilder.Root(new Uri("http://www.odata.org/service.svc"));
var entityset = ODataUriBuilder.EntitySet(personsEntitySet);
// Get operations
var queryOperation1 = emptyPayload.InRequestOperation(HttpVerb.Get, new ODataUri(new ODataUriSegment[] { root }), requestManager);
var queryOperation2 = emptyPayload.InRequestOperation(HttpVerb.Get, new ODataUri(new ODataUriSegment[] { root }), requestManager);
// Post operation containing a complex property
var postOperation = carPropertyPayload.InRequestOperation(HttpVerb.Post, new ODataUri(new ODataUriSegment[] { root, entityset }), requestManager, MimeTypes.ApplicationJsonLight);
// Delete operation with no payload
var deleteOperation = emptyPayload.InRequestOperation(HttpVerb.Delete, new ODataUri(new ODataUriSegment[] { root, entityset }), requestManager);
// Put operation where the payload is an EntityInstance
var putOperation = personPayload.InRequestOperation(HttpVerb.Put, new ODataUri(new ODataUriSegment[] { root, entityset }), requestManager);
// A changeset containing a delete with no payload and a put
var twoOperationsChangeset = BatchUtils.GetRequestChangeset(new IMimePart[] { postOperation, deleteOperation }, requestManager);
// A changeset containing a delete with no payload
var oneOperationChangeset = BatchUtils.GetRequestChangeset(new IMimePart[] { deleteOperation }, requestManager);
// A changeset containing a put, post and delete
var threeOperationsChangeset = BatchUtils.GetRequestChangeset(new IMimePart[] { putOperation, postOperation, deleteOperation }, requestManager);
// A changeset containing no operations
var emptyChangeset = BatchUtils.GetRequestChangeset(new IMimePart[] { }, requestManager);
// Empty Batch
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchRequestPayload()
.AddAnnotation(new BatchBoundaryAnnotation("bb_emptybatch")),
PayloadEdmModel = emptyPayload.PayloadEdmModel,
SkipTestConfiguration = (tc) => !tc.IsRequest,
});
// Single Operation
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchRequestPayload(queryOperation1)
.AddAnnotation(new BatchBoundaryAnnotation("bb_singleoperation")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => !tc.IsRequest,
});
// Multiple Operations
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchRequestPayload(queryOperation1, queryOperation2)
.AddAnnotation(new BatchBoundaryAnnotation("bb_multipleoperations")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => !tc.IsRequest,
});
// Single Changeset
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchRequestPayload(twoOperationsChangeset)
.AddAnnotation(new BatchBoundaryAnnotation("bb_singlechangeset")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => !tc.IsRequest,
});
// Multiple Changesets (different content types)
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchRequestPayload(twoOperationsChangeset, oneOperationChangeset, emptyChangeset)
.AddAnnotation(new BatchBoundaryAnnotation("bb_multiplechangesets")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => !tc.IsRequest,
});
// Operations and changesets
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchRequestPayload(twoOperationsChangeset, queryOperation1, oneOperationChangeset)
.AddAnnotation(new BatchBoundaryAnnotation("bb_operationsandchangesets_1")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => !tc.IsRequest,
});
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchRequestPayload(queryOperation1, oneOperationChangeset, queryOperation2)
.AddAnnotation(new BatchBoundaryAnnotation("bb_operationsandchangesets_2")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => !tc.IsRequest,
});
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchRequestPayload(queryOperation1, queryOperation2, twoOperationsChangeset, oneOperationChangeset)
.AddAnnotation(new BatchBoundaryAnnotation("bb_operationsandchangesets_3")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => !tc.IsRequest,
});
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchRequestPayload(queryOperation1, threeOperationsChangeset, queryOperation2, twoOperationsChangeset, queryOperation1, oneOperationChangeset)
.AddAnnotation(new BatchBoundaryAnnotation("bb_operationsandchangesets_4")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => !tc.IsRequest,
});
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchRequestPayload(queryOperation1, emptyChangeset, queryOperation1, threeOperationsChangeset, queryOperation2, oneOperationChangeset)
.AddAnnotation(new BatchBoundaryAnnotation("bb_operationsandchangesets_5")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => !tc.IsRequest,
});
}
/// <summary>
/// Creates several PayloadTestDescriptors containing Batch Responses
/// </summary>
/// <param name="requestManager">Used for building the requests/responses.</param>
/// <param name="model">The model to use for adding additional types.</param>
/// <param name="withTypeNames">Whether or not to use full type names.</param>
/// <returns>PayloadTestDescriptors</returns>
public static IEnumerable <PayloadTestDescriptor> CreateBatchResponseTestDescriptors(
IODataRequestManager requestManager,
EdmModel model,
bool withTypeNames = false)
{
EdmEntityType personType = null;
EdmComplexType carType = null;
if (model != null)
{
//TODO: CLONE for EdmModel
//model = model.Clone();
personType = model.FindDeclaredType("TestModel.TFPerson") as EdmEntityType;
carType = model.FindDeclaredType("TestModel.TFCar") as EdmComplexType;
// Create the metadata types for the entity instance used in the entity set
if (carType == null)
{
carType = new EdmComplexType("TestModel", "TFCar");
model.AddElement(carType);
carType.AddStructuralProperty("Make", EdmPrimitiveTypeKind.String, true);
carType.AddStructuralProperty("Color", EdmPrimitiveTypeKind.String, true);
}
if (personType == null)
{
personType = new EdmEntityType("TestModel", "TFPerson");
model.AddElement(personType);
personType.AddKeys(personType.AddStructuralProperty("Id", EdmPrimitiveTypeKind.Int32));
personType.AddStructuralProperty("Name", EdmPrimitiveTypeKind.String, true);
personType.AddStructuralProperty("Car", carType.ToTypeReference());
EdmEntityContainer container = (EdmEntityContainer)model.EntityContainer;
if (container == null)
{
container = new EdmEntityContainer("TestModel", "DefaultContainer");
model.AddElement(container);
container.AddEntitySet("Customers", personType);
container.AddEntitySet("TFPerson", personType);
}
}
}
ComplexInstance carInstance = PayloadBuilder.ComplexValue(withTypeNames ? "TestModel.TFCar" : null)
.Property("Make", PayloadBuilder.PrimitiveValue("Ford"))
.Property("Color", PayloadBuilder.PrimitiveValue("Blue"));
ComplexProperty carProperty = (ComplexProperty)PayloadBuilder.Property("Car", carInstance)
.WithTypeAnnotation(carType);
EntityInstance personInstance = PayloadBuilder.Entity(withTypeNames ? "TestModel.TFPerson" : null)
.Property("Id", PayloadBuilder.PrimitiveValue(1))
.Property("Name", PayloadBuilder.PrimitiveValue("John Doe"))
.Property("Car", carInstance)
.WithTypeAnnotation(personType);
ODataErrorPayload errorInstance = PayloadBuilder.Error("ErrorCode")
.Message("ErrorValue")
.InnerError(
PayloadBuilder.InnerError().Message("InnerErrorMessage").StackTrace("InnerErrorStackTrace").TypeName("InnerErrorTypeName"));
var carPropertyPayload = new PayloadTestDescriptor()
{
PayloadElement = carProperty,
PayloadEdmModel = model
};
var emptyPayload = new PayloadTestDescriptor()
{
PayloadEdmModel = CreateEmptyEdmModel()
};
var personPayload = new PayloadTestDescriptor()
{
// This payload will be serialised to JSON so we need the annotation to mark it as a response.
PayloadElement = personInstance.AddAnnotation(new PayloadFormatVersionAnnotation()
{
Response = true, ResponseWrapper = true
}),
PayloadEdmModel = model
};
var errorPayload = new PayloadTestDescriptor()
{
PayloadElement = errorInstance,
PayloadEdmModel = model,
};
// response operation with a status code of 5 containing a complex instance
var carPropertyPayloadOperation = carPropertyPayload.InResponseOperation(5, requestManager);
// response operation with no payload and a status code of 200
var emptyPayloadOperation = emptyPayload.InResponseOperation(200, requestManager);
// response operation with a status code of 418 containing an entity instance
var personPayloadOperation = personPayload.InResponseOperation(418, requestManager, MimeTypes.ApplicationJsonLight);
// response operation with a status code of 404 containing an error instance
var errorPayloadOperation = errorPayload.InResponseOperation(404, requestManager);
// changeset with multiple operations
var twoOperationsChangeset = BatchUtils.GetResponseChangeset(new IMimePart[] { carPropertyPayloadOperation, emptyPayloadOperation }, requestManager);
// changesets with a single operation
var oneOperationChangeset = BatchUtils.GetResponseChangeset(new IMimePart[] { personPayloadOperation }, requestManager);
var oneErrorChangeset = BatchUtils.GetResponseChangeset(new IMimePart[] { errorPayloadOperation }, requestManager);
// empty changeset
var emptyChangeset = BatchUtils.GetResponseChangeset(new IMimePart[] { }, requestManager);
// Empty Batch
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchResponsePayload(new IMimePart[] { })
.AddAnnotation(new BatchBoundaryAnnotation("bb_emptybatch")),
PayloadEdmModel = emptyPayload.PayloadEdmModel,
SkipTestConfiguration = (tc) => tc.IsRequest,
});
// Single Operation
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchResponsePayload(new IMimePart[] { carPropertyPayloadOperation })
.AddAnnotation(new BatchBoundaryAnnotation("bb_singleoperation")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => tc.IsRequest,
});
// Multiple Operations
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchResponsePayload(new IMimePart[] { carPropertyPayloadOperation, emptyPayloadOperation, errorPayloadOperation, personPayloadOperation })
.AddAnnotation(new BatchBoundaryAnnotation("bb_multipleoperations")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => tc.IsRequest,
});
// Single Changeset
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchResponsePayload(new IMimePart[] { twoOperationsChangeset })
.AddAnnotation(new BatchBoundaryAnnotation("bb_singlechangeset")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => tc.IsRequest,
});
// Multiple Changesets
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchResponsePayload(new IMimePart[] { twoOperationsChangeset, oneOperationChangeset })
.AddAnnotation(new BatchBoundaryAnnotation("bb_multiplechangesets")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => tc.IsRequest,
});
// Operations and changesets
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchResponsePayload(new IMimePart[] { twoOperationsChangeset, carPropertyPayloadOperation, oneOperationChangeset })
.AddAnnotation(new BatchBoundaryAnnotation("bb_operationsandchangesets_1")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => tc.IsRequest,
});
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchResponsePayload(new IMimePart[] { carPropertyPayloadOperation, oneOperationChangeset, emptyPayloadOperation })
.AddAnnotation(new BatchBoundaryAnnotation("bb_operationsandchangesets_2")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => tc.IsRequest,
});
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchResponsePayload(new IMimePart[] { errorPayloadOperation, carPropertyPayloadOperation, emptyPayloadOperation, twoOperationsChangeset, oneOperationChangeset })
.AddAnnotation(new BatchBoundaryAnnotation("bb_operationsandchangesets_3")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => tc.IsRequest,
});
yield return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchResponsePayload(
new IMimePart[] { oneErrorChangeset, carPropertyPayloadOperation, emptyChangeset, emptyPayloadOperation, twoOperationsChangeset, personPayloadOperation, oneOperationChangeset, errorPayloadOperation })
.AddAnnotation(new BatchBoundaryAnnotation("bb_operationsandchangesets_4")),
PayloadEdmModel = model,
SkipTestConfiguration = (tc) => tc.IsRequest,
});
}
public void DisplaySelection(bool force = false, int particleId = 10)
{
if (!force && !ShowSelection)
{
return; // don't render at all
}
if (force && ShowSelection)
{
return; // Will be rendered on regular tick instead
}
Level level = Player.Level;
if (!Monitor.TryEnter(_sync))
{
return;
}
try
{
BoundingBox box = GetSelection();
var numberOfParticles = box.Height * box.Width * 2 + box.Height * box.Depth * 2;
bool isBig = numberOfParticles > 500;
List <McpeLevelEvent> packets = new List <McpeLevelEvent>();
{
//if ((Math.Abs(box.Width) > 0) || (Math.Abs(box.Height) > 0) || (Math.Abs(box.Depth) > 0))
{
var minX = Math.Min(box.Min.X, box.Max.X);
var maxX = Math.Max(box.Min.X, box.Max.X) + 1;
var minY = Math.Max(0, Math.Min(box.Min.Y, box.Max.Y));
var maxY = Math.Min(255, Math.Max(box.Min.Y, box.Max.Y)) + 1;
var minZ = Math.Min(box.Min.Z, box.Max.Z);
var maxZ = Math.Max(box.Min.Z, box.Max.Z) + 1;
// x/y
for (float x = minX; x <= maxX; x++)
{
for (float y = minY; y <= maxY; y++)
{
foreach (var z in new float[] { minZ, maxZ })
{
if (isBig)
{
if (x != minX && x != maxX && y != minY && y != maxY)
{
continue;
}
}
if (!level.IsAir(new BlockCoordinates((int)x, (int)y, (int)z)))
{
continue;
}
//var particle = new Particle(particleId, Player.Level) {Position = new Vector3(x, y, z) + new Vector3(0.5f, 0.5f, 0.5f)};
//var particle = new Particle(particleId, Player.Level) { Position = new Vector3(x, y, z) };
//particle.Spawn(new[] { Player });
McpeLevelEvent particleEvent = McpeLevelEvent.CreateObject();
particleEvent.eventId = (short)(0x4000 | 10);
particleEvent.position = new Vector3(x, y, z);
particleEvent.data = 0;
packets.Add(particleEvent);
}
}
}
// x/z
//for (float x = minX; x <= maxX; x++)
//{
// foreach (var y in new float[] {minY, maxY})
// {
// for (float z = minZ; z <= maxZ; z++)
// {
// if (!level.IsAir(new BlockCoordinates((int) x, (int) y, (int) z))) continue;
// //var particle = new Particle(10, Player.Level) {Position = new Vector3(x, y, z) + new Vector3(0.5f, 0.5f, 0.5f)};
// var particle = new Particle(10, Player.Level) {Position = new Vector3(x, y, z)};
// particle.Spawn(new[] {Player});
// }
// }
//}
// z/y
foreach (var x in new float[] { minX, maxX })
{
for (float y = minY; y <= maxY; y++)
{
for (float z = minZ; z <= maxZ; z++)
{
if (isBig)
{
if (z != minZ && z != maxZ && y != minY && y != maxY)
{
continue;
}
}
if (!level.IsAir(new BlockCoordinates((int)x, (int)y, (int)z)))
{
continue;
}
//var particle = new Particle(10, Player.Level) {Position = new Vector3(x, y, z) + new Vector3(0.5f, 0.5f, 0.5f)};
//var particle = new Particle(10, Player.Level) { Position = new Vector3(x, y, z) };
//particle.Spawn(new[] { Player });
McpeLevelEvent particleEvent = McpeLevelEvent.CreateObject();
particleEvent.eventId = (short)(0x4000 | 10);
particleEvent.position = new Vector3(x, y, z);
particleEvent.data = 0;
packets.Add(particleEvent);
}
}
}
}
if (packets.Count > 500)
{
if (force)
{
Log.Warn($"Selection size is {numberOfParticles}. Number of particles is {packets.Count} ");
}
return; // too many particles
}
if (packets.Count > 0)
{
var packet = BatchUtils.CreateBatchPacket(CompressionLevel.Optimal, packets.ToArray());
Player.SendPacket(packet);
//level.RelayBroadcast(new[] { Player }, packet);
}
}
}
catch (Exception e)
{
Log.Error("Display selection", e);
}
finally
{
Monitor.Exit(_sync);
}
}
private static async Task MainAsync()
{
//const string nodeSize = "standard_a1_v2";
const string nodeSize = "standard_a4_v2";
const int nodeCount = 4;
const int maxTasksPerNode = 4; //can be set to 4* the number of cores on each node
const int taskCount = 32;
// Ensure there are enough tasks to help avoid hitting some timeout conditions below
int minimumTaskCount = nodeCount * maxTasksPerNode * 2;
if (taskCount < minimumTaskCount)
{
Console.WriteLine("You must specify at least two tasks per node core for this sample ({0} tasks in this configuration).", minimumTaskCount);
Console.WriteLine();
// Not enough tasks, exit the application
return;
}
// In this sample, the tasks simply ping localhost on the compute nodes; adjust these
// values to simulate variable task duration
const int minPings = 30;
const int maxPings = 60;
const string poolId = "ParallelTasksPoolDemo";
const string jobId = "ParallelTasksJob";
var settings = Config.LoadAccountSettings();
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(settings.BatchServiceUrl, settings.BatchAccountName, settings.BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
var pool = await BatchUtils.CreatePoolIfNotExistAsync(
batchClient,
poolId,
nodeSize : nodeSize,
maxTasksPerNode : maxTasksPerNode,
schedulingPolicy : new TaskSchedulingPolicy(ComputeNodeFillType.Pack),
lowPriorityNodes : nodeCount);
CloudJob job = await BatchUtils.CreateJobIfNotExistAsync(batchClient, poolId, jobId);
// The job's tasks ping localhost a random number of times between minPings and maxPings.
// Adjust the minPings/maxPings values above to experiment with different task durations.
Random rand = new Random();
List <CloudTask> tasks = new List <CloudTask>();
for (int i = 1; i <= taskCount; i++)
{
string taskId = "task" + i.ToString().PadLeft(3, '0');
string taskCommandLine = "ping -c " + rand.Next(minPings, maxPings + 1).ToString() + " localhost";
CloudTask task = new CloudTask(taskId, taskCommandLine);
tasks.Add(task);
}
// Pause execution until the pool is steady and its compute nodes are ready to accept jobs.
// NOTE: Such a pause is not necessary within your own code. Tasks can be added to a job at any point and will be
// scheduled to execute on a compute node as soon any node has reached Idle state. Because the focus of this sample
// is the demonstration of running tasks in parallel on multiple compute nodes, we wait for all compute nodes to
// complete initialization and reach the Idle state in order to maximize the number of compute nodes available for
// parallelization.
await BatchUtils.WaitForPoolToReachStateAsync(batchClient, poolId, AllocationState.Steady, TimeSpan.FromMinutes(30));
await BatchUtils.WaitForNodesToReachStateAsync(batchClient, poolId, ComputeNodeState.Idle, TimeSpan.FromMinutes(30));
// Add the tasks in one API call as opposed to a separate AddTask call for each. Bulk task
// submission helps to ensure efficient underlying API calls to the Batch service.
Console.WriteLine("Submitting tasks and awaiting completion...");
await batchClient.JobOperations.AddTaskAsync(job.Id, tasks);
// Pause again to wait until *all* nodes are running tasks
await BatchUtils.WaitForNodesToReachStateAsync(batchClient, poolId, ComputeNodeState.Running, TimeSpan.FromMinutes(2));
Stopwatch stopwatch = Stopwatch.StartNew();
// Print out task assignment information.
Console.WriteLine();
await BatchUtils.PrintNodeTasksAsync(batchClient, poolId);
Console.WriteLine();
// Pause execution while we wait for all of the tasks to complete
Console.WriteLine("Waiting for task completion...");
Console.WriteLine();
try
{
await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
job.ListTasks(),
TaskState.Completed,
TimeSpan.FromMinutes(30));
}
catch (TimeoutException e)
{
Console.WriteLine(e.ToString());
}
stopwatch.Stop();
// Obtain the tasks, specifying a detail level to limit the number of properties returned for each task.
// If you have a large number of tasks, specifying a DetailLevel is extremely important in reducing the
// amount of data transferred, lowering your query response times in increasing performance.
ODATADetailLevel detail = new ODATADetailLevel(selectClause: "id,commandLine,nodeInfo,state");
IPagedEnumerable <CloudTask> allTasks = batchClient.JobOperations.ListTasks(job.Id, detail);
// Get a collection of the completed tasks sorted by the compute nodes on which they executed
List <CloudTask> completedTasks = allTasks
.Where(t => t.State == TaskState.Completed)
.OrderBy(t => t.ComputeNodeInformation.ComputeNodeId)
.ToList();
// Print the completed task information
Console.WriteLine();
Console.WriteLine("Completed tasks:");
string lastNodeId = string.Empty;
foreach (CloudTask task in completedTasks)
{
if (!string.Equals(lastNodeId, task.ComputeNodeInformation.ComputeNodeId))
{
Console.WriteLine();
Console.WriteLine(task.ComputeNodeInformation.ComputeNodeId);
}
lastNodeId = task.ComputeNodeInformation.ComputeNodeId;
Console.WriteLine("\t{0}: {1}", task.Id, task.CommandLine);
}
// Get a collection of the uncompleted tasks which may exist if the TaskMonitor timeout was hit
List <CloudTask> uncompletedTasks = allTasks
.Where(t => t.State != TaskState.Completed)
.OrderBy(t => t.Id)
.ToList();
// Print a list of uncompleted tasks, if any
Console.WriteLine();
Console.WriteLine("Uncompleted tasks:");
Console.WriteLine();
if (uncompletedTasks.Any())
{
foreach (CloudTask task in uncompletedTasks)
{
Console.WriteLine("\t{0}: {1}", task.Id, task.CommandLine);
}
}
else
{
Console.WriteLine("\t<none>");
}
// Print some summary information
Console.WriteLine();
Console.WriteLine(" Nodes: " + nodeCount);
Console.WriteLine(" Node size: " + nodeSize);
Console.WriteLine("Max tasks per node: " + pool.MaxTasksPerComputeNode);
Console.WriteLine(" Tasks: " + tasks.Count);
Console.WriteLine(" Duration: " + stopwatch.Elapsed);
Console.WriteLine();
Console.WriteLine("Done!");
Console.WriteLine();
// Clean up the resources we've created in the Batch account
Console.WriteLine();
Console.WriteLine("Delete job? [yes] no");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
// Note that deleting the job will execute the job release task if the job was not previously terminated
await batchClient.JobOperations.DeleteJobAsync(job.Id);
}
Console.WriteLine("Delete pool? [yes] no");
response = Console.ReadLine();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(poolId);
}
}
}
private static async Task MainAsync()
{
const int nodeCount = 1;
string poolId = "TaskDependenciesSamplePool";
string jobId = "TaskDependenciesJob";
var settings = Config.LoadAccountSettings();
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(settings.BatchServiceUrl, settings.BatchAccountName, settings.BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
var pool = await BatchUtils.CreatePoolIfNotExistAsync(batchClient, poolId, lowPriorityNodes : nodeCount);
var job = await BatchUtils.CreateJobIfNotExistAsync(batchClient, pool.Id, jobId, usesTaskDependencies : true);
string taskOutputFile = "$AZ_BATCH_NODE_SHARED_DIR/task_output.txt";
// Create the collection of tasks that will be added to the job.
List <CloudTask> tasks = new List <CloudTask>
{
// 'Rain' and 'Sun' don't depend on any other tasks
new CloudTask("Rain", $"/bin/bash -c \"echo Rain >> {taskOutputFile}\""),
new CloudTask("Sun", $"/bin/bash -c \"echo Sun >> {taskOutputFile}\""),
// Task 'Flowers' depends on completion of both 'Rain' and 'Sun'
// before it is run.
new CloudTask("Flowers", $"/bin/bash -c \"echo Flowers >> {taskOutputFile}\"")
{
DependsOn = TaskDependencies.OnIds("Rain", "Sun")
},
// Tasks 1, 2, and 3 don't depend on any other tasks. Because
// we will be using them for a task range dependency, we must
// specify string representations of integers as their ids.
new CloudTask("1", $"/bin/bash -c \"echo 1 >> {taskOutputFile}\""),
new CloudTask("2", $"/bin/bash -c \"echo 2 >> {taskOutputFile}\""),
new CloudTask("3", $"/bin/bash -c \"echo 3 >> {taskOutputFile}\""),
// Task dependency on ID range
new CloudTask("Final", $"/bin/bash -c \"echo Final >> {taskOutputFile}\"")
{
DependsOn = TaskDependencies.OnIdRange(1, 3)
},
// Task A is the parent task.
new CloudTask("A", $"/bin/bash -c \"echo A >> {taskOutputFile}\"")
{
// Specify exit conditions for task A and their dependency actions.
ExitConditions = new ExitConditions
{
// If task A exits with a pre-processing error, block any downstream tasks (in this example, task B).
PreProcessingError = new ExitOptions
{
DependencyAction = DependencyAction.Block
},
// If task A exits with the specified error codes, block any downstream tasks (in this example, task B).
ExitCodes = new List <ExitCodeMapping>
{
new ExitCodeMapping(10, new ExitOptions()
{
DependencyAction = DependencyAction.Block
}),
new ExitCodeMapping(20, new ExitOptions()
{
DependencyAction = DependencyAction.Block
})
},
// If task A succeeds or fails with any other error, any downstream tasks become eligible to run
// (in this example, task B).
Default = new ExitOptions
{
DependencyAction = DependencyAction.Satisfy
},
}
},
// Task B depends on task A. Whether it becomes eligible to run depends on how task A exits.
new CloudTask("B", $"/bin/bash -c \"echo B >> {taskOutputFile}\"")
{
DependsOn = TaskDependencies.OnId("A")
},
};
// Add the tasks in one API call as opposed to a separate AddTask call for each. Bulk task
// submission helps to ensure efficient underlying API calls to the Batch service.
Console.WriteLine("Submitting tasks and awaiting completion...");
await batchClient.JobOperations.AddTaskAsync(job.Id, tasks);
// Wait for the tasks to complete before proceeding. The long timeout here is to allow time
// for the nodes within the pool to be created and started if the pool had not yet been created.
await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
job.ListTasks(),
TaskState.Completed,
TimeSpan.FromMinutes(30));
Console.WriteLine("All tasks completed.");
Console.WriteLine();
// Print the contents of the shared text file modified by the job preparation and other tasks.
ODATADetailLevel nodeDetail = new ODATADetailLevel(selectClause: "id, state");
IPagedEnumerable <ComputeNode> nodes = batchClient.PoolOperations.ListComputeNodes(poolId, nodeDetail);
await nodes.ForEachAsync(async (node) =>
{
// Check to ensure that the node is Idle before attempting to pull the text file.
// If the pool was just created, there is a chance that another node completed all
// of the tasks prior to the other node(s) completing their startup procedure.
if (node.State == ComputeNodeState.Idle)
{
NodeFile sharedTextFile = await node.GetNodeFileAsync("shared/task_output.txt");
Console.WriteLine("Contents of {0} on {1}:", sharedTextFile.Path, node.Id);
Console.WriteLine("-------------------------------------------");
Console.WriteLine(await sharedTextFile.ReadAsStringAsync());
}
});
// Clean up the resources we've created in the Batch account
Console.WriteLine();
Console.WriteLine("Delete job? [yes] no");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
// Note that deleting the job will execute the job release task if the job was not previously terminated
await batchClient.JobOperations.DeleteJobAsync(job.Id);
}
Console.WriteLine("Delete pool? [yes] no");
response = Console.ReadLine();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(poolId);
}
}
}
/// <summary>
/// Helper method to run a single iteration of the URI reading tests in a specified configuration.
/// </summary>
/// <typeparam name="T">The type of the payload to read.</typeparam>
/// <param name="payloadElement">The payload to read.</param>
/// <param name="setExpectedUriAction">An action to set the URI in question on the payload.</param>
/// <param name="model">The metadata model.</param>
/// <param name="payloadUri">The payload URI for the current iteration.</param>
/// <param name="baseUriValue">The base URI value for the current iteration.</param>
/// <param name="resolver">The resolver to use.</param>
/// <param name="testConfiguration">The reader test configuration.</param>
private void RunBaseUriReadingTest <T>(
T payloadElement,
Action <T, Uri, ReaderTestConfiguration> setExpectedUriAction,
IEdmModel model,
Uri payloadUri,
BaseUriValue baseUriValue,
KeyValuePair <Func <Uri, Uri, Uri, Uri>, Uri> resolver,
ReaderTestConfiguration testConfiguration,
bool runInBatch = false) where T : ODataPayloadElement
{
this.Assert.IsNull(testConfiguration.MessageReaderSettings.BaseUri, "No base URI expected on reader settings.");
ExpectedException expectedException = null;
Uri settingsBaseUri = baseUriValue.ReaderSettingBaseUri;
// Set the base URI on the message reader settings if specified
if (settingsBaseUri != null)
{
testConfiguration = new ReaderTestConfiguration(testConfiguration);
testConfiguration.MessageReaderSettings.BaseUri = settingsBaseUri;
}
// Create the payload element
T clonedPayloadElement = payloadElement.DeepCopy();
setExpectedUriAction(clonedPayloadElement, payloadUri, testConfiguration);
if (testConfiguration.Format == ODataFormat.Atom)
{
XElement xmlRepresentation = this.PayloadElementToXmlConverter.ConvertToXml(clonedPayloadElement);
// add an xml:base attribute if specified
Uri xmlBaseUri = baseUriValue.XmlBaseUri;
if (xmlBaseUri != null)
{
xmlRepresentation.Add(new XAttribute(XNamespace.Xml.GetName("base"), xmlBaseUri.OriginalString));
}
clonedPayloadElement.XmlRepresentation(xmlRepresentation);
if (resolver.Value != null)
{
setExpectedUriAction(clonedPayloadElement, resolver.Value, testConfiguration);
}
else
{
// compute the expected URI value for ATOM
if (!payloadUri.IsAbsoluteUri)
{
if (xmlBaseUri != null)
{
setExpectedUriAction(clonedPayloadElement, new Uri(xmlBaseUri, payloadUri), testConfiguration);
}
else if (settingsBaseUri != null)
{
setExpectedUriAction(clonedPayloadElement, new Uri(settingsBaseUri, payloadUri), testConfiguration);
}
else
{
// fail for relative URIs without base URI
expectedException = ODataExpectedExceptions.ODataException("ODataAtomDeserializer_RelativeUriUsedWithoutBaseUriSpecified", payloadUri.OriginalString);
}
}
}
}
else
{
throw new NotSupportedException("Unsupported configuration format: " + testConfiguration.Format.ToString());
}
PayloadReaderTestDescriptor testDescriptor = new PayloadReaderTestDescriptor(resolver.Key == null ? this.Settings : this.NoValidatorSettings)
{
PayloadElement = clonedPayloadElement,
PayloadEdmModel = model,
ExpectedException = expectedException,
UrlResolver = resolver.Key == null ? null : new TestUrlResolver {
ResolutionCallback = (baseUri, realPayloadUri) => resolver.Key(payloadUri, baseUri, realPayloadUri)
},
SkipTestConfiguration = tc => ODataPayloadElementConfigurationValidator.GetSkipTestConfiguration(payloadElement, ODataPayloadElementConfigurationValidator.AllValidators)(tc),
};
if (runInBatch)
{
// TODO: Batch reader does not enter Exception state upon cross reference error in payload.
// Once fixed allow the batch tests to run even for error cases.
if (expectedException != null)
{
return;
}
if (testConfiguration.IsRequest)
{
testDescriptor = new PayloadReaderTestDescriptor(testDescriptor)
{
PayloadElement =
PayloadBuilder.BatchRequestPayload(
BatchUtils.GetRequestChangeset(
new IMimePart[] { testDescriptor.PayloadDescriptor.InRequestOperation(
HttpVerb.Put,
new ODataUri(ODataUriBuilder.Root(new Uri("http://odata.org/service"))),
this.RequestManager,
TestMediaTypeUtils.GetDefaultContentType(testDescriptor.PayloadDescriptor.PayloadKind, testConfiguration.Format)) },
this.RequestManager))
.AddAnnotation(new BatchBoundaryAnnotation("bb_request"))
};
}
else
{
testDescriptor = new PayloadReaderTestDescriptor(testDescriptor)
{
PayloadElement =
PayloadBuilder.BatchResponsePayload(
testDescriptor.PayloadDescriptor.InResponseOperation(
200,
this.RequestManager,
TestMediaTypeUtils.GetDefaultContentType(testDescriptor.PayloadDescriptor.PayloadKind, testConfiguration.Format)))
.AddAnnotation(new BatchBoundaryAnnotation("bb_response"))
};
}
testConfiguration = new ReaderTestConfiguration(null, testConfiguration.MessageReaderSettings, testConfiguration.IsRequest, testConfiguration.Synchronous, testConfiguration.Version);
}
testDescriptor.RunTest(testConfiguration);
}
internal static McpeBatch CreateMcpeBatch(byte[] bytes)
{
return(BatchUtils.CreateBatchPacket(bytes, 0, (int)bytes.Length, CompressionLevel.Optimal, true));
}
protected virtual void BroadCastMovement(Player[] players, Entity[] entities)
{
DateTime now = DateTime.UtcNow;
if (players.Length == 0)
{
return;
}
if (players.Length <= 1 && entities.Length == 0)
{
return;
}
if (now - _lastBroadcast < TimeSpan.FromMilliseconds(50))
{
return;
}
DateTime tickTime = _lastSendTime;
_lastSendTime = DateTime.UtcNow;
using (MemoryStream stream = MiNetServer.MemoryStreamManager.GetStream())
{
int playerMoveCount = 0;
int entiyMoveCount = 0;
foreach (var player in players)
{
if (now - player.LastUpdatedTime <= now - tickTime)
{
PlayerLocation knownPosition = player.KnownPosition;
McpeMovePlayer move = McpeMovePlayer.CreateObject();
move.entityId = player.EntityId;
move.x = knownPosition.X;
move.y = knownPosition.Y + 1.62f;
move.z = knownPosition.Z;
move.yaw = knownPosition.Yaw;
move.pitch = knownPosition.Pitch;
move.headYaw = knownPosition.HeadYaw;
move.mode = 0;
byte[] bytes = move.Encode();
BatchUtils.WriteLength(stream, bytes.Length);
stream.Write(bytes, 0, bytes.Length);
move.PutPool();
playerMoveCount++;
}
}
foreach (var entity in entities)
{
if (now - entity.LastUpdatedTime <= now - tickTime)
{
{
McpeMoveEntity moveEntity = McpeMoveEntity.CreateObject();
moveEntity.entityId = entity.EntityId;
moveEntity.position = (PlayerLocation)entity.KnownPosition.Clone();
moveEntity.position.Y += entity.PositionOffset;
byte[] bytes = moveEntity.Encode();
BatchUtils.WriteLength(stream, bytes.Length);
stream.Write(bytes, 0, bytes.Length);
moveEntity.PutPool();
}
{
McpeSetEntityMotion entityMotion = McpeSetEntityMotion.CreateObject();
entityMotion.entityId = entity.EntityId;
entityMotion.velocity = entity.Velocity;
byte[] bytes = entityMotion.Encode();
BatchUtils.WriteLength(stream, bytes.Length);
stream.Write(bytes, 0, bytes.Length);
entityMotion.PutPool();
}
entiyMoveCount++;
}
}
if (playerMoveCount == 0 && entiyMoveCount == 0)
{
return;
}
if (players.Length == 1 && entiyMoveCount == 0)
{
return;
}
McpeBatch batch = BatchUtils.CreateBatchPacket(stream.GetBuffer(), 0, (int)stream.Length, CompressionLevel.Optimal, false);
batch.AddReferences(players.Length - 1);
batch.Encode();
//batch.ValidUntil = now + TimeSpan.FromMilliseconds(50);
foreach (var player in players)
{
MiNetServer.FastThreadPool.QueueUserWorkItem(() => player.SendPackage(batch));
}
_lastBroadcast = DateTime.UtcNow;
}
}
/// <summary>
/// Creates a batch payload with the specified number of changesets and the specified number of operations in each changeset.
/// </summary>
/// <param name="requestManager">Used for building the requests/responses.</param>
/// <param name="changeSetCount">The number of changesets to create in the batch payload.</param>
/// <param name="changeSetSizes">The size of each changeset.</param>
/// <param name="forRequest">true if creating a batch request payload; otherwise false.</param>
/// <param name="changeSetBoundary">The changeset boundary to use; or null to use an auto-generated one.</param>
/// <returns>A <see cref="PayloadTestDescriptor"/> for the batch payload.</returns>
public static PayloadTestDescriptor CreateDefaultChangeSetBatch(
IODataRequestManager requestManager,
int changeSetCount,
int[] changeSetSizes,
bool forRequest,
string changeSetBoundary = null)
{
Debug.Assert(changeSetCount >= 0, "batchSize >= 0");
Debug.Assert(changeSetSizes != null, "changeSetSizes != null");
Debug.Assert(changeSetSizes.Length == changeSetCount, "Size of the batch must match the length of the change set sizes array!");
var emptyPayload = new PayloadTestDescriptor()
{
PayloadEdmModel = CreateEmptyEdmModel()
};
var root = ODataUriBuilder.Root(new Uri("http://www.odata.org/service.svc"));
IMimePart[] parts = new IMimePart[changeSetCount];
if (forRequest)
{
// Delete operation with no payload
var deleteOperation = emptyPayload.InRequestOperation(HttpVerb.Delete, new ODataUri(new ODataUriSegment[] { root }), requestManager);
for (int i = 0; i < changeSetCount; ++i)
{
int changeSetSize = changeSetSizes[i];
var deleteOperations = new IMimePart[changeSetSize];
for (int j = 0; j < changeSetSize; ++j)
{
deleteOperations[j] = deleteOperation;
}
var changeset = BatchUtils.GetRequestChangeset(deleteOperations, requestManager);
parts[i] = changeset;
}
;
string requestBoundary = changeSetBoundary ?? "bb_multiple_request_changesets_" + changeSetCount;
return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchRequestPayload(parts)
.AddAnnotation(new BatchBoundaryAnnotation(requestBoundary)),
});
}
// Response operation with no payload and a status code of 200
var emptyPayloadOperation = emptyPayload.InResponseOperation(200, requestManager);
for (int i = 0; i < changeSetCount; ++i)
{
int changeSetSize = changeSetSizes[i];
var operationResponses = new IMimePart[changeSetSize];
for (int j = 0; j < changeSetSize; ++j)
{
operationResponses[j] = emptyPayloadOperation;
}
var changeset = BatchUtils.GetResponseChangeset(operationResponses, requestManager);
parts[i] = changeset;
}
;
string responseBoundary = changeSetBoundary ?? "bb_multiple_response_changesets_" + changeSetCount;
return(new PayloadTestDescriptor()
{
PayloadElement = PayloadBuilder.BatchResponsePayload(parts)
.AddAnnotation(new BatchBoundaryAnnotation(responseBoundary)),
});
}
private void SendQueue()
{
if (_sendQueueNotConcurrent.Count == 0)
{
return;
}
if (!Monitor.TryEnter(_syncHack))
{
return;
}
try
{
using (MemoryStream memStream = MiNetServer.MemoryStreamManager.GetStream())
{
Queue <Package> queue = _sendQueueNotConcurrent;
int messageCount = 0;
int lenght = queue.Count;
for (int i = 0; i < lenght; i++)
{
Package package = null;
lock (_queueSync)
{
if (queue.Count == 0)
{
break;
}
try
{
package = queue.Dequeue();
}
catch (Exception e)
{
}
}
if (package == null)
{
continue;
}
if (State == ConnectionState.Unconnected)
{
package.PutPool();
continue;
}
if (lenght == 1)
{
Server.SendPackage(this, package);
}
else if (package is McpeWrapper)
{
SendBuffered(messageCount, memStream);
messageCount = 0;
Server.SendPackage(this, package);
Thread.Sleep(1); // Really important to slow down speed a bit
}
else if (package.NoBatch)
{
SendBuffered(messageCount, memStream);
messageCount = 0;
Server.SendPackage(this, package);
}
else
{
if (messageCount == 0)
{
memStream.Position = 0;
memStream.SetLength(0);
}
byte[] bytes = package.Encode();
if (bytes != null)
{
messageCount++;
BatchUtils.WriteLength(memStream, bytes.Length);
//memStream.Write(BitConverter.GetBytes(Endian.SwapInt32(bytes.Length)), 0, 4);
memStream.Write(bytes, 0, bytes.Length);
}
package.PutPool();
}
}
if (State == ConnectionState.Unconnected)
{
return;
}
SendBuffered(messageCount, memStream);
}
}
finally
{
Monitor.Exit(_syncHack);
}
}
private FileSystemPath GetPlatformCommand(FileSystemPath command)
{
return(PlatformUtil.RuntimePlatform == PlatformUtil.Platform.Windows ? BatchUtils.GetPathToCmd() : command);
}
internal void HandlePackage(Package message, PlayerNetworkSession playerSession)
{
//SignalTick();
try
{
if (message == null)
{
return;
}
if (typeof(UnknownPackage) == message.GetType())
{
UnknownPackage packet = (UnknownPackage)message;
if (Log.IsDebugEnabled)
{
Log.Warn($"Received unknown package 0x{message.Id:X2}\n{Package.HexDump(packet.Message)}");
}
message.PutPool();
return;
}
if (typeof(McpeWrapper) == message.GetType())
{
McpeWrapper batch = (McpeWrapper)message;
var messages = new List <Package>();
// Get bytes
byte[] payload = batch.payload;
if (playerSession.CryptoContext != null && playerSession.CryptoContext.UseEncryption)
{
throw new Exception("No crypto enabled");
}
// Decompress bytes
MemoryStream stream = new MemoryStream(payload);
if (stream.ReadByte() != 0x78)
{
throw new InvalidDataException("Incorrect ZLib header. Expected 0x78 0x9C");
}
stream.ReadByte();
using (var defStream2 = new DeflateStream(stream, CompressionMode.Decompress, false))
{
// Get actual package out of bytes
using (MemoryStream destination = MiNetServer.MemoryStreamManager.GetStream())
{
defStream2.CopyTo(destination);
destination.Position = 0;
NbtBinaryReader reader = new NbtBinaryReader(destination, true);
while (destination.Position < destination.Length)
{
//int len = reader.ReadInt32();
int len = BatchUtils.ReadLength(destination);
byte[] internalBuffer = reader.ReadBytes(len);
//if (Log.IsDebugEnabled)
// Log.Debug($"0x{internalBuffer[0]:x2}\n{Package.HexDump(internalBuffer)}");
messages.Add(PackageFactory.CreatePackage(internalBuffer[0], internalBuffer, "mcpe") ??
new UnknownPackage(internalBuffer[0], internalBuffer));
}
if (destination.Length > destination.Position)
{
throw new Exception("Have more data");
}
}
}
foreach (var msg in messages)
{
// Temp fix for performance, take 1.
var interact = msg as McpeInteract;
if (interact?.actionId == 4 && interact.targetRuntimeEntityId == 0)
{
continue;
}
msg.DatagramSequenceNumber = batch.DatagramSequenceNumber;
msg.Reliability = batch.Reliability;
msg.ReliableMessageNumber = batch.ReliableMessageNumber;
msg.OrderingChannel = batch.OrderingChannel;
msg.OrderingIndex = batch.OrderingIndex;
HandlePackage(msg, playerSession);
}
message.PutPool();
return;
}
MiNetServer.TraceReceive(message);
if (CryptoContext != null && CryptoContext.UseEncryption)
{
MiNetServer.FastThreadPool.QueueUserWorkItem(delegate()
{
HandlePackage(MessageHandler, message as Package);
message.PutPool();
});
}
else
{
HandlePackage(MessageHandler, message);
message.PutPool();
}
}
catch (Exception e)
{
Log.Error("Package handling", e);
throw;
}
}
protected override Expression VisitSelect(SelectExpression selectExpression)
{
if (!IsForBatchEF)
{
return(base.VisitSelect(selectExpression));
}
if (BatchUtils.IsNonComposedSetOperation(selectExpression))
{
GenerateSetOperation((SetOperationBase)selectExpression.Tables[0]);
return(selectExpression);
}
IDisposable disposable = null;
if (selectExpression.Alias != null)
{
Sql.AppendLine("(");
disposable = Sql.Indent();
}
Sql.Append("SELECT ");
if (selectExpression.IsDistinct)
{
Sql.Append("DISTINCT ");
}
GenerateTop(selectExpression);
if (selectExpression.Projection.Any())
{
BatchUtils.GenerateList(selectExpression.Projection, Sql, delegate(ProjectionExpression e)
{
var oldSQL = Sql.Build().CommandText; //zack's code
Visit(e);
string column = BatchUtils.Diff(oldSQL, this.Sql.Build().CommandText); //zack's code
this._projectionSQL.Add(column); //zack's code
});
}
else
{
Sql.Append("1");
this._projectionSQL.Add("1"); //zack's code
}
if (selectExpression.Tables.Any())
{
Sql.AppendLine().Append("FROM ");
BatchUtils.GenerateList(selectExpression.Tables, Sql, delegate(TableExpressionBase e)
{
Visit(e);
}, delegate(IRelationalCommandBuilder sql)
{
sql.AppendLine();
});
}
else
{
GeneratePseudoFromClause();
}
if (selectExpression.Predicate != null)
{
Sql.AppendLine().Append("WHERE ");
var oldSQL = Sql.Build().CommandText; //zack's code
Visit(selectExpression.Predicate);
this.PredicateSQL = BatchUtils.Diff(oldSQL, this.Sql.Build().CommandText); //zack's code
}
if (selectExpression.GroupBy.Count > 0)
{
Sql.AppendLine().Append("GROUP BY ");
BatchUtils.GenerateList(selectExpression.GroupBy, Sql, delegate(SqlExpression e)
{
Visit(e);
});
}
if (selectExpression.Having != null)
{
Sql.AppendLine().Append("HAVING ");
Visit(selectExpression.Having);
}
GenerateOrderings(selectExpression);
GenerateLimitOffset(selectExpression);
if (selectExpression.Alias != null)
{
disposable.Dispose();
Sql.AppendLine().Append(")" + AliasSeparator + _sqlGenerationHelper.DelimitIdentifier(selectExpression.Alias));
}
return(selectExpression);
}
