internal Fetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V> scheduler, MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics , ExceptionReporter exceptionReporter, SecretKey shuffleKey, int id) { /* Default read timeout (in milliseconds) */ // Initiative value is 0, which means it hasn't retried yet. this.jobConf = job; this.reporter = reporter; this.scheduler = scheduler; this.merger = merger; this.metrics = metrics; this.exceptionReporter = exceptionReporter; this.id = id; this.reduce = reduceId.GetTaskID().GetId(); this.shuffleSecretKey = shuffleKey; ioErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.IoError.ToString ()); wrongLengthErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongLength .ToString()); badIdErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.BadId.ToString ()); wrongMapErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongMap .ToString()); connectionErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.Connection .ToString()); wrongReduceErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongReduce .ToString()); this.connectionTimeout = job.GetInt(MRJobConfig.ShuffleConnectTimeout, DefaultStalledCopyTimeout ); this.readTimeout = job.GetInt(MRJobConfig.ShuffleReadTimeout, DefaultReadTimeout); this.fetchRetryInterval = job.GetInt(MRJobConfig.ShuffleFetchRetryIntervalMs, MRJobConfig .DefaultShuffleFetchRetryIntervalMs); this.fetchRetryTimeout = job.GetInt(MRJobConfig.ShuffleFetchRetryTimeoutMs, DefaultStalledCopyTimeout ); bool shuffleFetchEnabledDefault = job.GetBoolean(YarnConfiguration.NmRecoveryEnabled , YarnConfiguration.DefaultNmRecoveryEnabled); this.fetchRetryEnabled = job.GetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, shuffleFetchEnabledDefault ); SetName("fetcher#" + id); SetDaemon(true); lock (typeof(Org.Apache.Hadoop.Mapreduce.Task.Reduce.Fetcher)) { sslShuffle = job.GetBoolean(MRConfig.ShuffleSslEnabledKey, MRConfig.ShuffleSslEnabledDefault ); if (sslShuffle && sslFactory == null) { sslFactory = new SSLFactory(SSLFactory.Mode.Client, job); try { sslFactory.Init(); } catch (Exception ex) { sslFactory.Destroy(); throw new RuntimeException(ex); } } } }
public void SpecialInvalidCase(InvalidCase caseName, TestCoreFramework.Enums.ResourceType resourceId, int baseIndex, int mergeIndex) { MergeManager mergeHandler = new MergeManager(); Common.SendAndVerifyMergeRequest(RecordsSpecialMergeCreator, mergeHandler, resourceId.ToPrivateApiResource(), 0, 1); Common.SendAndVerifyMergeRequest(RecordsSpecialMergeCreator, mergeHandler, resourceId.ToPrivateApiResource(), baseIndex, mergeIndex, false); }
public Fetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V> scheduler , MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics, ExceptionReporter exceptionReporter, SecretKey shuffleKey) : this(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, shuffleKey , ++nextId) { }
public static void SendAndVerifyMergeRequest(HrbcRecordCreator record, MergeManager mergeHandler, ResourceId resourceId, int baseIndex, int mergeIndex, bool IsValidRequest = true) { var requestContent = new Dictionary <string, object>() { ["baseId"] = record.Data[$"{resourceId}{baseIndex}"].Id, ["mergedId"] = new List <ulong>() { record.Data[$"{resourceId}{mergeIndex}"].Id }, ["mapping"] = GetMapping(resourceId, record.Data[$"{resourceId}{baseIndex}"].Id, record.Data[$"{ResourceId.Client}{baseIndex}"].Id), }; var response = mergeHandler.MergeRecords <MergeResponse>(resourceId, requestContent, System.Net.Http.HttpMethod.Post); if (IsValidRequest) { PrAssert.That(response, PrIs.SuccessfulResponse()); PrAssert.That(response.Result.Id, PrIs.Not.Null); LoopAndVerifyCompleteMerge(response.Result.Id); } else { PrAssert.That(response, PrIs.ErrorResponse().With.HttpCode((int)HttpStatusCode.BadRequest)); PrAssert.That(response.Errors.Code, PrIs.EqualTo((int)ResultCode.InvalidParameter)); PrAssert.That(response.Errors.Error, PrIs.EqualTo("Invalid Parameter.")); } }
public void MergeInvalid(ResourceId resourceId, InvalidTestCases inputType, Parameters parameter) { var mergeRequest = GetParameters(resourceId, inputType, parameter); var mergeHandler = new MergeManager(); var response = mergeHandler.MergeRecords <MergeResponse>(resourceId, mergeRequest, System.Net.Http.HttpMethod.Post); PrAssert.That(response, PrIs.ErrorResponse().And.HttpCode(System.Net.HttpStatusCode.BadRequest)); }
public void MergeInjection(ResourceId resourceId, InjectionInput injection, Parameters parameter) { var mergeRequest = GetDefaultParameters(resourceId); mergeRequest[Char.ToLowerInvariant(parameter.ToString()[0]) + parameter.ToString().Substring(1)] = InjectionInputData.InjectionInputMapper[injection]; var mergeHandler = new MergeManager(); var response = mergeHandler.MergeRecords <MergeResponse>(resourceId, mergeRequest, System.Net.Http.HttpMethod.Post); PrAssert.That(response, PrIs.ErrorResponse().And.HttpCode(System.Net.HttpStatusCode.BadRequest)); }
public async Task StepAsync() { var propagator = new PropagateManager(PropagateManager.DefaultNeighborsGetter); var merger = new MergeManager(); var absorber = new AbsorptionManager(AbsorptionManager.DefaultAbsorbtion); var context = new WaterContext(Maps["plato"], propagator, merger, absorber); for (var i = 0; i < 10; i++) { await context.StepAsync(); } }
public void Start() { this.mergeManager = GameObject.Find("Merge Manager").GetComponent <MergeManager>(); if (this.mergeManager == null) { Debug.LogError("Merge Manager is wrong here."); } this.ownerSelectable = this.GetComponent <Selectable>(); this.playerNetworkView = this.GetComponent <NetworkView>(); this.ownerAttackable = this.GetComponent <Attackable>(); //this.mergeLevel = 1; }
public void MergeValid(ResourceId resourceId, ValidTestCases inputType) { var mergeRequest = GetParameters(resourceId, inputType); var mergeHandler = new MergeManager(); var response = mergeHandler.MergeRecords <MergeResponse>(resourceId, mergeRequest, HttpMethod.Post); PrAssert.That(response, PrIs.SuccessfulResponse().And.HttpCode(System.Net.HttpStatusCode.OK)); //Waiting for merge task successed before verifing data Assert.That(GetQueueMergeStatus(response.Result.Id.ToString()), "The merging is not finished yet!"); //Verify data VerifyData(resourceId, inputType); }
public void MergeByOtherHttpMethod(HttpMethod inputType) { var mergeRequest = GetDefaultParameters(ResourceId.Client); var httpMethod = new Dictionary <HttpMethod, System.Net.Http.HttpMethod>() { [HttpMethod.GET] = System.Net.Http.HttpMethod.Get, [HttpMethod.DELETE] = System.Net.Http.HttpMethod.Delete, [HttpMethod.PUT] = System.Net.Http.HttpMethod.Put, }; var mergeHandler = new MergeManager(); var response = mergeHandler.MergeRecords <MergeResponse>(ResourceId.Client, mergeRequest, httpMethod[inputType]); PrAssert.That(response, PrIs.ErrorResponse().And.HttpCode(System.Net.HttpStatusCode.BadRequest)); }
private void okButton_Click(object sender, EventArgs e) { Form1 form = (Form1)Owner; XtraTabControl pagesTabControl = form.pagesTabControl; if (pagesTabControl.TabPages.Count != itemValues.Length) { WindowManager.ShowInfoBox(this, LanguageUtil.GetCurrentLanguageString("Conflict", Name)); tabPagesListBox.Items.Clear(); InitializeForm(); return; } MergeManager.MergeTabs(this, itemValues); WindowManager.CloseForm(this); }
public void MergeRecords(ResourceId resourceId, TestCoreFramework.Enums.ResourceType readResource, string refField) { MergeManager mergeHandler = new MergeManager(); Common.SendAndVerifyMergeRequest(RecordsMergeCreator, mergeHandler, resourceId, 0, 1); var readRecordData = ReadRecords(readResource, new[] { (int)RecordsMergeCreator.Data[$"{readResource.ToPrivateApiResource()}{0}"].Id, (int)RecordsMergeCreator.Data[$"{readResource.ToPrivateApiResource()}{1}"].Id }, new string[] { refField }); PrAssert.That(readRecordData, PrIs.SuccessfulResponse()); string readValue = readRecordData.Result.Items.First()[refField].ToString(); string actualValue = readValue.StartsWith("[") && readValue.EndsWith("]") ? JsonConvert.DeserializeObject <List <int> >(readValue).SingleOrDefault().ToString() : readValue; PrAssert.That(actualValue, PrIs.EqualTo(RecordsMergeCreator.Data[$"{resourceId}{0}"].Id.ToString()), $"The first {readResource} is not reference to base id record"); readValue = readRecordData.Result.Items.Last()[refField].ToString(); actualValue = readValue.StartsWith("[") && readValue.EndsWith("]") ? JsonConvert.DeserializeObject <List <int> >(readValue).SingleOrDefault().ToString() : readValue; PrAssert.That(actualValue, PrIs.EqualTo(RecordsMergeCreator.Data[$"{resourceId}{0}"].Id.ToString()), $"The second {readResource} is not reference to base id record"); }
static void Main(string[] args) { string parentPath; string leftChildPath; string rightChildPath; string resultPath; Console.WriteLine("Input path to parent file:"); parentPath = Console.ReadLine(); CheckIfFileExists(ref parentPath); Console.WriteLine(); Console.WriteLine("Input path to first child file:"); leftChildPath = Console.ReadLine(); CheckIfFileExists(ref leftChildPath); Console.WriteLine(); Console.WriteLine("Input path to second child file:"); rightChildPath = Console.ReadLine(); CheckIfFileExists(ref rightChildPath); Console.WriteLine(); Console.WriteLine("Input path to result file:"); resultPath = Console.ReadLine(); Console.WriteLine(); Console.WriteLine(); try { MergeManager manager = new MergeManager(); List <ThreeWayMergedDocumentLine> result = manager.GetThreeWayMergeResultAsync(parentPath, leftChildPath, rightChildPath).Result; MergePrintManager printManager = new MergePrintManager(); printManager.PrintThreeWayMergeResult(resultPath, result); Console.WriteLine("Merge completed!"); Console.ReadKey(); } catch (Exception ex) { Console.WriteLine(ex.Message); Console.ReadKey(); } }
//Used for status updates public virtual void Init(ShuffleConsumerPlugin.Context context) { this.context = context; this.reduceId = context.GetReduceId(); this.jobConf = context.GetJobConf(); this.umbilical = context.GetUmbilical(); this.reporter = context.GetReporter(); this.metrics = new ShuffleClientMetrics(reduceId, jobConf); this.copyPhase = context.GetCopyPhase(); this.taskStatus = context.GetStatus(); this.reduceTask = context.GetReduceTask(); this.localMapFiles = context.GetLocalMapFiles(); scheduler = new ShuffleSchedulerImpl <K, V>(jobConf, taskStatus, reduceId, this, copyPhase , context.GetShuffledMapsCounter(), context.GetReduceShuffleBytes(), context.GetFailedShuffleCounter ()); merger = CreateMergeManager(context); }
public void MergeTwoLevelClientRecords(TestCoreFramework.Enums.ResourceType resourceId) { MergeManager mergeHandler = new MergeManager(); Common.SendAndVerifyMergeRequest(RecordsTwoLevelMergeCreator, mergeHandler, resourceId.ToPrivateApiResource(), 0, 1); Common.SendAndVerifyMergeRequest(RecordsTwoLevelMergeCreator, mergeHandler, resourceId.ToPrivateApiResource(), 2, 3); Common.SendAndVerifyMergeRequest(RecordsTwoLevelMergeCreator, mergeHandler, resourceId.ToPrivateApiResource(), 0, 2); var readRecordData = ReadRecords(resourceId, new[] { (int)RecordsTwoLevelMergeCreator.Data[$"{resourceId.ToPrivateApiResource()}{0}"].Id }, mappingDictionary[resourceId.ToPrivateApiResource()].Keys.ToArray()); PrAssert.That(readRecordData, PrIs.SuccessfulResponse()); foreach (var item in mappingDictionary[resourceId.ToPrivateApiResource()]) { string readValue = readRecordData.Result.Items.SingleOrDefault()[item.Key].ToString(); string actualValue = readValue.StartsWith("[") && readValue.EndsWith("]") ? JsonConvert.DeserializeObject <List <int> >(readValue).SingleOrDefault().ToString() : readValue; PrAssert.That(actualValue, PrIs.EqualTo(item.Key == $"{resourceId}.P_Name" ? Common.MappingName : item.Value), $"The BaseId record field {item.Key} contains unexpected value"); } }
public void MergeInvalidCommon(ResourceId resourceId, InvalidCommonTestCases inputType) { var parameters = GetDefaultParameters(resourceId); switch (inputType) { case InvalidCommonTestCases.AdditonalDuplicateId: parameters["baseId"] = GetRecordId(resourceId, 3); parameters["mergedId"] = GetRecordId(resourceId, 4); parameters["duplicateId"] = GetRecordId(resourceId, 5); break; case InvalidCommonTestCases.NoneExistedField: parameters["baseId"] = GetRecordId(resourceId, 6); parameters["mergedId"] = GetRecordId(resourceId, 7); parameters["mapping"] = GetMapping(resourceId, $"{resourceId.ToString()}.P_NonExistField", "TEST"); break; case InvalidCommonTestCases.NoneExistedResource: parameters["baseId"] = GetRecordId(resourceId, 8); parameters["mergedId"] = GetRecordId(resourceId, 9); parameters["mapping"] = GetMapping(resourceId, $"Resource.P_Zipcode", "000-0000"); break; case InvalidCommonTestCases.WrongResource: parameters["baseId"] = GetRecordId(resourceId, 10); parameters["mergedId"] = GetRecordId(resourceId, 11); parameters["mapping"] = GetMapping(resourceId, $"Job.P_Zipcode", "000-0000"); break; case InvalidCommonTestCases.ReverseField: parameters["baseId"] = GetRecordId(resourceId, 12); parameters["mergedId"] = GetRecordId(resourceId, 13); parameters["mapping"] = GetMapping(resourceId, $"{resourceId.ToString()}.P_Zipcode", "{\"" + resourceId.ToString() + ".P_Zipcode\": \"000 - 0000\"}"); break; } var mergeHandler = new MergeManager(); var response = mergeHandler.MergeRecords <MergeResponse>(resourceId, parameters, System.Net.Http.HttpMethod.Post); PrAssert.That(response, PrIs.ErrorResponse().And.HttpCode(System.Net.HttpStatusCode.BadRequest)); }
public void MergeTest() { var mergeManager = new MergeManager(); var _0_0 = new PointInt(0, 0); var _0_1 = new PointInt(0, 1); var _1_1 = new PointInt(1, 1); var m = 0.1; var drops = new Dictionary <WaterDrop, PointInt> { { new WaterDrop(m), _0_0 }, { new WaterDrop(m), _0_0 }, { new WaterDrop(m, new Vector(0, 1)), _0_1 }, { new WaterDrop(m, new Vector(0, -1)), _0_1 }, { new WaterDrop(m, new Vector(0, 1)), _1_1 }, { new WaterDrop(m, new Vector(1, 0)), _1_1 } }; var newDrops = mergeManager.Merge(_map, drops); Assert.Equal(3, newDrops.Count); }
public void DeletedIds(TestCoreFramework.Enums.ResourceType resourceId) { var baseId = DeletedRecord.Data[resourceId.ToPrivateApiResource()].Ids[0][0]; var mergeId = DeletedRecord.Data[resourceId.ToPrivateApiResource()].Ids[1][0]; var clientId = DeletedRecord.Data[ResourceId.Client].Ids[0][0]; var requestContent = new Dictionary <string, object>() { ["baseId"] = baseId, ["mergedId"] = new List <int>() { mergeId }, ["mapping"] = Common.GetMapping(resourceId.ToPrivateApiResource(), (ulong)baseId, (ulong)clientId), }; MergeManager mergeHandler = new MergeManager(); var response = mergeHandler.MergeRecords <MergeResponse>(resourceId.ToPrivateApiResource(), requestContent, System.Net.Http.HttpMethod.Post); PrAssert.That(response, PrIs.ErrorResponse().With.HttpCode((int)HttpStatusCode.BadRequest)); PrAssert.That(response.Errors.Code, PrIs.EqualTo((int)ResultCode.InvalidParameter)); PrAssert.That(response.Errors.Error, PrIs.EqualTo("Invalid Parameter.")); }
public void MergeDeletedId(ResourceId resourceId, Parameters parameter) { var parameters = GetDefaultParameters(resourceId); switch (parameter) { case Parameters.MergedId: parameters["mergedId"] = PrepareDeletedIdsData.Data[resourceId].Ids[0][0].ToString(); break; case Parameters.BaseId: parameters["baseId"] = PrepareDeletedIdsData.Data[resourceId].Ids[0][0].ToString(); parameters["mergedId"] = PrepareDeletedIdsData.Data[resourceId].Ids[1][0].ToString(); parameters["mapping"] = GetMapping(resourceId, $"{resourceId.ToString()}.P_Id", PrepareDeletedIdsData.Data[resourceId].Ids[0][0].ToString()); break; } var mergeHandler = new MergeManager(); var response = mergeHandler.MergeRecords <MergeResponse>(resourceId, parameters, System.Net.Http.HttpMethod.Post); PrAssert.That(response, PrIs.ErrorResponse().And.HttpCode(System.Net.HttpStatusCode.BadRequest)); }
public IActionResult Partition() { foreach (var file in Request.Form.Files) { if (file.Length == 0) { throw new ArgumentException(); } string v_Path = $"Tmp{Path.DirectorySeparatorChar}{file.FileName}"; using (var flux = new FileStream(v_Path, FileMode.Create, FileAccess.Write)) { file.CopyTo(flux); } MergeManager v_Manager = new MergeManager(); v_Manager.Merge(v_Path); } return(Ok()); }
//public ActionResult TopMerge() //{ // HomeViewModels vm = new HomeViewModels(); // return View(vm); //} //[HttpPost] //public ActionResult TopMerge(HomeViewModels vm, int? id1, int? id2) //{ // MergeManager mgr = new MergeManager(); // if (id1 == null || id2 == null || id1 == id2) // { // vm.ErrorMessage = true; // return View(vm); // } // vm.SuccessMessage = true; // //mgr.MergedIds(vm.Selected_MCB); // return View(vm); //} public void Merge(int id1, int id2) { MergeManager mgr = new MergeManager(); mgr.mergeAll(id1, id2); }
public LocalFetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V> scheduler, MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics, ExceptionReporter exceptionReporter, SecretKey shuffleKey, IDictionary <TaskAttemptID, MapOutputFile> localMapFiles) : base(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, shuffleKey ) { this.job = job; this.localMapFiles = localMapFiles; SetName("localfetcher#" + id); SetDaemon(true); }
private void moveDownButton_Click(object sender, EventArgs e) { itemValues = MergeManager.MoveItemsDown(this, itemValues); }