internal Fetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V>
scheduler, MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics
, ExceptionReporter exceptionReporter, SecretKey shuffleKey, int id)
{
/* Default read timeout (in milliseconds) */
// Initiative value is 0, which means it hasn't retried yet.
this.jobConf = job;
this.reporter = reporter;
this.scheduler = scheduler;
this.merger = merger;
this.metrics = metrics;
this.exceptionReporter = exceptionReporter;
this.id = id;
this.reduce = reduceId.GetTaskID().GetId();
this.shuffleSecretKey = shuffleKey;
ioErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.IoError.ToString
());
wrongLengthErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongLength
.ToString());
badIdErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.BadId.ToString
());
wrongMapErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongMap
.ToString());
connectionErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.Connection
.ToString());
wrongReduceErrs = reporter.GetCounter(ShuffleErrGrpName, Fetcher.ShuffleErrors.WrongReduce
.ToString());
this.connectionTimeout = job.GetInt(MRJobConfig.ShuffleConnectTimeout, DefaultStalledCopyTimeout
);
this.readTimeout = job.GetInt(MRJobConfig.ShuffleReadTimeout, DefaultReadTimeout);
this.fetchRetryInterval = job.GetInt(MRJobConfig.ShuffleFetchRetryIntervalMs, MRJobConfig
.DefaultShuffleFetchRetryIntervalMs);
this.fetchRetryTimeout = job.GetInt(MRJobConfig.ShuffleFetchRetryTimeoutMs, DefaultStalledCopyTimeout
);
bool shuffleFetchEnabledDefault = job.GetBoolean(YarnConfiguration.NmRecoveryEnabled
, YarnConfiguration.DefaultNmRecoveryEnabled);
this.fetchRetryEnabled = job.GetBoolean(MRJobConfig.ShuffleFetchRetryEnabled, shuffleFetchEnabledDefault
);
SetName("fetcher#" + id);
SetDaemon(true);
lock (typeof(Org.Apache.Hadoop.Mapreduce.Task.Reduce.Fetcher))
{
sslShuffle = job.GetBoolean(MRConfig.ShuffleSslEnabledKey, MRConfig.ShuffleSslEnabledDefault
);
if (sslShuffle && sslFactory == null)
{
sslFactory = new SSLFactory(SSLFactory.Mode.Client, job);
try
{
sslFactory.Init();
}
catch (Exception ex)
{
sslFactory.Destroy();
throw new RuntimeException(ex);
}
}
}
}
public void SpecialInvalidCase(InvalidCase caseName, TestCoreFramework.Enums.ResourceType resourceId, int baseIndex, int mergeIndex)
{
MergeManager mergeHandler = new MergeManager();
Common.SendAndVerifyMergeRequest(RecordsSpecialMergeCreator, mergeHandler, resourceId.ToPrivateApiResource(), 0, 1);
Common.SendAndVerifyMergeRequest(RecordsSpecialMergeCreator, mergeHandler, resourceId.ToPrivateApiResource(), baseIndex, mergeIndex, false);
}
public Fetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V> scheduler
, MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics metrics, ExceptionReporter
exceptionReporter, SecretKey shuffleKey)
: this(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, shuffleKey
, ++nextId)
{
}
public static void SendAndVerifyMergeRequest(HrbcRecordCreator record, MergeManager mergeHandler, ResourceId resourceId, int baseIndex, int mergeIndex, bool IsValidRequest = true)
{
var requestContent = new Dictionary <string, object>()
{
["baseId"] = record.Data[$"{resourceId}{baseIndex}"].Id,
["mergedId"] = new List <ulong>()
{
record.Data[$"{resourceId}{mergeIndex}"].Id
},
["mapping"] = GetMapping(resourceId, record.Data[$"{resourceId}{baseIndex}"].Id, record.Data[$"{ResourceId.Client}{baseIndex}"].Id),
};
var response = mergeHandler.MergeRecords <MergeResponse>(resourceId, requestContent, System.Net.Http.HttpMethod.Post);
if (IsValidRequest)
{
PrAssert.That(response, PrIs.SuccessfulResponse());
PrAssert.That(response.Result.Id, PrIs.Not.Null);
LoopAndVerifyCompleteMerge(response.Result.Id);
}
else
{
PrAssert.That(response, PrIs.ErrorResponse().With.HttpCode((int)HttpStatusCode.BadRequest));
PrAssert.That(response.Errors.Code, PrIs.EqualTo((int)ResultCode.InvalidParameter));
PrAssert.That(response.Errors.Error, PrIs.EqualTo("Invalid Parameter."));
}
}
public void MergeInvalid(ResourceId resourceId, InvalidTestCases inputType, Parameters parameter)
{
var mergeRequest = GetParameters(resourceId, inputType, parameter);
var mergeHandler = new MergeManager();
var response = mergeHandler.MergeRecords <MergeResponse>(resourceId, mergeRequest, System.Net.Http.HttpMethod.Post);
PrAssert.That(response, PrIs.ErrorResponse().And.HttpCode(System.Net.HttpStatusCode.BadRequest));
}
public void MergeInjection(ResourceId resourceId, InjectionInput injection, Parameters parameter)
{
var mergeRequest = GetDefaultParameters(resourceId);
mergeRequest[Char.ToLowerInvariant(parameter.ToString()[0]) + parameter.ToString().Substring(1)] = InjectionInputData.InjectionInputMapper[injection];
var mergeHandler = new MergeManager();
var response = mergeHandler.MergeRecords <MergeResponse>(resourceId, mergeRequest, System.Net.Http.HttpMethod.Post);
PrAssert.That(response, PrIs.ErrorResponse().And.HttpCode(System.Net.HttpStatusCode.BadRequest));
}
public async Task StepAsync()
{
var propagator = new PropagateManager(PropagateManager.DefaultNeighborsGetter);
var merger = new MergeManager();
var absorber = new AbsorptionManager(AbsorptionManager.DefaultAbsorbtion);
var context = new WaterContext(Maps["plato"], propagator, merger, absorber);
for (var i = 0; i < 10; i++)
{
await context.StepAsync();
}
}
public void Start()
{
this.mergeManager = GameObject.Find("Merge Manager").GetComponent <MergeManager>();
if (this.mergeManager == null)
{
Debug.LogError("Merge Manager is wrong here.");
}
this.ownerSelectable = this.GetComponent <Selectable>();
this.playerNetworkView = this.GetComponent <NetworkView>();
this.ownerAttackable = this.GetComponent <Attackable>();
//this.mergeLevel = 1;
}
public void MergeValid(ResourceId resourceId, ValidTestCases inputType)
{
var mergeRequest = GetParameters(resourceId, inputType);
var mergeHandler = new MergeManager();
var response = mergeHandler.MergeRecords <MergeResponse>(resourceId, mergeRequest, HttpMethod.Post);
PrAssert.That(response, PrIs.SuccessfulResponse().And.HttpCode(System.Net.HttpStatusCode.OK));
//Waiting for merge task successed before verifing data
Assert.That(GetQueueMergeStatus(response.Result.Id.ToString()), "The merging is not finished yet!");
//Verify data
VerifyData(resourceId, inputType);
}
public void MergeByOtherHttpMethod(HttpMethod inputType)
{
var mergeRequest = GetDefaultParameters(ResourceId.Client);
var httpMethod = new Dictionary <HttpMethod, System.Net.Http.HttpMethod>()
{
[HttpMethod.GET] = System.Net.Http.HttpMethod.Get,
[HttpMethod.DELETE] = System.Net.Http.HttpMethod.Delete,
[HttpMethod.PUT] = System.Net.Http.HttpMethod.Put,
};
var mergeHandler = new MergeManager();
var response = mergeHandler.MergeRecords <MergeResponse>(ResourceId.Client, mergeRequest, httpMethod[inputType]);
PrAssert.That(response, PrIs.ErrorResponse().And.HttpCode(System.Net.HttpStatusCode.BadRequest));
}
private void okButton_Click(object sender, EventArgs e)
{
Form1 form = (Form1)Owner;
XtraTabControl pagesTabControl = form.pagesTabControl;
if (pagesTabControl.TabPages.Count != itemValues.Length)
{
WindowManager.ShowInfoBox(this, LanguageUtil.GetCurrentLanguageString("Conflict", Name));
tabPagesListBox.Items.Clear();
InitializeForm();
return;
}
MergeManager.MergeTabs(this, itemValues);
WindowManager.CloseForm(this);
}
public void MergeRecords(ResourceId resourceId, TestCoreFramework.Enums.ResourceType readResource, string refField)
{
MergeManager mergeHandler = new MergeManager();
Common.SendAndVerifyMergeRequest(RecordsMergeCreator, mergeHandler, resourceId, 0, 1);
var readRecordData = ReadRecords(readResource, new[] { (int)RecordsMergeCreator.Data[$"{readResource.ToPrivateApiResource()}{0}"].Id, (int)RecordsMergeCreator.Data[$"{readResource.ToPrivateApiResource()}{1}"].Id }, new string[] { refField });
PrAssert.That(readRecordData, PrIs.SuccessfulResponse());
string readValue = readRecordData.Result.Items.First()[refField].ToString();
string actualValue = readValue.StartsWith("[") && readValue.EndsWith("]") ? JsonConvert.DeserializeObject <List <int> >(readValue).SingleOrDefault().ToString() : readValue;
PrAssert.That(actualValue, PrIs.EqualTo(RecordsMergeCreator.Data[$"{resourceId}{0}"].Id.ToString()), $"The first {readResource} is not reference to base id record");
readValue = readRecordData.Result.Items.Last()[refField].ToString();
actualValue = readValue.StartsWith("[") && readValue.EndsWith("]") ? JsonConvert.DeserializeObject <List <int> >(readValue).SingleOrDefault().ToString() : readValue;
PrAssert.That(actualValue, PrIs.EqualTo(RecordsMergeCreator.Data[$"{resourceId}{0}"].Id.ToString()), $"The second {readResource} is not reference to base id record");
}
static void Main(string[] args)
{
string parentPath;
string leftChildPath;
string rightChildPath;
string resultPath;
Console.WriteLine("Input path to parent file:");
parentPath = Console.ReadLine();
CheckIfFileExists(ref parentPath);
Console.WriteLine();
Console.WriteLine("Input path to first child file:");
leftChildPath = Console.ReadLine();
CheckIfFileExists(ref leftChildPath);
Console.WriteLine();
Console.WriteLine("Input path to second child file:");
rightChildPath = Console.ReadLine();
CheckIfFileExists(ref rightChildPath);
Console.WriteLine();
Console.WriteLine("Input path to result file:");
resultPath = Console.ReadLine();
Console.WriteLine();
Console.WriteLine();
try
{
MergeManager manager = new MergeManager();
List <ThreeWayMergedDocumentLine> result = manager.GetThreeWayMergeResultAsync(parentPath, leftChildPath, rightChildPath).Result;
MergePrintManager printManager = new MergePrintManager();
printManager.PrintThreeWayMergeResult(resultPath, result);
Console.WriteLine("Merge completed!");
Console.ReadKey();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
Console.ReadKey();
}
}
//Used for status updates
public virtual void Init(ShuffleConsumerPlugin.Context context)
{
this.context = context;
this.reduceId = context.GetReduceId();
this.jobConf = context.GetJobConf();
this.umbilical = context.GetUmbilical();
this.reporter = context.GetReporter();
this.metrics = new ShuffleClientMetrics(reduceId, jobConf);
this.copyPhase = context.GetCopyPhase();
this.taskStatus = context.GetStatus();
this.reduceTask = context.GetReduceTask();
this.localMapFiles = context.GetLocalMapFiles();
scheduler = new ShuffleSchedulerImpl <K, V>(jobConf, taskStatus, reduceId, this, copyPhase
, context.GetShuffledMapsCounter(), context.GetReduceShuffleBytes(), context.GetFailedShuffleCounter
());
merger = CreateMergeManager(context);
}
public void MergeTwoLevelClientRecords(TestCoreFramework.Enums.ResourceType resourceId)
{
MergeManager mergeHandler = new MergeManager();
Common.SendAndVerifyMergeRequest(RecordsTwoLevelMergeCreator, mergeHandler, resourceId.ToPrivateApiResource(), 0, 1);
Common.SendAndVerifyMergeRequest(RecordsTwoLevelMergeCreator, mergeHandler, resourceId.ToPrivateApiResource(), 2, 3);
Common.SendAndVerifyMergeRequest(RecordsTwoLevelMergeCreator, mergeHandler, resourceId.ToPrivateApiResource(), 0, 2);
var readRecordData = ReadRecords(resourceId, new[] { (int)RecordsTwoLevelMergeCreator.Data[$"{resourceId.ToPrivateApiResource()}{0}"].Id }, mappingDictionary[resourceId.ToPrivateApiResource()].Keys.ToArray());
PrAssert.That(readRecordData, PrIs.SuccessfulResponse());
foreach (var item in mappingDictionary[resourceId.ToPrivateApiResource()])
{
string readValue = readRecordData.Result.Items.SingleOrDefault()[item.Key].ToString();
string actualValue = readValue.StartsWith("[") && readValue.EndsWith("]") ? JsonConvert.DeserializeObject <List <int> >(readValue).SingleOrDefault().ToString() : readValue;
PrAssert.That(actualValue, PrIs.EqualTo(item.Key == $"{resourceId}.P_Name" ? Common.MappingName : item.Value), $"The BaseId record field {item.Key} contains unexpected value");
}
}
public void MergeInvalidCommon(ResourceId resourceId, InvalidCommonTestCases inputType)
{
var parameters = GetDefaultParameters(resourceId);
switch (inputType)
{
case InvalidCommonTestCases.AdditonalDuplicateId:
parameters["baseId"] = GetRecordId(resourceId, 3);
parameters["mergedId"] = GetRecordId(resourceId, 4);
parameters["duplicateId"] = GetRecordId(resourceId, 5);
break;
case InvalidCommonTestCases.NoneExistedField:
parameters["baseId"] = GetRecordId(resourceId, 6);
parameters["mergedId"] = GetRecordId(resourceId, 7);
parameters["mapping"] = GetMapping(resourceId, $"{resourceId.ToString()}.P_NonExistField", "TEST");
break;
case InvalidCommonTestCases.NoneExistedResource:
parameters["baseId"] = GetRecordId(resourceId, 8);
parameters["mergedId"] = GetRecordId(resourceId, 9);
parameters["mapping"] = GetMapping(resourceId, $"Resource.P_Zipcode", "000-0000");
break;
case InvalidCommonTestCases.WrongResource:
parameters["baseId"] = GetRecordId(resourceId, 10);
parameters["mergedId"] = GetRecordId(resourceId, 11);
parameters["mapping"] = GetMapping(resourceId, $"Job.P_Zipcode", "000-0000");
break;
case InvalidCommonTestCases.ReverseField:
parameters["baseId"] = GetRecordId(resourceId, 12);
parameters["mergedId"] = GetRecordId(resourceId, 13);
parameters["mapping"] = GetMapping(resourceId, $"{resourceId.ToString()}.P_Zipcode", "{\"" + resourceId.ToString() + ".P_Zipcode\": \"000 - 0000\"}");
break;
}
var mergeHandler = new MergeManager();
var response = mergeHandler.MergeRecords <MergeResponse>(resourceId, parameters, System.Net.Http.HttpMethod.Post);
PrAssert.That(response, PrIs.ErrorResponse().And.HttpCode(System.Net.HttpStatusCode.BadRequest));
}
public void MergeTest()
{
var mergeManager = new MergeManager();
var _0_0 = new PointInt(0, 0);
var _0_1 = new PointInt(0, 1);
var _1_1 = new PointInt(1, 1);
var m = 0.1;
var drops = new Dictionary <WaterDrop, PointInt> {
{ new WaterDrop(m), _0_0 },
{ new WaterDrop(m), _0_0 },
{ new WaterDrop(m, new Vector(0, 1)), _0_1 },
{ new WaterDrop(m, new Vector(0, -1)), _0_1 },
{ new WaterDrop(m, new Vector(0, 1)), _1_1 },
{ new WaterDrop(m, new Vector(1, 0)), _1_1 }
};
var newDrops = mergeManager.Merge(_map, drops);
Assert.Equal(3, newDrops.Count);
}
public void DeletedIds(TestCoreFramework.Enums.ResourceType resourceId)
{
var baseId = DeletedRecord.Data[resourceId.ToPrivateApiResource()].Ids[0][0];
var mergeId = DeletedRecord.Data[resourceId.ToPrivateApiResource()].Ids[1][0];
var clientId = DeletedRecord.Data[ResourceId.Client].Ids[0][0];
var requestContent = new Dictionary <string, object>()
{
["baseId"] = baseId,
["mergedId"] = new List <int>()
{
mergeId
},
["mapping"] = Common.GetMapping(resourceId.ToPrivateApiResource(), (ulong)baseId, (ulong)clientId),
};
MergeManager mergeHandler = new MergeManager();
var response = mergeHandler.MergeRecords <MergeResponse>(resourceId.ToPrivateApiResource(), requestContent, System.Net.Http.HttpMethod.Post);
PrAssert.That(response, PrIs.ErrorResponse().With.HttpCode((int)HttpStatusCode.BadRequest));
PrAssert.That(response.Errors.Code, PrIs.EqualTo((int)ResultCode.InvalidParameter));
PrAssert.That(response.Errors.Error, PrIs.EqualTo("Invalid Parameter."));
}
public void MergeDeletedId(ResourceId resourceId, Parameters parameter)
{
var parameters = GetDefaultParameters(resourceId);
switch (parameter)
{
case Parameters.MergedId:
parameters["mergedId"] = PrepareDeletedIdsData.Data[resourceId].Ids[0][0].ToString();
break;
case Parameters.BaseId:
parameters["baseId"] = PrepareDeletedIdsData.Data[resourceId].Ids[0][0].ToString();
parameters["mergedId"] = PrepareDeletedIdsData.Data[resourceId].Ids[1][0].ToString();
parameters["mapping"] = GetMapping(resourceId, $"{resourceId.ToString()}.P_Id", PrepareDeletedIdsData.Data[resourceId].Ids[0][0].ToString());
break;
}
var mergeHandler = new MergeManager();
var response = mergeHandler.MergeRecords <MergeResponse>(resourceId, parameters, System.Net.Http.HttpMethod.Post);
PrAssert.That(response, PrIs.ErrorResponse().And.HttpCode(System.Net.HttpStatusCode.BadRequest));
}
public IActionResult Partition()
{
foreach (var file in Request.Form.Files)
{
if (file.Length == 0)
{
throw new ArgumentException();
}
string v_Path = $"Tmp{Path.DirectorySeparatorChar}{file.FileName}";
using (var flux = new FileStream(v_Path, FileMode.Create, FileAccess.Write))
{
file.CopyTo(flux);
}
MergeManager v_Manager = new MergeManager();
v_Manager.Merge(v_Path);
}
return(Ok());
}
//public ActionResult TopMerge()
//{
// HomeViewModels vm = new HomeViewModels();
// return View(vm);
//}
//[HttpPost]
//public ActionResult TopMerge(HomeViewModels vm, int? id1, int? id2)
//{
// MergeManager mgr = new MergeManager();
// if (id1 == null || id2 == null || id1 == id2)
// {
// vm.ErrorMessage = true;
// return View(vm);
// }
// vm.SuccessMessage = true;
// //mgr.MergedIds(vm.Selected_MCB);
// return View(vm);
//}
public void Merge(int id1, int id2)
{
MergeManager mgr = new MergeManager();
mgr.mergeAll(id1, id2);
}
public LocalFetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K,
V> scheduler, MergeManager <K, V> merger, Reporter reporter, ShuffleClientMetrics
metrics, ExceptionReporter exceptionReporter, SecretKey shuffleKey, IDictionary
<TaskAttemptID, MapOutputFile> localMapFiles)
: base(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, shuffleKey
)
{
this.job = job;
this.localMapFiles = localMapFiles;
SetName("localfetcher#" + id);
SetDaemon(true);
}
private void moveDownButton_Click(object sender, EventArgs e)
{
itemValues = MergeManager.MoveItemsDown(this, itemValues);
}
