/// <summary>
/// Initializes a new instance of the <see cref="MemberSettings"/> class.
/// </summary>
/// <param name="ignoredMembers">A collection of members to ignore. Can be null.</param>
/// <param name="ignoredTypes">A collection of types to ignore. Can be null.</param>
/// <param name="immutableTypes">A collection of types to treat as immutable. Can be null.</param>
/// <param name="comparers">A map of types with custom comparers. Can be null.</param>
/// <param name="copyers">A map of custom copy implementations for types. Can be null.</param>
/// <param name="referenceHandling">How reference values are handled.</param>
/// <param name="bindingFlags">The bindingflags used for getting members.</param>
protected MemberSettings(
IEnumerable<MemberInfo> ignoredMembers,
IEnumerable<Type> ignoredTypes,
IEnumerable<Type> immutableTypes,
IReadOnlyDictionary<Type, CastingComparer> comparers,
IReadOnlyDictionary<Type, CustomCopy> copyers,
ReferenceHandling referenceHandling,
BindingFlags bindingFlags)
{
this.ReferenceHandling = referenceHandling;
this.BindingFlags = bindingFlags;
if (ignoredMembers != null)
{
foreach (var ignoredMember in ignoredMembers)
{
this.ignoredMembers.TryAdd(ignoredMember, true);
}
}
this.ignoredTypes = IgnoredTypes.Create(ignoredTypes);
this.immutableTypes = ImmutableSet<Type>.Create(immutableTypes);
this.comparers = comparers;
this.copyers = copyers;
}
private void UpdateResourceMappings(IDictionary <string, CommonNodeLabelsManager.Host
> before, IDictionary <string, CommonNodeLabelsManager.Host> after)
{
// Get NMs in before only
ICollection <NodeId> allNMs = new HashSet <NodeId>();
foreach (KeyValuePair <string, CommonNodeLabelsManager.Host> entry in before)
{
Sharpen.Collections.AddAll(allNMs, entry.Value.nms.Keys);
}
foreach (KeyValuePair <string, CommonNodeLabelsManager.Host> entry_1 in after)
{
Sharpen.Collections.AddAll(allNMs, entry_1.Value.nms.Keys);
}
// Map used to notify RM
IDictionary <NodeId, ICollection <string> > newNodeToLabelsMap = new Dictionary <NodeId
, ICollection <string> >();
// traverse all nms
foreach (NodeId nodeId in allNMs)
{
CommonNodeLabelsManager.Node oldNM;
if ((oldNM = GetNMInNodeSet(nodeId, before, true)) != null)
{
ICollection <string> oldLabels = GetLabelsByNode(nodeId, before);
// no label in the past
if (oldLabels.IsEmpty())
{
// update labels
NodeLabel label = labelCollections[NoLabel];
label.RemoveNode(oldNM.resource);
// update queues, all queue can access this node
foreach (RMNodeLabelsManager.Queue q in queueCollections.Values)
{
Resources.SubtractFrom(q.resource, oldNM.resource);
}
}
else
{
// update labels
foreach (string labelName in oldLabels)
{
NodeLabel label = labelCollections[labelName];
if (null == label)
{
continue;
}
label.RemoveNode(oldNM.resource);
}
// update queues, only queue can access this node will be subtract
foreach (RMNodeLabelsManager.Queue q in queueCollections.Values)
{
if (IsNodeUsableByQueue(oldLabels, q))
{
Resources.SubtractFrom(q.resource, oldNM.resource);
}
}
}
}
CommonNodeLabelsManager.Node newNM;
if ((newNM = GetNMInNodeSet(nodeId, after, true)) != null)
{
ICollection <string> newLabels = GetLabelsByNode(nodeId, after);
newNodeToLabelsMap[nodeId] = ImmutableSet.CopyOf(newLabels);
// no label in the past
if (newLabels.IsEmpty())
{
// update labels
NodeLabel label = labelCollections[NoLabel];
label.AddNode(newNM.resource);
// update queues, all queue can access this node
foreach (RMNodeLabelsManager.Queue q in queueCollections.Values)
{
Resources.AddTo(q.resource, newNM.resource);
}
}
else
{
// update labels
foreach (string labelName in newLabels)
{
NodeLabel label = labelCollections[labelName];
label.AddNode(newNM.resource);
}
// update queues, only queue can access this node will be subtract
foreach (RMNodeLabelsManager.Queue q in queueCollections.Values)
{
if (IsNodeUsableByQueue(newLabels, q))
{
Resources.AddTo(q.resource, newNM.resource);
}
}
}
}
}
// Notify RM
if (rmContext != null && rmContext.GetDispatcher() != null)
{
rmContext.GetDispatcher().GetEventHandler().Handle(new NodeLabelsUpdateSchedulerEvent
(newNodeToLabelsMap));
}
}
/*
* Following methods are used for setting if a node is up and running, and it
* will update running nodes resource
*/
public virtual void ActivateNode(NodeId nodeId, Resource resource)
{
try
{
writeLock.Lock();
// save if we have a node before
IDictionary <string, CommonNodeLabelsManager.Host> before = CloneNodeMap(ImmutableSet
.Of(nodeId));
CreateHostIfNonExisted(nodeId.GetHost());
try
{
CreateNodeIfNonExisted(nodeId);
}
catch (IOException)
{
Log.Error("This shouldn't happen, cannot get host in nodeCollection" + " associated to the node being activated"
);
return;
}
CommonNodeLabelsManager.Node nm = GetNMInNodeSet(nodeId);
nm.resource = resource;
nm.running = true;
// Add node in labelsCollection
ICollection <string> labelsForNode = GetLabelsByNode(nodeId);
if (labelsForNode != null)
{
foreach (string label in labelsForNode)
{
NodeLabel labelInfo = labelCollections[label];
if (labelInfo != null)
{
labelInfo.AddNodeId(nodeId);
}
}
}
// get the node after edition
IDictionary <string, CommonNodeLabelsManager.Host> after = CloneNodeMap(ImmutableSet
.Of(nodeId));
UpdateResourceMappings(before, after);
}
finally
{
writeLock.Unlock();
}
}
//-------------------------------------------------------------------------
public virtual void test_allCurrencies()
{
Swap test = Swap.of(MOCK_GBP1, MOCK_USD1);
assertEquals(test.allCurrencies(), ImmutableSet.of(GBP, USD, EUR));
}
public virtual void TestStorageRestore()
{
int numDatanodes = 0;
cluster = new MiniDFSCluster.Builder(config).NumDataNodes(numDatanodes).ManageNameDfsDirs
(false).Build();
cluster.WaitActive();
SecondaryNameNode secondary = new SecondaryNameNode(config);
System.Console.Out.WriteLine("****testStorageRestore: Cluster and SNN started");
PrintStorages(cluster.GetNameNode().GetFSImage());
FileSystem fs = cluster.GetFileSystem();
Path path = new Path("/", "test");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path));
System.Console.Out.WriteLine("****testStorageRestore: dir 'test' created, invalidating storage..."
);
InvalidateStorage(cluster.GetNameNode().GetFSImage(), ImmutableSet.Of(path2, path3
));
PrintStorages(cluster.GetNameNode().GetFSImage());
System.Console.Out.WriteLine("****testStorageRestore: storage invalidated");
path = new Path("/", "test1");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path));
System.Console.Out.WriteLine("****testStorageRestore: dir 'test1' created");
// We did another edit, so the still-active directory at 'path1'
// should now differ from the others
FSImageTestUtil.AssertFileContentsDifferent(2, new FilePath(path1, "current/" + NNStorage.GetInProgressEditsFileName
(1)), new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName(1)),
new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(1)));
FSImageTestUtil.AssertFileContentsSame(new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName
(1)), new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(1)));
System.Console.Out.WriteLine("****testStorageRestore: checkfiles(false) run");
secondary.DoCheckpoint();
///should enable storage..
// We should have a checkpoint through txid 4 in the two image dirs
// (txid=4 for BEGIN, mkdir, mkdir, END)
FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetImageFileName
(4)), new FilePath(path2, "current/" + NNStorage.GetImageFileName(4)));
NUnit.Framework.Assert.IsFalse("Should not have any image in an edits-only directory"
, new FilePath(path3, "current/" + NNStorage.GetImageFileName(4)).Exists());
// Should have finalized logs in the directory that didn't fail
NUnit.Framework.Assert.IsTrue("Should have finalized logs in the directory that didn't fail"
, new FilePath(path1, "current/" + NNStorage.GetFinalizedEditsFileName(1, 4)).Exists
());
// Should not have finalized logs in the failed directories
NUnit.Framework.Assert.IsFalse("Should not have finalized logs in the failed directories"
, new FilePath(path2, "current/" + NNStorage.GetFinalizedEditsFileName(1, 4)).Exists
());
NUnit.Framework.Assert.IsFalse("Should not have finalized logs in the failed directories"
, new FilePath(path3, "current/" + NNStorage.GetFinalizedEditsFileName(1, 4)).Exists
());
// The new log segment should be in all of the directories.
FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetInProgressEditsFileName
(5)), new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName(5)),
new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(5)));
string md5BeforeEdit = FSImageTestUtil.GetFileMD5(new FilePath(path1, "current/"
+ NNStorage.GetInProgressEditsFileName(5)));
// The original image should still be the previously failed image
// directory after it got restored, since it's still useful for
// a recovery!
FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetImageFileName
(0)), new FilePath(path2, "current/" + NNStorage.GetImageFileName(0)));
// Do another edit to verify that all the logs are active.
path = new Path("/", "test2");
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(path));
// Logs should be changed by the edit.
string md5AfterEdit = FSImageTestUtil.GetFileMD5(new FilePath(path1, "current/" +
NNStorage.GetInProgressEditsFileName(5)));
NUnit.Framework.Assert.IsFalse(md5BeforeEdit.Equals(md5AfterEdit));
// And all logs should be changed.
FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetInProgressEditsFileName
(5)), new FilePath(path2, "current/" + NNStorage.GetInProgressEditsFileName(5)),
new FilePath(path3, "current/" + NNStorage.GetInProgressEditsFileName(5)));
secondary.Shutdown();
cluster.Shutdown();
// All logs should be finalized by clean shutdown
FSImageTestUtil.AssertFileContentsSame(new FilePath(path1, "current/" + NNStorage.GetFinalizedEditsFileName
(5, 7)), new FilePath(path2, "current/" + NNStorage.GetFinalizedEditsFileName(5,
7)), new FilePath(path3, "current/" + NNStorage.GetFinalizedEditsFileName(5, 7))
);
}
//-------------------------------------------------------------------------
public virtual FunctionRequirements requirements(T target, ISet <Measure> measures, CalculationParameters parameters, ReferenceData refData)
{
// extract data from product
CapitalIndexedBond product = target.Product;
Currency currency = product.Currency;
SecurityId securityId = product.SecurityId;
LegalEntityId legalEntityId = product.LegalEntityId;
// use lookup to build requirements
RatesMarketDataLookup ratesLookup = parameters.getParameter(typeof(RatesMarketDataLookup));
FunctionRequirements ratesReqs = ratesLookup.requirements(ImmutableSet.of(), ImmutableSet.of(product.RateCalculation.Index));
LegalEntityDiscountingMarketDataLookup ledLookup = parameters.getParameter(typeof(LegalEntityDiscountingMarketDataLookup));
FunctionRequirements ledReqs = ledLookup.requirements(securityId, legalEntityId, currency);
return(ratesReqs.combinedWith(ledReqs));
}
//-------------------------------------------------------------------------
public ImmutableSet <Currency> allCurrencies()
{
return(ImmutableSet.of(currency));
}
private static bool InternalLessEqual(SymGraph <TFunc, TADomain> thisG, SymGraph <TFunc, TADomain> thatG,
out IImmutableMap <SymValue, LispList <SymValue> > forward,
out IImmutableMap <SymValue, SymValue> backward)
{
int updateSize;
SymGraph <TFunc, TADomain> commonTail = ComputeCommonTail(thisG, thatG, out updateSize);
if (thisG.IsImmutable)
{
thisG = thisG.Clone();
}
var workList = new WorkList <EqualityPair <TFunc, TADomain> > ();
workList.Add(new EqualityPair <TFunc, TADomain> (thisG.const_root, thatG.const_root));
IImmutableSet <SymValue> backwardManifested = ImmutableSet <SymValue> .Empty(SymValue.GetUniqueKey);
IImmutableMap <SymValue, SymValue> backwardMap = ImmutableIntKeyMap <SymValue, SymValue> .Empty(SymValue.GetUniqueKey);
IImmutableMap <SymValue, LispList <SymValue> > forwardMap = ImmutableIntKeyMap <SymValue, LispList <SymValue> > .Empty(SymValue.GetUniqueKey);
IImmutableMap <SymValue, int> triggers = ImmutableIntKeyMap <SymValue, int> .Empty(SymValue.GetUniqueKey);
while (!workList.IsEmpty())
{
EqualityPair <TFunc, TADomain> equalityPair = workList.Pull();
SymValue sv1 = equalityPair.Sv1;
SymValue sv2 = equalityPair.Sv2;
SymValue s;
if (VisitedBefore(sv2, backwardManifested, backwardMap, out s))
{
if (s != null && s == sv1)
{
continue;
}
if (DebugOptions.Debug)
{
Console.WriteLine("---LessEqual fails due to pre-existing relation: {0} <- {1}", s, sv2);
}
forward = null;
backward = null;
return(false);
}
TADomain val1 = sv1 == null?thisG.UnderlyingTopValue.ForManifestedField() : thisG [sv1];
TADomain val2 = thatG [sv2];
if (!val1.LessEqual(val2))
{
if (DebugOptions.Debug)
{
Console.WriteLine("---LessEqual fails due to abstract values: !({0} <= {1})", val1, val2);
}
forward = null;
backward = null;
return(false);
}
if (sv1 != null)
{
backwardMap = backwardMap.Add(sv2, sv1);
forwardMap = forwardMap.Add(sv1, forwardMap [sv1].Cons(sv2));
}
else
{
backwardManifested = backwardManifested.Add(sv2);
}
if (thisG.HasAllBottomFields(sv1))
{
continue;
}
if (thatG.HasAllBottomFields(sv2))
{
if (DebugOptions.Debug)
{
Console.WriteLine("---LessEqual fails due to bottom field difference");
}
forward = null;
backward = null;
return(false);
}
foreach (TFunc function in thatG.Functions(sv2))
{
SymValue v1 = thisG [function, sv1];
SymValue v2 = thatG [function, sv2];
if (DebugOptions.Debug)
{
Console.WriteLine(" {0}-{1}->{2} <=? {3}-{4}->{5}", sv1, function, v1, sv2, function, v2);
}
workList.Add(new EqualityPair <TFunc, TADomain> (v1, v2));
}
foreach (var e in thatG.MultiEdges(sv2))
{
foreach (SymValue sv in thatG.MultiEdgeMap[sv2, e].AsEnumerable())
{
if (!UpdateTrigger(sv, e, ref triggers))
{
continue;
}
SymGraphTerm <TFunc> term = thatG.EqualMultiTermsMap [sv];
var args = new SymValue[term.Args.Length];
for (int i = 0; i < args.Length; i++)
{
args [i] = backwardMap [term.Args [i]];
}
SymValue v1 = thisG.LookupWithoutManifesting(args, e.Function);
if (v1 == null)
{
if (DebugOptions.Debug)
{
Console.WriteLine("---LessEqual fails due to missing multi term {0}({1})",
e.Function,
string.Join(", ", term.Args.Select(it => it.ToString())));
}
forward = null;
backward = null;
return(false);
}
workList.Add(new EqualityPair <TFunc, TADomain> (v1, sv));
}
}
}
forward = forwardMap;
backward = CompleteWithCommon(backwardMap, thisG, commonTail.IdGenerator);
return(true);
}
public virtual void TestZlibCompressorDecompressor()
{
try
{
int Size = 44 * 1024;
byte[] rawData = Generate(Size);
CompressDecompressTester.Of(rawData).WithCompressDecompressPair(new ZlibCompressor
(), new ZlibDecompressor()).WithTestCases(ImmutableSet.Of(CompressDecompressTester.CompressionTestStrategy
.CompressDecompressSingleBlock, CompressDecompressTester.CompressionTestStrategy
.CompressDecompressBlock, CompressDecompressTester.CompressionTestStrategy.CompressDecompressErrors
, CompressDecompressTester.CompressionTestStrategy.CompressDecompressWithEmptyStream
)).Test();
}
catch (Exception ex)
{
NUnit.Framework.Assert.Fail("testCompressorDecompressor error !!!" + ex);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerAllocateWithDefaultQueueLabels()
{
// This test is pretty much similar to testContainerAllocateWithLabel.
// Difference is, this test doesn't specify label expression in ResourceRequest,
// instead, it uses default queue label expression
// set node -> label
mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId
.NewInstance("h2", 0), ToSet("y")));
// inject node label manager
MockRM rm1 = new _MockRM_714(this, GetConfigurationWithDefaultQueueLabels(conf));
rm1.GetRMContext().SetNodeLabelManager(mgr);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("h1:1234", 8000);
// label = x
MockNM nm2 = rm1.RegisterNode("h2:1234", 8000);
// label = y
MockNM nm3 = rm1.RegisterNode("h3:1234", 8000);
// label = <empty>
ContainerId containerId;
// launch an app to queue a1 (label = x), and check all container will
// be allocated in h1
RMApp app1 = rm1.SubmitApp(200, "app", "user", null, "a1");
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
// request a container.
am1.Allocate("*", 1024, 1, new AList <ContainerId>());
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am1.GetApplicationAttemptId(), containerId, rm1, "h1");
// launch an app to queue b1 (label = y), and check all container will
// be allocated in h2
RMApp app2 = rm1.SubmitApp(200, "app", "user", null, "b1");
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm2);
// request a container.
am2.Allocate("*", 1024, 1, new AList <ContainerId>());
containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am2.GetApplicationAttemptId(), containerId, rm1, "h2");
// launch an app to queue c1 (label = ""), and check all container will
// be allocated in h3
RMApp app3 = rm1.SubmitApp(200, "app", "user", null, "c1");
MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm3);
// request a container.
am3.Allocate("*", 1024, 1, new AList <ContainerId>());
containerId = ContainerId.NewContainerId(am3.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am3.GetApplicationAttemptId(), containerId, rm1, "h3");
rm1.Close();
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerAllocateWithComplexLabels()
{
/*
* Queue structure:
* root (*)
* ________________
* / \
* a x(100%), y(50%) b y(50%), z(100%)
* ________________ ______________
* / / \
* a1 (x,y) b1(no) b2(y,z)
* 100% y = 100%, z = 100%
*
* Node structure:
* h1 : x
* h2 : y
* h3 : y
* h4 : z
* h5 : NO
*
* Total resource:
* x: 4G
* y: 6G
* z: 2G
* *: 2G
*
* Resource of
* a1: x=4G, y=3G, NO=0.2G
* b1: NO=0.9G (max=1G)
* b2: y=3, z=2G, NO=0.9G (max=1G)
*
* Each node can only allocate two containers
*/
// set node -> label
mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y", "z"));
mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId
.NewInstance("h2", 0), ToSet("y"), NodeId.NewInstance("h3", 0), ToSet("y"), NodeId
.NewInstance("h4", 0), ToSet("z"), NodeId.NewInstance("h5", 0), RMNodeLabelsManager
.EmptyStringSet));
// inject node label manager
MockRM rm1 = new _MockRM_557(this, GetComplexConfigurationWithQueueLabels(conf));
rm1.GetRMContext().SetNodeLabelManager(mgr);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("h1:1234", 2048);
MockNM nm2 = rm1.RegisterNode("h2:1234", 2048);
MockNM nm3 = rm1.RegisterNode("h3:1234", 2048);
MockNM nm4 = rm1.RegisterNode("h4:1234", 2048);
MockNM nm5 = rm1.RegisterNode("h5:1234", 2048);
ContainerId containerId;
// launch an app to queue a1 (label = x), and check all container will
// be allocated in h1
RMApp app1 = rm1.SubmitApp(1024, "app", "user", null, "a1");
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
// request a container (label = y). can be allocated on nm2
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "y");
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 2L);
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am1.GetApplicationAttemptId(), containerId, rm1, "h2");
// launch an app to queue b1 (label = y), and check all container will
// be allocated in h5
RMApp app2 = rm1.SubmitApp(1024, "app", "user", null, "b1");
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm5);
// request a container for AM, will succeed
// and now b1's queue capacity will be used, cannot allocate more containers
// (Maximum capacity reached)
am2.Allocate("*", 1024, 1, new AList <ContainerId>());
containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm4, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm5, containerId, RMContainerState
.Allocated, 10 * 1000));
// launch an app to queue b2
RMApp app3 = rm1.SubmitApp(1024, "app", "user", null, "b2");
MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm5);
// request a container. try to allocate on nm1 (label = x) and nm3 (label =
// y,z). Will successfully allocate on nm3
am3.Allocate("*", 1024, 1, new AList <ContainerId>(), "y");
containerId = ContainerId.NewContainerId(am3.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am3.GetApplicationAttemptId(), containerId, rm1, "h3");
// try to allocate container (request label = z) on nm4 (label = y,z).
// Will successfully allocate on nm4 only.
am3.Allocate("*", 1024, 1, new AList <ContainerId>(), "z");
containerId = ContainerId.NewContainerId(am3.GetApplicationAttemptId(), 3L);
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm4, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am3.GetApplicationAttemptId(), containerId, rm1, "h4");
rm1.Close();
}
/// <summary>
/// Returns a set containing any Ibor indices in the arguments.
/// </summary>
private static ISet <Index> indices(Index index, params Index[] otherIndices)
{
// The type parameter is needed for the benefit of the Eclipse compiler
return(ImmutableSet.builder <Index>().add(index).add(otherIndices).build());
}
/// <summary>
/// Adds a curve to the curve group definition which is used to provide discount rates and forward rates.
/// <para>
/// A curve added with this method cannot be calibrated by the market data system as it does not include
/// a curve definition. It is intended to be used with curves which are supplied by the user.
///
/// </para>
/// </summary>
/// <param name="curveName"> the name of the curve </param>
/// <param name="currency"> the currency for which the curve provides discount rates </param>
/// <param name="index"> the index for which the curve provides forward rates </param>
/// <param name="otherIndices"> the additional indices for which the curve provides forward rates </param>
/// <returns> this builder </returns>
public RatesCurveGroupDefinitionBuilder addCurve(CurveName curveName, Currency currency, RateIndex index, params RateIndex[] otherIndices)
{
RatesCurveGroupEntry entry = RatesCurveGroupEntry.builder().curveName(curveName).discountCurrencies(ImmutableSet.of(currency)).indices(indices(index, otherIndices)).build();
return(mergeEntry(entry));
}
//-------------------------------------------------------------------------
/// <summary>
/// Adds the definition of a curve to the curve group definition which is used to provide
/// discount rates and forward rates.
/// </summary>
/// <param name="curveDefinition"> the definition of the forward curve </param>
/// <param name="currency"> the currency for which the curve provides discount rates </param>
/// <param name="index"> the index for which the curve provides forward rates </param>
/// <param name="otherIndices"> the additional indices for which the curve provides forward rates </param>
/// <returns> this builder </returns>
public RatesCurveGroupDefinitionBuilder addCurve(CurveDefinition curveDefinition, Currency currency, RateIndex index, params RateIndex[] otherIndices)
{
ArgChecker.notNull(curveDefinition, "curveDefinition");
ArgChecker.notNull(currency, "currency");
ArgChecker.notNull(index, "index");
RatesCurveGroupEntry entry = RatesCurveGroupEntry.builder().curveName(curveDefinition.Name).discountCurrencies(ImmutableSet.of(currency)).indices(indices(index, otherIndices)).build();
return(merge(entry, curveDefinition));
}
/// <summary>
/// Adds the definition of a discount curve to the curve group definition.
/// <para>
/// A curve added with this method cannot be calibrated by the market data system as it does not include
/// a curve definition. It is intended to be used with curves which are supplied by the user.
///
/// </para>
/// </summary>
/// <param name="curveName"> the name of the curve </param>
/// <param name="otherCurrencies"> additional currencies for which the curve can provide discount factors </param>
/// <param name="currency"> the currency for which the curve provides discount rates </param>
/// <returns> this builder </returns>
public RatesCurveGroupDefinitionBuilder addDiscountCurve(CurveName curveName, Currency currency, params Currency[] otherCurrencies)
{
ArgChecker.notNull(curveName, "curveName");
ArgChecker.notNull(currency, "currency");
RatesCurveGroupEntry entry = RatesCurveGroupEntry.builder().curveName(curveName).discountCurrencies(ImmutableSet.copyOf(Lists.asList(currency, otherCurrencies))).build();
return(mergeEntry(entry));
}
public virtual void TestNameEditsConfigs()
{
Path file1 = new Path("TestNameEditsConfigs1");
Path file2 = new Path("TestNameEditsConfigs2");
Path file3 = new Path("TestNameEditsConfigs3");
MiniDFSCluster cluster = null;
SecondaryNameNode secondary = null;
Configuration conf = null;
FileSystem fileSys = null;
FilePath newNameDir = new FilePath(base_dir, "name");
FilePath newEditsDir = new FilePath(base_dir, "edits");
FilePath nameAndEdits = new FilePath(base_dir, "name_and_edits");
FilePath checkpointNameDir = new FilePath(base_dir, "secondname");
FilePath checkpointEditsDir = new FilePath(base_dir, "secondedits");
FilePath checkpointNameAndEdits = new FilePath(base_dir, "second_name_and_edits");
ImmutableList <FilePath> allCurrentDirs = ImmutableList.Of(new FilePath(nameAndEdits
, "current"), new FilePath(newNameDir, "current"), new FilePath(newEditsDir, "current"
), new FilePath(checkpointNameAndEdits, "current"), new FilePath(checkpointNameDir
, "current"), new FilePath(checkpointEditsDir, "current"));
ImmutableList <FilePath> imageCurrentDirs = ImmutableList.Of(new FilePath(nameAndEdits
, "current"), new FilePath(newNameDir, "current"), new FilePath(checkpointNameAndEdits
, "current"), new FilePath(checkpointNameDir, "current"));
// Start namenode with same dfs.namenode.name.dir and dfs.namenode.edits.dir
conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameAndEdits.GetPath());
conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, nameAndEdits.GetPath());
conf.Set(DFSConfigKeys.DfsNamenodeCheckpointDirKey, checkpointNameAndEdits.GetPath
());
conf.Set(DFSConfigKeys.DfsNamenodeCheckpointEditsDirKey, checkpointNameAndEdits.GetPath
());
replication = (short)conf.GetInt(DFSConfigKeys.DfsReplicationKey, 3);
// Manage our own dfs directories
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).ManageNameDfsDirs
(false).Build();
cluster.WaitActive();
secondary = StartSecondaryNameNode(conf);
fileSys = cluster.GetFileSystem();
try
{
NUnit.Framework.Assert.IsTrue(!fileSys.Exists(file1));
WriteFile(fileSys, file1, replication);
CheckFile(fileSys, file1, replication);
secondary.DoCheckpoint();
}
finally
{
fileSys.Close();
cluster.Shutdown();
secondary.Shutdown();
}
// Start namenode with additional dfs.namenode.name.dir and dfs.namenode.edits.dir
conf = new HdfsConfiguration();
NUnit.Framework.Assert.IsTrue(newNameDir.Mkdir());
NUnit.Framework.Assert.IsTrue(newEditsDir.Mkdir());
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameAndEdits.GetPath() + "," + newNameDir
.GetPath());
conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, nameAndEdits.GetPath() + "," + newEditsDir
.GetPath());
conf.Set(DFSConfigKeys.DfsNamenodeCheckpointDirKey, checkpointNameDir.GetPath() +
"," + checkpointNameAndEdits.GetPath());
conf.Set(DFSConfigKeys.DfsNamenodeCheckpointEditsDirKey, checkpointEditsDir.GetPath
() + "," + checkpointNameAndEdits.GetPath());
replication = (short)conf.GetInt(DFSConfigKeys.DfsReplicationKey, 3);
// Manage our own dfs directories. Do not format.
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Format(false
).ManageNameDfsDirs(false).Build();
cluster.WaitActive();
secondary = StartSecondaryNameNode(conf);
fileSys = cluster.GetFileSystem();
try
{
NUnit.Framework.Assert.IsTrue(fileSys.Exists(file1));
CheckFile(fileSys, file1, replication);
CleanupFile(fileSys, file1);
WriteFile(fileSys, file2, replication);
CheckFile(fileSys, file2, replication);
secondary.DoCheckpoint();
}
finally
{
fileSys.Close();
cluster.Shutdown();
secondary.Shutdown();
}
FSImageTestUtil.AssertParallelFilesAreIdentical(allCurrentDirs, ImmutableSet.Of("VERSION"
));
FSImageTestUtil.AssertSameNewestImage(imageCurrentDirs);
// Now remove common directory both have and start namenode with
// separate name and edits dirs
conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, newNameDir.GetPath());
conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, newEditsDir.GetPath());
conf.Set(DFSConfigKeys.DfsNamenodeCheckpointDirKey, checkpointNameDir.GetPath());
conf.Set(DFSConfigKeys.DfsNamenodeCheckpointEditsDirKey, checkpointEditsDir.GetPath
());
replication = (short)conf.GetInt(DFSConfigKeys.DfsReplicationKey, 3);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Format(false
).ManageNameDfsDirs(false).Build();
cluster.WaitActive();
secondary = StartSecondaryNameNode(conf);
fileSys = cluster.GetFileSystem();
try
{
NUnit.Framework.Assert.IsTrue(!fileSys.Exists(file1));
NUnit.Framework.Assert.IsTrue(fileSys.Exists(file2));
CheckFile(fileSys, file2, replication);
CleanupFile(fileSys, file2);
WriteFile(fileSys, file3, replication);
CheckFile(fileSys, file3, replication);
secondary.DoCheckpoint();
}
finally
{
fileSys.Close();
cluster.Shutdown();
secondary.Shutdown();
}
// No edit logs in new name dir
CheckImageAndEditsFilesExistence(newNameDir, true, false);
CheckImageAndEditsFilesExistence(newEditsDir, false, true);
CheckImageAndEditsFilesExistence(checkpointNameDir, true, false);
CheckImageAndEditsFilesExistence(checkpointEditsDir, false, true);
// Add old name_and_edits dir. File system should not read image or edits
// from old dir
NUnit.Framework.Assert.IsTrue(FileUtil.FullyDelete(new FilePath(nameAndEdits, "current"
)));
NUnit.Framework.Assert.IsTrue(FileUtil.FullyDelete(new FilePath(checkpointNameAndEdits
, "current")));
conf = new HdfsConfiguration();
conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, nameAndEdits.GetPath() + "," + newNameDir
.GetPath());
conf.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, nameAndEdits + "," + newEditsDir.GetPath
());
conf.Set(DFSConfigKeys.DfsNamenodeCheckpointDirKey, checkpointNameDir.GetPath() +
"," + checkpointNameAndEdits.GetPath());
conf.Set(DFSConfigKeys.DfsNamenodeCheckpointEditsDirKey, checkpointEditsDir.GetPath
() + "," + checkpointNameAndEdits.GetPath());
replication = (short)conf.GetInt(DFSConfigKeys.DfsReplicationKey, 3);
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDataNodes).Format(false
).ManageNameDfsDirs(false).Build();
cluster.WaitActive();
secondary = StartSecondaryNameNode(conf);
fileSys = cluster.GetFileSystem();
try
{
NUnit.Framework.Assert.IsTrue(!fileSys.Exists(file1));
NUnit.Framework.Assert.IsTrue(!fileSys.Exists(file2));
NUnit.Framework.Assert.IsTrue(fileSys.Exists(file3));
CheckFile(fileSys, file3, replication);
secondary.DoCheckpoint();
}
finally
{
fileSys.Close();
cluster.Shutdown();
secondary.Shutdown();
}
CheckImageAndEditsFilesExistence(nameAndEdits, true, true);
CheckImageAndEditsFilesExistence(checkpointNameAndEdits, true, true);
}
public virtual Org.Apache.Hadoop.IO.Compress.CompressDecompressTester <T, E> WithTestCases
(ImmutableSet <CompressDecompressTester.CompressionTestStrategy> stateges)
{
this.stateges = ImmutableSet.CopyOf(stateges);
return(this);
}
//-------------------------------------------------------------------------
/// <summary>
/// Returns the set of keys of this property set.
/// <para>
/// The iteration order of the map matches that of the input data.
///
/// </para>
/// </summary>
/// <returns> the set of keys </returns>
public ImmutableSet <string> keys()
{
return(ImmutableSet.copyOf(keyValueMap.Keys));
}
/// <summary>
/// Sets the {@code currencies} property in the builder
/// from an array of objects. </summary>
/// <param name="currencies"> the new value, not null </param>
/// <returns> this, for chaining, not null </returns>
public Builder currencies(params Currency[] currencies)
{
return(this.currencies(ImmutableSet.copyOf(currencies)));
}
public ISet <Measure> supportedMeasures()
{
return(ImmutableSet.of(measure));
}
//-------------------------------------------------------------------------
//JAVA TO C# CONVERTER WARNING: Java wildcard generics have no direct equivalent in .NET:
//ORIGINAL LINE: @Override public java.util.Set<? extends com.opengamma.strata.data.MarketDataId<?>> requirements()
public ISet <MarketDataId <object> > requirements()
{
return(ImmutableSet.of(fxRateId, spreadId));
}
//-------------------------------------------------------------------------
public ISet <ObservableId> requirements()
{
return(ImmutableSet.of(rateId));
}
/// <summary>
/// Gets the supported tokens on the given object.
/// </summary>
/// <param name="object"> the object for which to return the valid tokens </param>
/// <returns> the tokens </returns>
public static ISet <string> tokens(object @object)
{
return(getEvaluator(@object.GetType()).map(evaluator => evaluator.tokens(@object)).orElse(ImmutableSet.of()));
}
private Stream <string> tokensForSensitivity(CurrencyParameterSensitivity sensitivity)
{
return(ImmutableSet.of(sensitivity.Currency.Code, sensitivity.MarketDataName.Name).Select(v => v.ToLower(Locale.ENGLISH)));
}
internal DerivedFn(Measure measure) : this(measure, ImmutableSet.of(CASH_FLOWS, PAR_RATE))
{
}
//-------------------------------------------------------------------------
public virtual void test_allIndices()
{
Swap test = Swap.of(MOCK_GBP1, MOCK_USD1);
assertEquals(test.allIndices(), ImmutableSet.of(IborIndices.GBP_LIBOR_3M, FxIndices.EUR_GBP_ECB, OvernightIndices.EUR_EONIA));
}
public override ISet <string> tokens(CurrencyParameterSensitivity sensitivity)
{
return(ImmutableSet.of(sensitivity.Currency.Code.ToLower(Locale.ENGLISH), sensitivity.MarketDataName.Name.ToLower(Locale.ENGLISH)));
}
/*
* Following methods are used for setting if a node unregistered to RM
*/
public virtual void DeactivateNode(NodeId nodeId)
{
try
{
writeLock.Lock();
// save if we have a node before
IDictionary <string, CommonNodeLabelsManager.Host> before = CloneNodeMap(ImmutableSet
.Of(nodeId));
CommonNodeLabelsManager.Node nm = GetNMInNodeSet(nodeId);
if (null != nm)
{
if (null == nm.labels)
{
// When node deactivated, remove the nm from node collection if no
// labels explicitly set for this particular nm
// Save labels first, we need to remove label->nodes relation later
ICollection <string> savedNodeLabels = GetLabelsOnNode(nodeId);
// Remove this node in nodes collection
Sharpen.Collections.Remove(nodeCollections[nodeId.GetHost()].nms, nodeId);
// Remove this node in labels->node
RemoveNodeFromLabels(nodeId, savedNodeLabels);
}
else
{
// set nm is not running, and its resource = 0
nm.running = false;
nm.resource = Resource.NewInstance(0, 0);
}
}
// get the node after edition
IDictionary <string, CommonNodeLabelsManager.Host> after = CloneNodeMap(ImmutableSet
.Of(nodeId));
UpdateResourceMappings(before, after);
}
finally
{
writeLock.Unlock();
}
}
//-------------------------------------------------------------------------
public virtual FunctionRequirements requirements(T target, ISet <Measure> measures, CalculationParameters parameters, ReferenceData refData)
{
// extract data from product
OvernightFuture product = target.Product;
QuoteId quoteId = QuoteId.of(target.Product.SecurityId.StandardId, FieldName.SETTLEMENT_PRICE);
OvernightIndex index = product.Index;
// use lookup to build requirements
RatesMarketDataLookup ratesLookup = parameters.getParameter(typeof(RatesMarketDataLookup));
FunctionRequirements ratesReqs = ratesLookup.requirements(ImmutableSet.of(), ImmutableSet.of(index));
//JAVA TO C# CONVERTER WARNING: Java wildcard generics have no direct equivalent in .NET:
//ORIGINAL LINE: com.google.common.collect.ImmutableSet<com.opengamma.strata.data.MarketDataId<?>> valueReqs = com.google.common.collect.ImmutableSet.builder<com.opengamma.strata.data.MarketDataId<?>>().add(quoteId).addAll(ratesReqs.getValueRequirements()).build();
ImmutableSet <MarketDataId <object> > valueReqs = ImmutableSet.builder <MarketDataId <object> >().add(quoteId).addAll(ratesReqs.ValueRequirements).build();
return(ratesReqs.toBuilder().valueRequirements(valueReqs).build());
}
/// <summary>
/// Returns the set of currencies referred to by the swap.
/// <para>
/// This returns the complete set of currencies for the swap, not just the payment currencies.
///
/// </para>
/// </summary>
/// <returns> the set of currencies referred to by this swap </returns>
public ImmutableSet <Currency> allCurrencies()
{
ImmutableSet.Builder <Currency> builder = ImmutableSet.builder();
legs.ForEach(leg => leg.collectCurrencies(builder));
return(builder.build());
}
/// <summary>
/// Test init a queue configuration, children's capacity for a given label
/// doesn't equals to 100%.
/// </summary>
/// <remarks>
/// Test init a queue configuration, children's capacity for a given label
/// doesn't equals to 100%. This expect IllegalArgumentException thrown.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void TestQueueParsingWithSumOfChildLabelCapacityNot100PercentWithWildCard
()
{
nodeLabelManager.AddToCluserNodeLabels(ImmutableSet.Of("red", "blue"));
YarnConfiguration conf = new YarnConfiguration();
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
SetupQueueConfigurationWithLabels(csConf);
csConf.SetCapacityByLabel(CapacitySchedulerConfiguration.Root + ".b.b3", "red", 24
);
csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root, ImmutableSet.
Of(RMNodeLabelsManager.Any));
csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root + ".b", ImmutableSet
.Of(RMNodeLabelsManager.Any));
CapacityScheduler capacityScheduler = new CapacityScheduler();
RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new
RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new
ClientToAMTokenSecretManagerInRM(), null);
rmContext.SetNodeLabelManager(nodeLabelManager);
capacityScheduler.SetConf(csConf);
capacityScheduler.SetRMContext(rmContext);
capacityScheduler.Init(csConf);
capacityScheduler.Start();
ServiceOperations.StopQuietly(capacityScheduler);
}
