public void TestSerializeMessageSet()
{
using (var serialized = new ReusableMemoryStream(null))
{
var set = new PartitionData
{
Partition = 42,
CompressionCodec = CompressionCodec.None,
Messages = new[]
{
new Message {
Key = Key, Value = Value
},
new Message {
Key = Key, Value = Value
}
}
};
set.Serialize(serialized, SerializationConfig.ByteArraySerializers);
serialized.Position = 0;
Assert.AreEqual(42, BigEndianConverter.ReadInt32(serialized)); // Partition
Assert.AreEqual(serialized.Length - 4 - 4, BigEndianConverter.ReadInt32(serialized)); // MessageSet size
Assert.AreEqual(0, BigEndianConverter.ReadInt64(serialized)); // First message offset
int firstMsgSize = BigEndianConverter.ReadInt32(serialized);
serialized.Position += firstMsgSize;
Assert.AreEqual(0, BigEndianConverter.ReadInt64(serialized)); // First message offset
int secondMsgSize = BigEndianConverter.ReadInt32(serialized);
serialized.Position += secondMsgSize;
Assert.AreEqual(serialized.Length, serialized.Position);
}
}
public void TestDeserializeMessageSet(CompressionCodec codec)
{
using (var serialized = Pool.Reserve())
{
var set = new PartitionData
{
Partition = 42,
CompressionCodec = codec,
Messages = new[]
{
new Message {
Key = Key, Value = Value
},
new Message {
Key = Key, Value = Value
}
}
};
set.Serialize(serialized, SerializationConfig.ByteArraySerializers);
serialized.Position = 4;
var deserialized = FetchPartitionResponse.DeserializeMessageSet(serialized, SerializationConfig.ByteArrayDeserializers);
Assert.AreEqual(2, deserialized.Count);
foreach (var msg in deserialized)
{
Assert.AreEqual(0, msg.Offset);
CollectionAssert.AreEqual(Key, msg.Message.Key as byte[]);
CollectionAssert.AreEqual(Value, msg.Message.Value as byte[]);
}
}
}
public PLTGenertorFactory(PartitionData PD, COLCollection _COLSet, SubSamplingAnalysisSetting subSamplingSettings, DateTime _start, TimeStyle _timestyle, LossStyle _lossstyle)
{
Start = _start;
End = _start.AddYears(6);
pData = new PartitionDataAdpator(PD, subSamplingSettings);
GeneratorFactory = new GUInputGeneratorFactory(pData, _COLSet, _timestyle, _lossstyle);
}
public PLTGenertorFactory(PartitionData PD, HashSet <string> _subperils, SubSamplingAnalysisSetting subSamplingSettings, DateTime _start, TimeStyle _timestyle, LossStyle _lossstyle)
{
Start = _start;
End = _start.AddYears(6);
pData = new PartitionDataAdpator(PD, subSamplingSettings);
GeneratorFactory = new GUInputGeneratorFactory(pData, _subperils, _timestyle, _lossstyle);
}
public void TestBadCompression()
{
using (var serialized = Pool.Reserve())
{
var set = new PartitionData
{
Partition = 42,
CompressionCodec = CompressionCodec.Gzip,
Messages = new[]
{
new Message {
Key = Key, Value = Value
}
}
};
set.Serialize(serialized, SerializationConfig.ByteArraySerializers);
// corrupt compressed data
serialized[37] = 8;
// recompute crc
var crc = (int)Crc32.Compute(serialized, 24, serialized.Length - 24);
serialized.Position = 20;
BigEndianConverter.Write(serialized, crc);
// go
serialized.Position = 4;
Assert.That(() => FetchPartitionResponse.DeserializeMessageSet(serialized, SerializationConfig.ByteArrayDeserializers), Throws.InstanceOf <UncompressException>());
}
}
/// <summary>
/// MANIFOLD use
/// </summary>
public FetchResponseWrapper FetchAndGetDetail(string clientId, string topic, int correlationId, int partitionId, long fetchOffset, int fetchSize
, int maxWaitTime, int minWaitSize)
{
FetchResponse response = this.Fetch(clientId,
topic,
correlationId,
partitionId,
fetchOffset,
fetchSize,
maxWaitTime,
minWaitSize);
if (response == null)
{
throw new KafkaConsumeException(string.Format("FetchRequest returned null FetchResponse,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, this.Config.Broker, topic, partitionId));
}
PartitionData partitionData = response.PartitionData(topic, partitionId);
if (partitionData == null)
{
throw new KafkaConsumeException(string.Format("PartitionData int FetchResponse is null,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, this.Config.Broker, topic, partitionId));
}
if (partitionData.Error != ErrorMapping.NoError)
{
string s = string.Format("Partition data in FetchResponse has error. {0} {1} fetchOffset={2},leader={3},topic={4},partition={5}"
, partitionData.Error, KafkaException.GetMessage(partitionData.Error)
, fetchOffset, this.Config.Broker, topic, partitionId);
Logger.Error(s);
throw new KafkaConsumeException(s, partitionData.Error);
}
return(new FetchResponseWrapper(partitionData.GetMessageAndOffsets(), response.Size, response.CorrelationId, topic, partitionId));
}
public PartitionDataAdpator(PartitionData _PData, SubSamplingAnalysisSetting _subSamplingSettings)
{
PData = _PData;
expData = new Dictionary <long, ExposureDataAdaptor>();
JSParser = new JavascriptParser(GetParserSettings());
SubSamplingSettings = _subSamplingSettings;
}
public GUInputGeneratorFactory(PartitionData PD, HashSet <string> _subperils, SubSamplingAnalysisSetting subSmaplingSettings, TimeStyle _timestyle, LossStyle _lossstyle)
{
PData = new PartitionDataAdpator(PD, subSmaplingSettings);
NeedDisposePData = true;
subperils = _subperils;
timestyle = _timestyle;
lossstyle = _lossstyle;
}
public GUInputGeneratorFactory(PartitionData PD, COLCollection _COLSet, SubSamplingAnalysisSetting subSmaplingSettings, TimeStyle _timestyle, LossStyle _lossstyle)
{
PData = new PartitionDataAdpator(PD, subSmaplingSettings);
NeedDisposePData = true;
subperils = _COLSet.GetSubperils();
timestyle = _timestyle;
lossstyle = _lossstyle;
}
static void Main(string[] args)
{
//string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases\NewRL131_RMS_EDM_FinMod_Automation_RL131_72514_PortID_120_1000002113_1\4248\2050305\635447355784670000\rites_batch0.dat";
//string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases\PerRisk_RMS_EDM_FinMod_Automation_RL131_72514_PortID_896_1000002164_1\4438\2142576\635447551688220000\rites_batch0.dat";
//string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases\PerRisk_RMS_EDM_FinMod_Automation_RL131_72514_PortID_1013_1000002151_1\4413\2142553\635447537560220000\rites_batch0.dat";
// string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases Overlap Max Ded\NewRL131_RMS_EDM_FinMod_Automation_RL131_BT4IT4Aug30thNGtoRLAndInvalidationScriptsRanOnThisEDM_PortID_923_1000001480_1\6611\11352686\635416896292050000\rites_batch0.dat";
//string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases Overlap Max Ded\PerRisk_RMS_EDM_FinMod_Automation_RL13_BT4IT4Aug30thNGtoRLAndInvalidationScriptsRanOnThisEDM_PortID_958_1000001574_1\7066\11353699\635417441336090000\rites_batch0.dat";
//string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases Overlap Max Ded\EDM_RLFM_RL13_BT4IT4Aug30thNGtoRLAndInvalidationScriptsRanOnThisEDM_PortID_132_1000001835_1\8170\11366277\635419079253350000\rites_batch0.dat";
//string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases Overlap Max Ded\RMS_RLFM_EDM2_ForPlatformQA_PortID_1_1000001611_1\7239\11360603\635417579575500000\rites_batch0.dat";
//string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases\PerRisk_RMS_EDM_FinMod_Automation_RL131_72514_PortID_1021_1000002108_1\4233\2050284\635447346491010000\rites_batch0.dat";
//string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases\MTH_200_rites_batch0.dat";
//string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases\rites_batch0_MTH1.dat";
//string filepath = @" C:\LocalNGFM\EDS Extracts\FromRain\Large Extracts\rites_batch62001_PerRisk.dat";
//string filepath = @"D:\Nina_Automation_Testing\NTA_EDS_Extract\PerRisk_RMS_EDM_FinMod_Automation_RL13_BT4IT4Aug30thNGtoRLAndInvalidationScriptsRanOnThisEDM_PortID_897_1000001508_1\6752\11353141\635417040157160000\rites_batch0.dat";
//string filepath = @"D:\Matrix_Prototype_Storage\EDS Extract\JPTY_Multibuilding_PerRisk.dat";
string filepath = @"D:\Nina_Automation_Testing\Treaties\NoHoursClause\No_Reinstatement_NoAgg\From_Slava\rites_batch0_FromSlava.dat";
//string filepath = @"C:\LocalNGFM\EDS Extracts\From Swapna\rites_batch260197453_swapna_profile_case_Updated.dat";
//string filepath1 = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\rites_batch1.dat";
//string filepath1 = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\rites_SingleBuilding_overlap_subperil.dat";
// string filepath1 = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\rites_FixedGraph1.dat";
//string filepath1 = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\rites_FixedGraphOverlap.dat";
//string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\rites_batch0.dat";
//string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\rites_batch0_SingleBuilding_overlap.dat";
//string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\rites_batch0_SingleBuilding.dat"; //step policy
//string filepath = @"C:\LocalNGFM\EDS Extracts\SunnyCreated\TreatyTestCaseWithPositions.dat";
//string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\Test1EQ_StepPGU_CDL.dat";
// string filepath = @"C:\LocalNGFM\EDS Extracts\SunnyCreated\TreatyTestCaseWithPositions.dat";
PartitionData pd = GetPDFromPath(filepath);
UpdateCDLFOrGUSublimits(pd, 0);
long conID = 11331;
//long conID = 42586728;
//long conID = 2050306;
//TestMatrixHDFM(pd, conID);
COLCollection COLSet = new COLCollection(new HashSet <string>()
{
"EQ"
}); //{ "EQ", "WS" });
//COLCollection COLSet = new COLCollection(new HashSet<string>() { "CSWT" }); //{ "EQ", "WS" });
//TestRITEMap(pd, conID);
//TestGULossVector(pd, conID);
//TestReference(GraphType.FixedGraph1, pd, (int)conID, COLSet); //CDL: 3
//TestReference(GraphType.FixedGraphOverlap, pd, (int)conID, COLSet); //CDL:6
//TestReference(GraphType.FixedGraphOverlapSubperil, pd, (int)conID, COLSet); //CDL: 7
//TestReference(GraphType.FixedGraphOverlapSubperil2, pd, (int)conID, COLSet); //CDL: 8
//TestReference(GraphType.StepPolicyGraph, pd, (int)conID, COLSet); //CDL: 10
TestReference(GraphType.Auto, pd, (int)conID, COLSet); //CDL: 10
//TestReferenceWithPeriods(GraphType.Auto, pd, (int)conID, COLSet); /
//TestReferenceSpeed(GraphType.Auto, pd, (int)conID, COLSet);
//testPerformance();
//TestExposureDataAdaptor(pd, (int)conID);
}
public ReferencePrototype(PartitionData PD, ModuleSettingsProvider _moduleSettings, JavascriptContext JSContext)
{
SetModelList(_moduleSettings);
RAPSettings _rapsettings = GetRAPSettings(_moduleSettings);
SubSamplingAnalysisSetting _subSamplingSettings = GetSubSamplingSettings(_moduleSettings);
InitilizePrototype(PD, _rapsettings, _subSamplingSettings, JSContext);
}
private static void UpdateEDSExtractWithCDL(string CDl, string filepath, int ConIndex)
{
PartitionData PD = Deserialize(filepath);
PD.Exposures[ConIndex].Contract.CDLString = CDl;
string newfilepath = @"C:\LocalNGFM\EDS Extracts\From Swapna\rites_batch260197453_swapna_profile_case_Updated.dat";
Serialize(PD, newfilepath);
}
public void Prepare(PartitionData partitionData, bool append = false, bool OnlyBuildCOM = false)
{
JavaScriptHarness.Construct(underscore_js, grammar_ast_js);
Initialize(partitionData, append);
Prepare(OnlyBuildCOM);
JavaScriptHarness.Destruct();
}
//RITE.ID -> numBldgs -> <timestamp -> list of loss>
public GUInputGenerator(PartitionData PD, long conID, COLCollection _COLSet, TimeStyle _timestyle, LossStyle _lossstyle)
{
PartitionDataAdpator PData = new PartitionDataAdpator(PD);
expData = PData.GetExposureAdaptor(conID);
COLSet = _COLSet;
timestyle = _timestyle;
lossstyle = _lossstyle;
// Dictionary<long, Dictionary<int, Tuple<int, List<float>>>> GULosses = new Dictionary<long, Dictionary<int, Tuple<int, List<float>>>>();
}
public async Task OpenConnection(string subscription, string resourceGroup, string ehNamespace, PartitionContext partitionContext)
{
var partitionData = new PartitionData()
{
Name = partitionContext.PartitionId,
Sequence = partitionContext.RuntimeInformation.LastSequenceNumber,
RetrievalTime = partitionContext.RuntimeInformation.RetrievalTime,
LastEnqueueTime = partitionContext.RuntimeInformation.LastEnqueuedTimeUtc
};
_ = _content.AddOrUpdate(ehNamespace, new EhNamespaceData()
{
Namespace = ehNamespace,
Subscription = subscription,
ResourceGroup = resourceGroup,
Ehs = new Dictionary <string, EhData>()
{
{ partitionContext.EventHubPath, new EhData()
{
Name = partitionContext.EventHubPath,
Partitions = new Dictionary <string, PartitionData>()
{
{ partitionContext.PartitionId, partitionData }
}
} }
}
}, (e, newValue) =>
{
var b = partitionData;
if (newValue.Ehs.ContainsKey(partitionContext.EventHubPath))
{
if (newValue.Ehs[partitionContext.EventHubPath].Partitions.ContainsKey(partitionContext.PartitionId))
{
newValue.Ehs[partitionContext.EventHubPath].Partitions[partitionContext.PartitionId] = partitionData;
}
else
{
newValue.Ehs[partitionContext.EventHubPath].Partitions.Add(partitionContext.PartitionId, partitionData);
}
}
else
{
newValue.Ehs.Add(partitionContext.EventHubPath, new EhData()
{
Name = partitionContext.EventHubPath,
Partitions = new Dictionary <string, PartitionData>()
{
{ partitionContext.PartitionId, partitionData }
}
});
}
return(newValue);
});
await hub.Clients.All.SendAsync("updateData", _content);
}
public PullResponse(PartitionData pData)
{
this.Partition = pData.Partition;
this.Error = pData.Error;
this.Payload = new List <byte[]>();
foreach (MessageAndOffset m in pData.MessageSet)
{
this.Payload.Add(m.Message.Payload);
}
}
private static void UpdateCDLFOrGUSublimits(PartitionData pd, int ConIndex)
{
Regex regex = new Regex(Regex.Escape("Declarations\r\n"));
string cdlString = pd.Exposures[ConIndex].Contract.CDLString;
//string NewCDL = regex.Replace(cdlString, "Declarations\r\n Claims Adjustment Options Are (SUBLIMITS ARE GROUND UP)\r\n", 1);
//string NewCDL = regex.Replace(cdlString, "Declarations\r\n Claims Adjustment Options Are (Deductibles are , Sublimits are Net of Deductible)\r\n", 1);
string NewCDL = regex.Replace(cdlString, "Declarations\r\n Claims Adjustment Options Are (Deductibles are absorbable,SUBLIMITS ARE GROUND UP)\r\n", 1);
//string NewCDL = regex.Replace(cdlString, "Declarations\r\n Claims Adjustment Options Are (Sublimits are Net of Deductible)\r\n", 1);
pd.Exposures[ConIndex].Contract.CDLString = NewCDL;
}
public PLTGenerator(PartitionData _PD, long _conID, int pid, COLCollection _COLSet, TimeStyle _timestyle, LossStyle _lossStyle, DateTime _start, DateTime _end)
{
PartitionDataAdpator PData = new PartitionDataAdpator(_PD);
expData = PData.GetExposureAdaptor(conID);
PD = _PD;
conID = _conID;
COLSet = _COLSet;
timestyle = _timestyle;
lossStyle = _lossStyle;
PLT = new PeriodLossTable(pid, _COLSet, _timestyle, _lossStyle, _start, _end);
}
public PullResponseExt(PartitionData pData)
{
this.Partition = pData.Partition;
this.Error = pData.Error;
this.OffsetsAndPayLoads = new List <Tuple <long, byte[]> >();
foreach (MessageAndOffset m in pData.MessageSet)
{
this.OffsetsAndPayLoads.Add(new Tuple <long, byte[]>(m.Message.Offset, m.Message.Payload));
}
}
private static void TestRITEMap(PartitionData pd, long conID)
{
//Default SubSampling Settings
SubSamplingAnalysisSetting subSamplingSettings = new SubSamplingAnalysisSetting(false, 1, 0, 250, "", "");
PartitionDataAdpator PDataAdap = new PartitionDataAdpator(pd, subSamplingSettings);
ExposureDataAdaptor expData = PDataAdap.GetExposureAdaptor(conID);
ISubPerilConfig subperilInfo = new RMSSubPerilConfig();
RITEmapper1 mapper = new RITEmapper1(expData, new RAPSettings(new HashSet <string> {
"EQ"
}), subperilInfo);
}
public VectorGUInputGeneratorFactory(PartitionData PD, HashSet <string> _subperils, TimeStyle _timestyle, LossStyle _lossstyle, bool UseAggregatedRITEs, SubSamplingAnalysisSetting subSamplingSettings)
{
subperils = _subperils;
bool subperilsSupported = subperils.All(sp => subperilConfig.GetSubPerilList().Contains(sp));
if (!subperilsSupported)
{
throw new InvalidOperationException("Subperils specified cannot be supported at this time");
}
pData = new PartitionDataAdpator(PD, subSamplingSettings);
GeneratorFactory = new GUInputGeneratorFactory(pData, subperils, _timestyle, _lossstyle);
}
public static void Serialize(PartitionData pData, string file)
{
using (var wc = new WebClient().OpenWrite(file))
{
try
{
ProtoBuf.Serializer.Serialize <PartitionData>(wc, pData);
}
catch (ProtoBuf.ProtoException p)
{
//Log.Fatal("Error Writing of protobuf to file" + p.Message);
Console.WriteLine("Error Writing of protobuf to file \"{0}\": {1}", file, p.Message);
}
}
}
public void Prepare(string[] ContractExposuresProtobufExtractFiles)
{
string directoryName = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
Directory.SetCurrentDirectory(directoryName);
_handleHDFM.Clear();
foreach (string file in Utilities.GetExtantFiles(ContractExposuresProtobufExtractFiles))
{
if (file.EndsWith(".dat"))
{
PartitionData partitionData = Utilities.DeserializePartitionData(file);
_handleHDFM.Append(partitionData);
}
}
}
private static void Write(MemoryStream stream, PartitionData partition)
{
BigEndianConverter.Write(stream, partition.Partition);
if (partition.CompressionType == CompressionType.None)
{
var messageArray = partition.Messages.ToArray(); // make sure linq evaluation happen just once
var messageSetSize = EstimateMessageSetSize(messageArray);
BigEndianConverter.Write(stream, messageSetSize);
Write(stream, partition.Messages);
}
else
{
var compressedMessage = Compress(partition.CompressionType, partition.Messages);
WriteCompressedMessageSet(stream, compressedMessage, partition.CompressionType);
}
}
public static PartitionData DeserializePartitionData(string[] files, CancellationTokenSource ct)
{
List <PartitionData> PDs = new List <PartitionData>();
foreach (var f in GetExtantFiles(files))
{
if (ct.IsCancellationRequested)
{
return(null);
}
var pd = DeserializePartitionData(f);
if (pd != null)
{
PDs.Add(pd);
}
}
return((PDs.Count > 0) ? PartitionData.Merge(PDs) : null);
}
public static PartitionData Deserialize(string file)
{
PartitionData result = null;
using (var wc = new WebClient().OpenRead(file))
{
try
{
result = ProtoBuf.Serializer.Deserialize <PartitionData>(wc);
}
catch (ProtoBuf.ProtoException p)
{
//Log.Fatal("Error Reading a protobuf file" + p.Message);
Console.WriteLine("Error Reading a protobuf file \"{0}\": {1}", file, p.Message);
}
}
return(result);
}
private void Initialize(PartitionData partitionData, bool append = false)
{
Initialize(append);
#region VECTORIZED REFERENCE PROTOTYPE INTEGRATION
_ReferencePrototype = null;
UseReference = Convert.ToBoolean(ConfigurationManager.AppSettings["UseReference"]);
if (UseReference)
{
//Sunny hack... need to change to read from Model Setings Provider from the DLM..
RAPSettings RapSettings = new RAPSettings(new HashSet <string> {
"WS", "WI", "WA"
});
SubSamplingAnalysisSetting subSamplingSetiings = new SubSamplingAnalysisSetting(false, 1, 0, 250, "", "");
if (RapSettings != null || subSamplingSetiings != null)
{
_ReferencePrototype = new ReferencePrototype(partitionData, RapSettings, subSamplingSetiings);
}
else
{
throw new InvalidOperationException("Cannot run in Array Mode and have null RapSettings or null SubSampling Settings!");
}
}
#endregion
if (null != partitionData && null != partitionData.Exposures)
{
foreach (ContractExposure ce in partitionData.Exposures)
{
if ((ce.ContractType != null) && ce.ContractType.IsReinsuranceContract())
{ //Treaty Contract
ContractExposureDataIndex.TryAdd(ce.ExposureID, new TreatyContractExposureData(JavaScriptHarness, false, ce));
}
else if (null != ce.ContractSubjectExposures && null != ce.Subschedules)
{ // Primary Contract
ContractExposureDataIndex.TryAdd(ce.ExposureID, new PrimaryContractExposureData(JavaScriptHarness, false, ce, "", _ReferencePrototype));
}
}
}
}
public void TestUnsupportedMagicByte()
{
using (var serialized = new ReusableMemoryStream(null))
{
var set = new PartitionData
{
Partition = 42,
CompressionCodec = CompressionCodec.None,
Messages = new[]
{
new Message {
Key = Key, Value = Value
}
}
};
set.Serialize(serialized, SerializationConfig.ByteArraySerializers);
serialized[24] = 8; // set non supported magic byte
serialized.Position = 4;
Assert.That(() => FetchPartitionResponse.DeserializeMessageSet(serialized, SerializationConfig.ByteArrayDeserializers), Throws.InstanceOf <UnsupportedMagicByteVersion>());
}
}
public void TestBadCrc()
{
using (var serialized = new ReusableMemoryStream(null))
{
var set = new PartitionData
{
Partition = 42,
CompressionCodec = CompressionCodec.None,
Messages = new[]
{
new Message {
Key = Key, Value = Value
}
}
};
set.Serialize(serialized, SerializationConfig.ByteArraySerializers);
serialized[20] = 8; // change crc
serialized.Position = 4;
Assert.That(() => FetchPartitionResponse.DeserializeMessageSet(serialized, SerializationConfig.ByteArrayDeserializers), Throws.InstanceOf <CrcException>());
}
}
// Gets the table column names and rows, removing data from the table as it is returned.
public PartitionData Flush(string[] distinctValues)
{
// Put the columns in order of { Key0, Key1, ..., KeyN, Col1, Col2, ..., ColN } where the key columns are in the order
// specified, and the non-key columns follow in alphabetical order
_sortedColumns = new KeyValuePair <string, LinkedList <ColumnSegment> > [_columns.Count];
for (int i = 0; i < _keyNames.Length; ++i)
{
_sortedColumns[i] = new KeyValuePair <string, LinkedList <ColumnSegment> >(_keyNames[i], _columns[_keyNames[i]]);
_columns.Remove(_keyNames[i]);
}
Array.Copy(_columns.ToArray(), 0, _sortedColumns, _keyNames.Length, _sortedColumns.Length - _keyNames.Length);
_columns.Clear(); // No longer need
var sortingFunc = Comparer <KeyValuePair <string, LinkedList <ColumnSegment> > > .Create((x, y) => x.Key.CompareTo(y.Key));
Array.Sort(_sortedColumns, _keyNames.Length, _sortedColumns.Length - _keyNames.Length, sortingFunc); // Sort non-key columns
var result = new PartitionData {
ColumnNames = _sortedColumns.Select(col => col.Key).ToArray(),
Rows = get_rows(distinctValues)
};
return(result);
}
// 既存バージョン情報から作成したデータから、比較用ハッシュ作成
static private string CreateHashStringFromVersionPartitionData(PartitionData partitionData) {
string versionHashString = "";
foreach (var rowData in partitionData.rowDataList) {
versionHashString += rowData.Hash;
}
string hashString = CreateMD5Hash(versionHashString);
// 結果出力
Debug.Log("=====CreateHashStringFromVersionPartitionData:OutputStart=====");
Debug.Log("hashString : ↓");
Debug.Log(hashString);
Debug.Log("=====CreateHashStringFromVersionPartitionData:OutputEnd=====");
return hashString;
}
// バージョンファイルのデータを見て、新しいバージョン情報を取得する
static private string GetUpdatedVersionString(PartitionData partitionData) {
string versionString = "";
if (partitionData.dataVersion == "start") {
versionString = "1";
} else {
int versionNumber = int.Parse(partitionData.dataVersion);
versionNumber++;
versionString = versionNumber.ToString();
}
// 結果出力
Debug.Log("=====GetUpdatedVersionString:OutputStart=====");
Debug.Log("versionString : ↓");
Debug.Log(versionString);
Debug.Log("=====GetUpdatedVersionString:OutputEnd=====");
return versionString;
}
// バージョン情報から、その状態のバージョン文字列を作成
static private string CreateCurrentPartitionString(PartitionData partitionData, int partitionNumber) {
string outputString = "";
string dataCount = partitionData.rowDataList.Count.ToString();
string rowDataDetail = String.Format("{0},{1},{2},{3}",
"version",
partitionData.dataVersion,
dataCount,
partitionNumber.ToString());
outputString += rowDataDetail;
outputString += Environment.NewLine;
string temp = "";
foreach (var rowData in partitionData.rowDataList) {
temp += String.Format("{0},{1}", rowData.AssetBundleName, rowData.Hash);
temp += Environment.NewLine;
}
outputString += temp;
// 結果出力
Debug.Log("=====CreateCurrentPartitionString:OutputStart=====");
Debug.Log("outputString : ↓");
Debug.Log(outputString);
Debug.Log("=====CreateCurrentPartitionString:OutputEnd=====");
return outputString;
}
// 既存バージョンファイルから、差分確認用ディクショナリーを作成
static private Dictionary<int, PartitionData> CreateCheckDifferenceDictionaryFromPrevVersionFile(string versionFileSource) {
Dictionary<int, PartitionData> partitionDataDict = new Dictionary<int, PartitionData>();
// 改行で分けた文字列
string[] tempList = versionFileSource.Split(Environment.NewLine[0]);
ArrayList lineList = new ArrayList();
// 空データの削除
for (int i = 0; i < tempList.Length; i++) {
if (string.IsNullOrEmpty(tempList[i]) != true) {
lineList.Add(tempList[i]);
}
}
int index = 0;
while (index < lineList.Count) {
string line = lineList[index] as string;
string[] rowDetail = line.Split(","[0]);
string version = rowDetail[1];
string dataCount = rowDetail[2];
string partitionNumber = rowDetail[3];
PartitionData partitionData = new PartitionData();
partitionData.dataVersion = version;
for (int i = 0; i < int.Parse(dataCount); i++) {
line = lineList[index+1+i] as string;// 詳細情報の次のデータから参照する為の+1
string[] parameter = line.Split(","[0]);
RowData rowData = new RowData();
rowData.AssetBundleName = parameter[0];
rowData.Hash = parameter[1];
partitionData.rowDataList.Add(rowData);
}
partitionDataDict.Add(int.Parse(partitionNumber), partitionData);
index += (int.Parse(dataCount) + 1);// バージョン詳細の分
}
// 結果出力
Debug.Log("=====CreateCheckDifferenceDictionaryFromPrevVersionFile:OutputStart=====");
Debug.Log("versionFileSource : ↓");
Debug.Log(versionFileSource);
Debug.Log("partitionDataDict : ↓");
foreach(var partitionData in partitionDataDict) {
Debug.Log("KEY : " + partitionData.Key);
Debug.Log("VALUE.dataVersion : " + partitionData.Value.dataVersion);
Debug.Log("VALUE.rowDataList : ↓");
foreach(var rowData in partitionData.Value.rowDataList) {
Debug.Log("rowData.AssetBundleName : " + rowData.AssetBundleName);
Debug.Log("rowData.Hash : " + rowData.Hash);
}
}
Debug.Log("=====CreateCheckDifferenceDictionaryFromPrevVersionFile:OutputEnd=====");
return partitionDataDict;
}
public void GetBytesValidStructure()
{
string topicName = "topic";
int correlationId = 1;
string clientId = "TestClient";
short requiredAcks = 5;
int ackTimeout = 345;
var partition = 2;
short error = 0;
var payload = Encoding.UTF8.GetBytes("testMessage");
BufferedMessageSet messageSet = new BufferedMessageSet(new List<Message>() { new Message(payload) }, 0);
var partitionData = new PartitionData(partition, ErrorMapper.ToError(error), messageSet);
var topicData = new TopicData(topicName, new List<PartitionData>() { partitionData });
var request = new ProducerRequest(correlationId, clientId, requiredAcks, ackTimeout, new List<TopicData>() { topicData });
int requestSize = 2 + //request type id
2 + //versionId
4 + //correlation id
request.GetShortStringWriteLength(clientId) + // actual client id
2 + //required acks
4 + //ack timeout
4 + //data count
//=== data part
request.GetShortStringWriteLength(topicName) + //topic
4 + //partition data count
4 + //partition id
4 + //messages set size
messageSet.SetSize;
var ms = new MemoryStream();
request.WriteTo(ms);
byte[] bytes = ms.ToArray();
Assert.IsNotNull(bytes);
// add 4 bytes for the length of the message at the beginning
Assert.AreEqual(requestSize + 4, bytes.Length);
// first 4 bytes = the message length
Assert.AreEqual(requestSize, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray<byte>()), 0));
// next 2 bytes = the request type
Assert.AreEqual((short)RequestTypes.Produce, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray<byte>()), 0));
// next 2 bytes = the version id
Assert.AreEqual((short)ProducerRequest.CurrentVersion, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray<byte>()), 0));
// next 2 bytes = the correlation id
Assert.AreEqual(correlationId, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(8).Take(4).ToArray<byte>()), 0));
// next 2 bytes = the client id length
Assert.AreEqual((short)clientId.Length, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(12).Take(2).ToArray<byte>()), 0));
// next few bytes = the client id
Assert.AreEqual(clientId, Encoding.ASCII.GetString(bytes.Skip(14).Take(clientId.Length).ToArray<byte>()));
// next 2 bytes = the required acks
Assert.AreEqual((short)requiredAcks, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(14 + clientId.Length).Take(2).ToArray<byte>()), 0));
// next 4 bytes = the ack timeout
Assert.AreEqual(ackTimeout, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(16 + clientId.Length).Take(4).ToArray<byte>()), 0));
// next 4 bytes = the data count
Assert.AreEqual(1, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(20 + clientId.Length).Take(4).ToArray<byte>()), 0));
// next 2 bytes = the tppic length
Assert.AreEqual((short)topicName.Length, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(24 + clientId.Length).Take(2).ToArray<byte>()), 0));
// next few bytes = the topic
Assert.AreEqual(topicName, Encoding.ASCII.GetString(bytes.Skip(26 + clientId.Length).Take(topicName.Length).ToArray<byte>()));
// next 4 bytes = the partition data count
Assert.AreEqual(topicData.PartitionData.Count(), BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(26 + clientId.Length + topicName.Length).Take(4).ToArray<byte>()), 0));
// next 4 bytes = the partition
Assert.AreEqual(partition, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(30 + clientId.Length + topicName.Length).Take(4).ToArray<byte>()), 0));
// skipping MessageSet check - this could be done separately in another unit test
}
public static PartitionData GetPartition(double[] data, int nClasses)
{
if ((data == null) || (nClasses < 1))
{
return null;
}
int n = data.Length;
if (n < 2)
{
return null;
}
if ((nClasses % 2) == 0)
{
++nClasses;
}
double vMin = (from x in data select x).Min();
double vMax = (from x in data select x).Max();
if (vMin >= vMax)
{
return null;
}
double vMean = (from x in data select x).Average();
PartitionData oRet = new PartitionData();
oRet.ClassesCount = nClasses;
oRet.Limits = new double[nClasses + 1];
oRet.Classes = new int[n];
double dn = (double)2.5;
double deltaSup = (double)((vMax - vMean) / dn);
double deltaInf = (double)((vMean - vMin) / dn);
int iSup = nClasses;
int iInf = 0;
double fMin = vMin;
double fMax = vMax;
while (iInf < iSup)
{
oRet.Limits[iInf++] = fMin;
oRet.Limits[iSup--] = fMax;
fMin = (double)(fMin + deltaInf);
fMax = (double)(fMax - deltaSup);
}// while
int zMax = nClasses - 1;
for (int i = 0; i < n; ++i)
{
double x = data[i];
if (x <= vMin)
{
oRet.Classes[i] = 0;
}
else if (x >= vMax)
{
oRet.Classes[i] = zMax;
}
else
{
for (int j = 0; j < nClasses; ++j)
{
if ((x >= oRet.Limits[j]) && (x < oRet.Limits[j + 1]))
{
oRet.Classes[i] = j;
break;
}
}// j
}
}// i
return oRet;
}
public static PartitionData GetPartition(String[] data)
{
if (data == null)
{
return null;
}
Dictionary<String, int> oDict = new Dictionary<string, int>();
int n = data.Length;
int iCur = 0;
for (int i = 0; i < n; ++i)
{
String s = data[i];
if (!String.IsNullOrEmpty(s))
{
s = s.Trim().ToLower();
if (!String.IsNullOrEmpty(s))
{
if (!oDict.ContainsKey(s))
{
oDict[s] = iCur++;
}
}
}
}// i
if (iCur < 2)
{
return null;
}
PartitionData oRet = new PartitionData();
oRet.ClassesCount = iCur;
oRet.Classes = new int[n];
for (int i = 0; i < n; ++i)
{
String s = data[i];
if (!String.IsNullOrEmpty(s))
{
s = s.Trim().ToLower();
if (!String.IsNullOrEmpty(s))
{
if (!oDict.ContainsKey(s))
{
oRet.Classes[i] = -1;
}
else
{
oRet.Classes[i] = oDict[s];
}
}
}
}// i
return oRet;
}
public static PartitionData GetPartition(bool[] data)
{
if (data == null)
{
return null;
}
int n = data.Length;
if (n < 2)
{
return null;
}
PartitionData oRet = new PartitionData();
oRet.ClassesCount = 2;
oRet.Classes = new int[n];
for (int i = 0; i < n; ++i)
{
if (data[i])
{
oRet.Classes[i] = 1;
}
else
{
oRet.Classes[i] = 0;
}
}// i
return oRet;
}
