public void TestSerializeMessageSet() { using (var serialized = new ReusableMemoryStream(null)) { var set = new PartitionData { Partition = 42, CompressionCodec = CompressionCodec.None, Messages = new[] { new Message { Key = Key, Value = Value }, new Message { Key = Key, Value = Value } } }; set.Serialize(serialized, SerializationConfig.ByteArraySerializers); serialized.Position = 0; Assert.AreEqual(42, BigEndianConverter.ReadInt32(serialized)); // Partition Assert.AreEqual(serialized.Length - 4 - 4, BigEndianConverter.ReadInt32(serialized)); // MessageSet size Assert.AreEqual(0, BigEndianConverter.ReadInt64(serialized)); // First message offset int firstMsgSize = BigEndianConverter.ReadInt32(serialized); serialized.Position += firstMsgSize; Assert.AreEqual(0, BigEndianConverter.ReadInt64(serialized)); // First message offset int secondMsgSize = BigEndianConverter.ReadInt32(serialized); serialized.Position += secondMsgSize; Assert.AreEqual(serialized.Length, serialized.Position); } }
public void TestDeserializeMessageSet(CompressionCodec codec) { using (var serialized = Pool.Reserve()) { var set = new PartitionData { Partition = 42, CompressionCodec = codec, Messages = new[] { new Message { Key = Key, Value = Value }, new Message { Key = Key, Value = Value } } }; set.Serialize(serialized, SerializationConfig.ByteArraySerializers); serialized.Position = 4; var deserialized = FetchPartitionResponse.DeserializeMessageSet(serialized, SerializationConfig.ByteArrayDeserializers); Assert.AreEqual(2, deserialized.Count); foreach (var msg in deserialized) { Assert.AreEqual(0, msg.Offset); CollectionAssert.AreEqual(Key, msg.Message.Key as byte[]); CollectionAssert.AreEqual(Value, msg.Message.Value as byte[]); } } }
public PLTGenertorFactory(PartitionData PD, COLCollection _COLSet, SubSamplingAnalysisSetting subSamplingSettings, DateTime _start, TimeStyle _timestyle, LossStyle _lossstyle) { Start = _start; End = _start.AddYears(6); pData = new PartitionDataAdpator(PD, subSamplingSettings); GeneratorFactory = new GUInputGeneratorFactory(pData, _COLSet, _timestyle, _lossstyle); }
public PLTGenertorFactory(PartitionData PD, HashSet <string> _subperils, SubSamplingAnalysisSetting subSamplingSettings, DateTime _start, TimeStyle _timestyle, LossStyle _lossstyle) { Start = _start; End = _start.AddYears(6); pData = new PartitionDataAdpator(PD, subSamplingSettings); GeneratorFactory = new GUInputGeneratorFactory(pData, _subperils, _timestyle, _lossstyle); }
public void TestBadCompression() { using (var serialized = Pool.Reserve()) { var set = new PartitionData { Partition = 42, CompressionCodec = CompressionCodec.Gzip, Messages = new[] { new Message { Key = Key, Value = Value } } }; set.Serialize(serialized, SerializationConfig.ByteArraySerializers); // corrupt compressed data serialized[37] = 8; // recompute crc var crc = (int)Crc32.Compute(serialized, 24, serialized.Length - 24); serialized.Position = 20; BigEndianConverter.Write(serialized, crc); // go serialized.Position = 4; Assert.That(() => FetchPartitionResponse.DeserializeMessageSet(serialized, SerializationConfig.ByteArrayDeserializers), Throws.InstanceOf <UncompressException>()); } }
/// <summary> /// MANIFOLD use /// </summary> public FetchResponseWrapper FetchAndGetDetail(string clientId, string topic, int correlationId, int partitionId, long fetchOffset, int fetchSize , int maxWaitTime, int minWaitSize) { FetchResponse response = this.Fetch(clientId, topic, correlationId, partitionId, fetchOffset, fetchSize, maxWaitTime, minWaitSize); if (response == null) { throw new KafkaConsumeException(string.Format("FetchRequest returned null FetchResponse,fetchOffset={0},leader={1},topic={2},partition={3}", fetchOffset, this.Config.Broker, topic, partitionId)); } PartitionData partitionData = response.PartitionData(topic, partitionId); if (partitionData == null) { throw new KafkaConsumeException(string.Format("PartitionData int FetchResponse is null,fetchOffset={0},leader={1},topic={2},partition={3}", fetchOffset, this.Config.Broker, topic, partitionId)); } if (partitionData.Error != ErrorMapping.NoError) { string s = string.Format("Partition data in FetchResponse has error. {0} {1} fetchOffset={2},leader={3},topic={4},partition={5}" , partitionData.Error, KafkaException.GetMessage(partitionData.Error) , fetchOffset, this.Config.Broker, topic, partitionId); Logger.Error(s); throw new KafkaConsumeException(s, partitionData.Error); } return(new FetchResponseWrapper(partitionData.GetMessageAndOffsets(), response.Size, response.CorrelationId, topic, partitionId)); }
public PartitionDataAdpator(PartitionData _PData, SubSamplingAnalysisSetting _subSamplingSettings) { PData = _PData; expData = new Dictionary <long, ExposureDataAdaptor>(); JSParser = new JavascriptParser(GetParserSettings()); SubSamplingSettings = _subSamplingSettings; }
public GUInputGeneratorFactory(PartitionData PD, HashSet <string> _subperils, SubSamplingAnalysisSetting subSmaplingSettings, TimeStyle _timestyle, LossStyle _lossstyle) { PData = new PartitionDataAdpator(PD, subSmaplingSettings); NeedDisposePData = true; subperils = _subperils; timestyle = _timestyle; lossstyle = _lossstyle; }
public GUInputGeneratorFactory(PartitionData PD, COLCollection _COLSet, SubSamplingAnalysisSetting subSmaplingSettings, TimeStyle _timestyle, LossStyle _lossstyle) { PData = new PartitionDataAdpator(PD, subSmaplingSettings); NeedDisposePData = true; subperils = _COLSet.GetSubperils(); timestyle = _timestyle; lossstyle = _lossstyle; }
static void Main(string[] args) { //string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases\NewRL131_RMS_EDM_FinMod_Automation_RL131_72514_PortID_120_1000002113_1\4248\2050305\635447355784670000\rites_batch0.dat"; //string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases\PerRisk_RMS_EDM_FinMod_Automation_RL131_72514_PortID_896_1000002164_1\4438\2142576\635447551688220000\rites_batch0.dat"; //string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases\PerRisk_RMS_EDM_FinMod_Automation_RL131_72514_PortID_1013_1000002151_1\4413\2142553\635447537560220000\rites_batch0.dat"; // string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases Overlap Max Ded\NewRL131_RMS_EDM_FinMod_Automation_RL131_BT4IT4Aug30thNGtoRLAndInvalidationScriptsRanOnThisEDM_PortID_923_1000001480_1\6611\11352686\635416896292050000\rites_batch0.dat"; //string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases Overlap Max Ded\PerRisk_RMS_EDM_FinMod_Automation_RL13_BT4IT4Aug30thNGtoRLAndInvalidationScriptsRanOnThisEDM_PortID_958_1000001574_1\7066\11353699\635417441336090000\rites_batch0.dat"; //string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases Overlap Max Ded\EDM_RLFM_RL13_BT4IT4Aug30thNGtoRLAndInvalidationScriptsRanOnThisEDM_PortID_132_1000001835_1\8170\11366277\635419079253350000\rites_batch0.dat"; //string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases Overlap Max Ded\RMS_RLFM_EDM2_ForPlatformQA_PortID_1_1000001611_1\7239\11360603\635417579575500000\rites_batch0.dat"; //string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases\PerRisk_RMS_EDM_FinMod_Automation_RL131_72514_PortID_1021_1000002108_1\4233\2050284\635447346491010000\rites_batch0.dat"; //string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases\MTH_200_rites_batch0.dat"; //string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\MTH Cases\rites_batch0_MTH1.dat"; //string filepath = @" C:\LocalNGFM\EDS Extracts\FromRain\Large Extracts\rites_batch62001_PerRisk.dat"; //string filepath = @"D:\Nina_Automation_Testing\NTA_EDS_Extract\PerRisk_RMS_EDM_FinMod_Automation_RL13_BT4IT4Aug30thNGtoRLAndInvalidationScriptsRanOnThisEDM_PortID_897_1000001508_1\6752\11353141\635417040157160000\rites_batch0.dat"; //string filepath = @"D:\Matrix_Prototype_Storage\EDS Extract\JPTY_Multibuilding_PerRisk.dat"; string filepath = @"D:\Nina_Automation_Testing\Treaties\NoHoursClause\No_Reinstatement_NoAgg\From_Slava\rites_batch0_FromSlava.dat"; //string filepath = @"C:\LocalNGFM\EDS Extracts\From Swapna\rites_batch260197453_swapna_profile_case_Updated.dat"; //string filepath1 = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\rites_batch1.dat"; //string filepath1 = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\rites_SingleBuilding_overlap_subperil.dat"; // string filepath1 = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\rites_FixedGraph1.dat"; //string filepath1 = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\rites_FixedGraphOverlap.dat"; //string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\rites_batch0.dat"; //string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\rites_batch0_SingleBuilding_overlap.dat"; //string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\rites_batch0_SingleBuilding.dat"; //step policy //string filepath = @"C:\LocalNGFM\EDS Extracts\SunnyCreated\TreatyTestCaseWithPositions.dat"; //string filepath = @"C:\LocalNGFM\EDS Extracts\FromRain\Version2\Test1EQ_StepPGU_CDL.dat"; // string filepath = @"C:\LocalNGFM\EDS Extracts\SunnyCreated\TreatyTestCaseWithPositions.dat"; PartitionData pd = GetPDFromPath(filepath); UpdateCDLFOrGUSublimits(pd, 0); long conID = 11331; //long conID = 42586728; //long conID = 2050306; //TestMatrixHDFM(pd, conID); COLCollection COLSet = new COLCollection(new HashSet <string>() { "EQ" }); //{ "EQ", "WS" }); //COLCollection COLSet = new COLCollection(new HashSet<string>() { "CSWT" }); //{ "EQ", "WS" }); //TestRITEMap(pd, conID); //TestGULossVector(pd, conID); //TestReference(GraphType.FixedGraph1, pd, (int)conID, COLSet); //CDL: 3 //TestReference(GraphType.FixedGraphOverlap, pd, (int)conID, COLSet); //CDL:6 //TestReference(GraphType.FixedGraphOverlapSubperil, pd, (int)conID, COLSet); //CDL: 7 //TestReference(GraphType.FixedGraphOverlapSubperil2, pd, (int)conID, COLSet); //CDL: 8 //TestReference(GraphType.StepPolicyGraph, pd, (int)conID, COLSet); //CDL: 10 TestReference(GraphType.Auto, pd, (int)conID, COLSet); //CDL: 10 //TestReferenceWithPeriods(GraphType.Auto, pd, (int)conID, COLSet); / //TestReferenceSpeed(GraphType.Auto, pd, (int)conID, COLSet); //testPerformance(); //TestExposureDataAdaptor(pd, (int)conID); }
public ReferencePrototype(PartitionData PD, ModuleSettingsProvider _moduleSettings, JavascriptContext JSContext) { SetModelList(_moduleSettings); RAPSettings _rapsettings = GetRAPSettings(_moduleSettings); SubSamplingAnalysisSetting _subSamplingSettings = GetSubSamplingSettings(_moduleSettings); InitilizePrototype(PD, _rapsettings, _subSamplingSettings, JSContext); }
private static void UpdateEDSExtractWithCDL(string CDl, string filepath, int ConIndex) { PartitionData PD = Deserialize(filepath); PD.Exposures[ConIndex].Contract.CDLString = CDl; string newfilepath = @"C:\LocalNGFM\EDS Extracts\From Swapna\rites_batch260197453_swapna_profile_case_Updated.dat"; Serialize(PD, newfilepath); }
public void Prepare(PartitionData partitionData, bool append = false, bool OnlyBuildCOM = false) { JavaScriptHarness.Construct(underscore_js, grammar_ast_js); Initialize(partitionData, append); Prepare(OnlyBuildCOM); JavaScriptHarness.Destruct(); }
//RITE.ID -> numBldgs -> <timestamp -> list of loss> public GUInputGenerator(PartitionData PD, long conID, COLCollection _COLSet, TimeStyle _timestyle, LossStyle _lossstyle) { PartitionDataAdpator PData = new PartitionDataAdpator(PD); expData = PData.GetExposureAdaptor(conID); COLSet = _COLSet; timestyle = _timestyle; lossstyle = _lossstyle; // Dictionary<long, Dictionary<int, Tuple<int, List<float>>>> GULosses = new Dictionary<long, Dictionary<int, Tuple<int, List<float>>>>(); }
public async Task OpenConnection(string subscription, string resourceGroup, string ehNamespace, PartitionContext partitionContext) { var partitionData = new PartitionData() { Name = partitionContext.PartitionId, Sequence = partitionContext.RuntimeInformation.LastSequenceNumber, RetrievalTime = partitionContext.RuntimeInformation.RetrievalTime, LastEnqueueTime = partitionContext.RuntimeInformation.LastEnqueuedTimeUtc }; _ = _content.AddOrUpdate(ehNamespace, new EhNamespaceData() { Namespace = ehNamespace, Subscription = subscription, ResourceGroup = resourceGroup, Ehs = new Dictionary <string, EhData>() { { partitionContext.EventHubPath, new EhData() { Name = partitionContext.EventHubPath, Partitions = new Dictionary <string, PartitionData>() { { partitionContext.PartitionId, partitionData } } } } } }, (e, newValue) => { var b = partitionData; if (newValue.Ehs.ContainsKey(partitionContext.EventHubPath)) { if (newValue.Ehs[partitionContext.EventHubPath].Partitions.ContainsKey(partitionContext.PartitionId)) { newValue.Ehs[partitionContext.EventHubPath].Partitions[partitionContext.PartitionId] = partitionData; } else { newValue.Ehs[partitionContext.EventHubPath].Partitions.Add(partitionContext.PartitionId, partitionData); } } else { newValue.Ehs.Add(partitionContext.EventHubPath, new EhData() { Name = partitionContext.EventHubPath, Partitions = new Dictionary <string, PartitionData>() { { partitionContext.PartitionId, partitionData } } }); } return(newValue); }); await hub.Clients.All.SendAsync("updateData", _content); }
public PullResponse(PartitionData pData) { this.Partition = pData.Partition; this.Error = pData.Error; this.Payload = new List <byte[]>(); foreach (MessageAndOffset m in pData.MessageSet) { this.Payload.Add(m.Message.Payload); } }
private static void UpdateCDLFOrGUSublimits(PartitionData pd, int ConIndex) { Regex regex = new Regex(Regex.Escape("Declarations\r\n")); string cdlString = pd.Exposures[ConIndex].Contract.CDLString; //string NewCDL = regex.Replace(cdlString, "Declarations\r\n Claims Adjustment Options Are (SUBLIMITS ARE GROUND UP)\r\n", 1); //string NewCDL = regex.Replace(cdlString, "Declarations\r\n Claims Adjustment Options Are (Deductibles are , Sublimits are Net of Deductible)\r\n", 1); string NewCDL = regex.Replace(cdlString, "Declarations\r\n Claims Adjustment Options Are (Deductibles are absorbable,SUBLIMITS ARE GROUND UP)\r\n", 1); //string NewCDL = regex.Replace(cdlString, "Declarations\r\n Claims Adjustment Options Are (Sublimits are Net of Deductible)\r\n", 1); pd.Exposures[ConIndex].Contract.CDLString = NewCDL; }
public PLTGenerator(PartitionData _PD, long _conID, int pid, COLCollection _COLSet, TimeStyle _timestyle, LossStyle _lossStyle, DateTime _start, DateTime _end) { PartitionDataAdpator PData = new PartitionDataAdpator(_PD); expData = PData.GetExposureAdaptor(conID); PD = _PD; conID = _conID; COLSet = _COLSet; timestyle = _timestyle; lossStyle = _lossStyle; PLT = new PeriodLossTable(pid, _COLSet, _timestyle, _lossStyle, _start, _end); }
public PullResponseExt(PartitionData pData) { this.Partition = pData.Partition; this.Error = pData.Error; this.OffsetsAndPayLoads = new List <Tuple <long, byte[]> >(); foreach (MessageAndOffset m in pData.MessageSet) { this.OffsetsAndPayLoads.Add(new Tuple <long, byte[]>(m.Message.Offset, m.Message.Payload)); } }
private static void TestRITEMap(PartitionData pd, long conID) { //Default SubSampling Settings SubSamplingAnalysisSetting subSamplingSettings = new SubSamplingAnalysisSetting(false, 1, 0, 250, "", ""); PartitionDataAdpator PDataAdap = new PartitionDataAdpator(pd, subSamplingSettings); ExposureDataAdaptor expData = PDataAdap.GetExposureAdaptor(conID); ISubPerilConfig subperilInfo = new RMSSubPerilConfig(); RITEmapper1 mapper = new RITEmapper1(expData, new RAPSettings(new HashSet <string> { "EQ" }), subperilInfo); }
public VectorGUInputGeneratorFactory(PartitionData PD, HashSet <string> _subperils, TimeStyle _timestyle, LossStyle _lossstyle, bool UseAggregatedRITEs, SubSamplingAnalysisSetting subSamplingSettings) { subperils = _subperils; bool subperilsSupported = subperils.All(sp => subperilConfig.GetSubPerilList().Contains(sp)); if (!subperilsSupported) { throw new InvalidOperationException("Subperils specified cannot be supported at this time"); } pData = new PartitionDataAdpator(PD, subSamplingSettings); GeneratorFactory = new GUInputGeneratorFactory(pData, subperils, _timestyle, _lossstyle); }
public static void Serialize(PartitionData pData, string file) { using (var wc = new WebClient().OpenWrite(file)) { try { ProtoBuf.Serializer.Serialize <PartitionData>(wc, pData); } catch (ProtoBuf.ProtoException p) { //Log.Fatal("Error Writing of protobuf to file" + p.Message); Console.WriteLine("Error Writing of protobuf to file \"{0}\": {1}", file, p.Message); } } }
public void Prepare(string[] ContractExposuresProtobufExtractFiles) { string directoryName = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); Directory.SetCurrentDirectory(directoryName); _handleHDFM.Clear(); foreach (string file in Utilities.GetExtantFiles(ContractExposuresProtobufExtractFiles)) { if (file.EndsWith(".dat")) { PartitionData partitionData = Utilities.DeserializePartitionData(file); _handleHDFM.Append(partitionData); } } }
private static void Write(MemoryStream stream, PartitionData partition) { BigEndianConverter.Write(stream, partition.Partition); if (partition.CompressionType == CompressionType.None) { var messageArray = partition.Messages.ToArray(); // make sure linq evaluation happen just once var messageSetSize = EstimateMessageSetSize(messageArray); BigEndianConverter.Write(stream, messageSetSize); Write(stream, partition.Messages); } else { var compressedMessage = Compress(partition.CompressionType, partition.Messages); WriteCompressedMessageSet(stream, compressedMessage, partition.CompressionType); } }
public static PartitionData DeserializePartitionData(string[] files, CancellationTokenSource ct) { List <PartitionData> PDs = new List <PartitionData>(); foreach (var f in GetExtantFiles(files)) { if (ct.IsCancellationRequested) { return(null); } var pd = DeserializePartitionData(f); if (pd != null) { PDs.Add(pd); } } return((PDs.Count > 0) ? PartitionData.Merge(PDs) : null); }
public static PartitionData Deserialize(string file) { PartitionData result = null; using (var wc = new WebClient().OpenRead(file)) { try { result = ProtoBuf.Serializer.Deserialize <PartitionData>(wc); } catch (ProtoBuf.ProtoException p) { //Log.Fatal("Error Reading a protobuf file" + p.Message); Console.WriteLine("Error Reading a protobuf file \"{0}\": {1}", file, p.Message); } } return(result); }
private void Initialize(PartitionData partitionData, bool append = false) { Initialize(append); #region VECTORIZED REFERENCE PROTOTYPE INTEGRATION _ReferencePrototype = null; UseReference = Convert.ToBoolean(ConfigurationManager.AppSettings["UseReference"]); if (UseReference) { //Sunny hack... need to change to read from Model Setings Provider from the DLM.. RAPSettings RapSettings = new RAPSettings(new HashSet <string> { "WS", "WI", "WA" }); SubSamplingAnalysisSetting subSamplingSetiings = new SubSamplingAnalysisSetting(false, 1, 0, 250, "", ""); if (RapSettings != null || subSamplingSetiings != null) { _ReferencePrototype = new ReferencePrototype(partitionData, RapSettings, subSamplingSetiings); } else { throw new InvalidOperationException("Cannot run in Array Mode and have null RapSettings or null SubSampling Settings!"); } } #endregion if (null != partitionData && null != partitionData.Exposures) { foreach (ContractExposure ce in partitionData.Exposures) { if ((ce.ContractType != null) && ce.ContractType.IsReinsuranceContract()) { //Treaty Contract ContractExposureDataIndex.TryAdd(ce.ExposureID, new TreatyContractExposureData(JavaScriptHarness, false, ce)); } else if (null != ce.ContractSubjectExposures && null != ce.Subschedules) { // Primary Contract ContractExposureDataIndex.TryAdd(ce.ExposureID, new PrimaryContractExposureData(JavaScriptHarness, false, ce, "", _ReferencePrototype)); } } } }
public void TestUnsupportedMagicByte() { using (var serialized = new ReusableMemoryStream(null)) { var set = new PartitionData { Partition = 42, CompressionCodec = CompressionCodec.None, Messages = new[] { new Message { Key = Key, Value = Value } } }; set.Serialize(serialized, SerializationConfig.ByteArraySerializers); serialized[24] = 8; // set non supported magic byte serialized.Position = 4; Assert.That(() => FetchPartitionResponse.DeserializeMessageSet(serialized, SerializationConfig.ByteArrayDeserializers), Throws.InstanceOf <UnsupportedMagicByteVersion>()); } }
public void TestBadCrc() { using (var serialized = new ReusableMemoryStream(null)) { var set = new PartitionData { Partition = 42, CompressionCodec = CompressionCodec.None, Messages = new[] { new Message { Key = Key, Value = Value } } }; set.Serialize(serialized, SerializationConfig.ByteArraySerializers); serialized[20] = 8; // change crc serialized.Position = 4; Assert.That(() => FetchPartitionResponse.DeserializeMessageSet(serialized, SerializationConfig.ByteArrayDeserializers), Throws.InstanceOf <CrcException>()); } }
// Gets the table column names and rows, removing data from the table as it is returned. public PartitionData Flush(string[] distinctValues) { // Put the columns in order of { Key0, Key1, ..., KeyN, Col1, Col2, ..., ColN } where the key columns are in the order // specified, and the non-key columns follow in alphabetical order _sortedColumns = new KeyValuePair <string, LinkedList <ColumnSegment> > [_columns.Count]; for (int i = 0; i < _keyNames.Length; ++i) { _sortedColumns[i] = new KeyValuePair <string, LinkedList <ColumnSegment> >(_keyNames[i], _columns[_keyNames[i]]); _columns.Remove(_keyNames[i]); } Array.Copy(_columns.ToArray(), 0, _sortedColumns, _keyNames.Length, _sortedColumns.Length - _keyNames.Length); _columns.Clear(); // No longer need var sortingFunc = Comparer <KeyValuePair <string, LinkedList <ColumnSegment> > > .Create((x, y) => x.Key.CompareTo(y.Key)); Array.Sort(_sortedColumns, _keyNames.Length, _sortedColumns.Length - _keyNames.Length, sortingFunc); // Sort non-key columns var result = new PartitionData { ColumnNames = _sortedColumns.Select(col => col.Key).ToArray(), Rows = get_rows(distinctValues) }; return(result); }
// 既存バージョン情報から作成したデータから、比較用ハッシュ作成 static private string CreateHashStringFromVersionPartitionData(PartitionData partitionData) { string versionHashString = ""; foreach (var rowData in partitionData.rowDataList) { versionHashString += rowData.Hash; } string hashString = CreateMD5Hash(versionHashString); // 結果出力 Debug.Log("=====CreateHashStringFromVersionPartitionData:OutputStart====="); Debug.Log("hashString : ↓"); Debug.Log(hashString); Debug.Log("=====CreateHashStringFromVersionPartitionData:OutputEnd====="); return hashString; }
// バージョンファイルのデータを見て、新しいバージョン情報を取得する static private string GetUpdatedVersionString(PartitionData partitionData) { string versionString = ""; if (partitionData.dataVersion == "start") { versionString = "1"; } else { int versionNumber = int.Parse(partitionData.dataVersion); versionNumber++; versionString = versionNumber.ToString(); } // 結果出力 Debug.Log("=====GetUpdatedVersionString:OutputStart====="); Debug.Log("versionString : ↓"); Debug.Log(versionString); Debug.Log("=====GetUpdatedVersionString:OutputEnd====="); return versionString; }
// バージョン情報から、その状態のバージョン文字列を作成 static private string CreateCurrentPartitionString(PartitionData partitionData, int partitionNumber) { string outputString = ""; string dataCount = partitionData.rowDataList.Count.ToString(); string rowDataDetail = String.Format("{0},{1},{2},{3}", "version", partitionData.dataVersion, dataCount, partitionNumber.ToString()); outputString += rowDataDetail; outputString += Environment.NewLine; string temp = ""; foreach (var rowData in partitionData.rowDataList) { temp += String.Format("{0},{1}", rowData.AssetBundleName, rowData.Hash); temp += Environment.NewLine; } outputString += temp; // 結果出力 Debug.Log("=====CreateCurrentPartitionString:OutputStart====="); Debug.Log("outputString : ↓"); Debug.Log(outputString); Debug.Log("=====CreateCurrentPartitionString:OutputEnd====="); return outputString; }
// 既存バージョンファイルから、差分確認用ディクショナリーを作成 static private Dictionary<int, PartitionData> CreateCheckDifferenceDictionaryFromPrevVersionFile(string versionFileSource) { Dictionary<int, PartitionData> partitionDataDict = new Dictionary<int, PartitionData>(); // 改行で分けた文字列 string[] tempList = versionFileSource.Split(Environment.NewLine[0]); ArrayList lineList = new ArrayList(); // 空データの削除 for (int i = 0; i < tempList.Length; i++) { if (string.IsNullOrEmpty(tempList[i]) != true) { lineList.Add(tempList[i]); } } int index = 0; while (index < lineList.Count) { string line = lineList[index] as string; string[] rowDetail = line.Split(","[0]); string version = rowDetail[1]; string dataCount = rowDetail[2]; string partitionNumber = rowDetail[3]; PartitionData partitionData = new PartitionData(); partitionData.dataVersion = version; for (int i = 0; i < int.Parse(dataCount); i++) { line = lineList[index+1+i] as string;// 詳細情報の次のデータから参照する為の+1 string[] parameter = line.Split(","[0]); RowData rowData = new RowData(); rowData.AssetBundleName = parameter[0]; rowData.Hash = parameter[1]; partitionData.rowDataList.Add(rowData); } partitionDataDict.Add(int.Parse(partitionNumber), partitionData); index += (int.Parse(dataCount) + 1);// バージョン詳細の分 } // 結果出力 Debug.Log("=====CreateCheckDifferenceDictionaryFromPrevVersionFile:OutputStart====="); Debug.Log("versionFileSource : ↓"); Debug.Log(versionFileSource); Debug.Log("partitionDataDict : ↓"); foreach(var partitionData in partitionDataDict) { Debug.Log("KEY : " + partitionData.Key); Debug.Log("VALUE.dataVersion : " + partitionData.Value.dataVersion); Debug.Log("VALUE.rowDataList : ↓"); foreach(var rowData in partitionData.Value.rowDataList) { Debug.Log("rowData.AssetBundleName : " + rowData.AssetBundleName); Debug.Log("rowData.Hash : " + rowData.Hash); } } Debug.Log("=====CreateCheckDifferenceDictionaryFromPrevVersionFile:OutputEnd====="); return partitionDataDict; }
public void GetBytesValidStructure() { string topicName = "topic"; int correlationId = 1; string clientId = "TestClient"; short requiredAcks = 5; int ackTimeout = 345; var partition = 2; short error = 0; var payload = Encoding.UTF8.GetBytes("testMessage"); BufferedMessageSet messageSet = new BufferedMessageSet(new List<Message>() { new Message(payload) }, 0); var partitionData = new PartitionData(partition, ErrorMapper.ToError(error), messageSet); var topicData = new TopicData(topicName, new List<PartitionData>() { partitionData }); var request = new ProducerRequest(correlationId, clientId, requiredAcks, ackTimeout, new List<TopicData>() { topicData }); int requestSize = 2 + //request type id 2 + //versionId 4 + //correlation id request.GetShortStringWriteLength(clientId) + // actual client id 2 + //required acks 4 + //ack timeout 4 + //data count //=== data part request.GetShortStringWriteLength(topicName) + //topic 4 + //partition data count 4 + //partition id 4 + //messages set size messageSet.SetSize; var ms = new MemoryStream(); request.WriteTo(ms); byte[] bytes = ms.ToArray(); Assert.IsNotNull(bytes); // add 4 bytes for the length of the message at the beginning Assert.AreEqual(requestSize + 4, bytes.Length); // first 4 bytes = the message length Assert.AreEqual(requestSize, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Take(4).ToArray<byte>()), 0)); // next 2 bytes = the request type Assert.AreEqual((short)RequestTypes.Produce, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(4).Take(2).ToArray<byte>()), 0)); // next 2 bytes = the version id Assert.AreEqual((short)ProducerRequest.CurrentVersion, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(6).Take(2).ToArray<byte>()), 0)); // next 2 bytes = the correlation id Assert.AreEqual(correlationId, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(8).Take(4).ToArray<byte>()), 0)); // next 2 bytes = the client id length Assert.AreEqual((short)clientId.Length, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(12).Take(2).ToArray<byte>()), 0)); // next few bytes = the client id Assert.AreEqual(clientId, Encoding.ASCII.GetString(bytes.Skip(14).Take(clientId.Length).ToArray<byte>())); // next 2 bytes = the required acks Assert.AreEqual((short)requiredAcks, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(14 + clientId.Length).Take(2).ToArray<byte>()), 0)); // next 4 bytes = the ack timeout Assert.AreEqual(ackTimeout, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(16 + clientId.Length).Take(4).ToArray<byte>()), 0)); // next 4 bytes = the data count Assert.AreEqual(1, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(20 + clientId.Length).Take(4).ToArray<byte>()), 0)); // next 2 bytes = the tppic length Assert.AreEqual((short)topicName.Length, BitConverter.ToInt16(BitWorks.ReverseBytes(bytes.Skip(24 + clientId.Length).Take(2).ToArray<byte>()), 0)); // next few bytes = the topic Assert.AreEqual(topicName, Encoding.ASCII.GetString(bytes.Skip(26 + clientId.Length).Take(topicName.Length).ToArray<byte>())); // next 4 bytes = the partition data count Assert.AreEqual(topicData.PartitionData.Count(), BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(26 + clientId.Length + topicName.Length).Take(4).ToArray<byte>()), 0)); // next 4 bytes = the partition Assert.AreEqual(partition, BitConverter.ToInt32(BitWorks.ReverseBytes(bytes.Skip(30 + clientId.Length + topicName.Length).Take(4).ToArray<byte>()), 0)); // skipping MessageSet check - this could be done separately in another unit test }
public static PartitionData GetPartition(double[] data, int nClasses) { if ((data == null) || (nClasses < 1)) { return null; } int n = data.Length; if (n < 2) { return null; } if ((nClasses % 2) == 0) { ++nClasses; } double vMin = (from x in data select x).Min(); double vMax = (from x in data select x).Max(); if (vMin >= vMax) { return null; } double vMean = (from x in data select x).Average(); PartitionData oRet = new PartitionData(); oRet.ClassesCount = nClasses; oRet.Limits = new double[nClasses + 1]; oRet.Classes = new int[n]; double dn = (double)2.5; double deltaSup = (double)((vMax - vMean) / dn); double deltaInf = (double)((vMean - vMin) / dn); int iSup = nClasses; int iInf = 0; double fMin = vMin; double fMax = vMax; while (iInf < iSup) { oRet.Limits[iInf++] = fMin; oRet.Limits[iSup--] = fMax; fMin = (double)(fMin + deltaInf); fMax = (double)(fMax - deltaSup); }// while int zMax = nClasses - 1; for (int i = 0; i < n; ++i) { double x = data[i]; if (x <= vMin) { oRet.Classes[i] = 0; } else if (x >= vMax) { oRet.Classes[i] = zMax; } else { for (int j = 0; j < nClasses; ++j) { if ((x >= oRet.Limits[j]) && (x < oRet.Limits[j + 1])) { oRet.Classes[i] = j; break; } }// j } }// i return oRet; }
public static PartitionData GetPartition(String[] data) { if (data == null) { return null; } Dictionary<String, int> oDict = new Dictionary<string, int>(); int n = data.Length; int iCur = 0; for (int i = 0; i < n; ++i) { String s = data[i]; if (!String.IsNullOrEmpty(s)) { s = s.Trim().ToLower(); if (!String.IsNullOrEmpty(s)) { if (!oDict.ContainsKey(s)) { oDict[s] = iCur++; } } } }// i if (iCur < 2) { return null; } PartitionData oRet = new PartitionData(); oRet.ClassesCount = iCur; oRet.Classes = new int[n]; for (int i = 0; i < n; ++i) { String s = data[i]; if (!String.IsNullOrEmpty(s)) { s = s.Trim().ToLower(); if (!String.IsNullOrEmpty(s)) { if (!oDict.ContainsKey(s)) { oRet.Classes[i] = -1; } else { oRet.Classes[i] = oDict[s]; } } } }// i return oRet; }
public static PartitionData GetPartition(bool[] data) { if (data == null) { return null; } int n = data.Length; if (n < 2) { return null; } PartitionData oRet = new PartitionData(); oRet.ClassesCount = 2; oRet.Classes = new int[n]; for (int i = 0; i < n; ++i) { if (data[i]) { oRet.Classes[i] = 1; } else { oRet.Classes[i] = 0; } }// i return oRet; }