public void Start() { byte[] request; try { while (true) { Log.Info("document server idle"); Socket handler = _listener.Accept(); request = new byte[sizeof(long) + sizeof(int)]; int received = handler.Receive(request); Log.InfoFormat("received a {0} byte request", received); var address = BlockSerializer.DeserializeBlock(request, 0); byte[] response = FindInCache(address); handler.Send(response); handler.Shutdown(SocketShutdown.Both); handler.Dispose(); Log.InfoFormat("responded with a {0} byte document", response.Length); } } catch (Exception e) { Log.Info(e.ToString()); } }
public static LcrsNode DeserializeNode(Stream stream) { var blockArr = new byte[SizeOfNode()]; stream.Read(blockArr, 0, blockArr.Length); if (!BitConverter.IsLittleEndian) { Array.Reverse(blockArr, 0, sizeof(char)); Array.Reverse(blockArr, sizeof(char), sizeof(short)); Array.Reverse(blockArr, sizeof(char) + sizeof(short), sizeof(int)); } var val = BitConverter.ToChar(blockArr, 0); var haveSibling = BitConverter.ToBoolean(blockArr, sizeof(char)); var haveChild = BitConverter.ToBoolean(blockArr, sizeof(char) + sizeof(byte)); var eow = BitConverter.ToBoolean(blockArr, sizeof(char) + 2 * sizeof(byte)); var depth = BitConverter.ToInt16(blockArr, sizeof(char) + 3 * sizeof(byte)); var weight = BitConverter.ToInt32(blockArr, sizeof(char) + 3 * sizeof(byte) + sizeof(short)); var block = BlockSerializer.DeserializeBlock(stream); return(new LcrsNode( val, haveSibling, haveChild, eow, depth, weight, block)); }
public static void Serialize(this LcrsNode node, Stream stream) { var valBytes = BitConverter.GetBytes(node.Value); var haveSiblingByte = EncodedBoolean[node.HaveSibling]; var haveChildByte = EncodedBoolean[node.HaveChild]; var eowByte = EncodedBoolean[node.EndOfWord]; var depthBytes = BitConverter.GetBytes(node.Depth); var weightBytes = BitConverter.GetBytes(node.Weight); if (!BitConverter.IsLittleEndian) { Array.Reverse(valBytes); Array.Reverse(depthBytes); Array.Reverse(weightBytes); } stream.Write(valBytes, 0, valBytes.Length); stream.WriteByte(haveSiblingByte); stream.WriteByte(haveChildByte); stream.WriteByte(eowByte); stream.Write(depthBytes, 0, depthBytes.Length); stream.Write(weightBytes, 0, weightBytes.Length); if (!BlockSerializer.Serialize(node.PostingsAddress, stream) && node.EndOfWord) { throw new InvalidOperationException("cannot store word without posting address"); } }
override public void Load(BlockSerializer bs) { LoadBlockData(bs); SurfaceBlockSerializer sbs = new SurfaceBlockSerializer(); GameMaster.DeserializeByteArray <SurfaceBlockSerializer>(bs.specificData, ref sbs); LoadSurfaceBlockData(sbs); if (sbs.haveStructures) { foreach (StructureSerializer ss in sbs.structuresList) { if (ss.id != Structure.PLANT_ID) { Structure s = Structure.GetStructureByID(ss.id); if (s != null) { s.Load(ss, this); } } else { Plant p = Plant.GetNewPlant(ss.id); if (p != null) { p.Load(ss, this); } } } } }
override public void Load(BlockSerializer bs) { LoadBlockData(bs); CaveBlockSerializer cbs = new CaveBlockSerializer(); GameMaster.DeserializeByteArray <CaveBlockSerializer>(bs.specificData, ref cbs); LoadCaveBlockData(cbs); }
public Searcher(IReadSessionFactory sessionFactory) { _parser = new QueryParser(); _scorerFactory = new TfIdfFactory(); _versions = Util.GetIndexVersionListInChronologicalOrder(sessionFactory.DirectoryName); _blockSize = BlockSerializer.SizeOfBlock(); _sessionFactory = sessionFactory; }
public Searcher(string directory, IQueryParser parser = null, IScoringSchemeFactory scorerFactory = null, IReadSessionFactory sessionFactory = null) { _parser = parser ?? new QueryParser(); _scorerFactory = scorerFactory ?? new TfIdfFactory(); _versions = Util.GetIndexVersionListInChronologicalOrder(directory); _blockSize = BlockSerializer.SizeOfBlock(); _sessionFactory = sessionFactory ?? new FullTextReadSessionFactory(directory); }
// This method is thread-safe. //public void AddBlock(string path, string[] demodulationConfigs) //{ // string[] splitPath = path.Split('\\'); // log("Loading block " + splitPath[splitPath.Length - 1]); // BlockSerializer bs = new BlockSerializer(); // Block b = bs.DeserializeBlockFromZippedXML(path, "block.xml"); // AddBlock(b, demodulationConfigs); //} public Block LoadBlockFromFile(string path) { string[] splitPath = path.Split('\\'); log("Loading block " + splitPath[splitPath.Length - 1]); BlockSerializer bs = new BlockSerializer(); Block b = bs.DeserializeBlockFromZippedXML(path, "block.xml"); return(b); }
public void TestSerialize() { using StreamReader r = new StreamReader("../../../Fixtures/block.json"); string json = r.ReadToEnd(); Block block = JsonConvert.DeserializeObject <Block>(json); var expected = "0x2403000014000000e4000000cc010000160300000000000033cf3d1dc816fd3c72010000d50a0000000000000400001b00300200009dd635fda0e093e537b7eea56bcf7272cb107170330a30da3270e0dd0daf027b8e8c8951749096f7a1e86b4204655a6e52b381468faa3b6e2dba699dd586862ddad1ff8aba40d0aa41a8a12f4ac61a03c1df244501b8650cd702f10f3fcdf3136b809c54f8612326529ec47945301cb97c442a32e46c7390f9165e31df8ba98437323ea9c2a42e4556e19a9e8723000918d97226c000000075093a730aff06f3070631d8a8cc2de0a8ea1c267f942ce800000008000000e00000000c000000dc0000000000000033cf3d1db39cfc3c72010000d40a0000000000000400001a003002003e6936ef975d2f77055f473d0b661bf9dcc6604fdca1e0018bd3468fe7ff46299bd730ba9f1c57da5970c737914711156ab921ae205b4855197d7089556a84af000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006b76a1fd3fc2a42e75b671879e8723003e3798c210c0000000d872ce710aff0620e103220145a7c17e5fa6b232670856000000004a01000008000000420100000c000000d9000000cd0000001c00000020000000240000002800000058000000c100000000000000000000000000000001000000d50a0000000000000000000000000000000000000000000000000000000000000000000000000000ffffffff690000000800000061000000100000001800000061000000fa35509053000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce80114000000da648442dbb7347e467d1d09da13e5cd3a0ef0e10c000000080000000000000069000000080000005d0000005d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce80114000000da648442dbb7347e467d1d09da13e5cd3a0ef0e104000000deadbeef01000000a08d86ccf2dd38685637"; Assert.Equal(BlockSerializer.HexStringToBytes(expected), new BlockSerializer(block).Serialize()); }
public async Task <Block> Invoke() { var request = new GetLastBlockRequestMessage(); var response = await Invoke(request); var bs = new BlockSerializer(); return(bs.Deserialize(response.Result)); }
public Searcher(string directory, long version, QueryParser parser, IScoringSchemeFactory scorerFactory, IReadSessionFactory sessionFactory = null) { _directory = directory; _parser = parser; _scorerFactory = scorerFactory; _versions = new[] { BatchInfo.Load(Path.Combine(directory, version + ".ix")) }; _blockSize = BlockSerializer.SizeOfBlock(); _sessionFactory = sessionFactory ?? new ReadSessionFactory(directory); }
// This method is thread-safe. public void AddBlock(string path, string[] demodulationConfigs) { string[] splitPath = path.Split('\\'); log("Loading block " + splitPath[splitPath.Length - 1]); BlockSerializer bs = new BlockSerializer(); Block b = bs.DeserializeBlockFromZippedXML(path, "block.xml"); AddBlock(b, demodulationConfigs); }
public async Task <Block> Invoke(BigInteger height) { var request = new GetBlockByHeightRequestMessage(height); var response = await Invoke(request); var bs = new BlockSerializer(); return(bs.Deserialize(response.Result)); }
public MappedTrieReader(string fileName) { var dir = Path.GetDirectoryName(fileName); var version = Path.GetFileNameWithoutExtension(fileName); _stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read, 4096 * 1, FileOptions.SequentialScan); _blockSize = Serializer.SizeOfNode() + BlockSerializer.SizeOfBlock(); }
protected BlockSerializer GetBlockSerializer() { BlockSerializer bs = new BlockSerializer(); bs.type = type; bs.isTransparent = isTransparent; bs.pos = pos; bs.material_id = material_id; return(bs); }
public Searcher(string directory, QueryParser parser, IScoringSchemeFactory scorerFactory, IReadSessionFactory sessionFactory = null) { _directory = directory; _parser = parser; _scorerFactory = scorerFactory; _versions = Util.GetIndexFileNamesInChronologicalOrder(directory) .Select(f => BatchInfo.Load(f)).ToArray(); _blockSize = BlockSerializer.SizeOfBlock(); _sessionFactory = sessionFactory ?? new ReadSessionFactory(directory); }
internal void ActuallyTestLiveAnalysis() { for (int i = 0; i <= 44; i++) { string fileRoot = Environs.FileSystem.Paths["edmDataPath"] + "\\2019\\November2019\\13Nov1903_"; BlockSerializer bs = new BlockSerializer(); Block b = bs.DeserializeBlockFromZippedXML(fileRoot + (i.ToString()) + ".zip", "block.xml"); AcquisitionFinished(b); } }
internal void ActuallyTestLiveAnalysis() { for (int i = 1; i <= 20; i++) { string fileRoot = Environs.FileSystem.Paths["edmDataPath"] + "\\2009\\April2009\\30apr0900_"; BlockSerializer bs = new BlockSerializer(); Block b = bs.DeserializeBlockFromZippedXML(fileRoot + (i.ToString()) + ".zip", "block.xml"); AcquisitionFinished(b); } }
override public BlockSerializer Save() { BlockSerializer bs = GetBlockSerializer(); using (System.IO.MemoryStream stream = new System.IO.MemoryStream()) { new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter().Serialize(stream, GetSurfaceBlockSerializer()); bs.specificData = stream.ToArray(); } return(bs); }
public ReadSession( BatchInfo version, PostingsReader postingsReader, DocHashReader docHashReader, DocumentAddressReader addressReader, Stream stream) { Version = version; _stream = stream; _docHashReader = docHashReader; _postingsReader = postingsReader; _addressReader = addressReader; _blockSize = BlockSerializer.SizeOfBlock(); }
public ReadSession( SegmentInfo version, DocHashReader docHashReader, BlockInfoReader addressReader, Stream stream) { Version = version; Stream = stream; DocHashReader = docHashReader; AddressReader = addressReader; BlockSize = BlockSerializer.SizeOfBlock(); Stream.Seek(Version.KeyIndexOffset, SeekOrigin.Begin); KeyIndex = DocumentSerializer.ReadKeyIndex(Stream, Version.KeyIndexSize); }
override public BlockSerializer Save() { BlockSerializer bs = GetBlockSerializer(); CaveBlockSerializer cbs = new CaveBlockSerializer(); cbs.upMaterial_ID = material_id; cbs.haveSurface = haveSurface; cbs.surfaceBlockSerializer = GetSurfaceBlockSerializer(); using (System.IO.MemoryStream stream = new System.IO.MemoryStream()) { new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter().Serialize(stream, cbs); bs.specificData = stream.ToArray(); } return(bs); }
//public void TOFDemodulateBlocks(string[] blockFiles, string savePath) //{ // // first of all test that the save location exists to avoid later disappointment. // if (!Directory.Exists(Path.GetDirectoryName(savePath))) // { // log("Save path does not exist!!"); // return; // } // // initialise the accumulator // tcsga = new TOFChannelSetGroupAccumulator(); // // queue the blocks - the last block analysed will take care of saving the results. // foreach (string blockFile in blockFiles) // { // tofDemodulateParams tdp = new tofDemodulateParams(); // tdp.blockPath = blockFile; // tdp.savePath = savePath; // threadManager.AddToQueue(TOFDemodulateThreadWrapper, tdp); // } //} //private void TOFDemodulateBlock(string blockPath, string savePath) //{ // BlockSerializer bs = new BlockSerializer(); // string[] splitPath = blockPath.Split('\\'); // log("Loading block " + splitPath[splitPath.Length - 1]); // Block b = bs.DeserializeBlockFromZippedXML(blockPath, "block.xml"); // log("Demodulating block " + b.Config.Settings["cluster"] + " - " + b.Config.Settings["clusterIndex"]); // BlockTOFDemodulator btd = new BlockTOFDemodulator(); // TOFChannelSet tcs = btd.TOFDemodulateBlock(b, 0, true); // log("Accumulating block " + b.Config.Settings["cluster"] + " - " + b.Config.Settings["clusterIndex"]); // lock (accumulatorLock) tcsga.Add(tcs); // // are we the last block to be added? If so, it's our job to save the results // if (threadManager.RemainingJobs == 1) // { // // this lock should not be needed // lock(accumulatorLock) // { // TOFChannelSetGroup tcsg = tcsga.GetResult(); // Stream fileStream = new FileStream(savePath, FileMode.Create); // (new BinaryFormatter()).Serialize(fileStream, tcsg); // fileStream.Close(); // } // } //} //private void TOFDemodulateThreadWrapper(object parametersIn) //{ // threadManager.QueueItemWrapper(delegate(object parms) // { // tofDemodulateParams parameters = (tofDemodulateParams)parms; // TOFDemodulateBlock(parameters.blockPath, parameters.savePath); // }, // parametersIn // ); //} //private struct tofDemodulateParams //{ // public string blockPath; // public string savePath; // // this struct has a ToString method defined for error reporting porpoises. // public override string ToString() // { // return blockPath; // } //} #endregion #region Testing // Somewhere for SirCachealot to store test results that's accessible by Mathematica. // Makes debugging easier and is needed as a workaround for the constant Mathematica // NET/Link errors. // public TOFChannelSetGroup ChanSetGroup; // workarounds for NET/Link bugs //public TOFChannelSet GetAveragedChannelSet(bool eSign, bool bSign, bool rfSign) //{ // return ChanSetGroup.AverageChannelSetSignedByMachineState(eSign, bSign, rfSign); //} //public void LoadChannelSetGroup(string path) //{ // BinaryFormatter bf = new BinaryFormatter(); // FileStream fs = new FileStream(path, FileMode.Open); // ChanSetGroup = (TOFChannelSetGroup)bf.Deserialize(fs); // fs.Close(); //} public void Test1() { BlockSerializer bs = new BlockSerializer(); Block b = bs.DeserializeBlockFromZippedXML( "C:\\Users\\jony\\Files\\Data\\SEDM\\v3\\2009\\October2009\\01Oct0900_0.zip", "block.xml"); BlockDemodulator bd = new BlockDemodulator(); DemodulatedBlock db = bd.DemodulateBlockNL(b, DemodulationConfig.GetStandardDemodulationConfig("cgate11Fixed", b)); //JsonSerializer serializer = new JsonSerializer(); //using (StreamWriter sw = new StreamWriter("c:\\Users\\jony\\Desktop\\test.json")) //using (JsonWriter writer = new JsonTextWriter(sw)) //{ // serializer.Serialize(writer, b.Config); //} //bs.SerializeBlockAsJSON("c:\\Users\\jony\\Desktop\\test.json", b); }
private IEnumerable <DocumentTableRow> StreamDocuments() { var skipped = 0; var took = 0; for (int docId = 0; docId < _ix.DocumentCount; docId++) { var hash = _hashReader.Read(docId); var address = _addressReader.Read(new[] { new BlockInfo(docId * BlockSerializer.SizeOfBlock(), BlockSerializer.SizeOfBlock()) }).First(); var document = _documentReader.Read(new List <BlockInfo> { address }).First(); if (!hash.IsObsolete) { if (skipped == _skip && took < _take) { yield return(document); took++; } else if (skipped < _skip) { skipped++; } else if (took == _take) { break; } } } }
virtual public void Load(BlockSerializer bs) { LoadBlockData(bs); }
private void AddBlock(string path, string normConfig) { Monitor.JobStarted(); string fileName = path.Split('\\').Last(); try { Controller.log("Adding block " + fileName); using (BlockDatabaseDataContext dc = new BlockDatabaseDataContext()) { BlockSerializer bls = new BlockSerializer(); Block b = bls.DeserializeBlockFromZippedXML(path, "block.xml"); Controller.log("Loaded " + fileName); // at the moment the block data is normalized by dividing each "top" TOF through // by the integral of the corresponding "norm" TOF over the gate in the function below. // TODO: this could be improved! //b.Normalise(DemodulationConfig.GetStandardDemodulationConfig(normConfig, b).GatedDetectorExtractSpecs["norm"]); // add some of the single point data to the Shot TOFs so that it gets analysed string[] spvsToTOFulise = new string[] { "NorthCurrent", "SouthCurrent", "MiniFlux1", "MiniFlux2", "MiniFlux3", "ProbePD", "PumpPD" }; b.TOFuliseSinglePointData(spvsToTOFulise); // extract the metadata and config into a DB object DBBlock dbb = new DBBlock(); dbb.cluster = (string)b.Config.Settings["cluster"]; dbb.clusterIndex = (int)b.Config.Settings["clusterIndex"]; dbb.include = false; dbb.eState = (bool)b.Config.Settings["eState"]; dbb.bState = (bool)b.Config.Settings["bState"]; try { dbb.rfState = (bool)b.Config.Settings["rfState"]; } catch (Exception) { // blocks from the old days had no rfState recorded in them. dbb.rfState = true; } dbb.ePlus = (double)b.Config.Settings["ePlus"]; dbb.eMinus = (double)b.Config.Settings["eMinus"]; dbb.blockTime = (DateTime)b.TimeStamp; byte[] bts = serializeAsByteArray(b.Config); dbb.configBytes = bts; // extract the TOFChannelSets List <string> detectorsToExtract = new List <string> { "top", "norm", "magnetometer", "gnd", "battery", "topNormed", "NorthCurrent", "SouthCurrent", "MiniFlux1", "MiniFlux2", "MiniFlux3", "ProbePD", "PumpPD" }; foreach (string detector in detectorsToExtract) { BlockDemodulator demod = new BlockDemodulator(); TOFChannelSet tcs = demod.TOFDemodulateBlock(b, b.detectors.IndexOf(detector), true); byte[] tcsBytes = serializeAsByteArray(tcs); DBTOFChannelSet t = new DBTOFChannelSet(); t.tcsData = tcsBytes; t.detector = detector; t.FileID = Guid.NewGuid(); dbb.DBTOFChannelSets.Add(t); } Controller.log("Demodulated " + fileName); // add to the database dc.DBBlocks.InsertOnSubmit(dbb); dc.SubmitChanges(); Controller.log("Added " + fileName); } } catch (Exception e) { Controller.errorLog("Error adding " + fileName); Controller.errorLog("Error adding block " + path + "\n" + e.StackTrace); } finally { Monitor.JobFinished(); } }
public MappedTrieReader(Stream stream) { _stream = stream; _blockSize = Serializer.SizeOfNode() + BlockSerializer.SizeOfBlock(); _leaveOpen = true; }
protected void LoadBlockData(BlockSerializer bs) { isTransparent = bs.isTransparent; }
protected override BlockInfo Deserialize(long offset, int size, Stream stream) { stream.Seek(offset, SeekOrigin.Begin); return(BlockSerializer.DeserializeBlock(stream)); }