public void Start()
{
byte[] request;
try
{
while (true)
{
Log.Info("document server idle");
Socket handler = _listener.Accept();
request = new byte[sizeof(long) + sizeof(int)];
int received = handler.Receive(request);
Log.InfoFormat("received a {0} byte request", received);
var address = BlockSerializer.DeserializeBlock(request, 0);
byte[] response = FindInCache(address);
handler.Send(response);
handler.Shutdown(SocketShutdown.Both);
handler.Dispose();
Log.InfoFormat("responded with a {0} byte document", response.Length);
}
}
catch (Exception e)
{
Log.Info(e.ToString());
}
}
public static LcrsNode DeserializeNode(Stream stream)
{
var blockArr = new byte[SizeOfNode()];
stream.Read(blockArr, 0, blockArr.Length);
if (!BitConverter.IsLittleEndian)
{
Array.Reverse(blockArr, 0, sizeof(char));
Array.Reverse(blockArr, sizeof(char), sizeof(short));
Array.Reverse(blockArr, sizeof(char) + sizeof(short), sizeof(int));
}
var val = BitConverter.ToChar(blockArr, 0);
var haveSibling = BitConverter.ToBoolean(blockArr, sizeof(char));
var haveChild = BitConverter.ToBoolean(blockArr, sizeof(char) + sizeof(byte));
var eow = BitConverter.ToBoolean(blockArr, sizeof(char) + 2 * sizeof(byte));
var depth = BitConverter.ToInt16(blockArr, sizeof(char) + 3 * sizeof(byte));
var weight = BitConverter.ToInt32(blockArr, sizeof(char) + 3 * sizeof(byte) + sizeof(short));
var block = BlockSerializer.DeserializeBlock(stream);
return(new LcrsNode(
val,
haveSibling,
haveChild,
eow,
depth,
weight,
block));
}
public static void Serialize(this LcrsNode node, Stream stream)
{
var valBytes = BitConverter.GetBytes(node.Value);
var haveSiblingByte = EncodedBoolean[node.HaveSibling];
var haveChildByte = EncodedBoolean[node.HaveChild];
var eowByte = EncodedBoolean[node.EndOfWord];
var depthBytes = BitConverter.GetBytes(node.Depth);
var weightBytes = BitConverter.GetBytes(node.Weight);
if (!BitConverter.IsLittleEndian)
{
Array.Reverse(valBytes);
Array.Reverse(depthBytes);
Array.Reverse(weightBytes);
}
stream.Write(valBytes, 0, valBytes.Length);
stream.WriteByte(haveSiblingByte);
stream.WriteByte(haveChildByte);
stream.WriteByte(eowByte);
stream.Write(depthBytes, 0, depthBytes.Length);
stream.Write(weightBytes, 0, weightBytes.Length);
if (!BlockSerializer.Serialize(node.PostingsAddress, stream) && node.EndOfWord)
{
throw new InvalidOperationException("cannot store word without posting address");
}
}
override public void Load(BlockSerializer bs)
{
LoadBlockData(bs);
SurfaceBlockSerializer sbs = new SurfaceBlockSerializer();
GameMaster.DeserializeByteArray <SurfaceBlockSerializer>(bs.specificData, ref sbs);
LoadSurfaceBlockData(sbs);
if (sbs.haveStructures)
{
foreach (StructureSerializer ss in sbs.structuresList)
{
if (ss.id != Structure.PLANT_ID)
{
Structure s = Structure.GetStructureByID(ss.id);
if (s != null)
{
s.Load(ss, this);
}
}
else
{
Plant p = Plant.GetNewPlant(ss.id);
if (p != null)
{
p.Load(ss, this);
}
}
}
}
}
override public void Load(BlockSerializer bs)
{
LoadBlockData(bs);
CaveBlockSerializer cbs = new CaveBlockSerializer();
GameMaster.DeserializeByteArray <CaveBlockSerializer>(bs.specificData, ref cbs);
LoadCaveBlockData(cbs);
}
public Searcher(IReadSessionFactory sessionFactory)
{
_parser = new QueryParser();
_scorerFactory = new TfIdfFactory();
_versions = Util.GetIndexVersionListInChronologicalOrder(sessionFactory.DirectoryName);
_blockSize = BlockSerializer.SizeOfBlock();
_sessionFactory = sessionFactory;
}
public Searcher(string directory, IQueryParser parser = null, IScoringSchemeFactory scorerFactory = null, IReadSessionFactory sessionFactory = null)
{
_parser = parser ?? new QueryParser();
_scorerFactory = scorerFactory ?? new TfIdfFactory();
_versions = Util.GetIndexVersionListInChronologicalOrder(directory);
_blockSize = BlockSerializer.SizeOfBlock();
_sessionFactory = sessionFactory ?? new FullTextReadSessionFactory(directory);
}
// This method is thread-safe.
//public void AddBlock(string path, string[] demodulationConfigs)
//{
// string[] splitPath = path.Split('\\');
// log("Loading block " + splitPath[splitPath.Length - 1]);
// BlockSerializer bs = new BlockSerializer();
// Block b = bs.DeserializeBlockFromZippedXML(path, "block.xml");
// AddBlock(b, demodulationConfigs);
//}
public Block LoadBlockFromFile(string path)
{
string[] splitPath = path.Split('\\');
log("Loading block " + splitPath[splitPath.Length - 1]);
BlockSerializer bs = new BlockSerializer();
Block b = bs.DeserializeBlockFromZippedXML(path, "block.xml");
return(b);
}
public void TestSerialize()
{
using StreamReader r = new StreamReader("../../../Fixtures/block.json");
string json = r.ReadToEnd();
Block block = JsonConvert.DeserializeObject <Block>(json);
var expected = "0x2403000014000000e4000000cc010000160300000000000033cf3d1dc816fd3c72010000d50a0000000000000400001b00300200009dd635fda0e093e537b7eea56bcf7272cb107170330a30da3270e0dd0daf027b8e8c8951749096f7a1e86b4204655a6e52b381468faa3b6e2dba699dd586862ddad1ff8aba40d0aa41a8a12f4ac61a03c1df244501b8650cd702f10f3fcdf3136b809c54f8612326529ec47945301cb97c442a32e46c7390f9165e31df8ba98437323ea9c2a42e4556e19a9e8723000918d97226c000000075093a730aff06f3070631d8a8cc2de0a8ea1c267f942ce800000008000000e00000000c000000dc0000000000000033cf3d1db39cfc3c72010000d40a0000000000000400001a003002003e6936ef975d2f77055f473d0b661bf9dcc6604fdca1e0018bd3468fe7ff46299bd730ba9f1c57da5970c737914711156ab921ae205b4855197d7089556a84af000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006b76a1fd3fc2a42e75b671879e8723003e3798c210c0000000d872ce710aff0620e103220145a7c17e5fa6b232670856000000004a01000008000000420100000c000000d9000000cd0000001c00000020000000240000002800000058000000c100000000000000000000000000000001000000d50a0000000000000000000000000000000000000000000000000000000000000000000000000000ffffffff690000000800000061000000100000001800000061000000fa35509053000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce80114000000da648442dbb7347e467d1d09da13e5cd3a0ef0e10c000000080000000000000069000000080000005d0000005d0000000c00000055000000490000001000000030000000310000009bd7e06f3ecf4be0f2fcd2188b23f1b9fcc88e5d4b65a8637b17723bbda3cce80114000000da648442dbb7347e467d1d09da13e5cd3a0ef0e104000000deadbeef01000000a08d86ccf2dd38685637";
Assert.Equal(BlockSerializer.HexStringToBytes(expected), new BlockSerializer(block).Serialize());
}
public async Task <Block> Invoke()
{
var request = new GetLastBlockRequestMessage();
var response = await Invoke(request);
var bs = new BlockSerializer();
return(bs.Deserialize(response.Result));
}
public Searcher(string directory, long version, QueryParser parser, IScoringSchemeFactory scorerFactory, IReadSessionFactory sessionFactory = null)
{
_directory = directory;
_parser = parser;
_scorerFactory = scorerFactory;
_versions = new[] { BatchInfo.Load(Path.Combine(directory, version + ".ix")) };
_blockSize = BlockSerializer.SizeOfBlock();
_sessionFactory = sessionFactory ?? new ReadSessionFactory(directory);
}
// This method is thread-safe.
public void AddBlock(string path, string[] demodulationConfigs)
{
string[] splitPath = path.Split('\\');
log("Loading block " + splitPath[splitPath.Length - 1]);
BlockSerializer bs = new BlockSerializer();
Block b = bs.DeserializeBlockFromZippedXML(path, "block.xml");
AddBlock(b, demodulationConfigs);
}
public async Task <Block> Invoke(BigInteger height)
{
var request = new GetBlockByHeightRequestMessage(height);
var response = await Invoke(request);
var bs = new BlockSerializer();
return(bs.Deserialize(response.Result));
}
public MappedTrieReader(string fileName)
{
var dir = Path.GetDirectoryName(fileName);
var version = Path.GetFileNameWithoutExtension(fileName);
_stream = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read, 4096 * 1, FileOptions.SequentialScan);
_blockSize = Serializer.SizeOfNode() + BlockSerializer.SizeOfBlock();
}
protected BlockSerializer GetBlockSerializer()
{
BlockSerializer bs = new BlockSerializer();
bs.type = type;
bs.isTransparent = isTransparent;
bs.pos = pos;
bs.material_id = material_id;
return(bs);
}
public Searcher(string directory, QueryParser parser, IScoringSchemeFactory scorerFactory, IReadSessionFactory sessionFactory = null)
{
_directory = directory;
_parser = parser;
_scorerFactory = scorerFactory;
_versions = Util.GetIndexFileNamesInChronologicalOrder(directory)
.Select(f => BatchInfo.Load(f)).ToArray();
_blockSize = BlockSerializer.SizeOfBlock();
_sessionFactory = sessionFactory ?? new ReadSessionFactory(directory);
}
internal void ActuallyTestLiveAnalysis()
{
for (int i = 0; i <= 44; i++)
{
string fileRoot = Environs.FileSystem.Paths["edmDataPath"] + "\\2019\\November2019\\13Nov1903_";
BlockSerializer bs = new BlockSerializer();
Block b = bs.DeserializeBlockFromZippedXML(fileRoot + (i.ToString()) + ".zip", "block.xml");
AcquisitionFinished(b);
}
}
internal void ActuallyTestLiveAnalysis()
{
for (int i = 1; i <= 20; i++)
{
string fileRoot = Environs.FileSystem.Paths["edmDataPath"] + "\\2009\\April2009\\30apr0900_";
BlockSerializer bs = new BlockSerializer();
Block b = bs.DeserializeBlockFromZippedXML(fileRoot + (i.ToString()) + ".zip", "block.xml");
AcquisitionFinished(b);
}
}
override public BlockSerializer Save()
{
BlockSerializer bs = GetBlockSerializer();
using (System.IO.MemoryStream stream = new System.IO.MemoryStream())
{
new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter().Serialize(stream, GetSurfaceBlockSerializer());
bs.specificData = stream.ToArray();
}
return(bs);
}
public ReadSession(
BatchInfo version,
PostingsReader postingsReader,
DocHashReader docHashReader,
DocumentAddressReader addressReader,
Stream stream)
{
Version = version;
_stream = stream;
_docHashReader = docHashReader;
_postingsReader = postingsReader;
_addressReader = addressReader;
_blockSize = BlockSerializer.SizeOfBlock();
}
public ReadSession(
SegmentInfo version,
DocHashReader docHashReader,
BlockInfoReader addressReader,
Stream stream)
{
Version = version;
Stream = stream;
DocHashReader = docHashReader;
AddressReader = addressReader;
BlockSize = BlockSerializer.SizeOfBlock();
Stream.Seek(Version.KeyIndexOffset, SeekOrigin.Begin);
KeyIndex = DocumentSerializer.ReadKeyIndex(Stream, Version.KeyIndexSize);
}
override public BlockSerializer Save()
{
BlockSerializer bs = GetBlockSerializer();
CaveBlockSerializer cbs = new CaveBlockSerializer();
cbs.upMaterial_ID = material_id;
cbs.haveSurface = haveSurface;
cbs.surfaceBlockSerializer = GetSurfaceBlockSerializer();
using (System.IO.MemoryStream stream = new System.IO.MemoryStream())
{
new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter().Serialize(stream, cbs);
bs.specificData = stream.ToArray();
}
return(bs);
}
//public void TOFDemodulateBlocks(string[] blockFiles, string savePath)
//{
// // first of all test that the save location exists to avoid later disappointment.
// if (!Directory.Exists(Path.GetDirectoryName(savePath)))
// {
// log("Save path does not exist!!");
// return;
// }
// // initialise the accumulator
// tcsga = new TOFChannelSetGroupAccumulator();
// // queue the blocks - the last block analysed will take care of saving the results.
// foreach (string blockFile in blockFiles)
// {
// tofDemodulateParams tdp = new tofDemodulateParams();
// tdp.blockPath = blockFile;
// tdp.savePath = savePath;
// threadManager.AddToQueue(TOFDemodulateThreadWrapper, tdp);
// }
//}
//private void TOFDemodulateBlock(string blockPath, string savePath)
//{
// BlockSerializer bs = new BlockSerializer();
// string[] splitPath = blockPath.Split('\\');
// log("Loading block " + splitPath[splitPath.Length - 1]);
// Block b = bs.DeserializeBlockFromZippedXML(blockPath, "block.xml");
// log("Demodulating block " + b.Config.Settings["cluster"] + " - " + b.Config.Settings["clusterIndex"]);
// BlockTOFDemodulator btd = new BlockTOFDemodulator();
// TOFChannelSet tcs = btd.TOFDemodulateBlock(b, 0, true);
// log("Accumulating block " + b.Config.Settings["cluster"] + " - " + b.Config.Settings["clusterIndex"]);
// lock (accumulatorLock) tcsga.Add(tcs);
// // are we the last block to be added? If so, it's our job to save the results
// if (threadManager.RemainingJobs == 1)
// {
// // this lock should not be needed
// lock(accumulatorLock)
// {
// TOFChannelSetGroup tcsg = tcsga.GetResult();
// Stream fileStream = new FileStream(savePath, FileMode.Create);
// (new BinaryFormatter()).Serialize(fileStream, tcsg);
// fileStream.Close();
// }
// }
//}
//private void TOFDemodulateThreadWrapper(object parametersIn)
//{
// threadManager.QueueItemWrapper(delegate(object parms)
// {
// tofDemodulateParams parameters = (tofDemodulateParams)parms;
// TOFDemodulateBlock(parameters.blockPath, parameters.savePath);
// },
// parametersIn
// );
//}
//private struct tofDemodulateParams
//{
// public string blockPath;
// public string savePath;
// // this struct has a ToString method defined for error reporting porpoises.
// public override string ToString()
// {
// return blockPath;
// }
//}
#endregion
#region Testing
// Somewhere for SirCachealot to store test results that's accessible by Mathematica.
// Makes debugging easier and is needed as a workaround for the constant Mathematica
// NET/Link errors.
// public TOFChannelSetGroup ChanSetGroup;
// workarounds for NET/Link bugs
//public TOFChannelSet GetAveragedChannelSet(bool eSign, bool bSign, bool rfSign)
//{
// return ChanSetGroup.AverageChannelSetSignedByMachineState(eSign, bSign, rfSign);
//}
//public void LoadChannelSetGroup(string path)
//{
// BinaryFormatter bf = new BinaryFormatter();
// FileStream fs = new FileStream(path, FileMode.Open);
// ChanSetGroup = (TOFChannelSetGroup)bf.Deserialize(fs);
// fs.Close();
//}
public void Test1()
{
BlockSerializer bs = new BlockSerializer();
Block b = bs.DeserializeBlockFromZippedXML(
"C:\\Users\\jony\\Files\\Data\\SEDM\\v3\\2009\\October2009\\01Oct0900_0.zip", "block.xml");
BlockDemodulator bd = new BlockDemodulator();
DemodulatedBlock db = bd.DemodulateBlockNL(b,
DemodulationConfig.GetStandardDemodulationConfig("cgate11Fixed", b));
//JsonSerializer serializer = new JsonSerializer();
//using (StreamWriter sw = new StreamWriter("c:\\Users\\jony\\Desktop\\test.json"))
//using (JsonWriter writer = new JsonTextWriter(sw))
//{
// serializer.Serialize(writer, b.Config);
//}
//bs.SerializeBlockAsJSON("c:\\Users\\jony\\Desktop\\test.json", b);
}
private IEnumerable <DocumentTableRow> StreamDocuments()
{
var skipped = 0;
var took = 0;
for (int docId = 0; docId < _ix.DocumentCount; docId++)
{
var hash = _hashReader.Read(docId);
var address = _addressReader.Read(new[]
{
new BlockInfo(docId * BlockSerializer.SizeOfBlock(), BlockSerializer.SizeOfBlock())
}).First();
var document = _documentReader.Read(new List <BlockInfo> {
address
}).First();
if (!hash.IsObsolete)
{
if (skipped == _skip && took < _take)
{
yield return(document);
took++;
}
else if (skipped < _skip)
{
skipped++;
}
else if (took == _take)
{
break;
}
}
}
}
virtual public void Load(BlockSerializer bs)
{
LoadBlockData(bs);
}
private void AddBlock(string path, string normConfig)
{
Monitor.JobStarted();
string fileName = path.Split('\\').Last();
try
{
Controller.log("Adding block " + fileName);
using (BlockDatabaseDataContext dc = new BlockDatabaseDataContext())
{
BlockSerializer bls = new BlockSerializer();
Block b = bls.DeserializeBlockFromZippedXML(path, "block.xml");
Controller.log("Loaded " + fileName);
// at the moment the block data is normalized by dividing each "top" TOF through
// by the integral of the corresponding "norm" TOF over the gate in the function below.
// TODO: this could be improved!
//b.Normalise(DemodulationConfig.GetStandardDemodulationConfig(normConfig, b).GatedDetectorExtractSpecs["norm"]);
// add some of the single point data to the Shot TOFs so that it gets analysed
string[] spvsToTOFulise = new string[] { "NorthCurrent", "SouthCurrent", "MiniFlux1",
"MiniFlux2", "MiniFlux3", "ProbePD", "PumpPD" };
b.TOFuliseSinglePointData(spvsToTOFulise);
// extract the metadata and config into a DB object
DBBlock dbb = new DBBlock();
dbb.cluster = (string)b.Config.Settings["cluster"];
dbb.clusterIndex = (int)b.Config.Settings["clusterIndex"];
dbb.include = false;
dbb.eState = (bool)b.Config.Settings["eState"];
dbb.bState = (bool)b.Config.Settings["bState"];
try
{
dbb.rfState = (bool)b.Config.Settings["rfState"];
}
catch (Exception)
{
// blocks from the old days had no rfState recorded in them.
dbb.rfState = true;
}
dbb.ePlus = (double)b.Config.Settings["ePlus"];
dbb.eMinus = (double)b.Config.Settings["eMinus"];
dbb.blockTime = (DateTime)b.TimeStamp;
byte[] bts = serializeAsByteArray(b.Config);
dbb.configBytes = bts;
// extract the TOFChannelSets
List <string> detectorsToExtract = new List <string>
{
"top", "norm", "magnetometer", "gnd", "battery", "topNormed", "NorthCurrent", "SouthCurrent",
"MiniFlux1", "MiniFlux2", "MiniFlux3", "ProbePD", "PumpPD"
};
foreach (string detector in detectorsToExtract)
{
BlockDemodulator demod = new BlockDemodulator();
TOFChannelSet tcs = demod.TOFDemodulateBlock(b, b.detectors.IndexOf(detector), true);
byte[] tcsBytes = serializeAsByteArray(tcs);
DBTOFChannelSet t = new DBTOFChannelSet();
t.tcsData = tcsBytes;
t.detector = detector;
t.FileID = Guid.NewGuid();
dbb.DBTOFChannelSets.Add(t);
}
Controller.log("Demodulated " + fileName);
// add to the database
dc.DBBlocks.InsertOnSubmit(dbb);
dc.SubmitChanges();
Controller.log("Added " + fileName);
}
}
catch (Exception e)
{
Controller.errorLog("Error adding " + fileName);
Controller.errorLog("Error adding block " + path + "\n" + e.StackTrace);
}
finally
{
Monitor.JobFinished();
}
}
public MappedTrieReader(Stream stream)
{
_stream = stream;
_blockSize = Serializer.SizeOfNode() + BlockSerializer.SizeOfBlock();
_leaveOpen = true;
}
protected void LoadBlockData(BlockSerializer bs)
{
isTransparent = bs.isTransparent;
}
protected override BlockInfo Deserialize(long offset, int size, Stream stream)
{
stream.Seek(offset, SeekOrigin.Begin);
return(BlockSerializer.DeserializeBlock(stream));
}
