public async Task Can_Receive_Query_Response_On_Observer()
{
var recipientPeerId = PeerIdHelper.GetPeerId();
var hp = new HashProvider(HashingAlgorithm.GetAlgorithmMetadata("keccak-256"));
var lastDeltaHash = hp.ComputeMultiHash(ByteUtil.GenerateRandomByteArray(32));
var collection = new List <DeltaIndex>();
//// this matches the fake mock
for (uint x = 0; x < 10; x++)
{
var delta = new Delta
{
PreviousDeltaDfsHash = lastDeltaHash.Digest.ToByteString()
};
var index = new DeltaIndex
{
Height = 10,
Cid = delta.ToByteString()
};
collection.Add(index);
lastDeltaHash = hp.ComputeMultiHash(ByteUtil.GenerateRandomByteArray(32));
}
var deltaHistoryResponse = new PeerDeltaHistoryResponse(recipientPeerId, collection);
_peerDeltaHistoryRequest.DeltaHistoryResponseMessageStreamer.OnNext(deltaHistoryResponse);
var response = await _peerDeltaHistoryRequest.DeltaHistoryAsync(recipientPeerId).ConfigureAwait(false);
response.DeltaCid.Count.Should().Be(10);
}
/// <exception cref="NGit.Errors.MissingObjectException"></exception>
/// <exception cref="NGit.Errors.IncorrectObjectTypeException"></exception>
/// <exception cref="System.IO.IOException"></exception>
/// <exception cref="NGit.Errors.LargeObjectException"></exception>
private DeltaIndex Index(DeltaWindowEntry ent)
{
DeltaIndex idx = ent.index;
if (idx == null)
{
try
{
idx = new DeltaIndex(Buffer(ent));
}
catch (OutOfMemoryException noMemory)
{
LargeObjectException.OutOfMemory e;
e = new LargeObjectException.OutOfMemory(noMemory);
e.SetObjectId(ent.@object);
throw e;
}
if (0 < maxMemory)
{
loaded += idx.GetIndexSize() - idx.GetSourceSize();
}
ent.index = idx;
}
return(idx);
}
private void Scan(byte[] raw, int end)
{
// We scan the input backwards, and always insert onto the
// front of the chain. This ensures that chains will have lower
// offsets at the front of the chain, allowing us to prefer the
// earlier match rather than the later match.
//
int lastHash = 0;
int ptr = end - DeltaIndex.BLKSZ;
do
{
int key = DeltaIndex.HashBlock(raw, ptr);
int tIdx = key & tableMask;
int head = table[tIdx];
if (head != 0 && lastHash == key)
{
// Two consecutive blocks have the same content hash,
// prefer the earlier block because we want to use the
// longest sequence we can during encoding.
//
entries[head] = (((long)key) << 32) | ptr;
}
else
{
int eIdx = ++entryCnt;
entries[eIdx] = (((long)key) << 32) | ptr;
next[eIdx] = head;
table[tIdx] = eIdx;
}
lastHash = key;
ptr -= DeltaIndex.BLKSZ;
}while (0 <= ptr);
}
public virtual void TestLimitObjectSize_Length130InsertFails()
{
src = GetRng().NextBytes(130);
dst = GetRng().NextBytes(130);
DeltaIndex di = new DeltaIndex(src);
NUnit.Framework.Assert.IsFalse(di.Encode(actDeltaBuf, dst, src.Length));
}
public virtual void TestIndexSize()
{
src = GetRng().NextBytes(1024);
DeltaIndex di = new DeltaIndex(src);
NUnit.Framework.Assert.AreEqual(1860, di.GetIndexSize());
NUnit.Framework.Assert.AreEqual("DeltaIndex[2 KiB]", di.ToString());
}
public virtual void TestLimitObjectSize_Length130CopyFails()
{
src = GetRng().NextBytes(130);
Copy(0, 130);
dst = dstBuf.ToByteArray();
// The header requires 4 bytes for these objects, so a target length
// of 5 is bigger than the copy instruction and should cause an abort.
//
DeltaIndex di = new DeltaIndex(src);
NUnit.Framework.Assert.IsFalse(di.Encode(actDeltaBuf, dst, 5));
NUnit.Framework.Assert.AreEqual(4, actDeltaBuf.Size());
}
public virtual void TestLimitObjectSize_Length130CopyOk()
{
src = GetRng().NextBytes(130);
Copy(0, 130);
dst = dstBuf.ToByteArray();
DeltaIndex di = new DeltaIndex(src);
NUnit.Framework.Assert.IsTrue(di.Encode(actDeltaBuf, dst, dst.Length));
byte[] actDelta = actDeltaBuf.ToByteArray();
byte[] expDelta = expDeltaBuf.ToByteArray();
NUnit.Framework.Assert.AreEqual(BinaryDelta.Format(expDelta, false), BinaryDelta.
Format(actDelta, false));
}
protected virtual void SetUp()
{
underlying = new HashIndex <string>();
underlying.Add("foo0");
underlying.Add("foo1");
underlying.Add("foo2");
underlying.Add("foo3");
underlying.Add("foo4");
NUnit.Framework.Assert.AreEqual(5, underlying.Count);
spillover = new DeltaIndex <string>(underlying);
spillover.Add("foo1");
spillover.Add("foo5");
spillover.Add("foo6");
}
/// <exception cref="System.IO.IOException"></exception>
private void DoTest()
{
dst = dstBuf.ToByteArray();
DeltaIndex di = new DeltaIndex(src);
di.Encode(actDeltaBuf, dst);
byte[] actDelta = actDeltaBuf.ToByteArray();
byte[] expDelta = expDeltaBuf.ToByteArray();
NUnit.Framework.Assert.AreEqual(BinaryDelta.Format(expDelta, false), BinaryDelta.
Format(actDelta, false));
//
NUnit.Framework.Assert.IsTrue(actDelta.Length > 0, "delta is not empty");
NUnit.Framework.Assert.AreEqual(src.Length, BinaryDelta.GetBaseSize(actDelta));
NUnit.Framework.Assert.AreEqual(dst.Length, BinaryDelta.GetResultSize(actDelta));
CollectionAssert.AreEquivalent(dst, BinaryDelta.Apply(src, actDelta));
}
public async Task Can_Process_DeltaHeightRequest_Correctly()
{
var fakeContext = Substitute.For <IChannelHandlerContext>();
var deltaHistoryRequestMessage = new DeltaHistoryRequest();
var channeledAny = new ObserverDto(fakeContext,
deltaHistoryRequestMessage.ToProtocolMessage(PeerIdHelper.GetPeerId(),
CorrelationId.GenerateCorrelationId()
)
);
var observableStream = new[] { channeledAny }.ToObservable(_testScheduler);
_deltaHistoryRequestObserver.StartObserving(observableStream);
_testScheduler.Start();
var response = new DeltaHistoryResponse();
var hp = new HashProvider(HashingAlgorithm.GetAlgorithmMetadata("keccak-256"));
var lastDeltaHash = hp.ComputeMultiHash(ByteUtil.GenerateRandomByteArray(32));
for (uint x = 0; x < 10; x++)
{
var delta = new Delta
{
PreviousDeltaDfsHash = lastDeltaHash.Digest.ToByteString()
};
var index = new DeltaIndex
{
Height = 10,
Cid = delta.ToByteString()
};
response.DeltaIndex.Add(index);
lastDeltaHash = hp.ComputeMultiHash(ByteUtil.GenerateRandomByteArray(32));
}
await fakeContext.Channel.ReceivedWithAnyArgs(1)
.WriteAndFlushAsync(response.ToProtocolMessage(PeerIdHelper.GetPeerId(), CorrelationId.GenerateCorrelationId())).ConfigureAwait(false);
_subbedLogger.ReceivedWithAnyArgs(1);
}
/// <param name="deltaHeightRequest"></param>
/// <param name="channelHandlerContext"></param>
/// <param name="senderPeerId"></param>
/// <param name="correlationId"></param>
/// <returns></returns>
protected override DeltaHistoryResponse HandleRequest(DeltaHistoryRequest deltaHeightRequest,
IChannelHandlerContext channelHandlerContext,
PeerId senderPeerId,
ICorrelationId correlationId)
{
Guard.Argument(deltaHeightRequest, nameof(deltaHeightRequest)).NotNull();
Guard.Argument(channelHandlerContext, nameof(channelHandlerContext)).NotNull();
Guard.Argument(senderPeerId, nameof(senderPeerId)).NotNull();
Logger.Debug("PeerId: {0} requests: {1} deltas from height: {2}", senderPeerId, deltaHeightRequest.Range, deltaHeightRequest.Height);
// @TODO actually get this from some where for now provide a bunch of fake linked deltas
/////////////////////////////////////////////////////////////////////////////////////////////////////
var response = new DeltaHistoryResponse();
var lastDeltaHash = _hashProvider.ComputeMultiHash(ByteUtil.GenerateRandomByteArray(32));
for (uint x = 0; x < deltaHeightRequest.Range; x++)
{
var delta = new Delta
{
PreviousDeltaDfsHash = lastDeltaHash.Digest.ToByteString()
};
var index = new DeltaIndex
{
Height = deltaHeightRequest.Height,
Cid = delta.ToByteString()
};
response.Result.Add(index);
lastDeltaHash = _hashProvider.ComputeMultiHash(ByteUtil.GenerateRandomByteArray(32));
}
/////////////////////////////////////////////////////////////////////////////////////////////////////
return(response);
}
// CDM 2007: I wonder what this does differently from segmentWordsWithMarkov???
private List <TaggedWord> BasicSegmentWords(string s)
{
// We don't want to accidentally register words that we don't know
// about in the wordIndex, so we wrap it with a DeltaIndex
DeltaIndex <string> deltaWordIndex = new DeltaIndex <string>(wordIndex);
int length = s.Length;
// Set<String> POSes = (Set<String>) POSDistribution.keySet(); // 1.5
// best score of span
double[][] scores = new double[length][];
// best (last index of) first word for this span
int[][] splitBacktrace = new int[length][];
// best tag for word over this span
int[][] POSbacktrace = new int[length][];
for (int i = 0; i < length; i++)
{
Arrays.Fill(scores[i], double.NegativeInfinity);
}
// first fill in word probabilities
for (int diff = 1; diff <= 10; diff++)
{
for (int start = 0; start + diff <= length; start++)
{
int end = start + diff;
StringBuilder wordBuf = new StringBuilder();
for (int pos = start; pos < end; pos++)
{
wordBuf.Append(s[pos]);
}
string word = wordBuf.ToString();
// for (String tag : POSes) { // 1.5
foreach (string tag in POSes)
{
IntTaggedWord itw = new IntTaggedWord(word, tag, deltaWordIndex, tagIndex);
double newScore = lex.Score(itw, 0, word, null) + Math.Log(lex.GetPOSDistribution().ProbabilityOf(tag));
if (newScore > scores[start][end])
{
scores[start][end] = newScore;
splitBacktrace[start][end] = end;
POSbacktrace[start][end] = itw.Tag();
}
}
}
}
// now fill in word combination probabilities
for (int diff_1 = 2; diff_1 <= length; diff_1++)
{
for (int start = 0; start + diff_1 <= length; start++)
{
int end = start + diff_1;
for (int split = start + 1; split < end && split - start <= 10; split++)
{
if (splitBacktrace[start][split] != split)
{
continue;
}
// only consider words on left
double newScore = scores[start][split] + scores[split][end];
if (newScore > scores[start][end])
{
scores[start][end] = newScore;
splitBacktrace[start][end] = split;
}
}
}
}
IList <TaggedWord> words = new List <TaggedWord>();
int start_1 = 0;
while (start_1 < length)
{
int end = splitBacktrace[start_1][length];
StringBuilder wordBuf = new StringBuilder();
for (int pos = start_1; pos < end; pos++)
{
wordBuf.Append(s[pos]);
}
string word = wordBuf.ToString();
string tag = tagIndex.Get(POSbacktrace[start_1][end]);
words.Add(new TaggedWord(word, tag));
start_1 = end;
}
return(new List <TaggedWord>(words));
}
/// <summary>Do max language model markov segmentation.</summary>
/// <remarks>
/// Do max language model markov segmentation.
/// Note that this algorithm inherently tags words as it goes, but that
/// we throw away the tags in the final result so that the segmented words
/// are untagged. (Note: for a couple of years till Aug 2007, a tagged
/// result was returned, but this messed up the parser, because it could
/// use no tagging but the given tagging, which often wasn't very good.
/// Or in particular it was a subcategorized tagging which never worked
/// with the current forceTags option which assumes that gold taggings are
/// inherently basic taggings.)
/// </remarks>
/// <param name="s">A String to segment</param>
/// <returns>The list of segmented words.</returns>
private List <IHasWord> SegmentWordsWithMarkov(string s)
{
// We don't want to accidentally register words that we don't know
// about in the wordIndex, so we wrap it with a DeltaIndex
DeltaIndex <string> deltaWordIndex = new DeltaIndex <string>(wordIndex);
int length = s.Length;
// Set<String> POSes = (Set<String>) POSDistribution.keySet(); // 1.5
int numTags = POSes.Count;
// score of span with initial word of this tag
double[][][] scores = new double[length][][];
// best (length of) first word for this span with this tag
int[][][] splitBacktrace = new int[length][][];
// best tag for second word over this span, if first is this tag
int[][][] POSbacktrace = new int[length][][];
for (int i = 0; i < length; i++)
{
for (int j = 0; j < length + 1; j++)
{
Arrays.Fill(scores[i][j], double.NegativeInfinity);
}
}
// first fill in word probabilities
for (int diff = 1; diff <= 10; diff++)
{
for (int start = 0; start + diff <= length; start++)
{
int end = start + diff;
StringBuilder wordBuf = new StringBuilder();
for (int pos = start; pos < end; pos++)
{
wordBuf.Append(s[pos]);
}
string word = wordBuf.ToString();
foreach (string tag in POSes)
{
IntTaggedWord itw = new IntTaggedWord(word, tag, deltaWordIndex, tagIndex);
double score = lex.Score(itw, 0, word, null);
if (start == 0)
{
score += Math.Log(initialPOSDist.ProbabilityOf(tag));
}
scores[start][end][itw.Tag()] = score;
splitBacktrace[start][end][itw.Tag()] = end;
}
}
}
// now fill in word combination probabilities
for (int diff_1 = 2; diff_1 <= length; diff_1++)
{
for (int start = 0; start + diff_1 <= length; start++)
{
int end = start + diff_1;
for (int split = start + 1; split < end && split - start <= 10; split++)
{
foreach (string tag in POSes)
{
int tagNum = tagIndex.AddToIndex(tag);
if (splitBacktrace[start][split][tagNum] != split)
{
continue;
}
Distribution <string> rTagDist = markovPOSDists[tag];
if (rTagDist == null)
{
continue;
}
// this happens with "*" POS
foreach (string rTag in POSes)
{
int rTagNum = tagIndex.AddToIndex(rTag);
double newScore = scores[start][split][tagNum] + scores[split][end][rTagNum] + Math.Log(rTagDist.ProbabilityOf(rTag));
if (newScore > scores[start][end][tagNum])
{
scores[start][end][tagNum] = newScore;
splitBacktrace[start][end][tagNum] = split;
POSbacktrace[start][end][tagNum] = rTagNum;
}
}
}
}
}
}
int nextPOS = ArrayMath.Argmax(scores[0][length]);
List <IHasWord> words = new List <IHasWord>();
int start_1 = 0;
while (start_1 < length)
{
int split = splitBacktrace[start_1][length][nextPOS];
StringBuilder wordBuf = new StringBuilder();
for (int i_1 = start_1; i_1 < split; i_1++)
{
wordBuf.Append(s[i_1]);
}
string word = wordBuf.ToString();
// String tag = tagIndex.get(nextPOS);
// words.add(new TaggedWord(word, tag));
words.Add(new Word(word));
if (split < length)
{
nextPOS = POSbacktrace[start_1][length][nextPOS];
}
start_1 = split;
}
return(words);
}
public static DeltaIndexDao ToDao <TProto>(DeltaIndex protoBuff, IMapperProvider mapperProvider)
where TProto : IMessage
{
return(mapperProvider.Mapper.Map <DeltaIndexDao>(protoBuff));
}
private static long EstimateSize(ObjectToPack ent)
{
return(DeltaIndex.EstimateIndexSize(ent.GetWeight()));
}
internal LexicalizedParserQuery(LexicalizedParser parser)
{
this.op = parser.GetOp();
BinaryGrammar bg = parser.bg;
UnaryGrammar ug = parser.ug;
ILexicon lex = parser.lex;
IDependencyGrammar dg = parser.dg;
IIndex <string> stateIndex = parser.stateIndex;
IIndex <string> wordIndex = new DeltaIndex <string>(parser.wordIndex);
IIndex <string> tagIndex = parser.tagIndex;
this.debinarizer = new Debinarizer(op.forceCNF);
this.boundaryRemover = new BoundaryRemover();
if (op.doPCFG)
{
if (op.testOptions.iterativeCKY)
{
pparser = new IterativeCKYPCFGParser(bg, ug, lex, op, stateIndex, wordIndex, tagIndex);
}
else
{
pparser = new ExhaustivePCFGParser(bg, ug, lex, op, stateIndex, wordIndex, tagIndex);
}
}
else
{
pparser = null;
}
if (op.doDep)
{
dg.SetLexicon(lex);
if (!op.testOptions.useFastFactored)
{
dparser = new ExhaustiveDependencyParser(dg, lex, op, wordIndex, tagIndex);
}
else
{
dparser = null;
}
}
else
{
dparser = null;
}
if (op.doDep && op.doPCFG)
{
if (op.testOptions.useFastFactored)
{
MLEDependencyGrammar mledg = (MLEDependencyGrammar)dg;
int numToFind = 1;
if (op.testOptions.printFactoredKGood > 0)
{
numToFind = op.testOptions.printFactoredKGood;
}
bparser = new FastFactoredParser(pparser, mledg, op, numToFind, wordIndex, tagIndex);
}
else
{
IScorer scorer = new TwinScorer(pparser, dparser);
//Scorer scorer = parser;
if (op.testOptions.useN5)
{
bparser = new BiLexPCFGParser.N5BiLexPCFGParser(scorer, pparser, dparser, bg, ug, dg, lex, op, stateIndex, wordIndex, tagIndex);
}
else
{
bparser = new BiLexPCFGParser(scorer, pparser, dparser, bg, ug, dg, lex, op, stateIndex, wordIndex, tagIndex);
}
}
}
else
{
bparser = null;
}
subcategoryStripper = op.tlpParams.SubcategoryStripper();
}
internal virtual void Set(ObjectToPack @object)
{
this.@object = @object;
this.index = null;
this.buffer = null;
}
/// <summary>
/// Creates a new metadata source as a delta from a baseline metadata source
/// </summary>
/// <param name="baseline"></param>
public CompressedMetadataStore(CompressedMetadataStore baseline)
{
Version = CurrentVersion;
DeltaIndex = baseline.DeltaIndex + 1;
// Generate a file name that follows the indexed naming scheme.
if (DeltaIndex == 1)
{
// If this is the first delta from a baseline, take the file name and add -1 to the end (before the extension)
FilePath = $"{Path.GetFileNameWithoutExtension(baseline.FilePath)}-{DeltaIndex}{Path.GetExtension(baseline.FilePath)}";
}
else
{
// If this is a higher delta from a baseline, make sure the current file matches the naming scheme
if (!TryGetDeltaIndexFromFilePath(baseline.FilePath, out ulong parsedIndex, out string baseFileName))
{
throw new Exception("Baseline file naming scheme is corrupt. Expected [file name]-[delta index].zip.");
}
// If it does, add the version to the base file name, before the extension
FilePath = $"{baseFileName}-{DeltaIndex.ToString()}.zip";
}
OutputFile = new ZipOutputStream(File.Create(FilePath));
BaselineSource = baseline;
BaselineChecksum = baseline.Checksum;
// Record the end of the index range for the baseline
BaselineIndexesEnd = baseline.IndexToIdentity.Keys.Count - 1;
BaselineIdentities = new SortedSet <Identity>(baseline.IndexToIdentity.Values);
Identities = new SortedSet <Identity>(baseline.Identities);
IndexToIdentity = new Dictionary <int, Identity>(baseline.IndexToIdentity);
IdentityToIndex = new Dictionary <Identity, int>(baseline.IdentityToIndex);
ProductsTree = new Dictionary <int, List <int> >(baseline.ProductsTree);
UpdateTypeMap = new Dictionary <int, uint>(baseline.UpdateTypeMap);
Updates = new ConcurrentDictionary <Identity, Update>(baseline.Updates);
Categories = new ConcurrentDictionary <Identity, Update>(baseline.Categories);
// Create update placeholders
InstantiateUpdatePlaceholders();
UpstreamSource = baseline.UpstreamSource;
// Populate indexes
UpdatesIndex = Updates;
CategoriesIndex = Categories;
ProductsIndex = Categories.Values.OfType <Product>().ToDictionary(p => p.Identity);
ClassificationsIndex = Categories.Values.OfType <Classification>().ToDictionary(c => c.Identity);
DetectoidsIndex = Categories.Values.OfType <Detectoid>().ToDictionary(d => d.Identity);
Filter = BaselineSource.Filter;
CategoriesAnchor = BaselineSource.CategoriesAnchor;
UpstreamAccountGuid = BaselineSource.UpstreamAccountGuid;
UpstreamAccountName = BaselineSource.UpstreamAccountName;
UpdateTitlesIndex = new Dictionary <int, string>();
// Initialize bundles information
OnDeltaStore_InitializeBundes();
// Initialize prerequisites information
OnDeltaStore_InitializePrerequisites();
// Initialize update classification and product information
OnDeltaStore_InitializeProductClassification();
// Initialize files index
OnDeltaStore_InitializeFilesIndex();
// Initialize superseding index
OnDeltaStore_InitializeSupersededIndex();
// Initialize driver indexes
OnDeltaStore_InitializeDriversIndex();
// Initialize KB articles index
OnDeltaStore_InitializeKbArticles();
}
