public void Build(MetricDB db, Random rand, int num_refs, int K=7, int maxcand=1024, SequenceBuilder seq_builder=null)
{
var sample = new SampleSpace ("", db, num_refs, rand);
var sat = new SAT_Distal ();
sat.Build (sample, RandomSets.GetRandom());
this.Build (db, sat, K, maxcand, seq_builder);
}
/// <summary>
/// Build the index
/// </summary>
public virtual void Build(MetricDB db, int num_centers, Random rand, SequenceBuilder seq_builder = null)
{
this.DB = db;
var n = this.DB.Count;
// randomized has very good performance, even compared with more "intelligent" strategies
this.node_list = new List<Node> (num_centers);
var subset = RandomSets.GetRandomSubSet (num_centers, this.DB.Count, rand);
for (int centerID = 0; centerID < num_centers; ++centerID) {
this.node_list.Add (new Node (subset [centerID]));
}
var H = new HashSet<int> (subset);
for (int docID = 0; docID < n; ++docID) {
if (docID % 1000 == 0) {
Console.WriteLine ("== {0} {1}/{2}, num_centers: {3}, db: {4}", this, docID + 1, n, num_centers, db.Name);
}
if (H.Contains(docID)) {
continue;
}
var far = new Result(1);
for (var centerID = 0; centerID < num_centers; ++centerID) {
var node = this.node_list[centerID];
var d = this.DB.Dist(this.DB[node.refID], this.DB[docID]);
far.Push(centerID, -d);
}
var _far = far.First;
this.node_list[_far.docid].Add(docID, -_far.dist);
}
}
public void Build(string sa_name, SequenceBuilder seq_builder = null, BitmapFromBitStream bitmap_builder = null)
{
if (seq_builder == null) {
seq_builder = SequenceBuilders.GetSeqXLB_DiffSetRL2_64(16, 63);
}
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".structs"))) {
this.newF = RankSelectGenericIO.Load (Input);
if (bitmap_builder != null) {
var newF_stream = new BitStream32();
for (int i = 0; i < this.newF.Count; ++i) {
newF_stream.Write (this.newF.Access(i));
}
this.newF = bitmap_builder(new FakeBitmap(newF_stream));
}
int len = this.newF.Count1;
this.charT = new int[len];
// Console.WriteLine ("*****>> charT => {0} bytes", this.charT.Length * 4);
PrimitiveIO<int>.ReadFromFile (Input, len, this.charT);
}
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".psi"))) {
int seqlen = this.newF.Count;
var seq = new int[seqlen];
var L = new List<int>(this.N/this.Sigma + 1);
int curr = 0;
for (int i = 1; i <= this.AlphabetSize; i++) {
int next;
if (i == this.AlphabetSize) {
next = this.newF.Count;
} else {
next = this.newF.Select1 (i + 1);
}
int len = next - curr;
L.Clear();
PrimitiveIO<int>.ReadFromFile (Input, len, L);
for (int j = 0; j < len; ++j) {
var x = L[j];
try {
seq[ x ] = i - 1;
} catch (Exception e) {
Console.WriteLine ("== i: {0}, j: {1}, x: {2}, seq-count: {3}, len: {4}",
i, j, x, seq.Length, len);
throw e;
}
}
curr = next;
}
this.SeqPsi = seq_builder(seq, this.AlphabetSize);
}
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".samples"))) {
this.SA_sample_step = Input.ReadInt16 ();
this.SA_marked = RankSelectGenericIO.Load (Input);
var _samples = new ListIFS ();
_samples.Load (Input);
var _invsamples = new ListIFS ();
_invsamples.Load (Input);
this.SA_samples = _samples;
this.SA_invsamples = _invsamples;
}
}
/// <summary>
/// Build the index
/// </summary>
public virtual void Build(MetricDB db, int num_centers, Random rand, SequenceBuilder seq_builder = null)
{
this.DB = db;
var n = this.DB.Count;
// randomized has very good performance, even compared with more "intelligent" strategies
this.node_list = new List<Node> (num_centers);
var subset = RandomSets.GetRandomSubSet (num_centers, this.DB.Count, rand);
for (int centerID = 0; centerID < num_centers; ++centerID) {
this.node_list.Add (new Node (subset [centerID]));
}
var H = new HashSet<int> (subset);
for (int docID = 0; docID < n; ++docID) {
if (H.Contains(docID)) {
continue;
}
var near = new Result(1);
var far = new Result(1);
for (var centerID = 0; centerID < num_centers; ++centerID) {
var node = this.node_list[centerID];
var d = this.DB.Dist(this.DB[node.refID], this.DB[docID]);
near.Push(centerID, d);
far.Push(centerID, -d);
}
var _near = near.First;
var _far = far.First;
this.node_list[_near.ObjID].AddNear(docID, _near.Dist);
this.node_list[_far.ObjID].AddFar(docID, -_far.Dist);
}
}
public override void Build(LAESA idx, int num_pivs, int num_rings, SequenceBuilder seq_builder = null)
{
if (seq_builder == null) {
seq_builder = SequenceBuilders.GetIISeq(BitmapBuilders.GetSArray());
}
base.Build (idx, num_pivs, num_rings, seq_builder);
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(CamundaMap), BpmnModelConstants.CAMUNDA_ELEMENT_MAP).namespaceUri(CAMUNDA_NS).instanceProvider(new ModelTypeInstanceProviderAnonymousInnerClass());
SequenceBuilder sequenceBuilder = typeBuilder.sequence();
camundaEntryCollection = sequenceBuilder.elementCollection(typeof(CamundaEntry)).build();
typeBuilder.build();
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(Edge), DI_ELEMENT_EDGE).namespaceUri(DI_NS).extendsType(typeof(DiagramElement)).abstractType();
SequenceBuilder sequenceBuilder = typeBuilder.sequence();
waypointCollection = sequenceBuilder.elementCollection(typeof(Waypoint)).minOccurs(2).build();
typeBuilder.build();
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(CamundaFormData), CAMUNDA_ELEMENT_FORM_DATA).namespaceUri(BpmnModelConstants.CAMUNDA_NS).instanceProvider(new ModelTypeInstanceProviderAnonymousInnerClass());
SequenceBuilder sequenceBuilder = typeBuilder.sequence();
camundaFormFieldCollection = sequenceBuilder.elementCollection(typeof(CamundaFormField)).build();
typeBuilder.build();
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(SubConversation), BPMN_ELEMENT_SUB_CONVERSATION).namespaceUri(BPMN20_NS).extendsType(typeof(ConversationNode)).instanceProvider(new ModelTypeInstanceProviderAnonymousInnerClass());
SequenceBuilder sequenceBuilder = typeBuilder.sequence();
conversationNodeCollection = sequenceBuilder.elementCollection(typeof(ConversationNode)).build();
typeBuilder.build();
}
public void EmptyStringTest()
{
// arrange
string str = String.Empty;
//act
SequenceBuilder sb = new SequenceBuilder();
//assert
Assert.Null(sb.CheckEven(str));
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(CamundaProperties), CAMUNDA_ELEMENT_PROPERTIES).namespaceUri(CAMUNDA_NS).instanceProvider(new ModelTypeInstanceProviderAnonymousInnerClass());
SequenceBuilder sequenceBuilder = typeBuilder.sequence();
camundaPropertyCollection = sequenceBuilder.elementCollection(typeof(CamundaProperty)).build();
typeBuilder.build();
}
public void Load(Score score)
{
if (_view == null)
{
throw new ViewModelNotFoundException();
}
var sequence = new SequenceBuilder(score).Build();
_view.Load(sequence);
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(Event), BPMN_ELEMENT_EVENT).namespaceUri(BPMN20_NS).extendsType(typeof(FlowNode)).abstractType();
SequenceBuilder sequence = typeBuilder.sequence();
propertyCollection = sequence.elementCollection(typeof(Property)).build();
typeBuilder.build();
}
public static void BatchTest()
{
var recordGeneratorObv = Observable.Create <ExampleClass1>(x => ExampleClass1Factory.Subscribe(x));
var batchedRecordsStream = SequenceBuilder.Batch(recordGeneratorObv, x => x.Group);
batchedRecordsStream.Subscribe(x =>
{
Console.WriteLine($"Processing {x.Value.Count} records for batch group: {x.Key}");
});
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(Plane), DI_ELEMENT_PLANE).namespaceUri(DI_NS).extendsType(typeof(Node)).abstractType();
SequenceBuilder sequenceBuilder = typeBuilder.sequence();
diagramElementCollection = sequenceBuilder.elementCollection(typeof(DiagramElement)).build();
typeBuilder.build();
}
public void Build(MetricDB db, int sample_size, int num_instances, SequenceBuilder seq_builder = null)
{
this.DB = db;
this.lsc_indexes = new LSC[num_instances];
// IPermutation perm = null;
for (int i = 0; i < num_instances; ++i) {
var lsc = new LSC_H8 ();
lsc.Build(db, sample_size, seq_builder);
this.lsc_indexes[i] = lsc;
}
}
public virtual void Build(MetricDB db, int numcenters, int lambda, SequenceBuilder seq_builder = null)
{
var A = new List<Action>();
this.LC_LIST = new LC_RNN[lambda];
for (int i = 0; i < lambda; ++i) {
A.Add(this.BuildOneClosure(this.LC_LIST, i, db, numcenters, seq_builder));
}
var ops = new ParallelOptions();
ops.MaxDegreeOfParallelism = -1;
Parallel.ForEach(A, ops, (action) => action());
}
public void StartsWith_GivenSequenceWithMultipleSegments_ShouldReturnTrue()
{
//Arrange
var sequence = new SequenceBuilder <byte>().Append(new byte[] { 0x00, 0x01 }).Append(new byte[] { 0x02, 0x03 }).Build();
//Act
var actual = sequence.StartsWith(new byte[] { 0x00, 0x01, 0x02 });
//Assert
actual.Should().BeTrue();
}
public void EvenIndexTest()
{
//arrange
string str = "ababab";
string result = "aaa";
//act
SequenceBuilder sb = new SequenceBuilder();
//assert
Assert.Equal(sb.CheckEven(str), result);
}
public void KeyFormatterTests(string input)
{
var transformer = new Utf8KeyFormatter();
using var builder = new SequenceBuilder(MemoryPool <byte> .Shared);
transformer.Serialize(builder, input);
var expected = new UTF8Encoding(false).GetBytes(input);
Assert.Equal(expected, builder.Commit().ToArray());
}
public void StartsWith_GivenSequenceWithSingleSegment_ShouldReturnFalse()
{
//Arrange
var sequence = new SequenceBuilder <byte>().Append(new byte[] { 0x00, 0x02, 0x01 }).Build();
//Act
var actual = sequence.StartsWith(new byte[] { 0x00, 0x01 });
//Assert
actual.Should().BeFalse();
}
public uint Serialize(SequenceBuilder output, object?value)
{
try
{
return(DoSerialize(output, value));
}
catch (Exception e) when(!(e is SerializationException))
{
throw new SerializationException($"Exception '{e.Message}' while serializing '{value}'. See inner exception for details.", e);
}
}
public void KeyFormatterTests(string prefix, string key)
{
var transformer = new NamespacingKeyFormatter(prefix);
using var builder = new SequenceBuilder(MemoryPool <byte> .Shared);
transformer.Serialize(builder, key);
var expected = new UTF8Encoding(false).GetBytes(prefix + key);
Assert.Equal(expected, builder.Commit().ToArray());
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(CorrelationSubscription), BPMN_ELEMENT_CORRELATION_SUBSCRIPTION).namespaceUri(BPMN20_NS).extendsType(typeof(BaseElement)).instanceProvider(new ModelTypeInstanceProviderAnonymousInnerClass());
correlationKeyAttribute = typeBuilder.stringAttribute(BPMN_ATTRIBUTE_CORRELATION_KEY_REF).required().qNameAttributeReference(typeof(CorrelationKey)).build();
SequenceBuilder sequenceBuilder = typeBuilder.sequence();
correlationPropertyBindingCollection = sequenceBuilder.elementCollection(typeof(CorrelationPropertyBinding)).build();
typeBuilder.build();
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(CallConversation), BPMN_ELEMENT_CALL_CONVERSATION).namespaceUri(BPMN20_NS).extendsType(typeof(ConversationNode)).instanceProvider(new ModelTypeInstanceProviderAnonymousInnerClass());
calledCollaborationRefAttribute = typeBuilder.stringAttribute(BPMN_ATTRIBUTE_CALLED_COLLABORATION_REF).qNameAttributeReference(typeof(GlobalConversation)).build();
SequenceBuilder sequenceBuilder = typeBuilder.sequence();
participantAssociationCollection = sequenceBuilder.elementCollection(typeof(ParticipantAssociation)).build();
typeBuilder.build();
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(CamundaInputOutput), CAMUNDA_ELEMENT_INPUT_OUTPUT).namespaceUri(CAMUNDA_NS).instanceProvider(new ModelTypeInstanceProviderAnonymousInnerClass());
SequenceBuilder sequenceBuilder = typeBuilder.sequence();
camundaInputParameterCollection = sequenceBuilder.elementCollection(typeof(CamundaInputParameter)).build();
camundaOutputParameterCollection = sequenceBuilder.elementCollection(typeof(CamundaOutputParameter)).build();
typeBuilder.build();
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(LaneSet), BPMN_ELEMENT_LANE_SET).namespaceUri(BPMN20_NS).extendsType(typeof(BaseElement)).instanceProvider(new ModelTypeInstanceProviderAnonymousInnerClass());
nameAttribute = typeBuilder.stringAttribute(BPMN_ATTRIBUTE_NAME).build();
SequenceBuilder sequenceBuilder = typeBuilder.sequence();
laneCollection = sequenceBuilder.elementCollection(typeof(Lane)).build();
typeBuilder.build();
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(Resource), BPMN_ELEMENT_RESOURCE).namespaceUri(BPMN20_NS).extendsType(typeof(RootElement)).instanceProvider(new ModelTypeInstanceProviderAnonymousInnerClass());
nameAttribute = typeBuilder.stringAttribute(BPMN_ATTRIBUTE_NAME).required().build();
SequenceBuilder sequenceBuilder = typeBuilder.sequence();
resourceParameterCollection = sequenceBuilder.elementCollection(typeof(ResourceParameter)).build();
typeBuilder.build();
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(CorrelationKey), BPMN_ELEMENT_CORRELATION_KEY).namespaceUri(BPMN20_NS).extendsType(typeof(BaseElement)).instanceProvider(new ModelTypeInstanceProviderAnonymousInnerClass());
nameAttribute = typeBuilder.stringAttribute(BPMN_ATTRIBUTE_NAME).build();
SequenceBuilder sequenceBuilder = typeBuilder.sequence();
correlationPropertyRefCollection = sequenceBuilder.elementCollection(typeof(CorrelationPropertyRef)).qNameElementReferenceCollection(typeof(CorrelationProperty)).build();
typeBuilder.build();
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(Animals), ELEMENT_NAME_ANIMALS).namespaceUri(MODEL_NAMESPACE).instanceProvider(new ModelElementTypeBuilder_ModelTypeInstanceProviderAnonymousInnerClass());
SequenceBuilder sequence = typeBuilder.sequence();
descriptionChild = sequence.element(typeof(Description)).build();
animalColl = sequence.elementCollection(typeof(Animal)).build();
typeBuilder.build();
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(BpmnDiagram), BPMNDI_ELEMENT_BPMN_DIAGRAM).namespaceUri(BPMNDI_NS).extendsType(typeof(Diagram)).instanceProvider(new ModelTypeInstanceProviderAnonymousInnerClass());
SequenceBuilder sequenceBuilder = typeBuilder.sequence();
bpmnPlaneChild = sequenceBuilder.element(typeof(BpmnPlane)).required().build();
bpmnLabelStyleCollection = sequenceBuilder.elementCollection(typeof(BpmnLabelStyle)).build();
typeBuilder.build();
}
public static void RunExample()
{
SequenceBuilder builder = new SequenceBuilder("textureChanger");
builder.Append(VP.Adone().Texture("stone1.jpg", targetName: "myPp01"), TimeSpan.FromSeconds(1));
builder.Append(VP.Adone().Texture("stone2.jpg", targetName: "myPp01"), TimeSpan.FromSeconds(1));
builder.Append(VP.Adone().Texture("stone3.jpg", targetName: "myPp01"), TimeSpan.FromSeconds(1));
foreach (Action action in builder.Build().ToActions())
{
Console.WriteLine(action);
}
}
public void ReadUInt32_GivenSequenceWithSingleSegmentAndNonZeroStart_ShouldGiveExceptedResult()
{
//Arrange
var sequence = new SequenceBuilder <byte>().Append(new byte[] { 0x09, 0x00, 0x01, 0x02, 0x03 }).Build();
//Act
var actual = sequence.ReadUInt32(1, true);
//Assert
const uint expected = 66051;
actual.Should().Be(expected);
}
public void ReadUInt32_GivenSequenceWithMultipleSegments_ShouldGiveExceptedResult()
{
//Arrange
var sequence = new SequenceBuilder <byte>().Append(new byte[] { 0x00, 0x01 }).Append(new byte[] { 0x02, 0x03 }).Build();
//Act
var actual = sequence.ReadUInt32(0, true);
//Assert
const uint expected = 66051;
Assert.Equal(expected, actual);
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(Process), BPMN_ELEMENT_PROCESS).namespaceUri(BPMN20_NS).extendsType(typeof(CallableElement)).instanceProvider(new ModelTypeInstanceProviderAnonymousInnerClass());
processTypeAttribute = typeBuilder.enumAttribute(BPMN_ATTRIBUTE_PROCESS_TYPE, typeof(ProcessType)).defaultValue(ProcessType.None).build();
isClosedAttribute = typeBuilder.booleanAttribute(BPMN_ATTRIBUTE_IS_CLOSED).defaultValue(false).build();
isExecutableAttribute = typeBuilder.booleanAttribute(BPMN_ATTRIBUTE_IS_EXECUTABLE).build();
// TODO: definitionalCollaborationRef
SequenceBuilder sequenceBuilder = typeBuilder.sequence();
auditingChild = sequenceBuilder.element(typeof(Auditing)).build();
monitoringChild = sequenceBuilder.element(typeof(Monitoring)).build();
propertyCollection = sequenceBuilder.elementCollection(typeof(Property)).build();
laneSetCollection = sequenceBuilder.elementCollection(typeof(LaneSet)).build();
flowElementCollection = sequenceBuilder.elementCollection(typeof(FlowElement)).build();
artifactCollection = sequenceBuilder.elementCollection(typeof(Artifact)).build();
resourceRoleCollection = sequenceBuilder.elementCollection(typeof(ResourceRole)).build();
correlationSubscriptionCollection = sequenceBuilder.elementCollection(typeof(CorrelationSubscription)).build();
supportsCollection = sequenceBuilder.elementCollection(typeof(Supports)).qNameElementReferenceCollection(typeof(Process)).build();
/// <summary>
/// camunda extensions </summary>
camundaCandidateStarterGroupsAttribute = typeBuilder.stringAttribute(CAMUNDA_ATTRIBUTE_CANDIDATE_STARTER_GROUPS).@namespace(CAMUNDA_NS).build();
camundaCandidateStarterUsersAttribute = typeBuilder.stringAttribute(CAMUNDA_ATTRIBUTE_CANDIDATE_STARTER_USERS).@namespace(CAMUNDA_NS).build();
camundaJobPriorityAttribute = typeBuilder.stringAttribute(CAMUNDA_ATTRIBUTE_JOB_PRIORITY).@namespace(CAMUNDA_NS).build();
camundaTaskPriorityAttribute = typeBuilder.stringAttribute(CAMUNDA_ATTRIBUTE_TASK_PRIORITY).@namespace(CAMUNDA_NS).build();
camundaHistoryTimeToLiveAttribute = typeBuilder.stringAttribute(CAMUNDA_ATTRIBUTE_HISTORY_TIME_TO_LIVE).@namespace(CAMUNDA_NS).build();
camundaIsStartableInTasklistAttribute = typeBuilder.booleanAttribute(CAMUNDA_ATTRIBUTE_IS_STARTABLE_IN_TASKLIST).defaultValue(true).@namespace(CAMUNDA_NS).build();
camundaVersionTagAttribute = typeBuilder.stringAttribute(CAMUNDA_ATTRIBUTE_VERSION_TAG).@namespace(CAMUNDA_NS).build();
typeBuilder.build();
}
public static SequenceInvertedIndex Build(InvertedIndex invindex, SequenceBuilder builder)
{
var newinvindex = new SequenceInvertedIndex ();
var xseq = new int[invindex.NumberOfItems];
for (int sym = 0; sym < invindex.Count; ++sym) {
var list = invindex [sym];
foreach (var objID in list) {
xseq [objID] = sym;
}
}
newinvindex.seq = builder.Invoke (xseq, invindex.Count);
return newinvindex;
}
public static void registerType(ModelBuilder modelBuilder)
{
ModelElementTypeBuilder typeBuilder = modelBuilder.defineType(typeof(FlyingAnimal), TYPE_NAME_FLYING_ANIMAL).namespaceUri(MODEL_NAMESPACE).extendsType(typeof(Animal)).abstractType();
wingspanAttribute = typeBuilder.doubleAttribute(ATTRIBUTE_NAME_WINGSPAN).build();
SequenceBuilder sequence = typeBuilder.sequence();
flightInstructorChild = sequence.element(typeof(FlightInstructor)).idElementReference(typeof(FlyingAnimal)).build();
flightPartnerRefsColl = sequence.elementCollection(typeof(FlightPartnerRef)).idElementReferenceCollection(typeof(FlyingAnimal)).build();
typeBuilder.build();
}
public void BuildWebGraph(string filename, SequenceBuilder seqbuilder, BitmapFromBitStream bitmapbuilder = null)
{
if (bitmapbuilder == null) {
bitmapbuilder = BitmapBuilders.GetGGMN_wt (12);
}
var len_stream = new BitStream32 ();
var seq = new List<int> ();
int prev_context = -1;
using (var Input = File.OpenText (filename)) {
string line;
int lineno = 0;
int counterlineno = 0;
while (true) {
{
if (lineno % 10000 == 0) {
if (counterlineno % 10 == 0) {
Console.WriteLine ();
Console.Write ("Processing lines: ");
}
++counterlineno;
Console.Write ("{0}, ", lineno);
}
++lineno;
}
line = Input.ReadLine ();
if (line == null) {
break;
}
if (line.StartsWith ("#")) {
continue;
}
var link = line.Split ('\t', ' ');
var start_node = int.Parse (link [0]);
var end_node = int.Parse (link [1]);
// on webgraph format, starting nodes are already sorted, just advance and count
if (start_node != prev_context) {
for (int diffcount = start_node - prev_context; diffcount > 0; --diffcount) {
len_stream.Write (true);
}
prev_context = start_node;
}
len_stream.Write (false);
seq.Add (end_node);
}
// a simple hack simplifying direct-neighbors's retrieval
len_stream.Write (true);
}
this.SEQ = seqbuilder (seq, prev_context + 1);
this.LENS = bitmapbuilder (new FakeBitmap (len_stream));
}
public static void registerType(ModelBuilder bpmnModelBuilder)
{
ModelElementTypeBuilder typeBuilder = bpmnModelBuilder.defineType(typeof(BaseElement), BPMN_ELEMENT_BASE_ELEMENT).namespaceUri(BPMN20_NS).abstractType();
idAttribute = typeBuilder.stringAttribute(BPMN_ATTRIBUTE_ID).idAttribute().build();
SequenceBuilder sequenceBuilder = typeBuilder.sequence();
documentationCollection = sequenceBuilder.elementCollection(typeof(Documentation)).build();
extensionElementsChild = sequenceBuilder.element(typeof(ExtensionElements)).build();
typeBuilder.build();
}
public virtual void Build(PolyIndexLC_Composite pmi, int lambda, SequenceBuilder seq_builder = null)
{
var list = new List<LC_RNN> ();
foreach (var lc in pmi.LC_LIST) {
list.Add (lc);
}
var as_pmi = pmi.IDX as PolyIndexLC;
if (as_pmi != null) {
foreach (var lc in as_pmi.LC_LIST) {
var _lc = new LC();
_lc.Build(lc, SequenceBuilders.GetIISeq(BitmapBuilders.GetPlainSortedList()));
list.Add(_lc);
}
}
this.Build(list, lambda, seq_builder);
}
/// <summary>
/// Build the index
/// </summary>
public virtual void Build(MetricDB db, int num_centers, SequenceBuilder seq_builder = null)
{
this.DB = db;
this.CENTERS = RandomSets.GetRandomSubSet (num_centers, this.DB.Count);
Sorting.Sort<int> (this.CENTERS);
BitStream32 IsCenter = new BitStream32 ();
IsCenter.Write (false, db.Count);
var seq = new int[db.Count];
this.COV = new float[num_centers];
for (int i = 0; i < num_centers; i++) {
IsCenter [this.CENTERS [i]] = true;
seq [this.CENTERS [i]] = this.CENTERS.Count;
}
this.BuildInternal (IsCenter, seq, seq_builder);
//this.Save (output_name, invindex);
}
public void Build(string path, int _num_cols, BasicTokenizer tokenizer, SequenceBuilder seq_builder)
{
Console.WriteLine ("*** building Table: '{0}', with {1} columns", path, _num_cols);
var input = new StreamReader (File.OpenRead (path));
var C = new ColumnBuilder[ _num_cols ];
int numcol = 0;
int numrec = 0;
var recsep = tokenizer.RecordSeparator.ToString();
for (int i = 0; i < _num_cols; ++i) {
C [i] = new ColumnBuilder ();
C [i].Add(recsep);
}
foreach (var p in tokenizer.Parse(input, false)) {
// Console.WriteLine("<{0}>", p.Data);
if (p.DataType == TokenType.FieldSeparator) {
C[numcol].Add(recsep);
++numcol;
continue;
}
if (p.DataType == TokenType.RecordSeparator) {
if (numrec % 10000 == 0) {
Console.WriteLine("-- record: {0}, date-time: {1}", numrec, DateTime.Now);
}
while (numcol < _num_cols) {
C[numcol].Add(recsep);
// C[numcol].Add("");
++numcol;
}
++numrec;
numcol = 0;
continue;
}
//if (p.DataType == TokenType.Data) {
C[numcol].Add(p.Data);
//}
//Console.WriteLine ("===> type: {0}, data: '{1}'", p.DataType, p.Data);
}
this.InputTokenizer = tokenizer;
this.Columns = new Column[_num_cols];
for (int i = 0; i < _num_cols; ++i) {
Console.WriteLine ("*** compressing column-{0} of '{1}'", i, path);
C[i].Add (recsep);
this.Columns[i] = C[i].Finish(recsep, seq_builder);
}
}
public virtual void Build(IEnumerable<string> list, SequenceBuilder seq_builder, IList<int> seq_container)
{
this.FileNames = new List<string> ();
int docid = 0;
this.InputTokenizer = new BasicTokenizer('\0', '\0', '\0');
var parser = new TextParser(this.InputTokenizer, seq_container);
foreach (var filename in list) {
this.FileNames.Add (filename);
parser.AddPlainString(parser.GetFileSeparator());
parser.Parse (File.ReadAllText (filename));
if (docid % 500 == 0) {
Console.WriteLine ("== reviewing docid {0}, date-time: {1}", docid, DateTime.Now);
}
++docid;
}
this.Voc = MapVocSeq.SortingVoc (parser.Voc, parser.Seq);
this.Seq = seq_builder (parser.Seq, this.Voc.Count);
this.sep_symbol = this.RankVoc (parser.GetFileSeparator ());
}
/// <summary>
/// Build the LC_FixedM
/// </summary>
public override void Build(MetricDB db, int num_centers, SequenceBuilder seq_builder)
{
int bsize = (db.Count - num_centers) / num_centers;
this.DB = db;
int n = db.Count;
this.CENTERS = new List<int> (num_centers + 1);
this.COV = new List<float> (this.CENTERS.Count);
this.build_rest_list = new List<int> (n);
for (int i = 0; i < n; ++i) {
this.build_rest_list.Add (i);
}
var seq = new int[n];
this._Build (seq, bsize);
foreach (var c in this.CENTERS) {
seq[c] = this.CENTERS.Count;
}
this.FixOrder(seq);
this.SEQ = seq_builder(seq, this.CENTERS.Count + 1);
}
public virtual void Build(MetricDB db, int sampleSize,
SequenceBuilder seq_builder = null, Func<int,object> get_item = null)
{
this.DB = db;
if (seq_builder == null) {
seq_builder = SequenceBuilders.GetSeqXLB_SArray64 (16);
}
this.H = new ushort[sampleSize];
Random rand = new Random ();
{
HashSet<int> _coordinates = new HashSet<int> ();
int i = 0;
while (_coordinates.Count < sampleSize) {
var p = (ushort)(rand.Next () % ushort.MaxValue);
if (_coordinates.Add (p)) {
this.H [i] = p;
++i;
}
}
Array.Sort (this.H);
}
int len = this.DB.Count;
int pc = len / 100 + 1;
int numbits = sampleSize > 32 ? 32 : sampleSize;
var seq = new ListIFS (numbits);
// Console.WriteLine ("DIMENSION: {0}, LENGTH: {1}", numbits, len);
for (int docid = 0; docid < len; docid++) {
if (docid % pc == 0) {
Console.WriteLine ("Advance: {0:0.00}%, docid: {1}, total: {2}", docid * 100.0 / len, docid, len);
}
int hash;
if (get_item == null) {
hash = this.ComputeHash (this.DB [docid]);
} else {
hash = this.ComputeHash (get_item (docid));
}
// Console.WriteLine ("hash: {0}, max: {1}, sample-size: {2}", hash, 1 << sampleSize, sampleSize);
seq.Add (hash);
}
Console.WriteLine ("*** Creating index of sequences");
this.Seq = seq_builder (seq, 1 << numbits);
// IndexLoader.Save(outname, this);
}
public virtual void Build(MetricDB db, int numcenters, int lambda_search, int lambda_filter, SequenceBuilder seq_builder = null)
{
var L = new LC_RNN[lambda_search];
var M = new LC_RNN[lambda_filter];
var builder = SequenceBuilders.GetSeqPlain (short.MaxValue, ListIBuilders.GetListIFS (), null, true);
var A = new List<Action>();
for (int i = 0; i < lambda_search; ++i) {
A.Add(this.BuildOneClosure(L, i, db, numcenters, seq_builder));
}
for (int i = 0; i < lambda_filter; ++i) {
A.Add(this.BuildOneClosure(M, i, db, numcenters, builder));
}
var ops = new ParallelOptions();
ops.MaxDegreeOfParallelism = -1;
Parallel.ForEach(A, ops, (action) => action());
var poly_filter = new PolyIndexLC();
poly_filter.Build(M, 0, null);
this.Build(poly_filter, L);
}
public void Build(MetricDB db, int sampleSize, int numInstances, SequenceBuilder seq_builder = null)
{
this.DB = db;
IPermutation perm = null;
for (int i = 0; i < numInstances; ++i) {
var lsc = new LSC_H8 ();
//var _indexName = outname + String.Format (".instance-{0:00}.xml", i);
//lsc.SeqBuilder = this.SeqBuilder;
if (i == 0) {
lsc.Build (db, sampleSize, seq_builder);
var seq = lsc.GetSeq () as SeqXLB;
if (seq == null) {
throw new ArgumentException ("seq_builder should return an SeqXLB instance");
}
perm = seq.GetPERM ();
} else {
lsc.Build (db, sampleSize, seq_builder, (int p) => this.DB [perm [p]]);
}
}
}
public virtual void Build(LAESA idx, int num_pivs, int num_rings, SequenceBuilder seq_builder = null)
{
if (seq_builder == null) {
seq_builder = SequenceBuilders.GetSeqPlain (32);
}
base.Build (idx, num_pivs, num_rings, null);
this.SEQ = new Sequence[num_pivs];
var build_one_pivot = new Action<int>(delegate(int p) {
var D = this.DIST[p];
this.SEQ[p] = seq_builder(D, this.MAX_SYMBOL+1);
if (p % 10 == 0 || p + 1 == num_pivs) {
Console.Write ("== advance: {0}/{1}, ", p, num_pivs);
if (p % 100 == 0 || p + 1 == num_pivs) {
Console.WriteLine ();
}
}
});
Parallel.For(0, num_pivs, build_one_pivot);
this.EmulateDIST();
}
/// <summary>
/// Build the LC_FixedM
/// </summary>
public override void Build(MetricDB db, int num_centers, SequenceBuilder seq_builder = null)
{
int bsize = (db.Count - num_centers) / num_centers;
this.DB = db;
int n = db.Count;
this.CENTERS = new List<int> (num_centers + 1);
IList<int> rest_list = new List<int> (n);
this.COV = new List<float> (num_centers + 1);
for (int i = 0; i < n; ++i) {
rest_list.Add (i);
}
var seq = this.InternalBuild (ref rest_list, bsize, RandomSets.GetRandom());
foreach (var c in this.CENTERS) {
seq[c] = this.CENTERS.Count;
}
this.FixOrder(seq);
if (seq_builder == null) {
seq_builder = SequenceBuilders.GetIISeq(BitmapBuilders.GetPlainSortedList());
}
this.SEQ = seq_builder(seq, this.CENTERS.Count + 1);
}
public void Build(MetricDB db, KnrFP knrfp, int maxcand=1024, SequenceBuilder seq_builder=null)
{
this.DB = db;
this.R = knrfp.IdxRefs;
this.K = knrfp.K;
this.MAXCAND = maxcand;
var M = knrfp.Fingerprints.seqs;
// var L = new int[this.K * this.DB.Count];
// int pos = 0;
// for (int objID = 0; objID< this.DB.Count; ++objID) {
// var u = M [objID];
// for (int i = 0; i < this.K; ++i, ++pos) {
// L [pos] = u [i];
// }
// }
var L = new ListGen<int> ((int i) => M [i / K] [i % K], this.DB.Count * this.K);
if (seq_builder == null) {
seq_builder = SequenceBuilders.GetSeqXLB_SArray64 (16);
}
Console.WriteLine ("xxxxx Build L: {0}, R: {1}, db-count: {2}, db: {3}, K: {4}", L.Count, this.R.DB.Count, db.Count, db, K);
this.SEQ = seq_builder (L, this.R.DB.Count);
}
void FinishBuild(WTM_Inner node, SequenceBuilder seq_builder, int sigma)
{
if (node == null) {
return;
}
var s = node.SEQ as FakeSeq;
node.SEQ = seq_builder (s.SEQ, s.Sigma);
foreach (var child in node.CHILDREN) {
this.FinishBuild(child as WTM_Inner, seq_builder, sigma);
}
}
public static SequenceBuilder GetWTM(byte bits_per_code, SequenceBuilder seq_builder = null)
{
return delegate (IList<int> seq, int sigma) {
var wt = new WTM();
var symcoder = new EqualSizeCoder(bits_per_code, sigma - 1);
wt.Build(seq, sigma, symcoder, seq_builder);
return wt;
};
}
public void Build(string filename, SequenceBuilder seqbuilder, BitmapFromBitStream bitmapbuilder = null)
{
this.BuildWebGraph (filename, seqbuilder, bitmapbuilder);
}
public static SequenceBuilder GetWTM( ISymbolCoder symcoder = null, SequenceBuilder seq_builder = null)
{
return delegate (IList<int> seq, int sigma) {
var wt = new WTM();
wt.Build(seq, sigma, symcoder, seq_builder);
return wt;
};
}
public void Build(string sa_name, SequenceBuilder seq_builder = null)
{
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".structs"))) {
this.newF = GenericIO<Bitmap>.Load (Input);
int len = (int)this.newF.Count1;
this.charT = new int[len];
PrimitiveIO<int>.LoadVector (Input, len, this.charT);
}
if (seq_builder == null) {
// seq_builder = SequenceBuilders.GetWT_BinaryCoding(BitmapBuilders.GetRRR_wt(16));
seq_builder = SequenceBuilders.GetSeqXLB_DiffSet64();
}
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".bwt"))) {
var L = new ListIFS ();
L.Load (Input);
this.seqIndex = seq_builder (L, this.charT.Length);
}
using (var Input = new BinaryReader (File.OpenRead (sa_name + ".samples"))) {
this.SA_sample_step = Input.ReadInt16 ();
this.SA_marked = GenericIO<Bitmap>.Load (Input);
var _samples = new ListIFS ();
_samples.Load (Input);
var _invsamples = new ListIFS ();
_invsamples.Load (Input);
this.SA_samples = _samples;
this.SA_invsamples = _invsamples;
}
}
public static SequenceBuilder GetWTM_TopKFreqCoder(byte bits_per_code, int K, SequenceBuilder seq_builder = null)
{
return delegate (IList<int> seq, int sigma) {
var wt = new WTM();
TopKFreqCoder symcoder;
var __coder = new EqualSizeCoder(bits_per_code, sigma - 1);
var freqs = new int[sigma];
for (int i = 0; i < seq.Count; ++i) {
freqs[seq[i]]++;
}
symcoder = new TopKFreqCoder(K, freqs, __coder);
wt.Build(seq, sigma, symcoder, seq_builder);
return wt;
};
}
public void BuildLAESA(IList<LC_RNN> indexlist, int max_instances = 0, int num_pivs = 0, SequenceBuilder seq_builder = null)
{
base.Build (indexlist, max_instances, seq_builder);
var laesa = new LAESA ();
if (num_pivs == 0) {
laesa.Build (this.DB, this.LC_LIST.Count);
} else {
laesa.Build (this.DB, num_pivs);
}
}
public void Build(IList<int> text, int alphabet_size, ISymbolCoder symbol_split = null, SequenceBuilder seq_builder = null)
{
if (symbol_split == null) {
symbol_split = new EqualSizeCoder(4, alphabet_size-1);
}
this.SymbolCoder = symbol_split;
var list = this.SymbolCoder.Encode(0, null);
var numbits = list[0].numbits;
var arity = (short)(1 << numbits);
this.Alphabet = new WTM_Leaf[alphabet_size];
this.Root = new WTM_Inner (arity, null, true);
for (int i = 0; i < text.Count; i++) {
this.Add (text [i], list);
}
if (seq_builder == null) {
seq_builder = SequenceBuilders.GetSeqPlain(arity);
}
this.FinishBuild (this.Root, seq_builder, arity);
}
public void Build(IndexSingle idx, IList<LC_RNN> indexlist, int max_instances = 0, SequenceBuilder seq_builder = null)
{
base.Build (indexlist, max_instances, seq_builder);
this.IDX = idx;
}
public virtual void Build(PolyIndexLC_Composite original, int lambda_search, int lambda_filter, SequenceBuilder seq_builder = null)
{
base.Build (original.LC_LIST, lambda_search, seq_builder);
var pmi = new PolyIndexLC();
pmi.Build((original.IDX as PolyIndexLC).LC_LIST, lambda_filter, null);
this.IDX = pmi;
}
