/**********************************************************/
/********************* PUBLIC METHODS *********************/
/**
* Returns the position of the element at the coordinates indexes
* @param indexes: the indexes of the element
* @return the position
*/
public int GetPositionFromCoord(Indexes indexes)
{
int i = indexes.i;
int j = indexes.j;
int temp;
if (i < 0 || j < 0 || i == j) {
ErrorManager.DisplayErrorMessage("ReferenceMatrix.GetPositionFromCoord: Invalid indexes ("+ i +","+ j + ")");
return -1;
}
if (i > j) {
temp = i;
i = j;
j = temp;
}
return referenceMatrix [i, j];
}
public AbbyyDocumentStructure(object value, string format = null, AbstractScripter scripter = null)
: base(value, format, scripter)
{
Indexes.Add(FirstIndex);
Indexes.Add(LastIndex);
}
void RefreshIndexes(int totalIndexCount)
{
Indexes.GetTotalResults = () => totalIndexCount;
Indexes.LoadPage();
}
public void AdditionalAssemblies_NuGet_Live()
{
using (var store = GetDocumentStore())
{
store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition
{
Name = "HtmlIndex",
Maps =
{
"from c in docs.Companies select new { Name = typeof(HtmlAgilityPack.HtmlNode).Assembly.FullName }"
},
AdditionalAssemblies =
{
AdditionalAssembly.FromNuGet("HtmlAgilityPack", "1.11.28")
}
}));
using (var session = store.OpenSession())
{
session.Store(new Company
{
Name = "HR"
});
session.SaveChanges();
}
Indexes.WaitForIndexing(store);
RavenTestHelper.AssertNoIndexErrors(store);
var terms = store.Maintenance.Send(new GetTermsOperation("HtmlIndex", "Name", null));
Assert.Equal(1, terms.Length);
Assert.Contains("1.11.28.0", terms[0]);
store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition
{
Name = "HtmlIndex",
Maps =
{
"from c in docs.Companies select new { Name = typeof(HtmlAgilityPack.HtmlNode).Assembly.FullName }"
},
AdditionalAssemblies =
{
AdditionalAssembly.FromNuGet("HtmlAgilityPack", "1.11.32")
}
}));
Indexes.WaitForIndexing(store);
RavenTestHelper.AssertNoIndexErrors(store);
terms = store.Maintenance.Send(new GetTermsOperation("HtmlIndex", "Name", null));
Assert.Equal(1, terms.Length);
Assert.Contains("1.11.32.0", terms[0]);
store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition
{
Name = "HtmlIndex_2",
Maps =
{
"from c in docs.Companies select new { Name = typeof(HtmlAgilityPack.HtmlNode).Assembly.FullName }"
},
AdditionalAssemblies =
{
AdditionalAssembly.FromNuGet("HtmlAgilityPack", "1.11.28")
}
}));
Indexes.WaitForIndexing(store);
RavenTestHelper.AssertNoIndexErrors(store);
terms = store.Maintenance.Send(new GetTermsOperation("HtmlIndex_2", "Name", null));
Assert.Equal(1, terms.Length);
Assert.Contains("1.11.28.0", terms[0]);
}
}
private void DoTest <TIndex, TResult>()
where TIndex : AbstractIndexCreationTask, new()
{
using (var store = GetDocumentStore())
{
store.ExecuteIndex(new TIndex());
using (var session = store.OpenSession())
{
#region Adding Sample Data
session.Store(new Submission
{
PersonName = "Alice",
SurveyId = "surveys/1",
Answers = new[]
{
new Answer
{
QuestionId = "questions/1",
Responses = new[] { "b", "d" }
},
new Answer
{
QuestionId = "questions/2",
Responses = new[] { "b" }
},
new Answer
{
QuestionId = "questions/3",
Responses = new[] { "a", "b", "d" }
}
}
});
session.Store(new Submission
{
PersonName = "Bob",
SurveyId = "surveys/1",
Answers = new[]
{
new Answer
{
QuestionId = "questions/1",
Responses = new[] { "b", "c" }
},
new Answer
{
QuestionId = "questions/2",
Responses = new[] { "a", "b", "d" }
},
new Answer
{
QuestionId = "questions/3",
Responses = new[] { "c" }
}
}
});
session.Store(new Submission
{
PersonName = "Charlie",
SurveyId = "surveys/2",
Answers = new[]
{
new Answer
{
QuestionId = "questions/4",
Responses = new[] { "a", "b", "e" }
},
new Answer
{
QuestionId = "questions/5",
Responses = new[] { "b", "c", "d" }
},
new Answer
{
QuestionId = "questions/6",
Responses = new[] { "a", "c", "e" }
}
}
});
session.Store(new Submission
{
PersonName = "David",
SurveyId = "surveys/2",
Answers = new[]
{
new Answer
{
QuestionId = "questions/4",
Responses = new[] { "d", "e" }
},
new Answer
{
QuestionId = "questions/5",
Responses = new[] { "a", "b", "d" }
},
new Answer
{
QuestionId = "questions/6",
Responses = new[] { "c" }
}
}
});
#endregion
session.SaveChanges();
}
Indexes.WaitForIndexing(store);
// WaitForUserToContinueTheTest(store);
using (var session = store.OpenSession())
{
var results = session.Query <TResult, TIndex>().ToList();
// Just testing that some data returns.
Assert.Equal(6, results.Count);
// (The main thing is that the index builds and the query can execute without throwing an exception.)
}
}
}
protected override void InitializeIndexes()
{
base.InitializeIndexes();
Indexes.Add(CreateISysSchemaManagerPackageIndex());
}
public Employee_QuickSearch()
{
AddMap <Employee>(employees =>
from employee in employees
select new
{
Id = employee.Id,
IsActive = employee.TerminationDate == null,
FirstName = employee.FirstName,
LastName = employee.LastName,
Skills = employee.Skills,
TechnicalSkills = employee.TechnicalSkills.Select(x => x.Name),
FullName1 = string.Format("{0}, {1}", employee.LastName, employee.FirstName),
FullName2 = string.Format("{0} {1}", employee.FirstName, employee.LastName),
AttachmentIds = employee.AllAttachmentReferences.Select(x => x.Attachment.Id).ToArray(),
AttachmentNames = employee.AllAttachmentReferences.Select(x => x.Attachment.FileName).ToArray(),
AttachmentContent = new string[0],
AttachmentsBySlot = employee.AttachmentsBySlot,
IsEmployee = true,
OrphanAttachments = new dynamic[0],
FileId = employee.FileId,
Platform = employee.Platform,
CurrentPosition = employee.CurrentPosition,
CurrentProject = employee.CurrentProject,
Notes = employee.Notes,
Photo = employee.Photo,
Terms = new object[]
{
string.Format("{0:yyyy-MM-dd}", employee.HiringDate),
string.Format("{0:dd-MM-yyyy}", employee.HiringDate),
string.Format("{0:MM-dd-yyyy}", employee.HiringDate),
string.Format("{0:MMMM}", employee.HiringDate),
string.Format("{0:yyyy-MM-dd}", employee.BirthDate),
string.Format("{0:dd-MM-yyyy}", employee.BirthDate),
string.Format("{0:MM-dd-yyyy}", employee.BirthDate),
string.Format("{0:MMMM}", employee.BirthDate),
employee.BankAccount,
employee.HealthInsurance,
employee.EnglishLevel,
employee.Seniority,
employee.Degree,
employee.College
},
HiringDate = employee.HiringDate ?? DateTime.MinValue,
TerminationDate = employee.TerminationDate ?? DateTime.MaxValue
});
AddMap <Attachment>(attachments =>
from attachment in attachments
select new
{
Id = attachment.RelatedEntityId,
IsActive = false,
FirstName = (string)null,
LastName = (string)null,
Skills = (string)null,
TechnicalSkills = new string[0],
FullName1 = (string)null,
FullName2 = (string)null,
AttachmentIds = new string[0],
AttachmentNames = new string[0],
AttachmentContent = new string[0],
AttachmentsBySlot = new dynamic[0],
IsEmployee = false,
FileId = (string)null,
Platform = (string)null,
CurrentPosition = (string)null,
CurrentProject = (string)null,
Notes = new object[0],
Photo = (object)null,
Terms = new object[0],
OrphanAttachments = new[] { new { Id = attachment.Id, FileName = attachment.FileName, PlainContent = attachment.PlainContent } },
HiringDate = DateTime.MinValue,
TerminationDate = DateTime.MaxValue
});
Reduce = docs =>
from doc in docs
group doc by doc.Id into g
select new
{
Id = g.Key,
IsActive = g.Any(x => x.IsActive),
FirstName = g.Where(x => x.FirstName != null).Select(x => x.FirstName).FirstOrDefault(),
LastName = g.Where(x => x.LastName != null).Select(x => x.LastName).FirstOrDefault(),
Skills = g.Where(x => x.Skills != null).Select(x => x.Skills).FirstOrDefault(),
TechnicalSkills = g.SelectMany(x => x.TechnicalSkills).Distinct().ToArray(),
FullName1 = g.Where(x => x.FullName1 != null).Select(x => x.FullName1).FirstOrDefault(),
FullName2 = g.Where(x => x.FullName2 != null).Select(x => x.FullName2).FirstOrDefault(),
FileId = g.Where(x => x.FileId != null).Select(x => x.FileId).FirstOrDefault(),
Platform = g.Where(x => x.Platform != null).Select(x => x.Platform).FirstOrDefault(),
CurrentPosition = g.Where(x => x.CurrentPosition != null).Select(x => x.CurrentPosition).FirstOrDefault(),
CurrentProject = g.Where(x => x.CurrentProject != null).Select(x => x.CurrentProject).FirstOrDefault(),
Notes = g.SelectMany(x => x.Notes).Distinct().ToArray(),
Photo = g.Where(x => x.Photo != null).Select(x => x.Photo).FirstOrDefault(),
Terms = g.SelectMany(x => x.Terms).Distinct().ToArray(),
AttachmentIds = g.SelectMany(x => x.AttachmentIds).Distinct().ToArray(),
AttachmentNames = g.SelectMany(x => x.AttachmentNames).Distinct().ToArray(),
IsEmployee = g.Any(x => x.IsEmployee),
AttachmentContent = g.SelectMany(x => x.AttachmentContent).Union(
g.SelectMany(x => x.OrphanAttachments).Where(x => g.SelectMany(y => y.AttachmentIds).Contains(x.Id)).Select(x => x.PlainContent)
).ToArray(),
OrphanAttachments = g.SelectMany(x => x.OrphanAttachments).Where(x => !g.SelectMany(y => y.AttachmentIds).Contains(x.Id)).ToArray(),
AttachmentsBySlot = g.SelectMany(x => x.AttachmentsBySlot).Where(x => x != null).ToArray(),
HiringDate = g.Max(x => x.HiringDate),
TerminationDate = g.Min(x => x.TerminationDate)
};
Indexes.Add(x => x.FullName1, FieldIndexing.Analyzed);
Indexes.Add(x => x.FullName2, FieldIndexing.Analyzed);
Indexes.Add(x => x.Skills, FieldIndexing.Analyzed);
Indexes.Add(x => x.TechnicalSkills, FieldIndexing.Analyzed);
Indexes.Add(x => x.AttachmentNames, FieldIndexing.Analyzed);
Indexes.Add(x => x.AttachmentContent, FieldIndexing.Analyzed);
Indexes.Add(x => x.Platform, FieldIndexing.Analyzed);
Indexes.Add(x => x.CurrentPosition, FieldIndexing.Analyzed);
Indexes.Add(x => x.CurrentProject, FieldIndexing.Analyzed);
Indexes.Add(x => x.Notes, FieldIndexing.Analyzed);
Indexes.Add(x => x.Terms, FieldIndexing.Analyzed);
}
private void Save(string indexFileName)
{
using (var file = File.Create(indexFileName))
using (var compressor = new ICSharpCode.SharpZipLib.BZip2.BZip2OutputStream(file))
{
var serializer = new NetSerializer.Serializer(new Type[] { typeof(Indexes) });
var indexes = new Indexes();
indexes.ChunkIdToFirstNonFreeObjectInChunk = _chunkIdToFirstNonFreeObjectInChunk;
indexes.StartOfChunkToChunkId = _startOfChunkToChunkId;
indexes.ChunkToReferencingChunks = _chunkToReferencingChunks;
indexes.DirectlyRooted = _directlyRooted.ToArray();
indexes.AllRoots = _allRoots;
indexes.StaticRootsEnumerated = _staticRootsEnumerated;
indexes.ChunkSize = _chunkSize;
serializer.Serialize(compressor, indexes);
}
_context.WriteLine("Wrote index file of size {0}",
((ulong)new FileInfo(indexFileName).Length).ToMemoryUnits());
}
public GuiTheme()
{
Indexes.Add(nameInvoker);
}
public void CanUpdateSuggestions_AfterRestart()
{
var dataDir = NewDataPath();
using (var store = GetDocumentStore(new Options
{
Path = dataDir
}))
{
using (var session = store.OpenSession())
{
session.Store(new { Name = "john" });
session.Store(new { Name = "darsy" });
session.SaveChanges();
}
store.Maintenance.Send(new PutIndexesOperation(new[] { new IndexDefinition
{
Name = "test",
Maps = new HashSet <string> {
"from doc in docs select new { doc.Name}"
},
Fields = new Dictionary <string, IndexFieldOptions>
{
{
"Name",
new IndexFieldOptions {
Suggestions = true
}
}
}
} }));
Indexes.WaitForIndexing(store);
//var suggestionQueryResult = store.DatabaseCommands.Suggest("test", new SuggestionQuery
//{
// Field = "Name",
// Term = "jhon"
//});
//Assert.NotEmpty(suggestionQueryResult.Suggestions);
}
using (var store = GetDocumentStore(new Options
{
Path = dataDir
}))
{
using (var session = store.OpenSession())
{
session.Store(new { Name = "oren" });
session.SaveChanges();
}
Indexes.WaitForIndexing(store);
//var suggestionQueryResult = store.DatabaseCommands.Suggest("test", new SuggestionQuery
//{
// Field = "Name",
// Term = "jhon"
//});
//Assert.NotEmpty(suggestionQueryResult.Suggestions);
//suggestionQueryResult = store.DatabaseCommands.Suggest("test", new SuggestionQuery
//{
// Field = "Name",
// Term = "orne"
//});
//Assert.NotEmpty(suggestionQueryResult.Suggestions);
}
}
public void Spatial_index_should_not_stop_indexing_after_one_bad_document()
{
var validPizzeriaDoc = new Pizzeria
{
Name = "Pizza Hot",
DeliveryArea = "POLYGON ((5 10, 10 5, 15 10, 10 15, 5 10))"
};
var anotherValidPizzeriaDoc = new Pizzeria
{
Name = "Dominox Pizza",
DeliveryArea = "POLYGON ((6 10, 10 6, 15 10, 10 15, 6 10))"
};
var yetAnotherValidPizzeriaDoc = new Pizzeria
{
Name = "Sharky Pizza",
DeliveryArea = "POLYGON ((7 10, 10 7, 15 10, 10 15, 7 10))"
};
var invalidPizzeriaDoc = new Pizzeria
{
Name = "Very evil pizza",
DeliveryArea = "POLYGON ((1 1, 3 3, 1 3, 3 1, 1 1))"
};
var invalidPizzeriaDoc2 = new Pizzeria
{
Name = "Very evil pizza2",
DeliveryArea = "POLYGON ((1 1, 3 3, 1 3, 3 1, 1 1))"
};
using (var store = GetDocumentStore())
{
using (var session = store.OpenSession())
{
session.Store(validPizzeriaDoc);
session.Store(anotherValidPizzeriaDoc);
session.Store(invalidPizzeriaDoc);
session.Store(yetAnotherValidPizzeriaDoc);
session.Store(invalidPizzeriaDoc2);
session.SaveChanges();
}
new SpatialIndex().Execute(store);
Indexes.WaitForIndexing(store);
using (var session = store.OpenSession())
{
var pizzeriaDocCount = session.Query <Pizzeria, SpatialIndex>().Count();
var pizzerias = session.Query <Pizzeria, SpatialIndex>().ToList();
var stats = store.Maintenance.Send(new GetStatisticsOperation());
/*
* var indexErrors = store.Admin.Send(new GetIndexErrorsOperation());
* if (indexErrors.Errors.Length != 2)
* {
* Assert.False(true, string.Join(Environment.NewLine, stats.Errors.Select(e => e.ToString())));
* }
*
* Assert.Equal(2, stats.Errors.Length);
* Assert.Equal("pizzerias/3", stats.Errors.First().Document);
* Assert.Equal("pizzerias/5", stats.Errors.Last().Document);
* Assert.NotEmpty(pizzerias);
* Assert.Equal(3,pizzeriaDocCount);
*/
}
}
}
public override void VisitIndexedEnumeratedValue(int index, object item, int depth)
{
Indexes.Add(index);
base.VisitIndexedEnumeratedValue(index, item, depth);
}
protected override void InitializeIndexes()
{
base.InitializeIndexes();
Indexes.Add(CreateIMBQHMXCKGjle42hQVBJkGeeQSBAIndex());
}
public EndPointReferenceList()
{
Indexes.Add(EndPointReference <T> .EndPointInvoker.Instance);
}
public void SupportAttachmentsForInIndex_JavaScript()
{
using (var store = GetDocumentStore())
{
var index = new AttachmentIndex();
index.Execute(store);
using (var session = store.OpenSession())
{
session.Store(new Employee
{
FirstName = "John",
LastName = "Doe"
}, "employees/1");
session.Store(new Employee
{
FirstName = "Bob",
LastName = "Doe"
}, "employees/2");
session.Store(new Employee
{
FirstName = "Edward",
LastName = "Doe"
}, "employees/3");
session.SaveChanges();
}
Indexes.WaitForIndexing(store);
using (var session = store.OpenSession())
{
var employees = session.Query <AttachmentIndex.Result, AttachmentIndex>()
.Where(x => x.AttachmentNames.Contains("photo.jpg"))
.OfType <Employee>()
.ToList();
Assert.Equal(0, employees.Count);
}
using (var session = store.OpenSession())
{
var employee1 = session.Load <Employee>("employees/1");
var employee2 = session.Load <Employee>("employees/2");
session.Advanced.Attachments.Store(employee1, "photo.jpg", new MemoryStream(Encoding.UTF8.GetBytes("123")), "image/jpeg");
session.Advanced.Attachments.Store(employee1, "cv.pdf", new MemoryStream(Encoding.UTF8.GetBytes("321")), "application/pdf");
session.Advanced.Attachments.Store(employee2, "photo.jpg", new MemoryStream(Encoding.UTF8.GetBytes("456789")), "image/jpeg");
session.SaveChanges();
}
Indexes.WaitForIndexing(store);
using (var session = store.OpenSession())
{
var employees = session.Query <AttachmentIndex.Result, AttachmentIndex>()
.Where(x => x.AttachmentNames.Contains("photo.jpg"))
.OfType <Employee>()
.ToList();
Assert.Equal(2, employees.Count);
Assert.Contains("John", employees.Select(x => x.FirstName));
Assert.Contains("Bob", employees.Select(x => x.FirstName));
employees = session.Query <AttachmentIndex.Result, AttachmentIndex>()
.Where(x => x.AttachmentNames.Contains("cv.pdf"))
.OfType <Employee>()
.ToList();
Assert.Equal(1, employees.Count);
Assert.Contains("John", employees.Select(x => x.FirstName));
}
var terms = store.Maintenance.Send(new GetTermsOperation(index.IndexName, nameof(AttachmentIndex.Result.AttachmentNames), fromValue: null));
Assert.Equal(2, terms.Length);
Assert.Contains("photo.jpg", terms);
Assert.Contains("cv.pdf", terms);
terms = store.Maintenance.Send(new GetTermsOperation(index.IndexName, nameof(AttachmentIndex.Result.AttachmentSizes), fromValue: null));
Assert.Equal(2, terms.Length);
Assert.Contains("3", terms);
Assert.Contains("6", terms);
terms = store.Maintenance.Send(new GetTermsOperation(index.IndexName, nameof(AttachmentIndex.Result.AttachmentHashes), fromValue: null));
Assert.Equal(3, terms.Length);
terms = store.Maintenance.Send(new GetTermsOperation(index.IndexName, nameof(AttachmentIndex.Result.AttachmentContentTypes), fromValue: null));
Assert.Equal(2, terms.Length);
Assert.Contains("image/jpeg", terms);
Assert.Contains("application/pdf", terms);
using (var session = store.OpenSession())
{
var employee1 = session.Load <Employee>("employees/1");
session.Advanced.Attachments.Delete(employee1, "photo.jpg");
session.SaveChanges();
}
Indexes.WaitForIndexing(store);
using (var session = store.OpenSession())
{
var employees = session.Query <AttachmentIndex.Result, AttachmentIndex>()
.Where(x => x.AttachmentNames.Contains("photo.jpg"))
.OfType <Employee>()
.ToList();
Assert.Equal(1, employees.Count);
Assert.Contains("Bob", employees.Select(x => x.FirstName));
}
}
}
public static Indexes IndexString(string contact)
{
var index = new Indexes();
var parts = contact.Split(new[] { "," }, StringSplitOptions.None).ToList();
index.Email = parts.IndexOf("email");
index.FirstName = parts.IndexOf("firstname");
index.LastName = parts.IndexOf("lastname");
index.MiddleName = parts.IndexOf("middlename");
index.Grouops = parts.IndexOf("groups");
index.Organization = parts.IndexOf("organization");
return index;
}
public async Task CanUseChangesApiAndToggleServer()
{
var server = GetNewServer(new ServerCreationOptions
{
RunInMemory = false
});
using (IDocumentStore store = GetDocumentStore(new Options
{
ModifyDocumentStore = s =>
{
var documentStore = (IDocumentStore)s;
documentStore.Conventions.OperationStatusFetchMode = OperationStatusFetchMode.ChangesApi;
documentStore.OnFailedRequest += (sender, args) => { };
},
Server = server,
RunInMemory = false
}))
{
new Company_ByName().Execute(store);
put_1500_companies(store);
Indexes.WaitForIndexing(store);
var task = Task.Run(async() =>
{
var operation = await store.Operations.SendAsync(new DeleteByQueryOperation(new IndexQuery {
Query = $"FROM INDEX '{new Company_ByName().IndexName}'"
}, new QueryOperationOptions
{
MaxOpsPerSecond = 1
}));
var serverToggled = false;
while (store.WasDisposed == false)
{
try
{
if (serverToggled)
{
await operation.WaitForCompletionAsync(TimeSpan.FromMinutes(5));
}
else
{
await operation.WaitForCompletionAsync(TimeSpan.FromSeconds(1));
}
break;
}
catch (Exception e)
{
if (serverToggled == false)
{
Assert.Equal(typeof(TimeoutException), e.GetType());
Assert.Contains(e.Message, "did not get a reply for operation");
while (server.Disposed == false)
{
await Task.Delay(500);
}
serverToggled = true;
}
Assert.Equal(typeof(InvalidOperationException), e.GetType());
Assert.StartsWith(e.Message, "Could not fetch state of operation");
await Task.Delay(1000);
}
}
});
await Task.Delay(1000);
server = await ToggleServer(server);
await Task.Delay(5000);
server = await ToggleServer(server);
await Task.Delay(5000);
}
}
public DBSchemaItemList(DBSchema schema)
: base()
{
Indexes.Add(DBSchemaItem.NameInvoker <T> .Instance);
Schema = schema;
}
public void Can_page_when_using_nested_property_index()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenSession())
{
for (int i = 1; i < 11; i++)
{
session.Store(CreateProvider(i));
}
session.SaveChanges();
}
new NestedPropertyIndex1().Execute(store);
Indexes.WaitForIndexing(store);
// Retrieves all 10 providers in a single result set.
using (var session = store.OpenSession())
{
var result = (from p in session.Query <Provider, NestedPropertyIndex1>()
.Customize(x => x.WaitForNonStaleResults())
where p.Zip == "97520"
select p).Take(1024).ToArray();
Assert.Equal(10, result.Count());
}
// Retrieves all 10 providers, 2 at a time.
using (var session = store.OpenSession())
{
const int pageSize = 2;
int pageNumber = 0;
int skippedResults = 0;
int recordsToSkip = 0;
var providers = new List <Provider>();
QueryStatistics statistics;
while (true)
{
var result = (from p in session.Query <Provider, NestedPropertyIndex1>()
.Customize(x => x.WaitForNonStaleResults())
.Statistics(out statistics)
where p.Zip == "97520"
select p)
.Take(pageSize)
.Skip(recordsToSkip)
.ToList();
providers.AddRange(result);
if (result.Count < pageSize)
{
break;
}
pageNumber++;
skippedResults += statistics.SkippedResults;
recordsToSkip = pageSize * pageNumber + skippedResults;
// I found this in the Raven.Tests.MailingList.Vlad.WillOnlyGetPost2Once() method
//recordsToSkip = pageSize * pageNumber + statistics.SkippedResults;
}
Assert.Equal(10, providers.Count);
Assert.Equal(5, pageNumber);
}
}
}
public void AddIndex(Index index)
{
Indexes.Add(index);
}
public void OptimizationShouldWork_NewIndexedWillGetPrecomputedDocumentsToIndexToAvoidRetrievingAllDocumentsFromDisk()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenSession())
{
for (int i = 0; i < 10; i++)
{
session.Store(new Foo
{
Item = "Ball/" + i % 2
});
session.Store(new Bar
{
Item = "Computer/" + i
});
}
for (int i = 0; i < 10000; i++)
{
session.Store(new Baz
{
Item = "Baz/" + i
});
}
session.SaveChanges();
}
Indexes.WaitForIndexing(store);
new SingleMapIndex().Execute(store);
new MultiMapIndex().Execute(store);
new FooMapReduceIndex().Execute(store);
Indexes.WaitForIndexing(store);
using (var session = store.OpenSession())
{
var count1 = session.Query <Foo, SingleMapIndex>().Count();
Assert.Equal(10, count1);
var count2 = session.Query <MultiMapOutput, MultiMapIndex>().Count();
Assert.Equal(20, count2);
var count3 = session.Query <FooMapReduceIndex.Result, FooMapReduceIndex>().ToList();
Assert.Equal(2, count3.Count);
}
using (var session = store.OpenSession())
{
session.Store(new Foo
{
Item = "Ball/" + 999
});
session.SaveChanges();
var count1 = session.Query <Foo, SingleMapIndex>().Customize(x => x.WaitForNonStaleResults()).Count();
Assert.Equal(11, count1);
var count2 =
session.Query <MultiMapOutput, MultiMapIndex>().Customize(x => x.WaitForNonStaleResults()).Count();
Assert.Equal(21, count2);
var count3 = session.Query <FooMapReduceIndex.Result, FooMapReduceIndex>().Customize(x => x.WaitForNonStaleResults()).ToList();
Assert.Equal(3, count3.Count);
}
}
}
public void CanUseMetadataFor()
{
using (var store = GetDocumentStore())
{
new Companies_CompanyByType().Execute(store);
Indexes.WaitForIndexing(store);
using (var session = store.OpenSession())
{
var contact1 = new Contact {
FirstName = "FirstName1"
};
var contact2 = new Contact {
FirstName = "FirstName2"
};
var contact3 = new Contact {
FirstName = "FirstName3"
};
session.SaveChanges();
session.Store(new Company
{
Type = Company.CompanyType.Public,
Contacts = new List <Contact> {
contact1, contact2, contact3
}
});
session.Store(new Company
{
Type = Company.CompanyType.Public,
Contacts = new List <Contact> {
contact3
}
});
session.Store(new Company
{
Type = Company.CompanyType.Public,
Contacts = new List <Contact> {
contact1, contact2
}
});
session.Store(new Company
{
Type = Company.CompanyType.Private,
Contacts = new List <Contact> {
contact1, contact2
}
});
session.Store(new Company
{
Type = Company.CompanyType.Private,
Contacts = new List <Contact> {
contact1, contact2, contact3
}
});
session.SaveChanges();
Indexes.WaitForIndexing(store);
Companies_CompanyByType.ReduceResult[] companies = session.Query <Companies_CompanyByType.ReduceResult, Companies_CompanyByType>()
.OrderBy(x => x.Type)
.ToArray();
Assert.Equal(2, companies.Length);
Assert.Equal(Company.CompanyType.Private, companies[0].Type);
Assert.Equal(5, companies[0].ContactsCount);
Assert.NotNull(companies[0].LastModified);
Assert.Equal(Company.CompanyType.Public, companies[1].Type);
Assert.Equal(6, companies[1].ContactsCount);
Assert.NotNull(companies[1].LastModified);
}
}
}
public void ShouldIndexAllDocuments()
{
using (var store = GetDocumentStore())
{
new Companies_ByPhone().Execute(store);
using (var commands = store.Commands())
{
commands.Put(
"companies/1",
null, new
{
Name = "HR",
Phone = "Hadera"
},
new System.Collections.Generic.Dictionary <string, object>
{
{ Raven.Client.Constants.Documents.Metadata.Collection, "Companies" }
});
commands.Put(
"companies/2",
null, new
{
Name = "CF"
},
new System.Collections.Generic.Dictionary <string, object>
{
{ Raven.Client.Constants.Documents.Metadata.Collection, "Companies" }
});
}
Indexes.WaitForIndexing(store);
using (var session = store.OpenSession())
{
var companies = session
.Advanced
.DocumentQuery <Company>()
.Statistics(out var stats1)
.Not
.WhereExists(x => x.Phone)
.ToList();
Assert.Equal(1, companies.Count);
Assert.Equal("CF", companies[0].Name);
companies = session
.Advanced
.DocumentQuery <Company>(stats1.IndexName)
.Statistics(out var stats2)
.ToList();
Assert.Equal(2, companies.Count);
}
using (var session = store.OpenSession())
{
var companies = session
.Advanced
.DocumentQuery <Company, Companies_ByPhone>()
.Not
.WhereExists(x => x.Phone)
.ToList();
Assert.Equal(1, companies.Count);
Assert.Equal("CF", companies[0].Name);
companies = session
.Advanced
.DocumentQuery <Company, Companies_ByPhone>()
.ToList();
Assert.Equal(2, companies.Count);
}
}
}
protected override void InitializeIndexes()
{
base.InitializeIndexes();
Indexes.Add(CreateINmH5G7I4Cb8CyfyNeNd2gfA3eEIndex());
}
protected override void InitializeIndexes()
{
base.InitializeIndexes();
Indexes.Add(CreateIBaseSysPackageIdLevelIndex());
}
public void CanUseMLNET_Omnx()
{
using (var store = GetDocumentStore())
{
store.Maintenance.Send(new PutIndexesOperation(new IndexDefinition
{
Name = "Employees/Tags",
Maps =
{
@"
from i in docs.Employees
let pic = LoadAttachment(i, ""photo.png"")
let classified = ObjectClassification.GetObjects(pic.GetContentAsStream())
select new
{
Tags = classified.Keys,
Count = classified.Count
}
"
},
AdditionalSources = new System.Collections.Generic.Dictionary <string, string>
{
{ "ImageClassifier", @"
using System;
using System.IO;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using Microsoft.ML;
using Microsoft.ML.Data;
using Microsoft.ML.Transforms.Image;
using Microsoft.ML.Transforms.Onnx;
using System.Net.Http;
namespace ObjectDetection
{
public class ObjectClassification
{
public const int ROW_COUNT = 13;
public const int COL_COUNT = 13;
public const int BOXES_PER_CELL = 5;
public const int CLASS_COUNT = 20;
public const int BOX_INFO_FEATURE_COUNT = 5;
private const int channelStride = ROW_COUNT * COL_COUNT;
private static string[] labels = new string[]
{
""aeroplane"", ""bicycle"", ""bird"", ""boat"", ""bottle"",
""bus"", ""car"", ""cat"", ""chair"", ""cow"",
""diningtable"", ""dog"", ""horse"", ""motorbike"", ""person"",
""pottedplant"", ""sheep"", ""sofa"", ""train"", ""tvmonitor""
};
[ThreadStatic]
private static Context _context;
public class Context
{
public TransformerChain<OnnxTransformer> Model;
public MLContext MLContext;
public MemoryStream Mem = new MemoryStream();
public InMemoryImageData[] Array = new InMemoryImageData[1]
{
new InMemoryImageData()
};
private struct ImageNetSettings
{
public const int imageHeight = 416;
public const int imageWidth = 416;
}
public class InMemoryImageData
{
[ImageType(ImageNetSettings.imageHeight, ImageNetSettings.imageWidth)]
[LoadColumn(0)]
public Bitmap Image;
}
private static string DownloadCache(string url)
{
var file = Path.GetFileName(new Uri(url).AbsolutePath);
var cache = Path.Combine(Path.GetTempPath(), file);
if (File.Exists(cache))
return cache;
try
{
using var client = new HttpClient();
using var f = File.Create(cache);
client.GetAsync(url).Result.Content.ReadAsStreamAsync().Result.CopyTo(f);
return cache;
}
catch (Exception)
{
File.Delete(cache);//cleanup on failure
throw;
}
}
public Context()
{
var modelFilePath = DownloadCache(""https://media.githubusercontent.com/media/onnx/models/master/vision/object_detection_segmentation/tiny-yolov2/model/tinyyolov2-7.onnx"");
MLContext = new MLContext();
var pipeline = MLContext.Transforms.ResizeImages(outputColumnName: ""image"", imageWidth: ImageNetSettings.imageWidth, imageHeight: ImageNetSettings.imageHeight, inputColumnName: ""Image"")
.Append(MLContext.Transforms.ExtractPixels(outputColumnName: ""image""))
.Append(MLContext.Transforms.ApplyOnnxModel(""grid"", ""image"", modelFilePath));
var data = MLContext.Data.LoadFromEnumerable(new List<InMemoryImageData>());
Model = pipeline.Fit(data);
}
}
public static dynamic GetObjects(Stream s)
{
_context ??= new Context();
_context.Mem.SetLength(0);
s.CopyTo(_context.Mem);
_context.Mem.Position = 0;
using var bitmap = (Bitmap)Bitmap.FromStream(_context.Mem);
_context.Array[0].Image = bitmap;
var scoredData = _context.Model.Transform(_context.MLContext.Data.LoadFromEnumerable(_context.Array));
var matches = new Dictionary<string, float>();
foreach (var prob in scoredData.GetColumn<float[]>(TinyYoloModelSettings.ModelOutput))
{
ParseOutputs(prob, matches);
}
_context.Array[0].Image = null;
return matches;
}
private static void ParseOutputs(float[] yoloModelOutputs, Dictionary<string, float> matches, float threshold = .3F)
{
for (int row = 0; row < ROW_COUNT; row++)
{
for (int column = 0; column < COL_COUNT; column++)
{
for (int box = 0; box < BOXES_PER_CELL; box++)
{
var channel = (box * (CLASS_COUNT + BOX_INFO_FEATURE_COUNT));
float confidence = GetConfidence(yoloModelOutputs, row, column, channel);
if (confidence < threshold)
continue;
float[] predictedClasses = ExtractClasses(yoloModelOutputs, row, column, channel);
var (topResultIndex, topResultScore) = GetTopResult(predictedClasses);
var topScore = topResultScore * confidence;
if (topScore < threshold)
continue;
if(matches.TryGetValue(labels[topResultIndex], out var f) == false || f < topResultScore)
{
matches[labels[topResultIndex]] = topResultScore;
}
}
}
}
}
private static float GetConfidence(float[] modelOutput, int x, int y, int channel)
{
return Sigmoid(modelOutput[GetOffset(x, y, channel + 4)]);
}
private static float Sigmoid(float value)
{
var k = (float)Math.Exp(value);
return k / (1.0f + k);
}
private static int GetOffset(int x, int y, int channel)
{
// YOLO outputs a tensor that has a shape of 125x13x13, which
// WinML flattens into a 1D array. To access a specific channel
// for a given (x,y) cell position, we need to calculate an offset
// into the array
return (channel * channelStride) + (y * COL_COUNT) + x;
}
private static float[] Softmax(float[] values)
{
var maxVal = values.Max();
var exp = values.Select(v => Math.Exp(v - maxVal));
var sumExp = exp.Sum();
return exp.Select(v => (float)(v / sumExp)).ToArray();
}
private static float[] ExtractClasses(float[] modelOutput, int x, int y, int channel)
{
float[] predictedClasses = new float[CLASS_COUNT];
int predictedClassOffset = channel + BOX_INFO_FEATURE_COUNT;
for (int predictedClass = 0; predictedClass < CLASS_COUNT; predictedClass++)
{
predictedClasses[predictedClass] = modelOutput[GetOffset(x, y, predictedClass + predictedClassOffset)];
}
return Softmax(predictedClasses);
}
private static ValueTuple<int, float> GetTopResult(float[] predictedClasses)
{
return predictedClasses
.Select((predictedClass, index) => (Index: index, Value: predictedClass))
.OrderByDescending(result => result.Value)
.First();
}
public struct TinyYoloModelSettings
{
// for checking Tiny yolo2 Model input and output parameter names,
//you can use tools like Netron,
// which is installed by Visual Studio AI Tools
// input tensor name
public const string ModelInput = ""image"";
// output tensor name
public const string ModelOutput = ""grid"";
}
}
}
" }
},
AdditionalAssemblies =
{
AdditionalAssembly.FromRuntime("System.IO.FileSystem"),
AdditionalAssembly.FromRuntime("System.Net.Http"),
AdditionalAssembly.FromNuGet("Microsoft.ML", "1.5.5"),
AdditionalAssembly.FromNuGet("Microsoft.ML.ImageAnalytics", "1.5.5"),
AdditionalAssembly.FromNuGet("Microsoft.ML.OnnxTransformer", "1.5.5"),
AdditionalAssembly.FromNuGet("Microsoft.ML.OnnxRuntime.Managed","1.7.1"),
AdditionalAssembly.FromNuGet("Microsoft.ML.OnnxRuntime", "1.7.0")
}
}));
using (var session = store.OpenSession())
{
var employee = new Employee {
FirstName = "John", LastName = "Doe"
};
session.Store(employee);
var image = GetImage("RavenDB_15753.cat.jpg"); // from: https://unsplash.com/photos/IuJc2qh2TcA
session.Advanced.Attachments.Store(employee, "photo.png", image);
session.SaveChanges();
}
Indexes.WaitForIndexing(store);
RavenTestHelper.AssertNoIndexErrors(store);
}
}
protected override void InitializeIndexes()
{
base.InitializeIndexes();
Indexes.Add(CreateIMLLeadModifiedOnIndex());
Indexes.Add(CreateIDX_LeadNameIndex());
}
protected override void InitializeIndexes()
{
base.InitializeIndexes();
Indexes.Add(CreateIUSysSqlScriptDependencyIndex());
}
public async Task CanUpdateMapReduceIndexWithLoadDocumentAfterDocumentDeletion()
{
using (var store = GetDocumentStore())
{
string companyId2;
using (var session = store.OpenAsyncSession())
{
var company1 = new Company
{
Name = "Hibernating Rhinos"
};
await session.StoreAsync(company1);
await session.StoreAsync(new Order
{
CompanyId = company1.Id
});
var company2 = new Company
{
Name = "Hibernating Rhinos"
};
await session.StoreAsync(company2);
companyId2 = company2.Id;
await session.StoreAsync(new Order
{
CompanyId = companyId2
});
await session.SaveChangesAsync();
}
new OrdersMapReduceIndex().Execute(store);
Indexes.WaitForIndexing(store);
using (var session = store.OpenSession())
{
var list = session.Query <OrdersMapReduceIndex.Result, OrdersMapReduceIndex>().ToList();
Assert.Equal(1, list.Count);
Assert.Equal("Hibernating Rhinos", list[0].CompanyName);
Assert.Equal(2, list[0].Count);
}
store.Maintenance.Send(new StopIndexOperation(nameof(OrdersMapReduceIndex)));
using (var session = store.OpenAsyncSession())
{
session.Delete(companyId2);
await session.SaveChangesAsync();
}
var database = await GetDatabase(store.Database);
await database.TombstoneCleaner.ExecuteCleanup();
store.Maintenance.Send(new StartIndexOperation(nameof(OrdersMapReduceIndex)));
Indexes.WaitForIndexing(store);
using (var session = store.OpenSession())
{
var list = session.Query <OrdersMapReduceIndex.Result, OrdersMapReduceIndex>().ToList();
Assert.Equal(2, list.Count);
Assert.Equal("Hibernating Rhinos", list[0].CompanyName);
Assert.Equal(1, list[0].Count);
Assert.Equal("HR", list[1].CompanyName);
Assert.Equal(1, list[1].Count);
}
}
}
public void CanUseTryConvertInIndex_JavaScript()
{
using (var store = GetDocumentStore())
{
var index = new ConvertIndex();
index.Execute(store);
using (var session = store.OpenSession())
{
session.Store(new Item
{
DblNullValue = 1.1,
DblValue = 2.1,
FltNullValue = 3.1f,
FltValue = 4.1f,
IntNullValue = 5,
IntValue = 6,
LngNullValue = 7,
LngValue = 8,
ObjValue = new Company {
Name = "HR"
},
StgValue = "str"
}, "items/1");
session.SaveChanges();
}
Indexes.WaitForIndexing(store);
RavenTestHelper.AssertNoIndexErrors(store);
var terms = store.Maintenance.Send(new GetTermsOperation(index.IndexName, nameof(Item.DblNullValue), fromValue: null));
Assert.Equal(1, terms.Length);
Assert.True(double.Parse(terms[0], CultureInfo.InvariantCulture).AlmostEquals(1.1));
terms = store.Maintenance.Send(new GetTermsOperation(index.IndexName, nameof(Item.DblValue), fromValue: null));
Assert.Equal(1, terms.Length);
Assert.True(double.Parse(terms[0], CultureInfo.InvariantCulture).AlmostEquals(2.1));
terms = store.Maintenance.Send(new GetTermsOperation(index.IndexName, nameof(Item.FltNullValue), fromValue: null));
Assert.Equal(1, terms.Length);
Assert.True(float.Parse(terms[0], CultureInfo.InvariantCulture).AlmostEquals(3.1f));
terms = store.Maintenance.Send(new GetTermsOperation(index.IndexName, nameof(Item.FltValue), fromValue: null));
Assert.Equal(1, terms.Length);
Assert.True(float.Parse(terms[0], CultureInfo.InvariantCulture).AlmostEquals(4.1f));
terms = store.Maintenance.Send(new GetTermsOperation(index.IndexName, nameof(Item.IntNullValue), fromValue: null));
Assert.Equal(1, terms.Length);
Assert.Equal("5", terms[0]);
terms = store.Maintenance.Send(new GetTermsOperation(index.IndexName, nameof(Item.IntValue), fromValue: null));
Assert.Equal(1, terms.Length);
Assert.Equal("6", terms[0]);
terms = store.Maintenance.Send(new GetTermsOperation(index.IndexName, nameof(Item.LngNullValue), fromValue: null));
Assert.Equal(1, terms.Length);
Assert.Equal("7", terms[0]);
terms = store.Maintenance.Send(new GetTermsOperation(index.IndexName, nameof(Item.LngValue), fromValue: null));
Assert.Equal(1, terms.Length);
Assert.Equal("8", terms[0]);
terms = store.Maintenance.Send(new GetTermsOperation(index.IndexName, nameof(Item.ObjValue), fromValue: null));
Assert.Equal(1, terms.Length);
Assert.Equal("-1", terms[0]);
terms = store.Maintenance.Send(new GetTermsOperation(index.IndexName, nameof(Item.StgValue), fromValue: null));
Assert.Equal(1, terms.Length);
Assert.Equal("-1", terms[0]);
}
}
public TestDocumentByName()
{
Map = docs => from doc in docs select new { doc.Name };
Indexes.Add(x => x.Name, FieldIndexing.Search);
}
/// <summary>
/// Register a field to be indexed
/// </summary>
protected void Index(Expression <Func <TReduceResult, object> > field, FieldIndexing indexing)
{
Indexes.Add(field, indexing);
}
public void BoundingBoxTest()
{
// X XXX X
// X XXX X
// X XXX X
// X X
// XXXXXXX
var polygon = "POLYGON ((0 0, 0 5, 1 5, 1 1, 5 1, 5 5, 6 5, 6 0, 0 0))";
var rectangle1 = "2 2 4 4";
var rectangle2 = "6 6 10 10";
var rectangle3 = "0 0 6 6";
using (var store = GetDocumentStore())
{
store.Initialize();
new BBoxIndex().Execute(store);
new QuadTreeIndex().Execute(store);
using (var session = store.OpenSession())
{
session.Store(new SpatialDoc {
Shape = polygon
});
session.SaveChanges();
}
Indexes.WaitForIndexing(store);
using (var session = store.OpenSession())
{
var result = session.Query <SpatialDoc>()
.Count();
Assert.Equal(1, result);
}
using (var session = store.OpenSession())
{
var result = session.Query <SpatialDoc, BBoxIndex>()
.Spatial(x => x.Shape, x => x.Intersects(rectangle1))
.Count();
Assert.Equal(1, result);
}
using (var session = store.OpenSession())
{
var result = session.Query <SpatialDoc, BBoxIndex>()
.Spatial(x => x.Shape, x => x.Intersects(rectangle2))
.Count();
Assert.Equal(0, result);
}
using (var session = store.OpenSession())
{
var result = session.Query <SpatialDoc, BBoxIndex>()
.Spatial(x => x.Shape, x => x.Disjoint(rectangle2))
.Count();
Assert.Equal(1, result);
}
using (var session = store.OpenSession())
{
var result = session.Query <SpatialDoc, BBoxIndex>()
.Spatial(x => x.Shape, x => x.Within(rectangle3))
.Count();
Assert.Equal(1, result);
}
using (var session = store.OpenSession())
{
var result = session.Query <SpatialDoc, QuadTreeIndex>()
.Spatial(x => x.Shape, x => x.Intersects(rectangle2))
.Count();
Assert.Equal(0, result);
}
using (var session = store.OpenSession())
{
var result = session.Query <SpatialDoc, QuadTreeIndex>()
.Spatial(x => x.Shape, x => x.Intersects(rectangle1))
.Count();
Assert.Equal(0, result);
}
}
}
private void CheckIndexes(Indexes indexes,
params int[] expectedIndexes)
{
if (expectedIndexes == null)
expectedIndexes = new int[0];
int i = 0;
foreach (int index in indexes) {
Assert.AreEqual(expectedIndexes[i], index);
i++;
}
Assert.AreEqual(expectedIndexes.Length, i);
}
public Table(Database database, string catalog, string schema, string name, string type, string remarks, string dial)
{
this.database = database;
this.catalog = catalog;
this.schema = schema;
this.name = name;
this.type = type;
this.remarks = remarks;
columns = new Columns(this);
columns.load();
primaryKey = new PrimaryKey(this);
primaryKey.load();
//foreignKey = new ForeignKey(this);
//foreignKey.load();
indexes = new Indexes(this);
triggers = new Triggers(this);
insert = new OdbcCommand();
//insert.CommandText = "insert into " + schema + "." + name;
insert.CommandText = "insert into " + name;
insert.CommandText += "(";
int ordinal = 0;
if (dial.ToUpper() == "DB2")
{
foreach (Column column in columns.Values)
{
if (ordinal != 0)
insert.CommandText += ",";
insert.CommandText += column.name;
ordinal++;
}
}
else
{
foreach (Column column in columns.Values)
{
if (ordinal != 0)
insert.CommandText += ",";
insert.CommandText += "[" + column.name + "]";
ordinal++;
}
}
insert.CommandText += ") values (";
insert.CommandType = CommandType.Text;
insert.Connection = database.connection;
ordinal = 0;
foreach (Column column in columns.Values)
{
if (ordinal != 0)
insert.CommandText += ",";
// ODBC does not support named parameters
//insert.CommandText += "@" + column.name;
insert.CommandText += "?";
ordinal++;
insert.Parameters.Add(column.parameter);
}
insert.CommandText += ")";
}
