public void QueryWithArrayParameterTest()
{
using (var session = SessionFactory.CreateQuerySession())
{
session.Execute(new QueryWithArrayParameter()).ShouldBe(1);
}
}
public void QueryWithCustomParameterTypeTest()
{
using (var session = SessionFactory.CreateQuerySession())
{
var result = session.Execute(new QueryPersonByName(_fixture.TestPerson1));
result.Dob.ShouldBe(_fixture.TestPerson1.Dob);
}
}
public void QueryWithCustomTableParameterTypeTest()
{
using (var session = SessionFactory.CreateQuerySession())
{
var peopleIds = new[] { 1L, 2L };
var result = session.Execute(new GetPeopleIdsQuery(peopleIds));
result.ShouldBe(peopleIds);
}
}
public override void Execute()
{
try
{
var query = _queryParser.Parse(
collections: Collections,
q: Q,
fields: Fields,
select: _select,
and: And,
or: Or);
var targetCollectionId = _target.ToHash();
IEnumerable <IDictionary <string, object> > documents;
using (var readSession = _sessionFactory.CreateQuerySession(_model))
{
documents = readSession.Query(query, _skip, _take).Documents;
}
//TODO: Remove this when cc_wat is rebuilt.
var c = "cc_wat".ToHash();
foreach (var d in documents)
{
d.TryAdd(SystemFields.CollectionId, c);
}
if (_truncate)
{
_sessionFactory.Truncate(targetCollectionId);
}
using (var documentWriter = new DocumentWriter(targetCollectionId, _sessionFactory))
{
foreach (var field in _indexFieldNames)
{
documentWriter.EnsureKeyExists(field);
}
}
_sessionFactory.SaveAs(
targetCollectionId,
documents,
_indexFieldNames,
new HashSet <string>(),
_model);
}
catch (Exception ex)
{
_logger.LogError($"error processing {this} {ex}");
}
}
public void Run(IDictionary <string, string> args, ILogger logger)
{
var time = Stopwatch.StartNew();
var images = new MnistReader(args["imageFileName"], args["labelFileName"]).Read();
var collection = args["collection"];
var count = 0;
var errors = 0;
var model = new ImageModel();
using (var sessionFactory = new SessionFactory(new KeyValueConfiguration("sir.ini"), logger))
using (var querySession = sessionFactory.CreateQuerySession(model))
{
var queryParser = new QueryParser <IImage>(sessionFactory, model, logger);
foreach (var image in images)
{
var query = queryParser.Parse(collection, image, field: "image", select: "label", and: true, or: false);
var result = querySession.Query(query, 0, 1);
count++;
if (result.Total == 0)
{
errors++;
}
else
{
var imageLabel = image.Label.ToString();
var documentLabel = result.Documents.First()["label"].ToString();
if (!documentLabel.Equals(imageLabel))
{
errors++;
logger.LogDebug($"error. label: {imageLabel} document label: {documentLabel}\n{((MnistImage)image).Print()}");
}
}
logger.LogInformation($"errors: {errors}. total tests {count}. error rate: {(float)errors / count * 100}%");
}
}
logger.LogInformation($"tested {count} mnist images in {time.Elapsed}");
}
private void DownloadAndIndexWetFile()
{
var writePayload = new List <IDictionary <string, object> >();
var originalQuery = _queryParser.Parse(
Collections,
Q,
Fields,
select: new string[] { "url", "title", "filename" },
and: And,
or: Or);
using (var readSession = _sessionFactory.CreateQuerySession(_model))
{
var originalResult = readSession.Query(originalQuery, _skip, _take)
.Documents
.ToDictionary(x => (string)x["url"]);
var wetFileIds = new SortedList <string, object>();
ReadResult wetResult = null;
var wetCollectionId = "cc_wet".ToHash();
foreach (var doc in originalResult.Values)
{
if (doc["filename"] is object[])
{
continue;
}
var wetFileId = ((string)doc["filename"]).Replace("/warc", "/wet").Replace(".gz", ".wet.gz");
wetFileIds.TryAdd(wetFileId, null);
break;
}
foreach (var fileName in wetFileIds.Keys)
{
var wetQuery = _queryParser.Parse(
collections: new string[] { "cc_wet" },
q: fileName,
fields: new string[] { "filename" },
select: new string[] { "filename" },
and: true,
or: false);
if (wetQuery != null)
{
wetResult = readSession.Query(wetQuery, 0, 1);
}
if (wetResult == null || wetResult.Total == 0)
{
var localFileName = Path.Combine(_sessionFactory.Dir, "wet", fileName);
var tmpFileName = Path.Combine(_sessionFactory.Dir, "tmp", Id, fileName);
if (!File.Exists(localFileName))
{
if (!Directory.Exists(Path.GetDirectoryName(tmpFileName)))
{
Directory.CreateDirectory(Path.GetDirectoryName(tmpFileName));
}
var remoteFileName = $"https://commoncrawl.s3.amazonaws.com/{fileName}";
const double payloadSize = 150000000;
using (var client = new WebClient())
{
var state = new State {
Completed = false
};
client.DownloadFileCompleted += Client_DownloadFileCompleted;
client.DownloadFileAsync(new Uri(remoteFileName), tmpFileName, state);
while (!state.Completed)
{
try
{
if (File.Exists(tmpFileName))
{
var fi = new FileInfo(tmpFileName);
if (fi.Length > 0)
{
var status = (fi.Length / (payloadSize * wetFileIds.Count)) * 100;
Status["download"] = status;
}
}
}
catch { }
finally
{
Thread.Sleep(1000);
}
}
}
}
if (!File.Exists(localFileName))
{
try
{
var localDir = Path.GetDirectoryName(localFileName);
if (!Directory.Exists(localDir))
{
Directory.CreateDirectory(localDir);
}
File.Move(tmpFileName, localFileName, true);
Thread.Sleep(100);
Directory.Delete(Path.GetDirectoryName(tmpFileName));
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
}
}
foreach (var document in ReadWetFile(localFileName, fileName))
{
IDictionary <string, object> originalDoc;
var key = (string)document["url"];
if (originalResult.TryGetValue(key, out originalDoc))
{
document["title"] = originalDoc["title"];
document["filename"] = originalDoc["filename"];
writePayload.Add(document);
}
}
}
}
Status["download"] = 100;
if (writePayload.Count > 0)
{
var time = Stopwatch.StartNew();
var writeJob = new WriteJob(
wetCollectionId,
writePayload,
new TextModel(),
_wetStoredFieldNames,
_wetIndexedFieldNames);
_sessionFactory.Write(writeJob, reportSize: 1000);
Status["index"] = 100;
_logger.LogInformation($"wet file write job took {time.Elapsed}");
}
}
}