private static bool TryGetDiskCacheResult(string source, string name, InMemoryRavenConfiguration configuration, out string indexFilePath,
out Type type)
{
// It's not in the in-memory cache. See if it's been cached on disk.
//
// Q. Why do we cache on disk?
// A. It decreases the duration of individual test runs. Instead of
// recompiling the index each test run, we can just load them from disk.
// It also decreases creation time for indexes that were
// previously created and deleted, affecting both production and test environments.
//
// For more info, see http://ayende.com/blog/161218/robs-sprint-idly-indexing?key=f37cf4dc-0e5c-43be-9b27-632f61ba044f#comments-form-location
var indexCacheDir = GetIndexCacheDir(configuration);
string sourceHashed;
using (var md5 = MD5.Create())
{
var hash = md5.ComputeHash(Encoding.UTF8.GetBytes(source));
sourceHashed = MonoHttpUtility.UrlEncode(Convert.ToBase64String(hash));
}
indexFilePath = Path.Combine(indexCacheDir,
IndexingUtil.StableInvariantIgnoreCaseStringHash(source) + "." + sourceHashed + "." +
(Debugger.IsAttached ? "debug" : "nodebug") + ".dll");
try
{
if (Directory.Exists(indexCacheDir) == false)
{
Directory.CreateDirectory(indexCacheDir);
}
type = TryGetIndexFromDisk(indexFilePath, name);
}
catch (UnauthorizedAccessException)
{
// permission issues
type = null;
return(false);
}
catch (IOException)
{
// permission issues, probably
type = null;
return(false);
}
if (type != null)
{
cacheEntries.TryAdd(source, new CacheEntry
{
Source = source,
Type = type,
Usages = 1
});
{
return(true);
}
}
return(false);
}
public void PutMappedResult(string view, string docId, string reduceKey, RavenJObject data)
{
Guid etag = uuidGenerator.CreateSequentialUuid();
using (var update = new Update(session, MappedResults, JET_prep.Insert))
{
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["view"], view, Encoding.Unicode);
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["document_key"], docId, Encoding.Unicode);
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["reduce_key"], reduceKey, Encoding.Unicode);
var mapBucket = IndexingUtil.MapBucket(docId);
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["bucket"], mapBucket);
using (Stream stream = new BufferedStream(new ColumnStream(session, MappedResults, tableColumnsCache.MappedResultsColumns["data"])))
{
using (var dataStream = documentCodecs.Aggregate(stream, (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds)))
{
data.WriteTo(dataStream);
dataStream.Flush();
}
}
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["etag"], etag.TransformToValueForEsentSorting());
Api.SetColumn(session, MappedResults, tableColumnsCache.MappedResultsColumns["timestamp"], SystemTime.UtcNow);
update.Save();
}
}
private static string GetIndexFilePath(string source, string indexCacheDir)
{
var hash = Encryptor.Current.Hash.Compute16(Encoding.UTF8.GetBytes(source));
var sourceHashed = MonoHttpUtility.UrlEncode(Convert.ToBase64String(hash));
var indexFilePath = Path.Combine(indexCacheDir,
IndexingUtil.StableInvariantIgnoreCaseStringHash(source) + "." + sourceHashed + "." +
(Debugger.IsAttached ? "debug" : "nodebug") + ".dll");
return(indexFilePath);
}
private static string GetIndexFilePath(string source, string indexCacheDir, out int numberHash)
{
var hash = Hashing.XXHash64.CalculateRaw(source);
var sourceHashed = hash.ToString("X");
numberHash = IndexingUtil.StableInvariantIgnoreCaseStringHash(source);
var indexFilePath = Path.Combine(indexCacheDir,
numberHash + "." + sourceHashed + "." +
(Debugger.IsAttached ? "debug" : "nodebug") + ".dll");
return(indexFilePath);
}
public void PutMappedResult(int view, string docId, string reduceKey, RavenJObject data)
{
var mappedResultsByViewAndDocumentId = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndDocumentId);
var mappedResultsByView = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByView);
var mappedResultsByViewAndReduceKey = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKey);
var mappedResultsByViewAndReduceKeyAndSourceBucket = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.ByViewAndReduceKeyAndSourceBucket);
var mappedResultsData = tableStorage.MappedResults.GetIndex(Tables.MappedResults.Indices.Data);
var ms = CreateStream();
using (var stream = documentCodecs.Aggregate((Stream) new UndisposableStream(ms), (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds)))
{
data.WriteTo(stream);
stream.Flush();
}
var id = generator.CreateSequentialUuid(UuidType.MappedResults);
var idAsString = id.ToString();
var bucket = IndexingUtil.MapBucket(docId);
var reduceKeyHash = HashKey(reduceKey);
tableStorage.MappedResults.Add(
writeBatch.Value,
idAsString,
new RavenJObject
{
{ "view", view },
{ "reduceKey", reduceKey },
{ "docId", docId },
{ "etag", id.ToByteArray() },
{ "bucket", bucket },
{ "timestamp", SystemTime.UtcNow }
}, 0);
ms.Position = 0;
mappedResultsData.Add(writeBatch.Value, idAsString, ms, 0);
mappedResultsByViewAndDocumentId.MultiAdd(writeBatch.Value, CreateKey(view, docId), idAsString);
mappedResultsByView.MultiAdd(writeBatch.Value, CreateKey(view), idAsString);
mappedResultsByViewAndReduceKey.MultiAdd(writeBatch.Value, CreateKey(view, reduceKey, reduceKeyHash), idAsString);
mappedResultsByViewAndReduceKeyAndSourceBucket.MultiAdd(writeBatch.Value, CreateKey(view, reduceKey, reduceKeyHash, bucket), idAsString);
}
public void NumberOfLoadedItemsToReduceShouldBeLimited(string storageType)
{
using (var storage = NewTransactionalStorage(requestedStorage: storageType))
{
storage.Batch(accessor =>
{
accessor.Indexing.AddIndex("test", true);
accessor.MapReduce.PutMappedResult("test", "a/1", "a", new RavenJObject()
{
{ "A", "a" }
});
accessor.MapReduce.PutMappedResult("test", "a/1", "a", new RavenJObject()
{
{ "B", "b" }
});
accessor.MapReduce.PutMappedResult("test", "b/1", "b", new RavenJObject()
{
{ "C", "c" }
});
accessor.MapReduce.PutMappedResult("test", "b/1", "b", new RavenJObject()
{
{ "D", "d" }
});
accessor.MapReduce.ScheduleReductions("test", 0,
new List <ReduceKeyAndBucket>()
{
new ReduceKeyAndBucket(IndexingUtil.MapBucket("a/1"), "a"),
new ReduceKeyAndBucket(IndexingUtil.MapBucket("b/1"), "b")
});
});
storage.Batch(accessor =>
{
var results = accessor.MapReduce.GetItemsToReduce("test", new[] { "a", "b" }, 0, true, 2, new List <object>(), new HashSet <Tuple <string, int> >()).ToList();
Assert.Equal(2, results.Count);
Assert.Equal(results[0].Bucket, results[1].Bucket);
});
}
}
public void PutMappedResult(string view, string docId, string reduceKey, RavenJObject data)
{
var ms = new MemoryStream();
using (var stream = documentCodecs.Aggregate((Stream)ms, (ds, codec) => codec.Value.Encode(reduceKey, data, null, ds)))
{
data.WriteTo(stream);
}
var byteArray = generator.CreateSequentialUuid().ToByteArray();
var key = new RavenJObject
{
{ "view", view },
{ "reduceKey", reduceKey },
{ "docId", docId },
{ "etag", byteArray },
{ "bucket", IndexingUtil.MapBucket(docId) },
{ "timestamp", SystemTime.UtcNow }
};
storage.MappedResults.Put(key, ms.ToArray());
}
private void FindIndexName(DocumentDatabase database, DynamicQueryMapping map, IndexQuery query)
{
var targetName = map.ForEntityName ?? "AllDocs";
var combinedFields = String.Join("And",
map.Items
.OrderBy(x => x.To)
.Select(x => x.To));
var indexName = combinedFields;
if (map.SortDescriptors != null && map.SortDescriptors.Length > 0)
{
indexName = string.Format("{0}SortBy{1}", indexName,
String.Join("",
map.SortDescriptors
.Select(x => x.Field)
.OrderBy(x => x)));
}
if (map.HighlightedFields != null && map.HighlightedFields.Length > 0)
{
indexName = string.Format("{0}Highlight{1}", indexName,
string.Join("", map.HighlightedFields.OrderBy(x => x)));
}
string groupBy = null;
if (database.Configuration.RunInUnreliableYetFastModeThatIsNotSuitableForProduction == false &&
database.Configuration.RunInMemory == false)
{
indexName = IndexingUtil.FixupIndexName(indexName, database.Configuration.DataDirectory);
}
var permanentIndexName = indexName.Length == 0
? string.Format("Auto/{0}{1}", targetName, groupBy)
: string.Format("Auto/{0}/By{1}{2}", targetName, indexName, groupBy);
map.IndexName = permanentIndexName;
}
public void LimitOfLoadedItemsShouldNotBreakInTheMiddleOfBucket(string storageType)
{
using (var storage = NewTransactionalStorage(requestedStorage: storageType))
{
storage.Batch(accessor =>
{
accessor.Indexing.AddIndex("test", true);
accessor.MapReduce.PutMappedResult("test", "a/1", "a", new RavenJObject()
{
{ "A", "a" }
});
accessor.MapReduce.PutMappedResult("test", "a/1", "a", new RavenJObject()
{
{ "B", "b" }
});
accessor.MapReduce.PutMappedResult("test", "b/1", "b", new RavenJObject()
{
{ "C", "c" }
});
accessor.MapReduce.PutMappedResult("test", "b/1", "b", new RavenJObject()
{
{ "D", "d" }
});
accessor.MapReduce.ScheduleReductions("test", 0, new ReduceKeyAndBucket(IndexingUtil.MapBucket("a/1"), "a"));
accessor.MapReduce.ScheduleReductions("test", 0, new ReduceKeyAndBucket(IndexingUtil.MapBucket("b/1"), "b"));
});
storage.Batch(accessor =>
{
var results = accessor.MapReduce.GetItemsToReduce(new GetItemsToReduceParams("test", new[] { "a", "b" }, 0, true, new List <object>())
{
Take = 3
}).ToList();
Assert.Equal(4, results.Count);
});
}
}
public void NumberOfLoadedItemsToReduceShouldBeLimited(string storageType)
{
using (var storage = NewTransactionalStorage(requestedStorage: storageType))
{
storage.Batch(accessor =>
{
accessor.Indexing.AddIndex(test, true);
accessor.MapReduce.PutMappedResult(test, "a/1", "a", new RavenJObject()
{
{ "A", "a" }
});
accessor.MapReduce.PutMappedResult(test, "a/1", "a", new RavenJObject()
{
{ "B", "b" }
});
accessor.MapReduce.PutMappedResult(test, "b/1", "b", new RavenJObject()
{
{ "C", "c" }
});
accessor.MapReduce.PutMappedResult(test, "b/1", "b", new RavenJObject()
{
{ "D", "d" }
});
accessor.MapReduce.ScheduleReductions(test, 0, new ReduceKeyAndBucket(IndexingUtil.MapBucket("a/1"), "a"));
accessor.MapReduce.ScheduleReductions(test, 0, new ReduceKeyAndBucket(IndexingUtil.MapBucket("b/1"), "b"));
});
storage.Batch(accessor =>
{
var results = accessor.MapReduce.GetItemsToReduce(new GetItemsToReduceParams(test, new HashSet <string> {
"a", "b"
}, 0, true, new ConcurrentSet <object>())
{
Take = 2
}, CancellationToken.None).ToList();
Assert.Equal(2, results.Count);
Assert.Equal(results[0].Bucket, results[1].Bucket);
});
}
}
private static void MyTest()
{
var tx = new Raven.Storage.Managed.TransactionalStorage(new RavenConfiguration
{
RunInMemory = true
}, () => { });
tx.Initialize(new DummyUuidGenerator(), new OrderedPartCollection <AbstractDocumentCodec>());
for (int xi = 0; xi < 5; xi++)
{
var wait = xi;
Task.Factory.StartNew(() =>
{
Thread.Sleep(15 * wait);
tx.Batch(accessor =>
{
var reduceKeysAndBuckets = new List <ReduceKeyAndBucket>();
for (int i = 0; i < 10; i++)
{
var docId = "users/" + i;
reduceKeysAndBuckets.Add(new ReduceKeyAndBucket(IndexingUtil.MapBucket(docId), "1"));
reduceKeysAndBuckets.Add(new ReduceKeyAndBucket(IndexingUtil.MapBucket(docId), "2"));
accessor.MapReduce.PutMappedResult("test", docId, "1", new RavenJObject());
accessor.MapReduce.PutMappedResult("test", docId, "2", new RavenJObject());
}
accessor.MapReduce.ScheduleReductions("test", 0, reduceKeysAndBuckets);
});
});
Task.Factory.StartNew(() =>
{
Thread.Sleep(15 * wait);
tx.Batch(accessor =>
{
var reduceKeysAndBuckets = new List <ReduceKeyAndBucket>();
for (int i = 0; i < 10; i++)
{
var docId = "users/" + i;
reduceKeysAndBuckets.Add(new ReduceKeyAndBucket(IndexingUtil.MapBucket(docId), "1"));
reduceKeysAndBuckets.Add(new ReduceKeyAndBucket(IndexingUtil.MapBucket(docId), "2"));
accessor.MapReduce.PutMappedResult("test3", docId, "1", new RavenJObject());
accessor.MapReduce.PutMappedResult("test3", docId, "2", new RavenJObject());
}
accessor.MapReduce.ScheduleReductions("test3", 0, reduceKeysAndBuckets);
});
});
}
var items = 0;
while (items != 100)
{
var itemsToDelete = new List <object>();
tx.Batch(accessor =>
{
var list = accessor.MapReduce.GetItemsToReduce(
index: "test",
level: 0,
take: 256,
itemsToDelete: itemsToDelete
).ToList();
items += list.Count;
Console.WriteLine(list.Count);
});
tx.Batch(accessor =>
{
accessor.MapReduce.DeleteScheduledReduction(itemsToDelete);
});
Thread.Sleep(10);
}
}
public void CanGetCorrectResults()
{
for (int x = 0; x < 100; x++)
{
Environment.SetEnvironmentVariable("Test", x.ToString(CultureInfo.InvariantCulture));
using (var store = CreateStore())
{
var test1 = IndexingUtil.MapBucket("8787bc88-d29e-4e1b-aa6a-8aa407537188");
var test2 = IndexingUtil.MapBucket("8787bc88-d29e-4e1b-aa6a-8aa407537187");
//var ids = new[]
// {
// "8787bc88-d29e-4e1b-aa6a-8aa407537188",
// "c70b1644-0c1f-4196-aed9-e3b9a47b329c", // maps to bucket 517 (map) then to bucket 0 (level 1) //this FAILS!!!!
// //"c70b1644-0c1f-4196-aed9-e3b9a47b329d", // maps to bucket 119848 (map) then to bucket 195 (level 1) //this passes!!!!!
// "55d306e0-5999-4994-887b-d0c409b358b8",
// //"37a304ed-f8a9-4e89-8bb5-24979259bae5",
// //"692fd6e1-ea0a-419d-8fb8-559e03e4fbfa",
// //"e5a5ccfa-3ba3-47ac-a050-55f74b6bc782",
// //"d2fc6934-696e-45e5-9aa2-505b0ea1f10c",
// //"410f4f7b-48eb-4efa-b3e9-c314095ef45e",
// //"2b3797ac-3529-4d12-99a1-f0f6da252b10",
// //"2a3ff02e-8ad9-4b7c-ae68-2052c7128a9a",
// //"19513de8-1692-4384-8081-5e736c9e2d30",
// //"b8fccdff-4261-440e-a952-54b22a60dab6",
// };
var ids = new[]
{
"cb113766-79cc-4a34-89c2-2ff66b13c9de",
"1aebbf97-874b-484a-9539-30cd063f0099",
"de4252df-4d08-4b82-842f-cf4a37a3e27d",
"3870f4aa-58b6-40e2-8085-c1d0d782efec",
"d8cf60cf-4004-4df1-b071-cd8fbcbd387e",
"86b2b9b6-1c2a-47d8-88f3-8575567e321d",
"c37fd4ec-264e-4498-9f4e-43daecdb4538",
"dae40ffc-4eb6-4648-82c3-8dfc6c51c772",
"30394662-24e0-4f12-afc4-d8f276180e30",
"f4965176-41cd-404d-9320-0d668f07c5be",
"19b34bfb-8c9c-4b16-a1b0-970802c6e43a",
"ade6b1a4-9fff-413f-80d0-5bb228729f9a"
};
for (int i = 0; i < ids.Length; i++)
//for (int i = 0; i < 12; i++)
{
var key = ids[i];
AddRecord(store, 1, key);
//AddRecord(store, 1, Guid.NewGuid().ToString());
if (i % 3 == 0)
{
Thread.Sleep(40);
}
else
{
Thread.Sleep(5);
}
}
ReadRecords(store, ids.Length * 5);
//ReadRecords(store, 60);
//Console.ReadLine();
Console.Clear();
}
}
}
private static void AddRecord(IDocumentStore store, int records, string key)
{
using (var session = store.OpenSession())
{
for (int i = 0; i < records; i++)
{
var item = new TestClass
{
Id = key,
Items = new List <Item>()
};
for (int j = 0; j < 5; j++)
{
item.Items.Add(new Item
{
Id = j + 1,
Email = string.Format("rob{0}@text.com", i + 1),
Name = string.Format("rob{0}", i + 1)
});
}
session.Store(item);
session.SaveChanges();
Console.WriteLine("@@@ SaveChanges() - Doc Id {0}, bucket {1}", item.Id, IndexingUtil.MapBucket(item.Id));
}
//session.SaveChanges();
}
}