public Task DeleteGrain(GrainId grain)
{
SiloAddress silo = CalculateTargetSilo(grain);
if (silo == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (log.IsVerbose)
{
log.Verbose("Silo {0} tries to lookup for {1}-->{2} ({3}-->{4})", MyAddress, grain, silo, grain.GetUniformHashCode(), silo.GetConsistentHashCode());
}
if (silo.Equals(MyAddress))
{
// remove from our partition
DirectoryPartition.RemoveGrain(grain);
return(TaskDone.Done);
}
// remove from the cache
DirectoryCache.Remove(grain);
// send request to the owner
return(GetDirectoryReference(silo).DeleteGrain(grain, NUM_RETRIES));
}
/// Adjust local cache following the removal of a silo by droping:
/// 1) entries that point to activations located on the removed silo
/// 2) entries for grains that are now owned by this silo (me)
/// 3) entries for grains that were owned by this removed silo - we currently do NOT do that.
/// If we did 3, we need to do that BEFORE we change the membershipRingList (based on old Membership).
/// We don't do that since first cache refresh handles that.
/// Second, since Membership events are not guaranteed to be ordered, we may remove a cache entry that does not really point to a failed silo.
/// To do that properly, we need to store for each cache entry who was the directory owner that registered this activation (the original partition owner).
protected void AdjustLocalCache(SiloAddress removedSilo)
{
// remove all records of activations located on the removed silo
foreach (Tuple <GrainId, List <Tuple <SiloAddress, ActivationId> >, int> tuple in DirectoryCache.KeyValues)
{
// 2) remove entries owned by me (they should be retrieved from my directory partition)
if (MyAddress.Equals(CalculateTargetSilo(tuple.Item1)))
{
DirectoryCache.Remove(tuple.Item1);
}
// 1) remove entries that point to activations located on the removed silo
if (tuple.Item2.RemoveAll(tuple2 => tuple2.Item1.Equals(removedSilo)) <= 0)
{
continue;
}
if (tuple.Item2.Count > 0)
{
DirectoryCache.AddOrUpdate(tuple.Item1, tuple.Item2, tuple.Item3);
}
else
{
DirectoryCache.Remove(tuple.Item1);
}
}
}
public async Task DeleteGrainAsync(GrainId grainId, int hopCount)
{
// see if the owner is somewhere else (returns null if we are owner)
var forwardAddress = this.CheckIfShouldForward(grainId, hopCount, "DeleteGrainAsync");
// on all silos other than first, we insert a retry delay and recheck owner before forwarding
if (hopCount > 0 && forwardAddress != null)
{
await Task.Delay(RETRY_DELAY);
forwardAddress = this.CheckIfShouldForward(grainId, hopCount, "DeleteGrainAsync");
this.log.LogWarning($"DeleteGrainAsync - It seems we are not the owner of grain {grainId}, trying to forward it to {forwardAddress} (hopCount={hopCount})");
}
if (forwardAddress == null)
{
// we are the owner
DirectoryPartition.RemoveGrain(grainId);
}
else
{
// otherwise, notify the owner
DirectoryCache.Remove(grainId);
await GetDirectoryReference(forwardAddress).DeleteGrainAsync(grainId, hopCount + 1);
}
}
private static ICache GetQueryCache()
{
if (EnableQueryCache == false)
{
return(new NullCache());
}
var cacheDir = QueryCacheDirectory;
if (String.IsNullOrEmpty(cacheDir) && (!String.IsNullOrEmpty(StoreLocation)))
{
cacheDir = Path.Combine(StoreLocation, "_bscache");
}
if (!String.IsNullOrEmpty(cacheDir))
{
if (!Directory.Exists(cacheDir))
{
Directory.CreateDirectory(cacheDir);
}
ICache directoryCache = new DirectoryCache(cacheDir, MegabytesToBytes * QueryCacheDiskSpace,
new LruCacheEvictionPolicy()),
memoryCache = new MemoryCache(MegabytesToBytes * QueryCacheMemory, new LruCacheEvictionPolicy());
return(new TwoLevelCache(memoryCache, directoryCache));
}
else
{
ICache memoryCache = new MemoryCache(MegabytesToBytes * QueryCacheMemory, new LruCacheEvictionPolicy());
return(memoryCache);
}
}
public static InstanceCache LoadSettings()
{
InstanceCache cache = new InstanceCache();
DataDeserializer dds = new DataContractXMLLoader();
cache.AddInstance(dds);
cache.AddInstance((DataSerializer)dds);
DirectoryCache dirCache = new DirectoryCache();
if (File.Exists(Settings.Default.DirectoryCacheFile))
{
dirCache = dds.Deserialize <DirectoryCache>(Settings.Default.DirectoryCacheFile);
}
dirCache.ContainerPath = Settings.Default.ContainerDirectory;
cache.AddInstance(dirCache);
ManagedVersionsService mvs = new ManagedVersionsService();
if (File.Exists(Settings.Default.ManagedVersionsFile))
{
mvs.Load(dds);
}
mvs.Serializer = (DataSerializer)dds;
cache.AddInstance(mvs);
return(cache);
}
private AzureBlockStore(AzureBlockStoreConfiguration configuration)
{
#if BLOCKSTORECACHE
_cache = new BlockStoreCache((long)configuration.MemoryCacheInMB * 1024 * 1024, configuration.LocalStorageKey);
#else
ICache memoryCache = new MemoryCache((long)configuration.MemoryCacheInMB * 1024 * 1024, new LruCacheEvictionPolicy());
if (!String.IsNullOrEmpty(configuration.LocalStorageKey))
{
var localDiskResource = RoleEnvironment.GetLocalResource(configuration.LocalStorageKey);
ICache diskCache = new DirectoryCache(localDiskResource.RootPath,
((long)localDiskResource.MaximumSizeInMegabytes - 1) * 1024 * 1024,
new LruCacheEvictionPolicy());
_cache = new TwoLevelCache(memoryCache, diskCache);
}
else
{
_cache = memoryCache;
}
#endif
_highestPageOffsetByPath = new Dictionary <string, long>();
_storageAccount = CloudStorageAccount.Parse(configuration.ConnectionString);
_commitList = new ConcurrentQueue <BlockInfo>();
_commitThread = new Thread(RunCommitThread);
_commitThread.Start();
_disconnected = configuration.Disconnected;
}
public List <ActivationAddress> GetLocalCacheData(GrainId grain)
{
IReadOnlyList <Tuple <SiloAddress, ActivationId> > cached;
return(DirectoryCache.LookUp(grain, out cached) ?
cached.Select(elem => ActivationAddress.GetAddress(elem.Item1, grain, elem.Item2)).Where(addr => IsValidSilo(addr.Silo)).ToList() :
null);
}
public GrainAddress GetLocalCacheData(GrainId grain)
{
if (DirectoryCache.LookUp(grain, out var cache) && IsValidSilo(cache.SiloAddress))
{
return(cache);
}
return(null);
}
/// <summary>
/// Task for inaccessible path
/// </summary>
/// <param name="path"></param>
/// <param name="rootCache"></param>
private static void DiskUsageTask(string path, DirectoryCache rootCache)
{
var currentCacheNode = rootCache.GetSubDirectoryCache(path);
if (currentCacheNode == null || currentCacheNode.Accessible)
{
return;
}
LoadDirectoryInfoByCmd(currentCacheNode);
}
public async Task <AddressAndTag> RegisterAsync(GrainAddress address, int hopCount)
{
var counterStatistic = hopCount > 0 ? this.RegistrationsSingleActRemoteReceived : this.registrationsSingleActIssued;
counterStatistic.Increment();
// see if the owner is somewhere else (returns null if we are owner)
var forwardAddress = this.CheckIfShouldForward(address.GrainId, hopCount, "RegisterAsync");
// on all silos other than first, we insert a retry delay and recheck owner before forwarding
if (hopCount > 0 && forwardAddress != null)
{
await Task.Delay(RETRY_DELAY);
forwardAddress = this.CheckIfShouldForward(address.GrainId, hopCount, "RegisterAsync");
if (forwardAddress is object)
{
int hash = unchecked ((int)address.GrainId.GetUniformHashCode());
this.log.LogWarning($"RegisterAsync - It seems we are not the owner of activation {address} (hash: {hash:X}), trying to forward it to {forwardAddress} (hopCount={hopCount})");
}
}
if (forwardAddress == null)
{
RegistrationsSingleActLocal.Increment();
var result = DirectoryPartition.AddSingleActivation(address);
return(result);
}
else
{
RegistrationsSingleActRemoteSent.Increment();
// otherwise, notify the owner
AddressAndTag result = await GetDirectoryReference(forwardAddress).RegisterAsync(address, hopCount + 1);
// Caching optimization:
// cache the result of a successfull RegisterSingleActivation call, only if it is not a duplicate activation.
// this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup!
if (result.Address == null)
{
return(result);
}
if (!address.Equals(result.Address) || !IsValidSilo(address.SiloAddress))
{
return(result);
}
// update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup.
DirectoryCache.AddOrUpdate(address, result.VersionTag);
return(result);
}
}
/// <summary>
/// This is the function that actually does the searching. It creates threads and searches and then
/// waits for all the results to be sent.
/// </summary>
public void internalSearch()
{
String searchDirectory = mSearchParams.SearchDirectory;
if (mRegex != null)
{
List <String> fileList = DirectoryCache.get().getFileList();
if (fileList != null)
{
Stats.TotalFilesToSearch = fileList.Count;
Logger.get().AddInfoFormat("{0} files to search", Stats.TotalFilesToSearch);
try
{
// Set up the parallel lambda expression. The parameters were trial and error trying to get the fastest execution
fileList.AsParallel().WithExecutionMode(ParallelExecutionMode.ForceParallelism).WithDegreeOfParallelism(63)
.WithCancellation(Stats.CancellationTokenSource.Token)
.WithMergeOptions(ParallelMergeOptions.FullyBuffered).ForAll(file =>
{
// Check for cancellation, this speeds up the cancel after the button is clicked.
if (!Stats.CancellationTokenSource.IsCancellationRequested)
{
if (mSearchParams.FileNameSearch)
{
if (mRegex.IsMatch(file))
{
SearchResultList list = new SearchResultList(PathExt.GetRelativePath(searchDirectory, file));
saveResults(list);
}
}
else
{
searchFile(file);
}
}
});
}
catch (OperationCanceledException)
{
// Cancelling is not an error, why do they throw an exception?
}
catch (Exception ex)
{
Logger.get().AddError(ex.Message);
}
}
else
{
SearchResultList results = new SearchResultList(DirectoryCache.get().ErrorMessage);
mPendingResults.Add(results);
}
}
waitForResultsToBeSent();
}
public void InvalidateCacheEntry(ActivationAddress activationAddress)
{
var grainId = activationAddress.Grain;
var activationId = activationAddress.Activation;
// look up grainId activations
if (DirectoryCache.LookUp(grainId, out var entry, out _) && activationId.Equals(entry.Activation))
{
DirectoryCache.Remove(grainId);
}
}
public void InvalidateCacheEntry(ActivationAddress activationAddress)
{
int version;
IReadOnlyList <Tuple <SiloAddress, ActivationId> > list;
var grainId = activationAddress.Grain;
var activationId = activationAddress.Activation;
// look up grain activations
if (DirectoryCache.LookUp(grainId, out list, out version))
{
RemoveActivations(DirectoryCache, grainId, list, version, t => t.Item2.Equals(activationId));
}
}
/// Adjust local cache following the removal of a silo by droping:
/// 1) entries that point to activations located on the removed silo
/// 2) entries for grains that are now owned by this silo (me)
/// 3) entries for grains that were owned by this removed silo - we currently do NOT do that.
/// If we did 3, we need to do that BEFORE we change the membershipRingList (based on old Membership).
/// We don't do that since first cache refresh handles that.
/// Second, since Membership events are not guaranteed to be ordered, we may remove a cache entry that does not really point to a failed silo.
/// To do that properly, we need to store for each cache entry who was the directory owner that registered this activation (the original partition owner).
protected void AdjustLocalCache(SiloAddress removedSilo)
{
// remove all records of activations located on the removed silo
foreach (Tuple <GrainId, IReadOnlyList <Tuple <SiloAddress, ActivationId> >, int> tuple in DirectoryCache.KeyValues)
{
// 2) remove entries now owned by me (they should be retrieved from my directory partition)
if (MyAddress.Equals(CalculateTargetSilo(tuple.Item1)))
{
DirectoryCache.Remove(tuple.Item1);
}
// 1) remove entries that point to activations located on the removed silo
RemoveActivations(DirectoryCache, tuple.Item1, tuple.Item2, tuple.Item3, t => t.Item1.Equals(removedSilo));
}
}
public async Task RegisterAsync(ActivationAddress address)
{
registrationsIssued.Increment();
SiloAddress owner = CalculateTargetSilo(address.Grain);
if (owner == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (owner.Equals(MyAddress))
{
RegistrationsLocal.Increment();
// if I am the owner, store the new activation locally
DirectoryPartition.AddActivation(address.Grain, address.Activation, address.Silo);
}
else
{
RegistrationsRemoteSent.Increment();
// otherwise, notify the owner
int eTag = await GetDirectoryReference(owner).Register(address, NUM_RETRIES);
if (IsValidSilo(address.Silo))
{
// Caching optimization:
// cache the result of a successfull RegisterActivation call, only if it is not a duplicate activation.
// this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup!
IReadOnlyList <Tuple <SiloAddress, ActivationId> > cached;
if (!DirectoryCache.LookUp(address.Grain, out cached))
{
cached = new List <Tuple <SiloAddress, ActivationId> >(1)
{
Tuple.Create(address.Silo, address.Activation)
};
}
else
{
var newcached = new List <Tuple <SiloAddress, ActivationId> >(cached.Count + 1);
newcached.AddRange(cached);
newcached.Add(Tuple.Create(address.Silo, address.Activation));
cached = newcached;
}
// update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup.
DirectoryCache.AddOrUpdate(address.Grain, cached, eTag);
}
}
}
/// <summary>
/// Registers a new activation, in single activation mode, with the directory service.
/// If there is already an activation registered for this grain, then the new activation will
/// not be registered and the address of the existing activation will be returned.
/// Otherwise, the passed-in address will be returned.
/// <para>This method must be called from a scheduler thread.</para>
/// </summary>
/// <param name="address">The address of the potential new activation.</param>
/// <returns>The address registered for the grain's single activation.</returns>
public async Task <ActivationAddress> RegisterSingleActivationAsync(ActivationAddress address)
{
registrationsSingleActIssued.Increment();
SiloAddress owner = CalculateTargetSilo(address.Grain);
if (owner == null)
{
// We don't know about any other silos, and we're stopping, so throw
throw new InvalidOperationException("Grain directory is stopping");
}
if (owner.Equals(MyAddress))
{
RegistrationsSingleActLocal.Increment();
// if I am the owner, store the new activation locally
Tuple <ActivationAddress, int> returnedAddress = DirectoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo);
return(returnedAddress == null ? null : returnedAddress.Item1);
}
else
{
RegistrationsSingleActRemoteSent.Increment();
// otherwise, notify the owner
Tuple <ActivationAddress, int> returnedAddress = await GetDirectoryReference(owner).RegisterSingleActivation(address, NUM_RETRIES);
// Caching optimization:
// cache the result of a successfull RegisterSingleActivation call, only if it is not a duplicate activation.
// this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup!
if (returnedAddress == null || returnedAddress.Item1 == null)
{
return(null);
}
if (!address.Equals(returnedAddress.Item1) || !IsValidSilo(address.Silo))
{
return(returnedAddress.Item1);
}
var cached = new List <Tuple <SiloAddress, ActivationId> >(1)
{
Tuple.Create(address.Silo, address.Activation)
};
// update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup.
DirectoryCache.AddOrUpdate(address.Grain, cached, returnedAddress.Item2);
return(returnedAddress.Item1);
}
}
// Note that this implementation stops processing directory change requests (Register, Unregister, etc.) when the Stop event is raised.
// This means that there may be a short period during which no silo believes that it is the owner of directory information for a set of
// grains (for update purposes), which could cause application requests that require a new activation to be created to time out.
// The alternative would be to allow the silo to process requests after it has handed off its partition, in which case those changes
// would receive successful responses but would not be reflected in the eventual state of the directory.
// It's easy to change this, if we think the trade-off is better the other way.
public void Stop()
{
// This will cause remote write requests to be forwarded to the silo that will become the new owner.
// Requests might bounce back and forth for a while as membership stabilizes, but they will either be served by the
// new owner of the grain, or will wind up failing. In either case, we avoid requests succeeding at this silo after we've
// begun stopping, which could cause them to not get handed off to the new owner.
//mark Running as false will exclude myself from CalculateGrainDirectoryPartition(grainId)
Running = false;
if (maintainer != null)
{
maintainer.Stop();
}
DirectoryPartition.Clear();
DirectoryCache.Clear();
}
public void InvalidateCacheEntry(ActivationAddress activationAddress)
{
var grainId = activationAddress.Grain;
var activationId = activationAddress.Activation;
List <Tuple <SiloAddress, ActivationId> > list;
if (!DirectoryCache.LookUp(grainId, out list))
{
return;
}
list.RemoveAll(tuple => tuple.Item2.Equals(activationId));
// if list empty, need to remove from cache
if (list.Count == 0)
{
DirectoryCache.Remove(grainId);
}
}
/// <summary>
/// recursive load directory info
/// </summary>
/// <param name="path"></param>
/// <returns></returns>
internal static DirectoryCache LoadDirectory(string path)
{
var directoryCache = new DirectoryCache();
DirectoryInfo directory = new DirectoryInfo(path);
directoryCache.LastSeen = DateTime.Now;
directoryCache.Uri = directory.FullName;
if (!CheckAccess(directory))
{
directoryCache.Accessible = false;
UnauthorizedAccessCache.GetInstance().InsertMissPath(directory.Parent?.FullName ?? "", path);
return(directoryCache);
}
var fileList = directory.GetFiles()
.Where(item =>
!FileExcludeConfig.GetInstance().TestFileMatch(item.FullName))
.ToList();
var directoryList = directory.GetDirectories()
.Where(item =>
!FileExcludeConfig.GetInstance().TestDirectionMatch(item.FullName))
.ToList();
Parallel.ForEach(fileList,
fileInfo => {
lock (directoryCache.FileCaches) {
directoryCache.FileCaches.Add(new FileCache(fileInfo.FullName, (ulong)fileInfo.Length,
DateTime.Now));
}
});
Parallel.ForEach(directoryList,
directoryInfo => {
var dir = LoadDirectory(directoryInfo.FullName);
lock (directoryCache.SubDirectoryCaches) {
directoryCache.SubDirectoryCaches.Add(dir);
}
});
return(directoryCache);
}
/// <summary>
///
/// </summary>
/// <param name="targetPath"></param>
/// <param name="files"></param>
public void UpdateCache(string targetPath, List <string> files)
{
if (Directory.Exists(targetPath))
{
foreach (var directory in DirectoryCaches)
{
if (directory.Path == targetPath)
{
directory.Files = new List <string>(files);
directory.UpdateTime = Directory.GetLastWriteTime(targetPath);
return;
}
}
var cache = new DirectoryCache();
cache.Path = targetPath;
cache.Files = new List <string>(files);
cache.UpdateTime = Directory.GetLastWriteTime(targetPath);
DirectoryCaches.Add(cache);
}
}
// Note that this implementation stops processing directory change requests (Register, Unregister, etc.) when the Stop event is raised.
// This means that there may be a short period during which no silo believes that it is the owner of directory information for a set of
// grains (for update purposes), which could cause application requests that require a new activation to be created to time out.
// The alternative would be to allow the silo to process requests after it has handed off its partition, in which case those changes
// would receive successful responses but would not be reflected in the eventual state of the directory.
// It's easy to change this, if we think the trade-off is better the other way.
public void Stop(bool doOnStopHandoff)
{
// This will cause remote write requests to be forwarded to the silo that will become the new owner.
// Requests might bounce back and forth for a while as membership stabilizes, but they will either be served by the
// new owner of the grain, or will wind up failing. In either case, we avoid requests succeeding at this silo after we've
// begun stopping, which could cause them to not get handed off to the new owner.
Running = false;
if (doOnStopHandoff)
{
HandoffManager.ProcessSiloStoppingEvent();
}
else
{
MarkStopPreparationCompleted();
}
if (maintainer != null)
{
maintainer.Stop();
}
DirectoryCache.Clear();
}
/// Adjust local cache following the removal of a silo by dropping:
/// 1) entries that point to activations located on the removed silo
/// 2) entries for grains that are now owned by this silo (me)
/// 3) entries for grains that were owned by this removed silo - we currently do NOT do that.
/// If we did 3, we need to do that BEFORE we change the membershipRingList (based on old Membership).
/// We don't do that since first cache refresh handles that.
/// Second, since Membership events are not guaranteed to be ordered, we may remove a cache entry that does not really point to a failed silo.
/// To do that properly, we need to store for each cache entry who was the directory owner that registered this activation (the original partition owner).
protected void AdjustLocalCache(SiloAddress silo, bool dead)
{
// remove all records of activations located on the removed silo
foreach (var tuple in DirectoryCache.KeyValues)
{
var activationAddress = tuple.ActivationAddress;
// 2) remove entries now owned by me (they should be retrieved from my directory partition)
if (MyAddress.Equals(CalculateGrainDirectoryPartition(activationAddress.GrainId)))
{
DirectoryCache.Remove(activationAddress.GrainId);
}
// 1) remove entries that point to activations located on the removed silo
// For dead silos, remove any activation registered to that silo or one of its predecessors.
// For new silos, remove any activation registered to one of its predecessors.
if (activationAddress.SiloAddress.IsPredecessorOf(silo) || (activationAddress.SiloAddress.Equals(silo) && dead))
{
DirectoryCache.Remove(activationAddress.GrainId);
}
}
}
public void TestDirectoryCacheBasicOperations()
{
string cacheDirPath = Path.Combine(Path.GetTempPath(), "cachetest_" + DateTime.Now.Ticks);
ICache cache = new DirectoryCache(cacheDirPath, 2000, new LruCacheEvictionPolicy());
RunCacheTests(cache);
ICache reopenedCache = new DirectoryCache(cacheDirPath, 2000, new LruCacheEvictionPolicy());
var byteArray = cache.Lookup("ByteArray");
Assert.IsNotNull(byteArray);
Assert.AreEqual(4, byteArray.Length);
for (int i = 0; i < 4; i++)
{
Assert.AreEqual(i, byteArray[i]);
}
var cachedString = cache.Lookup <string>("String");
Assert.IsNotNull(cachedString);
Assert.AreEqual("Hello World", cachedString);
Assert.IsFalse(reopenedCache.ContainsKey("Object"));
}
/// <summary>
/// get directory size info form du.exe/du64.exe
/// we will use size not size on disk to keep same with
/// C# DirectoryInfo report
/// du.exe output example:
/// Files: 130552
/// Directories: 45942
/// Size: 16,113,239,255 bytes
/// Size on disk: 14,142,668,800 bytes
/// </summary>
/// <param name="directoryCache"></param>
/// <returns></returns>
internal static DirectoryCache LoadDirectoryInfoByCmd(DirectoryCache directoryCache)
{
var output = CmdHelper.ExecDiskUsage(directoryCache.Uri);
// check if du.exe return "", means a junction
if (output == "")
{
return(directoryCache);
}
// make windows "\r\n" to "\n" only
output = output.Replace("\r", "");
var sizeStr = output.Split("\n")[2].Trim();
sizeStr = sizeStr.Replace("Size: ", "").Replace("bytes", "");
if (ulong.TryParse(sizeStr, out var size))
{
directoryCache.Length = size;
}
return(directoryCache);
}
public override string ToString()
{
var sb = new StringBuilder();
long localLookupsDelta;
long localLookupsCurrent = localLookups.GetCurrentValueAndDelta(out localLookupsDelta);
long localLookupsSucceededDelta;
long localLookupsSucceededCurrent = localSuccesses.GetCurrentValueAndDelta(out localLookupsSucceededDelta);
long fullLookupsDelta;
long fullLookupsCurrent = fullLookups.GetCurrentValueAndDelta(out fullLookupsDelta);
long directoryPartitionSize = directoryPartitionCount.GetCurrentValue();
sb.AppendLine("Local Grain Directory:");
sb.AppendFormat(" Local partition: {0} entries", directoryPartitionSize).AppendLine();
sb.AppendLine(" Since last call:");
sb.AppendFormat(" Local lookups: {0}", localLookupsDelta).AppendLine();
sb.AppendFormat(" Local found: {0}", localLookupsSucceededDelta).AppendLine();
if (localLookupsCurrent > 0)
{
sb.AppendFormat(" Hit rate: {0:F1}%", (100.0 * localLookupsSucceededDelta) / localLookupsDelta).AppendLine();
}
sb.AppendFormat(" Full lookups: {0}", fullLookupsDelta).AppendLine();
sb.AppendLine(" Since start:");
sb.AppendFormat(" Local lookups: {0}", localLookupsCurrent).AppendLine();
sb.AppendFormat(" Local found: {0}", localLookupsSucceededCurrent).AppendLine();
if (localLookupsCurrent > 0)
{
sb.AppendFormat(" Hit rate: {0:F1}%", (100.0 * localLookupsSucceededCurrent) / localLookupsCurrent).AppendLine();
}
sb.AppendFormat(" Full lookups: {0}", fullLookupsCurrent).AppendLine();
sb.Append(DirectoryCache.ToString());
return(sb.ToString());
}
public void Extract(DirectoryEntity entity, string root, Func <BaseEntity, string> fileToPath, DirectoryCache cache, IProgress <int> progress)
{
int totalFiles = entity.GetAllFileEntities(true).Count;
int processed = 0;
Progress <int> sumProgress = new Progress <int>(prog =>
{
int percent = ++processed * 100 / totalFiles;
progress?.Report(percent);
});
ExtractInner(entity, root, fileToPath, cache, (progress != null) ? sumProgress : null);
}
public override void Extract(string root, BaseEntity entity, Func <BaseEntity, string> entityToDir, DirectoryCache cache, IProgress <int> progress)
{
PackageEntity package = entity as PackageEntity;
using (ZipArchive arch = ZipFile.OpenRead(Path.Combine(root, entity.RelativePath)))
{
foreach (var entry in arch.Entries)
{
BaseEntity ent = package.GetEntityFromRelativePath(entry.FullName, false);
if (ent is null)
{
throw new InvalidOperationException("Couldn't find entity in structure: " + entry.FullName + " of package: " + entity.RelativePath);
}
if (ent is FileEntity)
{
string dir = entityToDir(ent);
if (!cache.CacheDirectory(dir))
{
entry.ExtractToFile(Path.Combine(dir, ent.Name), false);
}
progress?.Report(1);
}
}
}
}
public async Task <List <ActivationAddress> > FullLookup(GrainId grain)
{
fullLookups.Increment();
SiloAddress silo = CalculateTargetSilo(grain, false);
// No need to check that silo != null since we're passing excludeThisSiloIfStopping = false
if (log.IsVerbose)
{
log.Verbose("Silo {0} fully lookups for {1}-->{2} ({3}-->{4})", MyAddress, grain, silo, grain.GetUniformHashCode(), silo.GetConsistentHashCode());
}
// We assyme that getting here means the grain was not found locally (i.e., in TryFullLookup()).
// We still check if we own the grain locally to avoid races between the time TryFullLookup() and FullLookup() were called.
if (silo.Equals(MyAddress))
{
LocalDirectoryLookups.Increment();
var localResult = DirectoryPartition.LookUpGrain(grain);
if (localResult == null)
{
// it can happen that we cannot find the grain in our partition if there were
// some recent changes in the membership
if (log.IsVerbose2)
{
log.Verbose2("FullLookup mine {0}=none", grain);
}
return(new List <ActivationAddress>());
}
var a = localResult.Item1.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).Where(addr => IsValidSilo(addr.Silo)).ToList();
if (log.IsVerbose2)
{
log.Verbose2("FullLookup mine {0}={1}", grain, a.ToStrings());
}
LocalDirectorySuccesses.Increment();
return(a);
}
// Just a optimization. Why sending a message to someone we know is not valid.
if (!IsValidSilo(silo))
{
throw new OrleansException(String.Format("Current directory at {0} is not stable to perform the lookup for grain {1} (it maps to {2}, which is not a valid silo). Retry later.", MyAddress, grain, silo));
}
RemoteLookupsSent.Increment();
Tuple <List <Tuple <SiloAddress, ActivationId> >, int> result = await GetDirectoryReference(silo).LookUp(grain, NUM_RETRIES);
// update the cache
List <Tuple <SiloAddress, ActivationId> > entries = result.Item1.Where(t => IsValidSilo(t.Item1)).ToList();
List <ActivationAddress> addresses = entries.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).ToList();
if (log.IsVerbose2)
{
log.Verbose2("FullLookup remote {0}={1}", grain, addresses.ToStrings());
}
if (entries.Count > 0)
{
DirectoryCache.AddOrUpdate(grain, entries, result.Item2);
}
return(addresses);
}
private void ExtractInner(DirectoryEntity entity, string root, Func <BaseEntity, string> fileToPath, DirectoryCache cache, IProgress <int> progress)
{
foreach (BaseEntity ent in entity.GetAllFileEntities())
{
foreach (Extractor ex in _extractors)
{
if (ex.CanExtract(ent))
{
ex.Extract(root, ent, fileToPath, cache, progress);
break;
}
}
}
}
public GameDirectoryService(ManagedVersionsService mvs, DirectoryCache dirCache, DataSerializer ds)
{
_mvs = mvs;
_dirCache = dirCache;
_ds = ds;
}
