public Task DeleteGrain(GrainId grain) { SiloAddress silo = CalculateTargetSilo(grain); if (silo == null) { // We don't know about any other silos, and we're stopping, so throw throw new InvalidOperationException("Grain directory is stopping"); } if (log.IsVerbose) { log.Verbose("Silo {0} tries to lookup for {1}-->{2} ({3}-->{4})", MyAddress, grain, silo, grain.GetUniformHashCode(), silo.GetConsistentHashCode()); } if (silo.Equals(MyAddress)) { // remove from our partition DirectoryPartition.RemoveGrain(grain); return(TaskDone.Done); } // remove from the cache DirectoryCache.Remove(grain); // send request to the owner return(GetDirectoryReference(silo).DeleteGrain(grain, NUM_RETRIES)); }
/// Adjust local cache following the removal of a silo by droping: /// 1) entries that point to activations located on the removed silo /// 2) entries for grains that are now owned by this silo (me) /// 3) entries for grains that were owned by this removed silo - we currently do NOT do that. /// If we did 3, we need to do that BEFORE we change the membershipRingList (based on old Membership). /// We don't do that since first cache refresh handles that. /// Second, since Membership events are not guaranteed to be ordered, we may remove a cache entry that does not really point to a failed silo. /// To do that properly, we need to store for each cache entry who was the directory owner that registered this activation (the original partition owner). protected void AdjustLocalCache(SiloAddress removedSilo) { // remove all records of activations located on the removed silo foreach (Tuple <GrainId, List <Tuple <SiloAddress, ActivationId> >, int> tuple in DirectoryCache.KeyValues) { // 2) remove entries owned by me (they should be retrieved from my directory partition) if (MyAddress.Equals(CalculateTargetSilo(tuple.Item1))) { DirectoryCache.Remove(tuple.Item1); } // 1) remove entries that point to activations located on the removed silo if (tuple.Item2.RemoveAll(tuple2 => tuple2.Item1.Equals(removedSilo)) <= 0) { continue; } if (tuple.Item2.Count > 0) { DirectoryCache.AddOrUpdate(tuple.Item1, tuple.Item2, tuple.Item3); } else { DirectoryCache.Remove(tuple.Item1); } } }
public async Task DeleteGrainAsync(GrainId grainId, int hopCount) { // see if the owner is somewhere else (returns null if we are owner) var forwardAddress = this.CheckIfShouldForward(grainId, hopCount, "DeleteGrainAsync"); // on all silos other than first, we insert a retry delay and recheck owner before forwarding if (hopCount > 0 && forwardAddress != null) { await Task.Delay(RETRY_DELAY); forwardAddress = this.CheckIfShouldForward(grainId, hopCount, "DeleteGrainAsync"); this.log.LogWarning($"DeleteGrainAsync - It seems we are not the owner of grain {grainId}, trying to forward it to {forwardAddress} (hopCount={hopCount})"); } if (forwardAddress == null) { // we are the owner DirectoryPartition.RemoveGrain(grainId); } else { // otherwise, notify the owner DirectoryCache.Remove(grainId); await GetDirectoryReference(forwardAddress).DeleteGrainAsync(grainId, hopCount + 1); } }
private static ICache GetQueryCache() { if (EnableQueryCache == false) { return(new NullCache()); } var cacheDir = QueryCacheDirectory; if (String.IsNullOrEmpty(cacheDir) && (!String.IsNullOrEmpty(StoreLocation))) { cacheDir = Path.Combine(StoreLocation, "_bscache"); } if (!String.IsNullOrEmpty(cacheDir)) { if (!Directory.Exists(cacheDir)) { Directory.CreateDirectory(cacheDir); } ICache directoryCache = new DirectoryCache(cacheDir, MegabytesToBytes * QueryCacheDiskSpace, new LruCacheEvictionPolicy()), memoryCache = new MemoryCache(MegabytesToBytes * QueryCacheMemory, new LruCacheEvictionPolicy()); return(new TwoLevelCache(memoryCache, directoryCache)); } else { ICache memoryCache = new MemoryCache(MegabytesToBytes * QueryCacheMemory, new LruCacheEvictionPolicy()); return(memoryCache); } }
public static InstanceCache LoadSettings() { InstanceCache cache = new InstanceCache(); DataDeserializer dds = new DataContractXMLLoader(); cache.AddInstance(dds); cache.AddInstance((DataSerializer)dds); DirectoryCache dirCache = new DirectoryCache(); if (File.Exists(Settings.Default.DirectoryCacheFile)) { dirCache = dds.Deserialize <DirectoryCache>(Settings.Default.DirectoryCacheFile); } dirCache.ContainerPath = Settings.Default.ContainerDirectory; cache.AddInstance(dirCache); ManagedVersionsService mvs = new ManagedVersionsService(); if (File.Exists(Settings.Default.ManagedVersionsFile)) { mvs.Load(dds); } mvs.Serializer = (DataSerializer)dds; cache.AddInstance(mvs); return(cache); }
private AzureBlockStore(AzureBlockStoreConfiguration configuration) { #if BLOCKSTORECACHE _cache = new BlockStoreCache((long)configuration.MemoryCacheInMB * 1024 * 1024, configuration.LocalStorageKey); #else ICache memoryCache = new MemoryCache((long)configuration.MemoryCacheInMB * 1024 * 1024, new LruCacheEvictionPolicy()); if (!String.IsNullOrEmpty(configuration.LocalStorageKey)) { var localDiskResource = RoleEnvironment.GetLocalResource(configuration.LocalStorageKey); ICache diskCache = new DirectoryCache(localDiskResource.RootPath, ((long)localDiskResource.MaximumSizeInMegabytes - 1) * 1024 * 1024, new LruCacheEvictionPolicy()); _cache = new TwoLevelCache(memoryCache, diskCache); } else { _cache = memoryCache; } #endif _highestPageOffsetByPath = new Dictionary <string, long>(); _storageAccount = CloudStorageAccount.Parse(configuration.ConnectionString); _commitList = new ConcurrentQueue <BlockInfo>(); _commitThread = new Thread(RunCommitThread); _commitThread.Start(); _disconnected = configuration.Disconnected; }
public List <ActivationAddress> GetLocalCacheData(GrainId grain) { IReadOnlyList <Tuple <SiloAddress, ActivationId> > cached; return(DirectoryCache.LookUp(grain, out cached) ? cached.Select(elem => ActivationAddress.GetAddress(elem.Item1, grain, elem.Item2)).Where(addr => IsValidSilo(addr.Silo)).ToList() : null); }
public GrainAddress GetLocalCacheData(GrainId grain) { if (DirectoryCache.LookUp(grain, out var cache) && IsValidSilo(cache.SiloAddress)) { return(cache); } return(null); }
/// <summary> /// Task for inaccessible path /// </summary> /// <param name="path"></param> /// <param name="rootCache"></param> private static void DiskUsageTask(string path, DirectoryCache rootCache) { var currentCacheNode = rootCache.GetSubDirectoryCache(path); if (currentCacheNode == null || currentCacheNode.Accessible) { return; } LoadDirectoryInfoByCmd(currentCacheNode); }
public async Task <AddressAndTag> RegisterAsync(GrainAddress address, int hopCount) { var counterStatistic = hopCount > 0 ? this.RegistrationsSingleActRemoteReceived : this.registrationsSingleActIssued; counterStatistic.Increment(); // see if the owner is somewhere else (returns null if we are owner) var forwardAddress = this.CheckIfShouldForward(address.GrainId, hopCount, "RegisterAsync"); // on all silos other than first, we insert a retry delay and recheck owner before forwarding if (hopCount > 0 && forwardAddress != null) { await Task.Delay(RETRY_DELAY); forwardAddress = this.CheckIfShouldForward(address.GrainId, hopCount, "RegisterAsync"); if (forwardAddress is object) { int hash = unchecked ((int)address.GrainId.GetUniformHashCode()); this.log.LogWarning($"RegisterAsync - It seems we are not the owner of activation {address} (hash: {hash:X}), trying to forward it to {forwardAddress} (hopCount={hopCount})"); } } if (forwardAddress == null) { RegistrationsSingleActLocal.Increment(); var result = DirectoryPartition.AddSingleActivation(address); return(result); } else { RegistrationsSingleActRemoteSent.Increment(); // otherwise, notify the owner AddressAndTag result = await GetDirectoryReference(forwardAddress).RegisterAsync(address, hopCount + 1); // Caching optimization: // cache the result of a successfull RegisterSingleActivation call, only if it is not a duplicate activation. // this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup! if (result.Address == null) { return(result); } if (!address.Equals(result.Address) || !IsValidSilo(address.SiloAddress)) { return(result); } // update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup. DirectoryCache.AddOrUpdate(address, result.VersionTag); return(result); } }
/// <summary> /// This is the function that actually does the searching. It creates threads and searches and then /// waits for all the results to be sent. /// </summary> public void internalSearch() { String searchDirectory = mSearchParams.SearchDirectory; if (mRegex != null) { List <String> fileList = DirectoryCache.get().getFileList(); if (fileList != null) { Stats.TotalFilesToSearch = fileList.Count; Logger.get().AddInfoFormat("{0} files to search", Stats.TotalFilesToSearch); try { // Set up the parallel lambda expression. The parameters were trial and error trying to get the fastest execution fileList.AsParallel().WithExecutionMode(ParallelExecutionMode.ForceParallelism).WithDegreeOfParallelism(63) .WithCancellation(Stats.CancellationTokenSource.Token) .WithMergeOptions(ParallelMergeOptions.FullyBuffered).ForAll(file => { // Check for cancellation, this speeds up the cancel after the button is clicked. if (!Stats.CancellationTokenSource.IsCancellationRequested) { if (mSearchParams.FileNameSearch) { if (mRegex.IsMatch(file)) { SearchResultList list = new SearchResultList(PathExt.GetRelativePath(searchDirectory, file)); saveResults(list); } } else { searchFile(file); } } }); } catch (OperationCanceledException) { // Cancelling is not an error, why do they throw an exception? } catch (Exception ex) { Logger.get().AddError(ex.Message); } } else { SearchResultList results = new SearchResultList(DirectoryCache.get().ErrorMessage); mPendingResults.Add(results); } } waitForResultsToBeSent(); }
public void InvalidateCacheEntry(ActivationAddress activationAddress) { var grainId = activationAddress.Grain; var activationId = activationAddress.Activation; // look up grainId activations if (DirectoryCache.LookUp(grainId, out var entry, out _) && activationId.Equals(entry.Activation)) { DirectoryCache.Remove(grainId); } }
public void InvalidateCacheEntry(ActivationAddress activationAddress) { int version; IReadOnlyList <Tuple <SiloAddress, ActivationId> > list; var grainId = activationAddress.Grain; var activationId = activationAddress.Activation; // look up grain activations if (DirectoryCache.LookUp(grainId, out list, out version)) { RemoveActivations(DirectoryCache, grainId, list, version, t => t.Item2.Equals(activationId)); } }
/// Adjust local cache following the removal of a silo by droping: /// 1) entries that point to activations located on the removed silo /// 2) entries for grains that are now owned by this silo (me) /// 3) entries for grains that were owned by this removed silo - we currently do NOT do that. /// If we did 3, we need to do that BEFORE we change the membershipRingList (based on old Membership). /// We don't do that since first cache refresh handles that. /// Second, since Membership events are not guaranteed to be ordered, we may remove a cache entry that does not really point to a failed silo. /// To do that properly, we need to store for each cache entry who was the directory owner that registered this activation (the original partition owner). protected void AdjustLocalCache(SiloAddress removedSilo) { // remove all records of activations located on the removed silo foreach (Tuple <GrainId, IReadOnlyList <Tuple <SiloAddress, ActivationId> >, int> tuple in DirectoryCache.KeyValues) { // 2) remove entries now owned by me (they should be retrieved from my directory partition) if (MyAddress.Equals(CalculateTargetSilo(tuple.Item1))) { DirectoryCache.Remove(tuple.Item1); } // 1) remove entries that point to activations located on the removed silo RemoveActivations(DirectoryCache, tuple.Item1, tuple.Item2, tuple.Item3, t => t.Item1.Equals(removedSilo)); } }
public async Task RegisterAsync(ActivationAddress address) { registrationsIssued.Increment(); SiloAddress owner = CalculateTargetSilo(address.Grain); if (owner == null) { // We don't know about any other silos, and we're stopping, so throw throw new InvalidOperationException("Grain directory is stopping"); } if (owner.Equals(MyAddress)) { RegistrationsLocal.Increment(); // if I am the owner, store the new activation locally DirectoryPartition.AddActivation(address.Grain, address.Activation, address.Silo); } else { RegistrationsRemoteSent.Increment(); // otherwise, notify the owner int eTag = await GetDirectoryReference(owner).Register(address, NUM_RETRIES); if (IsValidSilo(address.Silo)) { // Caching optimization: // cache the result of a successfull RegisterActivation call, only if it is not a duplicate activation. // this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup! IReadOnlyList <Tuple <SiloAddress, ActivationId> > cached; if (!DirectoryCache.LookUp(address.Grain, out cached)) { cached = new List <Tuple <SiloAddress, ActivationId> >(1) { Tuple.Create(address.Silo, address.Activation) }; } else { var newcached = new List <Tuple <SiloAddress, ActivationId> >(cached.Count + 1); newcached.AddRange(cached); newcached.Add(Tuple.Create(address.Silo, address.Activation)); cached = newcached; } // update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup. DirectoryCache.AddOrUpdate(address.Grain, cached, eTag); } } }
/// <summary> /// Registers a new activation, in single activation mode, with the directory service. /// If there is already an activation registered for this grain, then the new activation will /// not be registered and the address of the existing activation will be returned. /// Otherwise, the passed-in address will be returned. /// <para>This method must be called from a scheduler thread.</para> /// </summary> /// <param name="address">The address of the potential new activation.</param> /// <returns>The address registered for the grain's single activation.</returns> public async Task <ActivationAddress> RegisterSingleActivationAsync(ActivationAddress address) { registrationsSingleActIssued.Increment(); SiloAddress owner = CalculateTargetSilo(address.Grain); if (owner == null) { // We don't know about any other silos, and we're stopping, so throw throw new InvalidOperationException("Grain directory is stopping"); } if (owner.Equals(MyAddress)) { RegistrationsSingleActLocal.Increment(); // if I am the owner, store the new activation locally Tuple <ActivationAddress, int> returnedAddress = DirectoryPartition.AddSingleActivation(address.Grain, address.Activation, address.Silo); return(returnedAddress == null ? null : returnedAddress.Item1); } else { RegistrationsSingleActRemoteSent.Increment(); // otherwise, notify the owner Tuple <ActivationAddress, int> returnedAddress = await GetDirectoryReference(owner).RegisterSingleActivation(address, NUM_RETRIES); // Caching optimization: // cache the result of a successfull RegisterSingleActivation call, only if it is not a duplicate activation. // this way next local lookup will find this ActivationAddress in the cache and we will save a full lookup! if (returnedAddress == null || returnedAddress.Item1 == null) { return(null); } if (!address.Equals(returnedAddress.Item1) || !IsValidSilo(address.Silo)) { return(returnedAddress.Item1); } var cached = new List <Tuple <SiloAddress, ActivationId> >(1) { Tuple.Create(address.Silo, address.Activation) }; // update the cache so next local lookup will find this ActivationAddress in the cache and we will save full lookup. DirectoryCache.AddOrUpdate(address.Grain, cached, returnedAddress.Item2); return(returnedAddress.Item1); } }
// Note that this implementation stops processing directory change requests (Register, Unregister, etc.) when the Stop event is raised. // This means that there may be a short period during which no silo believes that it is the owner of directory information for a set of // grains (for update purposes), which could cause application requests that require a new activation to be created to time out. // The alternative would be to allow the silo to process requests after it has handed off its partition, in which case those changes // would receive successful responses but would not be reflected in the eventual state of the directory. // It's easy to change this, if we think the trade-off is better the other way. public void Stop() { // This will cause remote write requests to be forwarded to the silo that will become the new owner. // Requests might bounce back and forth for a while as membership stabilizes, but they will either be served by the // new owner of the grain, or will wind up failing. In either case, we avoid requests succeeding at this silo after we've // begun stopping, which could cause them to not get handed off to the new owner. //mark Running as false will exclude myself from CalculateGrainDirectoryPartition(grainId) Running = false; if (maintainer != null) { maintainer.Stop(); } DirectoryPartition.Clear(); DirectoryCache.Clear(); }
public void InvalidateCacheEntry(ActivationAddress activationAddress) { var grainId = activationAddress.Grain; var activationId = activationAddress.Activation; List <Tuple <SiloAddress, ActivationId> > list; if (!DirectoryCache.LookUp(grainId, out list)) { return; } list.RemoveAll(tuple => tuple.Item2.Equals(activationId)); // if list empty, need to remove from cache if (list.Count == 0) { DirectoryCache.Remove(grainId); } }
/// <summary> /// recursive load directory info /// </summary> /// <param name="path"></param> /// <returns></returns> internal static DirectoryCache LoadDirectory(string path) { var directoryCache = new DirectoryCache(); DirectoryInfo directory = new DirectoryInfo(path); directoryCache.LastSeen = DateTime.Now; directoryCache.Uri = directory.FullName; if (!CheckAccess(directory)) { directoryCache.Accessible = false; UnauthorizedAccessCache.GetInstance().InsertMissPath(directory.Parent?.FullName ?? "", path); return(directoryCache); } var fileList = directory.GetFiles() .Where(item => !FileExcludeConfig.GetInstance().TestFileMatch(item.FullName)) .ToList(); var directoryList = directory.GetDirectories() .Where(item => !FileExcludeConfig.GetInstance().TestDirectionMatch(item.FullName)) .ToList(); Parallel.ForEach(fileList, fileInfo => { lock (directoryCache.FileCaches) { directoryCache.FileCaches.Add(new FileCache(fileInfo.FullName, (ulong)fileInfo.Length, DateTime.Now)); } }); Parallel.ForEach(directoryList, directoryInfo => { var dir = LoadDirectory(directoryInfo.FullName); lock (directoryCache.SubDirectoryCaches) { directoryCache.SubDirectoryCaches.Add(dir); } }); return(directoryCache); }
/// <summary> /// /// </summary> /// <param name="targetPath"></param> /// <param name="files"></param> public void UpdateCache(string targetPath, List <string> files) { if (Directory.Exists(targetPath)) { foreach (var directory in DirectoryCaches) { if (directory.Path == targetPath) { directory.Files = new List <string>(files); directory.UpdateTime = Directory.GetLastWriteTime(targetPath); return; } } var cache = new DirectoryCache(); cache.Path = targetPath; cache.Files = new List <string>(files); cache.UpdateTime = Directory.GetLastWriteTime(targetPath); DirectoryCaches.Add(cache); } }
// Note that this implementation stops processing directory change requests (Register, Unregister, etc.) when the Stop event is raised. // This means that there may be a short period during which no silo believes that it is the owner of directory information for a set of // grains (for update purposes), which could cause application requests that require a new activation to be created to time out. // The alternative would be to allow the silo to process requests after it has handed off its partition, in which case those changes // would receive successful responses but would not be reflected in the eventual state of the directory. // It's easy to change this, if we think the trade-off is better the other way. public void Stop(bool doOnStopHandoff) { // This will cause remote write requests to be forwarded to the silo that will become the new owner. // Requests might bounce back and forth for a while as membership stabilizes, but they will either be served by the // new owner of the grain, or will wind up failing. In either case, we avoid requests succeeding at this silo after we've // begun stopping, which could cause them to not get handed off to the new owner. Running = false; if (doOnStopHandoff) { HandoffManager.ProcessSiloStoppingEvent(); } else { MarkStopPreparationCompleted(); } if (maintainer != null) { maintainer.Stop(); } DirectoryCache.Clear(); }
/// Adjust local cache following the removal of a silo by dropping: /// 1) entries that point to activations located on the removed silo /// 2) entries for grains that are now owned by this silo (me) /// 3) entries for grains that were owned by this removed silo - we currently do NOT do that. /// If we did 3, we need to do that BEFORE we change the membershipRingList (based on old Membership). /// We don't do that since first cache refresh handles that. /// Second, since Membership events are not guaranteed to be ordered, we may remove a cache entry that does not really point to a failed silo. /// To do that properly, we need to store for each cache entry who was the directory owner that registered this activation (the original partition owner). protected void AdjustLocalCache(SiloAddress silo, bool dead) { // remove all records of activations located on the removed silo foreach (var tuple in DirectoryCache.KeyValues) { var activationAddress = tuple.ActivationAddress; // 2) remove entries now owned by me (they should be retrieved from my directory partition) if (MyAddress.Equals(CalculateGrainDirectoryPartition(activationAddress.GrainId))) { DirectoryCache.Remove(activationAddress.GrainId); } // 1) remove entries that point to activations located on the removed silo // For dead silos, remove any activation registered to that silo or one of its predecessors. // For new silos, remove any activation registered to one of its predecessors. if (activationAddress.SiloAddress.IsPredecessorOf(silo) || (activationAddress.SiloAddress.Equals(silo) && dead)) { DirectoryCache.Remove(activationAddress.GrainId); } } }
public void TestDirectoryCacheBasicOperations() { string cacheDirPath = Path.Combine(Path.GetTempPath(), "cachetest_" + DateTime.Now.Ticks); ICache cache = new DirectoryCache(cacheDirPath, 2000, new LruCacheEvictionPolicy()); RunCacheTests(cache); ICache reopenedCache = new DirectoryCache(cacheDirPath, 2000, new LruCacheEvictionPolicy()); var byteArray = cache.Lookup("ByteArray"); Assert.IsNotNull(byteArray); Assert.AreEqual(4, byteArray.Length); for (int i = 0; i < 4; i++) { Assert.AreEqual(i, byteArray[i]); } var cachedString = cache.Lookup <string>("String"); Assert.IsNotNull(cachedString); Assert.AreEqual("Hello World", cachedString); Assert.IsFalse(reopenedCache.ContainsKey("Object")); }
/// <summary> /// get directory size info form du.exe/du64.exe /// we will use size not size on disk to keep same with /// C# DirectoryInfo report /// du.exe output example: /// Files: 130552 /// Directories: 45942 /// Size: 16,113,239,255 bytes /// Size on disk: 14,142,668,800 bytes /// </summary> /// <param name="directoryCache"></param> /// <returns></returns> internal static DirectoryCache LoadDirectoryInfoByCmd(DirectoryCache directoryCache) { var output = CmdHelper.ExecDiskUsage(directoryCache.Uri); // check if du.exe return "", means a junction if (output == "") { return(directoryCache); } // make windows "\r\n" to "\n" only output = output.Replace("\r", ""); var sizeStr = output.Split("\n")[2].Trim(); sizeStr = sizeStr.Replace("Size: ", "").Replace("bytes", ""); if (ulong.TryParse(sizeStr, out var size)) { directoryCache.Length = size; } return(directoryCache); }
public override string ToString() { var sb = new StringBuilder(); long localLookupsDelta; long localLookupsCurrent = localLookups.GetCurrentValueAndDelta(out localLookupsDelta); long localLookupsSucceededDelta; long localLookupsSucceededCurrent = localSuccesses.GetCurrentValueAndDelta(out localLookupsSucceededDelta); long fullLookupsDelta; long fullLookupsCurrent = fullLookups.GetCurrentValueAndDelta(out fullLookupsDelta); long directoryPartitionSize = directoryPartitionCount.GetCurrentValue(); sb.AppendLine("Local Grain Directory:"); sb.AppendFormat(" Local partition: {0} entries", directoryPartitionSize).AppendLine(); sb.AppendLine(" Since last call:"); sb.AppendFormat(" Local lookups: {0}", localLookupsDelta).AppendLine(); sb.AppendFormat(" Local found: {0}", localLookupsSucceededDelta).AppendLine(); if (localLookupsCurrent > 0) { sb.AppendFormat(" Hit rate: {0:F1}%", (100.0 * localLookupsSucceededDelta) / localLookupsDelta).AppendLine(); } sb.AppendFormat(" Full lookups: {0}", fullLookupsDelta).AppendLine(); sb.AppendLine(" Since start:"); sb.AppendFormat(" Local lookups: {0}", localLookupsCurrent).AppendLine(); sb.AppendFormat(" Local found: {0}", localLookupsSucceededCurrent).AppendLine(); if (localLookupsCurrent > 0) { sb.AppendFormat(" Hit rate: {0:F1}%", (100.0 * localLookupsSucceededCurrent) / localLookupsCurrent).AppendLine(); } sb.AppendFormat(" Full lookups: {0}", fullLookupsCurrent).AppendLine(); sb.Append(DirectoryCache.ToString()); return(sb.ToString()); }
public void Extract(DirectoryEntity entity, string root, Func <BaseEntity, string> fileToPath, DirectoryCache cache, IProgress <int> progress) { int totalFiles = entity.GetAllFileEntities(true).Count; int processed = 0; Progress <int> sumProgress = new Progress <int>(prog => { int percent = ++processed * 100 / totalFiles; progress?.Report(percent); }); ExtractInner(entity, root, fileToPath, cache, (progress != null) ? sumProgress : null); }
public override void Extract(string root, BaseEntity entity, Func <BaseEntity, string> entityToDir, DirectoryCache cache, IProgress <int> progress) { PackageEntity package = entity as PackageEntity; using (ZipArchive arch = ZipFile.OpenRead(Path.Combine(root, entity.RelativePath))) { foreach (var entry in arch.Entries) { BaseEntity ent = package.GetEntityFromRelativePath(entry.FullName, false); if (ent is null) { throw new InvalidOperationException("Couldn't find entity in structure: " + entry.FullName + " of package: " + entity.RelativePath); } if (ent is FileEntity) { string dir = entityToDir(ent); if (!cache.CacheDirectory(dir)) { entry.ExtractToFile(Path.Combine(dir, ent.Name), false); } progress?.Report(1); } } } }
public async Task <List <ActivationAddress> > FullLookup(GrainId grain) { fullLookups.Increment(); SiloAddress silo = CalculateTargetSilo(grain, false); // No need to check that silo != null since we're passing excludeThisSiloIfStopping = false if (log.IsVerbose) { log.Verbose("Silo {0} fully lookups for {1}-->{2} ({3}-->{4})", MyAddress, grain, silo, grain.GetUniformHashCode(), silo.GetConsistentHashCode()); } // We assyme that getting here means the grain was not found locally (i.e., in TryFullLookup()). // We still check if we own the grain locally to avoid races between the time TryFullLookup() and FullLookup() were called. if (silo.Equals(MyAddress)) { LocalDirectoryLookups.Increment(); var localResult = DirectoryPartition.LookUpGrain(grain); if (localResult == null) { // it can happen that we cannot find the grain in our partition if there were // some recent changes in the membership if (log.IsVerbose2) { log.Verbose2("FullLookup mine {0}=none", grain); } return(new List <ActivationAddress>()); } var a = localResult.Item1.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).Where(addr => IsValidSilo(addr.Silo)).ToList(); if (log.IsVerbose2) { log.Verbose2("FullLookup mine {0}={1}", grain, a.ToStrings()); } LocalDirectorySuccesses.Increment(); return(a); } // Just a optimization. Why sending a message to someone we know is not valid. if (!IsValidSilo(silo)) { throw new OrleansException(String.Format("Current directory at {0} is not stable to perform the lookup for grain {1} (it maps to {2}, which is not a valid silo). Retry later.", MyAddress, grain, silo)); } RemoteLookupsSent.Increment(); Tuple <List <Tuple <SiloAddress, ActivationId> >, int> result = await GetDirectoryReference(silo).LookUp(grain, NUM_RETRIES); // update the cache List <Tuple <SiloAddress, ActivationId> > entries = result.Item1.Where(t => IsValidSilo(t.Item1)).ToList(); List <ActivationAddress> addresses = entries.Select(t => ActivationAddress.GetAddress(t.Item1, grain, t.Item2)).ToList(); if (log.IsVerbose2) { log.Verbose2("FullLookup remote {0}={1}", grain, addresses.ToStrings()); } if (entries.Count > 0) { DirectoryCache.AddOrUpdate(grain, entries, result.Item2); } return(addresses); }
private void ExtractInner(DirectoryEntity entity, string root, Func <BaseEntity, string> fileToPath, DirectoryCache cache, IProgress <int> progress) { foreach (BaseEntity ent in entity.GetAllFileEntities()) { foreach (Extractor ex in _extractors) { if (ex.CanExtract(ent)) { ex.Extract(root, ent, fileToPath, cache, progress); break; } } } }
public GameDirectoryService(ManagedVersionsService mvs, DirectoryCache dirCache, DataSerializer ds) { _mvs = mvs; _dirCache = dirCache; _ds = ds; }