public static List <ProxyItem> GetProxies() { List <ProxyItem> result = new List <ProxyItem>(); Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + " ** START Proxy Helper **"); try { WebClient client = new WebClient(); Stream stream = client.OpenRead("https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list.txt"); StreamReader reader = new StreamReader(stream); string line = ""; while ((line = reader.ReadLine()) != null) { if (!String.IsNullOrWhiteSpace(line) && !line.StartsWith("Proxy") && !line.StartsWith("Mirrors") && !line.StartsWith("IP") && !line.StartsWith("Free")) { string[] tabProxy = line.Split(' '); ProxyItem item = new ProxyItem(); item.Proxy = line; item.CountryCode = tabProxy[1].Substring(0, 2); item.CountryToAvoid = CountriesToAvoid.Contains(item.CountryCode); result.Add(item); } } reader.Close(); } catch (Exception e) { FlightsEngine.Utils.Logger.GenerateError(e, System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); } Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + " ** END Proxy Helper **"); return(result); }
// POST api/values public async Task PostAsync([FromBody] ProxyItem proxy) { if (proxy.TrafficPercentage > 0 && proxy.TrafficPercentage <= 100) { ServicePartitionResolver resolver = ServicePartitionResolver.GetDefault(); ResolvedServicePartition partition = await resolver.ResolveAsync(new Uri("fabric:/ReverseProxy/ReverseProxyService"), new ServicePartitionKey(), new CancellationToken()); var totalEndpoints = partition.Endpoints.Count; var targetEndpointCount = Math.Floor(totalEndpoints * (proxy.TrafficPercentage / 100.0)); foreach (var endpoint in partition.Endpoints) { if (targetEndpointCount > 0) { JObject addresses = JObject.Parse(endpoint.Address); string address = (string)addresses["Endpoints"].First; // Send request to subset of nodes to set A:B test policy HttpContent content = new StringContent(JsonConvert.SerializeObject(proxy)); await client.PostAsync(address + "/proxy", content); targetEndpointCount--; } } } }
public virtual void AddVersion(ItemDefinition itemDefinition, VersionUri baseVersion, CallContext context) { if (DisableSerialization) { return; } Assert.ArgumentNotNull(itemDefinition, "itemDefinition"); var sourceItem = GetSourceFromIdIfIncluded(itemDefinition); if (sourceItem == null) { return; // not an included item } // we make a clone of the item so that we can insert a new version on it var versionAddProxy = new ProxyItem(sourceItem); // determine what the next version number should be in the current language // (highest number currently present + 1) var newVersionNumber = 1 + versionAddProxy.Versions .Where(v => v.Language.Equals(baseVersion.Language.CultureInfo)) .Select(v => v.VersionNumber) .DefaultIfEmpty() .Max(); IItemVersion newVersion; // if the base version is 0 or less, that means add a blank version (per SC DP behavior). If 1 or more we should copy all fields on that version into the new version. if (baseVersion.Version.Number > 0) { newVersion = versionAddProxy.Versions.FirstOrDefault(v => v.Language.Equals(baseVersion.Language.CultureInfo) && v.VersionNumber.Equals(baseVersion.Version.Number)); // the new version may not exist if we are using language fallback and adding a new version. If that's the case we should create a blank version, as that's what Sitecore does. if (newVersion != null) { newVersion = new ProxyItemVersion(newVersion) { VersionNumber = newVersionNumber } } ; // creating a new proxyversion essentially clones the existing version else { newVersion = new ProxyItemVersion(baseVersion.Language.CultureInfo, newVersionNumber); } } else { newVersion = new ProxyItemVersion(baseVersion.Language.CultureInfo, newVersionNumber); } // inject the new version we created into the proxy item var newVersions = versionAddProxy.Versions.Concat(new[] { newVersion }).ToArray(); versionAddProxy.Versions = newVersions; // flush to serialization data store SerializeItemIfIncluded(versionAddProxy, "Version Added"); }
public void Deserialize_IgnoresField_ExcludedWithFieldFilter() { var ignoredFieldId = ID.NewID; var fieldFilter = Substitute.For <IFieldFilter>(); fieldFilter.Includes(Arg.Any <Guid>()).Returns(true); fieldFilter.Includes(ignoredFieldId.Guid).Returns(false); var deserializer = new DefaultDeserializer(Substitute.For <IDefaultDeserializerLogger>(), fieldFilter); deserializer.ParentDataStore = new SitecoreDataStore(deserializer); using (var db = new Db()) { var itemId = ID.NewID; db.Add(new DbItem("Test Item", itemId) { { ignoredFieldId, "Test Value" } }); var itemData = new ProxyItem(new ItemData(db.GetItem(itemId))); var fields = new List <IItemFieldValue>(); fields.Add(new FakeFieldValue("Changed Ignored Value", fieldId: ignoredFieldId.Guid)); ((ProxyItemVersion)itemData.Versions.First()).Fields = fields; deserializer.Deserialize(itemData); var fromDb = db.GetItem(itemId); Assert.Equal(fromDb[ignoredFieldId], "Test Value"); } }
private IList <ProxyItem> GetProxyList(string proxyFile) { IList <ProxyItem> proxyList = new List <ProxyItem>(); if (string.IsNullOrEmpty(proxyFile) || !File.Exists(proxyFile)) { return(proxyList); } using (StreamReader sr = new StreamReader(proxyFile)) { string ip = string.Empty; int port = 0; int index = 1; while (!sr.EndOfStream) { string line = sr.ReadLine(); if (TextToItemHelper.GetProxy(line, ref ip, ref port)) { ProxyItem item = new ProxyItem(ip, port) { ID = index++ }; proxyList.Add(item); } } } return(proxyList); }
public void ChangeTemplate(ItemDefinition itemDefinition, TemplateChangeList changeList, CallContext context) { if (DisableSerialization) { return; } Assert.ArgumentNotNull(itemDefinition, nameof(itemDefinition)); Assert.ArgumentNotNull(changeList, nameof(changeList)); var sourceItem = GetSourceFromId(itemDefinition.ID); var existingSerializedItem = _targetDataStore.GetByPathAndId(sourceItem.Path, sourceItem.Id, sourceItem.DatabaseName); if (existingSerializedItem == null) { return; } var newItem = new ProxyItem(sourceItem); // note: sourceItem gets dumped. Because it has field changes made to it. newItem.TemplateId = changeList.Target.ID.Guid; _targetDataStore.Save(newItem); _logger.SavedItem(_targetDataStore.FriendlyName, sourceItem, "TemplateChanged"); }
public virtual void CopyItem(ItemDefinition source, ItemDefinition destination, string copyName, ID copyId, CallContext context) { if (DisableSerialization) { return; } // copying is easy - all we have to do is serialize the item using the copyID and new target destination as the parent. // Copied children will all result in multiple calls to CopyItem so we don't even need to worry about them. var existingItem = GetSourceFromId(source.ID, true); Assert.IsNotNull(existingItem, "Existing item to copy was not in the database!"); var destinationItem = GetSourceFromId(destination.ID); Assert.IsNotNull(existingItem, "Copy destination was not in the database!"); // wrap the existing item in a proxy so we can mutate it into a copy var copyTargetItem = new ProxyItem(existingItem); copyTargetItem.Path = $"{destinationItem.Path}/{copyName}"; copyTargetItem.Name = copyName; copyTargetItem.Id = copyId.Guid; if (!_predicate.Includes(copyTargetItem).IsIncluded) { return; // destination parent is not in a path that we are serializing, so skip out } _targetDataStore.Save(copyTargetItem); _logger.CopiedItem(_targetDataStore.FriendlyName, existingItem, copyTargetItem); }
private async Task <AngleSharp.Dom.IDocument> _getDocument(Uri uri) { var success = false; var content = string.Empty; var tryCount = 1; while (!success) { try { Status = string.Format("Найдено ресурсов: {2} Страница: {1} Попытка : {0}", tryCount, uri, _sitesFound); using (var web = new WebClient()) { ProxyItem proxy = _proxy.GetItem(); if (proxy != null && Settings.Default.UseProxy) { web.Proxy = new WebProxy(proxy.ip, proxy.port); } web.Credentials = CredentialCache.DefaultNetworkCredentials; web.Encoding = System.Text.Encoding.UTF8; web.Headers.Add("User-Agent", _userAgent.GetItem()); try { content = web.DownloadString(uri); if (content.Contains("yandex.ru/captcha")) { throw new CaptchaRequestException(); } success = true; } catch (WebException ex) { if (ex.Status == WebExceptionStatus.ProtocolError && ex.Response != null) { var resp = (HttpWebResponse)ex.Response; if (resp.StatusCode == HttpStatusCode.NotFound) // HTTP 404 { if (uri.ToString().IsMailRu()) { throw new EndCatalogExeptionException(); } } } throw; } } } catch (EndCatalogExeptionException) { throw; } catch (Exception ex) { tryCount++; Logger.Error(ex); } } return(await BrowsingContext.New().OpenAsync(r => r.Address(uri).Content(content))); }
public void ShouldReturnExpectedValue() { var fake = new ProxyItem { Path = "/sitecore/content/test item", DatabaseName = "master" }; Assert.Equal("master:/sitecore/content/test item (" + fake.Id + ")", fake.GetDisplayIdentifier()); }
public static ScrappingExecutionResult SearchViaScrapping(ScrappingSearch scrappingSearch, int SearchTripProviderId) { ScrappingExecutionResult result = new ScrappingExecutionResult(); try { int nbMaxAttempts = 50; bool continueProcess = true; int attemtNumber = 0; while (continueProcess) { attemtNumber = attemtNumber + 1; Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + " ** START SearchViaScrapping ** : " + attemtNumber); result = Run(scrappingSearch, SearchTripProviderId); if (scrappingSearch.ProxiesList == null || scrappingSearch.ProxiesList.Count == 0 || (!result.Success && (attemtNumber == 10 || attemtNumber == 20 || attemtNumber == 30))) { scrappingSearch.ProxiesList = ProxyHelper.GetProxies(); } else { ProxyItem proxyItemToModify = scrappingSearch.ProxiesList.Find(p => p.Proxy == scrappingSearch.Proxy); if (proxyItemToModify != null) { scrappingSearch.ProxiesList.Find(p => p.Proxy == scrappingSearch.Proxy).UseNumber = proxyItemToModify.UseNumber + 1; if (!result.Success) { scrappingSearch.ProxiesList.Find(p => p.Proxy == scrappingSearch.Proxy).Failure = proxyItemToModify.Failure + 1; } } } if (!result.Success) { scrappingSearch.Proxy = ProxyHelper.GetBestProxy(scrappingSearch.ProxiesList); scrappingSearch.NewProxy = true; } continueProcess = !result.Success && attemtNumber < nbMaxAttempts; } result.LastProxy = scrappingSearch.Proxy; result.AttemptsNumber = attemtNumber; result.ProxiesList = scrappingSearch.ProxiesList; } catch (Exception e) { result.Success = false; result.Error = e.ToString(); FlightsEngine.Utils.Logger.GenerateError(e, System.Reflection.MethodBase.GetCurrentMethod().DeclaringType, "Provider = " + scrappingSearch.Provider + " and Proxy = " + scrappingSearch.Proxy + " and url = " + scrappingSearch.Url); } finally { Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + " ** END SearchViaScrapping **"); } return(result); }
public void LoadTree_SkipsRootWhenExcluded() { var serializedRootItem = new ProxyItem(); var predicate = CreateExclusiveTestPredicate(); var logger = Substitute.For <ISerializationLoaderLogger>(); var loader = CreateTestLoader(predicate: predicate, logger: logger); TestLoadTree(loader, serializedRootItem); logger.Received().SkippedItemPresentInSerializationProvider(serializedRootItem, Arg.Any <string>(), Arg.Any <string>(), Arg.Any <string>()); }
public void ShouldPreserveBaseItemSharedFields(Item item) { var proxy = new ProxyItem(new ItemData(item)); proxy.SharedFields = new[] { new ProxyFieldValue(Guid.Empty, "hello") }; var changes = new ItemChanges(item); var sut = new ItemChangeApplyingItemData(proxy, changes); sut.SharedFields.First().Value.Should().Be("hello"); }
/// <summary> /// perform insert/update proxy settings into database /// </summary> /// <param name="proxyItem"></param> public void InsertProxy(ProxyItem proxyItem) { if (proxyItem.ServiceId > 0) { Appleseed.Framework.Data.DBHelper.ExeSQL("Update rb_Proxy set ServiceTitle ='" + proxyItem.ServiceTitle + "',ServiceUrl='" + proxyItem.ServiceUrl + "',ForwardHeaders='" + proxyItem.ForwardHeaders + "',EnabledContentAccess='" + proxyItem.EnabledContentAccess + "',ContentAccessRoles='" + proxyItem.ContentAccessRoles + "' where ServiceId=" + proxyItem.ServiceId + ""); } else { Appleseed.Framework.Data.DBHelper.ExeSQL("Insert into rb_Proxy values ('" + proxyItem.ServiceTitle + "','" + proxyItem.ServiceUrl + "','" + proxyItem.ForwardHeaders + "','" + proxyItem.EnabledContentAccess + "','" + proxyItem.ContentAccessRoles + "')"); } }
public static async Task <bool> TestAsync(ProxyItem proxy, string testUrl) { if (proxy == null || string.IsNullOrWhiteSpace(testUrl)) { return(false); } var client = new Client(); client.Proxy = proxy; client.TimeOut = 20 * 1000; var html = await client.GetAsync(testUrl); return(html != null); }
public void LoadTree_LoadsRootWhenIncluded() { var serializedRootItem = new ProxyItem(); var evaluator = Substitute.For <IEvaluator>(); var sourceData = Substitute.For <ISourceDataStore>(); sourceData.GetByPathAndId(serializedRootItem.Path, serializedRootItem.Id, serializedRootItem.DatabaseName).Returns((IItemData)null); var loader = CreateTestLoader(evaluator: evaluator, sourceDataStore: sourceData); TestLoadTree(loader, serializedRootItem); evaluator.Received().EvaluateNewSerializedItem(serializedRootItem); }
void ApplyProxy(WebRequest request, ProxyItem proxy) { if (proxy == null || string.IsNullOrEmpty(proxy.Host)) { return; } request.Proxy = new WebProxy(proxy.Host, proxy.Port); if (!string.IsNullOrEmpty(proxy.UserName)) { var p = request.Proxy as WebProxy; p.UseDefaultCredentials = false; p.Credentials = new NetworkCredential(proxy.UserName, proxy.Password); } }
public override bool SupportsItem(Item item) { item = ProxyItem.Unwrap(item); if (!SupportedItemTypes.Any(type => type.IsInstanceOfType(item))) { return(false); } try { return(Act.SupportsItem(item)); } catch (Exception e) { SafeElement.LogSafeError(Act, e, "SupportsItem"); } return(false); }
public override bool SupportsModifierItemForItems(IEnumerable <Item> items, Item modItem) { items = ProxyItem.Unwrap(items); modItem = ProxyItem.Unwrap(modItem); if (!SupportedModifierItemTypes.Any(type => type.IsInstanceOfType(modItem))) { return(false); } try { return(Act.SupportsModifierItemForItems(items, modItem)); } catch (Exception e) { SafeElement.LogSafeError(Act, e, "SupportsModifierItemForItems"); } return(false); }
public void ShouldExcludeChildrenExcludedByPredicate() { var child = new ProxyItem(); var parent = new ProxyItem(); parent.SetProxyChildren(new[] { child }); var predicate = Substitute.For <IPredicate>(); predicate.Includes(child).Returns(new PredicateResult(false)); Assert.NotEmpty(parent.GetChildren()); var filtered = new PredicateFilteredItemData(parent, predicate); Assert.Empty(filtered.GetChildren()); }
public void ShouldPreserveBaseItemVersionedFields(Item item) { var proxy = new ProxyItem(new ItemData(item)); proxy.Versions = new[] { new ProxyItemVersion(new CultureInfo("en"), 1) { Fields = new[] { new ProxyFieldValue(Guid.Empty, "hello") } } }; var changes = new ItemChanges(item); var sut = new ItemChangeApplyingItemData(proxy, changes); sut.Versions.First().Fields.First().Value.Should().Be("hello"); }
public void LoadTree_LoadsChildOfRootWhenIncluded() { var dataStore = Substitute.For <ITargetDataStore>(); var root = new ProxyItem(); var child = new ProxyItem { ParentId = root.Id }; dataStore.GetChildren(root).Returns(new[] { child }); var evaluator = Substitute.For <IEvaluator>(); var loader = CreateTestLoader(evaluator: evaluator, targetDataStore: dataStore); TestLoadTree(loader, root); evaluator.Received().EvaluateUpdate(Arg.Any <IItemData>(), child); }
public void LoadTree_SkipsChildOfRootWhenExcluded() { var dataStore = Substitute.For <ITargetDataStore>(); var root = new ProxyItem(); var child = new ProxyItem { ParentId = root.Id }; dataStore.GetChildren(root).Returns(new[] { child }); var predicate = CreateExclusiveTestPredicate(new[] { root }); var logger = Substitute.For <ISerializationLoaderLogger>(); var loader = CreateTestLoader(predicate: predicate, logger: logger, targetDataStore: dataStore); TestLoadTree(loader, root); logger.Received().SkippedItemPresentInSerializationProvider(Arg.Is <IItemData>(data => data.Id == child.Id), Arg.Any <string>(), Arg.Any <string>(), Arg.Any <string>()); }
private void RunItemChangeTest(Action <ProxyItem> setup, Action <Item> assert) { using (var db = new Db()) { var deserializer = CreateTestDeserializer(db); var itemId = AddExistingItem(db); var itemData = new ProxyItem(new ItemData(db.GetItem(itemId))); setup(itemData); deserializer.Deserialize(itemData); var fromDb = db.GetItem(itemId); Assert.NotNull(fromDb); assert(fromDb); } }
public static PythonExecutionResult SearchViaScrapping(AirlineSearch filter, ScrappingSearch scrappingSearch) { PythonExecutionResult result = new PythonExecutionResult(); try { int nbMaxAttempts = 3; int nbAttempts = 0; bool continueProcess = true; while (continueProcess) { nbAttempts = nbAttempts + 1; Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + " ** START SearchViaScrapping ** : " + nbAttempts); result = Run(filter, scrappingSearch); ProxyItem proxyItemToModify = scrappingSearch.ProxiesList.Find(p => p.Proxy == scrappingSearch.Proxy); scrappingSearch.ProxiesList.Find(p => p.Proxy == scrappingSearch.Proxy).UseNumber = proxyItemToModify.UseNumber + 1; if (!result.Success) { scrappingSearch.ProxiesList.Find(p => p.Proxy == scrappingSearch.Proxy).Failure = proxyItemToModify.Failure + 1; scrappingSearch.Proxy = ProxyHelper.GetBestProxy(scrappingSearch.ProxiesList); } continueProcess = !result.Success && nbAttempts < nbMaxAttempts; } result.ProxiesList = scrappingSearch.ProxiesList; } catch (Exception e) { result.Success = false; result.Error = e.ToString(); } finally { Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + " ** END SearchViaScrapping **"); } return(result); }
public override IEnumerable <Item> ChildrenOfItem(Item item) { IEnumerable <Item> children = null; item = ProxyItem.Unwrap(item); if (!SupportedItemTypes.Any(type => type.IsInstanceOfType(item))) { return(Enumerable.Empty <Item> ()); } try { // Strictly evaluate the IEnumerable before we leave the try block. children = ItemSource.ChildrenOfItem(item).ToArray(); } catch (Exception e) { children = null; SafeElement.LogSafeError(ItemSource, e, "ChildrenOfItem"); } finally { children = children ?? Enumerable.Empty <Item> (); } return(children); }
public override IEnumerable <Item> Perform(IEnumerable <Item> items, IEnumerable <Item> modItems) { IEnumerable <Item> results = null; items = ProxyItem.Unwrap(items); modItems = ProxyItem.Unwrap(modItems); try { // Strictly evaluate the IEnumerable before we leave the try block. results = Act.Perform(items, modItems); if (results != null) { results = results.ToArray(); } } catch (Exception e) { results = null; SafeElement.LogSafeError(Act, e, "Perform"); } finally { results = results ?? Enumerable.Empty <Item> (); } return(results); }
protected virtual void WriteItem(IItemData item, string path) { Assert.ArgumentNotNull(item, "item"); Assert.ArgumentNotNullOrEmpty(path, "path"); var proxiedItem = new ProxyItem(item) { SerializedItemId = path }; lock (FileUtil.GetFileLock(path)) { try { _treeWatcher.PushKnownUpdate(path, TreeWatcher.TreeWatcherChangeType.ChangeOrAdd); var directory = Path.GetDirectoryName(path); if (directory != null && !Directory.Exists(directory)) { Directory.CreateDirectory(directory); } using (var writer = File.Open(path, FileMode.Create, FileAccess.Write, FileShare.None)) { _formatter.WriteSerializedItem(proxiedItem, writer); } } catch (Exception exception) { if (File.Exists(path)) { File.Delete(path); } throw new SfsWriteException("Error while writing SFS item " + path, exception); } } AddToMetadataCache(item, path); _dataCache.AddOrUpdate(path, proxiedItem); }
public virtual void CreateItem(ItemDefinition newItem, ID templateId, ItemDefinition parent, CallContext context) { if (DisableSerialization) { return; } Assert.ArgumentNotNull(newItem, "itemDefinition"); // get the item's parent. We need to know the path to be able to serialize it, and the ItemDefinition doesn't have it. var parentItem = Database.GetItem(parent.ID); Assert.IsNotNull(parentItem, "New item parent {0} did not exist!", parent.ID); // create a new skeleton item based on the ItemDefinition; at this point the item has no fields, data, etc var newItemProxy = new ProxyItem(newItem.Name, newItem.ID.Guid, parent.ID.Guid, newItem.TemplateID.Guid, parentItem.Paths.Path + "/" + newItem.Name, Database.Name); newItemProxy.BranchId = newItem.BranchId.Guid; // serialize the skeleton if the predicate includes it SerializeItemIfIncluded(newItemProxy, "Created"); }
public override IEnumerable <Item> DynamicModifierItemsForItem(Item item) { IEnumerable <Item> modItems = null; item = ProxyItem.Unwrap(item); // This is a duplicate check, so we may want to remove it. if (!SupportedItemTypes.Any(type => type.IsInstanceOfType(item))) { return(Enumerable.Empty <Item> ()); } try { // Strictly evaluate the IEnumerable before we leave the try block. modItems = Act.DynamicModifierItemsForItem(item).ToArray(); } catch (Exception e) { modItems = null; SafeElement.LogSafeError(Act, e, "DynamicModifierItemsForItem"); } finally { modItems = modItems ?? Enumerable.Empty <Item> (); } return(modItems); }
public virtual void RemoveVersion(ItemDefinition itemDefinition, VersionUri version, CallContext context) { if (DisableSerialization) { return; } Assert.ArgumentNotNull(itemDefinition, "itemDefinition"); var sourceItem = GetSourceFromIdIfIncluded(itemDefinition); if (sourceItem == null) { return; // predicate excluded item } // create a clone of the item to remove the version from var versionRemovingProxy = new ProxyItem(sourceItem); // exclude the removed version versionRemovingProxy.Versions = versionRemovingProxy.Versions.Where(v => !v.Language.Equals(version.Language.CultureInfo) || !v.VersionNumber.Equals(version.Version.Number)); SerializeItemIfIncluded(versionRemovingProxy, "Version Removed"); }