public static HashSet <string> ExtractDistinctStrings(IEnumerable <string> extractFrom, IEnumerable <string> stringsToExclude, bool convertToLowerInVariant, bool extractAlphaNumericCharacters, bool removeSpaces, bool removeDoubleSpaces, int minimumDistinctStringLength, bool includeEmptyString, bool includeNull, bool removeNumericOnlyStrings)
{
if (extractFrom == null)
{
return(null);
}
int tryParse;
HashSet <string> distinctStrings = new HashSet <string>();
foreach (string @string in extractFrom)
{
string string2 = @string.Replace("�", string.Empty);
if (!string.IsNullOrEmpty(string2))
{
string2 = string2.Trim();
if (convertToLowerInVariant)
{
string2 = string2.ToLowerInvariant();
}
if (extractAlphaNumericCharacters)
{
string2 = UserDefinedFunctions.ExtractAlphaNumericCharacters(string2).Value;
}
if (removeDoubleSpaces)
{
while (string2.IndexOf(" ") != -1)
{
string2 = string2.Replace(" ", " ");
}
}
if (string2.Length < minimumDistinctStringLength)
{
continue;
}
}
if (!includeEmptyString)
{
if (string2 == string.Empty)
{
continue;
}
}
if (!includeNull)
{
if (string2 == null)
{
continue;
}
}
if (removeNumericOnlyStrings)
{
if (int.TryParse(string2, out tryParse))
{
continue;
}
}
if (!distinctStrings.Contains(string2))
{
if (stringsToExclude == null || stringsToExclude.Count() == 0)
{
distinctStrings.Add(string2);
}
else
{
if (!stringsToExclude.Contains(string2))
{
distinctStrings.Add(string2);
}
}
}
}
return(distinctStrings);
}
/// <summary>
/// Manages the image.
/// </summary>
/// <param name = "crawlRequest">The crawl request.</param>
/// <param name = "imageID">The image ID.</param>
/// <param name = "absoluteUri">The absolute URI.</param>
/// <param name = "source">The source.</param>
/// <param name = "fullTextIndexType">Full type of the text index.</param>
/// <param name = "extractImageMetaData">if set to <c>true</c> [extract image meta data].</param>
/// <param name = "insertImageMetaData">if set to <c>true</c> [insert image meta data].</param>
/// <param name = "saveImageToDisk">if set to <c>true</c> [save image to disk].</param>
/// <returns></returns>
public override ManagedImage ManageImage(CrawlRequest <TArachnodeDAO> crawlRequest, long imageID, string absoluteUri, byte[] source, string fullTextIndexType, bool extractImageMetaData, bool insertImageMetaData, bool saveImageToDisk)
{
try
{
using (MemoryStream memoryStream = new MemoryStream(source, true))
{
ManagedImage managedImage = new ManagedImage();
managedImage.Image = Image.FromStream(memoryStream);
if (extractImageMetaData)
{
XmlDocument xmlDocument = new XmlDocument();
XmlElement xmlElement;
xmlDocument.AppendChild(xmlDocument.CreateNode(XmlNodeType.XmlDeclaration, "", ""));
xmlDocument.AppendChild(xmlDocument.CreateElement("", "EXIFData", ""));
Dictionary <string, string> dictionary = new Dictionary <string, string>();
foreach (Pair pair in new EXIFExtractor(managedImage.Image, "", ""))
{
dictionary.Add(pair.First.ToString(), pair.Second.ToString());
}
foreach (KeyValuePair <string, string> keyValuePair in dictionary)
{
xmlElement = xmlDocument.CreateElement("", keyValuePair.Key.Replace(" ", "_"), "");
string value = UserDefinedFunctions.ExtractAlphaNumericCharacters(keyValuePair.Value).Value ?? string.Empty;
xmlElement.AppendChild(xmlDocument.CreateTextNode(value));
xmlDocument.ChildNodes.Item(1).AppendChild(xmlElement);
}
managedImage.EXIFData = xmlDocument;
if (insertImageMetaData)
{
_arachnodeDAO.InsertImageMetaData(absoluteUri, imageID, xmlDocument.InnerXml, managedImage.Image.Flags, managedImage.Image.Height, managedImage.Image.HorizontalResolution, managedImage.Image.VerticalResolution, managedImage.Image.Width);
}
}
if (saveImageToDisk)
{
managedImage.DiscoveryPath = _discoveryManager.GetDiscoveryPath(ApplicationSettings.DownloadedImagesDirectory, absoluteUri, fullTextIndexType);
managedImage.Image.Save(managedImage.DiscoveryPath);
}
return(managedImage);
} //ANODET: Parameter is not valid in the exception handler...
}
catch (Exception exception)
{
//ANODET: Images of 7 bytes (Generic GDI Error)...
#if !DEMO
if (crawlRequest != null)
{
_arachnodeDAO.InsertException(crawlRequest.Parent.Uri.AbsoluteUri, absoluteUri, exception, false);
}
else
{
_arachnodeDAO.InsertException(null, absoluteUri, exception, false);
}
#endif
}
return(null);
}
public static bool IsValidUSPhoneNumber(string input, bool verifyArithmetically)
{
if (string.IsNullOrEmpty(input))
{
return(false);
}
try
{
input = input.ToLowerInvariant();
if (input.Contains("x"))
{
input = input.Split('x')[0];
}
if (input.Contains("e"))
{
input = input.Split('e')[0];
}
string alphaNumericCharacters = UserDefinedFunctions.ExtractAlphaNumericCharacters(input).Value;
if (alphaNumericCharacters.Length > 11 || alphaNumericCharacters.Length <= 9)
{
return(false);
}
if (alphaNumericCharacters.Length == 11)
{
if (alphaNumericCharacters[0] != '1')
{
return(false);
}
alphaNumericCharacters = alphaNumericCharacters.Substring(1, 10);
}
foreach (char c in alphaNumericCharacters)
{
if (!char.IsDigit(c))
{
return(false);
}
}
if (!verifyArithmetically)
{
return(true);
}
int areaCode = int.Parse(alphaNumericCharacters.Substring(0, 3));
int firstThree = int.Parse(alphaNumericCharacters.Substring(3, 3));
int lastFour = int.Parse(alphaNumericCharacters.Substring(6, 4));
double phoneNumber = firstThree * 80;
phoneNumber += 1;
phoneNumber *= 250;
phoneNumber += lastFour;
phoneNumber += lastFour;
phoneNumber -= 250;
phoneNumber /= 2;
if (firstThree.ToString() + lastFour.ToString() != ((int)phoneNumber).ToString())
{
return(false);
}
// Multiply by 80
//Add 1
//Multiply by 250
//Add the last 4 digits of your phone number
//Add the last 4 digits of your number again
//Subtract 250
//Divide by 2
return(true);
}
catch
{
}
return(false);
}