internal UrlViewModel(UrlContent urlContent)
{
if (urlContent == null)
throw new ArgumentNullException("urlContent");
m_UrlContent = urlContent;
}
private async Task <bool> SaveToDb(List <string> links)
{
var host = new Uri(links.FirstOrDefault()).Host;
var domain = context.Domain.Where(domain => domain.Name == host).FirstOrDefault();
if (domain == default)
{
context.Domain.Add(new Domain()
{
Name = host
});
context.SaveChanges();
}
domain = context.Domain.Where(domain => domain.Name == host).FirstOrDefault();
foreach (var link in links)
{
var urlContent = new UrlContent()
{
DomainId = domain.Id, Url = link, Content = await PageLinkGetter.GetBody(link)
};
context.UrlContent.Add(urlContent);
}
context.UrlContent.Distinct();
context.SaveChanges();
return(true);
}
public FileInfo GetWithContent(ObjectIdentity objectIdentity, String geoLoc, ContentTransferMode transferMode, String objectID)
{
try
{
// Console.WriteLine("GetWithContent");
//ObjectId objectId = new ObjectId(objectID);
//ObjectIdentity objectIdent = new ObjectIdentity(objectId, objectIdentity.RepositoryName);
//ObjectIdentitySet objectIdentitySet = new ObjectIdentitySet(objectIdent);
//DataPackage dataPackageas = objectService.Get(objectIdentitySet, null);
ContentTransferProfile transferProfile = new ContentTransferProfile();
transferProfile.Geolocation = geoLoc;
transferProfile.TransferMode = transferMode;
DemoServiceContext.SetProfile(transferProfile);
ContentProfile contentProfile = new ContentProfile();
contentProfile.FormatFilter = FormatFilter.ANY;
contentProfile.UrlReturnPolicy = UrlReturnPolicy.PREFER;
OperationOptions operationOptions = new OperationOptions();
operationOptions.ContentProfile = contentProfile;
operationOptions.SetProfile(contentProfile);
ObjectIdentitySet objectIdSet = new ObjectIdentitySet(objectIdentity);
List <ObjectIdentity> objIdList = objectIdSet.Identities;
objIdList.Add(objectIdentity);
DataPackage dataPackage = objectService.Get(objectIdSet, operationOptions);
DataObject dataObject = dataPackage.DataObjects[0];
Content resultContent = dataObject.Contents[0];
String contentClassName = resultContent.GetType().FullName;
//Console.WriteLine("Returned content as type " + contentClassName);
if (contentClassName.Equals("Emc.Documentum.FS.DataModel.Core.Content.UrlContent"))
{
UrlContent urlContent = (UrlContent)resultContent;
// Console.WriteLine("Content ACS URL is: " + (urlContent.Url));
}
if (resultContent.CanGetAsFile())
{
FileInfo fileInfo = resultContent.GetAsFile();
return(fileInfo);
//Console.WriteLine("Got content as file " + fileInfo.FullName);
}
else if (contentClassName.Equals("Emc.Documentum.FS.DataModel.Core.Content.UcfContent"))
{
UcfContent ucfContent = (UcfContent)resultContent;
// Console.WriteLine("Got content as file " + ucfContent.LocalFilePath);
}
return(null);
}
catch (Exception ex)
{
Console.WriteLine(ex.StackTrace);
throw ex;
}
}
internal UrlViewModel(UrlContent urlContent)
{
if (urlContent == null)
{
throw new ArgumentNullException("urlContent");
}
m_UrlContent = urlContent;
}
public override List <UrlContent> GetHtmlUrlList(int depth, string rootUrl)
{
HashSet <UrlContent> ret = new HashSet <UrlContent>();
MatchCollection matches = Regex.Matches(HtmlRawContent, @"(href="")[-a-zA-Z0-9@:%_\+.~#?&//=]*("")");
foreach (var oneMatch in matches)
{
UrlContent oneUrl = new UrlContent();
oneUrl.Depth = depth;
oneUrl.Url = UrlProcess(rootUrl, oneMatch.ToString());
ret.Add(oneUrl);
}
return(ret.ToList());
}
public string GetImgurIdNum()
{
if (GetContentType() != ContentType.ImgurGallery)
{
return("");
}
else if (UrlContent.Contains("https"))
{
return(UrlContent.Replace("https://imgur.com/a/", ""));
}
return(UrlContent.Replace("http://imgur.com/a/", ""));
}
public FileInfo GetObjectWithUrl(ObjectIdentity objIdentity)
{
objIdentity.RepositoryName = this.DefaultRepository;
ObjectIdentitySet objectIdSet = new ObjectIdentitySet();
List <ObjectIdentity> objIdList = objectIdSet.Identities;
objIdList.Add(objIdentity);
List <ObjectContentSet> urlList = objectService.GetObjectContentUrls(objectIdSet);
ObjectContentSet objectContentSet = (ObjectContentSet)urlList[0];
UrlContent urlContent = (UrlContent)objectContentSet.Contents[0];
if (urlContent.CanGetAsFile())
{
// downloads the file using the ACS URL
FileInfo file = urlContent.GetAsFile();
Console.WriteLine("File exists: " + file.Exists);
Console.WriteLine("File full name: " + file.FullName);
return(file);
}
return(null);
}
public DataObject GetFilterContentSpecific(String qualificationString)
{
ContentProfile contentProfile = new ContentProfile();
contentProfile.FormatFilter = FormatFilter.SPECIFIED;
contentProfile.Format = "gif";
contentProfile.UrlReturnPolicy = UrlReturnPolicy.PREFER;
OperationOptions operationOptions = new OperationOptions();
operationOptions.ContentProfile = contentProfile;
ObjectIdentity objectIdentity =
new ObjectIdentity
(new Qualification(qualificationString), DefaultRepository);
ObjectIdentitySet objectIdSet = new ObjectIdentitySet(objectIdentity);
DataPackage dataPackage = null;
dataPackage = objectService.Get(objectIdSet, operationOptions);
DataObject dataObject = dataPackage.DataObjects[0];
Content resultContent = dataObject.Contents[0];
String contentClassName = resultContent.GetType().FullName;
Console.WriteLine("Returned content as type " + contentClassName);
if (contentClassName.Equals("Emc.Documentum.FS.DataModel.Core.Content.UrlContent"))
{
UrlContent urlContent = (UrlContent)resultContent;
Console.WriteLine("Content ACS URL is: " + (urlContent.Url));
}
if (resultContent.CanGetAsFile())
{
FileInfo fileInfo = resultContent.GetAsFile();
Console.WriteLine("Returned content format is " + resultContent.Format);
Console.WriteLine("Got content as file " + fileInfo.FullName);
}
return(dataObject);
}
private async Task <List <UrlContent> > GetLinksByUrl(IReadOnlyCollection <UrlContent> addedLink, UrlContent urlContent, int amount)
{
var html = await GetHtmlAsync(urlContent.Url);
var host = urlContent.Url.GetHost();
var nodes = html?.DocumentNode?.SelectNodes("//a[@href]");
if (nodes != null)
{
return(html.DocumentNode.SelectNodes("//a[@href]")
.Where(x => x.GetHref().GetHost() == host &&
!addedLink.Select(content => content.Url).Contains(x.GetHref()))
.Distinct()
.Take(amount)
.Select(x => new UrlContent(x.GetHref(), html.DocumentNode.InnerText.Trim()))
.ToList());
}
return(new List <UrlContent>());
}
internal UrlViewModel()
{
m_UrlContent = new UrlContent();
}
internal UrlViewModel()
{
m_UrlContent = new UrlContent();
}
public ContentType GetContentType()
{
ContentType type = ContentType.Blank;
if (UrlContent.Contains("v.redd.it"))
{
if (UrlContent.Contains(" "))
{
type = ContentType.Vreddit;
}
else
{
type = ContentType.VredditPostOnly;
}
}
else if (UrlContent.Contains("https://rule34video.com/videos/"))
{
type = ContentType.R34Video;
}
else if (UrlContent.Contains("imgur.com/a/"))
{
if (UrlContent.Contains(" ")) // Was split
{
int numOfUrls = UrlContent.Count(letter => letter == ' ') + 1;
if (numOfUrls > 2)
{
type = ContentType.Gallery;
}
else // Only one image in gallery
{
if (UrlContent.Contains(".gif"))
{
type = ContentType.Gif;
}
else
{
type = ContentType.Image;
}
}
}
else // Hasn't been split
{
type = ContentType.ImgurGallery;
}
}
else if (UrlContent.Contains(" ") && UrlContent.Contains("i.imgur.com") && !UrlContent.Contains(".gifv") && !UrlContent.Contains(".mp4"))
{
type = ContentType.Image;
if (UrlContent.Contains(".gif"))
{
type = ContentType.Gif;
}
}
else if (UrlContent.Contains(" "))
{
type = ContentType.Gallery;
}
else if (UrlContent.Contains("//imgur.com/") && !UrlContent.Contains(".gifv") && !UrlContent.Contains(".mp4"))
{
type = ContentType.ImgurImage;
}
else if (UrlContent.Contains(".jpg") || UrlContent.Contains(".png") || UrlContent.Contains(".jpeg"))
{
type = ContentType.Image;
}
else if (UrlContent.Contains(".mp4"))
{
type = ContentType.Mp4;
}
else if (UrlContent.Contains("redgifs") && UrlContent.Contains("watch"))
{
type = ContentType.RedGifWatch;
}
else if (UrlContent.Contains("twitter.com/"))
{
type = ContentType.Twitter;
}
else if (UrlContent.Contains("youtu.be") || UrlContent.Contains("youtube.com"))
{
type = ContentType.Youtube;
}
else if (UrlContent.Contains(".gifv"))
{
type = ContentType.Gifv;
}
else if (UrlContent.Contains(".gif"))
{
type = ContentType.Gif;
}
else if (UrlContent.Contains("gfycat.com"))
{
type = ContentType.GfyCat;
}
else if (!UrlThumbnail.Contains("self") && !UrlThumbnail.Contains("default"))
{
type = ContentType.UrlPreview;
}
return(type);
}
public string[] GetContentUrls()
{
string[] toReturn = new string[] { UrlContent };
if (UrlContent.Contains("reddit.com/gallery/"))
{
toReturn = UrlContent.Split(" ");
}
else if (UrlContent.Contains("https://rule34video.com/videos/"))
{
string vidId = toReturn[0].Replace("https://rule34video.com/videos/", "");
vidId = vidId.Substring(0, vidId.IndexOf("/"));
toReturn[0] = "https://rule34video.com/embed/" + vidId;
}
else if (UrlContent.Contains(".gifv"))
{
toReturn[0] = toReturn[0].Replace(".gifv", ".mp4");
toReturn[0] = toReturn[0].Replace("//imgur.com/", "//i.imgur.com/");
}
else if (UrlContent.Contains("https://gfycat.com/"))
{
toReturn[0] = toReturn[0].Replace("https://gfycat.com/", "https://gfycat.com/ifr/");
}
else if (UrlContent.Contains(" ") && UrlContent.Contains("i.imgur.com") && (GetContentType() == ContentType.Gif || GetContentType() == ContentType.Image))
{
toReturn[0] = UrlContent.Split(" ")[1];
}
else if (UrlContent.Contains("//imgur.com/") && !UrlContent.Contains("/a/")) // An Imgur pic that wasn't posted with the direct link
{
toReturn[0] = toReturn[0].Replace("//imgur.com/", "//i.imgur.com/") + ".png";
}
else if (UrlContent.Contains("v.redd.it") && UrlContent.Contains(" "))
{
toReturn = UrlContent.Split(" ");
}
else if (UrlContent.Contains("youtu.be"))
{
toReturn[0] = toReturn[0].Replace("https://youtu.be/", "https://www.youtube.com/embed/");
}
else if (UrlContent.Contains("youtube.com"))
{
toReturn[0] = toReturn[0].Replace("https://www.youtube.com/watch?v=", "https://www.youtube.com/embed/");
}
else if (UrlContent.Contains("imgur.com/a/") && GetContentType() == ContentType.Gallery)
{
toReturn = UrlContent.Split(" ");
}
else if (UrlContent.Contains("imgur.com/a/") && (GetContentType() == ContentType.Image || GetContentType() == ContentType.Gif))
{
toReturn[0] = UrlContent.Split(" ")[1];
}
else if (UrlContent.Contains("https://rule34.xxx//samples/") && UrlContent.Contains("?"))
{
toReturn[0] = toReturn[0].Replace("https://rule34.xxx//samples/", "https://us.rule34.xxx//images/");
toReturn[0] = toReturn[0].Replace("sample_", "");
toReturn[0] = toReturn[0].Substring(0, toReturn[0].LastIndexOf("?"));
toReturn[0] = toReturn[0].Replace("jpg", "jpeg");
}
return(toReturn);
}
