public void TestGetHashFromStreams()
{
Stream[] streams =
{
new MemoryStream(Encoding.UTF8.GetBytes(dataToTimestamp)),
new MemoryStream(Encoding.UTF8.GetBytes(dataToTimestamp2))
};
TimestampData timestampData = new TimestampData(streams);
byte[] result = timestampData.GetHashedData(HashAlgorithm.SHA1);
Stream[] streams2 =
{
new MemoryStream(Encoding.UTF8.GetBytes(dataToTimestamp)),
new MemoryStream(Encoding.UTF8.GetBytes(dataToTimestamp2))
};
System.Security.Cryptography.HashAlgorithm algorithm = new System.Security.Cryptography.SHA1Cng();
using (MemoryStream finalStream = new MemoryStream())
{
foreach (Stream stream in streams2)
{
stream.CopyTo(finalStream);
}
byte[] expectedResult = algorithm.ComputeHash(finalStream.ToArray());
Assert.IsTrue(result.SequenceEqual(expectedResult));
}
}
private static void CalTzp()
{
String appSecret = "0221860e";
String nonce = "1234567890";
String curTime = "1610717873";
//curTime = ((long)(DateTime.Now.ToUniversalTime() - new DateTime(1970, 1, 1)).TotalSeconds).ToString();
var tmp = appSecret + nonce + curTime;
System.Security.Cryptography.SHA1Cng sha = new System.Security.Cryptography.SHA1Cng();
var buffer = sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(tmp));
var lst = buffer.Select(x => x.ToString("x2")).ToList();
string rt = string.Concat(lst);
}
public static Int64 GetInt64HashCodeFast2(this string strText)
{
Int64 hashCode = 0;
if (!string.IsNullOrEmpty(strText))
{
//Unicode Encode Covering all characterset
byte[] byteContents = Encoding.Unicode.GetBytes(strText);
System.Security.Cryptography.SHA1Cng hash =
new System.Security.Cryptography.SHA1Cng();
byte[] hashText = hash.ComputeHash(byteContents);
//32Byte hashText separate
//hashCodeStart = 0~7 8Byte
//hashCodeMedium = 8~23 8Byte
//hashCodeEnd = 24~31 8Byte
//and Fold
Int64 hashCodeStart = BitConverter.ToInt64(hashText, 0);
Int64 hashCodeMedium = BitConverter.ToInt64(hashText, 5);
Int64 hashCodeEnd = BitConverter.ToInt64(hashText, 8);
hashCode = hashCodeStart ^ hashCodeMedium ^ hashCodeEnd;
}
return(hashCode);
}
private void buttonAdd_Click(object sender, RoutedEventArgs e)
{
if (String.IsNullOrWhiteSpace(textBoxPath.Text))
{
MessageBox.Show("You must select an image to add.", "No image path", MessageBoxButton.OK, MessageBoxImage.Error);
return;
}
else if (!File.Exists(textBoxPath.Text))
{
MessageBox.Show("The specified file does not exist.", "Invalid path", MessageBoxButton.OK, MessageBoxImage.Error);
return;
}
ImageData img = new ImageData();
img.path = textBoxPath.Text;
using (var stream = new FileStream(img.path, FileMode.Open)) {
try {
using (var bmp = System.Drawing.Image.FromStream(stream)) {
img.width = bmp.Width;
img.height = bmp.Height;
}
} catch (ArgumentException ex) {
MessageBox.Show("The specified file is not an image.", "Invalid file", MessageBoxButton.OK, MessageBoxImage.Error);
return;
}
stream.Seek(0, SeekOrigin.Begin);
using (var hasher = new System.Security.Cryptography.SHA1Cng()) {
img.sha1Hash = BitConverter.ToString(hasher.ComputeHash(stream)).Replace("-", "");
hasher.Clear();
}
stream.Seek(0, SeekOrigin.Begin);
img.thumbnailMD5 = Thumbnails.Add(stream, img.path);
}
if (ImgData == null)
{
ImageData otherImg = null;
for (int i = 0; i < ImageDataList.Images.Count; i++)
{
otherImg = ImageDataList.Images [i];
if (otherImg == null) // Wat?
{
continue;
}
if (string.Equals(otherImg.path, img.path, StringComparison.CurrentCultureIgnoreCase))
{
MessageBox.Show("There is already an entry with the specified path in the database.", "Duplicate image", MessageBoxButton.OK, MessageBoxImage.Error);
return;
}
else if (string.Equals(otherImg.sha1Hash, img.sha1Hash, StringComparison.CurrentCultureIgnoreCase))
{
MessageBox.Show("The specified image matches the SHA1 hash of another entry in the database.", "Duplicate image", MessageBoxButton.OK, MessageBoxImage.Error);
return;
}
}
}
img.description = textBoxDescription.Text;
img.size = (new FileInfo(img.path)).Length;
List <string> tags = new List <string> (), sources = new List <string> ();
foreach (string tag in this.listBoxTags.Items)
{
if (tag.StartsWith("source:")) // If the tag starts with "source:",
{
sources.Add(tag.Remove(0, 7)); // Remove the "source:" from the tag and add it to the sources list
}
else // If not,
{
tags.Add(tag); // Add it to the tags list
}
}
img.sources = sources.ToArray(); sources.Clear(); sources = null;
img.tags = tags.ToArray(); tags.Clear(); tags = null;
if (ImgData != null)
{
for (int i = 0; i < ImageDataList.Images.Count; i++)
{
if (ImageDataList.Images [i] == this.ImgData)
{
ImageDataList.Images [i] = img;
}
}
}
else
{
ImageDataList.Images.Add(img);
}
this.Close();
}
private static async Task <string> DownloadAttemptAsync(Uri url, bool force, IProgress <DataTransferProgress> progress)
{
var originalUrl = url;
string hash;
using (var sha1 = new System.Security.Cryptography.SHA1Cng())
{
hash = ToHex(sha1.ComputeHash(Encoding.UTF8.GetBytes(url.AbsoluteUri))).ToLower().Substring(0, 16);
}
lock (typeof(Paper))
{
if (cachedPapers == null)
{
cachedPapers = new Dictionary <string, string>();
foreach (var x in Directory.EnumerateFiles("/Awdee/SciHub", "*.pdf"))
{
var name = Path.GetFileName(x);
var p = name.IndexOf('-');
cachedPapers[p == -1 ? x : name.Substring(0, p)] = x;
}
}
}
var r = cachedPapers.TryGetValue(hash);
if (r != null)
{
return(r);
}
progress.Report("Initializing");
// return;
WebFile pdfFile = null;
string title = null;
HtmlNode original = null;
var cookies = new IsolatedCookieContainer();
string doi = null;
if (url.IsHostedOn("dx.doi.org"))
{
doi = HttpUtils.UnescapeDataString(url.AbsolutePath.Substring(1));
}
(pdfFile, title) = await TryGetLibgenAsync(doi, null, progress);
if (pdfFile == null && url.IsHostedOn("academia.edu"))
{
cookies.AddRange(HttpUtils.ParseCookies(Configuration_AcademiaEduCookie.Trim()));
progress.Report("Retrieving from Academia.edu");
original = await url.GetHtmlNodeAsync(null, cookies);
var u = new LazyUri(original.GetLinkUrl("a[data-download],.js-swp-download-button"));
u.AppendCookies(cookies);
u.AppendFragmentParameter("$header-Referer", url.AbsoluteUri);
title = original.GetValue(":property('citation_title')");
pdfFile = WebFile.FromUrlUntracked(u.Url);
}
if (pdfFile == null)
{
try
{
progress.Report("Retrieving plain page");
original = await url.GetHtmlNodeAsync();
doi = original.TryGetValue(":property('citation_doi'),meta[scheme='doi']:property('dc.Identifier')");
if (doi == null && url.IsHostedOn("nih.gov"))
{
doi = original.TryGetValue("a[ref='aid_type=doi'],.doi > a");
if (doi == null && url.AbsolutePath.StartsWith("/pubmed/"))
{
progress.Report("Finding DOI on EuropePMC.org");
var alt = await HttpUtils.FormatEscaped("http://europepmc.org/abstract/med/{0}", url.GetPathComponent(1)).GetHtmlNodeAsync();
doi = alt.TryGetValue("meta[name='citation_doi']", "content");
}
}
if (doi == null && url.IsHostedOn("sciencedirect.com"))
{
doi = original.TryGetValue("script:json-token('SDM.doi = ')");
}
if (doi != null)
{
(pdfFile, title) = await TryGetLibgenAsync(doi, null, progress);
}
if (pdfFile == null && url.IsHostedOn("researchgate.net"))
{
var u = FindPdfLink(original);
if (u != null)
{
pdfFile = WebFile.FromUrlUntracked(u);
}
}
if (title == null)
{
title = original.TryGetValue(":property('citation_title')");
if (title == null)
{
title = original.TryGetValue("title")?.TrimEnd(" - PubMed - NCBI");
}
}
}
catch (NotSupportedResponseException ex) when(ex.ContentType == "application/pdf")
{
pdfFile = WebFile.FromUrlUntracked(url);
}
}
if (pdfFile == null)
{
if (url.IsHostedOn("nlm.nih.gov"))
{
var a = original.TryGetLinkUrl(".portlet a");
if (a != null)
{
url = a;
}
else
{
var k = FindPdfLink(original);
if (k != null)
{
pdfFile = WebFile.FromUrlUntracked(k);
}
}
}
if (pdfFile == null)
{
if (!url.IsHostedOn("scielo.br"))
{
var u = new LazyUri("http://" + url.Host + ".sci-hub.cc" + url.AbsolutePath + url.Query + url.Fragment);
progress.Report("Trying on SciHub");
u.AppendFragmentParameter("$allow-same-redirect", "1");
url = u.Url;
}
else
{
progress.Report("Trying on " + url.Host);
}
var scihub = await url.GetHtmlNodeAsync(null, cookies);
if (scihub.FindSingle("img#captcha") != null)
{
throw new CaptchaException(scihub.OwnerDocument.PageUrl);
}
if (scihub.OwnerDocument.PageUrl.IsHostedOn("libgen.io"))
{
var u = scihub.GetLinkUrl("a[href*='/ads.php?']");
progress.Report("Found on LibGen.IO");
(pdfFile, title) = await TryGetLibgenAsync(null, u, progress);
}
else
{
var pdflink = scihub.TryGetLinkUrl("iframe#pdf") ??
FindPdfLink(scihub);
if (pdflink != null)
{
var u = new LazyUri(pdflink);
u.AppendCookies(cookies);
pdfFile = WebFile.FromUrlUntracked(u.Url);
}
}
}
}
if (pdfFile != null)
{
var uu = new LazyUri(pdfFile.Url);
uu.AppendFragmentParameter("$allow-same-redirect", "1");
uu.AppendFragmentParameter("$forbid-html", "1");
pdfFile = WebFile.FromUrlUntracked(uu.Url);
if (title == null)
{
var z = pdfFile.SuggestedFileName;
if (z != null)
{
title = Path.GetFileNameWithoutExtension(z);
}
}
else
{
title = title.Trim().TrimEnd(".").RegexReplace(@"\s+", "-");
}
progress.Report("Downloading from " + pdfFile.Url.Host);
string path;
try
{
path = await pdfFile.DownloadAsync("/Awdee/SciHub", hash + "-" + title + ".pdf", WebFile.FileOverwriteMode.Skip, CancellationToken.None, progress);
}
catch (NotSupportedResponseException ex)
{
if (ex.Page != null && ex.Page.FindSingle("img#captcha") != null)
{
throw new CaptchaException(ex.Page.OwnerDocument.PageUrl);
}
throw;
}
var filename = Path.GetFileName(path);
lock (typeof(Paper))
{
cachedPapers[hash] = path;
File.AppendAllText("/Awdee/SciHubDownloads.csv", string.Join("\t", originalUrl, title, doi, filename, new FileInfo(path).Length) + "\r\n", Encoding.UTF8);
}
progress.Report("Done.");
return(path);
}
throw new Exception("Could not find any PDF links.");
}
